1//===- MemorySanitizer.cpp - detector of uninitialized reads --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file is a part of MemorySanitizer, a detector of uninitialized
11/// reads.
12///
13/// The algorithm of the tool is similar to Memcheck
14/// (https://static.usenix.org/event/usenix05/tech/general/full_papers/seward/seward_html/usenix2005.html)
15/// We associate a few shadow bits with every byte of the application memory,
16/// poison the shadow of the malloc-ed or alloca-ed memory, load the shadow,
17/// bits on every memory read, propagate the shadow bits through some of the
18/// arithmetic instruction (including MOV), store the shadow bits on every
19/// memory write, report a bug on some other instructions (e.g. JMP) if the
20/// associated shadow is poisoned.
21///
22/// But there are differences too. The first and the major one:
23/// compiler instrumentation instead of binary instrumentation. This
24/// gives us much better register allocation, possible compiler
25/// optimizations and a fast start-up. But this brings the major issue
26/// as well: msan needs to see all program events, including system
27/// calls and reads/writes in system libraries, so we either need to
28/// compile *everything* with msan or use a binary translation
29/// component (e.g. DynamoRIO) to instrument pre-built libraries.
30/// Another difference from Memcheck is that we use 8 shadow bits per
31/// byte of application memory and use a direct shadow mapping. This
32/// greatly simplifies the instrumentation code and avoids races on
33/// shadow updates (Memcheck is single-threaded so races are not a
34/// concern there. Memcheck uses 2 shadow bits per byte with a slow
35/// path storage that uses 8 bits per byte).
36///
37/// The default value of shadow is 0, which means "clean" (not poisoned).
38///
39/// Every module initializer should call __msan_init to ensure that the
40/// shadow memory is ready. On error, __msan_warning is called. Since
41/// parameters and return values may be passed via registers, we have a
42/// specialized thread-local shadow for return values
43/// (__msan_retval_tls) and parameters (__msan_param_tls).
44///
45/// Origin tracking.
46///
47/// MemorySanitizer can track origins (allocation points) of all uninitialized
48/// values. This behavior is controlled with a flag (msan-track-origins) and is
49/// disabled by default.
50///
51/// Origins are 4-byte values created and interpreted by the runtime library.
52/// They are stored in a second shadow mapping, one 4-byte value for 4 bytes
53/// of application memory. Propagation of origins is basically a bunch of
54/// "select" instructions that pick the origin of a dirty argument, if an
55/// instruction has one.
56///
57/// Every 4 aligned, consecutive bytes of application memory have one origin
58/// value associated with them. If these bytes contain uninitialized data
59/// coming from 2 different allocations, the last store wins. Because of this,
60/// MemorySanitizer reports can show unrelated origins, but this is unlikely in
61/// practice.
62///
63/// Origins are meaningless for fully initialized values, so MemorySanitizer
64/// avoids storing origin to memory when a fully initialized value is stored.
65/// This way it avoids needless overwriting origin of the 4-byte region on
66/// a short (i.e. 1 byte) clean store, and it is also good for performance.
67///
68/// Atomic handling.
69///
70/// Ideally, every atomic store of application value should update the
71/// corresponding shadow location in an atomic way. Unfortunately, atomic store
72/// of two disjoint locations can not be done without severe slowdown.
73///
74/// Therefore, we implement an approximation that may err on the safe side.
75/// In this implementation, every atomically accessed location in the program
76/// may only change from (partially) uninitialized to fully initialized, but
77/// not the other way around. We load the shadow _after_ the application load,
78/// and we store the shadow _before_ the app store. Also, we always store clean
79/// shadow (if the application store is atomic). This way, if the store-load
80/// pair constitutes a happens-before arc, shadow store and load are correctly
81/// ordered such that the load will get either the value that was stored, or
82/// some later value (which is always clean).
83///
84/// This does not work very well with Compare-And-Swap (CAS) and
85/// Read-Modify-Write (RMW) operations. To follow the above logic, CAS and RMW
86/// must store the new shadow before the app operation, and load the shadow
87/// after the app operation. Computers don't work this way. Current
88/// implementation ignores the load aspect of CAS/RMW, always returning a clean
89/// value. It implements the store part as a simple atomic store by storing a
90/// clean shadow.
91///
92/// Instrumenting inline assembly.
93///
94/// For inline assembly code LLVM has little idea about which memory locations
95/// become initialized depending on the arguments. It can be possible to figure
96/// out which arguments are meant to point to inputs and outputs, but the
97/// actual semantics can be only visible at runtime. In the Linux kernel it's
98/// also possible that the arguments only indicate the offset for a base taken
99/// from a segment register, so it's dangerous to treat any asm() arguments as
100/// pointers. We take a conservative approach generating calls to
101/// __msan_instrument_asm_store(ptr, size)
102/// , which defer the memory unpoisoning to the runtime library.
103/// The latter can perform more complex address checks to figure out whether
104/// it's safe to touch the shadow memory.
105/// Like with atomic operations, we call __msan_instrument_asm_store() before
106/// the assembly call, so that changes to the shadow memory will be seen by
107/// other threads together with main memory initialization.
108///
109/// KernelMemorySanitizer (KMSAN) implementation.
110///
111/// The major differences between KMSAN and MSan instrumentation are:
112/// - KMSAN always tracks the origins and implies msan-keep-going=true;
113/// - KMSAN allocates shadow and origin memory for each page separately, so
114/// there are no explicit accesses to shadow and origin in the
115/// instrumentation.
116/// Shadow and origin values for a particular X-byte memory location
117/// (X=1,2,4,8) are accessed through pointers obtained via the
118/// __msan_metadata_ptr_for_load_X(ptr)
119/// __msan_metadata_ptr_for_store_X(ptr)
120/// functions. The corresponding functions check that the X-byte accesses
121/// are possible and returns the pointers to shadow and origin memory.
122/// Arbitrary sized accesses are handled with:
123/// __msan_metadata_ptr_for_load_n(ptr, size)
124/// __msan_metadata_ptr_for_store_n(ptr, size);
125/// Note that the sanitizer code has to deal with how shadow/origin pairs
126/// returned by the these functions are represented in different ABIs. In
127/// the X86_64 ABI they are returned in RDX:RAX, in PowerPC64 they are
128/// returned in r3 and r4, and in the SystemZ ABI they are written to memory
129/// pointed to by a hidden parameter.
130/// - TLS variables are stored in a single per-task struct. A call to a
131/// function __msan_get_context_state() returning a pointer to that struct
132/// is inserted into every instrumented function before the entry block;
133/// - __msan_warning() takes a 32-bit origin parameter;
134/// - local variables are poisoned with __msan_poison_alloca() upon function
135/// entry and unpoisoned with __msan_unpoison_alloca() before leaving the
136/// function;
137/// - the pass doesn't declare any global variables or add global constructors
138/// to the translation unit.
139///
140/// Also, KMSAN currently ignores uninitialized memory passed into inline asm
141/// calls, making sure we're on the safe side wrt. possible false positives.
142///
143/// KernelMemorySanitizer only supports X86_64, SystemZ and PowerPC64 at the
144/// moment.
145///
146//
147// FIXME: This sanitizer does not yet handle scalable vectors
148//
149//===----------------------------------------------------------------------===//
150
151#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
152#include "InstrumentationOptions.h"
153#include "llvm/ADT/APInt.h"
154#include "llvm/ADT/ArrayRef.h"
155#include "llvm/ADT/DenseMap.h"
156#include "llvm/ADT/DepthFirstIterator.h"
157#include "llvm/ADT/SetVector.h"
158#include "llvm/ADT/SmallPtrSet.h"
159#include "llvm/ADT/SmallVector.h"
160#include "llvm/ADT/StringExtras.h"
161#include "llvm/ADT/StringRef.h"
162#include "llvm/Analysis/GlobalsModRef.h"
163#include "llvm/Analysis/TargetLibraryInfo.h"
164#include "llvm/Analysis/ValueTracking.h"
165#include "llvm/IR/Argument.h"
166#include "llvm/IR/AttributeMask.h"
167#include "llvm/IR/Attributes.h"
168#include "llvm/IR/BasicBlock.h"
169#include "llvm/IR/CallingConv.h"
170#include "llvm/IR/Constant.h"
171#include "llvm/IR/Constants.h"
172#include "llvm/IR/DataLayout.h"
173#include "llvm/IR/DerivedTypes.h"
174#include "llvm/IR/Function.h"
175#include "llvm/IR/GlobalValue.h"
176#include "llvm/IR/GlobalVariable.h"
177#include "llvm/IR/IRBuilder.h"
178#include "llvm/IR/InlineAsm.h"
179#include "llvm/IR/InstVisitor.h"
180#include "llvm/IR/InstrTypes.h"
181#include "llvm/IR/Instruction.h"
182#include "llvm/IR/Instructions.h"
183#include "llvm/IR/IntrinsicInst.h"
184#include "llvm/IR/Intrinsics.h"
185#include "llvm/IR/IntrinsicsAArch64.h"
186#include "llvm/IR/IntrinsicsX86.h"
187#include "llvm/IR/MDBuilder.h"
188#include "llvm/IR/Module.h"
189#include "llvm/IR/Type.h"
190#include "llvm/IR/Value.h"
191#include "llvm/IR/ValueMap.h"
192#include "llvm/Support/Alignment.h"
193#include "llvm/Support/AtomicOrdering.h"
194#include "llvm/Support/Casting.h"
195#include "llvm/Support/Debug.h"
196#include "llvm/Support/DebugCounter.h"
197#include "llvm/Support/ErrorHandling.h"
198#include "llvm/Support/MathExtras.h"
199#include "llvm/Support/raw_ostream.h"
200#include "llvm/TargetParser/Triple.h"
201#include "llvm/Transforms/Utils/BasicBlockUtils.h"
202#include "llvm/Transforms/Utils/Instrumentation.h"
203#include "llvm/Transforms/Utils/Local.h"
204#include "llvm/Transforms/Utils/ModuleUtils.h"
205#include <algorithm>
206#include <cassert>
207#include <cstddef>
208#include <cstdint>
209#include <memory>
210#include <numeric>
211#include <string>
212#include <tuple>
213
214using namespace llvm;
215
216#define DEBUG_TYPE "msan"
217
218DEBUG_COUNTER(DebugInsertCheck, "msan-insert-check",
219 "Controls which checks to insert");
220
221DEBUG_COUNTER(DebugInstrumentInstruction, "msan-instrument-instruction",
222 "Controls which instruction to instrument");
223
224static const unsigned kOriginSize = 4;
225static const Align kMinOriginAlignment = Align(4);
226static const Align kShadowTLSAlignment = Align(8);
227
228// These constants must be kept in sync with the ones in msan.h.
229// TODO: increase size to match SVE/SVE2/SME/SME2 limits
230static const unsigned kParamTLSSize = 800;
231static const unsigned kRetvalTLSSize = 800;
232
233// Accesses sizes are powers of two: 1, 2, 4, 8.
234static const size_t kNumberOfAccessSizes = 4;
235
236const char kMsanModuleCtorName[] = "msan.module_ctor";
237const char kMsanInitName[] = "__msan_init";
238
239namespace {
240
241// Memory map parameters used in application-to-shadow address calculation.
242// Offset = (Addr & ~AndMask) ^ XorMask
243// Shadow = ShadowBase + Offset
244// Origin = OriginBase + Offset
245struct MemoryMapParams {
246 uint64_t AndMask;
247 uint64_t XorMask;
248 uint64_t ShadowBase;
249 uint64_t OriginBase;
250};
251
252struct PlatformMemoryMapParams {
253 const MemoryMapParams *bits32;
254 const MemoryMapParams *bits64;
255};
256
257} // end anonymous namespace
258
259// i386 Linux
260static const MemoryMapParams Linux_I386_MemoryMapParams = {
261 .AndMask: 0x000080000000, // AndMask
262 .XorMask: 0, // XorMask (not used)
263 .ShadowBase: 0, // ShadowBase (not used)
264 .OriginBase: 0x000040000000, // OriginBase
265};
266
267// x86_64 Linux
268static const MemoryMapParams Linux_X86_64_MemoryMapParams = {
269 .AndMask: 0, // AndMask (not used)
270 .XorMask: 0x500000000000, // XorMask
271 .ShadowBase: 0, // ShadowBase (not used)
272 .OriginBase: 0x100000000000, // OriginBase
273};
274
275// mips32 Linux
276// FIXME: Remove -msan-origin-base -msan-and-mask added by PR #109284 to tests
277// after picking good constants
278
279// mips64 Linux
280static const MemoryMapParams Linux_MIPS64_MemoryMapParams = {
281 .AndMask: 0, // AndMask (not used)
282 .XorMask: 0x008000000000, // XorMask
283 .ShadowBase: 0, // ShadowBase (not used)
284 .OriginBase: 0x002000000000, // OriginBase
285};
286
287// ppc32 Linux
288// FIXME: Remove -msan-origin-base -msan-and-mask added by PR #109284 to tests
289// after picking good constants
290
291// ppc64 Linux
292static const MemoryMapParams Linux_PowerPC64_MemoryMapParams = {
293 .AndMask: 0xE00000000000, // AndMask
294 .XorMask: 0x100000000000, // XorMask
295 .ShadowBase: 0x080000000000, // ShadowBase
296 .OriginBase: 0x1C0000000000, // OriginBase
297};
298
299// s390x Linux
300static const MemoryMapParams Linux_S390X_MemoryMapParams = {
301 .AndMask: 0xC00000000000, // AndMask
302 .XorMask: 0, // XorMask (not used)
303 .ShadowBase: 0x080000000000, // ShadowBase
304 .OriginBase: 0x1C0000000000, // OriginBase
305};
306
307// arm32 Linux
308// FIXME: Remove -msan-origin-base -msan-and-mask added by PR #109284 to tests
309// after picking good constants
310
311// aarch64 Linux
312static const MemoryMapParams Linux_AArch64_MemoryMapParams = {
313 .AndMask: 0, // AndMask (not used)
314 .XorMask: 0x0B00000000000, // XorMask
315 .ShadowBase: 0, // ShadowBase (not used)
316 .OriginBase: 0x0200000000000, // OriginBase
317};
318
319// loongarch64 Linux
320static const MemoryMapParams Linux_LoongArch64_MemoryMapParams = {
321 .AndMask: 0, // AndMask (not used)
322 .XorMask: 0x500000000000, // XorMask
323 .ShadowBase: 0, // ShadowBase (not used)
324 .OriginBase: 0x100000000000, // OriginBase
325};
326
327// hexagon Linux
328static const MemoryMapParams Linux_Hexagon_MemoryMapParams = {
329 .AndMask: 0, // AndMask (not used)
330 .XorMask: 0x20000000, // XorMask
331 .ShadowBase: 0, // ShadowBase (not used)
332 .OriginBase: 0x50000000, // OriginBase
333};
334
335// riscv32 Linux
336// FIXME: Remove -msan-origin-base -msan-and-mask added by PR #109284 to tests
337// after picking good constants
338
339// aarch64 FreeBSD
340static const MemoryMapParams FreeBSD_AArch64_MemoryMapParams = {
341 .AndMask: 0x1800000000000, // AndMask
342 .XorMask: 0x0400000000000, // XorMask
343 .ShadowBase: 0x0200000000000, // ShadowBase
344 .OriginBase: 0x0700000000000, // OriginBase
345};
346
347// i386 FreeBSD
348static const MemoryMapParams FreeBSD_I386_MemoryMapParams = {
349 .AndMask: 0x000180000000, // AndMask
350 .XorMask: 0x000040000000, // XorMask
351 .ShadowBase: 0x000020000000, // ShadowBase
352 .OriginBase: 0x000700000000, // OriginBase
353};
354
355// x86_64 FreeBSD
356static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams = {
357 .AndMask: 0xc00000000000, // AndMask
358 .XorMask: 0x200000000000, // XorMask
359 .ShadowBase: 0x100000000000, // ShadowBase
360 .OriginBase: 0x380000000000, // OriginBase
361};
362
363// x86_64 NetBSD
364static const MemoryMapParams NetBSD_X86_64_MemoryMapParams = {
365 .AndMask: 0, // AndMask
366 .XorMask: 0x500000000000, // XorMask
367 .ShadowBase: 0, // ShadowBase
368 .OriginBase: 0x100000000000, // OriginBase
369};
370
371static const PlatformMemoryMapParams Linux_X86_MemoryMapParams = {
372 .bits32: &Linux_I386_MemoryMapParams,
373 .bits64: &Linux_X86_64_MemoryMapParams,
374};
375
376static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams = {
377 .bits32: nullptr,
378 .bits64: &Linux_MIPS64_MemoryMapParams,
379};
380
381static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams = {
382 .bits32: nullptr,
383 .bits64: &Linux_PowerPC64_MemoryMapParams,
384};
385
386static const PlatformMemoryMapParams Linux_S390_MemoryMapParams = {
387 .bits32: nullptr,
388 .bits64: &Linux_S390X_MemoryMapParams,
389};
390
391static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams = {
392 .bits32: nullptr,
393 .bits64: &Linux_AArch64_MemoryMapParams,
394};
395
396static const PlatformMemoryMapParams Linux_LoongArch_MemoryMapParams = {
397 .bits32: nullptr,
398 .bits64: &Linux_LoongArch64_MemoryMapParams,
399};
400
401static const PlatformMemoryMapParams Linux_Hexagon_MemoryMapParams_P = {
402 .bits32: &Linux_Hexagon_MemoryMapParams,
403 .bits64: nullptr,
404};
405
406static const PlatformMemoryMapParams FreeBSD_ARM_MemoryMapParams = {
407 .bits32: nullptr,
408 .bits64: &FreeBSD_AArch64_MemoryMapParams,
409};
410
411static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams = {
412 .bits32: &FreeBSD_I386_MemoryMapParams,
413 .bits64: &FreeBSD_X86_64_MemoryMapParams,
414};
415
416static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams = {
417 .bits32: nullptr,
418 .bits64: &NetBSD_X86_64_MemoryMapParams,
419};
420
421enum OddOrEvenLanes { kBothLanes, kEvenLanes, kOddLanes };
422
423namespace {
424
425/// Instrument functions of a module to detect uninitialized reads.
426///
427/// Instantiating MemorySanitizer inserts the msan runtime library API function
428/// declarations into the module if they don't exist already. Instantiating
429/// ensures the __msan_init function is in the list of global constructors for
430/// the module.
431class MemorySanitizer {
432public:
433 MemorySanitizer(const InstrumentationOptions &Opts, Module &M,
434 MemorySanitizerOptions Options)
435 : Opts(Opts), CompileKernel(Options.Kernel),
436 TrackOrigins(Options.TrackOrigins), Recover(Options.Recover),
437 EagerChecks(Options.EagerChecks) {
438 initializeModule(M);
439 }
440
441 // MSan cannot be moved or copied because of MapParams.
442 MemorySanitizer(MemorySanitizer &&) = delete;
443 MemorySanitizer &operator=(MemorySanitizer &&) = delete;
444 MemorySanitizer(const MemorySanitizer &) = delete;
445 MemorySanitizer &operator=(const MemorySanitizer &) = delete;
446
447 bool sanitizeFunction(Function &F, TargetLibraryInfo &TLI);
448
449private:
450 friend struct MemorySanitizerVisitor;
451 friend struct VarArgHelperBase;
452 friend struct VarArgAMD64Helper;
453 friend struct VarArgAArch64Helper;
454 friend struct VarArgPowerPC64Helper;
455 friend struct VarArgPowerPC32Helper;
456 friend struct VarArgSystemZHelper;
457 friend struct VarArgI386Helper;
458 friend struct VarArgGenericHelper;
459
460 void initializeModule(Module &M);
461 void initializeCallbacks(Module &M, const TargetLibraryInfo &TLI);
462 void createKernelApi(Module &M, const TargetLibraryInfo &TLI);
463 void createUserspaceApi(Module &M, const TargetLibraryInfo &TLI);
464
465 template <typename... ArgsTy>
466 FunctionCallee getOrInsertMsanMetadataFunction(Module &M, StringRef Name,
467 ArgsTy... Args);
468
469 const InstrumentationOptions &Opts;
470 /// True if we're compiling the Linux kernel.
471 bool CompileKernel;
472 /// Track origins (allocation points) of uninitialized values.
473 int TrackOrigins;
474 bool Recover;
475 bool EagerChecks;
476
477 Triple TargetTriple;
478 LLVMContext *C;
479 Type *IntptrTy; ///< Integer type with the size of a ptr in default AS.
480 Type *OriginTy;
481 PointerType *PtrTy; ///< Integer type with the size of a ptr in default AS.
482
483 // XxxTLS variables represent the per-thread state in MSan and per-task state
484 // in KMSAN.
485 // For the userspace these point to thread-local globals. In the kernel land
486 // they point to the members of a per-task struct obtained via a call to
487 // __msan_get_context_state().
488
489 /// Thread-local shadow storage for function parameters.
490 Value *ParamTLS;
491
492 /// Thread-local origin storage for function parameters.
493 Value *ParamOriginTLS;
494
495 /// Thread-local shadow storage for function return value.
496 Value *RetvalTLS;
497
498 /// Thread-local origin storage for function return value.
499 Value *RetvalOriginTLS;
500
501 /// Thread-local shadow storage for in-register va_arg function.
502 Value *VAArgTLS;
503
504 /// Thread-local shadow storage for in-register va_arg function.
505 Value *VAArgOriginTLS;
506
507 /// Thread-local shadow storage for va_arg overflow area.
508 Value *VAArgOverflowSizeTLS;
509
510 /// Are the instrumentation callbacks set up?
511 bool CallbacksInitialized = false;
512
513 /// The run-time callback to print a warning.
514 FunctionCallee WarningFn;
515
516 // These arrays are indexed by log2(AccessSize).
517 FunctionCallee MaybeWarningFn[kNumberOfAccessSizes];
518 FunctionCallee MaybeWarningVarSizeFn;
519 FunctionCallee MaybeStoreOriginFn[kNumberOfAccessSizes];
520
521 /// Run-time helper that generates a new origin value for a stack
522 /// allocation.
523 FunctionCallee MsanSetAllocaOriginWithDescriptionFn;
524 // No description version
525 FunctionCallee MsanSetAllocaOriginNoDescriptionFn;
526
527 /// Run-time helper that poisons stack on function entry.
528 FunctionCallee MsanPoisonStackFn;
529
530 /// Run-time helper that records a store (or any event) of an
531 /// uninitialized value and returns an updated origin id encoding this info.
532 FunctionCallee MsanChainOriginFn;
533
534 /// Run-time helper that paints an origin over a region.
535 FunctionCallee MsanSetOriginFn;
536
537 /// MSan runtime replacements for memmove, memcpy and memset.
538 FunctionCallee MemmoveFn, MemcpyFn, MemsetFn;
539
540 /// KMSAN callback for task-local function argument shadow.
541 StructType *MsanContextStateTy;
542 FunctionCallee MsanGetContextStateFn;
543
544 /// Functions for poisoning/unpoisoning local variables
545 FunctionCallee MsanPoisonAllocaFn, MsanUnpoisonAllocaFn;
546
547 /// Pair of shadow/origin pointers.
548 Type *MsanMetadata;
549
550 /// Each of the MsanMetadataPtrXxx functions returns a MsanMetadata.
551 FunctionCallee MsanMetadataPtrForLoadN, MsanMetadataPtrForStoreN;
552 FunctionCallee MsanMetadataPtrForLoad_1_8[4];
553 FunctionCallee MsanMetadataPtrForStore_1_8[4];
554 FunctionCallee MsanInstrumentAsmStoreFn;
555
556 /// Storage for return values of the MsanMetadataPtrXxx functions.
557 Value *MsanMetadataAlloca;
558
559 /// Helper to choose between different MsanMetadataPtrXxx().
560 FunctionCallee getKmsanShadowOriginAccessFn(bool isStore, int size);
561
562 /// Memory map parameters used in application-to-shadow calculation.
563 const MemoryMapParams *MapParams;
564
565 /// Custom memory map parameters used when -msan-shadow-base or
566 // -msan-origin-base is provided.
567 MemoryMapParams CustomMapParams;
568
569 MDNode *ColdCallWeights;
570
571 /// Branch weights for origin store.
572 MDNode *OriginStoreWeights;
573};
574
575void insertModuleCtor(const InstrumentationOptions &Opts, Module &M) {
576 getOrCreateSanitizerCtorAndInitFunctions(
577 M, CtorName: kMsanModuleCtorName, InitName: kMsanInitName,
578 /*InitArgTypes=*/{},
579 /*InitArgs=*/{},
580 // This callback is invoked when the functions are created the first
581 // time. Hook them into the global ctors list in that case:
582 FunctionsCreatedCallback: [&](Function *Ctor, FunctionCallee) {
583 if (!Opts.msan_with_comdat) {
584 appendToGlobalCtors(M, F: Ctor, Priority: 0);
585 return;
586 }
587 Comdat *MsanCtorComdat = M.getOrInsertComdat(Name: kMsanModuleCtorName);
588 Ctor->setComdat(MsanCtorComdat);
589 appendToGlobalCtors(M, F: Ctor, Priority: 0, Data: Ctor);
590 });
591}
592
593} // end anonymous namespace
594
595MemorySanitizerOptions::MemorySanitizerOptions(int TO, bool R, bool K, bool EC)
596 : Kernel(K), TrackOrigins(TO), Recover(R), EagerChecks(EC) {}
597
598static MemorySanitizerOptions resolveOptions(const InstrumentationOptions &Opts,
599 MemorySanitizerOptions O) {
600 O.Kernel = valueOr(X: Opts.msan_kernel, Default: O.Kernel);
601 O.TrackOrigins =
602 Opts.msan_track_origins.value_or(u: O.Kernel ? 2 : O.TrackOrigins);
603 O.Recover = valueOr(X: Opts.msan_keep_going, Default: O.Kernel || O.Recover);
604 O.EagerChecks = valueOr(X: Opts.msan_eager_checks, Default: O.EagerChecks);
605 return O;
606}
607
608PreservedAnalyses MemorySanitizerPass::run(Module &M,
609 ModuleAnalysisManager &AM) {
610 // Return early if nosanitize_memory module flag is present for the module.
611 if (checkIfAlreadyInstrumented(M, Flag: "nosanitize_memory"))
612 return PreservedAnalyses::all();
613 const InstrumentationOptions &Opts = getInstrumentationOptions(M);
614 MemorySanitizerOptions MSanOpts = resolveOptions(Opts, O: Options);
615 bool Modified = false;
616 if (!MSanOpts.Kernel) {
617 insertModuleCtor(Opts, M);
618 Modified = true;
619 }
620
621 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
622 for (Function &F : M) {
623 if (F.empty())
624 continue;
625 MemorySanitizer Msan(Opts, *F.getParent(), MSanOpts);
626 Modified |=
627 Msan.sanitizeFunction(F, TLI&: FAM.getResult<TargetLibraryAnalysis>(IR&: F));
628 }
629
630 if (!Modified)
631 return PreservedAnalyses::all();
632
633 PreservedAnalyses PA = PreservedAnalyses::none();
634 // GlobalsAA is considered stateless and does not get invalidated unless
635 // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
636 // make changes that require GlobalsAA to be invalidated.
637 PA.abandon<GlobalsAA>();
638 return PA;
639}
640
641void MemorySanitizerPass::printPipeline(
642 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
643 static_cast<PassInfoMixin<MemorySanitizerPass> *>(this)->printPipeline(
644 OS, MapClassName2PassName);
645 OS << '<';
646 if (Options.Recover)
647 OS << "recover;";
648 if (Options.Kernel)
649 OS << "kernel;";
650 if (Options.EagerChecks)
651 OS << "eager-checks;";
652 OS << "track-origins=" << Options.TrackOrigins;
653 OS << '>';
654}
655
656/// Create a non-const global initialized with the given string.
657///
658/// Creates a writable global for Str so that we can pass it to the
659/// run-time lib. Runtime uses first 4 bytes of the string to store the
660/// frame ID, so the string needs to be mutable.
661static GlobalVariable *createPrivateConstGlobalForString(Module &M,
662 StringRef Str) {
663 Constant *StrConst = ConstantDataArray::getString(Context&: M.getContext(), Initializer: Str);
664 return new GlobalVariable(M, StrConst->getType(), /*isConstant=*/true,
665 GlobalValue::PrivateLinkage, StrConst, "");
666}
667
668template <typename... ArgsTy>
669FunctionCallee
670MemorySanitizer::getOrInsertMsanMetadataFunction(Module &M, StringRef Name,
671 ArgsTy... Args) {
672 if (TargetTriple.getArch() == Triple::systemz) {
673 // SystemZ ABI: shadow/origin pair is returned via a hidden parameter.
674 return M.getOrInsertFunction(Name, Type::getVoidTy(C&: *C), PtrTy,
675 std::forward<ArgsTy>(Args)...);
676 }
677
678 return M.getOrInsertFunction(Name, MsanMetadata,
679 std::forward<ArgsTy>(Args)...);
680}
681
682/// Create KMSAN API callbacks.
683void MemorySanitizer::createKernelApi(Module &M, const TargetLibraryInfo &TLI) {
684 IRBuilder<> IRB(M);
685
686 // These will be initialized in insertKmsanPrologue().
687 RetvalTLS = nullptr;
688 RetvalOriginTLS = nullptr;
689 ParamTLS = nullptr;
690 ParamOriginTLS = nullptr;
691 VAArgTLS = nullptr;
692 VAArgOriginTLS = nullptr;
693 VAArgOverflowSizeTLS = nullptr;
694
695 WarningFn = M.getOrInsertFunction(Name: "__msan_warning",
696 AttributeList: TLI.getAttrList(C, ArgNos: {0}, /*Signed=*/false),
697 RetTy: IRB.getVoidTy(), Args: IRB.getInt32Ty());
698
699 // Requests the per-task context state (kmsan_context_state*) from the
700 // runtime library.
701 MsanContextStateTy = StructType::get(
702 elt1: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8),
703 elts: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kRetvalTLSSize / 8),
704 elts: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8),
705 elts: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8), /* va_arg_origin */
706 elts: IRB.getInt64Ty(), elts: ArrayType::get(ElementType: OriginTy, NumElements: kParamTLSSize / 4), elts: OriginTy,
707 elts: OriginTy);
708 MsanGetContextStateFn =
709 M.getOrInsertFunction(Name: "__msan_get_context_state", RetTy: PtrTy);
710
711 MsanMetadata = StructType::get(elt1: PtrTy, elts: PtrTy);
712
713 for (int ind = 0, size = 1; ind < 4; ind++, size <<= 1) {
714 std::string name_load =
715 "__msan_metadata_ptr_for_load_" + std::to_string(val: size);
716 std::string name_store =
717 "__msan_metadata_ptr_for_store_" + std::to_string(val: size);
718 MsanMetadataPtrForLoad_1_8[ind] =
719 getOrInsertMsanMetadataFunction(M, Name: name_load, Args: PtrTy);
720 MsanMetadataPtrForStore_1_8[ind] =
721 getOrInsertMsanMetadataFunction(M, Name: name_store, Args: PtrTy);
722 }
723
724 MsanMetadataPtrForLoadN = getOrInsertMsanMetadataFunction(
725 M, Name: "__msan_metadata_ptr_for_load_n", Args: PtrTy, Args: IntptrTy);
726 MsanMetadataPtrForStoreN = getOrInsertMsanMetadataFunction(
727 M, Name: "__msan_metadata_ptr_for_store_n", Args: PtrTy, Args: IntptrTy);
728
729 // Functions for poisoning and unpoisoning memory.
730 MsanPoisonAllocaFn = M.getOrInsertFunction(
731 Name: "__msan_poison_alloca", RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy, Args: PtrTy);
732 MsanUnpoisonAllocaFn = M.getOrInsertFunction(
733 Name: "__msan_unpoison_alloca", RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy);
734}
735
736static Constant *getOrInsertGlobal(Module &M, StringRef Name, Type *Ty) {
737 return M.getOrInsertGlobal(Name, Ty, CreateGlobalCallback: [&] {
738 return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
739 nullptr, Name, nullptr,
740 GlobalVariable::InitialExecTLSModel);
741 });
742}
743
744/// Insert declarations for userspace-specific functions and globals.
745void MemorySanitizer::createUserspaceApi(Module &M,
746 const TargetLibraryInfo &TLI) {
747 IRBuilder<> IRB(M);
748
749 // Create the callback.
750 // FIXME: this function should have "Cold" calling conv,
751 // which is not yet implemented.
752 if (TrackOrigins) {
753 StringRef WarningFnName = Recover ? "__msan_warning_with_origin"
754 : "__msan_warning_with_origin_noreturn";
755 WarningFn = M.getOrInsertFunction(Name: WarningFnName,
756 AttributeList: TLI.getAttrList(C, ArgNos: {0}, /*Signed=*/false),
757 RetTy: IRB.getVoidTy(), Args: IRB.getInt32Ty());
758 } else {
759 StringRef WarningFnName =
760 Recover ? "__msan_warning" : "__msan_warning_noreturn";
761 WarningFn = M.getOrInsertFunction(Name: WarningFnName, RetTy: IRB.getVoidTy());
762 }
763
764 // Create the global TLS variables.
765 RetvalTLS =
766 getOrInsertGlobal(M, Name: "__msan_retval_tls",
767 Ty: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kRetvalTLSSize / 8));
768
769 RetvalOriginTLS = getOrInsertGlobal(M, Name: "__msan_retval_origin_tls", Ty: OriginTy);
770
771 ParamTLS =
772 getOrInsertGlobal(M, Name: "__msan_param_tls",
773 Ty: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8));
774
775 ParamOriginTLS =
776 getOrInsertGlobal(M, Name: "__msan_param_origin_tls",
777 Ty: ArrayType::get(ElementType: OriginTy, NumElements: kParamTLSSize / 4));
778
779 VAArgTLS =
780 getOrInsertGlobal(M, Name: "__msan_va_arg_tls",
781 Ty: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8));
782
783 VAArgOriginTLS =
784 getOrInsertGlobal(M, Name: "__msan_va_arg_origin_tls",
785 Ty: ArrayType::get(ElementType: OriginTy, NumElements: kParamTLSSize / 4));
786
787 VAArgOverflowSizeTLS = getOrInsertGlobal(M, Name: "__msan_va_arg_overflow_size_tls",
788 Ty: IRB.getIntPtrTy(DL: M.getDataLayout()));
789
790 for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
791 AccessSizeIndex++) {
792 unsigned AccessSize = 1 << AccessSizeIndex;
793 std::string FunctionName = "__msan_maybe_warning_" + itostr(X: AccessSize);
794 MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction(
795 Name: FunctionName, AttributeList: TLI.getAttrList(C, ArgNos: {0, 1}, /*Signed=*/false),
796 RetTy: IRB.getVoidTy(), Args: IRB.getIntNTy(N: AccessSize * 8), Args: IRB.getInt32Ty());
797 MaybeWarningVarSizeFn = M.getOrInsertFunction(
798 Name: "__msan_maybe_warning_N", AttributeList: TLI.getAttrList(C, ArgNos: {}, /*Signed=*/false),
799 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IRB.getInt64Ty(), Args: IRB.getInt32Ty());
800 FunctionName = "__msan_maybe_store_origin_" + itostr(X: AccessSize);
801 MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction(
802 Name: FunctionName, AttributeList: TLI.getAttrList(C, ArgNos: {0, 2}, /*Signed=*/false),
803 RetTy: IRB.getVoidTy(), Args: IRB.getIntNTy(N: AccessSize * 8), Args: PtrTy,
804 Args: IRB.getInt32Ty());
805 }
806
807 MsanSetAllocaOriginWithDescriptionFn =
808 M.getOrInsertFunction(Name: "__msan_set_alloca_origin_with_descr",
809 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy, Args: PtrTy, Args: PtrTy);
810 MsanSetAllocaOriginNoDescriptionFn =
811 M.getOrInsertFunction(Name: "__msan_set_alloca_origin_no_descr",
812 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy, Args: PtrTy);
813 MsanPoisonStackFn = M.getOrInsertFunction(Name: "__msan_poison_stack",
814 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy);
815}
816
817/// Insert extern declaration of runtime-provided functions and globals.
818void MemorySanitizer::initializeCallbacks(Module &M,
819 const TargetLibraryInfo &TLI) {
820 // Only do this once.
821 if (CallbacksInitialized)
822 return;
823
824 IRBuilder<> IRB(M);
825 // Initialize callbacks that are common for kernel and userspace
826 // instrumentation.
827 MsanChainOriginFn = M.getOrInsertFunction(
828 Name: "__msan_chain_origin",
829 AttributeList: TLI.getAttrList(C, ArgNos: {0}, /*Signed=*/false, /*Ret=*/true), RetTy: IRB.getInt32Ty(),
830 Args: IRB.getInt32Ty());
831 MsanSetOriginFn = M.getOrInsertFunction(
832 Name: "__msan_set_origin", AttributeList: TLI.getAttrList(C, ArgNos: {2}, /*Signed=*/false),
833 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy, Args: IRB.getInt32Ty());
834 MemmoveFn =
835 M.getOrInsertFunction(Name: "__msan_memmove", RetTy: PtrTy, Args: PtrTy, Args: PtrTy, Args: IntptrTy);
836 MemcpyFn =
837 M.getOrInsertFunction(Name: "__msan_memcpy", RetTy: PtrTy, Args: PtrTy, Args: PtrTy, Args: IntptrTy);
838 MemsetFn = M.getOrInsertFunction(Name: "__msan_memset",
839 AttributeList: TLI.getAttrList(C, ArgNos: {1}, /*Signed=*/true),
840 RetTy: PtrTy, Args: PtrTy, Args: IRB.getInt32Ty(), Args: IntptrTy);
841
842 MsanInstrumentAsmStoreFn = M.getOrInsertFunction(
843 Name: "__msan_instrument_asm_store", RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy);
844
845 if (CompileKernel) {
846 createKernelApi(M, TLI);
847 } else {
848 createUserspaceApi(M, TLI);
849 }
850 CallbacksInitialized = true;
851}
852
853FunctionCallee MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore,
854 int size) {
855 FunctionCallee *Fns =
856 isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
857 switch (size) {
858 case 1:
859 return Fns[0];
860 case 2:
861 return Fns[1];
862 case 4:
863 return Fns[2];
864 case 8:
865 return Fns[3];
866 default:
867 return nullptr;
868 }
869}
870
871/// Module-level initialization.
872///
873/// inserts a call to __msan_init to the module's constructor list.
874void MemorySanitizer::initializeModule(Module &M) {
875 auto &DL = M.getDataLayout();
876
877 TargetTriple = M.getTargetTriple();
878
879 bool ShadowPassed = Opts.msan_shadow_base.has_value();
880 bool OriginPassed = Opts.msan_origin_base.has_value();
881 // Check the overrides first
882 if (ShadowPassed || OriginPassed) {
883 CustomMapParams.AndMask = Opts.msan_and_mask;
884 CustomMapParams.XorMask = Opts.msan_xor_mask;
885 CustomMapParams.ShadowBase = Opts.msan_shadow_base.value_or(u: 0);
886 CustomMapParams.OriginBase = Opts.msan_origin_base.value_or(u: 0);
887 MapParams = &CustomMapParams;
888 } else {
889 switch (TargetTriple.getOS()) {
890 case Triple::FreeBSD:
891 switch (TargetTriple.getArch()) {
892 case Triple::aarch64:
893 MapParams = FreeBSD_ARM_MemoryMapParams.bits64;
894 break;
895 case Triple::x86_64:
896 MapParams = FreeBSD_X86_MemoryMapParams.bits64;
897 break;
898 case Triple::x86:
899 MapParams = FreeBSD_X86_MemoryMapParams.bits32;
900 break;
901 default:
902 report_fatal_error(reason: "unsupported architecture");
903 }
904 break;
905 case Triple::NetBSD:
906 switch (TargetTriple.getArch()) {
907 case Triple::x86_64:
908 MapParams = NetBSD_X86_MemoryMapParams.bits64;
909 break;
910 default:
911 report_fatal_error(reason: "unsupported architecture");
912 }
913 break;
914 case Triple::Linux:
915 switch (TargetTriple.getArch()) {
916 case Triple::x86_64:
917 MapParams = Linux_X86_MemoryMapParams.bits64;
918 break;
919 case Triple::x86:
920 MapParams = Linux_X86_MemoryMapParams.bits32;
921 break;
922 case Triple::mips64:
923 case Triple::mips64el:
924 MapParams = Linux_MIPS_MemoryMapParams.bits64;
925 break;
926 case Triple::ppc64:
927 case Triple::ppc64le:
928 MapParams = Linux_PowerPC_MemoryMapParams.bits64;
929 break;
930 case Triple::systemz:
931 MapParams = Linux_S390_MemoryMapParams.bits64;
932 break;
933 case Triple::aarch64:
934 case Triple::aarch64_be:
935 MapParams = Linux_ARM_MemoryMapParams.bits64;
936 break;
937 case Triple::loongarch64:
938 MapParams = Linux_LoongArch_MemoryMapParams.bits64;
939 break;
940 case Triple::hexagon:
941 MapParams = Linux_Hexagon_MemoryMapParams_P.bits32;
942 break;
943 default:
944 report_fatal_error(reason: "unsupported architecture");
945 }
946 break;
947 default:
948 report_fatal_error(reason: "unsupported operating system");
949 }
950 }
951
952 C = &(M.getContext());
953 IRBuilder<> IRB(M);
954 IntptrTy = IRB.getIntPtrTy(DL);
955 OriginTy = IRB.getInt32Ty();
956 PtrTy = IRB.getPtrTy();
957
958 ColdCallWeights = MDBuilder(*C).createUnlikelyBranchWeights();
959 OriginStoreWeights = MDBuilder(*C).createUnlikelyBranchWeights();
960
961 if (!CompileKernel) {
962 if (TrackOrigins)
963 M.getOrInsertGlobal(Name: "__msan_track_origins", Ty: IRB.getInt32Ty(), CreateGlobalCallback: [&] {
964 return new GlobalVariable(
965 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
966 IRB.getInt32(C: TrackOrigins), "__msan_track_origins");
967 });
968
969 if (Recover)
970 M.getOrInsertGlobal(Name: "__msan_keep_going", Ty: IRB.getInt32Ty(), CreateGlobalCallback: [&] {
971 return new GlobalVariable(M, IRB.getInt32Ty(), true,
972 GlobalValue::WeakODRLinkage,
973 IRB.getInt32(C: Recover), "__msan_keep_going");
974 });
975 }
976}
977
978namespace {
979
980/// A helper class that handles instrumentation of VarArg
981/// functions on a particular platform.
982///
983/// Implementations are expected to insert the instrumentation
984/// necessary to propagate argument shadow through VarArg function
985/// calls. Visit* methods are called during an InstVisitor pass over
986/// the function, and should avoid creating new basic blocks. A new
987/// instance of this class is created for each instrumented function.
988struct VarArgHelper {
989 virtual ~VarArgHelper() = default;
990
991 /// Visit a CallBase.
992 virtual void visitCallBase(CallBase &CB, IRBuilder<> &IRB) = 0;
993
994 /// Visit a va_start call.
995 virtual void visitVAStartInst(VAStartInst &I) = 0;
996
997 /// Visit a va_copy call.
998 virtual void visitVACopyInst(VACopyInst &I) = 0;
999
1000 /// Finalize function instrumentation.
1001 ///
1002 /// This method is called after visiting all interesting (see above)
1003 /// instructions in a function.
1004 virtual void finalizeInstrumentation() = 0;
1005};
1006
1007struct MemorySanitizerVisitor;
1008
1009} // end anonymous namespace
1010
1011static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
1012 MemorySanitizerVisitor &Visitor);
1013
1014static unsigned TypeSizeToSizeIndex(TypeSize TS) {
1015 if (TS.isScalable())
1016 // Scalable types unconditionally take slowpaths.
1017 return kNumberOfAccessSizes;
1018 unsigned TypeSizeFixed = TS.getFixedValue();
1019 if (TypeSizeFixed <= 8)
1020 return 0;
1021 return Log2_32_Ceil(Value: (TypeSizeFixed + 7) / 8);
1022}
1023
1024namespace {
1025
1026/// Helper class to attach debug information of the given instruction onto new
1027/// instructions inserted after.
1028class NextNodeIRBuilder : public IRBuilder<> {
1029public:
1030 explicit NextNodeIRBuilder(Instruction *IP) : IRBuilder<>(IP->getNextNode()) {
1031 SetCurrentDebugLocation(IP->getDebugLoc());
1032 }
1033};
1034
1035/// This class does all the work for a given function. Store and Load
1036/// instructions store and load corresponding shadow and origin
1037/// values. Most instructions propagate shadow from arguments to their
1038/// return values. Certain instructions (most importantly, BranchInst)
1039/// test their argument shadow and print reports (with a runtime call) if it's
1040/// non-zero.
1041struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
1042 const InstrumentationOptions &Opts;
1043 Function &F;
1044 MemorySanitizer &MS;
1045 SmallVector<PHINode *, 16> ShadowPHINodes, OriginPHINodes;
1046 ValueMap<Value *, Value *> ShadowMap, OriginMap;
1047 std::unique_ptr<VarArgHelper> VAHelper;
1048 const TargetLibraryInfo *TLI;
1049 Instruction *FnPrologueEnd;
1050 SmallVector<Instruction *, 16> Instructions;
1051
1052 // The following flags disable parts of MSan instrumentation based on
1053 // exclusion list contents and command-line options.
1054 bool InsertChecks;
1055 bool PropagateShadow;
1056 bool PoisonStack;
1057 bool PoisonUndef;
1058 bool PoisonUndefVectors;
1059
1060 struct ShadowOriginAndInsertPoint {
1061 Value *Shadow;
1062 Value *Origin;
1063 Instruction *OrigIns;
1064
1065 ShadowOriginAndInsertPoint(Value *S, Value *O, Instruction *I)
1066 : Shadow(S), Origin(O), OrigIns(I) {}
1067 };
1068 SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList;
1069 DenseMap<const DILocation *, int> LazyWarningDebugLocationCount;
1070 SmallSetVector<AllocaInst *, 16> AllocaSet;
1071 SmallVector<std::pair<IntrinsicInst *, AllocaInst *>, 16> LifetimeStartList;
1072 SmallVector<StoreInst *, 16> StoreList;
1073 int64_t SplittableBlocksCount = 0;
1074
1075 MemorySanitizerVisitor(Function &F, MemorySanitizer &MS,
1076 const TargetLibraryInfo &TLI)
1077 : Opts(MS.Opts), F(F), MS(MS), VAHelper(CreateVarArgHelper(Func&: F, Msan&: MS, Visitor&: *this)),
1078 TLI(&TLI) {
1079 bool SanitizeFunction = F.hasFnAttribute(Kind: Attribute::SanitizeMemory) &&
1080 !Opts.msan_disable_checks;
1081 InsertChecks = SanitizeFunction;
1082 PropagateShadow = SanitizeFunction;
1083 PoisonStack = SanitizeFunction && Opts.msan_poison_stack;
1084 PoisonUndef = SanitizeFunction && Opts.msan_poison_undef;
1085 PoisonUndefVectors = SanitizeFunction && Opts.msan_poison_undef_vectors;
1086
1087 // In the presence of unreachable blocks, we may see Phi nodes with
1088 // incoming nodes from such blocks. Since InstVisitor skips unreachable
1089 // blocks, such nodes will not have any shadow value associated with them.
1090 // It's easier to remove unreachable blocks than deal with missing shadow.
1091 removeUnreachableBlocks(F);
1092
1093 MS.initializeCallbacks(M&: *F.getParent(), TLI);
1094 FnPrologueEnd =
1095 IRBuilder<>(F.getEntryBlock().getFirstNonPHIIt())
1096 .CreateIntrinsicWithoutFolding(ID: Intrinsic::donothing, Args: {});
1097
1098 if (MS.CompileKernel) {
1099 IRBuilder<> IRB(FnPrologueEnd);
1100 insertKmsanPrologue(IRB);
1101 }
1102
1103 LLVM_DEBUG(if (!InsertChecks) dbgs()
1104 << "MemorySanitizer is not inserting checks into '"
1105 << F.getName() << "'\n");
1106 }
1107
1108 bool instrumentWithCalls(Value *V) {
1109 // Constants likely will be eliminated by follow-up passes.
1110 if (isa<Constant>(Val: V))
1111 return false;
1112 ++SplittableBlocksCount;
1113 return Opts.msan_instrumentation_with_call_threshold >= 0 &&
1114 SplittableBlocksCount >
1115 Opts.msan_instrumentation_with_call_threshold;
1116 }
1117
1118 bool isInPrologue(Instruction &I) {
1119 return I.getParent() == FnPrologueEnd->getParent() &&
1120 (&I == FnPrologueEnd || I.comesBefore(Other: FnPrologueEnd));
1121 }
1122
1123 // Creates a new origin and records the stack trace. In general we can call
1124 // this function for any origin manipulation we like. However it will cost
1125 // runtime resources. So use this wisely only if it can provide additional
1126 // information helpful to a user.
1127 Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
1128 if (MS.TrackOrigins <= 1)
1129 return V;
1130 return IRB.CreateCall(Callee: MS.MsanChainOriginFn, Args: V);
1131 }
1132
1133 Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) {
1134 const DataLayout &DL = F.getDataLayout();
1135 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
1136 if (IntptrSize == kOriginSize)
1137 return Origin;
1138 assert(IntptrSize == kOriginSize * 2);
1139 Origin = IRB.CreateIntCast(V: Origin, DestTy: MS.IntptrTy, /* isSigned */ false);
1140 return IRB.CreateOr(LHS: Origin, RHS: IRB.CreateShl(LHS: Origin, RHS: kOriginSize * 8));
1141 }
1142
1143 /// Fill memory range with the given origin value.
1144 void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
1145 TypeSize TS, Align Alignment) {
1146 const DataLayout &DL = F.getDataLayout();
1147 const Align IntptrAlignment = DL.getABITypeAlign(Ty: MS.IntptrTy);
1148 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
1149 assert(IntptrAlignment >= kMinOriginAlignment);
1150 assert(IntptrSize >= kOriginSize);
1151
1152 // Note: The loop based formation works for fixed length vectors too,
1153 // however we prefer to unroll and specialize alignment below.
1154 if (TS.isScalable()) {
1155 Value *Size = IRB.CreateTypeSize(Ty: MS.IntptrTy, Size: TS);
1156 Value *RoundUp =
1157 IRB.CreateAdd(LHS: Size, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kOriginSize - 1));
1158 Value *End =
1159 IRB.CreateUDiv(LHS: RoundUp, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kOriginSize));
1160 auto [InsertPt, Index] =
1161 SplitBlockAndInsertSimpleForLoop(End, SplitBefore: IRB.GetInsertPoint());
1162 IRB.SetInsertPoint(InsertPt);
1163
1164 Value *GEP = IRB.CreateGEP(Ty: MS.OriginTy, Ptr: OriginPtr, IdxList: Index);
1165 IRB.CreateAlignedStore(Val: Origin, Ptr: GEP, Align: kMinOriginAlignment);
1166 return;
1167 }
1168
1169 unsigned Size = TS.getFixedValue();
1170
1171 unsigned Ofs = 0;
1172 Align CurrentAlignment = Alignment;
1173 if (Alignment >= IntptrAlignment && IntptrSize > kOriginSize) {
1174 Value *IntptrOrigin = originToIntptr(IRB, Origin);
1175 Value *IntptrOriginPtr = IRB.CreatePointerCast(V: OriginPtr, DestTy: MS.PtrTy);
1176 for (unsigned i = 0; i < Size / IntptrSize; ++i) {
1177 Value *Ptr = i ? IRB.CreateConstGEP1_32(Ty: MS.IntptrTy, Ptr: IntptrOriginPtr, Idx0: i)
1178 : IntptrOriginPtr;
1179 IRB.CreateAlignedStore(Val: IntptrOrigin, Ptr, Align: CurrentAlignment);
1180 Ofs += IntptrSize / kOriginSize;
1181 CurrentAlignment = IntptrAlignment;
1182 }
1183 }
1184
1185 for (unsigned i = Ofs; i < (Size + kOriginSize - 1) / kOriginSize; ++i) {
1186 Value *GEP =
1187 i ? IRB.CreateConstGEP1_32(Ty: MS.OriginTy, Ptr: OriginPtr, Idx0: i) : OriginPtr;
1188 IRB.CreateAlignedStore(Val: Origin, Ptr: GEP, Align: CurrentAlignment);
1189 CurrentAlignment = kMinOriginAlignment;
1190 }
1191 }
1192
1193 void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
1194 Value *OriginPtr, Align Alignment) {
1195 const DataLayout &DL = F.getDataLayout();
1196 const Align OriginAlignment = std::max(a: kMinOriginAlignment, b: Alignment);
1197 TypeSize StoreSize = DL.getTypeStoreSize(Ty: Shadow->getType());
1198 // ZExt cannot convert between vector and scalar
1199 Value *ConvertedShadow = convertShadowToScalar(V: Shadow, IRB);
1200 if (auto *ConstantShadow = dyn_cast<Constant>(Val: ConvertedShadow)) {
1201 if (!Opts.msan_check_constant_shadow || ConstantShadow->isNullValue()) {
1202 // Origin is not needed: value is initialized or const shadow is
1203 // ignored.
1204 return;
1205 }
1206 if (llvm::isKnownNonZero(V: ConvertedShadow, Q: DL)) {
1207 // Copy origin as the value is definitely uninitialized.
1208 paintOrigin(IRB, Origin: updateOrigin(V: Origin, IRB), OriginPtr, TS: StoreSize,
1209 Alignment: OriginAlignment);
1210 return;
1211 }
1212 // Fallback to runtime check, which still can be optimized out later.
1213 }
1214
1215 TypeSize TypeSizeInBits = DL.getTypeSizeInBits(Ty: ConvertedShadow->getType());
1216 unsigned SizeIndex = TypeSizeToSizeIndex(TS: TypeSizeInBits);
1217 if (instrumentWithCalls(V: ConvertedShadow) &&
1218 SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
1219 FunctionCallee Fn = MS.MaybeStoreOriginFn[SizeIndex];
1220 Value *ConvertedShadow2 =
1221 IRB.CreateZExt(V: ConvertedShadow, DestTy: IRB.getIntNTy(N: 8 * (1 << SizeIndex)));
1222 CallBase *CB = IRB.CreateCall(Callee: Fn, Args: {ConvertedShadow2, Addr, Origin});
1223 CB->addParamAttr(ArgNo: 0, Kind: Attribute::ZExt);
1224 CB->addParamAttr(ArgNo: 2, Kind: Attribute::ZExt);
1225 } else {
1226 Value *Cmp = convertToBool(V: ConvertedShadow, IRB, name: "_mscmp");
1227 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1228 Cond: Cmp, SplitBefore: &*IRB.GetInsertPoint(), Unreachable: false, BranchWeights: MS.OriginStoreWeights);
1229 IRBuilder<> IRBNew(CheckTerm);
1230 paintOrigin(IRB&: IRBNew, Origin: updateOrigin(V: Origin, IRB&: IRBNew), OriginPtr, TS: StoreSize,
1231 Alignment: OriginAlignment);
1232 }
1233 }
1234
1235 void materializeStores() {
1236 for (StoreInst *SI : StoreList) {
1237 IRBuilder<> IRB(SI);
1238 Value *Val = SI->getValueOperand();
1239 Value *Addr = SI->getPointerOperand();
1240 Value *Shadow = SI->isAtomic() ? getCleanShadow(V: Val) : getShadow(V: Val);
1241 Value *ShadowPtr, *OriginPtr;
1242 Type *ShadowTy = Shadow->getType();
1243 const Align Alignment = SI->getAlign();
1244 const Align OriginAlignment = std::max(a: kMinOriginAlignment, b: Alignment);
1245 std::tie(args&: ShadowPtr, args&: OriginPtr) =
1246 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
1247
1248 [[maybe_unused]] StoreInst *NewSI =
1249 IRB.CreateAlignedStore(Val: Shadow, Ptr: ShadowPtr, Align: Alignment);
1250 LLVM_DEBUG(dbgs() << " STORE: " << *NewSI << "\n");
1251
1252 if (SI->isAtomic())
1253 SI->setOrdering(addReleaseOrdering(a: SI->getOrdering()));
1254
1255 if (MS.TrackOrigins && !SI->isAtomic())
1256 storeOrigin(IRB, Addr, Shadow, Origin: getOrigin(V: Val), OriginPtr,
1257 Alignment: OriginAlignment);
1258 }
1259 }
1260
1261 // Returns true if Debug Location corresponds to multiple warnings.
1262 bool shouldDisambiguateWarningLocation(const DebugLoc &DebugLoc) {
1263 if (MS.TrackOrigins < 2)
1264 return false;
1265
1266 if (LazyWarningDebugLocationCount.empty())
1267 for (const auto &I : InstrumentationList)
1268 ++LazyWarningDebugLocationCount[I.OrigIns->getDebugLoc()];
1269
1270 return LazyWarningDebugLocationCount[DebugLoc] >=
1271 Opts.msan_disambiguate_warning_threshold;
1272 }
1273
1274 /// Helper function to insert a warning at IRB's current insert point.
1275 void insertWarningFn(IRBuilder<> &IRB, Value *Origin) {
1276 if (!Origin)
1277 Origin = (Value *)IRB.getInt32(C: 0);
1278 assert(Origin->getType()->isIntegerTy());
1279
1280 if (shouldDisambiguateWarningLocation(DebugLoc: IRB.getCurrentDebugLocation())) {
1281 // Try to create additional origin with debug info of the last origin
1282 // instruction. It may provide additional information to the user.
1283 if (Instruction *OI = dyn_cast_or_null<Instruction>(Val: Origin)) {
1284 assert(MS.TrackOrigins);
1285 auto NewDebugLoc = OI->getDebugLoc();
1286 // Origin update with missing or the same debug location provides no
1287 // additional value.
1288 if (NewDebugLoc && NewDebugLoc != IRB.getCurrentDebugLocation()) {
1289 // Insert update just before the check, so we call runtime only just
1290 // before the report.
1291 IRBuilder<> IRBOrigin(&*IRB.GetInsertPoint());
1292 IRBOrigin.SetCurrentDebugLocation(NewDebugLoc);
1293 Origin = updateOrigin(V: Origin, IRB&: IRBOrigin);
1294 }
1295 }
1296 }
1297
1298 if (MS.CompileKernel || MS.TrackOrigins)
1299 IRB.CreateCall(Callee: MS.WarningFn, Args: Origin)->setCannotMerge();
1300 else
1301 IRB.CreateCall(Callee: MS.WarningFn)->setCannotMerge();
1302 // FIXME: Insert UnreachableInst if !MS.Recover?
1303 // This may invalidate some of the following checks and needs to be done
1304 // at the very end.
1305 }
1306
1307 void materializeOneCheck(IRBuilder<> &IRB, Value *ConvertedShadow,
1308 Value *Origin) {
1309 const DataLayout &DL = F.getDataLayout();
1310 TypeSize TypeSizeInBits = DL.getTypeSizeInBits(Ty: ConvertedShadow->getType());
1311 unsigned SizeIndex = TypeSizeToSizeIndex(TS: TypeSizeInBits);
1312 if (instrumentWithCalls(V: ConvertedShadow) && !MS.CompileKernel) {
1313 // ZExt cannot convert between vector and scalar
1314 ConvertedShadow = convertShadowToScalar(V: ConvertedShadow, IRB);
1315 Value *ConvertedShadow2 =
1316 IRB.CreateZExt(V: ConvertedShadow, DestTy: IRB.getIntNTy(N: 8 * (1 << SizeIndex)));
1317
1318 if (SizeIndex < kNumberOfAccessSizes) {
1319 FunctionCallee Fn = MS.MaybeWarningFn[SizeIndex];
1320 CallBase *CB = IRB.CreateCall(
1321 Callee: Fn,
1322 Args: {ConvertedShadow2,
1323 MS.TrackOrigins && Origin ? Origin : (Value *)IRB.getInt32(C: 0)});
1324 CB->addParamAttr(ArgNo: 0, Kind: Attribute::ZExt);
1325 CB->addParamAttr(ArgNo: 1, Kind: Attribute::ZExt);
1326 } else {
1327 FunctionCallee Fn = MS.MaybeWarningVarSizeFn;
1328 Value *ShadowAlloca = IRB.CreateAlloca(Ty: ConvertedShadow2->getType(), AddrSpace: 0u);
1329 IRB.CreateStore(Val: ConvertedShadow2, Ptr: ShadowAlloca);
1330 unsigned ShadowSize = DL.getTypeAllocSize(Ty: ConvertedShadow2->getType());
1331 CallBase *CB = IRB.CreateCall(
1332 Callee: Fn,
1333 Args: {ShadowAlloca, ConstantInt::get(Ty: IRB.getInt64Ty(), V: ShadowSize),
1334 MS.TrackOrigins && Origin ? Origin : (Value *)IRB.getInt32(C: 0)});
1335 CB->addParamAttr(ArgNo: 1, Kind: Attribute::ZExt);
1336 CB->addParamAttr(ArgNo: 2, Kind: Attribute::ZExt);
1337 }
1338 } else {
1339 Value *Cmp = convertToBool(V: ConvertedShadow, IRB, name: "_mscmp");
1340 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1341 Cond: Cmp, SplitBefore: &*IRB.GetInsertPoint(),
1342 /* Unreachable */ !MS.Recover, BranchWeights: MS.ColdCallWeights);
1343
1344 IRB.SetInsertPoint(CheckTerm);
1345 insertWarningFn(IRB, Origin);
1346 LLVM_DEBUG(dbgs() << " CHECK: " << *Cmp << "\n");
1347 }
1348 }
1349
1350 void materializeInstructionChecks(
1351 ArrayRef<ShadowOriginAndInsertPoint> InstructionChecks) {
1352 const DataLayout &DL = F.getDataLayout();
1353 // Disable combining in some cases. TrackOrigins checks each shadow to pick
1354 // correct origin.
1355 bool Combine = !MS.TrackOrigins;
1356 Instruction *Instruction = InstructionChecks.front().OrigIns;
1357 Value *Shadow = nullptr;
1358 for (const auto &ShadowData : InstructionChecks) {
1359 assert(ShadowData.OrigIns == Instruction);
1360 IRBuilder<> IRB(Instruction);
1361
1362 Value *ConvertedShadow = ShadowData.Shadow;
1363
1364 if (auto *ConstantShadow = dyn_cast<Constant>(Val: ConvertedShadow)) {
1365 if (!Opts.msan_check_constant_shadow || ConstantShadow->isNullValue()) {
1366 // Skip, value is initialized or const shadow is ignored.
1367 continue;
1368 }
1369 if (llvm::isKnownNonZero(V: ConvertedShadow, Q: DL)) {
1370 // Report as the value is definitely uninitialized.
1371 insertWarningFn(IRB, Origin: ShadowData.Origin);
1372 if (!MS.Recover)
1373 return; // Always fail and stop here, not need to check the rest.
1374 // Skip entire instruction,
1375 continue;
1376 }
1377 // Fallback to runtime check, which still can be optimized out later.
1378 }
1379
1380 if (!Combine) {
1381 materializeOneCheck(IRB, ConvertedShadow, Origin: ShadowData.Origin);
1382 continue;
1383 }
1384
1385 if (!Shadow) {
1386 Shadow = ConvertedShadow;
1387 continue;
1388 }
1389
1390 Shadow = convertToBool(V: Shadow, IRB, name: "_mscmp");
1391 ConvertedShadow = convertToBool(V: ConvertedShadow, IRB, name: "_mscmp");
1392 Shadow = IRB.CreateOr(LHS: Shadow, RHS: ConvertedShadow, Name: "_msor");
1393 }
1394
1395 if (Shadow) {
1396 assert(Combine);
1397 IRBuilder<> IRB(Instruction);
1398 materializeOneCheck(IRB, ConvertedShadow: Shadow, Origin: nullptr);
1399 }
1400 }
1401
1402 static bool isAArch64SVCount(Type *Ty) {
1403 if (TargetExtType *TTy = dyn_cast<TargetExtType>(Val: Ty))
1404 return TTy->getName() == "aarch64.svcount";
1405 return false;
1406 }
1407
1408 // This is intended to match the "AArch64 Predicate-as-Counter Type" (aka
1409 // 'target("aarch64.svcount")', but not e.g., <vscale x 4 x i32>.
1410 static bool isScalableNonVectorType(Type *Ty) {
1411 if (!isAArch64SVCount(Ty))
1412 LLVM_DEBUG(dbgs() << "isScalableNonVectorType: Unexpected type " << *Ty
1413 << "\n");
1414
1415 return Ty->isScalableTy() && !isa<VectorType>(Val: Ty);
1416 }
1417
1418 void materializeChecks() {
1419#ifndef NDEBUG
1420 // For assert below.
1421 SmallPtrSet<Instruction *, 16> Done;
1422#endif
1423
1424 for (auto I = InstrumentationList.begin();
1425 I != InstrumentationList.end();) {
1426 auto OrigIns = I->OrigIns;
1427 // Checks are grouped by the original instruction. We call all
1428 // `insertShadowCheck` for an instruction at once.
1429 assert(Done.insert(OrigIns).second);
1430 auto J = std::find_if(first: I + 1, last: InstrumentationList.end(),
1431 pred: [OrigIns](const ShadowOriginAndInsertPoint &R) {
1432 return OrigIns != R.OrigIns;
1433 });
1434 // Process all checks of instruction at once.
1435 materializeInstructionChecks(InstructionChecks: ArrayRef<ShadowOriginAndInsertPoint>(I, J));
1436 I = J;
1437 }
1438
1439 LLVM_DEBUG(dbgs() << "DONE:\n" << F);
1440 }
1441
1442 // Returns the last instruction in the new prologue
1443 void insertKmsanPrologue(IRBuilder<> &IRB) {
1444 Value *ContextState = IRB.CreateCall(Callee: MS.MsanGetContextStateFn, Args: {});
1445 Constant *Zero = IRB.getInt32(C: 0);
1446 MS.ParamTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1447 IdxList: {Zero, IRB.getInt32(C: 0)}, Name: "param_shadow");
1448 MS.RetvalTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1449 IdxList: {Zero, IRB.getInt32(C: 1)}, Name: "retval_shadow");
1450 MS.VAArgTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1451 IdxList: {Zero, IRB.getInt32(C: 2)}, Name: "va_arg_shadow");
1452 MS.VAArgOriginTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1453 IdxList: {Zero, IRB.getInt32(C: 3)}, Name: "va_arg_origin");
1454 MS.VAArgOverflowSizeTLS =
1455 IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1456 IdxList: {Zero, IRB.getInt32(C: 4)}, Name: "va_arg_overflow_size");
1457 MS.ParamOriginTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1458 IdxList: {Zero, IRB.getInt32(C: 5)}, Name: "param_origin");
1459 MS.RetvalOriginTLS =
1460 IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1461 IdxList: {Zero, IRB.getInt32(C: 6)}, Name: "retval_origin");
1462 if (MS.TargetTriple.getArch() == Triple::systemz)
1463 MS.MsanMetadataAlloca = IRB.CreateAlloca(Ty: MS.MsanMetadata, AddrSpace: 0u);
1464 }
1465
1466 /// Add MemorySanitizer instrumentation to a function.
1467 bool runOnFunction() {
1468 // Iterate all BBs in depth-first order and create shadow instructions
1469 // for all instructions (where applicable).
1470 // For PHI nodes we create dummy shadow PHIs which will be finalized later.
1471 for (BasicBlock *BB : depth_first(G: FnPrologueEnd->getParent()))
1472 visit(BB&: *BB);
1473
1474 // `visit` above only collects instructions. Process them after iterating
1475 // CFG to avoid requirement on CFG transformations.
1476 for (Instruction *I : Instructions)
1477 InstVisitor<MemorySanitizerVisitor>::visit(I&: *I);
1478
1479 // Finalize PHI nodes.
1480 for (PHINode *PN : ShadowPHINodes) {
1481 PHINode *PNS = cast<PHINode>(Val: getShadow(V: PN));
1482 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(Val: getOrigin(V: PN)) : nullptr;
1483 size_t NumValues = PN->getNumIncomingValues();
1484 for (size_t v = 0; v < NumValues; v++) {
1485 PNS->addIncoming(V: getShadow(I: PN, i: v), BB: PN->getIncomingBlock(i: v));
1486 if (PNO)
1487 PNO->addIncoming(V: getOrigin(I: PN, i: v), BB: PN->getIncomingBlock(i: v));
1488 }
1489 }
1490
1491 VAHelper->finalizeInstrumentation();
1492
1493 // Poison llvm.lifetime.start intrinsics, if we haven't fallen back to
1494 // instrumenting only allocas.
1495 if (Opts.msan_handle_lifetime_intrinsics) {
1496 for (auto Item : LifetimeStartList) {
1497 instrumentAlloca(I&: *Item.second, InsPoint: Item.first);
1498 AllocaSet.remove(X: Item.second);
1499 }
1500 }
1501 // Poison the allocas for which we didn't instrument the corresponding
1502 // lifetime intrinsics.
1503 for (AllocaInst *AI : AllocaSet)
1504 instrumentAlloca(I&: *AI);
1505
1506 // Insert shadow value checks.
1507 materializeChecks();
1508
1509 // Delayed instrumentation of StoreInst.
1510 // This may not add new address checks.
1511 materializeStores();
1512
1513 return true;
1514 }
1515
1516 /// Compute the shadow type that corresponds to a given Value.
1517 Type *getShadowTy(Value *V) { return getShadowTy(OrigTy: V->getType()); }
1518
1519 /// Compute the shadow type that corresponds to a given Type.
1520 Type *getShadowTy(Type *OrigTy) {
1521 if (!OrigTy->isSized()) {
1522 return nullptr;
1523 }
1524 // For integer type, shadow is the same as the original type.
1525 // This may return weird-sized types like i1.
1526 if (IntegerType *IT = dyn_cast<IntegerType>(Val: OrigTy))
1527 return IT;
1528 const DataLayout &DL = F.getDataLayout();
1529 if (VectorType *VT = dyn_cast<VectorType>(Val: OrigTy)) {
1530 uint32_t EltSize = DL.getTypeSizeInBits(Ty: VT->getElementType());
1531 return VectorType::get(ElementType: IntegerType::get(C&: *MS.C, NumBits: EltSize),
1532 EC: VT->getElementCount());
1533 }
1534 if (ArrayType *AT = dyn_cast<ArrayType>(Val: OrigTy)) {
1535 return ArrayType::get(ElementType: getShadowTy(OrigTy: AT->getElementType()),
1536 NumElements: AT->getNumElements());
1537 }
1538 if (StructType *ST = dyn_cast<StructType>(Val: OrigTy)) {
1539 SmallVector<Type *, 4> Elements;
1540 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1541 Elements.push_back(Elt: getShadowTy(OrigTy: ST->getElementType(N: i)));
1542 StructType *Res = StructType::get(Context&: *MS.C, Elements, isPacked: ST->isPacked());
1543 LLVM_DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n");
1544 return Res;
1545 }
1546 if (isScalableNonVectorType(Ty: OrigTy)) {
1547 LLVM_DEBUG(dbgs() << "getShadowTy: Scalable non-vector type: " << *OrigTy
1548 << "\n");
1549 return OrigTy;
1550 }
1551
1552 uint32_t TypeSize = DL.getTypeSizeInBits(Ty: OrigTy);
1553 return IntegerType::get(C&: *MS.C, NumBits: TypeSize);
1554 }
1555
1556 /// Extract combined shadow of struct elements as a bool
1557 Value *collapseStructShadow(StructType *Struct, Value *Shadow,
1558 IRBuilder<> &IRB) {
1559 Value *FalseVal = IRB.getIntN(/* width */ N: 1, /* value */ C: 0);
1560 Value *Aggregator = FalseVal;
1561
1562 for (unsigned Idx = 0; Idx < Struct->getNumElements(); Idx++) {
1563 // Combine by ORing together each element's bool shadow
1564 Value *ShadowItem = IRB.CreateExtractValue(Agg: Shadow, Idxs: Idx);
1565 Value *ShadowBool = convertToBool(V: ShadowItem, IRB);
1566
1567 if (Aggregator != FalseVal)
1568 Aggregator = IRB.CreateOr(LHS: Aggregator, RHS: ShadowBool);
1569 else
1570 Aggregator = ShadowBool;
1571 }
1572
1573 return Aggregator;
1574 }
1575
1576 // Extract combined shadow of array elements
1577 Value *collapseArrayShadow(ArrayType *Array, Value *Shadow,
1578 IRBuilder<> &IRB) {
1579 if (!Array->getNumElements())
1580 return IRB.getIntN(/* width */ N: 1, /* value */ C: 0);
1581
1582 Value *FirstItem = IRB.CreateExtractValue(Agg: Shadow, Idxs: 0);
1583 Value *Aggregator = convertShadowToScalar(V: FirstItem, IRB);
1584
1585 for (unsigned Idx = 1; Idx < Array->getNumElements(); Idx++) {
1586 Value *ShadowItem = IRB.CreateExtractValue(Agg: Shadow, Idxs: Idx);
1587 Value *ShadowInner = convertShadowToScalar(V: ShadowItem, IRB);
1588 Aggregator = IRB.CreateOr(LHS: Aggregator, RHS: ShadowInner);
1589 }
1590 return Aggregator;
1591 }
1592
1593 /// Convert a shadow value to it's flattened variant. The resulting
1594 /// shadow may not necessarily have the same bit width as the input
1595 /// value, but it will always be comparable to zero.
1596 Value *convertShadowToScalar(Value *V, IRBuilder<> &IRB) {
1597 if (StructType *Struct = dyn_cast<StructType>(Val: V->getType()))
1598 return collapseStructShadow(Struct, Shadow: V, IRB);
1599 if (ArrayType *Array = dyn_cast<ArrayType>(Val: V->getType()))
1600 return collapseArrayShadow(Array, Shadow: V, IRB);
1601 if (isa<VectorType>(Val: V->getType())) {
1602 if (isa<ScalableVectorType>(Val: V->getType()))
1603 return convertShadowToScalar(V: IRB.CreateOrReduce(Src: V), IRB);
1604 unsigned BitWidth =
1605 V->getType()->getPrimitiveSizeInBits().getFixedValue();
1606 return IRB.CreateBitCast(V, DestTy: IntegerType::get(C&: *MS.C, NumBits: BitWidth));
1607 }
1608 return V;
1609 }
1610
1611 // Convert a scalar value to an i1 by comparing with 0
1612 Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &name = "") {
1613 Type *VTy = V->getType();
1614 if (!VTy->isIntegerTy())
1615 return convertToBool(V: convertShadowToScalar(V, IRB), IRB, name);
1616 if (VTy->getIntegerBitWidth() == 1)
1617 // Just converting a bool to a bool, so do nothing.
1618 return V;
1619 return IRB.CreateICmpNE(LHS: V, RHS: ConstantInt::get(Ty: VTy, V: 0), Name: name);
1620 }
1621
1622 Type *ptrToIntPtrType(Type *PtrTy) const {
1623 if (VectorType *VectTy = dyn_cast<VectorType>(Val: PtrTy)) {
1624 return VectorType::get(ElementType: ptrToIntPtrType(PtrTy: VectTy->getElementType()),
1625 EC: VectTy->getElementCount());
1626 }
1627 assert(PtrTy->isIntOrPtrTy());
1628 return MS.IntptrTy;
1629 }
1630
1631 Type *getPtrToShadowPtrType(Type *IntPtrTy, Type *ShadowTy) const {
1632 if (VectorType *VectTy = dyn_cast<VectorType>(Val: IntPtrTy)) {
1633 return VectorType::get(
1634 ElementType: getPtrToShadowPtrType(IntPtrTy: VectTy->getElementType(), ShadowTy),
1635 EC: VectTy->getElementCount());
1636 }
1637 assert(IntPtrTy == MS.IntptrTy);
1638 return MS.PtrTy;
1639 }
1640
1641 Constant *constToIntPtr(Type *IntPtrTy, uint64_t C) const {
1642 if (VectorType *VectTy = dyn_cast<VectorType>(Val: IntPtrTy)) {
1643 return ConstantVector::getSplat(
1644 EC: VectTy->getElementCount(),
1645 Elt: constToIntPtr(IntPtrTy: VectTy->getElementType(), C));
1646 }
1647 assert(IntPtrTy == MS.IntptrTy);
1648 // TODO: Avoid implicit trunc?
1649 // See https://github.com/llvm/llvm-project/issues/112510.
1650 return ConstantInt::get(Ty: MS.IntptrTy, V: C, /*IsSigned=*/false,
1651 /*ImplicitTrunc=*/true);
1652 }
1653
1654 /// Returns the integer shadow offset that corresponds to a given
1655 /// application address, whereby:
1656 ///
1657 /// Offset = (Addr & ~AndMask) ^ XorMask
1658 /// Shadow = ShadowBase + Offset
1659 /// Origin = (OriginBase + Offset) & ~Alignment
1660 ///
1661 /// Note: for efficiency, many shadow mappings only require use the XorMask
1662 /// and OriginBase; the AndMask and ShadowBase are often zero.
1663 Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) {
1664 Type *IntptrTy = ptrToIntPtrType(PtrTy: Addr->getType());
1665 Value *OffsetLong = IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy);
1666
1667 if (uint64_t AndMask = MS.MapParams->AndMask)
1668 OffsetLong = IRB.CreateAnd(LHS: OffsetLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: ~AndMask));
1669
1670 if (uint64_t XorMask = MS.MapParams->XorMask)
1671 OffsetLong = IRB.CreateXor(LHS: OffsetLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: XorMask));
1672 return OffsetLong;
1673 }
1674
1675 /// Compute the shadow and origin addresses corresponding to a given
1676 /// application address.
1677 ///
1678 /// Shadow = ShadowBase + Offset
1679 /// Origin = (OriginBase + Offset) & ~3ULL
1680 /// Addr can be a ptr or <N x ptr>. In both cases ShadowTy the shadow type of
1681 /// a single pointee.
1682 /// Returns <shadow_ptr, origin_ptr> or <<N x shadow_ptr>, <N x origin_ptr>>.
1683 std::pair<Value *, Value *>
1684 getShadowOriginPtrUserspace(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy,
1685 MaybeAlign Alignment) {
1686 VectorType *VectTy = dyn_cast<VectorType>(Val: Addr->getType());
1687 if (!VectTy) {
1688 assert(Addr->getType()->isPointerTy());
1689 } else {
1690 assert(VectTy->getElementType()->isPointerTy());
1691 }
1692 Type *IntptrTy = ptrToIntPtrType(PtrTy: Addr->getType());
1693 Value *ShadowOffset = getShadowPtrOffset(Addr, IRB);
1694 Value *ShadowLong = ShadowOffset;
1695 if (uint64_t ShadowBase = MS.MapParams->ShadowBase) {
1696 ShadowLong =
1697 IRB.CreateAdd(LHS: ShadowLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: ShadowBase));
1698 }
1699 Value *ShadowPtr = IRB.CreateIntToPtr(
1700 V: ShadowLong, DestTy: getPtrToShadowPtrType(IntPtrTy: IntptrTy, ShadowTy));
1701
1702 Value *OriginPtr = nullptr;
1703 if (MS.TrackOrigins) {
1704 Value *OriginLong = ShadowOffset;
1705 uint64_t OriginBase = MS.MapParams->OriginBase;
1706 if (OriginBase != 0)
1707 OriginLong =
1708 IRB.CreateAdd(LHS: OriginLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: OriginBase));
1709 if (!Alignment || *Alignment < kMinOriginAlignment) {
1710 uint64_t Mask = kMinOriginAlignment.value() - 1;
1711 OriginLong = IRB.CreateAnd(LHS: OriginLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: ~Mask));
1712 }
1713 OriginPtr = IRB.CreateIntToPtr(
1714 V: OriginLong, DestTy: getPtrToShadowPtrType(IntPtrTy: IntptrTy, ShadowTy: MS.OriginTy));
1715 }
1716 return std::make_pair(x&: ShadowPtr, y&: OriginPtr);
1717 }
1718
1719 template <typename... ArgsTy>
1720 Value *createMetadataCall(IRBuilder<> &IRB, FunctionCallee Callee,
1721 ArgsTy... Args) {
1722 if (MS.TargetTriple.getArch() == Triple::systemz) {
1723 IRB.CreateCall(Callee,
1724 {MS.MsanMetadataAlloca, std::forward<ArgsTy>(Args)...});
1725 return IRB.CreateLoad(Ty: MS.MsanMetadata, Ptr: MS.MsanMetadataAlloca);
1726 }
1727
1728 return IRB.CreateCall(Callee, {std::forward<ArgsTy>(Args)...});
1729 }
1730
1731 std::pair<Value *, Value *> getShadowOriginPtrKernelNoVec(Value *Addr,
1732 IRBuilder<> &IRB,
1733 Type *ShadowTy,
1734 bool isStore) {
1735 Value *ShadowOriginPtrs;
1736 const DataLayout &DL = F.getDataLayout();
1737 TypeSize Size = DL.getTypeStoreSize(Ty: ShadowTy);
1738
1739 FunctionCallee Getter = MS.getKmsanShadowOriginAccessFn(isStore, size: Size);
1740 Value *AddrCast = IRB.CreatePointerCast(V: Addr, DestTy: MS.PtrTy);
1741 if (Getter) {
1742 ShadowOriginPtrs = createMetadataCall(IRB, Callee: Getter, Args: AddrCast);
1743 } else {
1744 Value *SizeVal = ConstantInt::get(Ty: MS.IntptrTy, V: Size);
1745 ShadowOriginPtrs = createMetadataCall(
1746 IRB,
1747 Callee: isStore ? MS.MsanMetadataPtrForStoreN : MS.MsanMetadataPtrForLoadN,
1748 Args: AddrCast, Args: SizeVal);
1749 }
1750 Value *ShadowPtr = IRB.CreateExtractValue(Agg: ShadowOriginPtrs, Idxs: 0);
1751 ShadowPtr = IRB.CreatePointerCast(V: ShadowPtr, DestTy: MS.PtrTy);
1752 Value *OriginPtr = IRB.CreateExtractValue(Agg: ShadowOriginPtrs, Idxs: 1);
1753
1754 return std::make_pair(x&: ShadowPtr, y&: OriginPtr);
1755 }
1756
1757 /// Addr can be a ptr or <N x ptr>. In both cases ShadowTy the shadow type of
1758 /// a single pointee.
1759 /// Returns <shadow_ptr, origin_ptr> or <<N x shadow_ptr>, <N x origin_ptr>>.
1760 std::pair<Value *, Value *> getShadowOriginPtrKernel(Value *Addr,
1761 IRBuilder<> &IRB,
1762 Type *ShadowTy,
1763 bool isStore) {
1764 VectorType *VectTy = dyn_cast<VectorType>(Val: Addr->getType());
1765 if (!VectTy) {
1766 assert(Addr->getType()->isPointerTy());
1767 return getShadowOriginPtrKernelNoVec(Addr, IRB, ShadowTy, isStore);
1768 }
1769
1770 // TODO: Support callbacs with vectors of addresses.
1771 unsigned NumElements = cast<FixedVectorType>(Val: VectTy)->getNumElements();
1772 Value *ShadowPtrs = ConstantInt::getNullValue(
1773 Ty: FixedVectorType::get(ElementType: IRB.getPtrTy(), NumElts: NumElements));
1774 Value *OriginPtrs = nullptr;
1775 if (MS.TrackOrigins)
1776 OriginPtrs = ConstantInt::getNullValue(
1777 Ty: FixedVectorType::get(ElementType: IRB.getPtrTy(), NumElts: NumElements));
1778 for (unsigned i = 0; i < NumElements; ++i) {
1779 Value *OneAddr =
1780 IRB.CreateExtractElement(Vec: Addr, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
1781 auto [ShadowPtr, OriginPtr] =
1782 getShadowOriginPtrKernelNoVec(Addr: OneAddr, IRB, ShadowTy, isStore);
1783
1784 ShadowPtrs = IRB.CreateInsertElement(
1785 Vec: ShadowPtrs, NewElt: ShadowPtr, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
1786 if (MS.TrackOrigins)
1787 OriginPtrs = IRB.CreateInsertElement(
1788 Vec: OriginPtrs, NewElt: OriginPtr, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
1789 }
1790 return {ShadowPtrs, OriginPtrs};
1791 }
1792
1793 std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB,
1794 Type *ShadowTy,
1795 MaybeAlign Alignment,
1796 bool isStore) {
1797 if (MS.CompileKernel)
1798 return getShadowOriginPtrKernel(Addr, IRB, ShadowTy, isStore);
1799 return getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment);
1800 }
1801
1802 /// Compute the shadow address for a given function argument.
1803 ///
1804 /// Shadow = ParamTLS+ArgOffset.
1805 Value *getShadowPtrForArgument(IRBuilder<> &IRB, unsigned ArgOffset) {
1806 return IRB.CreatePtrAddWithUnsignedOffset(Ptr: MS.ParamTLS, Offset: ArgOffset, Name: "_msarg");
1807 }
1808
1809 /// Compute the origin address for a given function argument.
1810 Value *getOriginPtrForArgument(IRBuilder<> &IRB, unsigned ArgOffset) {
1811 if (!MS.TrackOrigins)
1812 return nullptr;
1813 return IRB.CreatePtrAddWithUnsignedOffset(Ptr: MS.ParamOriginTLS, Offset: ArgOffset,
1814 Name: "_msarg_o");
1815 }
1816
1817 /// Compute the shadow address for a retval.
1818 Value *getShadowPtrForRetval(IRBuilder<> &IRB) {
1819 return IRB.CreatePointerCast(V: MS.RetvalTLS, DestTy: IRB.getPtrTy(AddrSpace: 0), Name: "_msret");
1820 }
1821
1822 /// Compute the origin address for a retval.
1823 Value *getOriginPtrForRetval() {
1824 // We keep a single origin for the entire retval. Might be too optimistic.
1825 return MS.RetvalOriginTLS;
1826 }
1827
1828 /// Set SV to be the shadow value for V.
1829 void setShadow(Value *V, Value *SV) {
1830 assert(!ShadowMap.count(V) && "Values may only have one shadow");
1831
1832 // getShadowTy(VoidType) == nullptr
1833 if (SV && !V->getType()->isVoidTy() && getShadowTy(V) != SV->getType()) {
1834 LLVM_DEBUG(dbgs() << "V: " << *V << "\n");
1835 LLVM_DEBUG(dbgs() << "SV: " << *SV << "\n");
1836 assert(getShadowTy(V) == SV->getType() &&
1837 "setShadow called with mismatching types");
1838 }
1839
1840 ShadowMap[V] = PropagateShadow ? SV : getCleanShadow(V);
1841 }
1842
1843 /// Set Origin to be the origin value for V.
1844 void setOrigin(Value *V, Value *Origin) {
1845 if (!MS.TrackOrigins)
1846 return;
1847 assert(!OriginMap.count(V) && "Values may only have one origin");
1848 LLVM_DEBUG(dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n");
1849 OriginMap[V] = Origin;
1850 }
1851
1852 Constant *getCleanShadow(Type *OrigTy) {
1853 Type *ShadowTy = getShadowTy(OrigTy);
1854 if (!ShadowTy)
1855 return nullptr;
1856 return Constant::getNullValue(Ty: ShadowTy);
1857 }
1858
1859 /// Create a clean shadow value for a given value.
1860 ///
1861 /// Clean shadow (all zeroes) means all bits of the value are defined
1862 /// (initialized).
1863 Constant *getCleanShadow(Value *V) { return getCleanShadow(OrigTy: V->getType()); }
1864
1865 /// Create a dirty shadow of a given shadow type.
1866 Constant *getPoisonedShadow(Type *ShadowTy) {
1867 assert(ShadowTy);
1868 if (isa<IntegerType>(Val: ShadowTy) || isa<VectorType>(Val: ShadowTy))
1869 return Constant::getAllOnesValue(Ty: ShadowTy);
1870 if (ArrayType *AT = dyn_cast<ArrayType>(Val: ShadowTy)) {
1871 SmallVector<Constant *, 4> Vals(AT->getNumElements(),
1872 getPoisonedShadow(ShadowTy: AT->getElementType()));
1873 return ConstantArray::get(T: AT, V: Vals);
1874 }
1875 if (StructType *ST = dyn_cast<StructType>(Val: ShadowTy)) {
1876 SmallVector<Constant *, 4> Vals;
1877 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1878 Vals.push_back(Elt: getPoisonedShadow(ShadowTy: ST->getElementType(N: i)));
1879 return ConstantStruct::get(T: ST, V: Vals);
1880 }
1881 llvm_unreachable("Unexpected shadow type");
1882 }
1883
1884 /// Create a dirty shadow for a given value.
1885 Constant *getPoisonedShadow(Value *V) {
1886 Type *ShadowTy = getShadowTy(V);
1887 if (!ShadowTy)
1888 return nullptr;
1889 return getPoisonedShadow(ShadowTy);
1890 }
1891
1892 /// Create a clean (zero) origin.
1893 Value *getCleanOrigin() { return Constant::getNullValue(Ty: MS.OriginTy); }
1894
1895 /// Get the shadow value for a given Value.
1896 ///
1897 /// This function either returns the value set earlier with setShadow,
1898 /// or extracts if from ParamTLS (for function arguments).
1899 Value *getShadow(Value *V) {
1900 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
1901 if (!PropagateShadow || I->getMetadata(KindID: LLVMContext::MD_nosanitize))
1902 return getCleanShadow(V);
1903 // For instructions the shadow is already stored in the map.
1904 Value *Shadow = ShadowMap[V];
1905 if (!Shadow) {
1906 LLVM_DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()));
1907 assert(Shadow && "No shadow for a value");
1908 }
1909 return Shadow;
1910 }
1911 // Handle fully undefined values
1912 // (partially undefined constant vectors are handled later)
1913 if ([[maybe_unused]] UndefValue *U = dyn_cast<UndefValue>(Val: V)) {
1914 Value *AllOnes = (PropagateShadow && PoisonUndef) ? getPoisonedShadow(V)
1915 : getCleanShadow(V);
1916 LLVM_DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n");
1917 return AllOnes;
1918 }
1919 if (Argument *A = dyn_cast<Argument>(Val: V)) {
1920 // For arguments we compute the shadow on demand and store it in the map.
1921 Value *&ShadowPtr = ShadowMap[V];
1922 if (ShadowPtr)
1923 return ShadowPtr;
1924 Function *F = A->getParent();
1925 IRBuilder<> EntryIRB(FnPrologueEnd);
1926 unsigned ArgOffset = 0;
1927 const DataLayout &DL = F->getDataLayout();
1928 for (auto &FArg : F->args()) {
1929 if (!FArg.getType()->isSized() || FArg.getType()->isScalableTy()) {
1930 LLVM_DEBUG(dbgs() << (FArg.getType()->isScalableTy()
1931 ? "vscale not fully supported\n"
1932 : "Arg is not sized\n"));
1933 if (A == &FArg) {
1934 ShadowPtr = getCleanShadow(V);
1935 setOrigin(V: A, Origin: getCleanOrigin());
1936 break;
1937 }
1938 continue;
1939 }
1940
1941 unsigned Size = FArg.hasByValAttr()
1942 ? DL.getTypeAllocSize(Ty: FArg.getParamByValType())
1943 : DL.getTypeAllocSize(Ty: FArg.getType());
1944
1945 if (A == &FArg) {
1946 bool Overflow = ArgOffset + Size > kParamTLSSize;
1947 if (FArg.hasByValAttr()) {
1948 // ByVal pointer itself has clean shadow. We copy the actual
1949 // argument shadow to the underlying memory.
1950 // Figure out maximal valid memcpy alignment.
1951 const Align ArgAlign = DL.getValueOrABITypeAlignment(
1952 Alignment: FArg.getParamAlign(), Ty: FArg.getParamByValType());
1953 Value *CpShadowPtr, *CpOriginPtr;
1954 std::tie(args&: CpShadowPtr, args&: CpOriginPtr) =
1955 getShadowOriginPtr(Addr: V, IRB&: EntryIRB, ShadowTy: EntryIRB.getInt8Ty(), Alignment: ArgAlign,
1956 /*isStore*/ true);
1957 if (!PropagateShadow || Overflow) {
1958 // ParamTLS overflow.
1959 EntryIRB.CreateMemSet(
1960 Ptr: CpShadowPtr, Val: Constant::getNullValue(Ty: EntryIRB.getInt8Ty()),
1961 Size, Align: ArgAlign);
1962 } else {
1963 Value *Base = getShadowPtrForArgument(IRB&: EntryIRB, ArgOffset);
1964 const Align CopyAlign = std::min(a: ArgAlign, b: kShadowTLSAlignment);
1965 [[maybe_unused]] Value *Cpy = EntryIRB.CreateMemCpy(
1966 Dst: CpShadowPtr, DstAlign: CopyAlign, Src: Base, SrcAlign: CopyAlign, Size);
1967 LLVM_DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n");
1968
1969 if (MS.TrackOrigins) {
1970 Value *OriginPtr = getOriginPtrForArgument(IRB&: EntryIRB, ArgOffset);
1971 // FIXME: OriginSize should be:
1972 // alignTo(V % kMinOriginAlignment + Size, kMinOriginAlignment)
1973 unsigned OriginSize = alignTo(Size, A: kMinOriginAlignment);
1974 EntryIRB.CreateMemCpy(
1975 Dst: CpOriginPtr,
1976 /* by getShadowOriginPtr */ DstAlign: kMinOriginAlignment, Src: OriginPtr,
1977 /* by origin_tls[ArgOffset] */ SrcAlign: kMinOriginAlignment,
1978 Size: OriginSize);
1979 }
1980 }
1981 }
1982
1983 if (!PropagateShadow || Overflow || FArg.hasByValAttr() ||
1984 (MS.EagerChecks && FArg.hasAttribute(Kind: Attribute::NoUndef))) {
1985 ShadowPtr = getCleanShadow(V);
1986 setOrigin(V: A, Origin: getCleanOrigin());
1987 } else {
1988 // Shadow over TLS
1989 Value *Base = getShadowPtrForArgument(IRB&: EntryIRB, ArgOffset);
1990 ShadowPtr = EntryIRB.CreateAlignedLoad(Ty: getShadowTy(V: &FArg), Ptr: Base,
1991 Align: kShadowTLSAlignment);
1992 if (MS.TrackOrigins) {
1993 Value *OriginPtr = getOriginPtrForArgument(IRB&: EntryIRB, ArgOffset);
1994 setOrigin(V: A, Origin: EntryIRB.CreateLoad(Ty: MS.OriginTy, Ptr: OriginPtr));
1995 }
1996 }
1997 LLVM_DEBUG(dbgs()
1998 << " ARG: " << FArg << " ==> " << *ShadowPtr << "\n");
1999 break;
2000 }
2001
2002 ArgOffset += alignTo(Size, A: kShadowTLSAlignment);
2003 }
2004 assert(ShadowPtr && "Could not find shadow for an argument");
2005 return ShadowPtr;
2006 }
2007
2008 // Check for partially-undefined constant vectors
2009 // TODO: scalable vectors (this is hard because we do not have IRBuilder)
2010 if (isa<FixedVectorType>(Val: V->getType()) && isa<Constant>(Val: V) &&
2011 cast<Constant>(Val: V)->containsUndefOrPoisonElement() && PropagateShadow &&
2012 PoisonUndefVectors) {
2013 unsigned NumElems = cast<FixedVectorType>(Val: V->getType())->getNumElements();
2014 SmallVector<Constant *, 32> ShadowVector(NumElems);
2015 for (unsigned i = 0; i != NumElems; ++i) {
2016 Constant *Elem = cast<Constant>(Val: V)->getAggregateElement(Elt: i);
2017 ShadowVector[i] = isa<UndefValue>(Val: Elem) ? getPoisonedShadow(V: Elem)
2018 : getCleanShadow(V: Elem);
2019 }
2020
2021 Value *ShadowConstant = ConstantVector::get(V: ShadowVector);
2022 LLVM_DEBUG(dbgs() << "Partial undef constant vector: " << *V << " ==> "
2023 << *ShadowConstant << "\n");
2024
2025 return ShadowConstant;
2026 }
2027
2028 // TODO: partially-undefined constant arrays, structures, and nested types
2029
2030 // For everything else the shadow is zero.
2031 return getCleanShadow(V);
2032 }
2033
2034 /// Get the shadow for i-th argument of the instruction I.
2035 Value *getShadow(Instruction *I, int i) {
2036 return getShadow(V: I->getOperand(i));
2037 }
2038
2039 /// Get the origin for a value.
2040 Value *getOrigin(Value *V) {
2041 if (!MS.TrackOrigins)
2042 return nullptr;
2043 if (!PropagateShadow || isa<Constant>(Val: V) || isa<InlineAsm>(Val: V))
2044 return getCleanOrigin();
2045 assert((isa<Instruction>(V) || isa<Argument>(V)) &&
2046 "Unexpected value type in getOrigin()");
2047 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
2048 if (I->getMetadata(KindID: LLVMContext::MD_nosanitize))
2049 return getCleanOrigin();
2050 }
2051 Value *Origin = OriginMap[V];
2052 assert(Origin && "Missing origin");
2053 return Origin;
2054 }
2055
2056 /// Get the origin for i-th argument of the instruction I.
2057 Value *getOrigin(Instruction *I, int i) {
2058 return getOrigin(V: I->getOperand(i));
2059 }
2060
2061 /// Remember the place where a shadow check should be inserted.
2062 ///
2063 /// This location will be later instrumented with a check that will print a
2064 /// UMR warning in runtime if the shadow value is not 0.
2065 void insertCheckShadow(Value *Shadow, Value *Origin, Instruction *OrigIns) {
2066 assert(Shadow);
2067 if (!InsertChecks)
2068 return;
2069
2070 if (!DebugCounter::shouldExecute(Counter&: DebugInsertCheck)) {
2071 LLVM_DEBUG(dbgs() << "Skipping check of " << *Shadow << " before "
2072 << *OrigIns << "\n");
2073 return;
2074 }
2075
2076 Type *ShadowTy = Shadow->getType();
2077 if (isScalableNonVectorType(Ty: ShadowTy)) {
2078 LLVM_DEBUG(dbgs() << "Skipping check of scalable non-vector " << *Shadow
2079 << " before " << *OrigIns << "\n");
2080 return;
2081 }
2082#ifndef NDEBUG
2083 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
2084 isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&
2085 "Can only insert checks for integer, vector, and aggregate shadow "
2086 "types");
2087#endif
2088 InstrumentationList.push_back(
2089 Elt: ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
2090 }
2091
2092 /// Get shadow for value, and remember the place where a shadow check should
2093 /// be inserted.
2094 ///
2095 /// This location will be later instrumented with a check that will print a
2096 /// UMR warning in runtime if the value is not fully defined.
2097 void insertCheckShadowOf(Value *Val, Instruction *OrigIns) {
2098 assert(Val);
2099 Value *Shadow, *Origin;
2100 if (Opts.msan_check_constant_shadow) {
2101 Shadow = getShadow(V: Val);
2102 if (!Shadow)
2103 return;
2104 Origin = getOrigin(V: Val);
2105 } else {
2106 Shadow = dyn_cast_or_null<Instruction>(Val: getShadow(V: Val));
2107 if (!Shadow)
2108 return;
2109 Origin = dyn_cast_or_null<Instruction>(Val: getOrigin(V: Val));
2110 }
2111 insertCheckShadow(Shadow, Origin, OrigIns);
2112 }
2113
2114 AtomicOrdering addReleaseOrdering(AtomicOrdering a) {
2115 switch (a) {
2116 case AtomicOrdering::NotAtomic:
2117 return AtomicOrdering::NotAtomic;
2118 case AtomicOrdering::Unordered:
2119 case AtomicOrdering::Monotonic:
2120 case AtomicOrdering::Release:
2121 return AtomicOrdering::Release;
2122 case AtomicOrdering::Acquire:
2123 case AtomicOrdering::AcquireRelease:
2124 return AtomicOrdering::AcquireRelease;
2125 case AtomicOrdering::SequentiallyConsistent:
2126 return AtomicOrdering::SequentiallyConsistent;
2127 }
2128 llvm_unreachable("Unknown ordering");
2129 }
2130
2131 Value *makeAddReleaseOrderingTable(IRBuilder<> &IRB) {
2132 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2133 uint32_t OrderingTable[NumOrderings] = {};
2134
2135 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2136 OrderingTable[(int)AtomicOrderingCABI::release] =
2137 (int)AtomicOrderingCABI::release;
2138 OrderingTable[(int)AtomicOrderingCABI::consume] =
2139 OrderingTable[(int)AtomicOrderingCABI::acquire] =
2140 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
2141 (int)AtomicOrderingCABI::acq_rel;
2142 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2143 (int)AtomicOrderingCABI::seq_cst;
2144
2145 return ConstantDataVector::get(Context&: IRB.getContext(), Elts: OrderingTable);
2146 }
2147
2148 AtomicOrdering addAcquireOrdering(AtomicOrdering a) {
2149 switch (a) {
2150 case AtomicOrdering::NotAtomic:
2151 return AtomicOrdering::NotAtomic;
2152 case AtomicOrdering::Unordered:
2153 case AtomicOrdering::Monotonic:
2154 case AtomicOrdering::Acquire:
2155 return AtomicOrdering::Acquire;
2156 case AtomicOrdering::Release:
2157 case AtomicOrdering::AcquireRelease:
2158 return AtomicOrdering::AcquireRelease;
2159 case AtomicOrdering::SequentiallyConsistent:
2160 return AtomicOrdering::SequentiallyConsistent;
2161 }
2162 llvm_unreachable("Unknown ordering");
2163 }
2164
2165 Value *makeAddAcquireOrderingTable(IRBuilder<> &IRB) {
2166 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2167 uint32_t OrderingTable[NumOrderings] = {};
2168
2169 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2170 OrderingTable[(int)AtomicOrderingCABI::acquire] =
2171 OrderingTable[(int)AtomicOrderingCABI::consume] =
2172 (int)AtomicOrderingCABI::acquire;
2173 OrderingTable[(int)AtomicOrderingCABI::release] =
2174 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
2175 (int)AtomicOrderingCABI::acq_rel;
2176 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2177 (int)AtomicOrderingCABI::seq_cst;
2178
2179 return ConstantDataVector::get(Context&: IRB.getContext(), Elts: OrderingTable);
2180 }
2181
2182 // ------------------- Visitors.
2183 using InstVisitor<MemorySanitizerVisitor>::visit;
2184 void visit(Instruction &I) {
2185 if (I.getMetadata(KindID: LLVMContext::MD_nosanitize))
2186 return;
2187 // Don't want to visit if we're in the prologue
2188 if (isInPrologue(I))
2189 return;
2190 if (!DebugCounter::shouldExecute(Counter&: DebugInstrumentInstruction)) {
2191 LLVM_DEBUG(dbgs() << "Skipping instruction: " << I << "\n");
2192 // We still need to set the shadow and origin to clean values.
2193 setShadow(V: &I, SV: getCleanShadow(V: &I));
2194 setOrigin(V: &I, Origin: getCleanOrigin());
2195 return;
2196 }
2197
2198 Instructions.push_back(Elt: &I);
2199 }
2200
2201 /// Instrument LoadInst
2202 ///
2203 /// Loads the corresponding shadow and (optionally) origin.
2204 /// Optionally, checks that the load address is fully defined.
2205 void visitLoadInst(LoadInst &I) {
2206 assert(I.getType()->isSized() && "Load type must have size");
2207 assert(!I.getMetadata(LLVMContext::MD_nosanitize));
2208 NextNodeIRBuilder IRB(&I);
2209 Type *ShadowTy = getShadowTy(V: &I);
2210 Value *Addr = I.getPointerOperand();
2211 Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
2212 const Align Alignment = I.getAlign();
2213 if (PropagateShadow) {
2214 std::tie(args&: ShadowPtr, args&: OriginPtr) =
2215 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2216 setShadow(V: &I,
2217 SV: IRB.CreateAlignedLoad(Ty: ShadowTy, Ptr: ShadowPtr, Align: Alignment, Name: "_msld"));
2218 } else {
2219 setShadow(V: &I, SV: getCleanShadow(V: &I));
2220 }
2221
2222 if (Opts.msan_check_access_address)
2223 insertCheckShadowOf(Val: I.getPointerOperand(), OrigIns: &I);
2224
2225 if (I.isAtomic())
2226 I.setOrdering(addAcquireOrdering(a: I.getOrdering()));
2227
2228 if (MS.TrackOrigins) {
2229 if (PropagateShadow) {
2230 const Align OriginAlignment = std::max(a: kMinOriginAlignment, b: Alignment);
2231 setOrigin(
2232 V: &I, Origin: IRB.CreateAlignedLoad(Ty: MS.OriginTy, Ptr: OriginPtr, Align: OriginAlignment));
2233 } else {
2234 setOrigin(V: &I, Origin: getCleanOrigin());
2235 }
2236 }
2237 }
2238
2239 /// Instrument StoreInst
2240 ///
2241 /// Stores the corresponding shadow and (optionally) origin.
2242 /// Optionally, checks that the store address is fully defined.
2243 void visitStoreInst(StoreInst &I) {
2244 StoreList.push_back(Elt: &I);
2245 if (Opts.msan_check_access_address)
2246 insertCheckShadowOf(Val: I.getPointerOperand(), OrigIns: &I);
2247 }
2248
2249 void handleCASOrRMW(Instruction &I) {
2250 assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I));
2251
2252 IRBuilder<> IRB(&I);
2253 Value *Addr = I.getOperand(i: 0);
2254 Value *Val = I.getOperand(i: 1);
2255 Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, ShadowTy: getShadowTy(V: Val), Alignment: Align(1),
2256 /*isStore*/ true)
2257 .first;
2258
2259 if (Opts.msan_check_access_address)
2260 insertCheckShadowOf(Val: Addr, OrigIns: &I);
2261
2262 // Only test the conditional argument of cmpxchg instruction.
2263 // The other argument can potentially be uninitialized, but we can not
2264 // detect this situation reliably without possible false positives.
2265 if (isa<AtomicCmpXchgInst>(Val: I))
2266 insertCheckShadowOf(Val, OrigIns: &I);
2267
2268 IRB.CreateStore(Val: getCleanShadow(V: Val), Ptr: ShadowPtr);
2269
2270 setShadow(V: &I, SV: getCleanShadow(V: &I));
2271 setOrigin(V: &I, Origin: getCleanOrigin());
2272 }
2273
2274 void visitAtomicRMWInst(AtomicRMWInst &I) {
2275 handleCASOrRMW(I);
2276 I.setOrdering(addReleaseOrdering(a: I.getOrdering()));
2277 }
2278
2279 void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
2280 handleCASOrRMW(I);
2281 I.setSuccessOrdering(addReleaseOrdering(a: I.getSuccessOrdering()));
2282 }
2283
2284 /// Generic handler to compute shadow for == and != comparisons.
2285 ///
2286 /// This function is used by handleEqualityComparison and visitSwitchInst.
2287 ///
2288 /// Sometimes the comparison result is known even if some of the bits of the
2289 /// arguments are not.
2290 Value *propagateEqualityComparison(IRBuilder<> &IRB, Value *A, Value *B,
2291 Value *Sa, Value *Sb) {
2292 assert(getShadowTy(A) == Sa->getType());
2293 assert(getShadowTy(B) == Sb->getType());
2294
2295 // Get rid of pointers and vectors of pointers.
2296 // For ints (and vectors of ints), types of A and Sa match,
2297 // and this is a no-op.
2298 A = IRB.CreatePointerCast(V: A, DestTy: Sa->getType());
2299 B = IRB.CreatePointerCast(V: B, DestTy: Sb->getType());
2300
2301 // A == B <==> (C = A^B) == 0
2302 // A != B <==> (C = A^B) != 0
2303 // Sc = Sa | Sb
2304 Value *C = IRB.CreateXor(LHS: A, RHS: B);
2305 Value *Sc = IRB.CreateOr(LHS: Sa, RHS: Sb);
2306 // Now dealing with i = (C == 0) comparison (or C != 0, does not matter now)
2307 // Result is defined if one of the following is true
2308 // * there is a defined 1 bit in C
2309 // * C is fully defined
2310 // Si = !(C & ~Sc) && Sc
2311 Value *Zero = Constant::getNullValue(Ty: Sc->getType());
2312 Value *MinusOne = Constant::getAllOnesValue(Ty: Sc->getType());
2313 Value *LHS = IRB.CreateICmpNE(LHS: Sc, RHS: Zero);
2314 Value *RHS =
2315 IRB.CreateICmpEQ(LHS: IRB.CreateAnd(LHS: IRB.CreateXor(LHS: Sc, RHS: MinusOne), RHS: C), RHS: Zero);
2316 Value *Si = IRB.CreateAnd(LHS, RHS);
2317 Si->setName("_msprop_icmp");
2318
2319 return Si;
2320 }
2321
2322 // Instrument:
2323 // switch i32 %Val, label %else [ i32 0, label %A
2324 // i32 1, label %B
2325 // i32 2, label %C ]
2326 //
2327 // Typically, the switch input value (%Val) is fully initialized.
2328 //
2329 // Sometimes the compiler may convert (icmp + br) into a switch statement.
2330 // MSan allows icmp eq/ne with partly initialized inputs to still result in a
2331 // fully initialized output, if there exists a bit that is initialized in
2332 // both inputs with a differing value. For compatibility, we support this in
2333 // the switch instrumentation as well. Note that this edge case only applies
2334 // if the switch input value does not match *any* of the cases (matching any
2335 // of the cases requires an exact, fully initialized match).
2336 //
2337 // ShadowCases = 0
2338 // | propagateEqualityComparison(Val, 0)
2339 // | propagateEqualityComparison(Val, 1)
2340 // | propagateEqualityComparison(Val, 2))
2341 void visitSwitchInst(SwitchInst &SI) {
2342 IRBuilder<> IRB(&SI);
2343
2344 Value *Val = SI.getCondition();
2345 Value *ShadowVal = getShadow(V: Val);
2346 // TODO: add fast path - if the condition is fully initialized, we know
2347 // there is no UUM, without needing to consider the case values below.
2348
2349 // Some code (e.g., AMDGPUGenMCCodeEmitter.inc) has tens of thousands of
2350 // cases. This results in an extremely long chained expression for MSan's
2351 // switch instrumentation, which can cause the JumpThreadingPass to have a
2352 // stack overflow or excessive runtime. We limit the number of cases
2353 // considered, with the tradeoff of niche false negatives.
2354 // TODO: figure out a better solution.
2355 int casesToConsider = Opts.msan_switch_precision;
2356
2357 Value *ShadowCases = nullptr;
2358 for (auto Case : SI.cases()) {
2359 if (casesToConsider <= 0)
2360 break;
2361
2362 Value *Comparator = Case.getCaseValue();
2363 // TODO: some simplification is possible when comparing multiple cases
2364 // simultaneously.
2365 Value *ComparisonShadow = propagateEqualityComparison(
2366 IRB, A: Val, B: Comparator, Sa: ShadowVal, Sb: getShadow(V: Comparator));
2367
2368 if (ShadowCases)
2369 ShadowCases = IRB.CreateOr(LHS: ShadowCases, RHS: ComparisonShadow);
2370 else
2371 ShadowCases = ComparisonShadow;
2372
2373 casesToConsider--;
2374 }
2375
2376 if (ShadowCases)
2377 insertCheckShadow(Shadow: ShadowCases, Origin: getOrigin(V: Val), OrigIns: &SI);
2378 }
2379
2380 // Vector manipulation.
2381 void visitExtractElementInst(ExtractElementInst &I) {
2382 insertCheckShadowOf(Val: I.getOperand(i_nocapture: 1), OrigIns: &I);
2383 IRBuilder<> IRB(&I);
2384 setShadow(V: &I, SV: IRB.CreateExtractElement(Vec: getShadow(I: &I, i: 0), Idx: I.getOperand(i_nocapture: 1),
2385 Name: "_msprop"));
2386 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2387 }
2388
2389 void visitInsertElementInst(InsertElementInst &I) {
2390 insertCheckShadowOf(Val: I.getOperand(i_nocapture: 2), OrigIns: &I);
2391 IRBuilder<> IRB(&I);
2392 auto *Shadow0 = getShadow(I: &I, i: 0);
2393 auto *Shadow1 = getShadow(I: &I, i: 1);
2394 setShadow(V: &I, SV: IRB.CreateInsertElement(Vec: Shadow0, NewElt: Shadow1, Idx: I.getOperand(i_nocapture: 2),
2395 Name: "_msprop"));
2396 setOriginForNaryOp(I);
2397 }
2398
2399 void visitShuffleVectorInst(ShuffleVectorInst &I) {
2400 IRBuilder<> IRB(&I);
2401 auto *Shadow0 = getShadow(I: &I, i: 0);
2402 auto *Shadow1 = getShadow(I: &I, i: 1);
2403 setShadow(V: &I, SV: IRB.CreateShuffleVector(V1: Shadow0, V2: Shadow1, Mask: I.getShuffleMask(),
2404 Name: "_msprop"));
2405 setOriginForNaryOp(I);
2406 }
2407
2408 // Casts.
2409 void visitSExtInst(SExtInst &I) {
2410 IRBuilder<> IRB(&I);
2411 setShadow(V: &I, SV: IRB.CreateSExt(V: getShadow(I: &I, i: 0), DestTy: I.getType(), Name: "_msprop"));
2412 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2413 }
2414
2415 void visitZExtInst(ZExtInst &I) {
2416 IRBuilder<> IRB(&I);
2417 setShadow(V: &I, SV: IRB.CreateZExt(V: getShadow(I: &I, i: 0), DestTy: I.getType(), Name: "_msprop"));
2418 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2419 }
2420
2421 void visitTruncInst(TruncInst &I) {
2422 IRBuilder<> IRB(&I);
2423 setShadow(V: &I, SV: IRB.CreateTrunc(V: getShadow(I: &I, i: 0), DestTy: I.getType(), Name: "_msprop"));
2424 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2425 }
2426
2427 void visitBitCastInst(BitCastInst &I) {
2428 // Special case: if this is the bitcast (there is exactly 1 allowed) between
2429 // a musttail call and a ret, don't instrument. New instructions are not
2430 // allowed after a musttail call.
2431 if (auto *CI = dyn_cast<CallInst>(Val: I.getOperand(i_nocapture: 0)))
2432 if (CI->isMustTailCall())
2433 return;
2434 IRBuilder<> IRB(&I);
2435 setShadow(V: &I, SV: IRB.CreateBitCast(V: getShadow(I: &I, i: 0), DestTy: getShadowTy(V: &I)));
2436 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2437 }
2438
2439 void visitAddrSpaceCastInst(AddrSpaceCastInst &I) {
2440 IRBuilder<> IRB(&I);
2441 setShadow(V: &I, SV: IRB.CreateIntCast(V: getShadow(I: &I, i: 0), DestTy: getShadowTy(V: &I), isSigned: false,
2442 Name: "_msprop_addrspacecast"));
2443 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2444 }
2445
2446 void visitPtrToIntInst(PtrToIntInst &I) {
2447 IRBuilder<> IRB(&I);
2448 setShadow(V: &I, SV: IRB.CreateIntCast(V: getShadow(I: &I, i: 0), DestTy: getShadowTy(V: &I), isSigned: false,
2449 Name: "_msprop_ptrtoint"));
2450 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2451 }
2452
2453 void visitPtrToAddrInst(PtrToAddrInst &I) {
2454 IRBuilder<> IRB(&I);
2455 setShadow(V: &I, SV: IRB.CreateIntCast(V: getShadow(I: &I, i: 0), DestTy: getShadowTy(V: &I), isSigned: false,
2456 Name: "_msprop_ptrtoaddr"));
2457 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2458 }
2459
2460 void visitIntToPtrInst(IntToPtrInst &I) {
2461 IRBuilder<> IRB(&I);
2462 setShadow(V: &I, SV: IRB.CreateIntCast(V: getShadow(I: &I, i: 0), DestTy: getShadowTy(V: &I), isSigned: false,
2463 Name: "_msprop_inttoptr"));
2464 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2465 }
2466
2467 /// Handle LLVM and NEON vector convert intrinsics.
2468 ///
2469 /// e.g., <4 x i32> @llvm.aarch64.neon.fcvtpu.v4i32.v4f32(<4 x float>)
2470 /// i32 @llvm.aarch64.neon.fcvtms.i32.f64 (double)
2471 /// <2 x i32> @fptoui (<2 x float>)
2472 /// i64 @llvm.fptosi.sat.i64.f64(double)
2473 ///
2474 /// Note that the size of input/output elements can differ e.g.,
2475 /// double @sitofp(i32)
2476 /// but the number of elements must be the same.
2477 ///
2478 /// For conversions to or from fixed-point, there is a trailing argument to
2479 /// indicate the fixed-point precision:
2480 /// - <4 x float> llvm.aarch64.neon.vcvtfxs2fp.v4f32.v4i32(<4 x i32>, i32)
2481 /// - <4 x i32> llvm.aarch64.neon.vcvtfp2fxu.v4i32.v4f32(<4 x float>, i32)
2482 ///
2483 /// For x86 SSE vector convert intrinsics, see
2484 /// handleSSEVectorConvertIntrinsic().
2485 void handleGenericVectorConvertIntrinsic(Instruction &I, bool FixedPoint) {
2486 [[maybe_unused]] unsigned NumArgs = I.getNumOperands();
2487 if (auto *CI = dyn_cast<CallInst>(Val: &I))
2488 NumArgs = CI->arg_size();
2489
2490 if (FixedPoint) {
2491 assert(NumArgs == 2);
2492 Value *Precision = I.getOperand(i: 1);
2493 insertCheckShadowOf(Val: Precision, OrigIns: &I);
2494 } else {
2495 assert(NumArgs == 1);
2496 }
2497
2498 IRBuilder<> IRB(&I);
2499 Value *S0 = getShadow(I: &I, i: 0);
2500
2501 /// For scalars:
2502 /// Since they are converting from floating-point to integer, or between
2503 /// different width floating-point values, the output is:
2504 /// - fully uninitialized if *any* bit of the input is uninitialized
2505 /// - fully ininitialized if all bits of the input are ininitialized
2506 /// We apply the same principle on a per-field basis for vectors.
2507 Value *OutShadow = IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S0, RHS: getCleanShadow(V: S0)),
2508 DestTy: getShadowTy(V: &I));
2509 setShadow(V: &I, SV: OutShadow);
2510 setOriginForNaryOp(I);
2511 }
2512
2513 void visitFPToSIInst(CastInst &I) {
2514 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2515 }
2516 void visitFPToUIInst(CastInst &I) {
2517 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2518 }
2519 void visitSIToFPInst(CastInst &I) {
2520 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2521 }
2522 void visitUIToFPInst(CastInst &I) {
2523 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2524 }
2525
2526 void visitFPExtInst(CastInst &I) {
2527 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2528 }
2529 void visitFPTruncInst(CastInst &I) {
2530 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
2531 }
2532
2533 /// Generic handler to compute shadow for bitwise AND.
2534 ///
2535 /// This is used by 'visitAnd' but also as a primitive for other handlers.
2536 ///
2537 /// This code is precise: it implements the rule that "And" of an initialized
2538 /// zero bit always results in an initialized value:
2539 // 1&1 => 1; 0&1 => 0; p&1 => p;
2540 // 1&0 => 0; 0&0 => 0; p&0 => 0;
2541 // 1&p => p; 0&p => 0; p&p => p;
2542 //
2543 // S = (S1 & S2) | (V1 & S2) | (S1 & V2)
2544 Value *handleBitwiseAnd(IRBuilder<> &IRB, Value *V1, Value *V2, Value *S1,
2545 Value *S2) {
2546 // "The two arguments to the ‘and’ instruction must be integer or vector
2547 // of integer values. Both arguments must have identical types."
2548 //
2549 // We enforce this condition for all callers to handleBitwiseAnd(); callers
2550 // with non-integer types should call CreateAppToShadowCast() themselves.
2551 assert(V1->getType()->isIntOrIntVectorTy());
2552 assert(V1->getType() == V2->getType());
2553
2554 // Conveniently, getShadowTy() of Int/IntVector returns the original type.
2555 assert(V1->getType() == S1->getType());
2556 assert(V2->getType() == S2->getType());
2557
2558 Value *S1S2 = IRB.CreateAnd(LHS: S1, RHS: S2);
2559 Value *V1S2 = IRB.CreateAnd(LHS: V1, RHS: S2);
2560 Value *S1V2 = IRB.CreateAnd(LHS: S1, RHS: V2);
2561
2562 return IRB.CreateOr(Ops: {S1S2, V1S2, S1V2});
2563 }
2564
2565 /// Handler for bitwise AND operator.
2566 void visitAnd(BinaryOperator &I) {
2567 IRBuilder<> IRB(&I);
2568 Value *V1 = I.getOperand(i_nocapture: 0);
2569 Value *V2 = I.getOperand(i_nocapture: 1);
2570 Value *S1 = getShadow(I: &I, i: 0);
2571 Value *S2 = getShadow(I: &I, i: 1);
2572
2573 Value *OutShadow = handleBitwiseAnd(IRB, V1, V2, S1, S2);
2574
2575 setShadow(V: &I, SV: OutShadow);
2576 setOriginForNaryOp(I);
2577 }
2578
2579 void visitOr(BinaryOperator &I) {
2580 IRBuilder<> IRB(&I);
2581 // "Or" of 1 and a poisoned value results in unpoisoned value:
2582 // 1|1 => 1; 0|1 => 1; p|1 => 1;
2583 // 1|0 => 1; 0|0 => 0; p|0 => p;
2584 // 1|p => 1; 0|p => p; p|p => p;
2585 //
2586 // S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2)
2587 //
2588 // If the "disjoint OR" property is violated, the result is poison, and
2589 // hence the entire shadow is uninitialized:
2590 // S = S | SignExt(V1 & V2 != 0)
2591 Value *S1 = getShadow(I: &I, i: 0);
2592 Value *S2 = getShadow(I: &I, i: 1);
2593 Value *V1 = I.getOperand(i_nocapture: 0);
2594 Value *V2 = I.getOperand(i_nocapture: 1);
2595
2596 // "The two arguments to the ‘or’ instruction must be integer or vector
2597 // of integer values. Both arguments must have identical types."
2598 assert(V1->getType()->isIntOrIntVectorTy());
2599 assert(V1->getType() == V2->getType());
2600
2601 // Conveniently, getShadowTy() of Int/IntVector returns the original type.
2602 assert(V1->getType() == S1->getType());
2603 assert(V2->getType() == S2->getType());
2604
2605 Value *NotV1 = IRB.CreateNot(V: V1);
2606 Value *NotV2 = IRB.CreateNot(V: V2);
2607
2608 Value *S1S2 = IRB.CreateAnd(LHS: S1, RHS: S2);
2609 Value *S2NotV1 = IRB.CreateAnd(LHS: NotV1, RHS: S2);
2610 Value *S1NotV2 = IRB.CreateAnd(LHS: S1, RHS: NotV2);
2611
2612 Value *S = IRB.CreateOr(Ops: {S1S2, S2NotV1, S1NotV2});
2613
2614 if (Opts.msan_precise_disjoint_or &&
2615 cast<PossiblyDisjointInst>(Val: &I)->isDisjoint()) {
2616 Value *V1V2 = IRB.CreateAnd(LHS: V1, RHS: V2);
2617 Value *DisjointOrShadow = IRB.CreateSExt(
2618 V: IRB.CreateICmpNE(LHS: V1V2, RHS: getCleanShadow(V: V1V2)), DestTy: V1V2->getType());
2619 S = IRB.CreateOr(LHS: S, RHS: DisjointOrShadow, Name: "_ms_disjoint");
2620 }
2621
2622 setShadow(V: &I, SV: S);
2623 setOriginForNaryOp(I);
2624 }
2625
2626 /// Default propagation of shadow and/or origin.
2627 ///
2628 /// This class implements the general case of shadow propagation, used in all
2629 /// cases where we don't know and/or don't care about what the operation
2630 /// actually does. It converts all input shadow values to a common type
2631 /// (extending or truncating as necessary), and bitwise OR's them.
2632 ///
2633 /// This is much cheaper than inserting checks (i.e. requiring inputs to be
2634 /// fully initialized), and less prone to false positives.
2635 ///
2636 /// This class also implements the general case of origin propagation. For a
2637 /// Nary operation, result origin is set to the origin of an argument that is
2638 /// not entirely initialized. If there is more than one such arguments, the
2639 /// rightmost of them is picked. It does not matter which one is picked if all
2640 /// arguments are initialized.
2641 template <bool CombineShadow> class Combiner {
2642 Value *Shadow = nullptr;
2643 Value *Origin = nullptr;
2644 IRBuilder<> &IRB;
2645 MemorySanitizerVisitor *MSV;
2646
2647 public:
2648 Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB)
2649 : IRB(IRB), MSV(MSV) {}
2650
2651 /// Add a pair of shadow and origin values to the mix.
2652 Combiner &Add(Value *OpShadow, Value *OpOrigin) {
2653 if (CombineShadow) {
2654 assert(OpShadow);
2655 if (!Shadow)
2656 Shadow = OpShadow;
2657 else {
2658 OpShadow = MSV->CreateShadowCast(IRB, V: OpShadow, dstTy: Shadow->getType());
2659 Shadow = IRB.CreateOr(LHS: Shadow, RHS: OpShadow, Name: "_msprop");
2660 }
2661 }
2662
2663 if (MSV->MS.TrackOrigins) {
2664 assert(OpOrigin);
2665 if (!Origin) {
2666 Origin = OpOrigin;
2667 } else {
2668 Constant *ConstOrigin = dyn_cast<Constant>(Val: OpOrigin);
2669 // No point in adding something that might result in 0 origin value.
2670 if (!ConstOrigin || !ConstOrigin->isNullValue()) {
2671 Value *Cond = MSV->convertToBool(V: OpShadow, IRB);
2672 Origin = IRB.CreateSelect(C: Cond, True: OpOrigin, False: Origin);
2673 }
2674 }
2675 }
2676 return *this;
2677 }
2678
2679 /// Add an application value to the mix.
2680 Combiner &Add(Value *V) {
2681 Value *OpShadow = MSV->getShadow(V);
2682 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) : nullptr;
2683 return Add(OpShadow, OpOrigin);
2684 }
2685
2686 /// Set the current combined values as the given instruction's shadow
2687 /// and origin.
2688 void Done(Instruction *I) {
2689 if (CombineShadow) {
2690 assert(Shadow);
2691 Shadow = MSV->CreateShadowCast(IRB, V: Shadow, dstTy: MSV->getShadowTy(V: I));
2692 MSV->setShadow(V: I, SV: Shadow);
2693 }
2694 if (MSV->MS.TrackOrigins) {
2695 assert(Origin);
2696 MSV->setOrigin(V: I, Origin);
2697 }
2698 }
2699
2700 /// Store the current combined value at the specified origin
2701 /// location.
2702 void DoneAndStoreOrigin(TypeSize TS, Value *OriginPtr) {
2703 if (MSV->MS.TrackOrigins) {
2704 assert(Origin);
2705 MSV->paintOrigin(IRB, Origin, OriginPtr, TS, Alignment: kMinOriginAlignment);
2706 }
2707 }
2708 };
2709
2710 using ShadowAndOriginCombiner = Combiner<true>;
2711 using OriginCombiner = Combiner<false>;
2712
2713 /// Propagate origin for arbitrary operation.
2714 void setOriginForNaryOp(Instruction &I) {
2715 if (!MS.TrackOrigins)
2716 return;
2717 IRBuilder<> IRB(&I);
2718 OriginCombiner OC(this, IRB);
2719 for (Use &Op : I.operands())
2720 OC.Add(V: Op.get());
2721 OC.Done(I: &I);
2722 }
2723
2724 size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) {
2725 assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) &&
2726 "Vector of pointers is not a valid shadow type");
2727 return Ty->isVectorTy() ? cast<FixedVectorType>(Val: Ty)->getNumElements() *
2728 Ty->getScalarSizeInBits()
2729 : Ty->getPrimitiveSizeInBits();
2730 }
2731
2732 /// Cast between two shadow types, extending or truncating as
2733 /// necessary.
2734 Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy,
2735 bool Signed = false) {
2736 Type *srcTy = V->getType();
2737 if (srcTy == dstTy)
2738 return V;
2739 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(Ty: srcTy);
2740 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(Ty: dstTy);
2741 if (srcSizeInBits > 1 && dstSizeInBits == 1)
2742 return IRB.CreateICmpNE(LHS: V, RHS: getCleanShadow(V));
2743
2744 if (dstTy->isIntegerTy() && srcTy->isIntegerTy())
2745 return IRB.CreateIntCast(V, DestTy: dstTy, isSigned: Signed);
2746 if (dstTy->isVectorTy() && srcTy->isVectorTy() &&
2747 cast<VectorType>(Val: dstTy)->getElementCount() ==
2748 cast<VectorType>(Val: srcTy)->getElementCount())
2749 return IRB.CreateIntCast(V, DestTy: dstTy, isSigned: Signed);
2750 Value *V1 = IRB.CreateBitCast(V, DestTy: Type::getIntNTy(C&: *MS.C, N: srcSizeInBits));
2751 Value *V2 =
2752 IRB.CreateIntCast(V: V1, DestTy: Type::getIntNTy(C&: *MS.C, N: dstSizeInBits), isSigned: Signed);
2753 return IRB.CreateBitCast(V: V2, DestTy: dstTy);
2754 // TODO: handle struct types.
2755 }
2756
2757 /// Cast an application value to the type of its own shadow.
2758 Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) {
2759 Type *ShadowTy = getShadowTy(V);
2760 if (V->getType() == ShadowTy)
2761 return V;
2762 if (V->getType()->isPtrOrPtrVectorTy())
2763 return IRB.CreatePtrToInt(V, DestTy: ShadowTy);
2764 else
2765 return IRB.CreateBitCast(V, DestTy: ShadowTy);
2766 }
2767
2768 /// Propagate shadow for arbitrary operation.
2769 void handleShadowOr(Instruction &I) {
2770 IRBuilder<> IRB(&I);
2771 ShadowAndOriginCombiner SC(this, IRB);
2772 for (Use &Op : I.operands())
2773 SC.Add(V: Op.get());
2774 SC.Done(I: &I);
2775 }
2776
2777 // Perform a bitwise OR on the horizontal pairs (or other specified grouping)
2778 // of elements.
2779 //
2780 // For example, suppose we have:
2781 // VectorA: <a0, a1, a2, a3, a4, a5>
2782 // VectorB: <b0, b1, b2, b3, b4, b5>
2783 // ReductionFactor: 3
2784 // Shards: 1
2785 // The output would be:
2786 // <a0|a1|a2, a3|a4|a5, b0|b1|b2, b3|b4|b5>
2787 //
2788 // If we have:
2789 // VectorA: <a0, a1, a2, a3, a4, a5, a6, a7>
2790 // VectorB: <b0, b1, b2, b3, b4, b5, b6, b7>
2791 // ReductionFactor: 2
2792 // Shards: 2
2793 // then a and be each have 2 "shards", resulting in the output being
2794 // interleaved:
2795 // <a0|a1, a2|a3, b0|b1, b2|b3, a4|a5, a6|a7, b4|b5, b6|b7>
2796 //
2797 // This is convenient for instrumenting horizontal add/sub.
2798 // For bitwise OR on "vertical" pairs, see maybeHandleSimpleNomemIntrinsic().
2799 Value *horizontalReduce(IntrinsicInst &I, unsigned ReductionFactor,
2800 unsigned Shards, Value *VectorA, Value *VectorB) {
2801 assert(isa<FixedVectorType>(VectorA->getType()));
2802 unsigned NumElems =
2803 cast<FixedVectorType>(Val: VectorA->getType())->getNumElements();
2804
2805 [[maybe_unused]] unsigned TotalNumElems = NumElems;
2806 if (VectorB) {
2807 assert(VectorA->getType() == VectorB->getType());
2808 TotalNumElems *= 2;
2809 }
2810
2811 assert(NumElems % (ReductionFactor * Shards) == 0);
2812
2813 Value *Or = nullptr;
2814
2815 IRBuilder<> IRB(&I);
2816 for (unsigned i = 0; i < ReductionFactor; i++) {
2817 SmallVector<int, 16> Mask;
2818
2819 for (unsigned j = 0; j < Shards; j++) {
2820 unsigned Offset = NumElems / Shards * j;
2821
2822 for (unsigned X = 0; X < NumElems / Shards; X += ReductionFactor)
2823 Mask.push_back(Elt: Offset + X + i);
2824
2825 if (VectorB) {
2826 for (unsigned X = 0; X < NumElems / Shards; X += ReductionFactor)
2827 Mask.push_back(Elt: NumElems + Offset + X + i);
2828 }
2829 }
2830
2831 Value *Masked;
2832 if (VectorB)
2833 Masked = IRB.CreateShuffleVector(V1: VectorA, V2: VectorB, Mask);
2834 else
2835 Masked = IRB.CreateShuffleVector(V: VectorA, Mask);
2836
2837 if (Or)
2838 Or = IRB.CreateOr(LHS: Or, RHS: Masked);
2839 else
2840 Or = Masked;
2841 }
2842
2843 return Or;
2844 }
2845
2846 /// Propagate shadow for 1- or 2-vector intrinsics that combine adjacent
2847 /// fields.
2848 ///
2849 /// e.g., <2 x i32> @llvm.aarch64.neon.saddlp.v2i32.v4i16(<4 x i16>)
2850 /// <16 x i8> @llvm.aarch64.neon.addp.v16i8(<16 x i8>, <16 x i8>)
2851 void handlePairwiseShadowOrIntrinsic(IntrinsicInst &I, unsigned Shards) {
2852 assert(I.arg_size() == 1 || I.arg_size() == 2);
2853
2854 assert(I.getType()->isVectorTy());
2855 assert(I.getArgOperand(0)->getType()->isVectorTy());
2856
2857 [[maybe_unused]] FixedVectorType *ParamType =
2858 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType());
2859 assert((I.arg_size() != 2) ||
2860 (ParamType == cast<FixedVectorType>(I.getArgOperand(1)->getType())));
2861 [[maybe_unused]] FixedVectorType *ReturnType =
2862 cast<FixedVectorType>(Val: I.getType());
2863 assert(ParamType->getNumElements() * I.arg_size() ==
2864 2 * ReturnType->getNumElements());
2865
2866 IRBuilder<> IRB(&I);
2867
2868 // Horizontal OR of shadow
2869 Value *FirstArgShadow = getShadow(I: &I, i: 0);
2870 Value *SecondArgShadow = nullptr;
2871 if (I.arg_size() == 2)
2872 SecondArgShadow = getShadow(I: &I, i: 1);
2873
2874 Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/2, Shards,
2875 VectorA: FirstArgShadow, VectorB: SecondArgShadow);
2876
2877 OrShadow = CreateShadowCast(IRB, V: OrShadow, dstTy: getShadowTy(V: &I));
2878
2879 setShadow(V: &I, SV: OrShadow);
2880 setOriginForNaryOp(I);
2881 }
2882
2883 /// Propagate shadow for 1- or 2-vector intrinsics that combine adjacent
2884 /// fields, with the parameters reinterpreted to have elements of a specified
2885 /// width. For example:
2886 /// @llvm.x86.ssse3.phadd.w(<1 x i64> [[VAR1]], <1 x i64> [[VAR2]])
2887 /// conceptually operates on
2888 /// (<4 x i16> [[VAR1]], <4 x i16> [[VAR2]])
2889 /// and can be handled with ReinterpretElemWidth == 16.
2890 void handlePairwiseShadowOrIntrinsic(IntrinsicInst &I, unsigned Shards,
2891 int ReinterpretElemWidth) {
2892 assert(I.arg_size() == 1 || I.arg_size() == 2);
2893
2894 assert(I.getType()->isVectorTy());
2895 assert(I.getArgOperand(0)->getType()->isVectorTy());
2896
2897 FixedVectorType *ParamType =
2898 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType());
2899 assert((I.arg_size() != 2) ||
2900 (ParamType == cast<FixedVectorType>(I.getArgOperand(1)->getType())));
2901
2902 [[maybe_unused]] FixedVectorType *ReturnType =
2903 cast<FixedVectorType>(Val: I.getType());
2904 assert(ParamType->getNumElements() * I.arg_size() ==
2905 2 * ReturnType->getNumElements());
2906
2907 IRBuilder<> IRB(&I);
2908
2909 FixedVectorType *ReinterpretShadowTy = nullptr;
2910 assert(isAligned(Align(ReinterpretElemWidth),
2911 ParamType->getPrimitiveSizeInBits()));
2912 ReinterpretShadowTy = FixedVectorType::get(
2913 ElementType: IRB.getIntNTy(N: ReinterpretElemWidth),
2914 NumElts: ParamType->getPrimitiveSizeInBits() / ReinterpretElemWidth);
2915
2916 // Horizontal OR of shadow
2917 Value *FirstArgShadow = getShadow(I: &I, i: 0);
2918 FirstArgShadow = IRB.CreateBitCast(V: FirstArgShadow, DestTy: ReinterpretShadowTy);
2919
2920 // If we had two parameters each with an odd number of elements, the total
2921 // number of elements is even, but we have never seen this in extant
2922 // instruction sets, so we enforce that each parameter must have an even
2923 // number of elements.
2924 assert(isAligned(
2925 Align(2),
2926 cast<FixedVectorType>(FirstArgShadow->getType())->getNumElements()));
2927
2928 Value *SecondArgShadow = nullptr;
2929 if (I.arg_size() == 2) {
2930 SecondArgShadow = getShadow(I: &I, i: 1);
2931 SecondArgShadow = IRB.CreateBitCast(V: SecondArgShadow, DestTy: ReinterpretShadowTy);
2932 }
2933
2934 Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/2, Shards,
2935 VectorA: FirstArgShadow, VectorB: SecondArgShadow);
2936
2937 OrShadow = CreateShadowCast(IRB, V: OrShadow, dstTy: getShadowTy(V: &I));
2938
2939 setShadow(V: &I, SV: OrShadow);
2940 setOriginForNaryOp(I);
2941 }
2942
2943 void visitFNeg(UnaryOperator &I) { handleShadowOr(I); }
2944
2945 // Handle multiplication by constant.
2946 //
2947 // Handle a special case of multiplication by constant that may have one or
2948 // more zeros in the lower bits. This makes corresponding number of lower bits
2949 // of the result zero as well. We model it by shifting the other operand
2950 // shadow left by the required number of bits. Effectively, we transform
2951 // (X * (A * 2**B)) to ((X << B) * A) and instrument (X << B) as (Sx << B).
2952 // We use multiplication by 2**N instead of shift to cover the case of
2953 // multiplication by 0, which may occur in some elements of a vector operand.
2954 void handleMulByConstant(BinaryOperator &I, Constant *ConstArg,
2955 Value *OtherArg) {
2956 Constant *ShadowMul;
2957 Type *Ty = ConstArg->getType();
2958 if (auto *VTy = dyn_cast<VectorType>(Val: Ty)) {
2959 unsigned NumElements = cast<FixedVectorType>(Val: VTy)->getNumElements();
2960 Type *EltTy = VTy->getElementType();
2961 SmallVector<Constant *, 16> Elements;
2962 for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
2963 if (ConstantInt *Elt =
2964 dyn_cast<ConstantInt>(Val: ConstArg->getAggregateElement(Elt: Idx))) {
2965 const APInt &V = Elt->getValue();
2966 APInt V2 = APInt(V.getBitWidth(), 1) << V.countr_zero();
2967 Elements.push_back(Elt: ConstantInt::get(Ty: EltTy, V: V2));
2968 } else {
2969 Elements.push_back(Elt: ConstantInt::get(Ty: EltTy, V: 1));
2970 }
2971 }
2972 ShadowMul = ConstantVector::get(V: Elements);
2973 } else {
2974 if (ConstantInt *Elt = dyn_cast<ConstantInt>(Val: ConstArg)) {
2975 const APInt &V = Elt->getValue();
2976 APInt V2 = APInt(V.getBitWidth(), 1) << V.countr_zero();
2977 ShadowMul = ConstantInt::get(Ty, V: V2);
2978 } else {
2979 ShadowMul = ConstantInt::get(Ty, V: 1);
2980 }
2981 }
2982
2983 IRBuilder<> IRB(&I);
2984 setShadow(V: &I,
2985 SV: IRB.CreateMul(LHS: getShadow(V: OtherArg), RHS: ShadowMul, Name: "msprop_mul_cst"));
2986 setOrigin(V: &I, Origin: getOrigin(V: OtherArg));
2987 }
2988
2989 void visitMul(BinaryOperator &I) {
2990 Constant *constOp0 = dyn_cast<Constant>(Val: I.getOperand(i_nocapture: 0));
2991 Constant *constOp1 = dyn_cast<Constant>(Val: I.getOperand(i_nocapture: 1));
2992 if (constOp0 && !constOp1)
2993 handleMulByConstant(I, ConstArg: constOp0, OtherArg: I.getOperand(i_nocapture: 1));
2994 else if (constOp1 && !constOp0)
2995 handleMulByConstant(I, ConstArg: constOp1, OtherArg: I.getOperand(i_nocapture: 0));
2996 else
2997 handleShadowOr(I);
2998 }
2999
3000 void visitFAdd(BinaryOperator &I) { handleShadowOr(I); }
3001 void visitFSub(BinaryOperator &I) { handleShadowOr(I); }
3002 void visitFMul(BinaryOperator &I) { handleShadowOr(I); }
3003 void visitAdd(BinaryOperator &I) { handleShadowOr(I); }
3004 void visitSub(BinaryOperator &I) { handleShadowOr(I); }
3005 void visitXor(BinaryOperator &I) { handleShadowOr(I); }
3006
3007 void handleIntegerDiv(Instruction &I) {
3008 // Strict on the second argument.
3009 insertCheckShadowOf(Val: I.getOperand(i: 1), OrigIns: &I);
3010 setShadow(V: &I, SV: getShadow(I: &I, i: 0));
3011 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
3012 }
3013
3014 void visitUDiv(BinaryOperator &I) { handleIntegerDiv(I); }
3015 void visitSDiv(BinaryOperator &I) { handleIntegerDiv(I); }
3016 void visitURem(BinaryOperator &I) { handleIntegerDiv(I); }
3017 void visitSRem(BinaryOperator &I) { handleIntegerDiv(I); }
3018
3019 // Floating point division is side-effect free. We can not require that the
3020 // divisor is fully initialized and must propagate shadow. See PR37523.
3021 void visitFDiv(BinaryOperator &I) { handleShadowOr(I); }
3022 void visitFRem(BinaryOperator &I) { handleShadowOr(I); }
3023
3024 /// Instrument == and != comparisons.
3025 ///
3026 /// Sometimes the comparison result is known even if some of the bits of the
3027 /// arguments are not.
3028 void handleEqualityComparison(ICmpInst &I) {
3029 IRBuilder<> IRB(&I);
3030 Value *A = I.getOperand(i_nocapture: 0);
3031 Value *B = I.getOperand(i_nocapture: 1);
3032 Value *Sa = getShadow(V: A);
3033 Value *Sb = getShadow(V: B);
3034
3035 Value *Si = propagateEqualityComparison(IRB, A, B, Sa, Sb);
3036
3037 setShadow(V: &I, SV: Si);
3038 setOriginForNaryOp(I);
3039 }
3040
3041 /// Instrument relational comparisons.
3042 ///
3043 /// This function does exact shadow propagation for all relational
3044 /// comparisons of integers, pointers and vectors of those.
3045 /// FIXME: output seems suboptimal when one of the operands is a constant
3046 void handleRelationalComparisonExact(ICmpInst &I) {
3047 IRBuilder<> IRB(&I);
3048 Value *A = I.getOperand(i_nocapture: 0);
3049 Value *B = I.getOperand(i_nocapture: 1);
3050 Value *Sa = getShadow(V: A);
3051 Value *Sb = getShadow(V: B);
3052
3053 // Get rid of pointers and vectors of pointers.
3054 // For ints (and vectors of ints), types of A and Sa match,
3055 // and this is a no-op.
3056 A = IRB.CreatePointerCast(V: A, DestTy: Sa->getType());
3057 B = IRB.CreatePointerCast(V: B, DestTy: Sb->getType());
3058
3059 // Let [a0, a1] be the interval of possible values of A, taking into account
3060 // its undefined bits. Let [b0, b1] be the interval of possible values of B.
3061 // Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0).
3062 bool IsSigned = I.isSigned();
3063
3064 auto GetMinMaxUnsigned = [&](Value *V, Value *S) {
3065 if (IsSigned) {
3066 // Sign-flip to map from signed range to unsigned range. Relation A vs B
3067 // should be preserved, if checked with `getUnsignedPredicate()`.
3068 // Relationship between Amin, Amax, Bmin, Bmax also will not be
3069 // affected, as they are created by effectively adding/substructing from
3070 // A (or B) a value, derived from shadow, with no overflow, either
3071 // before or after sign flip.
3072 APInt MinVal =
3073 APInt::getSignedMinValue(numBits: V->getType()->getScalarSizeInBits());
3074 V = IRB.CreateXor(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: MinVal));
3075 }
3076 // Minimize undefined bits.
3077 Value *Min = IRB.CreateAnd(LHS: V, RHS: IRB.CreateNot(V: S));
3078 Value *Max = IRB.CreateOr(LHS: V, RHS: S);
3079 return std::make_pair(x&: Min, y&: Max);
3080 };
3081
3082 auto [Amin, Amax] = GetMinMaxUnsigned(A, Sa);
3083 auto [Bmin, Bmax] = GetMinMaxUnsigned(B, Sb);
3084 Value *S1 = IRB.CreateICmp(P: I.getUnsignedPredicate(), LHS: Amin, RHS: Bmax);
3085 Value *S2 = IRB.CreateICmp(P: I.getUnsignedPredicate(), LHS: Amax, RHS: Bmin);
3086
3087 Value *Si = IRB.CreateXor(LHS: S1, RHS: S2);
3088 setShadow(V: &I, SV: Si);
3089 setOriginForNaryOp(I);
3090 }
3091
3092 /// Instrument signed relational comparisons.
3093 ///
3094 /// Handle sign bit tests: x<0, x>=0, x<=-1, x>-1 by propagating the highest
3095 /// bit of the shadow. Everything else is delegated to handleShadowOr().
3096 void handleSignedRelationalComparison(ICmpInst &I) {
3097 Constant *constOp;
3098 Value *op = nullptr;
3099 CmpInst::Predicate pre;
3100 if ((constOp = dyn_cast<Constant>(Val: I.getOperand(i_nocapture: 1)))) {
3101 op = I.getOperand(i_nocapture: 0);
3102 pre = I.getPredicate();
3103 } else if ((constOp = dyn_cast<Constant>(Val: I.getOperand(i_nocapture: 0)))) {
3104 op = I.getOperand(i_nocapture: 1);
3105 pre = I.getSwappedPredicate();
3106 } else {
3107 handleShadowOr(I);
3108 return;
3109 }
3110
3111 if ((constOp->isNullValue() &&
3112 (pre == CmpInst::ICMP_SLT || pre == CmpInst::ICMP_SGE)) ||
3113 (constOp->isAllOnesValue() &&
3114 (pre == CmpInst::ICMP_SGT || pre == CmpInst::ICMP_SLE))) {
3115 IRBuilder<> IRB(&I);
3116 Value *Shadow = IRB.CreateICmpSLT(LHS: getShadow(V: op), RHS: getCleanShadow(V: op),
3117 Name: "_msprop_icmp_s");
3118 setShadow(V: &I, SV: Shadow);
3119 setOrigin(V: &I, Origin: getOrigin(V: op));
3120 } else {
3121 handleShadowOr(I);
3122 }
3123 }
3124
3125 void visitICmpInst(ICmpInst &I) {
3126 if (!Opts.msan_handle_icmp) {
3127 handleShadowOr(I);
3128 return;
3129 }
3130 if (I.isEquality()) {
3131 handleEqualityComparison(I);
3132 return;
3133 }
3134
3135 assert(I.isRelational());
3136 if (Opts.msan_handle_icmp_exact) {
3137 handleRelationalComparisonExact(I);
3138 return;
3139 }
3140 if (I.isSigned()) {
3141 handleSignedRelationalComparison(I);
3142 return;
3143 }
3144
3145 assert(I.isUnsigned());
3146 if ((isa<Constant>(Val: I.getOperand(i_nocapture: 0)) || isa<Constant>(Val: I.getOperand(i_nocapture: 1)))) {
3147 handleRelationalComparisonExact(I);
3148 return;
3149 }
3150
3151 handleShadowOr(I);
3152 }
3153
3154 void visitFCmpInst(FCmpInst &I) { handleShadowOr(I); }
3155
3156 void handleShift(BinaryOperator &I) {
3157 IRBuilder<> IRB(&I);
3158 // If any of the S2 bits are poisoned, the whole thing is poisoned.
3159 // Otherwise perform the same shift on S1.
3160 Value *S1 = getShadow(I: &I, i: 0);
3161 Value *S2 = getShadow(I: &I, i: 1);
3162 Value *S2Conv =
3163 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S2, RHS: getCleanShadow(V: S2)), DestTy: S2->getType());
3164 Value *V2 = I.getOperand(i_nocapture: 1);
3165 Value *Shift = IRB.CreateBinOp(Opc: I.getOpcode(), LHS: S1, RHS: V2);
3166 setShadow(V: &I, SV: IRB.CreateOr(LHS: Shift, RHS: S2Conv));
3167 setOriginForNaryOp(I);
3168 }
3169
3170 void visitShl(BinaryOperator &I) { handleShift(I); }
3171 void visitAShr(BinaryOperator &I) { handleShift(I); }
3172 void visitLShr(BinaryOperator &I) { handleShift(I); }
3173
3174 void handleFunnelShift(IntrinsicInst &I) {
3175 IRBuilder<> IRB(&I);
3176 // If any of the S2 bits are poisoned, the whole thing is poisoned.
3177 // Otherwise perform the same shift on S0 and S1.
3178 Value *S0 = getShadow(I: &I, i: 0);
3179 Value *S1 = getShadow(I: &I, i: 1);
3180 Value *S2 = getShadow(I: &I, i: 2);
3181 Value *S2Conv =
3182 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S2, RHS: getCleanShadow(V: S2)), DestTy: S2->getType());
3183 Value *V2 = I.getOperand(i_nocapture: 2);
3184 Value *Shift = IRB.CreateIntrinsic(ID: I.getIntrinsicID(), OverloadTypes: S2Conv->getType(),
3185 Args: {S0, S1, V2});
3186 setShadow(V: &I, SV: IRB.CreateOr(LHS: Shift, RHS: S2Conv));
3187 setOriginForNaryOp(I);
3188 }
3189
3190 // Instrument bit manipulation intrinsics.
3191 // All of these intrinsics are Z = I(SRC, MASK)
3192 // where the types of all operands and the result match.
3193 // The following instrumentation happens to work for all of them:
3194 // Sz = I(Ssrc, MASK) | (sext (Smask != 0))
3195 void handleGenericBitManipulation(IntrinsicInst &I) {
3196 IRBuilder<> IRB(&I);
3197 Type *ShadowTy = getShadowTy(V: &I);
3198
3199 // If any bit of the mask operand is poisoned, then the whole thing is.
3200 Value *SMask = getShadow(I: &I, i: 1);
3201 SMask = IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: SMask, RHS: getCleanShadow(OrigTy: ShadowTy)),
3202 DestTy: ShadowTy);
3203 // Apply the same intrinsic to the shadow of the first operand.
3204 Value *S;
3205 if (Function *Func = I.getCalledFunction())
3206 S = IRB.CreateCall(Callee: Func, Args: {getShadow(I: &I, i: 0), I.getOperand(i_nocapture: 1)});
3207 else
3208 S = IRB.CreateIntrinsic(ID: I.getIntrinsicID(), OverloadTypes: ShadowTy,
3209 Args: {getShadow(I: &I, i: 0), I.getOperand(i_nocapture: 1)});
3210
3211 setShadow(V: &I, SV: IRB.CreateOr(LHS: SMask, RHS: S));
3212 setOriginForNaryOp(I);
3213 }
3214
3215 /// Instrument llvm.memmove
3216 ///
3217 /// At this point we don't know if llvm.memmove will be inlined or not.
3218 /// If we don't instrument it and it gets inlined,
3219 /// our interceptor will not kick in and we will lose the memmove.
3220 /// If we instrument the call here, but it does not get inlined,
3221 /// we will memmove the shadow twice: which is bad in case
3222 /// of overlapping regions. So, we simply lower the intrinsic to a call.
3223 ///
3224 /// Similar situation exists for memcpy and memset.
3225 void visitMemMoveInst(MemMoveInst &I) {
3226 getShadow(V: I.getArgOperand(i: 1)); // Ensure shadow initialized
3227 IRBuilder<> IRB(&I);
3228 IRB.CreateCall(Callee: MS.MemmoveFn,
3229 Args: {I.getArgOperand(i: 0), I.getArgOperand(i: 1),
3230 IRB.CreateIntCast(V: I.getArgOperand(i: 2), DestTy: MS.IntptrTy, isSigned: false)});
3231 I.eraseFromParent();
3232 }
3233
3234 /// Instrument memcpy
3235 ///
3236 /// Similar to memmove: avoid copying shadow twice. This is somewhat
3237 /// unfortunate as it may slowdown small constant memcpys.
3238 /// FIXME: consider doing manual inline for small constant sizes and proper
3239 /// alignment.
3240 ///
3241 /// Note: This also handles memcpy.inline, which promises no calls to external
3242 /// functions as an optimization. However, with instrumentation enabled this
3243 /// is difficult to promise; additionally, we know that the MSan runtime
3244 /// exists and provides __msan_memcpy(). Therefore, we assume that with
3245 /// instrumentation it's safe to turn memcpy.inline into a call to
3246 /// __msan_memcpy(). Should this be wrong, such as when implementing memcpy()
3247 /// itself, instrumentation should be disabled with the no_sanitize attribute.
3248 void visitMemCpyInst(MemCpyInst &I) {
3249 getShadow(V: I.getArgOperand(i: 1)); // Ensure shadow initialized
3250 IRBuilder<> IRB(&I);
3251 IRB.CreateCall(Callee: MS.MemcpyFn,
3252 Args: {I.getArgOperand(i: 0), I.getArgOperand(i: 1),
3253 IRB.CreateIntCast(V: I.getArgOperand(i: 2), DestTy: MS.IntptrTy, isSigned: false)});
3254 I.eraseFromParent();
3255 }
3256
3257 // Same as memcpy.
3258 void visitMemSetInst(MemSetInst &I) {
3259 IRBuilder<> IRB(&I);
3260 IRB.CreateCall(
3261 Callee: MS.MemsetFn,
3262 Args: {I.getArgOperand(i: 0),
3263 IRB.CreateIntCast(V: I.getArgOperand(i: 1), DestTy: IRB.getInt32Ty(), isSigned: false),
3264 IRB.CreateIntCast(V: I.getArgOperand(i: 2), DestTy: MS.IntptrTy, isSigned: false)});
3265 I.eraseFromParent();
3266 }
3267
3268 void visitVAStartInst(VAStartInst &I) { VAHelper->visitVAStartInst(I); }
3269
3270 void visitVACopyInst(VACopyInst &I) { VAHelper->visitVACopyInst(I); }
3271
3272 /// Handle vector store-like intrinsics.
3273 ///
3274 /// Instrument intrinsics that look like a simple SIMD store: writes memory,
3275 /// has 1 pointer argument and 1 vector argument, returns void.
3276 bool handleVectorStoreIntrinsic(IntrinsicInst &I) {
3277 assert(I.arg_size() == 2);
3278
3279 IRBuilder<> IRB(&I);
3280 Value *Addr = I.getArgOperand(i: 0);
3281 Value *Shadow = getShadow(I: &I, i: 1);
3282 Value *ShadowPtr, *OriginPtr;
3283
3284 // We don't know the pointer alignment (could be unaligned SSE store!).
3285 // Have to assume to worst case.
3286 std::tie(args&: ShadowPtr, args&: OriginPtr) = getShadowOriginPtr(
3287 Addr, IRB, ShadowTy: Shadow->getType(), Alignment: Align(1), /*isStore*/ true);
3288 IRB.CreateAlignedStore(Val: Shadow, Ptr: ShadowPtr, Align: Align(1));
3289
3290 if (Opts.msan_check_access_address)
3291 insertCheckShadowOf(Val: Addr, OrigIns: &I);
3292
3293 // FIXME: factor out common code from materializeStores
3294 if (MS.TrackOrigins)
3295 IRB.CreateStore(Val: getOrigin(I: &I, i: 1), Ptr: OriginPtr);
3296 return true;
3297 }
3298
3299 /// Handle vector load-like intrinsics.
3300 ///
3301 /// Instrument intrinsics that look like a simple SIMD load: reads memory,
3302 /// has 1 pointer argument, returns a vector.
3303 bool handleVectorLoadIntrinsic(IntrinsicInst &I) {
3304 assert(I.arg_size() == 1);
3305
3306 IRBuilder<> IRB(&I);
3307 Value *Addr = I.getArgOperand(i: 0);
3308
3309 Type *ShadowTy = getShadowTy(V: &I);
3310 Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
3311 if (PropagateShadow) {
3312 // We don't know the pointer alignment (could be unaligned SSE load!).
3313 // Have to assume to worst case.
3314 const Align Alignment = Align(1);
3315 std::tie(args&: ShadowPtr, args&: OriginPtr) =
3316 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
3317 setShadow(V: &I,
3318 SV: IRB.CreateAlignedLoad(Ty: ShadowTy, Ptr: ShadowPtr, Align: Alignment, Name: "_msld"));
3319 } else {
3320 setShadow(V: &I, SV: getCleanShadow(V: &I));
3321 }
3322
3323 if (Opts.msan_check_access_address)
3324 insertCheckShadowOf(Val: Addr, OrigIns: &I);
3325
3326 if (MS.TrackOrigins) {
3327 if (PropagateShadow)
3328 setOrigin(V: &I, Origin: IRB.CreateLoad(Ty: MS.OriginTy, Ptr: OriginPtr));
3329 else
3330 setOrigin(V: &I, Origin: getCleanOrigin());
3331 }
3332 return true;
3333 }
3334
3335 /// Handle (SIMD arithmetic)-like intrinsics.
3336 ///
3337 /// Instrument intrinsics with any number of arguments of the same type [*],
3338 /// equal to the return type, plus a specified number of trailing flags of
3339 /// any type.
3340 ///
3341 /// [*] The type should be simple (no aggregates or pointers; vectors are
3342 /// fine).
3343 ///
3344 /// Caller guarantees that this intrinsic does not access memory.
3345 ///
3346 /// TODO: "horizontal"/"pairwise" intrinsics are often incorrectly matched by
3347 /// by this handler. See horizontalReduce().
3348 ///
3349 /// TODO: permutation intrinsics are also often incorrectly matched.
3350 [[maybe_unused]] bool
3351 maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I,
3352 unsigned int trailingFlags) {
3353 Type *RetTy = I.getType();
3354 if (!(RetTy->isIntOrIntVectorTy() || RetTy->isFPOrFPVectorTy()))
3355 return false;
3356
3357 unsigned NumArgOperands = I.arg_size();
3358 assert(NumArgOperands >= trailingFlags);
3359 for (unsigned i = 0; i < NumArgOperands - trailingFlags; ++i) {
3360 Type *Ty = I.getArgOperand(i)->getType();
3361 if (Ty != RetTy)
3362 return false;
3363 }
3364
3365 IRBuilder<> IRB(&I);
3366 ShadowAndOriginCombiner SC(this, IRB);
3367 for (unsigned i = 0; i < NumArgOperands; ++i)
3368 SC.Add(V: I.getArgOperand(i));
3369 SC.Done(I: &I);
3370
3371 return true;
3372 }
3373
3374 /// Returns whether it was able to heuristically instrument unknown
3375 /// intrinsics.
3376 ///
3377 /// The main purpose of this code is to do something reasonable with all
3378 /// random intrinsics we might encounter, most importantly - SIMD intrinsics.
3379 /// We recognize several classes of intrinsics by their argument types and
3380 /// ModRefBehaviour and apply special instrumentation when we are reasonably
3381 /// sure that we know what the intrinsic does.
3382 ///
3383 /// We special-case intrinsics where this approach fails. See llvm.bswap
3384 /// handling as an example of that.
3385 bool maybeHandleUnknownIntrinsicUnlogged(IntrinsicInst &I) {
3386 unsigned NumArgOperands = I.arg_size();
3387 if (NumArgOperands == 0)
3388 return false;
3389
3390 if (NumArgOperands == 2 && I.getArgOperand(i: 0)->getType()->isPointerTy() &&
3391 I.getArgOperand(i: 1)->getType()->isVectorTy() &&
3392 I.getType()->isVoidTy() && !I.onlyReadsMemory()) {
3393 // This looks like a vector store.
3394 return handleVectorStoreIntrinsic(I);
3395 }
3396
3397 if (NumArgOperands == 1 && I.getArgOperand(i: 0)->getType()->isPointerTy() &&
3398 I.getType()->isVectorTy() && I.onlyReadsMemory()) {
3399 // This looks like a vector load.
3400 return handleVectorLoadIntrinsic(I);
3401 }
3402
3403 if (I.doesNotAccessMemory())
3404 if (maybeHandleSimpleNomemIntrinsic(I, /*trailingFlags=*/0))
3405 return true;
3406
3407 // FIXME: detect and handle SSE maskstore/maskload?
3408 // Some cases are now handled in handleAVXMasked{Load,Store}.
3409 return false;
3410 }
3411
3412 bool maybeHandleUnknownIntrinsic(IntrinsicInst &I) {
3413 if (maybeHandleUnknownIntrinsicUnlogged(I)) {
3414 if (Opts.msan_dump_heuristic_instructions)
3415 dumpInst(I, Prefix: "Heuristic");
3416
3417 LLVM_DEBUG(dbgs() << "UNKNOWN INSTRUCTION HANDLED HEURISTICALLY: " << I
3418 << "\n");
3419 return true;
3420 } else
3421 return false;
3422 }
3423
3424 void handleInvariantGroup(IntrinsicInst &I) {
3425 setShadow(V: &I, SV: getShadow(I: &I, i: 0));
3426 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
3427 }
3428
3429 void handleLifetimeStart(IntrinsicInst &I) {
3430 if (!PoisonStack)
3431 return;
3432 AllocaInst *AI = dyn_cast<AllocaInst>(Val: I.getArgOperand(i: 0));
3433 if (AI)
3434 LifetimeStartList.push_back(Elt: std::make_pair(x: &I, y&: AI));
3435 }
3436
3437 void handleBswap(IntrinsicInst &I) {
3438 IRBuilder<> IRB(&I);
3439 Value *Op = I.getArgOperand(i: 0);
3440 Type *OpType = Op->getType();
3441 setShadow(V: &I, SV: IRB.CreateIntrinsic(ID: Intrinsic::bswap, OverloadTypes: ArrayRef(&OpType, 1),
3442 Args: getShadow(V: Op)));
3443 setOrigin(V: &I, Origin: getOrigin(V: Op));
3444 }
3445
3446 // Uninitialized bits are ok if they appear after the leading/trailing 0's
3447 // and a 1. If the input is all zero, it is fully initialized iff
3448 // !is_zero_poison.
3449 //
3450 // e.g., for ctlz, with little-endian, if 0/1 are initialized bits with
3451 // concrete value 0/1, and ? is an uninitialized bit:
3452 // - 0001 0??? is fully initialized
3453 // - 000? ???? is fully uninitialized (*)
3454 // - ???? ???? is fully uninitialized
3455 // - 0000 0000 is fully uninitialized if is_zero_poison,
3456 // fully initialized otherwise
3457 //
3458 // (*) TODO: arguably, since the number of zeros is in the range [3, 8], we
3459 // only need to poison 4 bits.
3460 //
3461 // OutputShadow =
3462 // ((ConcreteZerosCount >= ShadowZerosCount) && !AllZeroShadow)
3463 // || (is_zero_poison && AllZeroSrc)
3464 void handleCountLeadingTrailingZeros(IntrinsicInst &I) {
3465 IRBuilder<> IRB(&I);
3466 Value *Src = I.getArgOperand(i: 0);
3467 Value *SrcShadow = getShadow(V: Src);
3468
3469 Value *False = IRB.getInt1(V: false);
3470 Value *ConcreteZerosCount = IRB.CreateIntrinsic(
3471 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: {Src, /*is_zero_poison=*/False});
3472 Value *ShadowZerosCount = IRB.CreateIntrinsic(
3473 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: {SrcShadow, /*is_zero_poison=*/False});
3474
3475 Value *CompareConcreteZeros = IRB.CreateICmpUGE(
3476 LHS: ConcreteZerosCount, RHS: ShadowZerosCount, Name: "_mscz_cmp_zeros");
3477
3478 Value *NotAllZeroShadow =
3479 IRB.CreateIsNotNull(Arg: SrcShadow, Name: "_mscz_shadow_not_null");
3480 Value *OutputShadow =
3481 IRB.CreateAnd(LHS: CompareConcreteZeros, RHS: NotAllZeroShadow, Name: "_mscz_main");
3482
3483 // If zero poison is requested, mix in with the shadow
3484 Constant *IsZeroPoison = cast<Constant>(Val: I.getOperand(i_nocapture: 1));
3485 if (!IsZeroPoison->isNullValue()) {
3486 Value *BoolZeroPoison = IRB.CreateIsNull(Arg: Src, Name: "_mscz_bzp");
3487 OutputShadow = IRB.CreateOr(LHS: OutputShadow, RHS: BoolZeroPoison, Name: "_mscz_bs");
3488 }
3489
3490 OutputShadow = IRB.CreateSExt(V: OutputShadow, DestTy: getShadowTy(V: Src), Name: "_mscz_os");
3491
3492 setShadow(V: &I, SV: OutputShadow);
3493 setOriginForNaryOp(I);
3494 }
3495
3496 /// Some instructions have additional zero-elements in the return type
3497 /// e.g., <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64>, ...)
3498 ///
3499 /// This function will return a vector type with the same number of elements
3500 /// as the input, but same per-element width as the return value e.g.,
3501 /// <8 x i8>.
3502 FixedVectorType *maybeShrinkVectorShadowType(Value *Src, IntrinsicInst &I) {
3503 assert(isa<FixedVectorType>(getShadowTy(&I)));
3504 FixedVectorType *ShadowType = cast<FixedVectorType>(Val: getShadowTy(V: &I));
3505
3506 // TODO: generalize beyond 2x?
3507 if (ShadowType->getElementCount() ==
3508 cast<VectorType>(Val: Src->getType())->getElementCount() * 2)
3509 ShadowType = FixedVectorType::getHalfElementsVectorType(VTy: ShadowType);
3510
3511 assert(ShadowType->getElementCount() ==
3512 cast<VectorType>(Src->getType())->getElementCount());
3513
3514 return ShadowType;
3515 }
3516
3517 /// Doubles the length of a vector shadow (extending with zeros) if necessary
3518 /// to match the length of the shadow for the instruction.
3519 /// If scalar types of the vectors are different, it will use the type of the
3520 /// input vector.
3521 /// This is more type-safe than CreateShadowCast().
3522 Value *maybeExtendVectorShadowWithZeros(Value *Shadow, IntrinsicInst &I) {
3523 IRBuilder<> IRB(&I);
3524 assert(isa<FixedVectorType>(Shadow->getType()));
3525 assert(isa<FixedVectorType>(I.getType()));
3526
3527 Value *FullShadow = getCleanShadow(V: &I);
3528 unsigned ShadowNumElems =
3529 cast<FixedVectorType>(Val: Shadow->getType())->getNumElements();
3530 unsigned FullShadowNumElems =
3531 cast<FixedVectorType>(Val: FullShadow->getType())->getNumElements();
3532
3533 assert((ShadowNumElems == FullShadowNumElems) ||
3534 (ShadowNumElems * 2 == FullShadowNumElems));
3535
3536 if (ShadowNumElems == FullShadowNumElems) {
3537 FullShadow = Shadow;
3538 } else {
3539 // TODO: generalize beyond 2x?
3540 SmallVector<int, 32> ShadowMask(FullShadowNumElems);
3541 std::iota(first: ShadowMask.begin(), last: ShadowMask.end(), value: 0);
3542
3543 // Append zeros
3544 FullShadow =
3545 IRB.CreateShuffleVector(V1: Shadow, V2: getCleanShadow(V: Shadow), Mask: ShadowMask);
3546 }
3547
3548 return FullShadow;
3549 }
3550
3551 /// Handle x86 SSE vector conversion.
3552 ///
3553 /// e.g., single-precision to half-precision conversion:
3554 /// <8 x i16> @llvm.x86.vcvtps2ph.256(<8 x float> %a0, i32 0)
3555 /// <8 x i16> @llvm.x86.vcvtps2ph.128(<4 x float> %a0, i32 0)
3556 ///
3557 /// floating-point to integer:
3558 /// <4 x i32> @llvm.x86.sse2.cvtps2dq(<4 x float>)
3559 /// <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double>)
3560 ///
3561 /// Note: if the output has more elements, they are zero-initialized (and
3562 /// therefore the shadow will also be initialized).
3563 ///
3564 /// This differs from handleSSEVectorConvertIntrinsic() because it
3565 /// propagates uninitialized shadow (instead of checking the shadow).
3566 void handleSSEVectorConvertIntrinsicByProp(IntrinsicInst &I,
3567 bool HasRoundingMode) {
3568 if (HasRoundingMode) {
3569 assert(I.arg_size() == 2);
3570 [[maybe_unused]] Value *RoundingMode = I.getArgOperand(i: 1);
3571 assert(RoundingMode->getType()->isIntegerTy());
3572 } else {
3573 assert(I.arg_size() == 1);
3574 }
3575
3576 Value *Src = I.getArgOperand(i: 0);
3577 assert(Src->getType()->isVectorTy());
3578
3579 // The return type might have more elements than the input.
3580 // Temporarily shrink the return type's number of elements.
3581 VectorType *ShadowType = maybeShrinkVectorShadowType(Src, I);
3582
3583 IRBuilder<> IRB(&I);
3584 Value *S0 = getShadow(I: &I, i: 0);
3585
3586 /// For scalars:
3587 /// Since they are converting to and/or from floating-point, the output is:
3588 /// - fully uninitialized if *any* bit of the input is uninitialized
3589 /// - fully ininitialized if all bits of the input are ininitialized
3590 /// We apply the same principle on a per-field basis for vectors.
3591 Value *Shadow =
3592 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S0, RHS: getCleanShadow(V: S0)), DestTy: ShadowType);
3593
3594 // The return type might have more elements than the input.
3595 // Extend the return type back to its original width if necessary.
3596 Value *FullShadow = maybeExtendVectorShadowWithZeros(Shadow, I);
3597
3598 setShadow(V: &I, SV: FullShadow);
3599 setOriginForNaryOp(I);
3600 }
3601
3602 // Instrument x86 SSE vector convert intrinsic.
3603 //
3604 // This function instruments intrinsics like cvtsi2ss:
3605 // %Out = int_xxx_cvtyyy(%ConvertOp)
3606 // or
3607 // %Out = int_xxx_cvtyyy(%CopyOp, %ConvertOp)
3608 // Intrinsic converts \p NumUsedElements elements of \p ConvertOp to the same
3609 // number \p Out elements, and (if has 2 arguments) copies the rest of the
3610 // elements from \p CopyOp.
3611 // In most cases conversion involves floating-point value which may trigger a
3612 // hardware exception when not fully initialized. For this reason we require
3613 // \p ConvertOp[0:NumUsedElements] to be fully initialized and trap otherwise.
3614 // We copy the shadow of \p CopyOp[NumUsedElements:] to \p
3615 // Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always
3616 // return a fully initialized value.
3617 //
3618 // For Arm NEON vector convert intrinsics, see
3619 // handleNEONVectorConvertIntrinsic().
3620 void handleSSEVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements,
3621 bool HasRoundingMode = false) {
3622 IRBuilder<> IRB(&I);
3623 Value *CopyOp, *ConvertOp;
3624
3625 assert((!HasRoundingMode ||
3626 isa<ConstantInt>(I.getArgOperand(I.arg_size() - 1))) &&
3627 "Invalid rounding mode");
3628
3629 switch (I.arg_size() - HasRoundingMode) {
3630 case 2:
3631 CopyOp = I.getArgOperand(i: 0);
3632 ConvertOp = I.getArgOperand(i: 1);
3633 break;
3634 case 1:
3635 ConvertOp = I.getArgOperand(i: 0);
3636 CopyOp = nullptr;
3637 break;
3638 default:
3639 llvm_unreachable("Cvt intrinsic with unsupported number of arguments.");
3640 }
3641
3642 // The first *NumUsedElements* elements of ConvertOp are converted to the
3643 // same number of output elements. The rest of the output is copied from
3644 // CopyOp, or (if not available) filled with zeroes.
3645 // Combine shadow for elements of ConvertOp that are used in this operation,
3646 // and insert a check.
3647 // FIXME: consider propagating shadow of ConvertOp, at least in the case of
3648 // int->any conversion.
3649 Value *ConvertShadow = getShadow(V: ConvertOp);
3650 Value *AggShadow = nullptr;
3651 if (ConvertOp->getType()->isVectorTy()) {
3652 AggShadow = IRB.CreateExtractElement(
3653 Vec: ConvertShadow, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: 0));
3654 for (int i = 1; i < NumUsedElements; ++i) {
3655 Value *MoreShadow = IRB.CreateExtractElement(
3656 Vec: ConvertShadow, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
3657 AggShadow = IRB.CreateOr(LHS: AggShadow, RHS: MoreShadow);
3658 }
3659 } else {
3660 AggShadow = ConvertShadow;
3661 }
3662 assert(AggShadow->getType()->isIntegerTy());
3663 insertCheckShadow(Shadow: AggShadow, Origin: getOrigin(V: ConvertOp), OrigIns: &I);
3664
3665 // Build result shadow by zero-filling parts of CopyOp shadow that come from
3666 // ConvertOp.
3667 if (CopyOp) {
3668 assert(CopyOp->getType() == I.getType());
3669 assert(CopyOp->getType()->isVectorTy());
3670 Value *ResultShadow = getShadow(V: CopyOp);
3671 Type *EltTy = cast<VectorType>(Val: ResultShadow->getType())->getElementType();
3672 for (int i = 0; i < NumUsedElements; ++i) {
3673 ResultShadow = IRB.CreateInsertElement(
3674 Vec: ResultShadow, NewElt: ConstantInt::getNullValue(Ty: EltTy),
3675 Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
3676 }
3677 setShadow(V: &I, SV: ResultShadow);
3678 setOrigin(V: &I, Origin: getOrigin(V: CopyOp));
3679 } else {
3680 setShadow(V: &I, SV: getCleanShadow(V: &I));
3681 setOrigin(V: &I, Origin: getCleanOrigin());
3682 }
3683 }
3684
3685 // Given a scalar or vector, extract lower 64 bits (or less), and return all
3686 // zeroes if it is zero, and all ones otherwise.
3687 Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
3688 if (S->getType()->isVectorTy())
3689 S = CreateShadowCast(IRB, V: S, dstTy: IRB.getInt64Ty(), /* Signed */ true);
3690 assert(S->getType()->getPrimitiveSizeInBits() <= 64);
3691 Value *S2 = IRB.CreateICmpNE(LHS: S, RHS: getCleanShadow(V: S));
3692 return CreateShadowCast(IRB, V: S2, dstTy: T, /* Signed */ true);
3693 }
3694
3695 // Given a vector, extract its first element, and return all
3696 // zeroes if it is zero, and all ones otherwise.
3697 Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
3698 Value *S1 = IRB.CreateExtractElement(Vec: S, Idx: (uint64_t)0);
3699 Value *S2 = IRB.CreateICmpNE(LHS: S1, RHS: getCleanShadow(V: S1));
3700 return CreateShadowCast(IRB, V: S2, dstTy: T, /* Signed */ true);
3701 }
3702
3703 Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) {
3704 Type *T = S->getType();
3705 assert(T->isVectorTy());
3706 Value *S2 = IRB.CreateICmpNE(LHS: S, RHS: getCleanShadow(V: S));
3707 return IRB.CreateSExt(V: S2, DestTy: T);
3708 }
3709
3710 // Instrument vector shift intrinsic.
3711 //
3712 // This function instruments intrinsics like int_x86_avx2_psll_w.
3713 // Intrinsic shifts %In by %ShiftSize bits.
3714 // %ShiftSize may be a vector. In that case the lower 64 bits determine shift
3715 // size, and the rest is ignored. Behavior is defined even if shift size is
3716 // greater than register (or field) width.
3717 void handleVectorShiftIntrinsic(IntrinsicInst &I, bool Variable) {
3718 assert(I.arg_size() == 2);
3719 IRBuilder<> IRB(&I);
3720 // If any of the S2 bits are poisoned, the whole thing is poisoned.
3721 // Otherwise perform the same shift on S1.
3722 Value *S1 = getShadow(I: &I, i: 0);
3723 Value *S2 = getShadow(I: &I, i: 1);
3724 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S: S2)
3725 : Lower64ShadowExtend(IRB, S: S2, T: getShadowTy(V: &I));
3726 Value *V1 = I.getOperand(i_nocapture: 0);
3727 Value *V2 = I.getOperand(i_nocapture: 1);
3728 Value *Shift = IRB.CreateCall(FTy: I.getFunctionType(), Callee: I.getCalledOperand(),
3729 Args: {IRB.CreateBitCast(V: S1, DestTy: V1->getType()), V2});
3730 Shift = IRB.CreateBitCast(V: Shift, DestTy: getShadowTy(V: &I));
3731 setShadow(V: &I, SV: IRB.CreateOr(LHS: Shift, RHS: S2Conv));
3732 setOriginForNaryOp(I);
3733 }
3734
3735 // Get an MMX-sized (64-bit) vector type, or optionally, other sized
3736 // vectors.
3737 Type *getMMXVectorTy(unsigned EltSizeInBits,
3738 unsigned X86_MMXSizeInBits = 64) {
3739 assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&
3740 "Illegal MMX vector element size");
3741 return FixedVectorType::get(ElementType: IntegerType::get(C&: *MS.C, NumBits: EltSizeInBits),
3742 NumElts: X86_MMXSizeInBits / EltSizeInBits);
3743 }
3744
3745 // Returns a signed counterpart for an (un)signed-saturate-and-pack
3746 // intrinsic.
3747 Intrinsic::ID getSignedPackIntrinsic(Intrinsic::ID id) {
3748 switch (id) {
3749 case Intrinsic::x86_sse2_packsswb_128:
3750 case Intrinsic::x86_sse2_packuswb_128:
3751 return Intrinsic::x86_sse2_packsswb_128;
3752
3753 case Intrinsic::x86_sse2_packssdw_128:
3754 case Intrinsic::x86_sse41_packusdw:
3755 return Intrinsic::x86_sse2_packssdw_128;
3756
3757 case Intrinsic::x86_avx2_packsswb:
3758 case Intrinsic::x86_avx2_packuswb:
3759 return Intrinsic::x86_avx2_packsswb;
3760
3761 case Intrinsic::x86_avx2_packssdw:
3762 case Intrinsic::x86_avx2_packusdw:
3763 return Intrinsic::x86_avx2_packssdw;
3764
3765 case Intrinsic::x86_mmx_packsswb:
3766 case Intrinsic::x86_mmx_packuswb:
3767 return Intrinsic::x86_mmx_packsswb;
3768
3769 case Intrinsic::x86_mmx_packssdw:
3770 return Intrinsic::x86_mmx_packssdw;
3771
3772 case Intrinsic::x86_avx512_packssdw_512:
3773 case Intrinsic::x86_avx512_packusdw_512:
3774 return Intrinsic::x86_avx512_packssdw_512;
3775
3776 case Intrinsic::x86_avx512_packsswb_512:
3777 case Intrinsic::x86_avx512_packuswb_512:
3778 return Intrinsic::x86_avx512_packsswb_512;
3779
3780 default:
3781 llvm_unreachable("unexpected intrinsic id");
3782 }
3783 }
3784
3785 // Instrument vector pack intrinsic.
3786 //
3787 // This function instruments intrinsics like x86_mmx_packsswb, that
3788 // packs elements of 2 input vectors into half as many bits with saturation.
3789 // Shadow is propagated with the signed variant of the same intrinsic applied
3790 // to sext(Sa != zeroinitializer), sext(Sb != zeroinitializer).
3791 // MMXEltSizeInBits is used only for x86mmx arguments.
3792 //
3793 // TODO: consider using GetMinMaxUnsigned() to handle saturation precisely
3794 void handleVectorPackIntrinsic(IntrinsicInst &I,
3795 unsigned MMXEltSizeInBits = 0) {
3796 assert(I.arg_size() == 2);
3797 IRBuilder<> IRB(&I);
3798 Value *S1 = getShadow(I: &I, i: 0);
3799 Value *S2 = getShadow(I: &I, i: 1);
3800 assert(S1->getType()->isVectorTy());
3801
3802 // SExt and ICmpNE below must apply to individual elements of input vectors.
3803 // In case of x86mmx arguments, cast them to appropriate vector types and
3804 // back.
3805 Type *T =
3806 MMXEltSizeInBits ? getMMXVectorTy(EltSizeInBits: MMXEltSizeInBits) : S1->getType();
3807 if (MMXEltSizeInBits) {
3808 S1 = IRB.CreateBitCast(V: S1, DestTy: T);
3809 S2 = IRB.CreateBitCast(V: S2, DestTy: T);
3810 }
3811 Value *S1_ext =
3812 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S1, RHS: Constant::getNullValue(Ty: T)), DestTy: T);
3813 Value *S2_ext =
3814 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S2, RHS: Constant::getNullValue(Ty: T)), DestTy: T);
3815 if (MMXEltSizeInBits) {
3816 S1_ext = IRB.CreateBitCast(V: S1_ext, DestTy: getMMXVectorTy(EltSizeInBits: 64));
3817 S2_ext = IRB.CreateBitCast(V: S2_ext, DestTy: getMMXVectorTy(EltSizeInBits: 64));
3818 }
3819
3820 Value *S = IRB.CreateIntrinsic(ID: getSignedPackIntrinsic(id: I.getIntrinsicID()),
3821 Args: {S1_ext, S2_ext}, /*FMFSource=*/nullptr,
3822 Name: "_msprop_vector_pack");
3823 if (MMXEltSizeInBits)
3824 S = IRB.CreateBitCast(V: S, DestTy: getShadowTy(V: &I));
3825 setShadow(V: &I, SV: S);
3826 setOriginForNaryOp(I);
3827 }
3828
3829 // Convert `Mask` into `<n x i1>`.
3830 Constant *createDppMask(unsigned Width, unsigned Mask) {
3831 SmallVector<Constant *, 4> R(Width);
3832 for (auto &M : R) {
3833 M = ConstantInt::getBool(Context&: F.getContext(), V: Mask & 1);
3834 Mask >>= 1;
3835 }
3836 return ConstantVector::get(V: R);
3837 }
3838
3839 // Calculate output shadow as array of booleans `<n x i1>`, assuming if any
3840 // arg is poisoned, entire dot product is poisoned.
3841 Value *findDppPoisonedOutput(IRBuilder<> &IRB, Value *S, unsigned SrcMask,
3842 unsigned DstMask) {
3843 const unsigned Width =
3844 cast<FixedVectorType>(Val: S->getType())->getNumElements();
3845
3846 S = IRB.CreateSelect(C: createDppMask(Width, Mask: SrcMask), True: S,
3847 False: Constant::getNullValue(Ty: S->getType()));
3848 Value *SElem = IRB.CreateOrReduce(Src: S);
3849 Value *IsClean = IRB.CreateIsNull(Arg: SElem, Name: "_msdpp");
3850 Value *DstMaskV = createDppMask(Width, Mask: DstMask);
3851
3852 return IRB.CreateSelect(
3853 C: IsClean, True: Constant::getNullValue(Ty: DstMaskV->getType()), False: DstMaskV);
3854 }
3855
3856 // See `Intel Intrinsics Guide` for `_dp_p*` instructions.
3857 //
3858 // 2 and 4 element versions produce single scalar of dot product, and then
3859 // puts it into elements of output vector, selected by 4 lowest bits of the
3860 // mask. Top 4 bits of the mask control which elements of input to use for dot
3861 // product.
3862 //
3863 // 8 element version mask still has only 4 bit for input, and 4 bit for output
3864 // mask. According to the spec it just operates as 4 element version on first
3865 // 4 elements of inputs and output, and then on last 4 elements of inputs and
3866 // output.
3867 void handleDppIntrinsic(IntrinsicInst &I) {
3868 IRBuilder<> IRB(&I);
3869
3870 Value *S0 = getShadow(I: &I, i: 0);
3871 Value *S1 = getShadow(I: &I, i: 1);
3872 Value *S = IRB.CreateOr(LHS: S0, RHS: S1);
3873
3874 const unsigned Width =
3875 cast<FixedVectorType>(Val: S->getType())->getNumElements();
3876 assert(Width == 2 || Width == 4 || Width == 8);
3877
3878 const unsigned Mask = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getZExtValue();
3879 const unsigned SrcMask = Mask >> 4;
3880 const unsigned DstMask = Mask & 0xf;
3881
3882 // Calculate shadow as `<n x i1>`.
3883 Value *SI1 = findDppPoisonedOutput(IRB, S, SrcMask, DstMask);
3884 if (Width == 8) {
3885 // First 4 elements of shadow are already calculated. `makeDppShadow`
3886 // operats on 32 bit masks, so we can just shift masks, and repeat.
3887 SI1 = IRB.CreateOr(
3888 LHS: SI1, RHS: findDppPoisonedOutput(IRB, S, SrcMask: SrcMask << 4, DstMask: DstMask << 4));
3889 }
3890 // Extend to real size of shadow, poisoning either all or none bits of an
3891 // element.
3892 S = IRB.CreateSExt(V: SI1, DestTy: S->getType(), Name: "_msdpp");
3893
3894 setShadow(V: &I, SV: S);
3895 setOriginForNaryOp(I);
3896 }
3897
3898 Value *convertBlendvToSelectMask(IRBuilder<> &IRB, Value *C) {
3899 C = CreateAppToShadowCast(IRB, V: C);
3900 FixedVectorType *FVT = cast<FixedVectorType>(Val: C->getType());
3901 unsigned ElSize = FVT->getElementType()->getPrimitiveSizeInBits();
3902 C = IRB.CreateAShr(LHS: C, RHS: ElSize - 1);
3903 FVT = FixedVectorType::get(ElementType: IRB.getInt1Ty(), NumElts: FVT->getNumElements());
3904 return IRB.CreateTrunc(V: C, DestTy: FVT);
3905 }
3906
3907 // `blendv(f, t, c)` is effectively `select(c[top_bit], t, f)`.
3908 void handleBlendvIntrinsic(IntrinsicInst &I) {
3909 Value *C = I.getOperand(i_nocapture: 2);
3910 Value *T = I.getOperand(i_nocapture: 1);
3911 Value *F = I.getOperand(i_nocapture: 0);
3912
3913 Value *Sc = getShadow(I: &I, i: 2);
3914 Value *Oc = MS.TrackOrigins ? getOrigin(V: C) : nullptr;
3915
3916 {
3917 IRBuilder<> IRB(&I);
3918 // Extract top bit from condition and its shadow.
3919 C = convertBlendvToSelectMask(IRB, C);
3920 Sc = convertBlendvToSelectMask(IRB, C: Sc);
3921
3922 setShadow(V: C, SV: Sc);
3923 setOrigin(V: C, Origin: Oc);
3924 }
3925
3926 handleSelectLikeInst(I, B: C, C: T, D: F);
3927 }
3928
3929 // Instrument sum-of-absolute-differences intrinsic.
3930 void handleVectorSadIntrinsic(IntrinsicInst &I, bool IsMMX = false) {
3931 const unsigned SignificantBitsPerResultElement = 16;
3932 Type *ResTy = IsMMX ? IntegerType::get(C&: *MS.C, NumBits: 64) : I.getType();
3933 unsigned ZeroBitsPerResultElement =
3934 ResTy->getScalarSizeInBits() - SignificantBitsPerResultElement;
3935
3936 IRBuilder<> IRB(&I);
3937 auto *Shadow0 = getShadow(I: &I, i: 0);
3938 auto *Shadow1 = getShadow(I: &I, i: 1);
3939 Value *S = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
3940 S = IRB.CreateBitCast(V: S, DestTy: ResTy);
3941 S = IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S, RHS: Constant::getNullValue(Ty: ResTy)),
3942 DestTy: ResTy);
3943 S = IRB.CreateLShr(LHS: S, RHS: ZeroBitsPerResultElement);
3944 S = IRB.CreateBitCast(V: S, DestTy: getShadowTy(V: &I));
3945 setShadow(V: &I, SV: S);
3946 setOriginForNaryOp(I);
3947 }
3948
3949 // Instrument dot-product / multiply-add(-accumulate)? intrinsics.
3950 //
3951 // e.g., Two operands:
3952 // <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a, <8 x i16> %b)
3953 //
3954 // Two operands which require an EltSizeInBits override:
3955 // <1 x i64> @llvm.x86.mmx.pmadd.wd(<1 x i64> %a, <1 x i64> %b)
3956 //
3957 // Three operands:
3958 // <4 x i32> @llvm.x86.avx512.vpdpbusd.128
3959 // (<4 x i32> %s, <16 x i8> %a, <16 x i8> %b)
3960 // <2 x float> @llvm.aarch64.neon.bfdot.v2f32.v4bf16
3961 // (<2 x float> %acc, <4 x bfloat> %a, <4 x bfloat> %b)
3962 // (these are equivalent to multiply-add on %a and %b, followed by
3963 // adding/"accumulating" %s. "Accumulation" stores the result in one
3964 // of the source registers, but this accumulate vs. add distinction
3965 // is lost when dealing with LLVM intrinsics.)
3966 //
3967 // ZeroPurifies means that multiplying a known-zero with an uninitialized
3968 // value results in an initialized value. This is applicable for integer
3969 // multiplication, but not floating-point (counter-example: NaN).
3970 void handleVectorDotProductIntrinsic(IntrinsicInst &I,
3971 unsigned ReductionFactor,
3972 bool ZeroPurifies,
3973 unsigned EltSizeInBits,
3974 enum OddOrEvenLanes Lanes) {
3975 IRBuilder<> IRB(&I);
3976
3977 [[maybe_unused]] FixedVectorType *ReturnType =
3978 cast<FixedVectorType>(Val: I.getType());
3979 assert(isa<FixedVectorType>(ReturnType));
3980
3981 // Vectors A and B, and shadows
3982 Value *Va = nullptr;
3983 Value *Vb = nullptr;
3984 Value *Sa = nullptr;
3985 Value *Sb = nullptr;
3986
3987 assert(I.arg_size() == 2 || I.arg_size() == 3);
3988 if (I.arg_size() == 2) {
3989 assert(Lanes == kBothLanes);
3990
3991 Va = I.getOperand(i_nocapture: 0);
3992 Vb = I.getOperand(i_nocapture: 1);
3993
3994 Sa = getShadow(I: &I, i: 0);
3995 Sb = getShadow(I: &I, i: 1);
3996 } else if (I.arg_size() == 3) {
3997 // Operand 0 is the accumulator. We will deal with that below.
3998 Va = I.getOperand(i_nocapture: 1);
3999 Vb = I.getOperand(i_nocapture: 2);
4000
4001 Sa = getShadow(I: &I, i: 1);
4002 Sb = getShadow(I: &I, i: 2);
4003
4004 if (Lanes == kEvenLanes || Lanes == kOddLanes) {
4005 // Convert < S0, S1, S2, S3, S4, S5, S6, S7 >
4006 // to < S0, S0, S2, S2, S4, S4, S6, S6 > (if even)
4007 // to < S1, S1, S3, S3, S5, S5, S7, S7 > (if odd)
4008 //
4009 // Note: for aarch64.neon.bfmlalb/t, the odd/even-indexed values are
4010 // zeroed, not duplicated. However, for shadow propagation, this
4011 // distinction is unimportant because Step 1 below will squeeze
4012 // each pair of elements (e.g., [S0, S0]) into a single bit, and
4013 // we only care if it is fully initialized.
4014
4015 FixedVectorType *InputShadowType = cast<FixedVectorType>(Val: Sa->getType());
4016 unsigned Width = InputShadowType->getNumElements();
4017
4018 Sa = IRB.CreateShuffleVector(
4019 V: Sa, Mask: getPclmulMask(Width, /*OddElements=*/Lanes == kOddLanes));
4020 Sb = IRB.CreateShuffleVector(
4021 V: Sb, Mask: getPclmulMask(Width, /*OddElements=*/Lanes == kOddLanes));
4022 }
4023 }
4024
4025 FixedVectorType *ParamType = cast<FixedVectorType>(Val: Va->getType());
4026 assert(ParamType == Vb->getType());
4027
4028 assert(ParamType->getPrimitiveSizeInBits() ==
4029 ReturnType->getPrimitiveSizeInBits());
4030
4031 if (I.arg_size() == 3) {
4032 [[maybe_unused]] auto *AccumulatorType =
4033 cast<FixedVectorType>(Val: I.getOperand(i_nocapture: 0)->getType());
4034 assert(AccumulatorType == ReturnType);
4035 }
4036
4037 FixedVectorType *ImplicitReturnType =
4038 cast<FixedVectorType>(Val: getShadowTy(OrigTy: ReturnType));
4039 // Step 1: instrument multiplication of corresponding vector elements
4040 if (EltSizeInBits) {
4041 ImplicitReturnType = cast<FixedVectorType>(
4042 Val: getMMXVectorTy(EltSizeInBits: EltSizeInBits * ReductionFactor,
4043 X86_MMXSizeInBits: ParamType->getPrimitiveSizeInBits()));
4044 ParamType = cast<FixedVectorType>(
4045 Val: getMMXVectorTy(EltSizeInBits, X86_MMXSizeInBits: ParamType->getPrimitiveSizeInBits()));
4046
4047 Va = IRB.CreateBitCast(V: Va, DestTy: ParamType);
4048 Vb = IRB.CreateBitCast(V: Vb, DestTy: ParamType);
4049
4050 Sa = IRB.CreateBitCast(V: Sa, DestTy: getShadowTy(OrigTy: ParamType));
4051 Sb = IRB.CreateBitCast(V: Sb, DestTy: getShadowTy(OrigTy: ParamType));
4052 } else {
4053 assert(ParamType->getNumElements() ==
4054 ReturnType->getNumElements() * ReductionFactor);
4055 }
4056
4057 // Each element of the vector is represented by a single bit (poisoned or
4058 // not) e.g., <8 x i1>.
4059 Value *SaNonZero = IRB.CreateIsNotNull(Arg: Sa);
4060 Value *SbNonZero = IRB.CreateIsNotNull(Arg: Sb);
4061 Value *And;
4062 if (ZeroPurifies) {
4063 // Multiplying an *initialized* zero by an uninitialized element results
4064 // in an initialized zero element.
4065 //
4066 // This is analogous to bitwise AND, where "AND" of 0 and a poisoned value
4067 // results in an unpoisoned value.
4068 Value *VaInt = Va;
4069 Value *VbInt = Vb;
4070 if (!Va->getType()->isIntegerTy()) {
4071 VaInt = CreateAppToShadowCast(IRB, V: Va);
4072 VbInt = CreateAppToShadowCast(IRB, V: Vb);
4073 }
4074
4075 // We check for non-zero on a per-element basis, not per-bit.
4076 Value *VaNonZero = IRB.CreateIsNotNull(Arg: VaInt);
4077 Value *VbNonZero = IRB.CreateIsNotNull(Arg: VbInt);
4078
4079 And = handleBitwiseAnd(IRB, V1: VaNonZero, V2: VbNonZero, S1: SaNonZero, S2: SbNonZero);
4080 } else {
4081 And = IRB.CreateOr(Ops: {SaNonZero, SbNonZero});
4082 }
4083
4084 // Extend <8 x i1> to <8 x i16>.
4085 // (The real pmadd intrinsic would have computed intermediate values of
4086 // <8 x i32>, but that is irrelevant for our shadow purposes because we
4087 // consider each element to be either fully initialized or fully
4088 // uninitialized.)
4089 And = IRB.CreateSExt(V: And, DestTy: Sa->getType());
4090
4091 // Step 2: instrument horizontal add
4092 // We don't need bit-precise horizontalReduce because we only want to check
4093 // if each pair/quad of elements is fully zero.
4094 // Cast to <4 x i32>.
4095 Value *Horizontal = IRB.CreateBitCast(V: And, DestTy: ImplicitReturnType);
4096
4097 // Compute <4 x i1>, then extend back to <4 x i32>.
4098 Value *OutShadow = IRB.CreateSExt(
4099 V: IRB.CreateICmpNE(LHS: Horizontal,
4100 RHS: Constant::getNullValue(Ty: Horizontal->getType())),
4101 DestTy: ImplicitReturnType);
4102
4103 // Cast it back to the required fake return type (if MMX: <1 x i64>; for
4104 // AVX, it is already correct).
4105 if (EltSizeInBits)
4106 OutShadow = CreateShadowCast(IRB, V: OutShadow, dstTy: getShadowTy(V: &I));
4107
4108 // Step 3 (if applicable): instrument accumulator
4109 if (I.arg_size() == 3)
4110 OutShadow = IRB.CreateOr(LHS: OutShadow, RHS: getShadow(I: &I, i: 0));
4111
4112 setShadow(V: &I, SV: OutShadow);
4113 setOriginForNaryOp(I);
4114 }
4115
4116 // Instrument compare-packed intrinsic.
4117 //
4118 // x86 has the predicate as the third operand, which is ImmArg e.g.,
4119 // - <4 x double> @llvm.x86.avx.cmp.pd.256(<4 x double>, <4 x double>, i8)
4120 // - <2 x double> @llvm.x86.sse2.cmp.pd(<2 x double>, <2 x double>, i8)
4121 //
4122 // while Arm has separate intrinsics for >= and > e.g.,
4123 // - <2 x i32> @llvm.aarch64.neon.facge.v2i32.v2f32
4124 // (<2 x float> %A, <2 x float>)
4125 // - <2 x i32> @llvm.aarch64.neon.facgt.v2i32.v2f32
4126 // (<2 x float> %A, <2 x float>)
4127 //
4128 // Bonus: this also handles scalar cases e.g.,
4129 // - i32 @llvm.aarch64.neon.facgt.i32.f32(float %A, float %B)
4130 void handleVectorComparePackedIntrinsic(IntrinsicInst &I,
4131 bool PredicateAsOperand) {
4132 if (PredicateAsOperand) {
4133 assert(I.arg_size() == 3);
4134 assert(I.paramHasAttr(2, Attribute::ImmArg));
4135 } else
4136 assert(I.arg_size() == 2);
4137
4138 IRBuilder<> IRB(&I);
4139
4140 // Basically, an or followed by sext(icmp ne 0) to end up with all-zeros or
4141 // all-ones shadow.
4142 Type *ResTy = getShadowTy(V: &I);
4143 auto *Shadow0 = getShadow(I: &I, i: 0);
4144 auto *Shadow1 = getShadow(I: &I, i: 1);
4145 Value *S0 = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
4146 Value *S = IRB.CreateSExt(
4147 V: IRB.CreateICmpNE(LHS: S0, RHS: Constant::getNullValue(Ty: ResTy)), DestTy: ResTy);
4148 setShadow(V: &I, SV: S);
4149 setOriginForNaryOp(I);
4150 }
4151
4152 // Instrument compare-scalar intrinsic.
4153 // This handles both cmp* intrinsics which return the result in the first
4154 // element of a vector, and comi* which return the result as i32.
4155 void handleVectorCompareScalarIntrinsic(IntrinsicInst &I) {
4156 IRBuilder<> IRB(&I);
4157 auto *Shadow0 = getShadow(I: &I, i: 0);
4158 auto *Shadow1 = getShadow(I: &I, i: 1);
4159 Value *S0 = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
4160 Value *S = LowerElementShadowExtend(IRB, S: S0, T: getShadowTy(V: &I));
4161 setShadow(V: &I, SV: S);
4162 setOriginForNaryOp(I);
4163 }
4164
4165 // Instrument generic vector reduction intrinsics
4166 // by ORing together all their fields.
4167 //
4168 // If AllowShadowCast is true, the return type does not need to be the same
4169 // type as the fields
4170 // e.g., declare i32 @llvm.aarch64.neon.uaddv.i32.v16i8(<16 x i8>)
4171 void handleVectorReduceIntrinsic(IntrinsicInst &I, bool AllowShadowCast) {
4172 assert(I.arg_size() == 1);
4173
4174 IRBuilder<> IRB(&I);
4175 Value *S = IRB.CreateOrReduce(Src: getShadow(I: &I, i: 0));
4176 if (AllowShadowCast)
4177 S = CreateShadowCast(IRB, V: S, dstTy: getShadowTy(V: &I));
4178 else
4179 assert(S->getType() == getShadowTy(&I));
4180 setShadow(V: &I, SV: S);
4181 setOriginForNaryOp(I);
4182 }
4183
4184 // Similar to handleVectorReduceIntrinsic but with an initial starting value.
4185 // e.g., call float @llvm.vector.reduce.fadd.f32.v2f32(float %a0, <2 x float>
4186 // %a1)
4187 // shadow = shadow[a0] | shadow[a1.0] | shadow[a1.1]
4188 //
4189 // The type of the return value, initial starting value, and elements of the
4190 // vector must be identical.
4191 void handleVectorReduceWithStarterIntrinsic(IntrinsicInst &I) {
4192 assert(I.arg_size() == 2);
4193
4194 IRBuilder<> IRB(&I);
4195 Value *Shadow0 = getShadow(I: &I, i: 0);
4196 Value *Shadow1 = IRB.CreateOrReduce(Src: getShadow(I: &I, i: 1));
4197 assert(Shadow0->getType() == Shadow1->getType());
4198 Value *S = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
4199 assert(S->getType() == getShadowTy(&I));
4200 setShadow(V: &I, SV: S);
4201 setOriginForNaryOp(I);
4202 }
4203
4204 // e.g., <4 x i32> @llvm.vector.partial.reduce.add (<4 x i32>, <8 x i32>)
4205 // acc addends
4206 // <4 x fp16> @llvm.vector.partial.reduce.fadd(<4 x fp16>, <8 x fp16>)
4207 // acc addends
4208 //
4209 // output[i] = acc[i] + addend[i] + addend[i+4]
4210 //
4211 // We approximate this as "all-or-nothing" for each element of the vector
4212 // i.e., even a single uninitialized bit in acc[i], addend[i], or
4213 // addend[i+4] will make output[i] fully uninitialized.
4214 //
4215 // By converting each vector element to a single value (0 or 1) representing
4216 // whether it is uninitialized, we can apply vector.partial.reduce.add to
4217 // compute the output shadow.
4218 //
4219 // N.B. this function is ready for scalable vectors, but its dependencies
4220 // are not.
4221 void handleVectorPartialReduceAddIntrinsic(IntrinsicInst &I) {
4222 IRBuilder<> IRB(&I);
4223
4224 assert(I.arg_size() == 2);
4225 [[maybe_unused]] Value *Accumulator = I.getArgOperand(i: 0);
4226 [[maybe_unused]] Value *Addends = I.getArgOperand(i: 1);
4227 assert(isa<VectorType>(Accumulator->getType()));
4228 assert(isa<VectorType>(Addends->getType()));
4229
4230 assert(I.getType() == Accumulator->getType());
4231
4232 assert(cast<VectorType>(Addends->getType())->getElementType() ==
4233 cast<VectorType>(I.getType())->getElementType());
4234 assert(cast<VectorType>(Addends->getType())
4235 ->getElementCount()
4236 .isKnownMultipleOf(
4237 cast<VectorType>(I.getType())->getElementCount()));
4238
4239 // e.g., <4 x i32>, <8 x i32>
4240 Value *SAcc = getShadow(I: &I, i: 0);
4241 Value *SAdd = getShadow(I: &I, i: 1);
4242
4243 // All-or-nothing shadows
4244 // e.g., <4 x i1>, <8 x i1>
4245 SAcc = IRB.CreateICmpNE(LHS: SAcc, RHS: getCleanShadow(V: SAcc));
4246 SAdd = IRB.CreateICmpNE(LHS: SAdd, RHS: getCleanShadow(V: SAdd));
4247
4248 // If we applied vector.partial.reduce.add to (<M x i1>, <N x i1>) shadows,
4249 // the sums might wrap around. Zero-extend to avoid this.
4250 // 16-bit for each sum (up to 65535) ought to be enough for anybody.
4251 SAcc = IRB.CreateZExt(
4252 V: SAcc,
4253 DestTy: VectorType::get(ElementType: IRB.getInt16Ty(),
4254 EC: cast<VectorType>(Val: SAcc->getType())->getElementCount()));
4255 SAdd = IRB.CreateZExt(
4256 V: SAdd,
4257 DestTy: VectorType::get(ElementType: IRB.getInt16Ty(),
4258 EC: cast<VectorType>(Val: SAdd->getType())->getElementCount()));
4259
4260 Value *SOutput = IRB.CreateIntrinsic(
4261 RetTy: SAcc->getType(), ID: Intrinsic::vector_partial_reduce_add, Args: {SAcc, SAdd});
4262
4263 SOutput = IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: SOutput, RHS: getCleanShadow(V: SOutput)),
4264 DestTy: getShadowTy(V: &I));
4265
4266 setShadow(V: &I, SV: SOutput);
4267 setOriginForNaryOp(I);
4268 }
4269
4270 // Instrument vector.reduce.or intrinsic.
4271 // Valid (non-poisoned) set bits in the operand pull low the
4272 // corresponding shadow bits.
4273 void handleVectorReduceOrIntrinsic(IntrinsicInst &I) {
4274 assert(I.arg_size() == 1);
4275
4276 IRBuilder<> IRB(&I);
4277 Value *OperandShadow = getShadow(I: &I, i: 0);
4278 Value *OperandUnsetBits = IRB.CreateNot(V: I.getOperand(i_nocapture: 0));
4279 Value *OperandUnsetOrPoison = IRB.CreateOr(LHS: OperandUnsetBits, RHS: OperandShadow);
4280 // Bit N is clean if any field's bit N is 1 and unpoison
4281 Value *OutShadowMask = IRB.CreateAndReduce(Src: OperandUnsetOrPoison);
4282 // Otherwise, it is clean if every field's bit N is unpoison
4283 Value *OrShadow = IRB.CreateOrReduce(Src: OperandShadow);
4284 Value *S = IRB.CreateAnd(LHS: OutShadowMask, RHS: OrShadow);
4285
4286 setShadow(V: &I, SV: S);
4287 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
4288 }
4289
4290 // Instrument vector.reduce.and intrinsic.
4291 // Valid (non-poisoned) unset bits in the operand pull down the
4292 // corresponding shadow bits.
4293 void handleVectorReduceAndIntrinsic(IntrinsicInst &I) {
4294 assert(I.arg_size() == 1);
4295
4296 IRBuilder<> IRB(&I);
4297 Value *OperandShadow = getShadow(I: &I, i: 0);
4298 Value *OperandSetOrPoison = IRB.CreateOr(LHS: I.getOperand(i_nocapture: 0), RHS: OperandShadow);
4299 // Bit N is clean if any field's bit N is 0 and unpoison
4300 Value *OutShadowMask = IRB.CreateAndReduce(Src: OperandSetOrPoison);
4301 // Otherwise, it is clean if every field's bit N is unpoison
4302 Value *OrShadow = IRB.CreateOrReduce(Src: OperandShadow);
4303 Value *S = IRB.CreateAnd(LHS: OutShadowMask, RHS: OrShadow);
4304
4305 setShadow(V: &I, SV: S);
4306 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
4307 }
4308
4309 void handleStmxcsr(IntrinsicInst &I) {
4310 IRBuilder<> IRB(&I);
4311 Value *Addr = I.getArgOperand(i: 0);
4312 Type *Ty = IRB.getInt32Ty();
4313 Value *ShadowPtr =
4314 getShadowOriginPtr(Addr, IRB, ShadowTy: Ty, Alignment: Align(1), /*isStore*/ true).first;
4315
4316 IRB.CreateStore(Val: getCleanShadow(OrigTy: Ty), Ptr: ShadowPtr);
4317
4318 if (Opts.msan_check_access_address)
4319 insertCheckShadowOf(Val: Addr, OrigIns: &I);
4320 }
4321
4322 void handleLdmxcsr(IntrinsicInst &I) {
4323 if (!InsertChecks)
4324 return;
4325
4326 IRBuilder<> IRB(&I);
4327 Value *Addr = I.getArgOperand(i: 0);
4328 Type *Ty = IRB.getInt32Ty();
4329 const Align Alignment = Align(1);
4330 Value *ShadowPtr, *OriginPtr;
4331 std::tie(args&: ShadowPtr, args&: OriginPtr) =
4332 getShadowOriginPtr(Addr, IRB, ShadowTy: Ty, Alignment, /*isStore*/ false);
4333
4334 if (Opts.msan_check_access_address)
4335 insertCheckShadowOf(Val: Addr, OrigIns: &I);
4336
4337 Value *Shadow = IRB.CreateAlignedLoad(Ty, Ptr: ShadowPtr, Align: Alignment, Name: "_ldmxcsr");
4338 Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(Ty: MS.OriginTy, Ptr: OriginPtr)
4339 : getCleanOrigin();
4340 insertCheckShadow(Shadow, Origin, OrigIns: &I);
4341 }
4342
4343 void handleMaskedExpandLoad(IntrinsicInst &I) {
4344 IRBuilder<> IRB(&I);
4345 Value *Ptr = I.getArgOperand(i: 0);
4346 MaybeAlign Align = I.getParamAlign(ArgNo: 0);
4347 Value *Mask = I.getArgOperand(i: 1);
4348 Value *PassThru = I.getArgOperand(i: 2);
4349
4350 if (Opts.msan_check_access_address) {
4351 insertCheckShadowOf(Val: Ptr, OrigIns: &I);
4352 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4353 }
4354
4355 if (!PropagateShadow) {
4356 setShadow(V: &I, SV: getCleanShadow(V: &I));
4357 setOrigin(V: &I, Origin: getCleanOrigin());
4358 return;
4359 }
4360
4361 Type *ShadowTy = getShadowTy(V: &I);
4362 Type *ElementShadowTy = cast<VectorType>(Val: ShadowTy)->getElementType();
4363 auto [ShadowPtr, OriginPtr] =
4364 getShadowOriginPtr(Addr: Ptr, IRB, ShadowTy: ElementShadowTy, Alignment: Align, /*isStore*/ false);
4365
4366 Value *Shadow =
4367 IRB.CreateMaskedExpandLoad(Ty: ShadowTy, Ptr: ShadowPtr, Align, Mask,
4368 PassThru: getShadow(V: PassThru), Name: "_msmaskedexpload");
4369
4370 setShadow(V: &I, SV: Shadow);
4371
4372 // TODO: Store origins.
4373 setOrigin(V: &I, Origin: getCleanOrigin());
4374 }
4375
4376 void handleMaskedCompressStore(IntrinsicInst &I) {
4377 IRBuilder<> IRB(&I);
4378 Value *Values = I.getArgOperand(i: 0);
4379 Value *Ptr = I.getArgOperand(i: 1);
4380 MaybeAlign Align = I.getParamAlign(ArgNo: 1);
4381 Value *Mask = I.getArgOperand(i: 2);
4382
4383 if (Opts.msan_check_access_address) {
4384 insertCheckShadowOf(Val: Ptr, OrigIns: &I);
4385 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4386 }
4387
4388 Value *Shadow = getShadow(V: Values);
4389 Type *ElementShadowTy =
4390 getShadowTy(OrigTy: cast<VectorType>(Val: Values->getType())->getElementType());
4391 auto [ShadowPtr, OriginPtrs] =
4392 getShadowOriginPtr(Addr: Ptr, IRB, ShadowTy: ElementShadowTy, Alignment: Align, /*isStore*/ true);
4393
4394 IRB.CreateMaskedCompressStore(Val: Shadow, Ptr: ShadowPtr, Align, Mask);
4395
4396 // TODO: Store origins.
4397 }
4398
4399 void handleMaskedGather(IntrinsicInst &I) {
4400 IRBuilder<> IRB(&I);
4401 Value *Ptrs = I.getArgOperand(i: 0);
4402 const Align Alignment = I.getParamAlign(ArgNo: 0).valueOrOne();
4403 Value *Mask = I.getArgOperand(i: 1);
4404 Value *PassThru = I.getArgOperand(i: 2);
4405
4406 Type *PtrsShadowTy = getShadowTy(V: Ptrs);
4407 if (Opts.msan_check_access_address) {
4408 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4409 Value *MaskedPtrShadow = IRB.CreateSelect(
4410 C: Mask, True: getShadow(V: Ptrs), False: Constant::getNullValue(Ty: (PtrsShadowTy)),
4411 Name: "_msmaskedptrs");
4412 insertCheckShadow(Shadow: MaskedPtrShadow, Origin: getOrigin(V: Ptrs), OrigIns: &I);
4413 }
4414
4415 if (!PropagateShadow) {
4416 setShadow(V: &I, SV: getCleanShadow(V: &I));
4417 setOrigin(V: &I, Origin: getCleanOrigin());
4418 return;
4419 }
4420
4421 Type *ShadowTy = getShadowTy(V: &I);
4422 Type *ElementShadowTy = cast<VectorType>(Val: ShadowTy)->getElementType();
4423 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
4424 Addr: Ptrs, IRB, ShadowTy: ElementShadowTy, Alignment, /*isStore*/ false);
4425
4426 Value *Shadow =
4427 IRB.CreateMaskedGather(Ty: ShadowTy, Ptrs: ShadowPtrs, Alignment, Mask,
4428 PassThru: getShadow(V: PassThru), Name: "_msmaskedgather");
4429
4430 setShadow(V: &I, SV: Shadow);
4431
4432 // TODO: Store origins.
4433 setOrigin(V: &I, Origin: getCleanOrigin());
4434 }
4435
4436 void handleMaskedScatter(IntrinsicInst &I) {
4437 IRBuilder<> IRB(&I);
4438 Value *Values = I.getArgOperand(i: 0);
4439 Value *Ptrs = I.getArgOperand(i: 1);
4440 const Align Alignment = I.getParamAlign(ArgNo: 1).valueOrOne();
4441 Value *Mask = I.getArgOperand(i: 2);
4442
4443 Type *PtrsShadowTy = getShadowTy(V: Ptrs);
4444 if (Opts.msan_check_access_address) {
4445 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4446 Value *MaskedPtrShadow = IRB.CreateSelect(
4447 C: Mask, True: getShadow(V: Ptrs), False: Constant::getNullValue(Ty: (PtrsShadowTy)),
4448 Name: "_msmaskedptrs");
4449 insertCheckShadow(Shadow: MaskedPtrShadow, Origin: getOrigin(V: Ptrs), OrigIns: &I);
4450 }
4451
4452 Value *Shadow = getShadow(V: Values);
4453 Type *ElementShadowTy =
4454 getShadowTy(OrigTy: cast<VectorType>(Val: Values->getType())->getElementType());
4455 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
4456 Addr: Ptrs, IRB, ShadowTy: ElementShadowTy, Alignment, /*isStore*/ true);
4457
4458 IRB.CreateMaskedScatter(Val: Shadow, Ptrs: ShadowPtrs, Alignment, Mask);
4459
4460 // TODO: Store origin.
4461 }
4462
4463 // Intrinsic::masked_store
4464 //
4465 // Note: handleAVXMaskedStore handles AVX/AVX2 variants, though AVX512 masked
4466 // stores are lowered to Intrinsic::masked_store.
4467 void handleMaskedStore(IntrinsicInst &I) {
4468 IRBuilder<> IRB(&I);
4469 Value *V = I.getArgOperand(i: 0);
4470 Value *Ptr = I.getArgOperand(i: 1);
4471 const Align Alignment = I.getParamAlign(ArgNo: 1).valueOrOne();
4472 Value *Mask = I.getArgOperand(i: 2);
4473 Value *Shadow = getShadow(V);
4474
4475 if (Opts.msan_check_access_address) {
4476 insertCheckShadowOf(Val: Ptr, OrigIns: &I);
4477 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4478 }
4479
4480 Value *ShadowPtr;
4481 Value *OriginPtr;
4482 std::tie(args&: ShadowPtr, args&: OriginPtr) = getShadowOriginPtr(
4483 Addr: Ptr, IRB, ShadowTy: Shadow->getType(), Alignment, /*isStore*/ true);
4484
4485 IRB.CreateMaskedStore(Val: Shadow, Ptr: ShadowPtr, Alignment, Mask);
4486
4487 if (!MS.TrackOrigins)
4488 return;
4489
4490 auto &DL = F.getDataLayout();
4491 paintOrigin(IRB, Origin: getOrigin(V), OriginPtr,
4492 TS: DL.getTypeStoreSize(Ty: Shadow->getType()),
4493 Alignment: std::max(a: Alignment, b: kMinOriginAlignment));
4494 }
4495
4496 // Intrinsic::masked_load
4497 //
4498 // Note: handleAVXMaskedLoad handles AVX/AVX2 variants, though AVX512 masked
4499 // loads are lowered to Intrinsic::masked_load.
4500 void handleMaskedLoad(IntrinsicInst &I) {
4501 IRBuilder<> IRB(&I);
4502 Value *Ptr = I.getArgOperand(i: 0);
4503 const Align Alignment = I.getParamAlign(ArgNo: 0).valueOrOne();
4504 Value *Mask = I.getArgOperand(i: 1);
4505 Value *PassThru = I.getArgOperand(i: 2);
4506
4507 if (Opts.msan_check_access_address) {
4508 insertCheckShadowOf(Val: Ptr, OrigIns: &I);
4509 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4510 }
4511
4512 if (!PropagateShadow) {
4513 setShadow(V: &I, SV: getCleanShadow(V: &I));
4514 setOrigin(V: &I, Origin: getCleanOrigin());
4515 return;
4516 }
4517
4518 Type *ShadowTy = getShadowTy(V: &I);
4519 Value *ShadowPtr, *OriginPtr;
4520 std::tie(args&: ShadowPtr, args&: OriginPtr) =
4521 getShadowOriginPtr(Addr: Ptr, IRB, ShadowTy, Alignment, /*isStore*/ false);
4522 setShadow(V: &I, SV: IRB.CreateMaskedLoad(Ty: ShadowTy, Ptr: ShadowPtr, Alignment, Mask,
4523 PassThru: getShadow(V: PassThru), Name: "_msmaskedld"));
4524
4525 if (!MS.TrackOrigins)
4526 return;
4527
4528 // Choose between PassThru's and the loaded value's origins.
4529 Value *MaskedPassThruShadow = IRB.CreateAnd(
4530 LHS: getShadow(V: PassThru), RHS: IRB.CreateSExt(V: IRB.CreateNeg(V: Mask), DestTy: ShadowTy));
4531
4532 Value *NotNull = convertToBool(V: MaskedPassThruShadow, IRB, name: "_mscmp");
4533
4534 Value *PtrOrigin = IRB.CreateLoad(Ty: MS.OriginTy, Ptr: OriginPtr);
4535 Value *Origin = IRB.CreateSelect(C: NotNull, True: getOrigin(V: PassThru), False: PtrOrigin);
4536
4537 setOrigin(V: &I, Origin);
4538 }
4539
4540 // e.g., <4 x i32> @llvm.masked.udiv.v4i32(<4 x i32> %dividend,
4541 // <4 x i32> %divisor,
4542 // <4 x i1> %mask)
4543 //
4544 // As handleIntegerDiv(), but per-lane: strict on the divisor and propagating
4545 // the dividend, both only on the enabled lanes. Disabled lanes cannot cause
4546 // undefined behaviour, and their result is poison.
4547 void handleMaskedIntegerDivRem(IntrinsicInst &I) {
4548 assert(I.arg_size() == 3);
4549 IRBuilder<> IRB(&I);
4550 Value *Dividend = I.getArgOperand(i: 0);
4551 Value *Divisor = I.getArgOperand(i: 1);
4552 Value *Mask = I.getArgOperand(i: 2);
4553
4554 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4555
4556 Value *MaskedDivisorShadow = IRB.CreateSelect(
4557 C: Mask, True: getShadow(V: Divisor), False: getCleanShadow(V: Divisor), Name: "_msmaskeddivisor");
4558 insertCheckShadow(Shadow: MaskedDivisorShadow, Origin: getOrigin(V: Divisor), OrigIns: &I);
4559
4560 if (!PropagateShadow) {
4561 setShadow(V: &I, SV: getCleanShadow(V: &I));
4562 setOrigin(V: &I, Origin: getCleanOrigin());
4563 return;
4564 }
4565
4566 setShadow(V: &I, SV: IRB.CreateSelect(C: Mask, True: getShadow(V: Dividend),
4567 False: getPoisonedShadow(V: &I), Name: "_msmaskeddiv"));
4568 setOrigin(V: &I, Origin: getOrigin(V: Dividend));
4569 }
4570
4571 // e.g., i32 @llvm.experimental.vector.extract.last.active.v4i32(
4572 // <4 x i32> %data, <4 x i1> %mask, i32 %passthru)
4573 void handleVectorExtractLastActive(IntrinsicInst &I) {
4574 assert(I.arg_size() == 3);
4575 IRBuilder<> IRB(&I);
4576 Value *Data = I.getArgOperand(i: 0);
4577 Value *Mask = I.getArgOperand(i: 1);
4578 Value *PassThru = I.getArgOperand(i: 2);
4579
4580 assert(isa<VectorType>(Data->getType()));
4581 assert(isa<VectorType>(Mask->getType()));
4582 assert(!isa<VectorType>(PassThru->getType()));
4583 assert(!isa<VectorType>(I.getType()));
4584 assert(I.getType() == PassThru->getType());
4585 assert(cast<VectorType>(Data->getType())->getElementType() == I.getType());
4586 assert(cast<VectorType>(Mask->getType())->getElementType() ==
4587 IRB.getInt1Ty());
4588 assert(cast<VectorType>(Data->getType())->getElementCount() ==
4589 cast<VectorType>(Mask->getType())->getElementCount());
4590
4591 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4592
4593 if (!PropagateShadow) {
4594 setShadow(V: &I, SV: getCleanShadow(V: &I));
4595 setOrigin(V: &I, Origin: getCleanOrigin());
4596 return;
4597 }
4598
4599 Value *DataShadow = getShadow(V: Data);
4600 Value *Shadow = IRB.CreateIntrinsic(
4601 ID: Intrinsic::experimental_vector_extract_last_active,
4602 OverloadTypes: {DataShadow->getType()}, Args: {DataShadow, Mask, getShadow(V: PassThru)},
4603 /*FMFSource=*/{}, Name: "_msprop_vector_extract_last_active");
4604 setShadow(V: &I, SV: Shadow);
4605
4606 if (MS.TrackOrigins) {
4607 Value *AnyActive = IRB.CreateOrReduce(Src: Mask);
4608 setOrigin(V: &I, Origin: IRB.CreateSelect(C: AnyActive, True: getOrigin(V: Data),
4609 False: getOrigin(V: PassThru)));
4610 }
4611 }
4612
4613 // e.g., void @llvm.x86.avx.maskstore.ps.256(ptr, <8 x i32>, <8 x float>)
4614 // dst mask src
4615 //
4616 // AVX512 masked stores are lowered to Intrinsic::masked_load and are handled
4617 // by handleMaskedStore.
4618 //
4619 // This function handles AVX and AVX2 masked stores; these use the MSBs of a
4620 // vector of integers, unlike the LLVM masked intrinsics, which require a
4621 // vector of booleans. X86InstCombineIntrinsic.cpp::simplifyX86MaskedLoad
4622 // mentions that the x86 backend does not know how to efficiently convert
4623 // from a vector of booleans back into the AVX mask format; therefore, they
4624 // (and we) do not reduce AVX/AVX2 masked intrinsics into LLVM masked
4625 // intrinsics.
4626 void handleAVXMaskedStore(IntrinsicInst &I) {
4627 assert(I.arg_size() == 3);
4628
4629 IRBuilder<> IRB(&I);
4630
4631 Value *Dst = I.getArgOperand(i: 0);
4632 assert(Dst->getType()->isPointerTy() && "Destination is not a pointer!");
4633
4634 Value *Mask = I.getArgOperand(i: 1);
4635 assert(isa<VectorType>(Mask->getType()) && "Mask is not a vector!");
4636
4637 Value *Src = I.getArgOperand(i: 2);
4638 assert(isa<VectorType>(Src->getType()) && "Source is not a vector!");
4639
4640 const Align Alignment = Align(1);
4641
4642 Value *SrcShadow = getShadow(V: Src);
4643
4644 if (Opts.msan_check_access_address) {
4645 insertCheckShadowOf(Val: Dst, OrigIns: &I);
4646 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4647 }
4648
4649 Value *DstShadowPtr;
4650 Value *DstOriginPtr;
4651 std::tie(args&: DstShadowPtr, args&: DstOriginPtr) = getShadowOriginPtr(
4652 Addr: Dst, IRB, ShadowTy: SrcShadow->getType(), Alignment, /*isStore*/ true);
4653
4654 SmallVector<Value *, 2> ShadowArgs;
4655 ShadowArgs.append(NumInputs: 1, Elt: DstShadowPtr);
4656 ShadowArgs.append(NumInputs: 1, Elt: Mask);
4657 // The intrinsic may require floating-point but shadows can be arbitrary
4658 // bit patterns, of which some would be interpreted as "invalid"
4659 // floating-point values (NaN etc.); we assume the intrinsic will happily
4660 // copy them.
4661 ShadowArgs.append(NumInputs: 1, Elt: IRB.CreateBitCast(V: SrcShadow, DestTy: Src->getType()));
4662
4663 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
4664 RetTy: IRB.getVoidTy(), ID: I.getIntrinsicID(), Args: ShadowArgs);
4665 setShadow(V: &I, SV: CI);
4666
4667 if (!MS.TrackOrigins)
4668 return;
4669
4670 // Approximation only
4671 auto &DL = F.getDataLayout();
4672 paintOrigin(IRB, Origin: getOrigin(V: Src), OriginPtr: DstOriginPtr,
4673 TS: DL.getTypeStoreSize(Ty: SrcShadow->getType()),
4674 Alignment: std::max(a: Alignment, b: kMinOriginAlignment));
4675 }
4676
4677 // e.g., <8 x float> @llvm.x86.avx.maskload.ps.256(ptr, <8 x i32>)
4678 // return src mask
4679 //
4680 // Masked-off values are replaced with 0, which conveniently also represents
4681 // initialized memory.
4682 //
4683 // AVX512 masked stores are lowered to Intrinsic::masked_load and are handled
4684 // by handleMaskedStore.
4685 //
4686 // We do not combine this with handleMaskedLoad; see comment in
4687 // handleAVXMaskedStore for the rationale.
4688 //
4689 // This is subtly different than handleIntrinsicByApplyingToShadow(I, 1)
4690 // because we need to apply getShadowOriginPtr, not getShadow, to the first
4691 // parameter.
4692 void handleAVXMaskedLoad(IntrinsicInst &I) {
4693 assert(I.arg_size() == 2);
4694
4695 IRBuilder<> IRB(&I);
4696
4697 Value *Src = I.getArgOperand(i: 0);
4698 assert(Src->getType()->isPointerTy() && "Source is not a pointer!");
4699
4700 Value *Mask = I.getArgOperand(i: 1);
4701 assert(isa<VectorType>(Mask->getType()) && "Mask is not a vector!");
4702
4703 const Align Alignment = Align(1);
4704
4705 if (Opts.msan_check_access_address) {
4706 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4707 }
4708
4709 Type *SrcShadowTy = getShadowTy(V: Src);
4710 Value *SrcShadowPtr, *SrcOriginPtr;
4711 std::tie(args&: SrcShadowPtr, args&: SrcOriginPtr) =
4712 getShadowOriginPtr(Addr: Src, IRB, ShadowTy: SrcShadowTy, Alignment, /*isStore*/ false);
4713
4714 SmallVector<Value *, 2> ShadowArgs;
4715 ShadowArgs.append(NumInputs: 1, Elt: SrcShadowPtr);
4716 ShadowArgs.append(NumInputs: 1, Elt: Mask);
4717
4718 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
4719 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: ShadowArgs);
4720 // The AVX masked load intrinsics do not have integer variants. We use the
4721 // floating-point variants, which will happily copy the shadows even if
4722 // they are interpreted as "invalid" floating-point values (NaN etc.).
4723 setShadow(V: &I, SV: IRB.CreateBitCast(V: CI, DestTy: getShadowTy(V: &I)));
4724
4725 if (!MS.TrackOrigins)
4726 return;
4727
4728 // The "pass-through" value is always zero (initialized). To the extent
4729 // that that results in initialized aligned 4-byte chunks, the origin value
4730 // is ignored. It is therefore correct to simply copy the origin from src.
4731 Value *PtrSrcOrigin = IRB.CreateLoad(Ty: MS.OriginTy, Ptr: SrcOriginPtr);
4732 setOrigin(V: &I, Origin: PtrSrcOrigin);
4733 }
4734
4735 // Test whether the mask indices are initialized, only checking the bits that
4736 // are actually used.
4737 //
4738 // e.g., if Idx is <32 x i16>, only (log2(32) == 5) bits of each index are
4739 // used/checked.
4740 void maskedCheckAVXIndexShadow(IRBuilder<> &IRB, Value *Idx, Instruction *I) {
4741 assert(isFixedIntVector(Idx));
4742 auto IdxVectorSize =
4743 cast<FixedVectorType>(Val: Idx->getType())->getNumElements();
4744 assert(isPowerOf2_64(IdxVectorSize));
4745
4746 // Compiler isn't smart enough, let's help it
4747 if (isa<Constant>(Val: Idx))
4748 return;
4749
4750 auto *IdxShadow = getShadow(V: Idx);
4751 Value *Truncated = IRB.CreateTrunc(
4752 V: IdxShadow,
4753 DestTy: FixedVectorType::get(ElementType: Type::getIntNTy(C&: *MS.C, N: Log2_64(Value: IdxVectorSize)),
4754 NumElts: IdxVectorSize));
4755 insertCheckShadow(Shadow: Truncated, Origin: getOrigin(V: Idx), OrigIns: I);
4756 }
4757
4758 // Instrument AVX permutation intrinsic.
4759 // We apply the same permutation (argument index 1) to the shadow.
4760 void handleAVXVpermilvar(IntrinsicInst &I) {
4761 IRBuilder<> IRB(&I);
4762 Value *Shadow = getShadow(I: &I, i: 0);
4763 maskedCheckAVXIndexShadow(IRB, Idx: I.getArgOperand(i: 1), I: &I);
4764
4765 // Shadows are integer-ish types but some intrinsics require a
4766 // different (e.g., floating-point) type.
4767 Shadow = IRB.CreateBitCast(V: Shadow, DestTy: I.getArgOperand(i: 0)->getType());
4768 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
4769 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: {Shadow, I.getArgOperand(i: 1)});
4770
4771 setShadow(V: &I, SV: IRB.CreateBitCast(V: CI, DestTy: getShadowTy(V: &I)));
4772 setOriginForNaryOp(I);
4773 }
4774
4775 // Instrument AVX permutation intrinsic.
4776 // We apply the same permutation (argument index 1) to the shadows.
4777 void handleAVXVpermi2var(IntrinsicInst &I) {
4778 assert(I.arg_size() == 3);
4779 assert(isa<FixedVectorType>(I.getArgOperand(0)->getType()));
4780 assert(isa<FixedVectorType>(I.getArgOperand(1)->getType()));
4781 assert(isa<FixedVectorType>(I.getArgOperand(2)->getType()));
4782 [[maybe_unused]] auto ArgVectorSize =
4783 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements();
4784 assert(cast<FixedVectorType>(I.getArgOperand(1)->getType())
4785 ->getNumElements() == ArgVectorSize);
4786 assert(cast<FixedVectorType>(I.getArgOperand(2)->getType())
4787 ->getNumElements() == ArgVectorSize);
4788 assert(I.getArgOperand(0)->getType() == I.getArgOperand(2)->getType());
4789 assert(I.getType() == I.getArgOperand(0)->getType());
4790 assert(I.getArgOperand(1)->getType()->isIntOrIntVectorTy());
4791 IRBuilder<> IRB(&I);
4792 Value *AShadow = getShadow(I: &I, i: 0);
4793 Value *Idx = I.getArgOperand(i: 1);
4794 Value *BShadow = getShadow(I: &I, i: 2);
4795
4796 maskedCheckAVXIndexShadow(IRB, Idx, I: &I);
4797
4798 // Shadows are integer-ish types but some intrinsics require a
4799 // different (e.g., floating-point) type.
4800 AShadow = IRB.CreateBitCast(V: AShadow, DestTy: I.getArgOperand(i: 0)->getType());
4801 BShadow = IRB.CreateBitCast(V: BShadow, DestTy: I.getArgOperand(i: 2)->getType());
4802 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
4803 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: {AShadow, Idx, BShadow});
4804 setShadow(V: &I, SV: IRB.CreateBitCast(V: CI, DestTy: getShadowTy(V: &I)));
4805 setOriginForNaryOp(I);
4806 }
4807
4808 [[maybe_unused]] static bool isFixedIntVectorTy(const Type *T) {
4809 return isa<FixedVectorType>(Val: T) && T->isIntOrIntVectorTy();
4810 }
4811
4812 [[maybe_unused]] static bool isFixedFPVectorTy(const Type *T) {
4813 return isa<FixedVectorType>(Val: T) && T->isFPOrFPVectorTy();
4814 }
4815
4816 [[maybe_unused]] static bool isFixedIntVector(const Value *V) {
4817 return isFixedIntVectorTy(T: V->getType());
4818 }
4819
4820 [[maybe_unused]] static bool isFixedFPVector(const Value *V) {
4821 return isFixedFPVectorTy(T: V->getType());
4822 }
4823
4824 // e.g., <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512
4825 // (<16 x float> a, <16 x i32> writethru, i16 mask,
4826 // i32 rounding)
4827 //
4828 // Inconveniently, some similar intrinsics have a different operand order:
4829 // <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512
4830 // (<16 x float> a, i32 rounding, <16 x i16> writethru,
4831 // i16 mask)
4832 //
4833 // If the return type has more elements than A, the excess elements are
4834 // zeroed (and the corresponding shadow is initialized).
4835 // <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128
4836 // (<4 x float> a, i32 rounding, <8 x i16> writethru,
4837 // i8 mask)
4838 //
4839 // dst[i] = mask[i] ? convert(a[i]) : writethru[i]
4840 // dst_shadow[i] = mask[i] ? all_or_nothing(a_shadow[i]) : writethru_shadow[i]
4841 // where all_or_nothing(x) is fully uninitialized if x has any
4842 // uninitialized bits
4843 void handleAVX512VectorConvertFPToInt(IntrinsicInst &I, bool LastMask) {
4844 IRBuilder<> IRB(&I);
4845
4846 assert(I.arg_size() == 4);
4847 Value *A = I.getOperand(i_nocapture: 0);
4848 Value *WriteThrough;
4849 Value *Mask;
4850 Value *RoundingMode;
4851 if (LastMask) {
4852 WriteThrough = I.getOperand(i_nocapture: 2);
4853 Mask = I.getOperand(i_nocapture: 3);
4854 RoundingMode = I.getOperand(i_nocapture: 1);
4855 } else {
4856 WriteThrough = I.getOperand(i_nocapture: 1);
4857 Mask = I.getOperand(i_nocapture: 2);
4858 RoundingMode = I.getOperand(i_nocapture: 3);
4859 }
4860
4861 assert(isFixedFPVector(A));
4862 assert(isFixedIntVector(WriteThrough));
4863
4864 unsigned ANumElements =
4865 cast<FixedVectorType>(Val: A->getType())->getNumElements();
4866 [[maybe_unused]] unsigned WriteThruNumElements =
4867 cast<FixedVectorType>(Val: WriteThrough->getType())->getNumElements();
4868 assert(ANumElements == WriteThruNumElements ||
4869 ANumElements * 2 == WriteThruNumElements);
4870
4871 assert(Mask->getType()->isIntegerTy());
4872 unsigned MaskNumElements = Mask->getType()->getScalarSizeInBits();
4873 assert(ANumElements == MaskNumElements ||
4874 ANumElements * 2 == MaskNumElements);
4875
4876 assert(WriteThruNumElements == MaskNumElements);
4877
4878 // Some bits of the mask may be unused, though it's unusual to have partly
4879 // uninitialized bits.
4880 insertCheckShadowOf(Val: Mask, OrigIns: &I);
4881
4882 assert(RoundingMode->getType()->isIntegerTy());
4883 // Only some bits of the rounding mode are used, though it's very
4884 // unusual to have uninitialized bits there (more commonly, it's a
4885 // constant).
4886 insertCheckShadowOf(Val: RoundingMode, OrigIns: &I);
4887
4888 assert(I.getType() == WriteThrough->getType());
4889
4890 Value *AShadow = getShadow(V: A);
4891 AShadow = maybeExtendVectorShadowWithZeros(Shadow: AShadow, I);
4892
4893 if (ANumElements * 2 == MaskNumElements) {
4894 // Ensure that the irrelevant bits of the mask are zero, hence selecting
4895 // from the zeroed shadow instead of the writethrough's shadow.
4896 Mask =
4897 IRB.CreateTrunc(V: Mask, DestTy: IRB.getIntNTy(N: ANumElements), Name: "_ms_mask_trunc");
4898 Mask =
4899 IRB.CreateZExt(V: Mask, DestTy: IRB.getIntNTy(N: MaskNumElements), Name: "_ms_mask_zext");
4900 }
4901
4902 // Convert i16 mask to <16 x i1>
4903 Mask = IRB.CreateBitCast(
4904 V: Mask, DestTy: FixedVectorType::get(ElementType: IRB.getInt1Ty(), NumElts: MaskNumElements),
4905 Name: "_ms_mask_bitcast");
4906
4907 /// For floating-point to integer conversion, the output is:
4908 /// - fully uninitialized if *any* bit of the input is uninitialized
4909 /// - fully ininitialized if all bits of the input are ininitialized
4910 /// We apply the same principle on a per-element basis for vectors.
4911 ///
4912 /// We use the scalar width of the return type instead of A's.
4913 AShadow = IRB.CreateSExt(
4914 V: IRB.CreateICmpNE(LHS: AShadow, RHS: getCleanShadow(OrigTy: AShadow->getType())),
4915 DestTy: getShadowTy(V: &I), Name: "_ms_a_shadow");
4916
4917 Value *WriteThroughShadow = getShadow(V: WriteThrough);
4918 Value *Shadow = IRB.CreateSelect(C: Mask, True: AShadow, False: WriteThroughShadow,
4919 Name: "_ms_writethru_select");
4920
4921 setShadow(V: &I, SV: Shadow);
4922 setOriginForNaryOp(I);
4923 }
4924
4925 static SmallVector<int, 8> getPclmulMask(unsigned Width, bool OddElements) {
4926 SmallVector<int, 8> Mask;
4927 for (unsigned X = OddElements ? 1 : 0; X < Width; X += 2) {
4928 Mask.append(NumInputs: 2, Elt: X);
4929 }
4930 return Mask;
4931 }
4932
4933 // Instrument pclmul intrinsics.
4934 // These intrinsics operate either on odd or on even elements of the input
4935 // vectors, depending on the constant in the 3rd argument, ignoring the rest.
4936 // Replace the unused elements with copies of the used ones, ex:
4937 // (0, 1, 2, 3) -> (0, 0, 2, 2) (even case)
4938 // or
4939 // (0, 1, 2, 3) -> (1, 1, 3, 3) (odd case)
4940 // and then apply the usual shadow combining logic.
4941 void handlePclmulIntrinsic(IntrinsicInst &I) {
4942 IRBuilder<> IRB(&I);
4943 unsigned Width =
4944 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements();
4945 assert(isa<ConstantInt>(I.getArgOperand(2)) &&
4946 "pclmul 3rd operand must be a constant");
4947 unsigned Imm = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getZExtValue();
4948 Value *Shuf0 = IRB.CreateShuffleVector(V: getShadow(I: &I, i: 0),
4949 Mask: getPclmulMask(Width, OddElements: Imm & 0x01));
4950 Value *Shuf1 = IRB.CreateShuffleVector(V: getShadow(I: &I, i: 1),
4951 Mask: getPclmulMask(Width, OddElements: Imm & 0x10));
4952 ShadowAndOriginCombiner SOC(this, IRB);
4953 SOC.Add(OpShadow: Shuf0, OpOrigin: getOrigin(I: &I, i: 0));
4954 SOC.Add(OpShadow: Shuf1, OpOrigin: getOrigin(I: &I, i: 1));
4955 SOC.Done(I: &I);
4956 }
4957
4958 // Instrument _mm_*_sd|ss intrinsics
4959 void handleUnarySdSsIntrinsic(IntrinsicInst &I) {
4960 IRBuilder<> IRB(&I);
4961 unsigned Width =
4962 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements();
4963 Value *First = getShadow(I: &I, i: 0);
4964 Value *Second = getShadow(I: &I, i: 1);
4965 // First element of second operand, remaining elements of first operand
4966 SmallVector<int, 16> Mask;
4967 Mask.push_back(Elt: Width);
4968 for (unsigned i = 1; i < Width; i++)
4969 Mask.push_back(Elt: i);
4970 Value *Shadow = IRB.CreateShuffleVector(V1: First, V2: Second, Mask);
4971
4972 setShadow(V: &I, SV: Shadow);
4973 setOriginForNaryOp(I);
4974 }
4975
4976 void handleVtestIntrinsic(IntrinsicInst &I) {
4977 IRBuilder<> IRB(&I);
4978 Value *Shadow0 = getShadow(I: &I, i: 0);
4979 Value *Shadow1 = getShadow(I: &I, i: 1);
4980 Value *Or = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
4981 Value *NZ = IRB.CreateICmpNE(LHS: Or, RHS: Constant::getNullValue(Ty: Or->getType()));
4982 Value *Scalar = convertShadowToScalar(V: NZ, IRB);
4983 Value *Shadow = IRB.CreateZExt(V: Scalar, DestTy: getShadowTy(V: &I));
4984
4985 setShadow(V: &I, SV: Shadow);
4986 setOriginForNaryOp(I);
4987 }
4988
4989 void handleBinarySdSsIntrinsic(IntrinsicInst &I) {
4990 IRBuilder<> IRB(&I);
4991 unsigned Width =
4992 cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements();
4993 Value *First = getShadow(I: &I, i: 0);
4994 Value *Second = getShadow(I: &I, i: 1);
4995 Value *OrShadow = IRB.CreateOr(LHS: First, RHS: Second);
4996 // First element of both OR'd together, remaining elements of first operand
4997 SmallVector<int, 16> Mask;
4998 Mask.push_back(Elt: Width);
4999 for (unsigned i = 1; i < Width; i++)
5000 Mask.push_back(Elt: i);
5001 Value *Shadow = IRB.CreateShuffleVector(V1: First, V2: OrShadow, Mask);
5002
5003 setShadow(V: &I, SV: Shadow);
5004 setOriginForNaryOp(I);
5005 }
5006
5007 // _mm_round_ps / _mm_round_ps.
5008 // Similar to maybeHandleSimpleNomemIntrinsic except
5009 // the second argument is guaranteed to be a constant integer.
5010 void handleRoundPdPsIntrinsic(IntrinsicInst &I) {
5011 assert(I.getArgOperand(0)->getType() == I.getType());
5012 assert(I.arg_size() == 2);
5013 assert(isa<ConstantInt>(I.getArgOperand(1)));
5014
5015 IRBuilder<> IRB(&I);
5016 ShadowAndOriginCombiner SC(this, IRB);
5017 SC.Add(V: I.getArgOperand(i: 0));
5018 SC.Done(I: &I);
5019 }
5020
5021 // Instrument @llvm.abs intrinsic.
5022 //
5023 // e.g., i32 @llvm.abs.i32 (i32 <Src>, i1 <is_int_min_poison>)
5024 // <4 x i32> @llvm.abs.v4i32(<4 x i32> <Src>, i1 <is_int_min_poison>)
5025 void handleAbsIntrinsic(IntrinsicInst &I) {
5026 assert(I.arg_size() == 2);
5027 Value *Src = I.getArgOperand(i: 0);
5028 Value *IsIntMinPoison = I.getArgOperand(i: 1);
5029
5030 assert(I.getType()->isIntOrIntVectorTy());
5031
5032 assert(Src->getType() == I.getType());
5033
5034 assert(IsIntMinPoison->getType()->isIntegerTy());
5035 assert(IsIntMinPoison->getType()->getIntegerBitWidth() == 1);
5036
5037 IRBuilder<> IRB(&I);
5038 Value *SrcShadow = getShadow(V: Src);
5039
5040 APInt MinVal =
5041 APInt::getSignedMinValue(numBits: Src->getType()->getScalarSizeInBits());
5042 Value *MinValVec = ConstantInt::get(Ty: Src->getType(), V: MinVal);
5043 Value *SrcIsMin = IRB.CreateICmp(P: CmpInst::ICMP_EQ, LHS: Src, RHS: MinValVec);
5044
5045 Value *PoisonedShadow = getPoisonedShadow(V: Src);
5046 Value *PoisonedIfIntMinShadow =
5047 IRB.CreateSelect(C: SrcIsMin, True: PoisonedShadow, False: SrcShadow);
5048 Value *Shadow =
5049 IRB.CreateSelect(C: IsIntMinPoison, True: PoisonedIfIntMinShadow, False: SrcShadow);
5050
5051 setShadow(V: &I, SV: Shadow);
5052 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
5053 }
5054
5055 void handleIsFpClass(IntrinsicInst &I) {
5056 IRBuilder<> IRB(&I);
5057 Value *Shadow = getShadow(I: &I, i: 0);
5058 setShadow(V: &I, SV: IRB.CreateICmpNE(LHS: Shadow, RHS: getCleanShadow(V: Shadow)));
5059 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
5060 }
5061
5062 void handleArithmeticWithOverflow(IntrinsicInst &I) {
5063 IRBuilder<> IRB(&I);
5064 Value *Shadow0 = getShadow(I: &I, i: 0);
5065 Value *Shadow1 = getShadow(I: &I, i: 1);
5066 Value *ShadowElt0 = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
5067 Value *ShadowElt1 =
5068 IRB.CreateICmpNE(LHS: ShadowElt0, RHS: getCleanShadow(V: ShadowElt0));
5069
5070 Value *Shadow = PoisonValue::get(T: getShadowTy(V: &I));
5071 Shadow = IRB.CreateInsertValue(Agg: Shadow, Val: ShadowElt0, Idxs: 0);
5072 Shadow = IRB.CreateInsertValue(Agg: Shadow, Val: ShadowElt1, Idxs: 1);
5073
5074 setShadow(V: &I, SV: Shadow);
5075 setOriginForNaryOp(I);
5076 }
5077
5078 void handleModfOrSincos(IntrinsicInst &I) {
5079 IRBuilder<> IRB(&I);
5080 Value *ArgShadow = getShadow(I: &I, i: 0);
5081 Value *Shadow = PoisonValue::get(T: getShadowTy(V: &I));
5082 Shadow = IRB.CreateInsertValue(Agg: Shadow, Val: ArgShadow, Idxs: 0);
5083 Shadow = IRB.CreateInsertValue(Agg: Shadow, Val: ArgShadow, Idxs: 1);
5084 setShadow(V: &I, SV: Shadow);
5085 setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
5086 }
5087
5088 Value *extractLowerShadow(IRBuilder<> &IRB, Value *V) {
5089 assert(isa<FixedVectorType>(V->getType()));
5090 assert(cast<FixedVectorType>(V->getType())->getNumElements() > 0);
5091 Value *Shadow = getShadow(V);
5092 return IRB.CreateExtractElement(Vec: Shadow,
5093 Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: 0));
5094 }
5095
5096 // Handle llvm.x86.avx512.mask.pmov{,s,us}.*.{128,256,512}
5097 //
5098 // e.g., call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512
5099 // (<8 x i64>, <16 x i8>, i8)
5100 // A WriteThru Mask
5101 //
5102 // call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512
5103 // (<16 x i32>, <16 x i8>, i16)
5104 //
5105 // Dst[i] = Mask[i] ? truncate_or_saturate(A[i]) : WriteThru[i]
5106 // Dst_shadow[i] = Mask[i] ? truncate(A_shadow[i]) : WriteThru_shadow[i]
5107 //
5108 // If Dst has more elements than A, the excess elements are zeroed (and the
5109 // corresponding shadow is initialized).
5110 //
5111 // Note: for PMOV (truncation), handleIntrinsicByApplyingToShadow is precise
5112 // and is much faster than this handler.
5113 void handleAVX512VectorDownConvert(IntrinsicInst &I) {
5114 IRBuilder<> IRB(&I);
5115
5116 assert(I.arg_size() == 3);
5117 Value *A = I.getOperand(i_nocapture: 0);
5118 Value *WriteThrough = I.getOperand(i_nocapture: 1);
5119 Value *Mask = I.getOperand(i_nocapture: 2);
5120
5121 assert(isFixedIntVector(A));
5122 assert(isFixedIntVector(WriteThrough));
5123
5124 unsigned ANumElements =
5125 cast<FixedVectorType>(Val: A->getType())->getNumElements();
5126 unsigned OutputNumElements =
5127 cast<FixedVectorType>(Val: WriteThrough->getType())->getNumElements();
5128 assert(ANumElements == OutputNumElements ||
5129 ANumElements * 2 == OutputNumElements);
5130 // N.B. some PMOV{,S,US} instructions have a 4x or even 8x ratio in the
5131 // number of elements e.g.,
5132 // <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256
5133 // (<4 x i64>, <16 x i8>, i8)
5134 // <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128
5135 // (<2 x i64>, <16 x i8>, i8)
5136 // However, we currently handle those elsewhere.
5137
5138 assert(Mask->getType()->isIntegerTy());
5139 insertCheckShadowOf(Val: Mask, OrigIns: &I);
5140
5141 // The mask has 1 bit per element of A, but a minimum of 8 bits.
5142 if (Mask->getType()->getScalarSizeInBits() == 8 && OutputNumElements < 8)
5143 Mask = IRB.CreateTrunc(V: Mask, DestTy: Type::getIntNTy(C&: *MS.C, N: OutputNumElements));
5144 assert(Mask->getType()->getScalarSizeInBits() == ANumElements);
5145
5146 assert(I.getType() == WriteThrough->getType());
5147
5148 // Widen the mask, if necessary, to have one bit per element of the output
5149 // vector.
5150 // We want the extra bits to have '1's, so that the CreateSelect will
5151 // select the values from AShadow instead of WriteThroughShadow ("maskless"
5152 // versions of the intrinsics are sometimes implemented using an all-1's
5153 // mask and an undefined value for WriteThroughShadow). We accomplish this
5154 // by using bitwise NOT before and after the ZExt.
5155 if (ANumElements != OutputNumElements) {
5156 Mask = IRB.CreateNot(V: Mask);
5157 Mask = IRB.CreateZExt(V: Mask, DestTy: Type::getIntNTy(C&: *MS.C, N: OutputNumElements),
5158 Name: "_ms_widen_mask");
5159 Mask = IRB.CreateNot(V: Mask);
5160 }
5161 Mask = IRB.CreateBitCast(
5162 V: Mask, DestTy: FixedVectorType::get(ElementType: IRB.getInt1Ty(), NumElts: OutputNumElements));
5163
5164 Value *AShadow = getShadow(V: A);
5165
5166 // The return type might have more elements than the input.
5167 // Temporarily shrink the return type's number of elements.
5168 VectorType *ShadowType = maybeShrinkVectorShadowType(Src: A, I);
5169
5170 // PMOV truncates; PMOVS/PMOVUS uses signed/unsigned saturation.
5171 // This handler treats them all as truncation, which leads to some rare
5172 // false positives in the cases where the truncated bytes could
5173 // unambiguously saturate the value e.g., if A = ??????10 ????????
5174 // (big-endian), the unsigned saturated byte conversion is 11111111 i.e.,
5175 // fully defined, but the truncated byte is ????????.
5176 //
5177 // TODO: use GetMinMaxUnsigned() to handle saturation precisely.
5178 AShadow = IRB.CreateTrunc(V: AShadow, DestTy: ShadowType, Name: "_ms_trunc_shadow");
5179 AShadow = maybeExtendVectorShadowWithZeros(Shadow: AShadow, I);
5180
5181 Value *WriteThroughShadow = getShadow(V: WriteThrough);
5182
5183 Value *Shadow = IRB.CreateSelect(C: Mask, True: AShadow, False: WriteThroughShadow);
5184 setShadow(V: &I, SV: Shadow);
5185 setOriginForNaryOp(I);
5186 }
5187
5188 // Handle llvm.x86.avx512.* instructions that take vector(s) of floating-point
5189 // values and perform an operation whose shadow propagation should be handled
5190 // as all-or-nothing [*], with masking provided by a vector and a mask
5191 // supplied as an integer.
5192 //
5193 // [*] if all bits of a vector element are initialized, the output is fully
5194 // initialized; otherwise, the output is fully uninitialized
5195 //
5196 // e.g., <16 x float> @llvm.x86.avx512.rsqrt14.ps.512
5197 // (<16 x float>, <16 x float>, i16)
5198 // A WriteThru Mask
5199 //
5200 // <2 x double> @llvm.x86.avx512.rcp14.pd.128
5201 // (<2 x double>, <2 x double>, i8)
5202 // A WriteThru Mask
5203 //
5204 // <8 x double> @llvm.x86.avx512.mask.rndscale.pd.512
5205 // (<8 x double>, i32, <8 x double>, i8, i32)
5206 // A Imm WriteThru Mask Rounding
5207 //
5208 // <16 x float> @llvm.x86.avx512.mask.scalef.ps.512
5209 // (<16 x float>, <16 x float>, <16 x float>, i16, i32)
5210 // WriteThru A B Mask Rnd
5211 //
5212 // All operands other than A, B, ..., and WriteThru (e.g., Mask, Imm,
5213 // Rounding) must be fully initialized.
5214 //
5215 // Dst[i] = Mask[i] ? some_op(A[i], B[i], ...)
5216 // : WriteThru[i]
5217 // Dst_shadow[i] = Mask[i] ? all_or_nothing(A_shadow[i] | B_shadow[i] | ...)
5218 // : WriteThru_shadow[i]
5219 void handleAVX512VectorGenericMaskedFP(IntrinsicInst &I,
5220 SmallVector<unsigned, 4> DataIndices,
5221 unsigned WriteThruIndex,
5222 unsigned MaskIndex) {
5223 IRBuilder<> IRB(&I);
5224
5225 unsigned NumArgs = I.arg_size();
5226
5227 assert(WriteThruIndex < NumArgs);
5228 assert(MaskIndex < NumArgs);
5229 assert(WriteThruIndex != MaskIndex);
5230 Value *WriteThru = I.getOperand(i_nocapture: WriteThruIndex);
5231
5232 unsigned OutputNumElements =
5233 cast<FixedVectorType>(Val: WriteThru->getType())->getNumElements();
5234
5235 assert(DataIndices.size() > 0);
5236
5237 bool isData[16] = {false};
5238 assert(NumArgs <= 16);
5239 for (unsigned i : DataIndices) {
5240 assert(i < NumArgs);
5241 assert(i != WriteThruIndex);
5242 assert(i != MaskIndex);
5243
5244 isData[i] = true;
5245
5246 Value *A = I.getOperand(i_nocapture: i);
5247 assert(isFixedFPVector(A));
5248 [[maybe_unused]] unsigned ANumElements =
5249 cast<FixedVectorType>(Val: A->getType())->getNumElements();
5250 assert(ANumElements == OutputNumElements);
5251 }
5252
5253 Value *Mask = I.getOperand(i_nocapture: MaskIndex);
5254
5255 assert(isFixedFPVector(WriteThru));
5256
5257 for (unsigned i = 0; i < NumArgs; ++i) {
5258 if (!isData[i] && i != WriteThruIndex) {
5259 // Imm, Mask, Rounding etc. are "control" data, hence we require that
5260 // they be fully initialized.
5261 assert(I.getOperand(i)->getType()->isIntegerTy());
5262 insertCheckShadowOf(Val: I.getOperand(i_nocapture: i), OrigIns: &I);
5263 }
5264 }
5265
5266 // The mask has 1 bit per element of A, but a minimum of 8 bits.
5267 if (Mask->getType()->getScalarSizeInBits() == 8 && OutputNumElements < 8)
5268 Mask = IRB.CreateTrunc(V: Mask, DestTy: Type::getIntNTy(C&: *MS.C, N: OutputNumElements));
5269 assert(Mask->getType()->getScalarSizeInBits() == OutputNumElements);
5270
5271 assert(I.getType() == WriteThru->getType());
5272
5273 Mask = IRB.CreateBitCast(
5274 V: Mask, DestTy: FixedVectorType::get(ElementType: IRB.getInt1Ty(), NumElts: OutputNumElements));
5275
5276 Value *DataShadow = nullptr;
5277 for (unsigned i : DataIndices) {
5278 Value *A = I.getOperand(i_nocapture: i);
5279 if (DataShadow)
5280 DataShadow = IRB.CreateOr(LHS: DataShadow, RHS: getShadow(V: A));
5281 else
5282 DataShadow = getShadow(V: A);
5283 }
5284
5285 // All-or-nothing shadow
5286 DataShadow =
5287 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: DataShadow, RHS: getCleanShadow(V: DataShadow)),
5288 DestTy: DataShadow->getType());
5289
5290 Value *WriteThruShadow = getShadow(V: WriteThru);
5291
5292 Value *Shadow = IRB.CreateSelect(C: Mask, True: DataShadow, False: WriteThruShadow);
5293 setShadow(V: &I, SV: Shadow);
5294
5295 setOriginForNaryOp(I);
5296 }
5297
5298 // AVX512 Floating-Point Classification
5299 //
5300 // e.g.,
5301 // - < 8 x i1> @llvm.x86.avx512.fpclass.pd.512
5302 // (<8 x double> %input, i32 %classifiers)
5303 // - <16 x i1> @llvm.x86.avx512.fpclass.ps.512
5304 // (<16 x float> %input, i32 %classifiers)
5305 void handleAVX512FPClass(IntrinsicInst &I) {
5306 IRBuilder<> IRB(&I);
5307
5308 assert(I.arg_size() == 2);
5309
5310 Value *Input = I.getOperand(i_nocapture: 0);
5311 assert(isFixedFPVector(Input));
5312 [[maybe_unused]] FixedVectorType *InputType = cast<FixedVectorType>(Val: Input->getType());
5313
5314 Value *Classifiers = I.getOperand(i_nocapture: 1);
5315 assert(isa<ConstantInt>(Classifiers));
5316 // No shadow check needed for constants
5317
5318 assert(isFixedIntVectorTy(I.getType()));
5319 FixedVectorType *OutputType = cast<FixedVectorType>(Val: I.getType());
5320 assert(OutputType->getScalarSizeInBits() == 1);
5321
5322 assert(OutputType->getNumElements() == InputType->getNumElements());
5323
5324 Value *OutputShadow;
5325 if (cast<ConstantInt>(Val: Classifiers)->isZero())
5326 // Each bit specifies whether a particular classifier is enabled.
5327 // If Classifiers == 0, the output is trivially known to be zero, thus
5328 // the output is fully initialized.
5329 OutputShadow = getCleanShadow(OrigTy: OutputType);
5330 else
5331 // Approximate each bit of the output shadow based on whether the
5332 // corresponding input element is fully initialized. It is only
5333 // approximate because some classifications do not rely on all bits of
5334 // the input element.
5335 OutputShadow = IRB.CreateICmpNE(LHS: getShadow(V: Input), RHS: getCleanShadow(V: Input));
5336
5337 setShadow(V: &I, SV: OutputShadow);
5338
5339 setOriginForNaryOp(I);
5340 }
5341
5342 // For sh.* compiler intrinsics:
5343 // llvm.x86.avx512fp16.mask.{add/sub/mul/div/max/min}.sh.round
5344 // (<8 x half>, <8 x half>, <8 x half>, i8, i32)
5345 // A B WriteThru Mask RoundingMode
5346 //
5347 // DstShadow[0] = Mask[0] ? (AShadow[0] | BShadow[0]) : WriteThruShadow[0]
5348 // DstShadow[1..7] = AShadow[1..7]
5349 void visitGenericScalarHalfwordInst(IntrinsicInst &I) {
5350 IRBuilder<> IRB(&I);
5351
5352 assert(I.arg_size() == 5);
5353 Value *A = I.getOperand(i_nocapture: 0);
5354 Value *B = I.getOperand(i_nocapture: 1);
5355 Value *WriteThrough = I.getOperand(i_nocapture: 2);
5356 Value *Mask = I.getOperand(i_nocapture: 3);
5357 Value *RoundingMode = I.getOperand(i_nocapture: 4);
5358
5359 // Technically, we could probably just check whether the LSB is
5360 // initialized, but intuitively it feels like a partly uninitialized mask
5361 // is unintended, and we should warn the user immediately.
5362 insertCheckShadowOf(Val: Mask, OrigIns: &I);
5363 insertCheckShadowOf(Val: RoundingMode, OrigIns: &I);
5364
5365 assert(isa<FixedVectorType>(A->getType()));
5366 unsigned NumElements =
5367 cast<FixedVectorType>(Val: A->getType())->getNumElements();
5368 assert(NumElements == 8);
5369 assert(A->getType() == B->getType());
5370 assert(B->getType() == WriteThrough->getType());
5371 assert(Mask->getType()->getPrimitiveSizeInBits() == NumElements);
5372 assert(RoundingMode->getType()->isIntegerTy());
5373
5374 Value *ALowerShadow = extractLowerShadow(IRB, V: A);
5375 Value *BLowerShadow = extractLowerShadow(IRB, V: B);
5376
5377 Value *ABLowerShadow = IRB.CreateOr(LHS: ALowerShadow, RHS: BLowerShadow);
5378
5379 Value *WriteThroughLowerShadow = extractLowerShadow(IRB, V: WriteThrough);
5380
5381 Mask = IRB.CreateBitCast(
5382 V: Mask, DestTy: FixedVectorType::get(ElementType: IRB.getInt1Ty(), NumElts: NumElements));
5383 Value *MaskLower =
5384 IRB.CreateExtractElement(Vec: Mask, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: 0));
5385
5386 Value *AShadow = getShadow(V: A);
5387 Value *DstLowerShadow =
5388 IRB.CreateSelect(C: MaskLower, True: ABLowerShadow, False: WriteThroughLowerShadow);
5389 Value *DstShadow = IRB.CreateInsertElement(
5390 Vec: AShadow, NewElt: DstLowerShadow, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: 0),
5391 Name: "_msprop");
5392
5393 setShadow(V: &I, SV: DstShadow);
5394 setOriginForNaryOp(I);
5395 }
5396
5397 // Approximately handle AVX Galois Field Affine Transformation
5398 //
5399 // e.g.,
5400 // <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8>, <16 x i8>, i8)
5401 // <32 x i8> @llvm.x86.vgf2p8affineqb.256(<32 x i8>, <32 x i8>, i8)
5402 // <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8>, <64 x i8>, i8)
5403 // Out A x b
5404 // where A and x are packed matrices, b is a vector,
5405 // Out = A * x + b in GF(2)
5406 //
5407 // Multiplication in GF(2) is equivalent to bitwise AND. However, the matrix
5408 // computation also includes a parity calculation.
5409 //
5410 // For the bitwise AND of bits V1 and V2, the exact shadow is:
5411 // Out_Shadow = (V1_Shadow & V2_Shadow)
5412 // | (V1 & V2_Shadow)
5413 // | (V1_Shadow & V2 )
5414 //
5415 // We approximate the shadow of gf2p8affineqb using:
5416 // Out_Shadow = gf2p8affineqb(x_Shadow, A_shadow, 0)
5417 // | gf2p8affineqb(x, A_shadow, 0)
5418 // | gf2p8affineqb(x_Shadow, A, 0)
5419 // | set1_epi8(b_Shadow)
5420 //
5421 // This approximation has false negatives: if an intermediate dot-product
5422 // contains an even number of 1's, the parity is 0.
5423 // It has no false positives.
5424 void handleAVXGF2P8Affine(IntrinsicInst &I) {
5425 IRBuilder<> IRB(&I);
5426
5427 assert(I.arg_size() == 3);
5428 Value *A = I.getOperand(i_nocapture: 0);
5429 Value *X = I.getOperand(i_nocapture: 1);
5430 Value *B = I.getOperand(i_nocapture: 2);
5431
5432 assert(isFixedIntVector(A));
5433 assert(cast<VectorType>(A->getType())
5434 ->getElementType()
5435 ->getScalarSizeInBits() == 8);
5436
5437 assert(A->getType() == X->getType());
5438
5439 assert(B->getType()->isIntegerTy());
5440 assert(B->getType()->getScalarSizeInBits() == 8);
5441
5442 assert(I.getType() == A->getType());
5443
5444 Value *AShadow = getShadow(V: A);
5445 Value *XShadow = getShadow(V: X);
5446 Value *BZeroShadow = getCleanShadow(V: B);
5447
5448 Value *AShadowXShadow = IRB.CreateIntrinsic(
5449 RetTy: I.getType(), ID: I.getIntrinsicID(), Args: {XShadow, AShadow, BZeroShadow});
5450 Value *AShadowX = IRB.CreateIntrinsic(RetTy: I.getType(), ID: I.getIntrinsicID(),
5451 Args: {X, AShadow, BZeroShadow});
5452 Value *XShadowA = IRB.CreateIntrinsic(RetTy: I.getType(), ID: I.getIntrinsicID(),
5453 Args: {XShadow, A, BZeroShadow});
5454
5455 unsigned NumElements = cast<FixedVectorType>(Val: I.getType())->getNumElements();
5456 Value *BShadow = getShadow(V: B);
5457 Value *BBroadcastShadow = getCleanShadow(V: AShadow);
5458 // There is no LLVM IR intrinsic for _mm512_set1_epi8.
5459 // This loop generates a lot of LLVM IR, which we expect that CodeGen will
5460 // lower appropriately (e.g., VPBROADCASTB).
5461 // Besides, b is often a constant, in which case it is fully initialized.
5462 for (unsigned i = 0; i < NumElements; i++)
5463 BBroadcastShadow = IRB.CreateInsertElement(Vec: BBroadcastShadow, NewElt: BShadow, Idx: i);
5464
5465 setShadow(V: &I, SV: IRB.CreateOr(
5466 Ops: {AShadowXShadow, AShadowX, XShadowA, BBroadcastShadow}));
5467 setOriginForNaryOp(I);
5468 }
5469
5470 // Handle Arm NEON vector load intrinsics (vld*).
5471 //
5472 // The WithLane instructions (ld[234]lane) are similar to:
5473 // call {<4 x i32>, <4 x i32>, <4 x i32>}
5474 // @llvm.aarch64.neon.ld3lane.v4i32.p0
5475 // (<4 x i32> %L1, <4 x i32> %L2, <4 x i32> %L3, i64 %lane, ptr
5476 // %A)
5477 //
5478 // The non-WithLane instructions (ld[234], ld1x[234], ld[234]r) are similar
5479 // to:
5480 // call {<8 x i8>, <8 x i8>} @llvm.aarch64.neon.ld2.v8i8.p0(ptr %A)
5481 void handleNEONVectorLoad(IntrinsicInst &I, bool WithLane) {
5482 unsigned int numArgs = I.arg_size();
5483
5484 // Return type is a struct of vectors of integers or floating-point
5485 assert(I.getType()->isStructTy());
5486 [[maybe_unused]] StructType *RetTy = cast<StructType>(Val: I.getType());
5487 assert(RetTy->getNumElements() > 0);
5488 assert(RetTy->getElementType(0)->isIntOrIntVectorTy() ||
5489 RetTy->getElementType(0)->isFPOrFPVectorTy());
5490 for (unsigned int i = 0; i < RetTy->getNumElements(); i++)
5491 assert(RetTy->getElementType(i) == RetTy->getElementType(0));
5492
5493 if (WithLane) {
5494 // 2, 3 or 4 vectors, plus lane number, plus input pointer
5495 assert(4 <= numArgs && numArgs <= 6);
5496
5497 // Return type is a struct of the input vectors
5498 assert(RetTy->getNumElements() + 2 == numArgs);
5499 for (unsigned int i = 0; i < RetTy->getNumElements(); i++)
5500 assert(I.getArgOperand(i)->getType() == RetTy->getElementType(0));
5501 } else {
5502 assert(numArgs == 1);
5503 }
5504
5505 IRBuilder<> IRB(&I);
5506
5507 SmallVector<Value *, 6> ShadowArgs;
5508 if (WithLane) {
5509 for (unsigned int i = 0; i < numArgs - 2; i++)
5510 ShadowArgs.push_back(Elt: getShadow(V: I.getArgOperand(i)));
5511
5512 // Lane number, passed verbatim
5513 Value *LaneNumber = I.getArgOperand(i: numArgs - 2);
5514 ShadowArgs.push_back(Elt: LaneNumber);
5515
5516 // TODO: blend shadow of lane number into output shadow?
5517 insertCheckShadowOf(Val: LaneNumber, OrigIns: &I);
5518 }
5519
5520 Value *Src = I.getArgOperand(i: numArgs - 1);
5521 assert(Src->getType()->isPointerTy() && "Source is not a pointer!");
5522
5523 Type *SrcShadowTy = getShadowTy(V: Src);
5524 auto [SrcShadowPtr, SrcOriginPtr] =
5525 getShadowOriginPtr(Addr: Src, IRB, ShadowTy: SrcShadowTy, Alignment: Align(1), /*isStore*/ false);
5526 ShadowArgs.push_back(Elt: SrcShadowPtr);
5527
5528 // The NEON vector load instructions handled by this function all have
5529 // integer variants. It is easier to use those rather than trying to cast
5530 // a struct of vectors of floats into a struct of vectors of integers.
5531 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
5532 RetTy: getShadowTy(V: &I), ID: I.getIntrinsicID(), Args: ShadowArgs);
5533 setShadow(V: &I, SV: CI);
5534
5535 if (!MS.TrackOrigins)
5536 return;
5537
5538 Value *PtrSrcOrigin = IRB.CreateLoad(Ty: MS.OriginTy, Ptr: SrcOriginPtr);
5539 setOrigin(V: &I, Origin: PtrSrcOrigin);
5540 }
5541
5542 /// Handle Arm NEON vector store intrinsics (vst{2,3,4}, vst1x_{2,3,4},
5543 /// and vst{2,3,4}lane).
5544 ///
5545 /// Arm NEON vector store intrinsics have the output address (pointer) as the
5546 /// last argument, with the initial arguments being the inputs (and lane
5547 /// number for vst{2,3,4}lane). They return void.
5548 ///
5549 /// - st4 interleaves the output e.g., st4 (inA, inB, inC, inD, outP) writes
5550 /// abcdabcdabcdabcd... into *outP
5551 /// - st1_x4 is non-interleaved e.g., st1_x4 (inA, inB, inC, inD, outP)
5552 /// writes aaaa...bbbb...cccc...dddd... into *outP
5553 /// - st4lane has arguments of (inA, inB, inC, inD, lane, outP)
5554 /// These instructions can all be instrumented with essentially the same
5555 /// MSan logic, simply by applying the corresponding intrinsic to the shadow.
5556 void handleNEONVectorStoreIntrinsic(IntrinsicInst &I, bool useLane) {
5557 IRBuilder<> IRB(&I);
5558
5559 // Don't use getNumOperands() because it includes the callee
5560 int numArgOperands = I.arg_size();
5561
5562 // The last arg operand is the output (pointer)
5563 assert(numArgOperands >= 1);
5564 Value *Addr = I.getArgOperand(i: numArgOperands - 1);
5565 assert(Addr->getType()->isPointerTy());
5566 int skipTrailingOperands = 1;
5567
5568 if (Opts.msan_check_access_address)
5569 insertCheckShadowOf(Val: Addr, OrigIns: &I);
5570
5571 // Second-last operand is the lane number (for vst{2,3,4}lane)
5572 if (useLane) {
5573 skipTrailingOperands++;
5574 assert(numArgOperands >= static_cast<int>(skipTrailingOperands));
5575 assert(isa<IntegerType>(
5576 I.getArgOperand(numArgOperands - skipTrailingOperands)->getType()));
5577 }
5578
5579 SmallVector<Value *, 8> ShadowArgs;
5580 // All the initial operands are the inputs
5581 for (int i = 0; i < numArgOperands - skipTrailingOperands; i++) {
5582 assert(isa<FixedVectorType>(I.getArgOperand(i)->getType()));
5583 Value *Shadow = getShadow(I: &I, i);
5584 ShadowArgs.append(NumInputs: 1, Elt: Shadow);
5585 }
5586
5587 // MSan's GetShadowTy assumes the LHS is the type we want the shadow for
5588 // e.g., for:
5589 // [[TMP5:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
5590 // we know the type of the output (and its shadow) is <16 x i8>.
5591 //
5592 // Arm NEON VST is unusual because the last argument is the output address:
5593 // define void @st2_16b(<16 x i8> %A, <16 x i8> %B, ptr %P) {
5594 // call void @llvm.aarch64.neon.st2.v16i8.p0
5595 // (<16 x i8> [[A]], <16 x i8> [[B]], ptr [[P]])
5596 // and we have no type information about P's operand. We must manually
5597 // compute the type (<16 x i8> x 2).
5598 FixedVectorType *OutputVectorTy = FixedVectorType::get(
5599 ElementType: cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getElementType(),
5600 NumElts: cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements() *
5601 (numArgOperands - skipTrailingOperands));
5602 Type *OutputShadowTy = getShadowTy(OrigTy: OutputVectorTy);
5603
5604 if (useLane)
5605 ShadowArgs.append(NumInputs: 1,
5606 Elt: I.getArgOperand(i: numArgOperands - skipTrailingOperands));
5607
5608 Value *OutputShadowPtr, *OutputOriginPtr;
5609 // AArch64 NEON does not need alignment (unless OS requires it)
5610 std::tie(args&: OutputShadowPtr, args&: OutputOriginPtr) = getShadowOriginPtr(
5611 Addr, IRB, ShadowTy: OutputShadowTy, Alignment: Align(1), /*isStore*/ true);
5612 ShadowArgs.append(NumInputs: 1, Elt: OutputShadowPtr);
5613
5614 CallInst *CI = IRB.CreateIntrinsicWithoutFolding(
5615 RetTy: IRB.getVoidTy(), ID: I.getIntrinsicID(), Args: ShadowArgs);
5616 setShadow(V: &I, SV: CI);
5617
5618 if (MS.TrackOrigins) {
5619 // TODO: if we modelled the vst* instruction more precisely, we could
5620 // more accurately track the origins (e.g., if both inputs are
5621 // uninitialized for vst2, we currently blame the second input, even
5622 // though part of the output depends only on the first input).
5623 //
5624 // This is particularly imprecise for vst{2,3,4}lane, since only one
5625 // lane of each input is actually copied to the output.
5626 OriginCombiner OC(this, IRB);
5627 for (int i = 0; i < numArgOperands - skipTrailingOperands; i++)
5628 OC.Add(V: I.getArgOperand(i));
5629
5630 const DataLayout &DL = F.getDataLayout();
5631 OC.DoneAndStoreOrigin(TS: DL.getTypeStoreSize(Ty: OutputVectorTy),
5632 OriginPtr: OutputOriginPtr);
5633 }
5634 }
5635
5636 // Integer matrix multiplication:
5637 // - <4 x i32> @llvm.aarch64.neon.{s,u,us}mmla.v4i32.v16i8
5638 // (<4 x i32> %R, <16 x i8> %A, <16 x i8> %B)
5639 // - <4 x i32> is a 2x2 matrix
5640 // - <16 x i8> %A and %B are 2x8 and 8x2 matrices respectively
5641 //
5642 // Floating-point matrix multiplication:
5643 // - <4 x float> @llvm.aarch64.neon.bfmmla
5644 // (<4 x float> %R, <8 x bfloat> %A, <8 x bfloat> %B)
5645 // - <4 x float> is a 2x2 matrix
5646 // - <8 x bfloat> %A and %B are 2x4 and 4x2 matrices respectively
5647 //
5648 // The general shadow propagation approach is:
5649 // 1) get the shadows of the input matrices %A and %B
5650 // 2) map each shadow value to 0x1 if the corresponding value is fully
5651 // initialized, and 0x0 otherwise
5652 // 3) perform a matrix multiplication on the shadows of %A and %B [*].
5653 // The output will be a 2x2 matrix. For each element, a value of 0x8
5654 // (for {s,u,us}mmla) or 0x4 (for bfmmla) means all the corresponding
5655 // inputs were clean; if so, set the shadow to zero, otherwise set to -1.
5656 // 4) blend in the shadow of %R
5657 //
5658 // [*] Since shadows are integral, the obvious approach is to always apply
5659 // ummla to the shadows. Unfortunately, Armv8.2+bf16 supports bfmmla,
5660 // but not ummla. Thus, for bfmmla, our instrumentation reuses bfmmla.
5661 //
5662 // TODO: consider allowing multiplication of zero with an uninitialized value
5663 // to result in an initialized value.
5664 void handleNEONMatrixMultiply(IntrinsicInst &I) {
5665 IRBuilder<> IRB(&I);
5666
5667 assert(I.arg_size() == 3);
5668 Value *R = I.getArgOperand(i: 0);
5669 Value *A = I.getArgOperand(i: 1);
5670 Value *B = I.getArgOperand(i: 2);
5671
5672 assert(I.getType() == R->getType());
5673
5674 assert(isa<FixedVectorType>(R->getType()));
5675 assert(isa<FixedVectorType>(A->getType()));
5676 assert(isa<FixedVectorType>(B->getType()));
5677
5678 FixedVectorType *RTy = cast<FixedVectorType>(Val: R->getType());
5679 FixedVectorType *ATy = cast<FixedVectorType>(Val: A->getType());
5680 FixedVectorType *BTy = cast<FixedVectorType>(Val: B->getType());
5681 assert(ATy->getElementType() == BTy->getElementType());
5682
5683 if (RTy->getElementType()->isIntegerTy()) {
5684 // <4 x i32> @llvm.aarch64.neon.ummla.v4i32.v16i8
5685 // (<4 x i32> %R, <16 x i8> %X, <16 x i8> %Y)
5686 assert(RTy == FixedVectorType::get(IntegerType::get(*MS.C, 32), 4));
5687 assert(ATy == FixedVectorType::get(IntegerType::get(*MS.C, 8), 16));
5688 assert(BTy == FixedVectorType::get(IntegerType::get(*MS.C, 8), 16));
5689 } else {
5690 // <4 x float> @llvm.aarch64.neon.bfmmla
5691 // (<4 x float> %R, <8 x bfloat> %X, <8 x bfloat> %Y)
5692 assert(RTy == FixedVectorType::get(Type::getFloatTy(*MS.C), 4));
5693 assert(ATy == FixedVectorType::get(Type::getBFloatTy(*MS.C), 8));
5694 assert(BTy == FixedVectorType::get(Type::getBFloatTy(*MS.C), 8));
5695 }
5696
5697 Value *ShadowR = getShadow(I: &I, i: 0);
5698 Value *ShadowA = getShadow(I: &I, i: 1);
5699 Value *ShadowB = getShadow(I: &I, i: 2);
5700
5701 Value *ShadowAB;
5702 Value *FullyInit;
5703
5704 if (RTy->getElementType()->isIntegerTy()) {
5705 // If the value is fully initialized, the shadow will be 000...001.
5706 // Otherwise, the shadow will be all zero.
5707 // (This is the opposite of how we typically handle shadows.)
5708 ShadowA = IRB.CreateZExt(V: IRB.CreateICmpEQ(LHS: ShadowA, RHS: getCleanShadow(OrigTy: ATy)),
5709 DestTy: getShadowTy(OrigTy: ATy));
5710 ShadowB = IRB.CreateZExt(V: IRB.CreateICmpEQ(LHS: ShadowB, RHS: getCleanShadow(OrigTy: BTy)),
5711 DestTy: getShadowTy(OrigTy: BTy));
5712 // TODO: the CreateSelect approach used below for floating-point is more
5713 // generic than CreateZExt. Investigate whether it is worthwhile
5714 // unifying the two approaches.
5715
5716 ShadowAB = IRB.CreateIntrinsic(RetTy: RTy, ID: Intrinsic::aarch64_neon_ummla,
5717 Args: {getCleanShadow(OrigTy: RTy), ShadowA, ShadowB});
5718
5719 // ummla multiplies a 2x8 matrix with an 8x2 matrix. If all entries of the
5720 // input matrices are equal to 0x1, all entries of the output matrix will
5721 // be 0x8.
5722 FullyInit = ConstantVector::getSplat(
5723 EC: RTy->getElementCount(), Elt: ConstantInt::get(Ty: RTy->getElementType(), V: 0x8));
5724
5725 ShadowAB = IRB.CreateICmpNE(LHS: ShadowAB, RHS: FullyInit);
5726 } else {
5727 Constant *ABZeros = ConstantVector::getSplat(
5728 EC: ATy->getElementCount(), Elt: ConstantFP::get(Ty: ATy->getElementType(), V: 0));
5729 Constant *ABOnes = ConstantVector::getSplat(
5730 EC: ATy->getElementCount(), Elt: ConstantFP::get(Ty: ATy->getElementType(), V: 1));
5731
5732 // As per the integer case, if the shadow is clean, we store 0x1,
5733 // otherwise we store 0x0 (the opposite of usual shadow arithmetic).
5734 ShadowA = IRB.CreateSelect(C: IRB.CreateICmpEQ(LHS: ShadowA, RHS: getCleanShadow(OrigTy: ATy)),
5735 True: ABOnes, False: ABZeros);
5736 ShadowB = IRB.CreateSelect(C: IRB.CreateICmpEQ(LHS: ShadowB, RHS: getCleanShadow(OrigTy: BTy)),
5737 True: ABOnes, False: ABZeros);
5738
5739 Constant *RZeros = ConstantVector::getSplat(
5740 EC: RTy->getElementCount(), Elt: ConstantFP::get(Ty: RTy->getElementType(), V: 0));
5741
5742 ShadowAB = IRB.CreateIntrinsic(RetTy: RTy, ID: Intrinsic::aarch64_neon_bfmmla,
5743 Args: {RZeros, ShadowA, ShadowB});
5744
5745 // bfmmla multiplies a 2x4 matrix with an 4x2 matrix. If all entries of
5746 // the input matrices are equal to 0x1, all entries of the output matrix
5747 // will be 4.0. (To avoid floating-point error, we check if each entry
5748 // < 3.5.)
5749 FullyInit = ConstantVector::getSplat(
5750 EC: RTy->getElementCount(), Elt: ConstantFP::get(Ty: RTy->getElementType(), V: 3.5));
5751
5752 // FCmpULT: "yields true if either operand is a QNAN or op1 is less than"
5753 // op2"
5754 ShadowAB = IRB.CreateFCmpULT(LHS: ShadowAB, RHS: FullyInit);
5755 }
5756
5757 ShadowR = IRB.CreateICmpNE(LHS: ShadowR, RHS: getCleanShadow(OrigTy: RTy));
5758 ShadowR = IRB.CreateOr(LHS: ShadowAB, RHS: ShadowR);
5759
5760 setShadow(V: &I, SV: IRB.CreateSExt(V: ShadowR, DestTy: getShadowTy(OrigTy: RTy)));
5761
5762 setOriginForNaryOp(I);
5763 }
5764
5765 /// Handle intrinsics by applying the intrinsic to the shadows.
5766 ///
5767 /// For example, this can be applied to the Arm NEON vector table intrinsics
5768 /// (tbl{1,2,3,4}).
5769 ///
5770 /// Typically, shadowIntrinsicID will be specified by the caller to be
5771 /// I.getIntrinsicID(), but the caller can choose to replace it with another
5772 /// intrinsic of the same type.
5773 ///
5774 /// verbatimArgIndices specifies which arguments are passed unmodified to the
5775 /// intrinsic, though such arguments will also taint the shadow. Non-negative
5776 /// indices are zero-indexed, while negative indices count from the last
5777 /// argument. For example, if an intrinsic has one trailing verbatim argument
5778 /// (specified with verbatimArgIndices = {-1}):
5779 /// out = intrinsic(var1, var2, opType)
5780 /// we compute:
5781 /// shadow[out] =
5782 /// intrinsic(shadow[var1], shadow[var2], opType) | shadow[opType]
5783 ///
5784 /// If an intrinsic is called with floating-point arguments, we will
5785 /// typically cast the shadows to floating-point, apply the intrinsic [*],
5786 /// then cast the result back to integer/shadow.
5787 ///
5788 /// In cases where we know the intrinsic is compatible with integer
5789 /// arguments, 'forceIntegerIntrinsic' will apply the integer variant, even
5790 /// if the arguments are floating-point, thus avoiding unnecessary casts
5791 /// e.g., if I is:
5792 /// <16 x float> @llvm.x86.avx512.mask.compress
5793 /// (<16 x float>, <16 x float>, <16 x i1> %mask)
5794 /// we would prefer to compute the shadows using:
5795 /// <16 x i32> @llvm.x86.avx512.mask.compress
5796 /// (<16 x i32>, <16 x i32>, <16 x i1> %mask)
5797 ///
5798 /// [*] CAUTION: this assumes that the intrinsic will handle arbitrary
5799 /// bit-patterns (for example, if the intrinsic accepts floats
5800 /// for var1, we require that it doesn't care if inputs are
5801 /// NaNs).
5802 ///
5803 /// The origin is approximated using setOriginForNaryOp.
5804 void handleIntrinsicByApplyingToShadow(IntrinsicInst &I,
5805 Intrinsic::ID shadowIntrinsicID,
5806 ArrayRef<int> verbatimArgIndices,
5807 bool forceIntegerIntrinsic) {
5808 IRBuilder<> IRB(&I);
5809
5810 assert(verbatimArgIndices.size() <= I.arg_size());
5811
5812 // verbatimArgIndices may have negative indices, and the indices may be in
5813 // any order (e.g., [5, -1, 2]). Convert it to a lookup table.
5814 SmallVector<bool, 4> verbatimArgLookup(I.arg_size(), false);
5815
5816 for (int index : verbatimArgIndices) {
5817 if (index < 0)
5818 index += I.arg_size();
5819 assert(index >= 0 && static_cast<unsigned int>(index) < I.arg_size());
5820 verbatimArgLookup[index] = true;
5821 }
5822
5823 SmallVector<Value *, 8> ShadowArgs;
5824 // Don't use getNumOperands() because it includes the callee
5825 for (unsigned int i = 0; i < I.arg_size(); i++) {
5826 if (verbatimArgLookup[i]) {
5827 Value *Arg = I.getArgOperand(i);
5828
5829 if (forceIntegerIntrinsic)
5830 assert(Arg->getType()->isIntOrIntVectorTy());
5831
5832 ShadowArgs.push_back(Elt: Arg);
5833 } else {
5834 Value *Shadow = getShadow(I: &I, i);
5835
5836 if (forceIntegerIntrinsic)
5837 ShadowArgs.push_back(Elt: Shadow);
5838 else
5839 ShadowArgs.push_back(
5840 Elt: IRB.CreateBitCast(V: Shadow, DestTy: I.getArgOperand(i)->getType()));
5841 }
5842 }
5843
5844 Value *CombinedShadow;
5845 if (forceIntegerIntrinsic) {
5846 CombinedShadow =
5847 IRB.CreateIntrinsic(RetTy: getShadowTy(V: &I), ID: shadowIntrinsicID, Args: ShadowArgs);
5848 } else {
5849 Value *CI =
5850 IRB.CreateIntrinsic(RetTy: I.getType(), ID: shadowIntrinsicID, Args: ShadowArgs);
5851 CombinedShadow = IRB.CreateBitCast(V: CI, DestTy: getShadowTy(V: &I));
5852 }
5853
5854 // Combine the computed shadow with the shadow of verbatim args
5855 //
5856 // N.B. we could merge this with the earlier loop, but we keep it separate
5857 // for the IR to be byte-identical with the existing tests.
5858 for (unsigned int i = 0; i < I.arg_size(); i++) {
5859 if (!verbatimArgLookup[i])
5860 continue;
5861
5862 Value *Shadow =
5863 CreateShadowCast(IRB, V: getShadow(I: &I, i), dstTy: CombinedShadow->getType());
5864 CombinedShadow = IRB.CreateOr(LHS: Shadow, RHS: CombinedShadow, Name: "_msprop");
5865 }
5866
5867 setShadow(V: &I, SV: CombinedShadow);
5868
5869 setOriginForNaryOp(I);
5870 }
5871
5872 // Approximation only
5873 //
5874 // e.g., <16 x i8> @llvm.aarch64.neon.pmull64(i64, i64)
5875 void handleNEONVectorMultiplyIntrinsic(IntrinsicInst &I) {
5876 assert(I.arg_size() == 2);
5877
5878 handleShadowOr(I);
5879 }
5880
5881 // Handles:
5882 // <4 x half> @llvm.aarch64.neon.fp8.fdot2.lane
5883 // (<4 x half>, <8 x i8>, <16 x i8>, i32)
5884 // accumulator A B lane
5885 //
5886 // <8 x half> @llvm.aarch64.neon.fp8.fdot2.lane
5887 // (<8 x half>, <16 x i8>, <16 x i8>, i32)
5888 // <2 x float> @llvm.aarch64.neon.fp8.fdot4.lane
5889 // (<2 x float>, <8 x i8>, <16 x i8>, i32)
5890 // <4 x float> @llvm.aarch64.neon.fp8.fdot4.lane
5891 // (<4 x float>, <16 x i8>, <16 x i8>, i32)
5892 //
5893 // The lane specifies which pair (fdot2) or quad (fdot4) of numbers to
5894 // extract from B, which is then splatted before being used in the dot
5895 // products e.g., for
5896 // <4 x half> @llvm.aarch64.neon.fp8.fdot2.lane:
5897 // (<4 x half>, <8 x i8>, <16 x i8>, 1)
5898 //
5899 // acc[0] acc[1] acc[2] acc[3]
5900 // + + + + + + + +
5901 // A[0] A[1] A[2] A[3] A[4] A[5] A[6] A[7]
5902 // * * * * * * * *
5903 // B[2] B[3] B[2] B[3] B[2] B[3] B[2] B[3]
5904 //
5905 // Notice that if any bit of B[2] or B[3] is uninitialized, every accumulator
5906 // value will become tainted; we approximate this by marking the output as
5907 // fully uninitialized. This permits a 'Select' optimization.
5908 //
5909 // This function is separate from handleVectorDotProductIntrinsic(), because
5910 // the non-overlapping features (e.g., odd/even lanes vs. numbered lanes,
5911 // ZeroPurifies, EltSizeInBits) and optimizations make clean code reuse
5912 // difficult.
5913 void handleNEONDotProductLaneIntrinsic(IntrinsicInst &I,
5914 unsigned ReductionFactor) {
5915 IRBuilder<> IRB(&I);
5916 assert(I.arg_size() == 4);
5917
5918 [[maybe_unused]] Value *VAcc = I.getOperand(i_nocapture: 0);
5919 [[maybe_unused]] Value *Va = I.getOperand(i_nocapture: 1);
5920 [[maybe_unused]] Value *Vb = I.getOperand(i_nocapture: 2);
5921 Value *Lane = I.getOperand(i_nocapture: 3);
5922
5923 assert(isa<FixedVectorType>(VAcc->getType()));
5924 assert(VAcc->getType() == I.getType());
5925
5926 assert(isa<FixedVectorType>(Va->getType()));
5927 assert(Va->getType()->getPrimitiveSizeInBits() ==
5928 I.getType()->getPrimitiveSizeInBits());
5929
5930 assert(cast<FixedVectorType>(Va->getType())->getNumElements() ==
5931 cast<FixedVectorType>(I.getType())->getNumElements() *
5932 ReductionFactor);
5933
5934 assert(isa<FixedVectorType>(Vb->getType()));
5935 // Deliberately not strict equality
5936 assert(Vb->getType()->getPrimitiveSizeInBits() >=
5937 I.getType()->getPrimitiveSizeInBits());
5938
5939 assert(Lane->getType()->isIntegerTy());
5940
5941 // (<4 x 16>, <8 x i8>, <16 x i8>)
5942 // SAcc Sa Sb
5943 Value *SAcc = getShadow(I: &I, i: 0);
5944 Value *Sa = getShadow(I: &I, i: 1);
5945 Value *Sb = getShadow(I: &I, i: 2);
5946
5947 // Cast the shadows to:
5948 // (<4 x i16>, <4 x i16>, <8 x i16>)
5949 // SAcc Sa Sb
5950 Sa = IRB.CreateBitCast(V: Sa, DestTy: SAcc->getType());
5951 Sb = IRB.CreateBitCast(
5952 V: Sb, DestTy: FixedVectorType::getWithSizeAndScalar(
5953 SizeTy: cast<FixedVectorType>(Val: Sb->getType()),
5954 EltTy: cast<FixedVectorType>(Val: SAcc->getType())->getElementType()));
5955
5956 // All-or-nothing shadows
5957 Sa =
5958 IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: Sa, RHS: getCleanShadow(V: Sa)), DestTy: Sa->getType());
5959
5960 // Extract the specific lane from Sb to get i16, then turn it into a single
5961 // bit representing if it is fully initialized.
5962 Sb = IRB.CreateExtractElement(Vec: Sb, Idx: Lane);
5963 Value *SbClean = IRB.CreateIsNull(Arg: Sb);
5964
5965 Value *SOutput = IRB.CreateOr(LHS: SAcc, RHS: Sa);
5966
5967 // Select is cheaper than broadcasting Sb into <4 x i16>.
5968 SOutput = IRB.CreateSelect(C: SbClean, True: SOutput, False: getPoisonedShadow(V: SOutput));
5969
5970 setShadow(V: &I, SV: SOutput);
5971 setOriginForNaryOp(I);
5972 }
5973
5974 bool maybeHandleCrossPlatformIntrinsic(IntrinsicInst &I) {
5975 switch (I.getIntrinsicID()) {
5976 case Intrinsic::uadd_with_overflow:
5977 case Intrinsic::sadd_with_overflow:
5978 case Intrinsic::usub_with_overflow:
5979 case Intrinsic::ssub_with_overflow:
5980 case Intrinsic::umul_with_overflow:
5981 case Intrinsic::smul_with_overflow:
5982 handleArithmeticWithOverflow(I);
5983 break;
5984 case Intrinsic::modf:
5985 case Intrinsic::sincos:
5986 case Intrinsic::sincospi:
5987 handleModfOrSincos(I);
5988 break;
5989 case Intrinsic::abs:
5990 handleAbsIntrinsic(I);
5991 break;
5992 case Intrinsic::bitreverse:
5993 handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID: I.getIntrinsicID(),
5994 /*verbatimArgIndices=*/{},
5995 /*forceIntegerIntrinsic=*/false);
5996 break;
5997 case Intrinsic::is_fpclass:
5998 handleIsFpClass(I);
5999 break;
6000 case Intrinsic::lifetime_start:
6001 handleLifetimeStart(I);
6002 break;
6003 case Intrinsic::launder_invariant_group:
6004 handleInvariantGroup(I);
6005 break;
6006 case Intrinsic::bswap:
6007 handleBswap(I);
6008 break;
6009 case Intrinsic::ctlz:
6010 case Intrinsic::cttz:
6011 handleCountLeadingTrailingZeros(I);
6012 break;
6013 case Intrinsic::masked_compressstore:
6014 handleMaskedCompressStore(I);
6015 break;
6016 case Intrinsic::masked_expandload:
6017 handleMaskedExpandLoad(I);
6018 break;
6019 case Intrinsic::masked_gather:
6020 handleMaskedGather(I);
6021 break;
6022 case Intrinsic::masked_scatter:
6023 handleMaskedScatter(I);
6024 break;
6025 case Intrinsic::masked_store:
6026 handleMaskedStore(I);
6027 break;
6028 case Intrinsic::masked_load:
6029 handleMaskedLoad(I);
6030 break;
6031 case Intrinsic::masked_udiv:
6032 case Intrinsic::masked_sdiv:
6033 case Intrinsic::masked_urem:
6034 case Intrinsic::masked_srem:
6035 handleMaskedIntegerDivRem(I);
6036 break;
6037 case Intrinsic::vector_reduce_and:
6038 handleVectorReduceAndIntrinsic(I);
6039 break;
6040 case Intrinsic::vector_reduce_or:
6041 handleVectorReduceOrIntrinsic(I);
6042 break;
6043
6044 case Intrinsic::vector_reduce_add:
6045 case Intrinsic::vector_reduce_xor:
6046 case Intrinsic::vector_reduce_mul:
6047 // Signed/Unsigned Min/Max
6048 // TODO: handling similarly to AND/OR may be more precise.
6049 case Intrinsic::vector_reduce_smax:
6050 case Intrinsic::vector_reduce_smin:
6051 case Intrinsic::vector_reduce_umax:
6052 case Intrinsic::vector_reduce_umin:
6053 // TODO: this has no false positives, but arguably we should check that all
6054 // the bits are initialized.
6055 case Intrinsic::vector_reduce_fmax:
6056 case Intrinsic::vector_reduce_fmin:
6057 handleVectorReduceIntrinsic(I, /*AllowShadowCast=*/false);
6058 break;
6059
6060 case Intrinsic::vector_reduce_fadd:
6061 case Intrinsic::vector_reduce_fmul:
6062 handleVectorReduceWithStarterIntrinsic(I);
6063 break;
6064
6065 // e.g., <4 x i32> @llvm.vector.partial.reduce.add(<4 x i32>, <8 x i32>)
6066 case Intrinsic::vector_partial_reduce_add:
6067 case Intrinsic::vector_partial_reduce_fadd:
6068 handleVectorPartialReduceAddIntrinsic(I);
6069 break;
6070
6071 case Intrinsic::experimental_vector_extract_last_active:
6072 handleVectorExtractLastActive(I);
6073 break;
6074
6075 case Intrinsic::scmp:
6076 case Intrinsic::ucmp: {
6077 handleShadowOr(I);
6078 break;
6079 }
6080
6081 case Intrinsic::fshl:
6082 case Intrinsic::fshr:
6083 handleFunnelShift(I);
6084 break;
6085
6086 case Intrinsic::pdep:
6087 case Intrinsic::pext:
6088 handleGenericBitManipulation(I);
6089 break;
6090
6091 case Intrinsic::is_constant:
6092 // The result of llvm.is.constant() is always defined.
6093 setShadow(V: &I, SV: getCleanShadow(V: &I));
6094 setOrigin(V: &I, Origin: getCleanOrigin());
6095 break;
6096
6097 // The non-saturating versions are handled by visitFPTo[US]IInst().
6098 //
6099 // N.B. some platform-specific intrinsics, such as AArch64 fcvtz[us], are
6100 // lowered to these cross-platform intrinsics.
6101 case Intrinsic::fptosi_sat:
6102 case Intrinsic::fptoui_sat:
6103 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
6104 break;
6105
6106 // e.g.,
6107 // notail call void (...) @llvm.fake.use(i64 %x)
6108 // notail call void (...) @llvm.fake.use(i32 %y)
6109 // notail call void (...) @llvm.fake.use(ptr %z)
6110 case Intrinsic::fake_use:
6111 assert(I.getType()->isVoidTy());
6112 // fake_uses aren't real, they can't hurt you. If the use isn't real, it
6113 // can't be a real use-of-uninitialized memory. Silently skip over
6114 // fake_use.
6115 return true;
6116
6117 default:
6118 return false;
6119 }
6120
6121 return true;
6122 }
6123
6124 bool maybeHandleX86SIMDIntrinsic(IntrinsicInst &I) {
6125 switch (I.getIntrinsicID()) {
6126 case Intrinsic::x86_sse_stmxcsr:
6127 handleStmxcsr(I);
6128 break;
6129 case Intrinsic::x86_sse_ldmxcsr:
6130 handleLdmxcsr(I);
6131 break;
6132
6133 // Convert Scalar Double Precision Floating-Point Value
6134 // to Unsigned Doubleword Integer
6135 // etc.
6136 case Intrinsic::x86_avx512_vcvtsd2usi64:
6137 case Intrinsic::x86_avx512_vcvtsd2usi32:
6138 case Intrinsic::x86_avx512_vcvtss2usi64:
6139 case Intrinsic::x86_avx512_vcvtss2usi32:
6140 case Intrinsic::x86_avx512_cvttss2usi64:
6141 case Intrinsic::x86_avx512_cvttss2usi:
6142 case Intrinsic::x86_avx512_cvttsd2usi64:
6143 case Intrinsic::x86_avx512_cvttsd2usi:
6144 case Intrinsic::x86_avx512_cvtusi2ss:
6145 case Intrinsic::x86_avx512_cvtusi642sd:
6146 case Intrinsic::x86_avx512_cvtusi642ss:
6147 handleSSEVectorConvertIntrinsic(I, NumUsedElements: 1, HasRoundingMode: true);
6148 break;
6149 case Intrinsic::x86_sse2_cvtsd2si64:
6150 case Intrinsic::x86_sse2_cvtsd2si:
6151 case Intrinsic::x86_sse2_cvtsd2ss:
6152 case Intrinsic::x86_sse2_cvttsd2si64:
6153 case Intrinsic::x86_sse2_cvttsd2si:
6154 case Intrinsic::x86_sse_cvtss2si64:
6155 case Intrinsic::x86_sse_cvtss2si:
6156 case Intrinsic::x86_sse_cvttss2si64:
6157 case Intrinsic::x86_sse_cvttss2si:
6158 handleSSEVectorConvertIntrinsic(I, NumUsedElements: 1);
6159 break;
6160 case Intrinsic::x86_sse_cvtps2pi:
6161 case Intrinsic::x86_sse_cvttps2pi:
6162 handleSSEVectorConvertIntrinsic(I, NumUsedElements: 2);
6163 break;
6164
6165 // TODO:
6166 // <1 x i64> @llvm.x86.sse.cvtpd2pi(<2 x double>)
6167 // <2 x double> @llvm.x86.sse.cvtpi2pd(<1 x i64>)
6168 // <4 x float> @llvm.x86.sse.cvtpi2ps(<4 x float>, <1 x i64>)
6169
6170 case Intrinsic::x86_vcvtps2ph_128:
6171 case Intrinsic::x86_vcvtps2ph_256: {
6172 handleSSEVectorConvertIntrinsicByProp(I, /*HasRoundingMode=*/true);
6173 break;
6174 }
6175
6176 // Convert Packed Single Precision Floating-Point Values
6177 // to Packed Signed Doubleword Integer Values
6178 //
6179 // <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512
6180 // (<16 x float>, <16 x i32>, i16, i32)
6181 case Intrinsic::x86_avx512_mask_cvtps2dq_512:
6182 handleAVX512VectorConvertFPToInt(I, /*LastMask=*/false);
6183 break;
6184
6185 // Convert Packed Double Precision Floating-Point Values
6186 // to Packed Single Precision Floating-Point Values
6187 case Intrinsic::x86_sse2_cvtpd2ps:
6188 case Intrinsic::x86_sse2_cvtps2dq:
6189 case Intrinsic::x86_sse2_cvtpd2dq:
6190 case Intrinsic::x86_sse2_cvttps2dq:
6191 case Intrinsic::x86_sse2_cvttpd2dq:
6192 case Intrinsic::x86_avx_cvt_pd2_ps_256:
6193 case Intrinsic::x86_avx_cvt_ps2dq_256:
6194 case Intrinsic::x86_avx_cvt_pd2dq_256:
6195 case Intrinsic::x86_avx_cvtt_ps2dq_256:
6196 case Intrinsic::x86_avx_cvtt_pd2dq_256: {
6197 handleSSEVectorConvertIntrinsicByProp(I, /*HasRoundingMode=*/false);
6198 break;
6199 }
6200
6201 // Convert Single-Precision FP Value to 16-bit FP Value
6202 // <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512
6203 // (<16 x float>, i32, <16 x i16>, i16)
6204 // <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128
6205 // (<4 x float>, i32, <8 x i16>, i8)
6206 // <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.256
6207 // (<8 x float>, i32, <8 x i16>, i8)
6208 case Intrinsic::x86_avx512_mask_vcvtps2ph_512:
6209 case Intrinsic::x86_avx512_mask_vcvtps2ph_256:
6210 case Intrinsic::x86_avx512_mask_vcvtps2ph_128:
6211 handleAVX512VectorConvertFPToInt(I, /*LastMask=*/true);
6212 break;
6213
6214 // Shift Packed Data (Left Logical, Right Arithmetic, Right Logical)
6215 case Intrinsic::x86_avx512_psll_w_512:
6216 case Intrinsic::x86_avx512_psll_d_512:
6217 case Intrinsic::x86_avx512_psll_q_512:
6218 case Intrinsic::x86_avx512_pslli_w_512:
6219 case Intrinsic::x86_avx512_pslli_d_512:
6220 case Intrinsic::x86_avx512_pslli_q_512:
6221 case Intrinsic::x86_avx512_psrl_w_512:
6222 case Intrinsic::x86_avx512_psrl_d_512:
6223 case Intrinsic::x86_avx512_psrl_q_512:
6224 case Intrinsic::x86_avx512_psra_w_512:
6225 case Intrinsic::x86_avx512_psra_d_512:
6226 case Intrinsic::x86_avx512_psra_q_512:
6227 case Intrinsic::x86_avx512_psrli_w_512:
6228 case Intrinsic::x86_avx512_psrli_d_512:
6229 case Intrinsic::x86_avx512_psrli_q_512:
6230 case Intrinsic::x86_avx512_psrai_w_512:
6231 case Intrinsic::x86_avx512_psrai_d_512:
6232 case Intrinsic::x86_avx512_psrai_q_512:
6233 case Intrinsic::x86_avx512_psra_q_256:
6234 case Intrinsic::x86_avx512_psra_q_128:
6235 case Intrinsic::x86_avx512_psrai_q_256:
6236 case Intrinsic::x86_avx512_psrai_q_128:
6237 case Intrinsic::x86_avx2_psll_w:
6238 case Intrinsic::x86_avx2_psll_d:
6239 case Intrinsic::x86_avx2_psll_q:
6240 case Intrinsic::x86_avx2_pslli_w:
6241 case Intrinsic::x86_avx2_pslli_d:
6242 case Intrinsic::x86_avx2_pslli_q:
6243 case Intrinsic::x86_avx2_psrl_w:
6244 case Intrinsic::x86_avx2_psrl_d:
6245 case Intrinsic::x86_avx2_psrl_q:
6246 case Intrinsic::x86_avx2_psra_w:
6247 case Intrinsic::x86_avx2_psra_d:
6248 case Intrinsic::x86_avx2_psrli_w:
6249 case Intrinsic::x86_avx2_psrli_d:
6250 case Intrinsic::x86_avx2_psrli_q:
6251 case Intrinsic::x86_avx2_psrai_w:
6252 case Intrinsic::x86_avx2_psrai_d:
6253 case Intrinsic::x86_sse2_psll_w:
6254 case Intrinsic::x86_sse2_psll_d:
6255 case Intrinsic::x86_sse2_psll_q:
6256 case Intrinsic::x86_sse2_pslli_w:
6257 case Intrinsic::x86_sse2_pslli_d:
6258 case Intrinsic::x86_sse2_pslli_q:
6259 case Intrinsic::x86_sse2_psrl_w:
6260 case Intrinsic::x86_sse2_psrl_d:
6261 case Intrinsic::x86_sse2_psrl_q:
6262 case Intrinsic::x86_sse2_psra_w:
6263 case Intrinsic::x86_sse2_psra_d:
6264 case Intrinsic::x86_sse2_psrli_w:
6265 case Intrinsic::x86_sse2_psrli_d:
6266 case Intrinsic::x86_sse2_psrli_q:
6267 case Intrinsic::x86_sse2_psrai_w:
6268 case Intrinsic::x86_sse2_psrai_d:
6269 case Intrinsic::x86_mmx_psll_w:
6270 case Intrinsic::x86_mmx_psll_d:
6271 case Intrinsic::x86_mmx_psll_q:
6272 case Intrinsic::x86_mmx_pslli_w:
6273 case Intrinsic::x86_mmx_pslli_d:
6274 case Intrinsic::x86_mmx_pslli_q:
6275 case Intrinsic::x86_mmx_psrl_w:
6276 case Intrinsic::x86_mmx_psrl_d:
6277 case Intrinsic::x86_mmx_psrl_q:
6278 case Intrinsic::x86_mmx_psra_w:
6279 case Intrinsic::x86_mmx_psra_d:
6280 case Intrinsic::x86_mmx_psrli_w:
6281 case Intrinsic::x86_mmx_psrli_d:
6282 case Intrinsic::x86_mmx_psrli_q:
6283 case Intrinsic::x86_mmx_psrai_w:
6284 case Intrinsic::x86_mmx_psrai_d:
6285 handleVectorShiftIntrinsic(I, /* Variable */ false);
6286 break;
6287 case Intrinsic::x86_avx2_psllv_d:
6288 case Intrinsic::x86_avx2_psllv_d_256:
6289 case Intrinsic::x86_avx512_psllv_d_512:
6290 case Intrinsic::x86_avx2_psllv_q:
6291 case Intrinsic::x86_avx2_psllv_q_256:
6292 case Intrinsic::x86_avx512_psllv_q_512:
6293 case Intrinsic::x86_avx2_psrlv_d:
6294 case Intrinsic::x86_avx2_psrlv_d_256:
6295 case Intrinsic::x86_avx512_psrlv_d_512:
6296 case Intrinsic::x86_avx2_psrlv_q:
6297 case Intrinsic::x86_avx2_psrlv_q_256:
6298 case Intrinsic::x86_avx512_psrlv_q_512:
6299 case Intrinsic::x86_avx2_psrav_d:
6300 case Intrinsic::x86_avx2_psrav_d_256:
6301 case Intrinsic::x86_avx512_psrav_d_512:
6302 case Intrinsic::x86_avx512_psrav_q_128:
6303 case Intrinsic::x86_avx512_psrav_q_256:
6304 case Intrinsic::x86_avx512_psrav_q_512:
6305 handleVectorShiftIntrinsic(I, /* Variable */ true);
6306 break;
6307
6308 // Pack with Signed/Unsigned Saturation
6309 case Intrinsic::x86_sse2_packsswb_128:
6310 case Intrinsic::x86_sse2_packssdw_128:
6311 case Intrinsic::x86_sse2_packuswb_128:
6312 case Intrinsic::x86_sse41_packusdw:
6313 case Intrinsic::x86_avx2_packsswb:
6314 case Intrinsic::x86_avx2_packssdw:
6315 case Intrinsic::x86_avx2_packuswb:
6316 case Intrinsic::x86_avx2_packusdw:
6317 // e.g., <64 x i8> @llvm.x86.avx512.packsswb.512
6318 // (<32 x i16> %a, <32 x i16> %b)
6319 // <32 x i16> @llvm.x86.avx512.packssdw.512
6320 // (<16 x i32> %a, <16 x i32> %b)
6321 // Note: AVX512 masked variants are auto-upgraded by LLVM.
6322 case Intrinsic::x86_avx512_packsswb_512:
6323 case Intrinsic::x86_avx512_packssdw_512:
6324 case Intrinsic::x86_avx512_packuswb_512:
6325 case Intrinsic::x86_avx512_packusdw_512:
6326 handleVectorPackIntrinsic(I);
6327 break;
6328
6329 case Intrinsic::x86_sse41_pblendvb:
6330 case Intrinsic::x86_sse41_blendvpd:
6331 case Intrinsic::x86_sse41_blendvps:
6332 case Intrinsic::x86_avx_blendv_pd_256:
6333 case Intrinsic::x86_avx_blendv_ps_256:
6334 case Intrinsic::x86_avx2_pblendvb:
6335 handleBlendvIntrinsic(I);
6336 break;
6337
6338 case Intrinsic::x86_avx_dp_ps_256:
6339 case Intrinsic::x86_sse41_dppd:
6340 case Intrinsic::x86_sse41_dpps:
6341 handleDppIntrinsic(I);
6342 break;
6343
6344 case Intrinsic::x86_mmx_packsswb:
6345 case Intrinsic::x86_mmx_packuswb:
6346 handleVectorPackIntrinsic(I, MMXEltSizeInBits: 16);
6347 break;
6348
6349 case Intrinsic::x86_mmx_packssdw:
6350 handleVectorPackIntrinsic(I, MMXEltSizeInBits: 32);
6351 break;
6352
6353 case Intrinsic::x86_mmx_psad_bw:
6354 handleVectorSadIntrinsic(I, IsMMX: true);
6355 break;
6356 case Intrinsic::x86_sse2_psad_bw:
6357 case Intrinsic::x86_avx2_psad_bw:
6358 handleVectorSadIntrinsic(I);
6359 break;
6360
6361 // Multiply and Add Packed Words
6362 // < 4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16>, <8 x i16>)
6363 // < 8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>)
6364 // <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16>, <32 x i16>)
6365 //
6366 // Multiply and Add Packed Signed and Unsigned Bytes
6367 // < 8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8>, <16 x i8>)
6368 // <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8>, <32 x i8>)
6369 // <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8>, <64 x i8>)
6370 //
6371 // These intrinsics are auto-upgraded into non-masked forms:
6372 // < 4 x i32> @llvm.x86.avx512.mask.pmaddw.d.128
6373 // (<8 x i16>, <8 x i16>, <4 x i32>, i8)
6374 // < 8 x i32> @llvm.x86.avx512.mask.pmaddw.d.256
6375 // (<16 x i16>, <16 x i16>, <8 x i32>, i8)
6376 // <16 x i32> @llvm.x86.avx512.mask.pmaddw.d.512
6377 // (<32 x i16>, <32 x i16>, <16 x i32>, i16)
6378 // < 8 x i16> @llvm.x86.avx512.mask.pmaddubs.w.128
6379 // (<16 x i8>, <16 x i8>, <8 x i16>, i8)
6380 // <16 x i16> @llvm.x86.avx512.mask.pmaddubs.w.256
6381 // (<32 x i8>, <32 x i8>, <16 x i16>, i16)
6382 // <32 x i16> @llvm.x86.avx512.mask.pmaddubs.w.512
6383 // (<64 x i8>, <64 x i8>, <32 x i16>, i32)
6384 case Intrinsic::x86_sse2_pmadd_wd:
6385 case Intrinsic::x86_avx2_pmadd_wd:
6386 case Intrinsic::x86_avx512_pmaddw_d_512:
6387 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
6388 case Intrinsic::x86_avx2_pmadd_ub_sw:
6389 case Intrinsic::x86_avx512_pmaddubs_w_512:
6390 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6391 /*ZeroPurifies=*/true,
6392 /*EltSizeInBits=*/0,
6393 /*Lanes=*/kBothLanes);
6394 break;
6395
6396 // <1 x i64> @llvm.x86.ssse3.pmadd.ub.sw(<1 x i64>, <1 x i64>)
6397 case Intrinsic::x86_ssse3_pmadd_ub_sw:
6398 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6399 /*ZeroPurifies=*/true,
6400 /*EltSizeInBits=*/8,
6401 /*Lanes=*/kBothLanes);
6402 break;
6403
6404 // <1 x i64> @llvm.x86.mmx.pmadd.wd(<1 x i64>, <1 x i64>)
6405 case Intrinsic::x86_mmx_pmadd_wd:
6406 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6407 /*ZeroPurifies=*/true,
6408 /*EltSizeInBits=*/16,
6409 /*Lanes=*/kBothLanes);
6410 break;
6411
6412 // BFloat16 multiply-add to single-precision
6413 // <4 x float> llvm.aarch64.neon.bfmlalt
6414 // (<4 x float>, <8 x bfloat>, <8 x bfloat>)
6415 case Intrinsic::aarch64_neon_bfmlalt:
6416 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6417 /*ZeroPurifies=*/false,
6418 /*EltSizeInBits=*/0,
6419 /*Lanes=*/kOddLanes);
6420 break;
6421
6422 // <4 x float> llvm.aarch64.neon.bfmlalb
6423 // (<4 x float>, <8 x bfloat>, <8 x bfloat>)
6424 case Intrinsic::aarch64_neon_bfmlalb:
6425 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6426 /*ZeroPurifies=*/false,
6427 /*EltSizeInBits=*/0,
6428 /*Lanes=*/kEvenLanes);
6429 break;
6430
6431 // AVX Vector Neural Network Instructions: bytes
6432 //
6433 // Multiply and Add Signed Bytes
6434 // < 4 x i32> @llvm.x86.avx2.vpdpbssd.128
6435 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6436 // < 8 x i32> @llvm.x86.avx2.vpdpbssd.256
6437 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6438 // <16 x i32> @llvm.x86.avx10.vpdpbssd.512
6439 // (<16 x i32>, <64 x i8>, <64 x i8>)
6440 //
6441 // Multiply and Add Signed Bytes With Saturation
6442 // < 4 x i32> @llvm.x86.avx2.vpdpbssds.128
6443 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6444 // < 8 x i32> @llvm.x86.avx2.vpdpbssds.256
6445 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6446 // <16 x i32> @llvm.x86.avx10.vpdpbssds.512
6447 // (<16 x i32>, <64 x i8>, <64 x i8>)
6448 //
6449 // Multiply and Add Signed and Unsigned Bytes
6450 // < 4 x i32> @llvm.x86.avx2.vpdpbsud.128
6451 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6452 // < 8 x i32> @llvm.x86.avx2.vpdpbsud.256
6453 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6454 // <16 x i32> @llvm.x86.avx10.vpdpbsud.512
6455 // (<16 x i32>, <64 x i8>, <64 x i8>)
6456 //
6457 // Multiply and Add Signed and Unsigned Bytes With Saturation
6458 // < 4 x i32> @llvm.x86.avx2.vpdpbsuds.128
6459 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6460 // < 8 x i32> @llvm.x86.avx2.vpdpbsuds.256
6461 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6462 // <16 x i32> @llvm.x86.avx512.vpdpbusds.512
6463 // (<16 x i32>, <64 x i8>, <64 x i8>)
6464 //
6465 // Multiply and Add Unsigned and Signed Bytes
6466 // < 4 x i32> @llvm.x86.avx512.vpdpbusd.128
6467 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6468 // < 8 x i32> @llvm.x86.avx512.vpdpbusd.256
6469 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6470 // <16 x i32> @llvm.x86.avx512.vpdpbusd.512
6471 // (<16 x i32>, <64 x i8>, <64 x i8>)
6472 //
6473 // Multiply and Add Unsigned and Signed Bytes With Saturation
6474 // < 4 x i32> @llvm.x86.avx512.vpdpbusds.128
6475 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6476 // < 8 x i32> @llvm.x86.avx512.vpdpbusds.256
6477 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6478 // <16 x i32> @llvm.x86.avx10.vpdpbsuds.512
6479 // (<16 x i32>, <64 x i8>, <64 x i8>)
6480 //
6481 // Multiply and Add Unsigned Bytes
6482 // < 4 x i32> @llvm.x86.avx2.vpdpbuud.128
6483 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6484 // < 8 x i32> @llvm.x86.avx2.vpdpbuud.256
6485 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6486 // <16 x i32> @llvm.x86.avx10.vpdpbuud.512
6487 // (<16 x i32>, <64 x i8>, <64 x i8>)
6488 //
6489 // Multiply and Add Unsigned Bytes With Saturation
6490 // < 4 x i32> @llvm.x86.avx2.vpdpbuuds.128
6491 // (< 4 x i32>, <16 x i8>, <16 x i8>)
6492 // < 8 x i32> @llvm.x86.avx2.vpdpbuuds.256
6493 // (< 8 x i32>, <32 x i8>, <32 x i8>)
6494 // <16 x i32> @llvm.x86.avx10.vpdpbuuds.512
6495 // (<16 x i32>, <64 x i8>, <64 x i8>)
6496 //
6497 // These intrinsics are auto-upgraded into non-masked forms:
6498 // <4 x i32> @llvm.x86.avx512.mask.vpdpbusd.128
6499 // (<4 x i32>, <16 x i8>, <16 x i8>, i8)
6500 // <4 x i32> @llvm.x86.avx512.maskz.vpdpbusd.128
6501 // (<4 x i32>, <16 x i8>, <16 x i8>, i8)
6502 // <8 x i32> @llvm.x86.avx512.mask.vpdpbusd.256
6503 // (<8 x i32>, <32 x i8>, <32 x i8>, i8)
6504 // <8 x i32> @llvm.x86.avx512.maskz.vpdpbusd.256
6505 // (<8 x i32>, <32 x i8>, <32 x i8>, i8)
6506 // <16 x i32> @llvm.x86.avx512.mask.vpdpbusd.512
6507 // (<16 x i32>, <64 x i8>, <64 x i8>, i16)
6508 // <16 x i32> @llvm.x86.avx512.maskz.vpdpbusd.512
6509 // (<16 x i32>, <64 x i8>, <64 x i8>, i16)
6510 //
6511 // <4 x i32> @llvm.x86.avx512.mask.vpdpbusds.128
6512 // (<4 x i32>, <16 x i8>, <16 x i8>, i8)
6513 // <4 x i32> @llvm.x86.avx512.maskz.vpdpbusds.128
6514 // (<4 x i32>, <16 x i8>, <16 x i8>, i8)
6515 // <8 x i32> @llvm.x86.avx512.mask.vpdpbusds.256
6516 // (<8 x i32>, <32 x i8>, <32 x i8>, i8)
6517 // <8 x i32> @llvm.x86.avx512.maskz.vpdpbusds.256
6518 // (<8 x i32>, <32 x i8>, <32 x i8>, i8)
6519 // <16 x i32> @llvm.x86.avx512.mask.vpdpbusds.512
6520 // (<16 x i32>, <64 x i8>, <64 x i8>, i16)
6521 // <16 x i32> @llvm.x86.avx512.maskz.vpdpbusds.512
6522 // (<16 x i32>, <64 x i8>, <64 x i8>, i16)
6523 case Intrinsic::x86_avx512_vpdpbusd_128:
6524 case Intrinsic::x86_avx512_vpdpbusd_256:
6525 case Intrinsic::x86_avx512_vpdpbusd_512:
6526 case Intrinsic::x86_avx512_vpdpbusds_128:
6527 case Intrinsic::x86_avx512_vpdpbusds_256:
6528 case Intrinsic::x86_avx512_vpdpbusds_512:
6529 case Intrinsic::x86_avx2_vpdpbssd_128:
6530 case Intrinsic::x86_avx2_vpdpbssd_256:
6531 case Intrinsic::x86_avx10_vpdpbssd_512:
6532 case Intrinsic::x86_avx2_vpdpbssds_128:
6533 case Intrinsic::x86_avx2_vpdpbssds_256:
6534 case Intrinsic::x86_avx10_vpdpbssds_512:
6535 case Intrinsic::x86_avx2_vpdpbsud_128:
6536 case Intrinsic::x86_avx2_vpdpbsud_256:
6537 case Intrinsic::x86_avx10_vpdpbsud_512:
6538 case Intrinsic::x86_avx2_vpdpbsuds_128:
6539 case Intrinsic::x86_avx2_vpdpbsuds_256:
6540 case Intrinsic::x86_avx10_vpdpbsuds_512:
6541 case Intrinsic::x86_avx2_vpdpbuud_128:
6542 case Intrinsic::x86_avx2_vpdpbuud_256:
6543 case Intrinsic::x86_avx10_vpdpbuud_512:
6544 case Intrinsic::x86_avx2_vpdpbuuds_128:
6545 case Intrinsic::x86_avx2_vpdpbuuds_256:
6546 case Intrinsic::x86_avx10_vpdpbuuds_512:
6547 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/4,
6548 /*ZeroPurifies=*/true,
6549 /*EltSizeInBits=*/0,
6550 /*Lanes=*/kBothLanes);
6551 break;
6552
6553 // AVX Vector Neural Network Instructions: words
6554 //
6555 // Multiply and Add Signed Word Integers
6556 // < 4 x i32> @llvm.x86.avx512.vpdpwssd.128
6557 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6558 // < 8 x i32> @llvm.x86.avx512.vpdpwssd.256
6559 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6560 // <16 x i32> @llvm.x86.avx512.vpdpwssd.512
6561 // (<16 x i32>, <32 x i16>, <32 x i16>)
6562 //
6563 // Multiply and Add Signed Word Integers With Saturation
6564 // < 4 x i32> @llvm.x86.avx512.vpdpwssds.128
6565 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6566 // < 8 x i32> @llvm.x86.avx512.vpdpwssds.256
6567 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6568 // <16 x i32> @llvm.x86.avx512.vpdpwssds.512
6569 // (<16 x i32>, <32 x i16>, <32 x i16>)
6570 //
6571 // Multiply and Add Signed and Unsigned Word Integers
6572 // < 4 x i32> @llvm.x86.avx2.vpdpwsud.128
6573 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6574 // < 8 x i32> @llvm.x86.avx2.vpdpwsud.256
6575 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6576 // <16 x i32> @llvm.x86.avx10.vpdpwsud.512
6577 // (<16 x i32>, <32 x i16>, <32 x i16>)
6578 //
6579 // Multiply and Add Signed and Unsigned Word Integers With Saturation
6580 // < 4 x i32> @llvm.x86.avx2.vpdpwsuds.128
6581 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6582 // < 8 x i32> @llvm.x86.avx2.vpdpwsuds.256
6583 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6584 // <16 x i32> @llvm.x86.avx10.vpdpwsuds.512
6585 // (<16 x i32>, <32 x i16>, <32 x i16>)
6586 //
6587 // Multiply and Add Unsigned and Signed Word Integers
6588 // < 4 x i32> @llvm.x86.avx2.vpdpwusd.128
6589 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6590 // < 8 x i32> @llvm.x86.avx2.vpdpwusd.256
6591 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6592 // <16 x i32> @llvm.x86.avx10.vpdpwusd.512
6593 // (<16 x i32>, <32 x i16>, <32 x i16>)
6594 //
6595 // Multiply and Add Unsigned and Signed Word Integers With Saturation
6596 // < 4 x i32> @llvm.x86.avx2.vpdpwusds.128
6597 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6598 // < 8 x i32> @llvm.x86.avx2.vpdpwusds.256
6599 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6600 // <16 x i32> @llvm.x86.avx10.vpdpwusds.512
6601 // (<16 x i32>, <32 x i16>, <32 x i16>)
6602 //
6603 // Multiply and Add Unsigned and Unsigned Word Integers
6604 // < 4 x i32> @llvm.x86.avx2.vpdpwuud.128
6605 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6606 // < 8 x i32> @llvm.x86.avx2.vpdpwuud.256
6607 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6608 // <16 x i32> @llvm.x86.avx10.vpdpwuud.512
6609 // (<16 x i32>, <32 x i16>, <32 x i16>)
6610 //
6611 // Multiply and Add Unsigned and Unsigned Word Integers With Saturation
6612 // < 4 x i32> @llvm.x86.avx2.vpdpwuuds.128
6613 // (< 4 x i32>, < 8 x i16>, < 8 x i16>)
6614 // < 8 x i32> @llvm.x86.avx2.vpdpwuuds.256
6615 // (< 8 x i32>, <16 x i16>, <16 x i16>)
6616 // <16 x i32> @llvm.x86.avx10.vpdpwuuds.512
6617 // (<16 x i32>, <32 x i16>, <32 x i16>)
6618 //
6619 // These intrinsics are auto-upgraded into non-masked forms:
6620 // <4 x i32> @llvm.x86.avx512.mask.vpdpwssd.128
6621 // (<4 x i32>, <8 x i16>, <8 x i16>, i8)
6622 // <4 x i32> @llvm.x86.avx512.maskz.vpdpwssd.128
6623 // (<4 x i32>, <8 x i16>, <8 x i16>, i8)
6624 // <8 x i32> @llvm.x86.avx512.mask.vpdpwssd.256
6625 // (<8 x i32>, <16 x i16>, <16 x i16>, i8)
6626 // <8 x i32> @llvm.x86.avx512.maskz.vpdpwssd.256
6627 // (<8 x i32>, <16 x i16>, <16 x i16>, i8)
6628 // <16 x i32> @llvm.x86.avx512.mask.vpdpwssd.512
6629 // (<16 x i32>, <32 x i16>, <32 x i16>, i16)
6630 // <16 x i32> @llvm.x86.avx512.maskz.vpdpwssd.512
6631 // (<16 x i32>, <32 x i16>, <32 x i16>, i16)
6632 //
6633 // <4 x i32> @llvm.x86.avx512.mask.vpdpwssds.128
6634 // (<4 x i32>, <8 x i16>, <8 x i16>, i8)
6635 // <4 x i32> @llvm.x86.avx512.maskz.vpdpwssds.128
6636 // (<4 x i32>, <8 x i16>, <8 x i16>, i8)
6637 // <8 x i32> @llvm.x86.avx512.mask.vpdpwssds.256
6638 // (<8 x i32>, <16 x i16>, <16 x i16>, i8)
6639 // <8 x i32> @llvm.x86.avx512.maskz.vpdpwssds.256
6640 // (<8 x i32>, <16 x i16>, <16 x i16>, i8)
6641 // <16 x i32> @llvm.x86.avx512.mask.vpdpwssds.512
6642 // (<16 x i32>, <32 x i16>, <32 x i16>, i16)
6643 // <16 x i32> @llvm.x86.avx512.maskz.vpdpwssds.512
6644 // (<16 x i32>, <32 x i16>, <32 x i16>, i16)
6645 case Intrinsic::x86_avx512_vpdpwssd_128:
6646 case Intrinsic::x86_avx512_vpdpwssd_256:
6647 case Intrinsic::x86_avx512_vpdpwssd_512:
6648 case Intrinsic::x86_avx512_vpdpwssds_128:
6649 case Intrinsic::x86_avx512_vpdpwssds_256:
6650 case Intrinsic::x86_avx512_vpdpwssds_512:
6651 case Intrinsic::x86_avx2_vpdpwsud_128:
6652 case Intrinsic::x86_avx2_vpdpwsud_256:
6653 case Intrinsic::x86_avx10_vpdpwsud_512:
6654 case Intrinsic::x86_avx2_vpdpwsuds_128:
6655 case Intrinsic::x86_avx2_vpdpwsuds_256:
6656 case Intrinsic::x86_avx10_vpdpwsuds_512:
6657 case Intrinsic::x86_avx2_vpdpwusd_128:
6658 case Intrinsic::x86_avx2_vpdpwusd_256:
6659 case Intrinsic::x86_avx10_vpdpwusd_512:
6660 case Intrinsic::x86_avx2_vpdpwusds_128:
6661 case Intrinsic::x86_avx2_vpdpwusds_256:
6662 case Intrinsic::x86_avx10_vpdpwusds_512:
6663 case Intrinsic::x86_avx2_vpdpwuud_128:
6664 case Intrinsic::x86_avx2_vpdpwuud_256:
6665 case Intrinsic::x86_avx10_vpdpwuud_512:
6666 case Intrinsic::x86_avx2_vpdpwuuds_128:
6667 case Intrinsic::x86_avx2_vpdpwuuds_256:
6668 case Intrinsic::x86_avx10_vpdpwuuds_512:
6669 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6670 /*ZeroPurifies=*/true,
6671 /*EltSizeInBits=*/0,
6672 /*Lanes=*/kBothLanes);
6673 break;
6674
6675 // Dot Product of BF16 Pairs Accumulated Into Packed Single
6676 // Precision
6677 // <4 x float> @llvm.x86.avx512bf16.dpbf16ps.128
6678 // (<4 x float>, <8 x bfloat>, <8 x bfloat>)
6679 // <8 x float> @llvm.x86.avx512bf16.dpbf16ps.256
6680 // (<8 x float>, <16 x bfloat>, <16 x bfloat>)
6681 // <16 x float> @llvm.x86.avx512bf16.dpbf16ps.512
6682 // (<16 x float>, <32 x bfloat>, <32 x bfloat>)
6683 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6684 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6685 case Intrinsic::x86_avx512bf16_dpbf16ps_512:
6686 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
6687 /*ZeroPurifies=*/false,
6688 /*EltSizeInBits=*/0,
6689 /*Lanes=*/kBothLanes);
6690 break;
6691
6692 case Intrinsic::x86_sse_cmp_ss:
6693 case Intrinsic::x86_sse2_cmp_sd:
6694 case Intrinsic::x86_sse_comieq_ss:
6695 case Intrinsic::x86_sse_comilt_ss:
6696 case Intrinsic::x86_sse_comile_ss:
6697 case Intrinsic::x86_sse_comigt_ss:
6698 case Intrinsic::x86_sse_comige_ss:
6699 case Intrinsic::x86_sse_comineq_ss:
6700 case Intrinsic::x86_sse_ucomieq_ss:
6701 case Intrinsic::x86_sse_ucomilt_ss:
6702 case Intrinsic::x86_sse_ucomile_ss:
6703 case Intrinsic::x86_sse_ucomigt_ss:
6704 case Intrinsic::x86_sse_ucomige_ss:
6705 case Intrinsic::x86_sse_ucomineq_ss:
6706 case Intrinsic::x86_sse2_comieq_sd:
6707 case Intrinsic::x86_sse2_comilt_sd:
6708 case Intrinsic::x86_sse2_comile_sd:
6709 case Intrinsic::x86_sse2_comigt_sd:
6710 case Intrinsic::x86_sse2_comige_sd:
6711 case Intrinsic::x86_sse2_comineq_sd:
6712 case Intrinsic::x86_sse2_ucomieq_sd:
6713 case Intrinsic::x86_sse2_ucomilt_sd:
6714 case Intrinsic::x86_sse2_ucomile_sd:
6715 case Intrinsic::x86_sse2_ucomigt_sd:
6716 case Intrinsic::x86_sse2_ucomige_sd:
6717 case Intrinsic::x86_sse2_ucomineq_sd:
6718 handleVectorCompareScalarIntrinsic(I);
6719 break;
6720
6721 case Intrinsic::x86_avx_cmp_pd_256:
6722 case Intrinsic::x86_avx_cmp_ps_256:
6723 case Intrinsic::x86_sse2_cmp_pd:
6724 case Intrinsic::x86_sse_cmp_ps:
6725 handleVectorComparePackedIntrinsic(I, /*PredicateAsOperand=*/true);
6726 break;
6727
6728 case Intrinsic::x86_bmi_bextr_32:
6729 case Intrinsic::x86_bmi_bextr_64:
6730 case Intrinsic::x86_bmi_bzhi_32:
6731 case Intrinsic::x86_bmi_bzhi_64:
6732 handleGenericBitManipulation(I);
6733 break;
6734
6735 case Intrinsic::x86_pclmulqdq:
6736 case Intrinsic::x86_pclmulqdq_256:
6737 case Intrinsic::x86_pclmulqdq_512:
6738 handlePclmulIntrinsic(I);
6739 break;
6740
6741 case Intrinsic::x86_avx_round_pd_256:
6742 case Intrinsic::x86_avx_round_ps_256:
6743 case Intrinsic::x86_sse41_round_pd:
6744 case Intrinsic::x86_sse41_round_ps:
6745 handleRoundPdPsIntrinsic(I);
6746 break;
6747
6748 case Intrinsic::x86_sse41_round_sd:
6749 case Intrinsic::x86_sse41_round_ss:
6750 handleUnarySdSsIntrinsic(I);
6751 break;
6752
6753 case Intrinsic::x86_sse2_max_sd:
6754 case Intrinsic::x86_sse_max_ss:
6755 case Intrinsic::x86_sse2_min_sd:
6756 case Intrinsic::x86_sse_min_ss:
6757 handleBinarySdSsIntrinsic(I);
6758 break;
6759
6760 case Intrinsic::x86_avx_vtestc_pd:
6761 case Intrinsic::x86_avx_vtestc_pd_256:
6762 case Intrinsic::x86_avx_vtestc_ps:
6763 case Intrinsic::x86_avx_vtestc_ps_256:
6764 case Intrinsic::x86_avx_vtestnzc_pd:
6765 case Intrinsic::x86_avx_vtestnzc_pd_256:
6766 case Intrinsic::x86_avx_vtestnzc_ps:
6767 case Intrinsic::x86_avx_vtestnzc_ps_256:
6768 case Intrinsic::x86_avx_vtestz_pd:
6769 case Intrinsic::x86_avx_vtestz_pd_256:
6770 case Intrinsic::x86_avx_vtestz_ps:
6771 case Intrinsic::x86_avx_vtestz_ps_256:
6772 case Intrinsic::x86_avx_ptestc_256:
6773 case Intrinsic::x86_avx_ptestnzc_256:
6774 case Intrinsic::x86_avx_ptestz_256:
6775 case Intrinsic::x86_sse41_ptestc:
6776 case Intrinsic::x86_sse41_ptestnzc:
6777 case Intrinsic::x86_sse41_ptestz:
6778 handleVtestIntrinsic(I);
6779 break;
6780
6781 // Packed Horizontal Add/Subtract
6782 case Intrinsic::x86_ssse3_phadd_w:
6783 case Intrinsic::x86_ssse3_phadd_w_128:
6784 case Intrinsic::x86_ssse3_phsub_w:
6785 case Intrinsic::x86_ssse3_phsub_w_128:
6786 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/1,
6787 /*ReinterpretElemWidth=*/16);
6788 break;
6789
6790 case Intrinsic::x86_avx2_phadd_w:
6791 case Intrinsic::x86_avx2_phsub_w:
6792 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/2,
6793 /*ReinterpretElemWidth=*/16);
6794 break;
6795
6796 // Packed Horizontal Add/Subtract
6797 case Intrinsic::x86_ssse3_phadd_d:
6798 case Intrinsic::x86_ssse3_phadd_d_128:
6799 case Intrinsic::x86_ssse3_phsub_d:
6800 case Intrinsic::x86_ssse3_phsub_d_128:
6801 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/1,
6802 /*ReinterpretElemWidth=*/32);
6803 break;
6804
6805 case Intrinsic::x86_avx2_phadd_d:
6806 case Intrinsic::x86_avx2_phsub_d:
6807 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/2,
6808 /*ReinterpretElemWidth=*/32);
6809 break;
6810
6811 // Packed Horizontal Add/Subtract and Saturate
6812 case Intrinsic::x86_ssse3_phadd_sw:
6813 case Intrinsic::x86_ssse3_phadd_sw_128:
6814 case Intrinsic::x86_ssse3_phsub_sw:
6815 case Intrinsic::x86_ssse3_phsub_sw_128:
6816 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/1,
6817 /*ReinterpretElemWidth=*/16);
6818 break;
6819
6820 case Intrinsic::x86_avx2_phadd_sw:
6821 case Intrinsic::x86_avx2_phsub_sw:
6822 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/2,
6823 /*ReinterpretElemWidth=*/16);
6824 break;
6825
6826 // Packed Single/Double Precision Floating-Point Horizontal Add
6827 case Intrinsic::x86_sse3_hadd_ps:
6828 case Intrinsic::x86_sse3_hadd_pd:
6829 case Intrinsic::x86_sse3_hsub_ps:
6830 case Intrinsic::x86_sse3_hsub_pd:
6831 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/1);
6832 break;
6833
6834 case Intrinsic::x86_avx_hadd_pd_256:
6835 case Intrinsic::x86_avx_hadd_ps_256:
6836 case Intrinsic::x86_avx_hsub_pd_256:
6837 case Intrinsic::x86_avx_hsub_ps_256:
6838 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/2);
6839 break;
6840
6841 case Intrinsic::x86_avx_maskstore_ps:
6842 case Intrinsic::x86_avx_maskstore_pd:
6843 case Intrinsic::x86_avx_maskstore_ps_256:
6844 case Intrinsic::x86_avx_maskstore_pd_256:
6845 case Intrinsic::x86_avx2_maskstore_d:
6846 case Intrinsic::x86_avx2_maskstore_q:
6847 case Intrinsic::x86_avx2_maskstore_d_256:
6848 case Intrinsic::x86_avx2_maskstore_q_256: {
6849 handleAVXMaskedStore(I);
6850 break;
6851 }
6852
6853 case Intrinsic::x86_avx_maskload_ps:
6854 case Intrinsic::x86_avx_maskload_pd:
6855 case Intrinsic::x86_avx_maskload_ps_256:
6856 case Intrinsic::x86_avx_maskload_pd_256:
6857 case Intrinsic::x86_avx2_maskload_d:
6858 case Intrinsic::x86_avx2_maskload_q:
6859 case Intrinsic::x86_avx2_maskload_d_256:
6860 case Intrinsic::x86_avx2_maskload_q_256: {
6861 handleAVXMaskedLoad(I);
6862 break;
6863 }
6864
6865 // Packed
6866 case Intrinsic::x86_avx512fp16_add_ph_512:
6867 case Intrinsic::x86_avx512fp16_sub_ph_512:
6868 case Intrinsic::x86_avx512fp16_mul_ph_512:
6869 case Intrinsic::x86_avx512fp16_div_ph_512:
6870 case Intrinsic::x86_avx512fp16_max_ph_512:
6871 case Intrinsic::x86_avx512fp16_min_ph_512:
6872 case Intrinsic::x86_avx512_min_ps_512:
6873 case Intrinsic::x86_avx512_min_pd_512:
6874 case Intrinsic::x86_avx512_max_ps_512:
6875 case Intrinsic::x86_avx512_max_pd_512: {
6876 // These AVX512 variants contain the rounding mode as a trailing flag.
6877 // Earlier variants do not have a trailing flag and are already handled
6878 // by maybeHandleSimpleNomemIntrinsic(I, 0) via
6879 // maybeHandleUnknownIntrinsic.
6880 [[maybe_unused]] bool Success =
6881 maybeHandleSimpleNomemIntrinsic(I, /*trailingFlags=*/1);
6882 assert(Success);
6883 break;
6884 }
6885
6886 case Intrinsic::x86_avx_vpermilvar_pd:
6887 case Intrinsic::x86_avx_vpermilvar_pd_256:
6888 case Intrinsic::x86_avx512_vpermilvar_pd_512:
6889 case Intrinsic::x86_avx_vpermilvar_ps:
6890 case Intrinsic::x86_avx_vpermilvar_ps_256:
6891 case Intrinsic::x86_avx512_vpermilvar_ps_512: {
6892 handleAVXVpermilvar(I);
6893 break;
6894 }
6895
6896 case Intrinsic::x86_avx512_vpermi2var_d_128:
6897 case Intrinsic::x86_avx512_vpermi2var_d_256:
6898 case Intrinsic::x86_avx512_vpermi2var_d_512:
6899 case Intrinsic::x86_avx512_vpermi2var_hi_128:
6900 case Intrinsic::x86_avx512_vpermi2var_hi_256:
6901 case Intrinsic::x86_avx512_vpermi2var_hi_512:
6902 case Intrinsic::x86_avx512_vpermi2var_pd_128:
6903 case Intrinsic::x86_avx512_vpermi2var_pd_256:
6904 case Intrinsic::x86_avx512_vpermi2var_pd_512:
6905 case Intrinsic::x86_avx512_vpermi2var_ps_128:
6906 case Intrinsic::x86_avx512_vpermi2var_ps_256:
6907 case Intrinsic::x86_avx512_vpermi2var_ps_512:
6908 case Intrinsic::x86_avx512_vpermi2var_q_128:
6909 case Intrinsic::x86_avx512_vpermi2var_q_256:
6910 case Intrinsic::x86_avx512_vpermi2var_q_512:
6911 case Intrinsic::x86_avx512_vpermi2var_qi_128:
6912 case Intrinsic::x86_avx512_vpermi2var_qi_256:
6913 case Intrinsic::x86_avx512_vpermi2var_qi_512:
6914 handleAVXVpermi2var(I);
6915 break;
6916
6917 // Packed Shuffle
6918 // llvm.x86.sse.pshuf.w(<1 x i64>, i8)
6919 // llvm.x86.ssse3.pshuf.b(<1 x i64>, <1 x i64>)
6920 // llvm.x86.ssse3.pshuf.b.128(<16 x i8>, <16 x i8>)
6921 // llvm.x86.avx2.pshuf.b(<32 x i8>, <32 x i8>)
6922 // llvm.x86.avx512.pshuf.b.512(<64 x i8>, <64 x i8>)
6923 //
6924 // The following intrinsics are auto-upgraded:
6925 // llvm.x86.sse2.pshuf.d(<4 x i32>, i8)
6926 // llvm.x86.sse2.gpshufh.w(<8 x i16>, i8)
6927 // llvm.x86.sse2.pshufl.w(<8 x i16>, i8)
6928 case Intrinsic::x86_avx2_pshuf_b:
6929 case Intrinsic::x86_sse_pshuf_w:
6930 case Intrinsic::x86_ssse3_pshuf_b_128:
6931 case Intrinsic::x86_ssse3_pshuf_b:
6932 case Intrinsic::x86_avx512_pshuf_b_512:
6933 handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID: I.getIntrinsicID(),
6934 /*verbatimArgsIndices=*/verbatimArgIndices: {-1},
6935 /*forceIntegerIntrinsic=*/false);
6936 break;
6937
6938 // AVX512 PMOV: Packed MOV, with truncation
6939 // Precisely handled by applying the same intrinsic to the shadow
6940 case Intrinsic::x86_avx512_mask_pmov_dw_128:
6941 case Intrinsic::x86_avx512_mask_pmov_db_128:
6942 case Intrinsic::x86_avx512_mask_pmov_qb_128:
6943 case Intrinsic::x86_avx512_mask_pmov_qw_128:
6944 case Intrinsic::x86_avx512_mask_pmov_qd_128:
6945 case Intrinsic::x86_avx512_mask_pmov_wb_128:
6946 case Intrinsic::x86_avx512_mask_pmov_dw_256:
6947 case Intrinsic::x86_avx512_mask_pmov_db_256:
6948 case Intrinsic::x86_avx512_mask_pmov_qb_256:
6949 case Intrinsic::x86_avx512_mask_pmov_qw_256:
6950 case Intrinsic::x86_avx512_mask_pmov_dw_512:
6951 case Intrinsic::x86_avx512_mask_pmov_db_512:
6952 case Intrinsic::x86_avx512_mask_pmov_qb_512:
6953 case Intrinsic::x86_avx512_mask_pmov_qw_512: {
6954 // Intrinsic::x86_avx512_mask_pmov_{qd,wb}_{256,512} were removed in
6955 // f608dc1f5775ee880e8ea30e2d06ab5a4a935c22
6956 handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID: I.getIntrinsicID(),
6957 /*verbatimArgIndices=*/{-1},
6958 /*forceIntegerIntrinsic=*/false);
6959 break;
6960 }
6961
6962 // AVX512 PMOV{S,US}: Packed MOV, with signed/unsigned saturation
6963 // Approximately handled using the corresponding truncation intrinsic
6964 // TODO: improve handleAVX512VectorDownConvert to precisely model saturation
6965 case Intrinsic::x86_avx512_mask_pmovs_dw_512:
6966 case Intrinsic::x86_avx512_mask_pmovus_dw_512: {
6967 handleIntrinsicByApplyingToShadow(
6968 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_dw_512,
6969 /*verbatimArgIndices=*/{-1}, /*forceIntegerIntrinsic=*/false);
6970 break;
6971 }
6972
6973 case Intrinsic::x86_avx512_mask_pmovs_dw_256:
6974 case Intrinsic::x86_avx512_mask_pmovus_dw_256:
6975 handleIntrinsicByApplyingToShadow(
6976 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_dw_256,
6977 /*verbatimArgIndices=*/{-1}, /*forceIntegerIntrinsic=*/false);
6978 break;
6979
6980 case Intrinsic::x86_avx512_mask_pmovs_dw_128:
6981 case Intrinsic::x86_avx512_mask_pmovus_dw_128:
6982 handleIntrinsicByApplyingToShadow(
6983 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_dw_128,
6984 /*verbatimArgIndices=*/{-1}, /*forceIntegerIntrinsic=*/false);
6985 break;
6986
6987 case Intrinsic::x86_avx512_mask_pmovs_db_512:
6988 case Intrinsic::x86_avx512_mask_pmovus_db_512: {
6989 handleIntrinsicByApplyingToShadow(
6990 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_db_512,
6991 /*verbatimArgIndices=*/{-1}, /*forceIntegerIntrinsic=*/false);
6992 break;
6993 }
6994
6995 case Intrinsic::x86_avx512_mask_pmovs_db_256:
6996 case Intrinsic::x86_avx512_mask_pmovus_db_256:
6997 handleIntrinsicByApplyingToShadow(
6998 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_db_256,
6999 /*verbatimArgIndices=*/{-1}, /*forceIntegerIntrinsic=*/false);
7000 break;
7001
7002 case Intrinsic::x86_avx512_mask_pmovs_db_128:
7003 case Intrinsic::x86_avx512_mask_pmovus_db_128:
7004 handleIntrinsicByApplyingToShadow(
7005 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_db_128,
7006 /*verbatimArgIndices=*/{-1}, /*forceIntegerIntrinsic=*/false);
7007 break;
7008
7009 case Intrinsic::x86_avx512_mask_pmovs_qb_512:
7010 case Intrinsic::x86_avx512_mask_pmovus_qb_512: {
7011 handleIntrinsicByApplyingToShadow(
7012 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qb_512,
7013 /*verbatimArgIndices=*/{-1}, /*forceIntegerIntrinsic=*/false);
7014 break;
7015 }
7016
7017 case Intrinsic::x86_avx512_mask_pmovs_qb_256:
7018 case Intrinsic::x86_avx512_mask_pmovus_qb_256:
7019 handleIntrinsicByApplyingToShadow(
7020 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qb_256,
7021 /*verbatimArgIndices=*/{-1}, /*forceIntegerIntrinsic=*/false);
7022 break;
7023
7024 case Intrinsic::x86_avx512_mask_pmovs_qb_128:
7025 case Intrinsic::x86_avx512_mask_pmovus_qb_128:
7026 handleIntrinsicByApplyingToShadow(
7027 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qb_128,
7028 /*verbatimArgIndices=*/{-1}, /*forceIntegerIntrinsic=*/false);
7029 break;
7030
7031 case Intrinsic::x86_avx512_mask_pmovs_qw_512:
7032 case Intrinsic::x86_avx512_mask_pmovus_qw_512: {
7033 handleIntrinsicByApplyingToShadow(
7034 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qw_512,
7035 /*verbatimArgIndices=*/{-1}, /*forceIntegerIntrinsic=*/false);
7036 break;
7037 }
7038
7039 case Intrinsic::x86_avx512_mask_pmovs_qw_256:
7040 case Intrinsic::x86_avx512_mask_pmovus_qw_256:
7041 handleIntrinsicByApplyingToShadow(
7042 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qw_256,
7043 /*verbatimArgIndices=*/{-1}, /*forceIntegerIntrinsic=*/false);
7044 break;
7045
7046 case Intrinsic::x86_avx512_mask_pmovs_qw_128:
7047 case Intrinsic::x86_avx512_mask_pmovus_qw_128:
7048 handleIntrinsicByApplyingToShadow(
7049 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qw_128,
7050 /*verbatimArgIndices=*/{-1}, /*forceIntegerIntrinsic=*/false);
7051 break;
7052
7053 case Intrinsic::x86_avx512_mask_pmovs_qd_128:
7054 case Intrinsic::x86_avx512_mask_pmovus_qd_128:
7055 handleIntrinsicByApplyingToShadow(
7056 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_qd_128,
7057 /*verbatimArgIndices=*/{-1}, /*forceIntegerIntrinsic=*/false);
7058 break;
7059
7060 case Intrinsic::x86_avx512_mask_pmovs_wb_128:
7061 case Intrinsic::x86_avx512_mask_pmovus_wb_128:
7062 handleIntrinsicByApplyingToShadow(
7063 I, shadowIntrinsicID: Intrinsic::x86_avx512_mask_pmov_wb_128,
7064 /*verbatimArgIndices=*/{-1}, /*forceIntegerIntrinsic=*/false);
7065 break;
7066
7067 case Intrinsic::x86_avx512_mask_pmovs_qd_256:
7068 case Intrinsic::x86_avx512_mask_pmovus_qd_256:
7069 case Intrinsic::x86_avx512_mask_pmovs_wb_256:
7070 case Intrinsic::x86_avx512_mask_pmovus_wb_256:
7071 case Intrinsic::x86_avx512_mask_pmovs_qd_512:
7072 case Intrinsic::x86_avx512_mask_pmovus_qd_512:
7073 case Intrinsic::x86_avx512_mask_pmovs_wb_512:
7074 case Intrinsic::x86_avx512_mask_pmovus_wb_512: {
7075 // Since Intrinsic::x86_avx512_mask_pmov_{qd,wb}_{256,512} do not exist,
7076 // we cannot use handleIntrinsicByApplyingToShadow. Instead, we call the
7077 // slow-path handler.
7078 handleAVX512VectorDownConvert(I);
7079 break;
7080 }
7081
7082 // e.g.,
7083 // <16 x float> @llvm.x86.avx512.mask.compress
7084 // (<16 x float> %data, <16 x float> %passthru,
7085 // <16 x i1> %mask)
7086 // <16 x i32> @llvm.x86.avx512.mask.compress
7087 // (<16 x i32> %data, <16 x i32> %passthru,
7088 // <16 x i1> %mask)
7089 case Intrinsic::x86_avx512_mask_compress:
7090 handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID: I.getIntrinsicID(),
7091 /*verbatimArgIndices=*/{-1},
7092 /*forceIntegerIntrinsic=*/true);
7093 break;
7094
7095 // AVX512/AVX10 Reciprocal
7096 // <16 x float> @llvm.x86.avx512.rsqrt14.ps.512
7097 // (<16 x float>, <16 x float>, i16)
7098 // <8 x float> @llvm.x86.avx512.rsqrt14.ps.256
7099 // (<8 x float>, <8 x float>, i8)
7100 // <4 x float> @llvm.x86.avx512.rsqrt14.ps.128
7101 // (<4 x float>, <4 x float>, i8)
7102 //
7103 // <8 x double> @llvm.x86.avx512.rsqrt14.pd.512
7104 // (<8 x double>, <8 x double>, i8)
7105 // <4 x double> @llvm.x86.avx512.rsqrt14.pd.256
7106 // (<4 x double>, <4 x double>, i8)
7107 // <2 x double> @llvm.x86.avx512.rsqrt14.pd.128
7108 // (<2 x double>, <2 x double>, i8)
7109 //
7110 // <32 x bfloat> @llvm.x86.avx10.mask.rsqrt.bf16.512
7111 // (<32 x bfloat>, <32 x bfloat>, i32)
7112 // <16 x bfloat> @llvm.x86.avx10.mask.rsqrt.bf16.256
7113 // (<16 x bfloat>, <16 x bfloat>, i16)
7114 // <8 x bfloat> @llvm.x86.avx10.mask.rsqrt.bf16.128
7115 // (<8 x bfloat>, <8 x bfloat>, i8)
7116 //
7117 // <32 x half> @llvm.x86.avx512fp16.mask.rsqrt.ph.512
7118 // (<32 x half>, <32 x half>, i32)
7119 // <16 x half> @llvm.x86.avx512fp16.mask.rsqrt.ph.256
7120 // (<16 x half>, <16 x half>, i16)
7121 // <8 x half> @llvm.x86.avx512fp16.mask.rsqrt.ph.128
7122 // (<8 x half>, <8 x half>, i8)
7123 //
7124 // TODO: 3-operand variants are not handled:
7125 // <2 x double> @llvm.x86.avx512.rsqrt14.sd
7126 // (<2 x double>, <2 x double>, <2 x double>, i8)
7127 // <4 x float> @llvm.x86.avx512.rsqrt14.ss
7128 // (<4 x float>, <4 x float>, <4 x float>, i8)
7129 // <8 x half> @llvm.x86.avx512fp16.mask.rsqrt.sh
7130 // (<8 x half>, <8 x half>, <8 x half>, i8)
7131 case Intrinsic::x86_avx512_rsqrt14_ps_512:
7132 case Intrinsic::x86_avx512_rsqrt14_ps_256:
7133 case Intrinsic::x86_avx512_rsqrt14_ps_128:
7134 case Intrinsic::x86_avx512_rsqrt14_pd_512:
7135 case Intrinsic::x86_avx512_rsqrt14_pd_256:
7136 case Intrinsic::x86_avx512_rsqrt14_pd_128:
7137 case Intrinsic::x86_avx10_mask_rsqrt_bf16_512:
7138 case Intrinsic::x86_avx10_mask_rsqrt_bf16_256:
7139 case Intrinsic::x86_avx10_mask_rsqrt_bf16_128:
7140 case Intrinsic::x86_avx512fp16_mask_rsqrt_ph_512:
7141 case Intrinsic::x86_avx512fp16_mask_rsqrt_ph_256:
7142 case Intrinsic::x86_avx512fp16_mask_rsqrt_ph_128:
7143 handleAVX512VectorGenericMaskedFP(I, /*DataIndices=*/{0},
7144 /*WriteThruIndex=*/1,
7145 /*MaskIndex=*/2);
7146 break;
7147
7148 // AVX512/AVX10 Reciprocal Square Root
7149 // <16 x float> @llvm.x86.avx512.rcp14.ps.512
7150 // (<16 x float>, <16 x float>, i16)
7151 // <8 x float> @llvm.x86.avx512.rcp14.ps.256
7152 // (<8 x float>, <8 x float>, i8)
7153 // <4 x float> @llvm.x86.avx512.rcp14.ps.128
7154 // (<4 x float>, <4 x float>, i8)
7155 //
7156 // <8 x double> @llvm.x86.avx512.rcp14.pd.512
7157 // (<8 x double>, <8 x double>, i8)
7158 // <4 x double> @llvm.x86.avx512.rcp14.pd.256
7159 // (<4 x double>, <4 x double>, i8)
7160 // <2 x double> @llvm.x86.avx512.rcp14.pd.128
7161 // (<2 x double>, <2 x double>, i8)
7162 //
7163 // <32 x bfloat> @llvm.x86.avx10.mask.rcp.bf16.512
7164 // (<32 x bfloat>, <32 x bfloat>, i32)
7165 // <16 x bfloat> @llvm.x86.avx10.mask.rcp.bf16.256
7166 // (<16 x bfloat>, <16 x bfloat>, i16)
7167 // <8 x bfloat> @llvm.x86.avx10.mask.rcp.bf16.128
7168 // (<8 x bfloat>, <8 x bfloat>, i8)
7169 //
7170 // <32 x half> @llvm.x86.avx512fp16.mask.rcp.ph.512
7171 // (<32 x half>, <32 x half>, i32)
7172 // <16 x half> @llvm.x86.avx512fp16.mask.rcp.ph.256
7173 // (<16 x half>, <16 x half>, i16)
7174 // <8 x half> @llvm.x86.avx512fp16.mask.rcp.ph.128
7175 // (<8 x half>, <8 x half>, i8)
7176 //
7177 // TODO: 3-operand variants are not handled:
7178 // <2 x double> @llvm.x86.avx512.rcp14.sd
7179 // (<2 x double>, <2 x double>, <2 x double>, i8)
7180 // <4 x float> @llvm.x86.avx512.rcp14.ss
7181 // (<4 x float>, <4 x float>, <4 x float>, i8)
7182 // <8 x half> @llvm.x86.avx512fp16.mask.rcp.sh
7183 // (<8 x half>, <8 x half>, <8 x half>, i8)
7184 case Intrinsic::x86_avx512_rcp14_ps_512:
7185 case Intrinsic::x86_avx512_rcp14_ps_256:
7186 case Intrinsic::x86_avx512_rcp14_ps_128:
7187 case Intrinsic::x86_avx512_rcp14_pd_512:
7188 case Intrinsic::x86_avx512_rcp14_pd_256:
7189 case Intrinsic::x86_avx512_rcp14_pd_128:
7190 case Intrinsic::x86_avx10_mask_rcp_bf16_512:
7191 case Intrinsic::x86_avx10_mask_rcp_bf16_256:
7192 case Intrinsic::x86_avx10_mask_rcp_bf16_128:
7193 case Intrinsic::x86_avx512fp16_mask_rcp_ph_512:
7194 case Intrinsic::x86_avx512fp16_mask_rcp_ph_256:
7195 case Intrinsic::x86_avx512fp16_mask_rcp_ph_128:
7196 handleAVX512VectorGenericMaskedFP(I, /*DataIndices=*/{0},
7197 /*WriteThruIndex=*/1,
7198 /*MaskIndex=*/2);
7199 break;
7200
7201 // <32 x half> @llvm.x86.avx512fp16.mask.rndscale.ph.512
7202 // (<32 x half>, i32, <32 x half>, i32, i32)
7203 // <16 x half> @llvm.x86.avx512fp16.mask.rndscale.ph.256
7204 // (<16 x half>, i32, <16 x half>, i32, i16)
7205 // <8 x half> @llvm.x86.avx512fp16.mask.rndscale.ph.128
7206 // (<8 x half>, i32, <8 x half>, i32, i8)
7207 //
7208 // <16 x float> @llvm.x86.avx512.mask.rndscale.ps.512
7209 // (<16 x float>, i32, <16 x float>, i16, i32)
7210 // <8 x float> @llvm.x86.avx512.mask.rndscale.ps.256
7211 // (<8 x float>, i32, <8 x float>, i8)
7212 // <4 x float> @llvm.x86.avx512.mask.rndscale.ps.128
7213 // (<4 x float>, i32, <4 x float>, i8)
7214 //
7215 // <8 x double> @llvm.x86.avx512.mask.rndscale.pd.512
7216 // (<8 x double>, i32, <8 x double>, i8, i32)
7217 // A Imm WriteThru Mask Rounding
7218 // <4 x double> @llvm.x86.avx512.mask.rndscale.pd.256
7219 // (<4 x double>, i32, <4 x double>, i8)
7220 // <2 x double> @llvm.x86.avx512.mask.rndscale.pd.128
7221 // (<2 x double>, i32, <2 x double>, i8)
7222 // A Imm WriteThru Mask
7223 //
7224 // <32 x bfloat> @llvm.x86.avx10.mask.rndscale.bf16.512
7225 // (<32 x bfloat>, i32, <32 x bfloat>, i32)
7226 // <16 x bfloat> @llvm.x86.avx10.mask.rndscale.bf16.256
7227 // (<16 x bfloat>, i32, <16 x bfloat>, i16)
7228 // <8 x bfloat> @llvm.x86.avx10.mask.rndscale.bf16.128
7229 // (<8 x bfloat>, i32, <8 x bfloat>, i8)
7230 //
7231 // Not supported: three vectors
7232 // - <8 x half> @llvm.x86.avx512fp16.mask.rndscale.sh
7233 // (<8 x half>, <8 x half>,<8 x half>, i8, i32, i32)
7234 // - <4 x float> @llvm.x86.avx512.mask.rndscale.ss
7235 // (<4 x float>, <4 x float>, <4 x float>, i8, i32, i32)
7236 // - <2 x double> @llvm.x86.avx512.mask.rndscale.sd
7237 // (<2 x double>, <2 x double>, <2 x double>, i8, i32,
7238 // i32)
7239 // A B WriteThru Mask Imm
7240 // Rounding
7241 case Intrinsic::x86_avx512fp16_mask_rndscale_ph_512:
7242 case Intrinsic::x86_avx512fp16_mask_rndscale_ph_256:
7243 case Intrinsic::x86_avx512fp16_mask_rndscale_ph_128:
7244 case Intrinsic::x86_avx512_mask_rndscale_ps_512:
7245 case Intrinsic::x86_avx512_mask_rndscale_ps_256:
7246 case Intrinsic::x86_avx512_mask_rndscale_ps_128:
7247 case Intrinsic::x86_avx512_mask_rndscale_pd_512:
7248 case Intrinsic::x86_avx512_mask_rndscale_pd_256:
7249 case Intrinsic::x86_avx512_mask_rndscale_pd_128:
7250 case Intrinsic::x86_avx10_mask_rndscale_bf16_512:
7251 case Intrinsic::x86_avx10_mask_rndscale_bf16_256:
7252 case Intrinsic::x86_avx10_mask_rndscale_bf16_128:
7253 handleAVX512VectorGenericMaskedFP(I, /*DataIndices=*/{0},
7254 /*WriteThruIndex=*/2,
7255 /*MaskIndex=*/3);
7256 break;
7257
7258 // AVX512 Vector Scale Float* Packed
7259 //
7260 // < 8 x double> @llvm.x86.avx512.mask.scalef.pd.512
7261 // (<8 x double>, <8 x double>, <8 x double>, i8, i32)
7262 // A B WriteThru Msk Round
7263 // < 4 x double> @llvm.x86.avx512.mask.scalef.pd.256
7264 // (<4 x double>, <4 x double>, <4 x double>, i8)
7265 // < 2 x double> @llvm.x86.avx512.mask.scalef.pd.128
7266 // (<2 x double>, <2 x double>, <2 x double>, i8)
7267 //
7268 // <16 x float> @llvm.x86.avx512.mask.scalef.ps.512
7269 // (<16 x float>, <16 x float>, <16 x float>, i16, i32)
7270 // < 8 x float> @llvm.x86.avx512.mask.scalef.ps.256
7271 // (<8 x float>, <8 x float>, <8 x float>, i8)
7272 // < 4 x float> @llvm.x86.avx512.mask.scalef.ps.128
7273 // (<4 x float>, <4 x float>, <4 x float>, i8)
7274 //
7275 // <32 x half> @llvm.x86.avx512fp16.mask.scalef.ph.512
7276 // (<32 x half>, <32 x half>, <32 x half>, i32, i32)
7277 // <16 x half> @llvm.x86.avx512fp16.mask.scalef.ph.256
7278 // (<16 x half>, <16 x half>, <16 x half>, i16)
7279 // < 8 x half> @llvm.x86.avx512fp16.mask.scalef.ph.128
7280 // (<8 x half>, <8 x half>, <8 x half>, i8)
7281 //
7282 // TODO: AVX10
7283 // <32 x bfloat> @llvm.x86.avx10.mask.scalef.bf16.512
7284 // (<32 x bfloat>, <32 x bfloat>, <32 x bfloat>, i32)
7285 // <16 x bfloat> @llvm.x86.avx10.mask.scalef.bf16.256
7286 // (<16 x bfloat>, <16 x bfloat>, <16 x bfloat>, i16)
7287 // < 8 x bfloat> @llvm.x86.avx10.mask.scalef.bf16.128
7288 // (<8 x bfloat>, <8 x bfloat>, <8 x bfloat>, i8)
7289 case Intrinsic::x86_avx512_mask_scalef_pd_512:
7290 case Intrinsic::x86_avx512_mask_scalef_pd_256:
7291 case Intrinsic::x86_avx512_mask_scalef_pd_128:
7292 case Intrinsic::x86_avx512_mask_scalef_ps_512:
7293 case Intrinsic::x86_avx512_mask_scalef_ps_256:
7294 case Intrinsic::x86_avx512_mask_scalef_ps_128:
7295 case Intrinsic::x86_avx512fp16_mask_scalef_ph_512:
7296 case Intrinsic::x86_avx512fp16_mask_scalef_ph_256:
7297 case Intrinsic::x86_avx512fp16_mask_scalef_ph_128:
7298 // The AVX512 512-bit operand variants have an extra operand (the
7299 // Rounding mode). The extra operand, if present, will be
7300 // automatically checked by the handler.
7301 handleAVX512VectorGenericMaskedFP(I, /*DataIndices=*/{0, 1},
7302 /*WriteThruIndex=*/2,
7303 /*MaskIndex=*/3);
7304 break;
7305
7306 // TODO: AVX512 Vector Scale Float* Scalar
7307 //
7308 // This is different from the Packed variant, because some bits are copied,
7309 // and some bits are zeroed.
7310 //
7311 // < 4 x float> @llvm.x86.avx512.mask.scalef.ss
7312 // (<4 x float>, <4 x float>, <4 x float>, i8, i32)
7313 //
7314 // < 2 x double> @llvm.x86.avx512.mask.scalef.sd
7315 // (<2 x double>, <2 x double>, <2 x double>, i8, i32)
7316 //
7317 // < 8 x half> @llvm.x86.avx512fp16.mask.scalef.sh
7318 // (<8 x half>, <8 x half>, <8 x half>, i8, i32)
7319
7320 // AVX512 FP16 Arithmetic
7321 case Intrinsic::x86_avx512fp16_mask_add_sh_round:
7322 case Intrinsic::x86_avx512fp16_mask_sub_sh_round:
7323 case Intrinsic::x86_avx512fp16_mask_mul_sh_round:
7324 case Intrinsic::x86_avx512fp16_mask_div_sh_round:
7325 case Intrinsic::x86_avx512fp16_mask_max_sh_round:
7326 case Intrinsic::x86_avx512fp16_mask_min_sh_round: {
7327 visitGenericScalarHalfwordInst(I);
7328 break;
7329 }
7330
7331 // AVX512 Floating-Point Classification
7332 // - <8 x i1> @llvm.x86.avx512.fpclass.pd.512(<8 x double>, i32)
7333 // - <16 x i1> @llvm.x86.avx512.fpclass.ps.512(<16 x float>, i32)
7334 case Intrinsic::x86_avx512_fpclass_pd_512:
7335 case Intrinsic::x86_avx512_fpclass_ps_512:
7336 handleAVX512FPClass(I);
7337 break;
7338
7339 // AVX Galois Field New Instructions
7340 case Intrinsic::x86_vgf2p8affineqb_128:
7341 case Intrinsic::x86_vgf2p8affineqb_256:
7342 case Intrinsic::x86_vgf2p8affineqb_512:
7343 handleAVXGF2P8Affine(I);
7344 break;
7345
7346 default:
7347 return false;
7348 }
7349
7350 return true;
7351 }
7352
7353 bool maybeHandleArmSIMDIntrinsic(IntrinsicInst &I) {
7354 switch (I.getIntrinsicID()) {
7355 // Two operands e.g.,
7356 // - <8 x i8> @llvm.aarch64.neon.rshrn.v8i8 (<8 x i16>, i32)
7357 // - <4 x i16> @llvm.aarch64.neon.uqrshl.v4i16(<4 x i16>, <4 x i16>)
7358 case Intrinsic::aarch64_neon_rshrn:
7359 case Intrinsic::aarch64_neon_sqrshl:
7360 case Intrinsic::aarch64_neon_sqrshrn:
7361 case Intrinsic::aarch64_neon_sqrshrun:
7362 case Intrinsic::aarch64_neon_sqshl:
7363 case Intrinsic::aarch64_neon_sqshlu:
7364 case Intrinsic::aarch64_neon_sqshrn:
7365 case Intrinsic::aarch64_neon_sqshrun:
7366 case Intrinsic::aarch64_neon_srshl:
7367 case Intrinsic::aarch64_neon_sshl:
7368 case Intrinsic::aarch64_neon_uqrshl:
7369 case Intrinsic::aarch64_neon_uqrshrn:
7370 case Intrinsic::aarch64_neon_uqshl:
7371 case Intrinsic::aarch64_neon_uqshrn:
7372 case Intrinsic::aarch64_neon_urshl:
7373 case Intrinsic::aarch64_neon_ushl:
7374 handleVectorShiftIntrinsic(I, /* Variable */ false);
7375 break;
7376
7377 // Vector Shift Left/Right and Insert
7378 //
7379 // Three operands e.g.,
7380 // - <4 x i16> @llvm.aarch64.neon.vsli.v4i16
7381 // (<4 x i16> %a, <4 x i16> %b, i32 %n)
7382 // - <16 x i8> @llvm.aarch64.neon.vsri.v16i8
7383 // (<16 x i8> %a, <16 x i8> %b, i32 %n)
7384 //
7385 // %b is shifted by %n bits, and the "missing" bits are filled in with %a
7386 // (instead of zero-extending/sign-extending).
7387 case Intrinsic::aarch64_neon_vsli:
7388 case Intrinsic::aarch64_neon_vsri:
7389 handleIntrinsicByApplyingToShadow(I, shadowIntrinsicID: I.getIntrinsicID(),
7390 /*verbatimArgIndices=*/{-1},
7391 /*forceIntegerIntrinsic=*/false);
7392 break;
7393
7394 // TODO: handling max/min similarly to AND/OR may be more precise
7395 // Floating-Point Maximum/Minimum Pairwise
7396 case Intrinsic::aarch64_neon_fmaxp:
7397 case Intrinsic::aarch64_neon_fminp:
7398 // Floating-Point Maximum/Minimum Number Pairwise
7399 case Intrinsic::aarch64_neon_fmaxnmp:
7400 case Intrinsic::aarch64_neon_fminnmp:
7401 // Signed/Unsigned Maximum/Minimum Pairwise
7402 case Intrinsic::aarch64_neon_smaxp:
7403 case Intrinsic::aarch64_neon_sminp:
7404 case Intrinsic::aarch64_neon_umaxp:
7405 case Intrinsic::aarch64_neon_uminp:
7406 // Add Pairwise
7407 case Intrinsic::aarch64_neon_addp:
7408 // Floating-point Add Pairwise
7409 case Intrinsic::aarch64_neon_faddp:
7410 // Add Long Pairwise
7411 case Intrinsic::aarch64_neon_saddlp:
7412 case Intrinsic::aarch64_neon_uaddlp: {
7413 handlePairwiseShadowOrIntrinsic(I, /*Shards=*/1);
7414 break;
7415 }
7416
7417 // Floating-point Convert to integer, rounding to nearest with ties to Away
7418 case Intrinsic::aarch64_neon_fcvtas:
7419 case Intrinsic::aarch64_neon_fcvtau:
7420 // Floating-point convert to integer, rounding toward minus infinity
7421 case Intrinsic::aarch64_neon_fcvtms:
7422 case Intrinsic::aarch64_neon_fcvtmu:
7423 // Floating-point convert to integer, rounding to nearest with ties to even
7424 case Intrinsic::aarch64_neon_fcvtns:
7425 case Intrinsic::aarch64_neon_fcvtnu:
7426 // Floating-point convert to integer, rounding toward plus infinity
7427 case Intrinsic::aarch64_neon_fcvtps:
7428 case Intrinsic::aarch64_neon_fcvtpu:
7429 // Floating-point Convert to integer, rounding toward Zero
7430 case Intrinsic::aarch64_neon_fcvtzs:
7431 case Intrinsic::aarch64_neon_fcvtzu:
7432 // Floating-point convert to lower precision narrow, rounding to odd
7433 case Intrinsic::aarch64_neon_fcvtxn:
7434 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/false);
7435 break;
7436
7437 // Vector Conversions Between Fixed-Point and Floating-Point
7438 case Intrinsic::aarch64_neon_vcvtfxs2fp:
7439 case Intrinsic::aarch64_neon_vcvtfp2fxs:
7440 case Intrinsic::aarch64_neon_vcvtfxu2fp:
7441 case Intrinsic::aarch64_neon_vcvtfp2fxu:
7442 handleGenericVectorConvertIntrinsic(I, /*FixedPoint=*/true);
7443 break;
7444
7445 // TODO: bfloat conversions
7446 // - bfloat @llvm.aarch64.neon.bfcvt(float)
7447 // - <8 x bfloat> @llvm.aarch64.neon.bfcvtn(<4 x float>)
7448 // - <8 x bfloat> @llvm.aarch64.neon.bfcvtn2(<8 x bfloat>, <4 x float>)
7449
7450 // Add reduction to scalar
7451 case Intrinsic::aarch64_neon_faddv:
7452 case Intrinsic::aarch64_neon_saddv:
7453 case Intrinsic::aarch64_neon_uaddv:
7454 // Signed/Unsigned min/max (Vector)
7455 // TODO: handling similarly to AND/OR may be more precise.
7456 case Intrinsic::aarch64_neon_smaxv:
7457 case Intrinsic::aarch64_neon_sminv:
7458 case Intrinsic::aarch64_neon_umaxv:
7459 case Intrinsic::aarch64_neon_uminv:
7460 // Floating-point min/max (vector)
7461 // The f{min,max}"nm"v variants handle NaN differently than f{min,max}v,
7462 // but our shadow propagation is the same.
7463 case Intrinsic::aarch64_neon_fmaxv:
7464 case Intrinsic::aarch64_neon_fminv:
7465 case Intrinsic::aarch64_neon_fmaxnmv:
7466 case Intrinsic::aarch64_neon_fminnmv:
7467 // Sum long across vector
7468 case Intrinsic::aarch64_neon_saddlv:
7469 case Intrinsic::aarch64_neon_uaddlv:
7470 handleVectorReduceIntrinsic(I, /*AllowShadowCast=*/true);
7471 break;
7472
7473 case Intrinsic::aarch64_neon_ld1x2:
7474 case Intrinsic::aarch64_neon_ld1x3:
7475 case Intrinsic::aarch64_neon_ld1x4:
7476 case Intrinsic::aarch64_neon_ld2:
7477 case Intrinsic::aarch64_neon_ld3:
7478 case Intrinsic::aarch64_neon_ld4:
7479 case Intrinsic::aarch64_neon_ld2r:
7480 case Intrinsic::aarch64_neon_ld3r:
7481 case Intrinsic::aarch64_neon_ld4r: {
7482 handleNEONVectorLoad(I, /*WithLane=*/false);
7483 break;
7484 }
7485
7486 case Intrinsic::aarch64_neon_ld2lane:
7487 case Intrinsic::aarch64_neon_ld3lane:
7488 case Intrinsic::aarch64_neon_ld4lane: {
7489 handleNEONVectorLoad(I, /*WithLane=*/true);
7490 break;
7491 }
7492
7493 // Saturating extract narrow
7494 case Intrinsic::aarch64_neon_sqxtn:
7495 case Intrinsic::aarch64_neon_sqxtun:
7496 case Intrinsic::aarch64_neon_uqxtn:
7497 // These only have one argument, but we (ab)use handleShadowOr because it
7498 // does work on single argument intrinsics and will typecast the shadow
7499 // (and update the origin).
7500 handleShadowOr(I);
7501 break;
7502
7503 case Intrinsic::aarch64_neon_st1x2:
7504 case Intrinsic::aarch64_neon_st1x3:
7505 case Intrinsic::aarch64_neon_st1x4:
7506 case Intrinsic::aarch64_neon_st2:
7507 case Intrinsic::aarch64_neon_st3:
7508 case Intrinsic::aarch64_neon_st4: {
7509 handleNEONVectorStoreIntrinsic(I, useLane: false);
7510 break;
7511 }
7512
7513 case Intrinsic::aarch64_neon_st2lane:
7514 case Intrinsic::aarch64_neon_st3lane:
7515 case Intrinsic::aarch64_neon_st4lane: {
7516 handleNEONVectorStoreIntrinsic(I, useLane: true);
7517 break;
7518 }
7519
7520 // Arm NEON vector table intrinsics have the source/table register(s) as
7521 // arguments, followed by the index register. They return the output.
7522 //
7523 // 'TBL writes a zero if an index is out-of-range, while TBX leaves the
7524 // original value unchanged in the destination register.'
7525 // Conveniently, zero denotes a clean shadow, which means out-of-range
7526 // indices for TBL will initialize the user data with zero and also clean
7527 // the shadow. (For TBX, neither the user data nor the shadow will be
7528 // updated, which is also correct.)
7529 case Intrinsic::aarch64_neon_tbl1:
7530 case Intrinsic::aarch64_neon_tbl2:
7531 case Intrinsic::aarch64_neon_tbl3:
7532 case Intrinsic::aarch64_neon_tbl4:
7533 case Intrinsic::aarch64_neon_tbx1:
7534 case Intrinsic::aarch64_neon_tbx2:
7535 case Intrinsic::aarch64_neon_tbx3:
7536 case Intrinsic::aarch64_neon_tbx4: {
7537 // The last trailing argument (index register) should be handled verbatim
7538 handleIntrinsicByApplyingToShadow(
7539 I, /*shadowIntrinsicID=*/I.getIntrinsicID(),
7540 /*verbatimArgIndices=*/{-1}, /*forceIntegerIntrinsic=*/false);
7541 break;
7542 }
7543
7544 case Intrinsic::aarch64_neon_fmulx:
7545 case Intrinsic::aarch64_neon_pmul:
7546 case Intrinsic::aarch64_neon_pmull:
7547 case Intrinsic::aarch64_neon_smull:
7548 case Intrinsic::aarch64_neon_pmull64:
7549 case Intrinsic::aarch64_neon_umull: {
7550 handleNEONVectorMultiplyIntrinsic(I);
7551 break;
7552 }
7553
7554 case Intrinsic::aarch64_neon_smmla:
7555 case Intrinsic::aarch64_neon_ummla:
7556 case Intrinsic::aarch64_neon_usmmla:
7557 case Intrinsic::aarch64_neon_bfmmla:
7558 handleNEONMatrixMultiply(I);
7559 break;
7560
7561 // <2 x i32> @llvm.aarch64.neon.{u,s,us}dot.v2i32.v8i8
7562 // (<2 x i32> %acc, <8 x i8> %a, <8 x i8> %b)
7563 // <4 x i32> @llvm.aarch64.neon.{u,s,us}dot.v4i32.v16i8
7564 // (<4 x i32> %acc, <16 x i8> %a, <16 x i8> %b)
7565 case Intrinsic::aarch64_neon_sdot:
7566 case Intrinsic::aarch64_neon_udot:
7567 case Intrinsic::aarch64_neon_usdot:
7568 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/4,
7569 /*ZeroPurifies=*/true,
7570 /*EltSizeInBits=*/0,
7571 /*Lanes=*/kBothLanes);
7572 break;
7573
7574 // <2 x float> @llvm.aarch64.neon.bfdot.v2f32.v4bf16
7575 // (<2 x float> %acc, <4 x bfloat> %a, <4 x bfloat> %b)
7576 // <4 x float> @llvm.aarch64.neon.bfdot.v4f32.v8bf16
7577 // (<4 x float> %acc, <8 x bfloat> %a, <8 x bfloat> %b)
7578 case Intrinsic::aarch64_neon_bfdot:
7579 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
7580 /*ZeroPurifies=*/false,
7581 /*EltSizeInBits=*/0,
7582 /*Lanes=*/kBothLanes);
7583 break;
7584
7585 // <4 x half > @llvm.aarch64.neon.fp8.fdot2
7586 // (<4 x half >, < 8 x i8>, < 8 x i8>)
7587 // <8 x half > @llvm.aarch64.neon.fp8.fdot2
7588 // (<8 x half >, <16 x i8>, <16 x i8>)
7589 //
7590 // N.B. although the multiplicands are i8, they are actually fp8, thus
7591 // ZeroPurifies is not applicable.
7592 case Intrinsic::aarch64_neon_fp8_fdot2:
7593 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/2,
7594 /*ZeroPurifies=*/false,
7595 /*EltSizeInBits=*/0,
7596 /*Lanes=*/kBothLanes);
7597 break;
7598
7599 // <2 x float> @llvm.aarch64.neon.fp8.fdot4
7600 // (<2 x float>, < 8 x i8>, < 8 x i8>)
7601 // <4 x float> @llvm.aarch64.neon.fp8.fdot4
7602 // (<4 x float>, <16 x i8>, <16 x i8>)
7603 //
7604 // N.B. although the multiplicands are i8, they are actually fp8, thus
7605 // ZeroPurifies is not applicable.
7606 case Intrinsic::aarch64_neon_fp8_fdot4:
7607 handleVectorDotProductIntrinsic(I, /*ReductionFactor=*/4,
7608 /*ZeroPurifies=*/false,
7609 /*EltSizeInBits=*/0,
7610 /*Lanes=*/kBothLanes);
7611 break;
7612
7613 // <4 x half> @llvm.aarch64.neon.fp8.fdot2.lane
7614 // (<4 x half>, <8 x i8>, <16 x i8>, i32)
7615 // <8 x half> @llvm.aarch64.neon.fp8.fdot2.lane
7616 // (<8 x half>, <16 x i8>, <16 x i8>, i32)
7617 case Intrinsic::aarch64_neon_fp8_fdot2_lane:
7618 handleNEONDotProductLaneIntrinsic(I, /*ReductionFactor=*/2);
7619 break;
7620
7621 // <2 x float> @llvm.aarch64.neon.fp8.fdot4.lane
7622 // (<2 x float>, <8 x i8>, <16 x i8>, i32)
7623 // <4 x float> @llvm.aarch64.neon.fp8.fdot4.lane
7624 // (<4 x float>, <16 x i8>, <16 x i8>, i32)
7625 case Intrinsic::aarch64_neon_fp8_fdot4_lane:
7626 handleNEONDotProductLaneIntrinsic(I, /*ReductionFactor=*/4);
7627 break;
7628
7629 // Floating-Point Absolute Compare Greater Than/Equal
7630 case Intrinsic::aarch64_neon_facge:
7631 case Intrinsic::aarch64_neon_facgt:
7632 handleVectorComparePackedIntrinsic(I, /*PredicateAsOperand=*/false);
7633 break;
7634
7635 default:
7636 return false;
7637 }
7638
7639 return true;
7640 }
7641
7642 void visitIntrinsicInst(IntrinsicInst &I) {
7643 if (maybeHandleCrossPlatformIntrinsic(I))
7644 return;
7645
7646 if (maybeHandleX86SIMDIntrinsic(I))
7647 return;
7648
7649 if (maybeHandleArmSIMDIntrinsic(I))
7650 return;
7651
7652 if (maybeHandleUnknownIntrinsic(I))
7653 return;
7654
7655 visitInstruction(I);
7656 }
7657
7658 void visitLibAtomicLoad(CallBase &CB) {
7659 // Since we use getNextNode here, we can't have CB terminate the BB.
7660 assert(isa<CallInst>(CB));
7661
7662 IRBuilder<> IRB(&CB);
7663 Value *Size = CB.getArgOperand(i: 0);
7664 Value *SrcPtr = CB.getArgOperand(i: 1);
7665 Value *DstPtr = CB.getArgOperand(i: 2);
7666 Value *Ordering = CB.getArgOperand(i: 3);
7667 // Convert the call to have at least Acquire ordering to make sure
7668 // the shadow operations aren't reordered before it.
7669 Value *NewOrdering =
7670 IRB.CreateExtractElement(Vec: makeAddAcquireOrderingTable(IRB), Idx: Ordering);
7671 CB.setArgOperand(i: 3, v: NewOrdering);
7672
7673 NextNodeIRBuilder NextIRB(&CB);
7674 Value *SrcShadowPtr, *SrcOriginPtr;
7675 std::tie(args&: SrcShadowPtr, args&: SrcOriginPtr) =
7676 getShadowOriginPtr(Addr: SrcPtr, IRB&: NextIRB, ShadowTy: NextIRB.getInt8Ty(), Alignment: Align(1),
7677 /*isStore*/ false);
7678 Value *DstShadowPtr =
7679 getShadowOriginPtr(Addr: DstPtr, IRB&: NextIRB, ShadowTy: NextIRB.getInt8Ty(), Alignment: Align(1),
7680 /*isStore*/ true)
7681 .first;
7682
7683 NextIRB.CreateMemCpy(Dst: DstShadowPtr, DstAlign: Align(1), Src: SrcShadowPtr, SrcAlign: Align(1), Size);
7684 if (MS.TrackOrigins) {
7685 Value *SrcOrigin = NextIRB.CreateAlignedLoad(Ty: MS.OriginTy, Ptr: SrcOriginPtr,
7686 Align: kMinOriginAlignment);
7687 Value *NewOrigin = updateOrigin(V: SrcOrigin, IRB&: NextIRB);
7688 NextIRB.CreateCall(Callee: MS.MsanSetOriginFn, Args: {DstPtr, Size, NewOrigin});
7689 }
7690 }
7691
7692 void visitLibAtomicStore(CallBase &CB) {
7693 IRBuilder<> IRB(&CB);
7694 Value *Size = CB.getArgOperand(i: 0);
7695 Value *DstPtr = CB.getArgOperand(i: 2);
7696 Value *Ordering = CB.getArgOperand(i: 3);
7697 // Convert the call to have at least Release ordering to make sure
7698 // the shadow operations aren't reordered after it.
7699 Value *NewOrdering =
7700 IRB.CreateExtractElement(Vec: makeAddReleaseOrderingTable(IRB), Idx: Ordering);
7701 CB.setArgOperand(i: 3, v: NewOrdering);
7702
7703 Value *DstShadowPtr =
7704 getShadowOriginPtr(Addr: DstPtr, IRB, ShadowTy: IRB.getInt8Ty(), Alignment: Align(1),
7705 /*isStore*/ true)
7706 .first;
7707
7708 // Atomic store always paints clean shadow/origin. See file header.
7709 IRB.CreateMemSet(Ptr: DstShadowPtr, Val: getCleanShadow(OrigTy: IRB.getInt8Ty()), Size,
7710 Align: Align(1));
7711 }
7712
7713 void visitCallBase(CallBase &CB) {
7714 assert(!CB.getMetadata(LLVMContext::MD_nosanitize));
7715 if (CB.isInlineAsm()) {
7716 // For inline asm (either a call to asm function, or callbr instruction),
7717 // do the usual thing: check argument shadow and mark all outputs as
7718 // clean. Note that any side effects of the inline asm that are not
7719 // immediately visible in its constraints are not handled.
7720 if (Opts.msan_handle_asm_conservative)
7721 visitAsmInstruction(I&: CB);
7722 else
7723 visitInstruction(I&: CB);
7724 return;
7725 }
7726 LibFunc LF = TLI->getLibFunc(CB);
7727 if (LF != NotLibFunc) {
7728 // libatomic.a functions need to have special handling because there isn't
7729 // a good way to intercept them or compile the library with
7730 // instrumentation.
7731 switch (LF) {
7732 case LibFunc_atomic_load:
7733 if (!isa<CallInst>(Val: CB)) {
7734 llvm::errs() << "MSAN -- cannot instrument invoke of libatomic load."
7735 "Ignoring!\n";
7736 break;
7737 }
7738 visitLibAtomicLoad(CB);
7739 return;
7740 case LibFunc_atomic_store:
7741 visitLibAtomicStore(CB);
7742 return;
7743 default:
7744 break;
7745 }
7746 }
7747
7748 if (auto *Call = dyn_cast<CallInst>(Val: &CB)) {
7749 assert(!isa<IntrinsicInst>(Call) && "intrinsics are handled elsewhere");
7750
7751 // We are going to insert code that relies on the fact that the callee
7752 // will become a non-readonly function after it is instrumented by us. To
7753 // prevent this code from being optimized out, mark that function
7754 // non-readonly in advance.
7755 // TODO: We can likely do better than dropping memory() completely here.
7756 AttributeMask B;
7757 B.addAttribute(Val: Attribute::Memory).addAttribute(Val: Attribute::Speculatable);
7758
7759 Call->removeFnAttrs(AttrsToRemove: B);
7760 if (Function *Func = Call->getCalledFunction()) {
7761 Func->removeFnAttrs(Attrs: B);
7762 }
7763
7764 maybeMarkSanitizerLibraryCallNoBuiltin(CI: Call, TLI);
7765 }
7766 IRBuilder<> IRB(&CB);
7767 bool MayCheckCall = MS.EagerChecks;
7768 if (Function *Func = CB.getCalledFunction()) {
7769 // __sanitizer_unaligned_{load,store} functions may be called by users
7770 // and always expects shadows in the TLS. So don't check them.
7771 MayCheckCall &= !Func->getName().starts_with(Prefix: "__sanitizer_unaligned_");
7772 }
7773
7774 unsigned ArgOffset = 0;
7775 LLVM_DEBUG(dbgs() << " CallSite: " << CB << "\n");
7776 for (const auto &[i, A] : llvm::enumerate(First: CB.args())) {
7777 if (!A->getType()->isSized()) {
7778 LLVM_DEBUG(dbgs() << "Arg " << i << " is not sized: " << CB << "\n");
7779 continue;
7780 }
7781
7782 if (A->getType()->isScalableTy()) {
7783 LLVM_DEBUG(dbgs() << "Arg " << i << " is vscale: " << CB << "\n");
7784 // Handle as noundef, but don't reserve tls slots.
7785 insertCheckShadowOf(Val: A, OrigIns: &CB);
7786 continue;
7787 }
7788
7789 unsigned Size = 0;
7790 const DataLayout &DL = F.getDataLayout();
7791
7792 bool ByVal = CB.isByValArgument(ArgNo: i);
7793 bool NoUndef = CB.paramHasAttr(ArgNo: i, Kind: Attribute::NoUndef);
7794 bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
7795
7796 if (EagerCheck) {
7797 insertCheckShadowOf(Val: A, OrigIns: &CB);
7798 Size = DL.getTypeAllocSize(Ty: A->getType());
7799 } else {
7800 [[maybe_unused]] Value *Store = nullptr;
7801 // Compute the Shadow for arg even if it is ByVal, because
7802 // in that case getShadow() will copy the actual arg shadow to
7803 // __msan_param_tls.
7804 Value *ArgShadow = getShadow(V: A);
7805 Value *ArgShadowBase = getShadowPtrForArgument(IRB, ArgOffset);
7806 LLVM_DEBUG(dbgs() << " Arg#" << i << ": " << *A
7807 << " Shadow: " << *ArgShadow << "\n");
7808 if (ByVal) {
7809 // ByVal requires some special handling as it's too big for a single
7810 // load
7811 assert(A->getType()->isPointerTy() &&
7812 "ByVal argument is not a pointer!");
7813 Size = DL.getTypeAllocSize(Ty: CB.getParamByValType(ArgNo: i));
7814 if (ArgOffset + Size > kParamTLSSize)
7815 break;
7816 const MaybeAlign ParamAlignment(CB.getParamAlign(ArgNo: i));
7817 MaybeAlign Alignment = std::nullopt;
7818 if (ParamAlignment)
7819 Alignment = std::min(a: *ParamAlignment, b: kShadowTLSAlignment);
7820 Value *AShadowPtr, *AOriginPtr;
7821 std::tie(args&: AShadowPtr, args&: AOriginPtr) =
7822 getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(), Alignment,
7823 /*isStore*/ false);
7824 if (!PropagateShadow) {
7825 Store = IRB.CreateMemSet(Ptr: ArgShadowBase,
7826 Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
7827 Size, Align: Alignment);
7828 } else {
7829 Store = IRB.CreateMemCpy(Dst: ArgShadowBase, DstAlign: Alignment, Src: AShadowPtr,
7830 SrcAlign: Alignment, Size);
7831 if (MS.TrackOrigins) {
7832 Value *ArgOriginBase = getOriginPtrForArgument(IRB, ArgOffset);
7833 // FIXME: OriginSize should be:
7834 // alignTo(A % kMinOriginAlignment + Size, kMinOriginAlignment)
7835 unsigned OriginSize = alignTo(Size, A: kMinOriginAlignment);
7836 IRB.CreateMemCpy(
7837 Dst: ArgOriginBase,
7838 /* by origin_tls[ArgOffset] */ DstAlign: kMinOriginAlignment,
7839 Src: AOriginPtr,
7840 /* by getShadowOriginPtr */ SrcAlign: kMinOriginAlignment, Size: OriginSize);
7841 }
7842 }
7843 } else {
7844 // Any other parameters mean we need bit-grained tracking of uninit
7845 // data
7846 Size = DL.getTypeAllocSize(Ty: A->getType());
7847 if (ArgOffset + Size > kParamTLSSize)
7848 break;
7849 Store = IRB.CreateAlignedStore(Val: ArgShadow, Ptr: ArgShadowBase,
7850 Align: kShadowTLSAlignment);
7851 Constant *Cst = dyn_cast<Constant>(Val: ArgShadow);
7852 if (MS.TrackOrigins && !(Cst && Cst->isNullValue())) {
7853 IRB.CreateStore(Val: getOrigin(V: A),
7854 Ptr: getOriginPtrForArgument(IRB, ArgOffset));
7855 }
7856 }
7857 assert(Store != nullptr);
7858 LLVM_DEBUG(dbgs() << " Param:" << *Store << "\n");
7859 }
7860 assert(Size != 0);
7861 ArgOffset += alignTo(Size, A: kShadowTLSAlignment);
7862 }
7863 LLVM_DEBUG(dbgs() << " done with call args\n");
7864
7865 FunctionType *FT = CB.getFunctionType();
7866 if (FT->isVarArg()) {
7867 VAHelper->visitCallBase(CB, IRB);
7868 }
7869
7870 // Now, get the shadow for the RetVal.
7871 if (!CB.getType()->isSized())
7872 return;
7873 // Don't emit the epilogue for musttail call returns.
7874 if (isa<CallInst>(Val: CB) && cast<CallInst>(Val&: CB).isMustTailCall())
7875 return;
7876
7877 if (MayCheckCall && CB.hasRetAttr(Kind: Attribute::NoUndef)) {
7878 setShadow(V: &CB, SV: getCleanShadow(V: &CB));
7879 setOrigin(V: &CB, Origin: getCleanOrigin());
7880 return;
7881 }
7882
7883 IRBuilder<> IRBBefore(&CB);
7884 // Until we have full dynamic coverage, make sure the retval shadow is 0.
7885 Value *Base = getShadowPtrForRetval(IRB&: IRBBefore);
7886 IRBBefore.CreateAlignedStore(Val: getCleanShadow(V: &CB), Ptr: Base,
7887 Align: kShadowTLSAlignment);
7888 BasicBlock::iterator NextInsn;
7889 if (isa<CallInst>(Val: CB)) {
7890 NextInsn = ++CB.getIterator();
7891 assert(NextInsn != CB.getParent()->end());
7892 } else {
7893 BasicBlock *NormalDest = cast<InvokeInst>(Val&: CB).getNormalDest();
7894 if (!NormalDest->getSinglePredecessor()) {
7895 // FIXME: this case is tricky, so we are just conservative here.
7896 // Perhaps we need to split the edge between this BB and NormalDest,
7897 // but a naive attempt to use SplitEdge leads to a crash.
7898 setShadow(V: &CB, SV: getCleanShadow(V: &CB));
7899 setOrigin(V: &CB, Origin: getCleanOrigin());
7900 return;
7901 }
7902 // FIXME: NextInsn is likely in a basic block that has not been visited
7903 // yet. Anything inserted there will be instrumented by MSan later!
7904 NextInsn = NormalDest->getFirstInsertionPt();
7905 assert(NextInsn != NormalDest->end() &&
7906 "Could not find insertion point for retval shadow load");
7907 }
7908 IRBuilder<> IRBAfter(&*NextInsn);
7909 Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
7910 Ty: getShadowTy(V: &CB), Ptr: getShadowPtrForRetval(IRB&: IRBAfter), Align: kShadowTLSAlignment,
7911 Name: "_msret");
7912 setShadow(V: &CB, SV: RetvalShadow);
7913 if (MS.TrackOrigins)
7914 setOrigin(V: &CB, Origin: IRBAfter.CreateLoad(Ty: MS.OriginTy, Ptr: getOriginPtrForRetval()));
7915 }
7916
7917 bool isAMustTailRetVal(Value *RetVal) {
7918 if (auto *I = dyn_cast<BitCastInst>(Val: RetVal)) {
7919 RetVal = I->getOperand(i_nocapture: 0);
7920 }
7921 if (auto *I = dyn_cast<CallInst>(Val: RetVal)) {
7922 return I->isMustTailCall();
7923 }
7924 return false;
7925 }
7926
7927 void visitReturnInst(ReturnInst &I) {
7928 IRBuilder<> IRB(&I);
7929 Value *RetVal = I.getReturnValue();
7930 if (!RetVal)
7931 return;
7932 // Don't emit the epilogue for musttail call returns.
7933 if (isAMustTailRetVal(RetVal))
7934 return;
7935 Value *ShadowPtr = getShadowPtrForRetval(IRB);
7936 bool HasNoUndef = F.hasRetAttribute(Kind: Attribute::NoUndef);
7937 bool StoreShadow = !(MS.EagerChecks && HasNoUndef);
7938 // FIXME: Consider using SpecialCaseList to specify a list of functions that
7939 // must always return fully initialized values. For now, we hardcode "main".
7940 bool EagerCheck = (MS.EagerChecks && HasNoUndef) || (F.getName() == "main");
7941
7942 Value *Shadow = getShadow(V: RetVal);
7943 bool StoreOrigin = true;
7944 if (EagerCheck) {
7945 insertCheckShadowOf(Val: RetVal, OrigIns: &I);
7946 Shadow = getCleanShadow(V: RetVal);
7947 StoreOrigin = false;
7948 }
7949
7950 // The caller may still expect information passed over TLS if we pass our
7951 // check
7952 if (StoreShadow) {
7953 IRB.CreateAlignedStore(Val: Shadow, Ptr: ShadowPtr, Align: kShadowTLSAlignment);
7954 if (MS.TrackOrigins && StoreOrigin)
7955 IRB.CreateStore(Val: getOrigin(V: RetVal), Ptr: getOriginPtrForRetval());
7956 }
7957 }
7958
7959 void visitPHINode(PHINode &I) {
7960 IRBuilder<> IRB(&I);
7961 if (!PropagateShadow) {
7962 setShadow(V: &I, SV: getCleanShadow(V: &I));
7963 setOrigin(V: &I, Origin: getCleanOrigin());
7964 return;
7965 }
7966
7967 ShadowPHINodes.push_back(Elt: &I);
7968 setShadow(V: &I, SV: IRB.CreatePHI(Ty: getShadowTy(V: &I), NumReservedValues: I.getNumIncomingValues(),
7969 Name: "_msphi_s"));
7970 if (MS.TrackOrigins)
7971 setOrigin(
7972 V: &I, Origin: IRB.CreatePHI(Ty: MS.OriginTy, NumReservedValues: I.getNumIncomingValues(), Name: "_msphi_o"));
7973 }
7974
7975 Value *getLocalVarIdptr(AllocaInst &I) {
7976 ConstantInt *IntConst =
7977 ConstantInt::get(Ty: Type::getInt32Ty(C&: (*F.getParent()).getContext()), V: 0);
7978 return new GlobalVariable(*F.getParent(), IntConst->getType(),
7979 /*isConstant=*/false, GlobalValue::PrivateLinkage,
7980 IntConst);
7981 }
7982
7983 Value *getLocalVarDescription(AllocaInst &I) {
7984 return createPrivateConstGlobalForString(M&: *F.getParent(), Str: I.getName());
7985 }
7986
7987 void poisonAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
7988 if (PoisonStack && Opts.msan_poison_stack_with_call) {
7989 IRB.CreateCall(Callee: MS.MsanPoisonStackFn, Args: {&I, Len});
7990 } else {
7991 Value *ShadowBase, *OriginBase;
7992 std::tie(args&: ShadowBase, args&: OriginBase) = getShadowOriginPtr(
7993 Addr: &I, IRB, ShadowTy: IRB.getInt8Ty(), Alignment: Align(1), /*isStore*/ true);
7994
7995 Value *PoisonValue =
7996 IRB.getInt8(C: PoisonStack ? Opts.msan_poison_stack_pattern : 0);
7997 IRB.CreateMemSet(Ptr: ShadowBase, Val: PoisonValue, Size: Len, Align: I.getAlign());
7998 }
7999
8000 if (PoisonStack && MS.TrackOrigins) {
8001 Value *Idptr = getLocalVarIdptr(I);
8002 if (Opts.msan_print_stack_names) {
8003 Value *Descr = getLocalVarDescription(I);
8004 IRB.CreateCall(Callee: MS.MsanSetAllocaOriginWithDescriptionFn,
8005 Args: {&I, Len, Idptr, Descr});
8006 } else {
8007 IRB.CreateCall(Callee: MS.MsanSetAllocaOriginNoDescriptionFn, Args: {&I, Len, Idptr});
8008 }
8009 }
8010 }
8011
8012 void poisonAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
8013 Value *Descr = getLocalVarDescription(I);
8014 if (PoisonStack) {
8015 IRB.CreateCall(Callee: MS.MsanPoisonAllocaFn, Args: {&I, Len, Descr});
8016 } else {
8017 IRB.CreateCall(Callee: MS.MsanUnpoisonAllocaFn, Args: {&I, Len});
8018 }
8019 }
8020
8021 void instrumentAlloca(AllocaInst &I, Instruction *InsPoint = nullptr) {
8022 if (!InsPoint)
8023 InsPoint = &I;
8024 NextNodeIRBuilder IRB(InsPoint);
8025 Value *Len = IRB.CreateAllocationSize(DestTy: MS.IntptrTy, AI: &I);
8026
8027 if (MS.CompileKernel)
8028 poisonAllocaKmsan(I, IRB, Len);
8029 else
8030 poisonAllocaUserspace(I, IRB, Len);
8031 }
8032
8033 void visitAllocaInst(AllocaInst &I) {
8034 setShadow(V: &I, SV: getCleanShadow(V: &I));
8035 setOrigin(V: &I, Origin: getCleanOrigin());
8036 // We'll get to this alloca later unless it's poisoned at the corresponding
8037 // llvm.lifetime.start.
8038 AllocaSet.insert(X: &I);
8039 }
8040
8041 void visitSelectInst(SelectInst &I) {
8042 // a = select b, c, d
8043 Value *B = I.getCondition();
8044 Value *C = I.getTrueValue();
8045 Value *D = I.getFalseValue();
8046
8047 handleSelectLikeInst(I, B, C, D);
8048 }
8049
8050 void handleSelectLikeInst(Instruction &I, Value *B, Value *C, Value *D) {
8051 IRBuilder<> IRB(&I);
8052
8053 Value *Sb = getShadow(V: B);
8054 Value *Sc = getShadow(V: C);
8055 Value *Sd = getShadow(V: D);
8056
8057 Value *Ob = MS.TrackOrigins ? getOrigin(V: B) : nullptr;
8058 Value *Oc = MS.TrackOrigins ? getOrigin(V: C) : nullptr;
8059 Value *Od = MS.TrackOrigins ? getOrigin(V: D) : nullptr;
8060
8061 // Result shadow if condition shadow is 0.
8062 Value *Sa0 = IRB.CreateSelect(C: B, True: Sc, False: Sd);
8063 Value *Sa1;
8064 if (I.getType()->isAggregateType()) {
8065 // To avoid "sign extending" i1 to an arbitrary aggregate type, we just do
8066 // an extra "select". This results in much more compact IR.
8067 // Sa = select Sb, poisoned, (select b, Sc, Sd)
8068 Sa1 = getPoisonedShadow(ShadowTy: getShadowTy(OrigTy: I.getType()));
8069 } else if (isScalableNonVectorType(Ty: I.getType())) {
8070 // This is intended to handle target("aarch64.svcount"), which can't be
8071 // handled in the else branch because of incompatibility with CreateXor
8072 // ("The supported LLVM operations on this type are limited to load,
8073 // store, phi, select and alloca instructions").
8074
8075 // TODO: this currently underapproximates. Use Arm SVE EOR in the else
8076 // branch as needed instead.
8077 Sa1 = getCleanShadow(OrigTy: getShadowTy(OrigTy: I.getType()));
8078 } else {
8079 // Sa = select Sb, [ (c^d) | Sc | Sd ], [ b ? Sc : Sd ]
8080 // If Sb (condition is poisoned), look for bits in c and d that are equal
8081 // and both unpoisoned.
8082 // If !Sb (condition is unpoisoned), simply pick one of Sc and Sd.
8083
8084 // Cast arguments to shadow-compatible type.
8085 C = CreateAppToShadowCast(IRB, V: C);
8086 D = CreateAppToShadowCast(IRB, V: D);
8087
8088 // Result shadow if condition shadow is 1.
8089 Sa1 = IRB.CreateOr(Ops: {IRB.CreateXor(LHS: C, RHS: D), Sc, Sd});
8090 }
8091 Value *Sa = IRB.CreateSelect(C: Sb, True: Sa1, False: Sa0, Name: "_msprop_select");
8092 setShadow(V: &I, SV: Sa);
8093 if (MS.TrackOrigins) {
8094 // Origins are always i32, so any vector conditions must be flattened.
8095 // FIXME: consider tracking vector origins for app vectors?
8096 if (B->getType()->isVectorTy()) {
8097 B = convertToBool(V: B, IRB);
8098 Sb = convertToBool(V: Sb, IRB);
8099 }
8100 // a = select b, c, d
8101 // Oa = Sb ? Ob : (b ? Oc : Od)
8102 setOrigin(V: &I, Origin: IRB.CreateSelect(C: Sb, True: Ob, False: IRB.CreateSelect(C: B, True: Oc, False: Od)));
8103 }
8104 }
8105
8106 void visitLandingPadInst(LandingPadInst &I) {
8107 // Do nothing.
8108 // See https://github.com/google/sanitizers/issues/504
8109 setShadow(V: &I, SV: getCleanShadow(V: &I));
8110 setOrigin(V: &I, Origin: getCleanOrigin());
8111 }
8112
8113 void visitCatchSwitchInst(CatchSwitchInst &I) {
8114 setShadow(V: &I, SV: getCleanShadow(V: &I));
8115 setOrigin(V: &I, Origin: getCleanOrigin());
8116 }
8117
8118 void visitFuncletPadInst(FuncletPadInst &I) {
8119 setShadow(V: &I, SV: getCleanShadow(V: &I));
8120 setOrigin(V: &I, Origin: getCleanOrigin());
8121 }
8122
8123 void visitGetElementPtrInst(GetElementPtrInst &I) { handleShadowOr(I); }
8124
8125 void visitExtractValueInst(ExtractValueInst &I) {
8126 IRBuilder<> IRB(&I);
8127 Value *Agg = I.getAggregateOperand();
8128 LLVM_DEBUG(dbgs() << "ExtractValue: " << I << "\n");
8129 Value *AggShadow = getShadow(V: Agg);
8130 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
8131 Value *ResShadow = IRB.CreateExtractValue(Agg: AggShadow, Idxs: I.getIndices());
8132 LLVM_DEBUG(dbgs() << " ResShadow: " << *ResShadow << "\n");
8133 setShadow(V: &I, SV: ResShadow);
8134 setOriginForNaryOp(I);
8135 }
8136
8137 void visitInsertValueInst(InsertValueInst &I) {
8138 IRBuilder<> IRB(&I);
8139 LLVM_DEBUG(dbgs() << "InsertValue: " << I << "\n");
8140 Value *AggShadow = getShadow(V: I.getAggregateOperand());
8141 Value *InsShadow = getShadow(V: I.getInsertedValueOperand());
8142 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
8143 LLVM_DEBUG(dbgs() << " InsShadow: " << *InsShadow << "\n");
8144 Value *Res = IRB.CreateInsertValue(Agg: AggShadow, Val: InsShadow, Idxs: I.getIndices());
8145 LLVM_DEBUG(dbgs() << " Res: " << *Res << "\n");
8146 setShadow(V: &I, SV: Res);
8147 setOriginForNaryOp(I);
8148 }
8149
8150 void dumpInst(Instruction &I, const Twine &Prefix) {
8151 // Instruction name only
8152 // For intrinsics, the full/overloaded name is used
8153 //
8154 // e.g., "call llvm.aarch64.neon.uqsub.v16i8"
8155 if (CallInst *CI = dyn_cast<CallInst>(Val: &I)) {
8156 errs() << "ZZZ:" << Prefix << " call "
8157 << CI->getCalledFunction()->getName() << "\n";
8158 } else {
8159 errs() << "ZZZ:" << Prefix << " " << I.getOpcodeName() << "\n";
8160 }
8161
8162 // Instruction prototype (including return type and parameter types)
8163 // For intrinsics, we use the base/non-overloaded name
8164 //
8165 // e.g., "call <16 x i8> @llvm.aarch64.neon.uqsub(<16 x i8>, <16 x i8>)"
8166 unsigned NumOperands = I.getNumOperands();
8167 if (CallInst *CI = dyn_cast<CallInst>(Val: &I)) {
8168 errs() << "YYY:" << Prefix << " call " << *I.getType() << " @";
8169
8170 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: CI))
8171 errs() << Intrinsic::getBaseName(id: II->getIntrinsicID());
8172 else
8173 errs() << CI->getCalledFunction()->getName();
8174
8175 errs() << "(";
8176
8177 // The last operand of a CallInst is the function itself.
8178 NumOperands--;
8179 } else
8180 errs() << "YYY:" << Prefix << " " << *I.getType() << " "
8181 << I.getOpcodeName() << "(";
8182
8183 for (size_t i = 0; i < NumOperands; i++) {
8184 if (i > 0)
8185 errs() << ", ";
8186
8187 errs() << *(I.getOperand(i)->getType());
8188 }
8189
8190 errs() << ")\n";
8191
8192 // Full instruction, including types and operand values
8193 // For intrinsics, the full/overloaded name is used
8194 //
8195 // e.g., "%vqsubq_v.i15 = call noundef <16 x i8>
8196 // @llvm.aarch64.neon.uqsub.v16i8(<16 x i8> %vext21.i,
8197 // <16 x i8> splat (i8 1)), !dbg !66"
8198 errs() << "QQQ:" << Prefix << " " << I << "\n";
8199 }
8200
8201 void visitResumeInst(ResumeInst &I) {
8202 LLVM_DEBUG(dbgs() << "Resume: " << I << "\n");
8203 // Nothing to do here.
8204 }
8205
8206 void visitCleanupReturnInst(CleanupReturnInst &CRI) {
8207 LLVM_DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n");
8208 // Nothing to do here.
8209 }
8210
8211 void visitCatchReturnInst(CatchReturnInst &CRI) {
8212 LLVM_DEBUG(dbgs() << "CatchReturn: " << CRI << "\n");
8213 // Nothing to do here.
8214 }
8215
8216 void instrumentAsmArgument(Value *Operand, Type *ElemTy, Instruction &I,
8217 IRBuilder<> &IRB, const DataLayout &DL,
8218 bool isOutput) {
8219 // For each assembly argument, we check its value for being initialized.
8220 // If the argument is a pointer, we assume it points to a single element
8221 // of the corresponding type (or to a 8-byte word, if the type is unsized).
8222 // Each such pointer is instrumented with a call to the runtime library.
8223 Type *OpType = Operand->getType();
8224 // Check the operand value itself.
8225 insertCheckShadowOf(Val: Operand, OrigIns: &I);
8226 if (!OpType->isPointerTy() || !isOutput) {
8227 assert(!isOutput);
8228 return;
8229 }
8230 if (!ElemTy->isSized())
8231 return;
8232 auto Size = DL.getTypeStoreSize(Ty: ElemTy);
8233 Value *SizeVal = IRB.CreateTypeSize(Ty: MS.IntptrTy, Size);
8234 if (MS.CompileKernel) {
8235 IRB.CreateCall(Callee: MS.MsanInstrumentAsmStoreFn, Args: {Operand, SizeVal});
8236 } else {
8237 // ElemTy, derived from elementtype(), does not encode the alignment of
8238 // the pointer. Conservatively assume that the shadow memory is unaligned.
8239 // When Size is large, avoid StoreInst as it would expand to many
8240 // instructions.
8241 auto [ShadowPtr, _] =
8242 getShadowOriginPtrUserspace(Addr: Operand, IRB, ShadowTy: IRB.getInt8Ty(), Alignment: Align(1));
8243 if (Size <= 32)
8244 IRB.CreateAlignedStore(Val: getCleanShadow(OrigTy: ElemTy), Ptr: ShadowPtr, Align: Align(1));
8245 else
8246 IRB.CreateMemSet(Ptr: ShadowPtr, Val: ConstantInt::getNullValue(Ty: IRB.getInt8Ty()),
8247 Size: SizeVal, Align: Align(1));
8248 }
8249 }
8250
8251 /// Get the number of output arguments returned by pointers.
8252 int getNumOutputArgs(InlineAsm *IA, CallBase *CB) {
8253 int NumRetOutputs = 0;
8254 int NumOutputs = 0;
8255 Type *RetTy = cast<Value>(Val: CB)->getType();
8256 if (!RetTy->isVoidTy()) {
8257 // Register outputs are returned via the CallInst return value.
8258 auto *ST = dyn_cast<StructType>(Val: RetTy);
8259 if (ST)
8260 NumRetOutputs = ST->getNumElements();
8261 else
8262 NumRetOutputs = 1;
8263 }
8264 InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints();
8265 for (const InlineAsm::ConstraintInfo &Info : Constraints) {
8266 switch (Info.Type) {
8267 case InlineAsm::isOutput:
8268 NumOutputs++;
8269 break;
8270 default:
8271 break;
8272 }
8273 }
8274 return NumOutputs - NumRetOutputs;
8275 }
8276
8277 void visitAsmInstruction(Instruction &I) {
8278 // Conservative inline assembly handling: check for poisoned shadow of
8279 // asm() arguments, then unpoison the result and all the memory locations
8280 // pointed to by those arguments.
8281 // An inline asm() statement in C++ contains lists of input and output
8282 // arguments used by the assembly code. These are mapped to operands of the
8283 // CallInst as follows:
8284 // - nR register outputs ("=r) are returned by value in a single structure
8285 // (SSA value of the CallInst);
8286 // - nO other outputs ("=m" and others) are returned by pointer as first
8287 // nO operands of the CallInst;
8288 // - nI inputs ("r", "m" and others) are passed to CallInst as the
8289 // remaining nI operands.
8290 // The total number of asm() arguments in the source is nR+nO+nI, and the
8291 // corresponding CallInst has nO+nI+1 operands (the last operand is the
8292 // function to be called).
8293 const DataLayout &DL = F.getDataLayout();
8294 CallBase *CB = cast<CallBase>(Val: &I);
8295 IRBuilder<> IRB(&I);
8296 InlineAsm *IA = cast<InlineAsm>(Val: CB->getCalledOperand());
8297 int OutputArgs = getNumOutputArgs(IA, CB);
8298 // The last operand of a CallInst is the function itself.
8299 int NumOperands = CB->getNumOperands() - 1;
8300
8301 // Check input arguments. Doing so before unpoisoning output arguments, so
8302 // that we won't overwrite uninit values before checking them.
8303 for (int i = OutputArgs; i < NumOperands; i++) {
8304 Value *Operand = CB->getOperand(i_nocapture: i);
8305 instrumentAsmArgument(Operand, ElemTy: CB->getParamElementType(ArgNo: i), I, IRB, DL,
8306 /*isOutput*/ false);
8307 }
8308 // Unpoison output arguments. This must happen before the actual InlineAsm
8309 // call, so that the shadow for memory published in the asm() statement
8310 // remains valid.
8311 for (int i = 0; i < OutputArgs; i++) {
8312 Value *Operand = CB->getOperand(i_nocapture: i);
8313 instrumentAsmArgument(Operand, ElemTy: CB->getParamElementType(ArgNo: i), I, IRB, DL,
8314 /*isOutput*/ true);
8315 }
8316
8317 setShadow(V: &I, SV: getCleanShadow(V: &I));
8318 setOrigin(V: &I, Origin: getCleanOrigin());
8319 }
8320
8321 void visitFreezeInst(FreezeInst &I) {
8322 // Freeze always returns a fully defined value.
8323 setShadow(V: &I, SV: getCleanShadow(V: &I));
8324 setOrigin(V: &I, Origin: getCleanOrigin());
8325 }
8326
8327 void visitInstruction(Instruction &I) {
8328 // Everything else: stop propagating and check for poisoned shadow.
8329 if (Opts.msan_dump_strict_instructions)
8330 dumpInst(I, Prefix: "Strict");
8331 LLVM_DEBUG(dbgs() << "DEFAULT: " << I << "\n");
8332 for (size_t i = 0, n = I.getNumOperands(); i < n; i++) {
8333 Value *Operand = I.getOperand(i);
8334 if (Operand->getType()->isSized())
8335 insertCheckShadowOf(Val: Operand, OrigIns: &I);
8336 }
8337 setShadow(V: &I, SV: getCleanShadow(V: &I));
8338 setOrigin(V: &I, Origin: getCleanOrigin());
8339 }
8340};
8341
8342struct VarArgHelperBase : public VarArgHelper {
8343 Function &F;
8344 MemorySanitizer &MS;
8345 MemorySanitizerVisitor &MSV;
8346 SmallVector<CallInst *, 16> VAStartInstrumentationList;
8347 const unsigned VAListTagSize;
8348
8349 VarArgHelperBase(Function &F, MemorySanitizer &MS,
8350 MemorySanitizerVisitor &MSV, unsigned VAListTagSize)
8351 : F(F), MS(MS), MSV(MSV), VAListTagSize(VAListTagSize) {}
8352
8353 Value *getShadowAddrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset) {
8354 Value *Base = IRB.CreatePointerCast(V: MS.VAArgTLS, DestTy: MS.IntptrTy);
8355 return IRB.CreateAdd(LHS: Base, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset));
8356 }
8357
8358 /// Compute the shadow address for a given va_arg.
8359 Value *getShadowPtrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset) {
8360 return IRB.CreatePtrAddWithUnsignedOffset(Ptr: MS.VAArgTLS, Offset: ArgOffset,
8361 Name: "_msarg_va_s");
8362 }
8363
8364 /// Compute the shadow address for a given va_arg.
8365 Value *getShadowPtrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset,
8366 unsigned ArgSize) {
8367 // Make sure we don't overflow __msan_va_arg_tls.
8368 if (ArgOffset + ArgSize > kParamTLSSize)
8369 return nullptr;
8370 return getShadowPtrForVAArgument(IRB, ArgOffset);
8371 }
8372
8373 /// Compute the origin address for a given va_arg.
8374 Value *getOriginPtrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset) {
8375 // getOriginPtrForVAArgument() is always called after
8376 // getShadowPtrForVAArgument(), so __msan_va_arg_origin_tls can never
8377 // overflow.
8378 return IRB.CreatePtrAddWithUnsignedOffset(Ptr: MS.VAArgOriginTLS, Offset: ArgOffset,
8379 Name: "_msarg_va_o");
8380 }
8381
8382 void CleanUnusedTLS(IRBuilder<> &IRB, Value *ShadowBase,
8383 unsigned BaseOffset) {
8384 // The tails of __msan_va_arg_tls is not large enough to fit full
8385 // value shadow, but it will be copied to backup anyway. Make it
8386 // clean.
8387 if (BaseOffset >= kParamTLSSize)
8388 return;
8389 Value *TailSize =
8390 ConstantInt::getSigned(Ty: IRB.getInt32Ty(), V: kParamTLSSize - BaseOffset);
8391 IRB.CreateMemSet(Ptr: ShadowBase, Val: ConstantInt::getNullValue(Ty: IRB.getInt8Ty()),
8392 Size: TailSize, Align: Align(8));
8393 }
8394
8395 void unpoisonVAListTagForInst(IntrinsicInst &I) {
8396 IRBuilder<> IRB(&I);
8397 Value *VAListTag = I.getArgOperand(i: 0);
8398 const Align Alignment = Align(8);
8399 auto [ShadowPtr, OriginPtr] = MSV.getShadowOriginPtr(
8400 Addr: VAListTag, IRB, ShadowTy: IRB.getInt8Ty(), Alignment, /*isStore*/ true);
8401 // Unpoison the whole __va_list_tag.
8402 IRB.CreateMemSet(Ptr: ShadowPtr, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
8403 Size: VAListTagSize, Align: Alignment, isVolatile: false);
8404 }
8405
8406 void visitVAStartInst(VAStartInst &I) override {
8407 if (F.getCallingConv() == CallingConv::Win64)
8408 return;
8409 VAStartInstrumentationList.push_back(Elt: &I);
8410 unpoisonVAListTagForInst(I);
8411 }
8412
8413 void visitVACopyInst(VACopyInst &I) override {
8414 if (F.getCallingConv() == CallingConv::Win64)
8415 return;
8416 unpoisonVAListTagForInst(I);
8417 }
8418};
8419
8420/// AMD64-specific implementation of VarArgHelper.
8421struct VarArgAMD64Helper : public VarArgHelperBase {
8422 // An unfortunate workaround for asymmetric lowering of va_arg stuff.
8423 // See a comment in visitCallBase for more details.
8424 static const unsigned AMD64GpEndOffset = 48; // AMD64 ABI Draft 0.99.6 p3.5.7
8425 static const unsigned AMD64FpEndOffsetSSE = 176;
8426 // If SSE is disabled, fp_offset in va_list is zero.
8427 static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
8428
8429 unsigned AMD64FpEndOffset;
8430 AllocaInst *VAArgTLSCopy = nullptr;
8431 AllocaInst *VAArgTLSOriginCopy = nullptr;
8432 Value *VAArgOverflowSize = nullptr;
8433
8434 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
8435
8436 VarArgAMD64Helper(Function &F, MemorySanitizer &MS,
8437 MemorySanitizerVisitor &MSV)
8438 : VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/24) {
8439 AMD64FpEndOffset = AMD64FpEndOffsetSSE;
8440 for (const auto &Attr : F.getAttributes().getFnAttrs()) {
8441 if (Attr.isStringAttribute() &&
8442 (Attr.getKindAsString() == "target-features")) {
8443 if (Attr.getValueAsString().contains(Other: "-sse"))
8444 AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
8445 break;
8446 }
8447 }
8448 }
8449
8450 ArgKind classifyArgument(Value *arg) {
8451 // A very rough approximation of X86_64 argument classification rules.
8452 Type *T = arg->getType();
8453 if (T->isX86_FP80Ty())
8454 return AK_Memory;
8455 if (T->isFPOrFPVectorTy())
8456 return AK_FloatingPoint;
8457 if (T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
8458 return AK_GeneralPurpose;
8459 if (T->isPointerTy())
8460 return AK_GeneralPurpose;
8461 return AK_Memory;
8462 }
8463
8464 // For VarArg functions, store the argument shadow in an ABI-specific format
8465 // that corresponds to va_list layout.
8466 // We do this because Clang lowers va_arg in the frontend, and this pass
8467 // only sees the low level code that deals with va_list internals.
8468 // A much easier alternative (provided that Clang emits va_arg instructions)
8469 // would have been to associate each live instance of va_list with a copy of
8470 // MSanParamTLS, and extract shadow on va_arg() call in the argument list
8471 // order.
8472 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
8473 unsigned GpOffset = 0;
8474 unsigned FpOffset = AMD64GpEndOffset;
8475 unsigned OverflowOffset = AMD64FpEndOffset;
8476 const DataLayout &DL = F.getDataLayout();
8477
8478 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
8479 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
8480 bool IsByVal = CB.isByValArgument(ArgNo);
8481 if (IsByVal) {
8482 // ByVal arguments always go to the overflow area.
8483 // Fixed arguments passed through the overflow area will be stepped
8484 // over by va_start, so don't count them towards the offset.
8485 if (IsFixed)
8486 continue;
8487 assert(A->getType()->isPointerTy());
8488 Type *RealTy = CB.getParamByValType(ArgNo);
8489 uint64_t ArgSize = DL.getTypeAllocSize(Ty: RealTy);
8490 uint64_t AlignedSize = alignTo(Value: ArgSize, Align: 8);
8491 unsigned BaseOffset = OverflowOffset;
8492 Value *ShadowBase = getShadowPtrForVAArgument(IRB, ArgOffset: OverflowOffset);
8493 Value *OriginBase = nullptr;
8494 if (MS.TrackOrigins)
8495 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: OverflowOffset);
8496 OverflowOffset += AlignedSize;
8497
8498 if (OverflowOffset > kParamTLSSize) {
8499 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
8500 continue; // We have no space to copy shadow there.
8501 }
8502
8503 Value *ShadowPtr, *OriginPtr;
8504 std::tie(args&: ShadowPtr, args&: OriginPtr) =
8505 MSV.getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(), Alignment: kShadowTLSAlignment,
8506 /*isStore*/ false);
8507 IRB.CreateMemCpy(Dst: ShadowBase, DstAlign: kShadowTLSAlignment, Src: ShadowPtr,
8508 SrcAlign: kShadowTLSAlignment, Size: ArgSize);
8509 if (MS.TrackOrigins)
8510 IRB.CreateMemCpy(Dst: OriginBase, DstAlign: kShadowTLSAlignment, Src: OriginPtr,
8511 SrcAlign: kShadowTLSAlignment, Size: ArgSize);
8512 } else {
8513 ArgKind AK = classifyArgument(arg: A);
8514 if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
8515 AK = AK_Memory;
8516 if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
8517 AK = AK_Memory;
8518 Value *ShadowBase, *OriginBase = nullptr;
8519 switch (AK) {
8520 case AK_GeneralPurpose:
8521 ShadowBase = getShadowPtrForVAArgument(IRB, ArgOffset: GpOffset);
8522 if (MS.TrackOrigins)
8523 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: GpOffset);
8524 GpOffset += 8;
8525 assert(GpOffset <= kParamTLSSize);
8526 break;
8527 case AK_FloatingPoint:
8528 ShadowBase = getShadowPtrForVAArgument(IRB, ArgOffset: FpOffset);
8529 if (MS.TrackOrigins)
8530 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: FpOffset);
8531 FpOffset += 16;
8532 assert(FpOffset <= kParamTLSSize);
8533 break;
8534 case AK_Memory:
8535 if (IsFixed)
8536 continue;
8537 uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
8538 uint64_t AlignedSize = alignTo(Value: ArgSize, Align: 8);
8539 unsigned BaseOffset = OverflowOffset;
8540 ShadowBase = getShadowPtrForVAArgument(IRB, ArgOffset: OverflowOffset);
8541 if (MS.TrackOrigins) {
8542 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: OverflowOffset);
8543 }
8544 OverflowOffset += AlignedSize;
8545 if (OverflowOffset > kParamTLSSize) {
8546 // We have no space to copy shadow there.
8547 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
8548 continue;
8549 }
8550 }
8551 // Take fixed arguments into account for GpOffset and FpOffset,
8552 // but don't actually store shadows for them.
8553 // TODO(glider): don't call get*PtrForVAArgument() for them.
8554 if (IsFixed)
8555 continue;
8556 Value *Shadow = MSV.getShadow(V: A);
8557 IRB.CreateAlignedStore(Val: Shadow, Ptr: ShadowBase, Align: kShadowTLSAlignment);
8558 if (MS.TrackOrigins) {
8559 Value *Origin = MSV.getOrigin(V: A);
8560 TypeSize StoreSize = DL.getTypeStoreSize(Ty: Shadow->getType());
8561 MSV.paintOrigin(IRB, Origin, OriginPtr: OriginBase, TS: StoreSize,
8562 Alignment: std::max(a: kShadowTLSAlignment, b: kMinOriginAlignment));
8563 }
8564 }
8565 }
8566 Constant *OverflowSize =
8567 ConstantInt::get(Ty: IRB.getInt64Ty(), V: OverflowOffset - AMD64FpEndOffset);
8568 IRB.CreateStore(Val: OverflowSize, Ptr: MS.VAArgOverflowSizeTLS);
8569 }
8570
8571 void finalizeInstrumentation() override {
8572 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
8573 "finalizeInstrumentation called twice");
8574 if (!VAStartInstrumentationList.empty()) {
8575 // If there is a va_start in this function, make a backup copy of
8576 // va_arg_tls somewhere in the function entry block.
8577 IRBuilder<> IRB(MSV.FnPrologueEnd);
8578 VAArgOverflowSize =
8579 IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
8580 Value *CopySize = IRB.CreateAdd(
8581 LHS: ConstantInt::get(Ty: MS.IntptrTy, V: AMD64FpEndOffset), RHS: VAArgOverflowSize);
8582 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
8583 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
8584 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
8585 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
8586
8587 Value *SrcSize = IRB.CreateBinaryIntrinsic(
8588 ID: Intrinsic::umin, LHS: CopySize,
8589 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
8590 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
8591 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
8592 if (MS.TrackOrigins) {
8593 VAArgTLSOriginCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
8594 VAArgTLSOriginCopy->setAlignment(kShadowTLSAlignment);
8595 IRB.CreateMemCpy(Dst: VAArgTLSOriginCopy, DstAlign: kShadowTLSAlignment,
8596 Src: MS.VAArgOriginTLS, SrcAlign: kShadowTLSAlignment, Size: SrcSize);
8597 }
8598 }
8599
8600 // Instrument va_start.
8601 // Copy va_list shadow from the backup copy of the TLS contents.
8602 for (CallInst *OrigInst : VAStartInstrumentationList) {
8603 NextNodeIRBuilder IRB(OrigInst);
8604 Value *VAListTag = OrigInst->getArgOperand(i: 0);
8605
8606 Value *RegSaveAreaPtrPtr =
8607 IRB.CreatePtrAddWithUnsignedOffset(Ptr: VAListTag, Offset: 16);
8608 Value *RegSaveAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: RegSaveAreaPtrPtr);
8609 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
8610 const Align Alignment = Align(16);
8611 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
8612 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8613 Alignment, /*isStore*/ true);
8614 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
8615 Size: AMD64FpEndOffset);
8616 if (MS.TrackOrigins)
8617 IRB.CreateMemCpy(Dst: RegSaveAreaOriginPtr, DstAlign: Alignment, Src: VAArgTLSOriginCopy,
8618 SrcAlign: Alignment, Size: AMD64FpEndOffset);
8619 Value *OverflowArgAreaPtrPtr =
8620 IRB.CreatePtrAddWithUnsignedOffset(Ptr: VAListTag, Offset: 8);
8621 Value *OverflowArgAreaPtr =
8622 IRB.CreateLoad(Ty: MS.PtrTy, Ptr: OverflowArgAreaPtrPtr);
8623 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
8624 std::tie(args&: OverflowArgAreaShadowPtr, args&: OverflowArgAreaOriginPtr) =
8625 MSV.getShadowOriginPtr(Addr: OverflowArgAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8626 Alignment, /*isStore*/ true);
8627 Value *SrcPtr = IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(), Ptr: VAArgTLSCopy,
8628 Idx0: AMD64FpEndOffset);
8629 IRB.CreateMemCpy(Dst: OverflowArgAreaShadowPtr, DstAlign: Alignment, Src: SrcPtr, SrcAlign: Alignment,
8630 Size: VAArgOverflowSize);
8631 if (MS.TrackOrigins) {
8632 SrcPtr = IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(), Ptr: VAArgTLSOriginCopy,
8633 Idx0: AMD64FpEndOffset);
8634 IRB.CreateMemCpy(Dst: OverflowArgAreaOriginPtr, DstAlign: Alignment, Src: SrcPtr, SrcAlign: Alignment,
8635 Size: VAArgOverflowSize);
8636 }
8637 }
8638 }
8639};
8640
8641/// AArch64-specific implementation of VarArgHelper.
8642struct VarArgAArch64Helper : public VarArgHelperBase {
8643 static const unsigned kAArch64GrArgSize = 64;
8644 static const unsigned kAArch64VrArgSize = 128;
8645
8646 static const unsigned AArch64GrBegOffset = 0;
8647 static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
8648 // Make VR space aligned to 16 bytes.
8649 static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
8650 static const unsigned AArch64VrEndOffset =
8651 AArch64VrBegOffset + kAArch64VrArgSize;
8652 static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
8653
8654 AllocaInst *VAArgTLSCopy = nullptr;
8655 Value *VAArgOverflowSize = nullptr;
8656
8657 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
8658
8659 VarArgAArch64Helper(Function &F, MemorySanitizer &MS,
8660 MemorySanitizerVisitor &MSV)
8661 : VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/32) {}
8662
8663 // A very rough approximation of aarch64 argument classification rules.
8664 std::pair<ArgKind, uint64_t> classifyArgument(Type *T) {
8665 if (T->isIntOrPtrTy() && T->getPrimitiveSizeInBits() <= 64)
8666 return {AK_GeneralPurpose, 1};
8667 if (T->isFloatingPointTy() && T->getPrimitiveSizeInBits() <= 128)
8668 return {AK_FloatingPoint, 1};
8669
8670 if (T->isArrayTy()) {
8671 auto R = classifyArgument(T: T->getArrayElementType());
8672 R.second *= T->getScalarType()->getArrayNumElements();
8673 return R;
8674 }
8675
8676 if (const FixedVectorType *FV = dyn_cast<FixedVectorType>(Val: T)) {
8677 auto R = classifyArgument(T: FV->getScalarType());
8678 R.second *= FV->getNumElements();
8679 return R;
8680 }
8681
8682 LLVM_DEBUG(errs() << "Unknown vararg type: " << *T << "\n");
8683 return {AK_Memory, 0};
8684 }
8685
8686 // The instrumentation stores the argument shadow in a non ABI-specific
8687 // format because it does not know which argument is named (since Clang,
8688 // like x86_64 case, lowers the va_args in the frontend and this pass only
8689 // sees the low level code that deals with va_list internals).
8690 // The first seven GR registers are saved in the first 56 bytes of the
8691 // va_arg tls arra, followed by the first 8 FP/SIMD registers, and then
8692 // the remaining arguments.
8693 // Using constant offset within the va_arg TLS array allows fast copy
8694 // in the finalize instrumentation.
8695 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
8696 unsigned GrOffset = AArch64GrBegOffset;
8697 unsigned VrOffset = AArch64VrBegOffset;
8698 unsigned OverflowOffset = AArch64VAEndOffset;
8699
8700 const DataLayout &DL = F.getDataLayout();
8701 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
8702 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
8703 auto [AK, RegNum] = classifyArgument(T: A->getType());
8704 if (AK == AK_GeneralPurpose &&
8705 (GrOffset + RegNum * 8) > AArch64GrEndOffset)
8706 AK = AK_Memory;
8707 if (AK == AK_FloatingPoint &&
8708 (VrOffset + RegNum * 16) > AArch64VrEndOffset)
8709 AK = AK_Memory;
8710 Value *Base;
8711 switch (AK) {
8712 case AK_GeneralPurpose:
8713 Base = getShadowPtrForVAArgument(IRB, ArgOffset: GrOffset);
8714 GrOffset += 8 * RegNum;
8715 break;
8716 case AK_FloatingPoint:
8717 Base = getShadowPtrForVAArgument(IRB, ArgOffset: VrOffset);
8718 VrOffset += 16 * RegNum;
8719 break;
8720 case AK_Memory:
8721 // Don't count fixed arguments in the overflow area - va_start will
8722 // skip right over them.
8723 if (IsFixed)
8724 continue;
8725 uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
8726 uint64_t AlignedSize = alignTo(Value: ArgSize, Align: 8);
8727 unsigned BaseOffset = OverflowOffset;
8728 Base = getShadowPtrForVAArgument(IRB, ArgOffset: BaseOffset);
8729 OverflowOffset += AlignedSize;
8730 if (OverflowOffset > kParamTLSSize) {
8731 // We have no space to copy shadow there.
8732 CleanUnusedTLS(IRB, ShadowBase: Base, BaseOffset);
8733 continue;
8734 }
8735 break;
8736 }
8737 // Count Gp/Vr fixed arguments to their respective offsets, but don't
8738 // bother to actually store a shadow.
8739 if (IsFixed)
8740 continue;
8741 IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base, Align: kShadowTLSAlignment);
8742 }
8743 Constant *OverflowSize =
8744 ConstantInt::get(Ty: IRB.getInt64Ty(), V: OverflowOffset - AArch64VAEndOffset);
8745 IRB.CreateStore(Val: OverflowSize, Ptr: MS.VAArgOverflowSizeTLS);
8746 }
8747
8748 // Retrieve a va_list field of 'void*' size.
8749 Value *getVAField64(IRBuilder<> &IRB, Value *VAListTag, unsigned Offset) {
8750 Value *SaveAreaPtrPtr =
8751 IRB.CreatePtrAddWithUnsignedOffset(Ptr: VAListTag, Offset);
8752 return IRB.CreateLoad(Ty: Type::getInt64Ty(C&: *MS.C), Ptr: SaveAreaPtrPtr);
8753 }
8754
8755 // Retrieve a va_list field of 'int' size.
8756 Value *getVAField32(IRBuilder<> &IRB, Value *VAListTag, unsigned Offset) {
8757 Value *SaveAreaPtr = IRB.CreatePtrAddWithUnsignedOffset(Ptr: VAListTag, Offset);
8758 Value *SaveArea32 = IRB.CreateLoad(Ty: IRB.getInt32Ty(), Ptr: SaveAreaPtr);
8759 return IRB.CreateSExt(V: SaveArea32, DestTy: MS.IntptrTy);
8760 }
8761
8762 void finalizeInstrumentation() override {
8763 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
8764 "finalizeInstrumentation called twice");
8765 if (!VAStartInstrumentationList.empty()) {
8766 // If there is a va_start in this function, make a backup copy of
8767 // va_arg_tls somewhere in the function entry block.
8768 IRBuilder<> IRB(MSV.FnPrologueEnd);
8769 VAArgOverflowSize =
8770 IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
8771 Value *CopySize = IRB.CreateAdd(
8772 LHS: ConstantInt::get(Ty: MS.IntptrTy, V: AArch64VAEndOffset), RHS: VAArgOverflowSize);
8773 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
8774 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
8775 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
8776 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
8777
8778 Value *SrcSize = IRB.CreateBinaryIntrinsic(
8779 ID: Intrinsic::umin, LHS: CopySize,
8780 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
8781 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
8782 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
8783 }
8784
8785 Value *GrArgSize = ConstantInt::get(Ty: MS.IntptrTy, V: kAArch64GrArgSize);
8786 Value *VrArgSize = ConstantInt::get(Ty: MS.IntptrTy, V: kAArch64VrArgSize);
8787
8788 // Instrument va_start, copy va_list shadow from the backup copy of
8789 // the TLS contents.
8790 for (CallInst *OrigInst : VAStartInstrumentationList) {
8791 NextNodeIRBuilder IRB(OrigInst);
8792
8793 Value *VAListTag = OrigInst->getArgOperand(i: 0);
8794
8795 // The variadic ABI for AArch64 creates two areas to save the incoming
8796 // argument registers (one for 64-bit general register xn-x7 and another
8797 // for 128-bit FP/SIMD vn-v7).
8798 // We need then to propagate the shadow arguments on both regions
8799 // 'va::__gr_top + va::__gr_offs' and 'va::__vr_top + va::__vr_offs'.
8800 // The remaining arguments are saved on shadow for 'va::stack'.
8801 // One caveat is it requires only to propagate the non-named arguments,
8802 // however on the call site instrumentation 'all' the arguments are
8803 // saved. So to copy the shadow values from the va_arg TLS array
8804 // we need to adjust the offset for both GR and VR fields based on
8805 // the __{gr,vr}_offs value (since they are stores based on incoming
8806 // named arguments).
8807 Type *RegSaveAreaPtrTy = IRB.getPtrTy();
8808
8809 // Read the stack pointer from the va_list.
8810 Value *StackSaveAreaPtr =
8811 IRB.CreateIntToPtr(V: getVAField64(IRB, VAListTag, Offset: 0), DestTy: RegSaveAreaPtrTy);
8812
8813 // Read both the __gr_top and __gr_off and add them up.
8814 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, Offset: 8);
8815 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, Offset: 24);
8816
8817 Value *GrRegSaveAreaPtr = IRB.CreateIntToPtr(
8818 V: IRB.CreateAdd(LHS: GrTopSaveAreaPtr, RHS: GrOffSaveArea), DestTy: RegSaveAreaPtrTy);
8819
8820 // Read both the __vr_top and __vr_off and add them up.
8821 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, Offset: 16);
8822 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, Offset: 28);
8823
8824 Value *VrRegSaveAreaPtr = IRB.CreateIntToPtr(
8825 V: IRB.CreateAdd(LHS: VrTopSaveAreaPtr, RHS: VrOffSaveArea), DestTy: RegSaveAreaPtrTy);
8826
8827 // It does not know how many named arguments is being used and, on the
8828 // callsite all the arguments were saved. Since __gr_off is defined as
8829 // '0 - ((8 - named_gr) * 8)', the idea is to just propagate the variadic
8830 // argument by ignoring the bytes of shadow from named arguments.
8831 Value *GrRegSaveAreaShadowPtrOff =
8832 IRB.CreateAdd(LHS: GrArgSize, RHS: GrOffSaveArea);
8833
8834 Value *GrRegSaveAreaShadowPtr =
8835 MSV.getShadowOriginPtr(Addr: GrRegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8836 Alignment: Align(8), /*isStore*/ true)
8837 .first;
8838
8839 Value *GrSrcPtr =
8840 IRB.CreateInBoundsPtrAdd(Ptr: VAArgTLSCopy, Offset: GrRegSaveAreaShadowPtrOff);
8841 Value *GrCopySize = IRB.CreateSub(LHS: GrArgSize, RHS: GrRegSaveAreaShadowPtrOff);
8842
8843 IRB.CreateMemCpy(Dst: GrRegSaveAreaShadowPtr, DstAlign: Align(8), Src: GrSrcPtr, SrcAlign: Align(8),
8844 Size: GrCopySize);
8845
8846 // Again, but for FP/SIMD values.
8847 Value *VrRegSaveAreaShadowPtrOff =
8848 IRB.CreateAdd(LHS: VrArgSize, RHS: VrOffSaveArea);
8849
8850 Value *VrRegSaveAreaShadowPtr =
8851 MSV.getShadowOriginPtr(Addr: VrRegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8852 Alignment: Align(8), /*isStore*/ true)
8853 .first;
8854
8855 Value *VrSrcPtr = IRB.CreateInBoundsPtrAdd(
8856 Ptr: IRB.CreatePtrAddWithUnsignedOffset(Ptr: VAArgTLSCopy, Offset: AArch64VrBegOffset,
8857 Name: "", NW: GEPNoWrapFlags::inBounds()),
8858 Offset: VrRegSaveAreaShadowPtrOff);
8859 Value *VrCopySize = IRB.CreateSub(LHS: VrArgSize, RHS: VrRegSaveAreaShadowPtrOff);
8860
8861 IRB.CreateMemCpy(Dst: VrRegSaveAreaShadowPtr, DstAlign: Align(8), Src: VrSrcPtr, SrcAlign: Align(8),
8862 Size: VrCopySize);
8863
8864 // And finally for remaining arguments.
8865 Value *StackSaveAreaShadowPtr =
8866 MSV.getShadowOriginPtr(Addr: StackSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
8867 Alignment: Align(16), /*isStore*/ true)
8868 .first;
8869
8870 Value *StackSrcPtr = IRB.CreatePtrAddWithUnsignedOffset(
8871 Ptr: VAArgTLSCopy, Offset: AArch64VAEndOffset, Name: "", NW: GEPNoWrapFlags::inBounds());
8872
8873 IRB.CreateMemCpy(Dst: StackSaveAreaShadowPtr, DstAlign: Align(16), Src: StackSrcPtr,
8874 SrcAlign: Align(16), Size: VAArgOverflowSize);
8875 }
8876 }
8877};
8878
8879/// PowerPC64-specific implementation of VarArgHelper.
8880struct VarArgPowerPC64Helper : public VarArgHelperBase {
8881 AllocaInst *VAArgTLSCopy = nullptr;
8882 Value *VAArgSize = nullptr;
8883
8884 VarArgPowerPC64Helper(Function &F, MemorySanitizer &MS,
8885 MemorySanitizerVisitor &MSV)
8886 : VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/8) {}
8887
8888 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
8889 // For PowerPC, we need to deal with alignment of stack arguments -
8890 // they are mostly aligned to 8 bytes, but vectors and i128 arrays
8891 // are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
8892 // For that reason, we compute current offset from stack pointer (which is
8893 // always properly aligned), and offset for the first vararg, then subtract
8894 // them.
8895 unsigned VAArgBase;
8896 Triple TargetTriple(F.getParent()->getTargetTriple());
8897 // Parameter save area starts at 48 bytes from frame pointer for ABIv1,
8898 // and 32 bytes for ABIv2. This is usually determined by target
8899 // endianness, but in theory could be overridden by function attribute.
8900 if (TargetTriple.isPPC64ELFv2ABI())
8901 VAArgBase = 32;
8902 else
8903 VAArgBase = 48;
8904 unsigned VAArgOffset = VAArgBase;
8905 const DataLayout &DL = F.getDataLayout();
8906 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
8907 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
8908 bool IsByVal = CB.isByValArgument(ArgNo);
8909 if (IsByVal) {
8910 assert(A->getType()->isPointerTy());
8911 Type *RealTy = CB.getParamByValType(ArgNo);
8912 uint64_t ArgSize = DL.getTypeAllocSize(Ty: RealTy);
8913 Align ArgAlign = CB.getParamAlign(ArgNo).value_or(u: Align(8));
8914 if (ArgAlign < 8)
8915 ArgAlign = Align(8);
8916 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
8917 if (!IsFixed) {
8918 Value *Base =
8919 getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset - VAArgBase, ArgSize);
8920 if (Base) {
8921 Value *AShadowPtr, *AOriginPtr;
8922 std::tie(args&: AShadowPtr, args&: AOriginPtr) =
8923 MSV.getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(),
8924 Alignment: kShadowTLSAlignment, /*isStore*/ false);
8925
8926 IRB.CreateMemCpy(Dst: Base, DstAlign: kShadowTLSAlignment, Src: AShadowPtr,
8927 SrcAlign: kShadowTLSAlignment, Size: ArgSize);
8928 }
8929 }
8930 VAArgOffset += alignTo(Size: ArgSize, A: Align(8));
8931 } else {
8932 Value *Base;
8933 uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
8934 Align ArgAlign = Align(8);
8935 if (A->getType()->isArrayTy()) {
8936 // Arrays are aligned to element size, except for long double
8937 // arrays, which are aligned to 8 bytes.
8938 Type *ElementTy = A->getType()->getArrayElementType();
8939 if (!ElementTy->isPPC_FP128Ty())
8940 ArgAlign = Align(DL.getTypeAllocSize(Ty: ElementTy));
8941 } else if (A->getType()->isVectorTy()) {
8942 // Vectors are naturally aligned.
8943 ArgAlign = Align(ArgSize);
8944 }
8945 if (ArgAlign < 8)
8946 ArgAlign = Align(8);
8947 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
8948 if (DL.isBigEndian()) {
8949 // Adjusting the shadow for argument with size < 8 to match the
8950 // placement of bits in big endian system
8951 if (ArgSize < 8)
8952 VAArgOffset += (8 - ArgSize);
8953 }
8954 if (!IsFixed) {
8955 Base =
8956 getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset - VAArgBase, ArgSize);
8957 if (Base)
8958 IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base, Align: kShadowTLSAlignment);
8959 }
8960 VAArgOffset += ArgSize;
8961 VAArgOffset = alignTo(Size: VAArgOffset, A: Align(8));
8962 }
8963 if (IsFixed)
8964 VAArgBase = VAArgOffset;
8965 }
8966
8967 Constant *TotalVAArgSize =
8968 ConstantInt::get(Ty: MS.IntptrTy, V: VAArgOffset - VAArgBase);
8969 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
8970 // a new class member i.e. it is the total size of all VarArgs.
8971 IRB.CreateStore(Val: TotalVAArgSize, Ptr: MS.VAArgOverflowSizeTLS);
8972 }
8973
8974 void finalizeInstrumentation() override {
8975 assert(!VAArgSize && !VAArgTLSCopy &&
8976 "finalizeInstrumentation called twice");
8977 IRBuilder<> IRB(MSV.FnPrologueEnd);
8978 VAArgSize = IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
8979 Value *CopySize = VAArgSize;
8980
8981 if (!VAStartInstrumentationList.empty()) {
8982 // If there is a va_start in this function, make a backup copy of
8983 // va_arg_tls somewhere in the function entry block.
8984
8985 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
8986 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
8987 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
8988 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
8989
8990 Value *SrcSize = IRB.CreateBinaryIntrinsic(
8991 ID: Intrinsic::umin, LHS: CopySize,
8992 RHS: ConstantInt::get(Ty: IRB.getInt64Ty(), V: kParamTLSSize));
8993 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
8994 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
8995 }
8996
8997 // Instrument va_start.
8998 // Copy va_list shadow from the backup copy of the TLS contents.
8999 for (CallInst *OrigInst : VAStartInstrumentationList) {
9000 NextNodeIRBuilder IRB(OrigInst);
9001 Value *VAListTag = OrigInst->getArgOperand(i: 0);
9002 Value *RegSaveAreaPtrPtr = IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy);
9003
9004 RegSaveAreaPtrPtr = IRB.CreateIntToPtr(V: RegSaveAreaPtrPtr, DestTy: MS.PtrTy);
9005
9006 Value *RegSaveAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: RegSaveAreaPtrPtr);
9007 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
9008 const DataLayout &DL = F.getDataLayout();
9009 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
9010 const Align Alignment = Align(IntptrSize);
9011 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
9012 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
9013 Alignment, /*isStore*/ true);
9014 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
9015 Size: CopySize);
9016 }
9017 }
9018};
9019
9020/// PowerPC32-specific implementation of VarArgHelper.
9021struct VarArgPowerPC32Helper : public VarArgHelperBase {
9022 AllocaInst *VAArgTLSCopy = nullptr;
9023 Value *VAArgSize = nullptr;
9024
9025 VarArgPowerPC32Helper(Function &F, MemorySanitizer &MS,
9026 MemorySanitizerVisitor &MSV)
9027 : VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/12) {}
9028
9029 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
9030 unsigned VAArgBase;
9031 // Parameter save area is 8 bytes from frame pointer in PPC32
9032 VAArgBase = 8;
9033 unsigned VAArgOffset = VAArgBase;
9034 const DataLayout &DL = F.getDataLayout();
9035 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
9036 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
9037 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
9038 bool IsByVal = CB.isByValArgument(ArgNo);
9039 if (IsByVal) {
9040 assert(A->getType()->isPointerTy());
9041 Type *RealTy = CB.getParamByValType(ArgNo);
9042 uint64_t ArgSize = DL.getTypeAllocSize(Ty: RealTy);
9043 Align ArgAlign = CB.getParamAlign(ArgNo).value_or(u: Align(IntptrSize));
9044 if (ArgAlign < IntptrSize)
9045 ArgAlign = Align(IntptrSize);
9046 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
9047 if (!IsFixed) {
9048 Value *Base =
9049 getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset - VAArgBase, ArgSize);
9050 if (Base) {
9051 Value *AShadowPtr, *AOriginPtr;
9052 std::tie(args&: AShadowPtr, args&: AOriginPtr) =
9053 MSV.getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(),
9054 Alignment: kShadowTLSAlignment, /*isStore*/ false);
9055
9056 IRB.CreateMemCpy(Dst: Base, DstAlign: kShadowTLSAlignment, Src: AShadowPtr,
9057 SrcAlign: kShadowTLSAlignment, Size: ArgSize);
9058 }
9059 }
9060 VAArgOffset += alignTo(Size: ArgSize, A: Align(IntptrSize));
9061 } else {
9062 Value *Base;
9063 Type *ArgTy = A->getType();
9064
9065 // On PPC 32 floating point variable arguments are stored in separate
9066 // area: fp_save_area = reg_save_area + 4*8. We do not copy shaodow for
9067 // them as they will be found when checking call arguments.
9068 if (!ArgTy->isFloatingPointTy()) {
9069 uint64_t ArgSize = DL.getTypeAllocSize(Ty: ArgTy);
9070 Align ArgAlign = Align(IntptrSize);
9071 if (ArgTy->isArrayTy()) {
9072 // Arrays are aligned to element size, except for long double
9073 // arrays, which are aligned to 8 bytes.
9074 Type *ElementTy = ArgTy->getArrayElementType();
9075 if (!ElementTy->isPPC_FP128Ty())
9076 ArgAlign = Align(DL.getTypeAllocSize(Ty: ElementTy));
9077 } else if (ArgTy->isVectorTy()) {
9078 // Vectors are naturally aligned.
9079 ArgAlign = Align(ArgSize);
9080 }
9081 if (ArgAlign < IntptrSize)
9082 ArgAlign = Align(IntptrSize);
9083 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
9084 if (DL.isBigEndian()) {
9085 // Adjusting the shadow for argument with size < IntptrSize to match
9086 // the placement of bits in big endian system
9087 if (ArgSize < IntptrSize)
9088 VAArgOffset += (IntptrSize - ArgSize);
9089 }
9090 if (!IsFixed) {
9091 Base = getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset - VAArgBase,
9092 ArgSize);
9093 if (Base)
9094 IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base,
9095 Align: kShadowTLSAlignment);
9096 }
9097 VAArgOffset += ArgSize;
9098 VAArgOffset = alignTo(Size: VAArgOffset, A: Align(IntptrSize));
9099 }
9100 }
9101 }
9102
9103 Constant *TotalVAArgSize =
9104 ConstantInt::get(Ty: MS.IntptrTy, V: VAArgOffset - VAArgBase);
9105 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
9106 // a new class member i.e. it is the total size of all VarArgs.
9107 IRB.CreateStore(Val: TotalVAArgSize, Ptr: MS.VAArgOverflowSizeTLS);
9108 }
9109
9110 void finalizeInstrumentation() override {
9111 assert(!VAArgSize && !VAArgTLSCopy &&
9112 "finalizeInstrumentation called twice");
9113 IRBuilder<> IRB(MSV.FnPrologueEnd);
9114 VAArgSize = IRB.CreateLoad(Ty: MS.IntptrTy, Ptr: MS.VAArgOverflowSizeTLS);
9115 Value *CopySize = VAArgSize;
9116
9117 if (!VAStartInstrumentationList.empty()) {
9118 // If there is a va_start in this function, make a backup copy of
9119 // va_arg_tls somewhere in the function entry block.
9120
9121 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
9122 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
9123 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
9124 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
9125
9126 Value *SrcSize = IRB.CreateBinaryIntrinsic(
9127 ID: Intrinsic::umin, LHS: CopySize,
9128 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
9129 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
9130 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
9131 }
9132
9133 // Instrument va_start.
9134 // Copy va_list shadow from the backup copy of the TLS contents.
9135 for (CallInst *OrigInst : VAStartInstrumentationList) {
9136 NextNodeIRBuilder IRB(OrigInst);
9137 Value *VAListTag = OrigInst->getArgOperand(i: 0);
9138 Value *RegSaveAreaPtrPtr = IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy);
9139 Value *RegSaveAreaSize = CopySize;
9140
9141 // In PPC32 va_list_tag is a struct
9142 RegSaveAreaPtrPtr =
9143 IRB.CreateAdd(LHS: RegSaveAreaPtrPtr, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: 8));
9144
9145 // On PPC 32 reg_save_area can only hold 32 bytes of data
9146 RegSaveAreaSize = IRB.CreateBinaryIntrinsic(
9147 ID: Intrinsic::umin, LHS: CopySize, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: 32));
9148
9149 RegSaveAreaPtrPtr = IRB.CreateIntToPtr(V: RegSaveAreaPtrPtr, DestTy: MS.PtrTy);
9150 Value *RegSaveAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: RegSaveAreaPtrPtr);
9151
9152 const DataLayout &DL = F.getDataLayout();
9153 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
9154 const Align Alignment = Align(IntptrSize);
9155
9156 { // Copy reg save area
9157 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
9158 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
9159 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
9160 Alignment, /*isStore*/ true);
9161 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy,
9162 SrcAlign: Alignment, Size: RegSaveAreaSize);
9163
9164 RegSaveAreaShadowPtr =
9165 IRB.CreatePtrToInt(V: RegSaveAreaShadowPtr, DestTy: MS.IntptrTy);
9166 Value *FPSaveArea = IRB.CreateAdd(LHS: RegSaveAreaShadowPtr,
9167 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: 32));
9168 FPSaveArea = IRB.CreateIntToPtr(V: FPSaveArea, DestTy: MS.PtrTy);
9169 // We fill fp shadow with zeroes as uninitialized fp args should have
9170 // been found during call base check
9171 IRB.CreateMemSet(Ptr: FPSaveArea, Val: ConstantInt::getNullValue(Ty: IRB.getInt8Ty()),
9172 Size: ConstantInt::get(Ty: MS.IntptrTy, V: 32), Align: Alignment);
9173 }
9174
9175 { // Copy overflow area
9176 // RegSaveAreaSize is min(CopySize, 32) -> no overflow can occur
9177 Value *OverflowAreaSize = IRB.CreateSub(LHS: CopySize, RHS: RegSaveAreaSize);
9178
9179 Value *OverflowAreaPtrPtr = IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy);
9180 OverflowAreaPtrPtr =
9181 IRB.CreateAdd(LHS: OverflowAreaPtrPtr, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: 4));
9182 OverflowAreaPtrPtr = IRB.CreateIntToPtr(V: OverflowAreaPtrPtr, DestTy: MS.PtrTy);
9183
9184 Value *OverflowAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: OverflowAreaPtrPtr);
9185
9186 Value *OverflowAreaShadowPtr, *OverflowAreaOriginPtr;
9187 std::tie(args&: OverflowAreaShadowPtr, args&: OverflowAreaOriginPtr) =
9188 MSV.getShadowOriginPtr(Addr: OverflowAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
9189 Alignment, /*isStore*/ true);
9190
9191 Value *OverflowVAArgTLSCopyPtr =
9192 IRB.CreatePtrToInt(V: VAArgTLSCopy, DestTy: MS.IntptrTy);
9193 OverflowVAArgTLSCopyPtr =
9194 IRB.CreateAdd(LHS: OverflowVAArgTLSCopyPtr, RHS: RegSaveAreaSize);
9195
9196 OverflowVAArgTLSCopyPtr =
9197 IRB.CreateIntToPtr(V: OverflowVAArgTLSCopyPtr, DestTy: MS.PtrTy);
9198 IRB.CreateMemCpy(Dst: OverflowAreaShadowPtr, DstAlign: Alignment,
9199 Src: OverflowVAArgTLSCopyPtr, SrcAlign: Alignment, Size: OverflowAreaSize);
9200 }
9201 }
9202 }
9203};
9204
9205/// SystemZ-specific implementation of VarArgHelper.
9206struct VarArgSystemZHelper : public VarArgHelperBase {
9207 static const unsigned SystemZGpOffset = 16;
9208 static const unsigned SystemZGpEndOffset = 56;
9209 static const unsigned SystemZFpOffset = 128;
9210 static const unsigned SystemZFpEndOffset = 160;
9211 static const unsigned SystemZMaxVrArgs = 8;
9212 static const unsigned SystemZRegSaveAreaSize = 160;
9213 static const unsigned SystemZOverflowOffset = 160;
9214 static const unsigned SystemZVAListTagSize = 32;
9215 static const unsigned SystemZOverflowArgAreaPtrOffset = 16;
9216 static const unsigned SystemZRegSaveAreaPtrOffset = 24;
9217
9218 bool IsSoftFloatABI;
9219 AllocaInst *VAArgTLSCopy = nullptr;
9220 AllocaInst *VAArgTLSOriginCopy = nullptr;
9221 Value *VAArgOverflowSize = nullptr;
9222
9223 enum class ArgKind {
9224 GeneralPurpose,
9225 FloatingPoint,
9226 Vector,
9227 Memory,
9228 Indirect,
9229 };
9230
9231 enum class ShadowExtension { None, Zero, Sign };
9232
9233 VarArgSystemZHelper(Function &F, MemorySanitizer &MS,
9234 MemorySanitizerVisitor &MSV)
9235 : VarArgHelperBase(F, MS, MSV, SystemZVAListTagSize),
9236 IsSoftFloatABI(F.getFnAttribute(Kind: "use-soft-float").getValueAsBool()) {}
9237
9238 ArgKind classifyArgument(Type *T) {
9239 // T is a SystemZABIInfo::classifyArgumentType() output, and there are
9240 // only a few possibilities of what it can be. In particular, enums, single
9241 // element structs and large types have already been taken care of.
9242
9243 // Some i128 and fp128 arguments are converted to pointers only in the
9244 // back end.
9245 if (T->isIntegerTy(BitWidth: 128) || T->isFP128Ty())
9246 return ArgKind::Indirect;
9247 if (T->isFloatingPointTy())
9248 return IsSoftFloatABI ? ArgKind::GeneralPurpose : ArgKind::FloatingPoint;
9249 if (T->isIntegerTy() || T->isPointerTy())
9250 return ArgKind::GeneralPurpose;
9251 if (T->isVectorTy())
9252 return ArgKind::Vector;
9253 return ArgKind::Memory;
9254 }
9255
9256 ShadowExtension getShadowExtension(const CallBase &CB, unsigned ArgNo) {
9257 // ABI says: "One of the simple integer types no more than 64 bits wide.
9258 // ... If such an argument is shorter than 64 bits, replace it by a full
9259 // 64-bit integer representing the same number, using sign or zero
9260 // extension". Shadow for an integer argument has the same type as the
9261 // argument itself, so it can be sign or zero extended as well.
9262 bool ZExt = CB.paramHasAttr(ArgNo, Kind: Attribute::ZExt);
9263 bool SExt = CB.paramHasAttr(ArgNo, Kind: Attribute::SExt);
9264 if (ZExt) {
9265 assert(!SExt);
9266 return ShadowExtension::Zero;
9267 }
9268 if (SExt) {
9269 assert(!ZExt);
9270 return ShadowExtension::Sign;
9271 }
9272 return ShadowExtension::None;
9273 }
9274
9275 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
9276 unsigned GpOffset = SystemZGpOffset;
9277 unsigned FpOffset = SystemZFpOffset;
9278 unsigned VrIndex = 0;
9279 unsigned OverflowOffset = SystemZOverflowOffset;
9280 const DataLayout &DL = F.getDataLayout();
9281 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
9282 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
9283 // SystemZABIInfo does not produce ByVal parameters.
9284 assert(!CB.isByValArgument(ArgNo));
9285 Type *T = A->getType();
9286 ArgKind AK = classifyArgument(T);
9287 if (AK == ArgKind::Indirect) {
9288 T = MS.PtrTy;
9289 AK = ArgKind::GeneralPurpose;
9290 }
9291 if (AK == ArgKind::GeneralPurpose && GpOffset >= SystemZGpEndOffset)
9292 AK = ArgKind::Memory;
9293 if (AK == ArgKind::FloatingPoint && FpOffset >= SystemZFpEndOffset)
9294 AK = ArgKind::Memory;
9295 if (AK == ArgKind::Vector && (VrIndex >= SystemZMaxVrArgs || !IsFixed))
9296 AK = ArgKind::Memory;
9297 Value *ShadowBase = nullptr;
9298 Value *OriginBase = nullptr;
9299 ShadowExtension SE = ShadowExtension::None;
9300 switch (AK) {
9301 case ArgKind::GeneralPurpose: {
9302 // Always keep track of GpOffset, but store shadow only for varargs.
9303 uint64_t ArgSize = 8;
9304 if (GpOffset + ArgSize <= kParamTLSSize) {
9305 if (!IsFixed) {
9306 SE = getShadowExtension(CB, ArgNo);
9307 uint64_t GapSize = 0;
9308 if (SE == ShadowExtension::None) {
9309 uint64_t ArgAllocSize = DL.getTypeAllocSize(Ty: T);
9310 assert(ArgAllocSize <= ArgSize);
9311 GapSize = ArgSize - ArgAllocSize;
9312 }
9313 ShadowBase = getShadowAddrForVAArgument(IRB, ArgOffset: GpOffset + GapSize);
9314 if (MS.TrackOrigins)
9315 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: GpOffset + GapSize);
9316 }
9317 GpOffset += ArgSize;
9318 } else {
9319 GpOffset = kParamTLSSize;
9320 }
9321 break;
9322 }
9323 case ArgKind::FloatingPoint: {
9324 // Always keep track of FpOffset, but store shadow only for varargs.
9325 uint64_t ArgSize = 8;
9326 if (FpOffset + ArgSize <= kParamTLSSize) {
9327 if (!IsFixed) {
9328 // PoP says: "A short floating-point datum requires only the
9329 // left-most 32 bit positions of a floating-point register".
9330 // Therefore, in contrast to AK_GeneralPurpose and AK_Memory,
9331 // don't extend shadow and don't mind the gap.
9332 ShadowBase = getShadowAddrForVAArgument(IRB, ArgOffset: FpOffset);
9333 if (MS.TrackOrigins)
9334 OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: FpOffset);
9335 }
9336 FpOffset += ArgSize;
9337 } else {
9338 FpOffset = kParamTLSSize;
9339 }
9340 break;
9341 }
9342 case ArgKind::Vector: {
9343 // Keep track of VrIndex. No need to store shadow, since vector varargs
9344 // go through AK_Memory.
9345 assert(IsFixed);
9346 VrIndex++;
9347 break;
9348 }
9349 case ArgKind::Memory: {
9350 // Keep track of OverflowOffset and store shadow only for varargs.
9351 // Ignore fixed args, since we need to copy only the vararg portion of
9352 // the overflow area shadow.
9353 if (!IsFixed) {
9354 uint64_t ArgAllocSize = DL.getTypeAllocSize(Ty: T);
9355 uint64_t ArgSize = alignTo(Value: ArgAllocSize, Align: 8);
9356 if (OverflowOffset + ArgSize <= kParamTLSSize) {
9357 SE = getShadowExtension(CB, ArgNo);
9358 uint64_t GapSize =
9359 SE == ShadowExtension::None ? ArgSize - ArgAllocSize : 0;
9360 ShadowBase =
9361 getShadowAddrForVAArgument(IRB, ArgOffset: OverflowOffset + GapSize);
9362 if (MS.TrackOrigins)
9363 OriginBase =
9364 getOriginPtrForVAArgument(IRB, ArgOffset: OverflowOffset + GapSize);
9365 OverflowOffset += ArgSize;
9366 } else {
9367 OverflowOffset = kParamTLSSize;
9368 }
9369 }
9370 break;
9371 }
9372 case ArgKind::Indirect:
9373 llvm_unreachable("Indirect must be converted to GeneralPurpose");
9374 }
9375 if (ShadowBase == nullptr)
9376 continue;
9377 Value *Shadow = MSV.getShadow(V: A);
9378 if (SE != ShadowExtension::None)
9379 Shadow = MSV.CreateShadowCast(IRB, V: Shadow, dstTy: IRB.getInt64Ty(),
9380 /*Signed*/ SE == ShadowExtension::Sign);
9381 ShadowBase = IRB.CreateIntToPtr(V: ShadowBase, DestTy: MS.PtrTy, Name: "_msarg_va_s");
9382 IRB.CreateStore(Val: Shadow, Ptr: ShadowBase);
9383 if (MS.TrackOrigins) {
9384 Value *Origin = MSV.getOrigin(V: A);
9385 TypeSize StoreSize = DL.getTypeStoreSize(Ty: Shadow->getType());
9386 MSV.paintOrigin(IRB, Origin, OriginPtr: OriginBase, TS: StoreSize,
9387 Alignment: kMinOriginAlignment);
9388 }
9389 }
9390 Constant *OverflowSize = ConstantInt::get(
9391 Ty: IRB.getInt64Ty(), V: OverflowOffset - SystemZOverflowOffset);
9392 IRB.CreateStore(Val: OverflowSize, Ptr: MS.VAArgOverflowSizeTLS);
9393 }
9394
9395 void copyRegSaveArea(IRBuilder<> &IRB, Value *VAListTag) {
9396 Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
9397 V: IRB.CreateAdd(
9398 LHS: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
9399 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: SystemZRegSaveAreaPtrOffset)),
9400 DestTy: MS.PtrTy);
9401 Value *RegSaveAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: RegSaveAreaPtrPtr);
9402 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
9403 const Align Alignment = Align(8);
9404 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
9405 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(), Alignment,
9406 /*isStore*/ true);
9407 // TODO(iii): copy only fragments filled by visitCallBase()
9408 // TODO(iii): support packed-stack && !use-soft-float
9409 // For use-soft-float functions, it is enough to copy just the GPRs.
9410 unsigned RegSaveAreaSize =
9411 IsSoftFloatABI ? SystemZGpEndOffset : SystemZRegSaveAreaSize;
9412 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
9413 Size: RegSaveAreaSize);
9414 if (MS.TrackOrigins)
9415 IRB.CreateMemCpy(Dst: RegSaveAreaOriginPtr, DstAlign: Alignment, Src: VAArgTLSOriginCopy,
9416 SrcAlign: Alignment, Size: RegSaveAreaSize);
9417 }
9418
9419 // FIXME: This implementation limits OverflowOffset to kParamTLSSize, so we
9420 // don't know real overflow size and can't clear shadow beyond kParamTLSSize.
9421 void copyOverflowArea(IRBuilder<> &IRB, Value *VAListTag) {
9422 Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
9423 V: IRB.CreateAdd(
9424 LHS: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
9425 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: SystemZOverflowArgAreaPtrOffset)),
9426 DestTy: MS.PtrTy);
9427 Value *OverflowArgAreaPtr = IRB.CreateLoad(Ty: MS.PtrTy, Ptr: OverflowArgAreaPtrPtr);
9428 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
9429 const Align Alignment = Align(8);
9430 std::tie(args&: OverflowArgAreaShadowPtr, args&: OverflowArgAreaOriginPtr) =
9431 MSV.getShadowOriginPtr(Addr: OverflowArgAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
9432 Alignment, /*isStore*/ true);
9433 Value *SrcPtr = IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(), Ptr: VAArgTLSCopy,
9434 Idx0: SystemZOverflowOffset);
9435 IRB.CreateMemCpy(Dst: OverflowArgAreaShadowPtr, DstAlign: Alignment, Src: SrcPtr, SrcAlign: Alignment,
9436 Size: VAArgOverflowSize);
9437 if (MS.TrackOrigins) {
9438 SrcPtr = IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(), Ptr: VAArgTLSOriginCopy,
9439 Idx0: SystemZOverflowOffset);
9440 IRB.CreateMemCpy(Dst: OverflowArgAreaOriginPtr, DstAlign: Alignment, Src: SrcPtr, SrcAlign: Alignment,
9441 Size: VAArgOverflowSize);
9442 }
9443 }
9444
9445 void finalizeInstrumentation() override {
9446 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
9447 "finalizeInstrumentation called twice");
9448 if (!VAStartInstrumentationList.empty()) {
9449 // If there is a va_start in this function, make a backup copy of
9450 // va_arg_tls somewhere in the function entry block.
9451 IRBuilder<> IRB(MSV.FnPrologueEnd);
9452 VAArgOverflowSize =
9453 IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
9454 Value *CopySize =
9455 IRB.CreateAdd(LHS: ConstantInt::get(Ty: MS.IntptrTy, V: SystemZOverflowOffset),
9456 RHS: VAArgOverflowSize);
9457 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
9458 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
9459 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
9460 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
9461
9462 Value *SrcSize = IRB.CreateBinaryIntrinsic(
9463 ID: Intrinsic::umin, LHS: CopySize,
9464 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
9465 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
9466 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
9467 if (MS.TrackOrigins) {
9468 VAArgTLSOriginCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
9469 VAArgTLSOriginCopy->setAlignment(kShadowTLSAlignment);
9470 IRB.CreateMemCpy(Dst: VAArgTLSOriginCopy, DstAlign: kShadowTLSAlignment,
9471 Src: MS.VAArgOriginTLS, SrcAlign: kShadowTLSAlignment, Size: SrcSize);
9472 }
9473 }
9474
9475 // Instrument va_start.
9476 // Copy va_list shadow from the backup copy of the TLS contents.
9477 for (CallInst *OrigInst : VAStartInstrumentationList) {
9478 NextNodeIRBuilder IRB(OrigInst);
9479 Value *VAListTag = OrigInst->getArgOperand(i: 0);
9480 copyRegSaveArea(IRB, VAListTag);
9481 copyOverflowArea(IRB, VAListTag);
9482 }
9483 }
9484};
9485
9486/// i386-specific implementation of VarArgHelper.
9487struct VarArgI386Helper : public VarArgHelperBase {
9488 AllocaInst *VAArgTLSCopy = nullptr;
9489 Value *VAArgSize = nullptr;
9490
9491 VarArgI386Helper(Function &F, MemorySanitizer &MS,
9492 MemorySanitizerVisitor &MSV)
9493 : VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/4) {}
9494
9495 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
9496 const DataLayout &DL = F.getDataLayout();
9497 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
9498 unsigned VAArgOffset = 0;
9499 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
9500 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
9501 bool IsByVal = CB.isByValArgument(ArgNo);
9502 if (IsByVal) {
9503 assert(A->getType()->isPointerTy());
9504 Type *RealTy = CB.getParamByValType(ArgNo);
9505 uint64_t ArgSize = DL.getTypeAllocSize(Ty: RealTy);
9506 Align ArgAlign = CB.getParamAlign(ArgNo).value_or(u: Align(IntptrSize));
9507 if (ArgAlign < IntptrSize)
9508 ArgAlign = Align(IntptrSize);
9509 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
9510 if (!IsFixed) {
9511 Value *Base = getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset, ArgSize);
9512 if (Base) {
9513 Value *AShadowPtr, *AOriginPtr;
9514 std::tie(args&: AShadowPtr, args&: AOriginPtr) =
9515 MSV.getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(),
9516 Alignment: kShadowTLSAlignment, /*isStore*/ false);
9517
9518 IRB.CreateMemCpy(Dst: Base, DstAlign: kShadowTLSAlignment, Src: AShadowPtr,
9519 SrcAlign: kShadowTLSAlignment, Size: ArgSize);
9520 }
9521 VAArgOffset += alignTo(Size: ArgSize, A: Align(IntptrSize));
9522 }
9523 } else {
9524 Value *Base;
9525 uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
9526 Align ArgAlign = Align(IntptrSize);
9527 VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
9528 if (DL.isBigEndian()) {
9529 // Adjusting the shadow for argument with size < IntptrSize to match
9530 // the placement of bits in big endian system
9531 if (ArgSize < IntptrSize)
9532 VAArgOffset += (IntptrSize - ArgSize);
9533 }
9534 if (!IsFixed) {
9535 Base = getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset, ArgSize);
9536 if (Base)
9537 IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base, Align: kShadowTLSAlignment);
9538 VAArgOffset += ArgSize;
9539 VAArgOffset = alignTo(Size: VAArgOffset, A: Align(IntptrSize));
9540 }
9541 }
9542 }
9543
9544 Constant *TotalVAArgSize = ConstantInt::get(Ty: MS.IntptrTy, V: VAArgOffset);
9545 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
9546 // a new class member i.e. it is the total size of all VarArgs.
9547 IRB.CreateStore(Val: TotalVAArgSize, Ptr: MS.VAArgOverflowSizeTLS);
9548 }
9549
9550 void finalizeInstrumentation() override {
9551 assert(!VAArgSize && !VAArgTLSCopy &&
9552 "finalizeInstrumentation called twice");
9553 IRBuilder<> IRB(MSV.FnPrologueEnd);
9554 VAArgSize = IRB.CreateLoad(Ty: MS.IntptrTy, Ptr: MS.VAArgOverflowSizeTLS);
9555 Value *CopySize = VAArgSize;
9556
9557 if (!VAStartInstrumentationList.empty()) {
9558 // If there is a va_start in this function, make a backup copy of
9559 // va_arg_tls somewhere in the function entry block.
9560 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
9561 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
9562 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
9563 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
9564
9565 Value *SrcSize = IRB.CreateBinaryIntrinsic(
9566 ID: Intrinsic::umin, LHS: CopySize,
9567 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
9568 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
9569 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
9570 }
9571
9572 // Instrument va_start.
9573 // Copy va_list shadow from the backup copy of the TLS contents.
9574 for (CallInst *OrigInst : VAStartInstrumentationList) {
9575 NextNodeIRBuilder IRB(OrigInst);
9576 Value *VAListTag = OrigInst->getArgOperand(i: 0);
9577 Type *RegSaveAreaPtrTy = PointerType::getUnqual(C&: *MS.C);
9578 Value *RegSaveAreaPtrPtr =
9579 IRB.CreateIntToPtr(V: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
9580 DestTy: PointerType::get(C&: *MS.C, AddressSpace: 0));
9581 Value *RegSaveAreaPtr =
9582 IRB.CreateLoad(Ty: RegSaveAreaPtrTy, Ptr: RegSaveAreaPtrPtr);
9583 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
9584 const DataLayout &DL = F.getDataLayout();
9585 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
9586 const Align Alignment = Align(IntptrSize);
9587 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
9588 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
9589 Alignment, /*isStore*/ true);
9590 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
9591 Size: CopySize);
9592 }
9593 }
9594};
9595
9596/// Implementation of VarArgHelper that is used for ARM32, MIPS, RISCV,
9597/// LoongArch64.
9598struct VarArgGenericHelper : public VarArgHelperBase {
9599 AllocaInst *VAArgTLSCopy = nullptr;
9600 Value *VAArgSize = nullptr;
9601
9602 VarArgGenericHelper(Function &F, MemorySanitizer &MS,
9603 MemorySanitizerVisitor &MSV, const unsigned VAListTagSize)
9604 : VarArgHelperBase(F, MS, MSV, VAListTagSize) {}
9605
9606 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
9607 unsigned VAArgOffset = 0;
9608 const DataLayout &DL = F.getDataLayout();
9609 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
9610 for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
9611 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
9612 if (IsFixed)
9613 continue;
9614 uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
9615 if (DL.isBigEndian()) {
9616 // Adjusting the shadow for argument with size < IntptrSize to match the
9617 // placement of bits in big endian system
9618 if (ArgSize < IntptrSize)
9619 VAArgOffset += (IntptrSize - ArgSize);
9620 }
9621 Value *Base = getShadowPtrForVAArgument(IRB, ArgOffset: VAArgOffset, ArgSize);
9622 VAArgOffset += ArgSize;
9623 VAArgOffset = alignTo(Value: VAArgOffset, Align: IntptrSize);
9624 if (!Base)
9625 continue;
9626 IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base, Align: kShadowTLSAlignment);
9627 }
9628
9629 Constant *TotalVAArgSize = ConstantInt::get(Ty: MS.IntptrTy, V: VAArgOffset);
9630 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
9631 // a new class member i.e. it is the total size of all VarArgs.
9632 IRB.CreateStore(Val: TotalVAArgSize, Ptr: MS.VAArgOverflowSizeTLS);
9633 }
9634
9635 void finalizeInstrumentation() override {
9636 assert(!VAArgSize && !VAArgTLSCopy &&
9637 "finalizeInstrumentation called twice");
9638 IRBuilder<> IRB(MSV.FnPrologueEnd);
9639 VAArgSize = IRB.CreateLoad(Ty: MS.IntptrTy, Ptr: MS.VAArgOverflowSizeTLS);
9640 Value *CopySize = VAArgSize;
9641
9642 if (!VAStartInstrumentationList.empty()) {
9643 // If there is a va_start in this function, make a backup copy of
9644 // va_arg_tls somewhere in the function entry block.
9645 VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
9646 VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
9647 IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
9648 Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
9649
9650 Value *SrcSize = IRB.CreateBinaryIntrinsic(
9651 ID: Intrinsic::umin, LHS: CopySize,
9652 RHS: ConstantInt::get(Ty: MS.IntptrTy, V: kParamTLSSize));
9653 IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
9654 SrcAlign: kShadowTLSAlignment, Size: SrcSize);
9655 }
9656
9657 // Instrument va_start.
9658 // Copy va_list shadow from the backup copy of the TLS contents.
9659 for (CallInst *OrigInst : VAStartInstrumentationList) {
9660 NextNodeIRBuilder IRB(OrigInst);
9661 Value *VAListTag = OrigInst->getArgOperand(i: 0);
9662 Type *RegSaveAreaPtrTy = PointerType::getUnqual(C&: *MS.C);
9663 Value *RegSaveAreaPtrPtr =
9664 IRB.CreateIntToPtr(V: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
9665 DestTy: PointerType::get(C&: *MS.C, AddressSpace: 0));
9666 Value *RegSaveAreaPtr =
9667 IRB.CreateLoad(Ty: RegSaveAreaPtrTy, Ptr: RegSaveAreaPtrPtr);
9668 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
9669 const DataLayout &DL = F.getDataLayout();
9670 unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
9671 const Align Alignment = Align(IntptrSize);
9672 std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
9673 MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
9674 Alignment, /*isStore*/ true);
9675 IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
9676 Size: CopySize);
9677 }
9678 }
9679};
9680
9681// ARM32, Loongarch64, MIPS and RISCV share the same calling conventions
9682// regarding VAArgs.
9683using VarArgARM32Helper = VarArgGenericHelper;
9684using VarArgRISCVHelper = VarArgGenericHelper;
9685using VarArgMIPSHelper = VarArgGenericHelper;
9686using VarArgLoongArch64Helper = VarArgGenericHelper;
9687using VarArgHexagonHelper = VarArgGenericHelper;
9688
9689/// A no-op implementation of VarArgHelper.
9690struct VarArgNoOpHelper : public VarArgHelper {
9691 VarArgNoOpHelper(Function &F, MemorySanitizer &MS,
9692 MemorySanitizerVisitor &MSV) {}
9693
9694 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {}
9695
9696 void visitVAStartInst(VAStartInst &I) override {}
9697
9698 void visitVACopyInst(VACopyInst &I) override {}
9699
9700 void finalizeInstrumentation() override {}
9701};
9702
9703} // end anonymous namespace
9704
9705static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
9706 MemorySanitizerVisitor &Visitor) {
9707 // VarArg handling is only implemented on AMD64. False positives are possible
9708 // on other platforms.
9709 Triple TargetTriple(Func.getParent()->getTargetTriple());
9710
9711 if (TargetTriple.getArch() == Triple::x86)
9712 return new VarArgI386Helper(Func, Msan, Visitor);
9713
9714 if (TargetTriple.getArch() == Triple::x86_64)
9715 return new VarArgAMD64Helper(Func, Msan, Visitor);
9716
9717 if (TargetTriple.isARM())
9718 return new VarArgARM32Helper(Func, Msan, Visitor, /*VAListTagSize=*/4);
9719
9720 if (TargetTriple.isAArch64())
9721 return new VarArgAArch64Helper(Func, Msan, Visitor);
9722
9723 if (TargetTriple.isSystemZ())
9724 return new VarArgSystemZHelper(Func, Msan, Visitor);
9725
9726 // On PowerPC32 VAListTag is a struct
9727 // {char, char, i16 padding, char *, char *}
9728 if (TargetTriple.isPPC32())
9729 return new VarArgPowerPC32Helper(Func, Msan, Visitor);
9730
9731 if (TargetTriple.isPPC64())
9732 return new VarArgPowerPC64Helper(Func, Msan, Visitor);
9733
9734 if (TargetTriple.isRISCV32())
9735 return new VarArgRISCVHelper(Func, Msan, Visitor, /*VAListTagSize=*/4);
9736
9737 if (TargetTriple.isRISCV64())
9738 return new VarArgRISCVHelper(Func, Msan, Visitor, /*VAListTagSize=*/8);
9739
9740 if (TargetTriple.isMIPS32())
9741 return new VarArgMIPSHelper(Func, Msan, Visitor, /*VAListTagSize=*/4);
9742
9743 if (TargetTriple.isMIPS64())
9744 return new VarArgMIPSHelper(Func, Msan, Visitor, /*VAListTagSize=*/8);
9745
9746 if (TargetTriple.isLoongArch64())
9747 return new VarArgLoongArch64Helper(Func, Msan, Visitor,
9748 /*VAListTagSize=*/8);
9749
9750 if (TargetTriple.getArch() == Triple::hexagon)
9751 return new VarArgHexagonHelper(Func, Msan, Visitor, /*VAListTagSize=*/12);
9752
9753 return new VarArgNoOpHelper(Func, Msan, Visitor);
9754}
9755
9756bool MemorySanitizer::sanitizeFunction(Function &F, TargetLibraryInfo &TLI) {
9757 if (!CompileKernel && F.getName() == kMsanModuleCtorName)
9758 return false;
9759
9760 if (F.hasFnAttribute(Kind: Attribute::DisableSanitizerInstrumentation))
9761 return false;
9762
9763 MemorySanitizerVisitor Visitor(F, *this, TLI);
9764
9765 // Clear out memory attributes.
9766 AttributeMask B;
9767 B.addAttribute(Val: Attribute::Memory).addAttribute(Val: Attribute::Speculatable);
9768 F.removeFnAttrs(Attrs: B);
9769
9770 return Visitor.runOnFunction();
9771}
9772