1//===- MemProfInstrumentation.cpp - memory alloc and access instrumentation ==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of MemProf. Memory accesses are instrumented
10// to increment the access count held in a shadow memory location, or
11// alternatively to call into the runtime. Memory intrinsic calls (memmove,
12// memcpy, memset) are changed to call the memory profiling runtime version
13// instead.
14//
15//===----------------------------------------------------------------------===//
16
17#include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/ADT/Statistic.h"
20#include "llvm/ADT/StringRef.h"
21#include "llvm/Analysis/MemoryBuiltins.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/IR/Constant.h"
25#include "llvm/IR/DataLayout.h"
26#include "llvm/IR/DiagnosticInfo.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/GlobalValue.h"
29#include "llvm/IR/IRBuilder.h"
30#include "llvm/IR/Instruction.h"
31#include "llvm/IR/IntrinsicInst.h"
32#include "llvm/IR/Module.h"
33#include "llvm/IR/Type.h"
34#include "llvm/IR/Value.h"
35#include "llvm/ProfileData/InstrProf.h"
36#include "llvm/ProfileData/MemProf.h"
37#include "llvm/Support/CommandLine.h"
38#include "llvm/Support/Debug.h"
39#include "llvm/TargetParser/Triple.h"
40#include "llvm/Transforms/Utils/BasicBlockUtils.h"
41#include "llvm/Transforms/Utils/ModuleUtils.h"
42
43using namespace llvm;
44using namespace llvm::memprof;
45
46#define DEBUG_TYPE "memprof"
47
48constexpr int LLVM_MEM_PROFILER_VERSION = 1;
49
50// Size of memory mapped to a single shadow location.
51constexpr uint64_t DefaultMemGranularity = 64;
52
53// Size of memory mapped to a single histogram bucket.
54constexpr uint64_t HistogramGranularity = 8;
55
56// Scale from granularity down to shadow size.
57constexpr uint64_t DefaultShadowScale = 3;
58
59constexpr char MemProfModuleCtorName[] = "memprof.module_ctor";
60constexpr uint64_t MemProfCtorAndDtorPriority = 1;
61// On Emscripten, the system needs more than one priorities for constructors.
62constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50;
63constexpr char MemProfInitName[] = "__memprof_init";
64constexpr char MemProfVersionCheckNamePrefix[] =
65 "__memprof_version_mismatch_check_v";
66
67constexpr char MemProfShadowMemoryDynamicAddress[] =
68 "__memprof_shadow_memory_dynamic_address";
69
70constexpr char MemProfFilenameVar[] = "__memprof_profile_filename";
71
72constexpr char MemProfHistogramFlagVar[] = "__memprof_histogram";
73
74// Command-line flags.
75
76static cl::opt<bool> ClInsertVersionCheck(
77 "memprof-guard-against-version-mismatch",
78 cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,
79 cl::init(Val: true));
80
81// This flag may need to be replaced with -f[no-]memprof-reads.
82static cl::opt<bool> ClInstrumentReads("memprof-instrument-reads",
83 cl::desc("instrument read instructions"),
84 cl::Hidden, cl::init(Val: true));
85
86static cl::opt<bool>
87 ClInstrumentWrites("memprof-instrument-writes",
88 cl::desc("instrument write instructions"), cl::Hidden,
89 cl::init(Val: true));
90
91static cl::opt<bool> ClInstrumentAtomics(
92 "memprof-instrument-atomics",
93 cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
94 cl::init(Val: true));
95
96static cl::opt<bool> ClUseCalls(
97 "memprof-use-callbacks",
98 cl::desc("Use callbacks instead of inline instrumentation sequences."),
99 cl::Hidden, cl::init(Val: false));
100
101static cl::opt<std::string>
102 ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix",
103 cl::desc("Prefix for memory access callbacks"),
104 cl::Hidden, cl::init(Val: "__memprof_"));
105
106// These flags allow to change the shadow mapping.
107// The shadow mapping looks like
108// Shadow = ((Mem & mask) >> scale) + offset
109
110static cl::opt<int> ClMappingScale("memprof-mapping-scale",
111 cl::desc("scale of memprof shadow mapping"),
112 cl::Hidden, cl::init(Val: DefaultShadowScale));
113
114static cl::opt<int>
115 ClMappingGranularity("memprof-mapping-granularity",
116 cl::desc("granularity of memprof shadow mapping"),
117 cl::Hidden, cl::init(Val: DefaultMemGranularity));
118
119static cl::opt<bool> ClStack("memprof-instrument-stack",
120 cl::desc("Instrument scalar stack variables"),
121 cl::Hidden, cl::init(Val: false));
122
123// Debug flags.
124
125static cl::opt<int> ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden,
126 cl::init(Val: 0));
127
128static cl::opt<std::string> ClDebugFunc("memprof-debug-func", cl::Hidden,
129 cl::desc("Debug func"));
130
131static cl::opt<int> ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"),
132 cl::Hidden, cl::init(Val: -1));
133
134static cl::opt<int> ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"),
135 cl::Hidden, cl::init(Val: -1));
136
137static cl::opt<bool> ClHistogram("memprof-histogram",
138 cl::desc("Collect access count histograms"),
139 cl::Hidden, cl::init(Val: false));
140
141static cl::opt<std::string>
142 MemprofRuntimeDefaultOptions("memprof-runtime-default-options",
143 cl::desc("The default memprof options"),
144 cl::Hidden, cl::init(Val: ""));
145
146// Instrumentation statistics
147STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
148STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
149STATISTIC(NumSkippedStackReads, "Number of non-instrumented stack reads");
150STATISTIC(NumSkippedStackWrites, "Number of non-instrumented stack writes");
151
152namespace {
153
154/// This struct defines the shadow mapping using the rule:
155/// shadow = ((mem & mask) >> Scale) ADD DynamicShadowOffset.
156struct ShadowMapping {
157 ShadowMapping() {
158 Scale = ClMappingScale;
159 Granularity = ClHistogram ? HistogramGranularity : ClMappingGranularity;
160 Mask = ~(Granularity - 1);
161 }
162
163 int Scale;
164 int Granularity;
165 uint64_t Mask; // Computed as ~(Granularity-1)
166};
167
168static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) {
169 return TargetTriple.isOSEmscripten() ? MemProfEmscriptenCtorAndDtorPriority
170 : MemProfCtorAndDtorPriority;
171}
172
173struct InterestingMemoryAccess {
174 Value *Addr = nullptr;
175 bool IsWrite;
176 Type *AccessTy;
177 Value *MaybeMask = nullptr;
178};
179
180/// Instrument the code in module to profile memory accesses.
181class MemProfiler {
182public:
183 MemProfiler(Module &M) {
184 C = &(M.getContext());
185 LongSize = M.getDataLayout().getPointerSizeInBits();
186 IntptrTy = Type::getIntNTy(C&: *C, N: LongSize);
187 PtrTy = PointerType::getUnqual(C&: *C);
188 }
189
190 /// If it is an interesting memory access, populate information
191 /// about the access and return a InterestingMemoryAccess struct.
192 /// Otherwise return std::nullopt.
193 std::optional<InterestingMemoryAccess>
194 isInterestingMemoryAccess(Instruction *I) const;
195
196 void instrumentMop(Instruction *I, const DataLayout &DL,
197 InterestingMemoryAccess &Access);
198 void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
199 Value *Addr, bool IsWrite);
200 void instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
201 Instruction *I, Value *Addr, Type *AccessTy,
202 bool IsWrite);
203 void instrumentMemIntrinsic(MemIntrinsic *MI);
204 Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
205 bool instrumentFunction(Function &F);
206 bool maybeInsertMemProfInitAtFunctionEntry(Function &F);
207 bool insertDynamicShadowAtFunctionEntry(Function &F);
208
209private:
210 void initializeCallbacks(Module &M);
211
212 LLVMContext *C;
213 int LongSize;
214 Type *IntptrTy;
215 PointerType *PtrTy;
216 ShadowMapping Mapping;
217
218 // These arrays is indexed by AccessIsWrite
219 FunctionCallee MemProfMemoryAccessCallback[2];
220
221 FunctionCallee MemProfMemmove, MemProfMemcpy, MemProfMemset;
222 Value *DynamicShadowOffset = nullptr;
223};
224
225class ModuleMemProfiler {
226public:
227 ModuleMemProfiler(Module &M) { TargetTriple = M.getTargetTriple(); }
228
229 bool instrumentModule(Module &);
230
231private:
232 Triple TargetTriple;
233 ShadowMapping Mapping;
234 Function *MemProfCtorFunction = nullptr;
235};
236
237} // end anonymous namespace
238
239MemProfilerPass::MemProfilerPass() = default;
240
241PreservedAnalyses MemProfilerPass::run(Function &F,
242 AnalysisManager<Function> &AM) {
243 assert((!ClHistogram || ClMappingGranularity == DefaultMemGranularity) &&
244 "Memprof with histogram only supports default mapping granularity");
245 Module &M = *F.getParent();
246 MemProfiler Profiler(M);
247 if (Profiler.instrumentFunction(F))
248 return PreservedAnalyses::none();
249 return PreservedAnalyses::all();
250}
251
252ModuleMemProfilerPass::ModuleMemProfilerPass() = default;
253
254PreservedAnalyses ModuleMemProfilerPass::run(Module &M,
255 AnalysisManager<Module> &AM) {
256
257 ModuleMemProfiler Profiler(M);
258 if (Profiler.instrumentModule(M))
259 return PreservedAnalyses::none();
260 return PreservedAnalyses::all();
261}
262
263Value *MemProfiler::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
264 // (Shadow & mask) >> scale
265 Shadow = IRB.CreateAnd(LHS: Shadow, RHS: Mapping.Mask);
266 Shadow = IRB.CreateLShr(LHS: Shadow, RHS: Mapping.Scale);
267 // (Shadow >> scale) | offset
268 assert(DynamicShadowOffset);
269 return IRB.CreateAdd(LHS: Shadow, RHS: DynamicShadowOffset);
270}
271
272// Instrument memset/memmove/memcpy
273void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) {
274 IRBuilder<> IRB(MI);
275 if (isa<MemTransferInst>(Val: MI)) {
276 IRB.CreateCall(Callee: isa<MemMoveInst>(Val: MI) ? MemProfMemmove : MemProfMemcpy,
277 Args: {MI->getOperand(i_nocapture: 0), MI->getOperand(i_nocapture: 1),
278 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 2), DestTy: IntptrTy, isSigned: false)});
279 } else if (isa<MemSetInst>(Val: MI)) {
280 IRB.CreateCall(
281 Callee: MemProfMemset,
282 Args: {MI->getOperand(i_nocapture: 0),
283 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 1), DestTy: IRB.getInt32Ty(), isSigned: false),
284 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 2), DestTy: IntptrTy, isSigned: false)});
285 }
286 MI->eraseFromParent();
287}
288
289std::optional<InterestingMemoryAccess>
290MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
291 // Do not instrument the load fetching the dynamic shadow address.
292 if (DynamicShadowOffset == I)
293 return std::nullopt;
294
295 InterestingMemoryAccess Access;
296
297 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
298 if (!ClInstrumentReads)
299 return std::nullopt;
300 Access.IsWrite = false;
301 Access.AccessTy = LI->getType();
302 Access.Addr = LI->getPointerOperand();
303 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
304 if (!ClInstrumentWrites)
305 return std::nullopt;
306 Access.IsWrite = true;
307 Access.AccessTy = SI->getValueOperand()->getType();
308 Access.Addr = SI->getPointerOperand();
309 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: I)) {
310 if (!ClInstrumentAtomics)
311 return std::nullopt;
312 Access.IsWrite = true;
313 Access.AccessTy = RMW->getValOperand()->getType();
314 Access.Addr = RMW->getPointerOperand();
315 } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
316 if (!ClInstrumentAtomics)
317 return std::nullopt;
318 Access.IsWrite = true;
319 Access.AccessTy = XCHG->getCompareOperand()->getType();
320 Access.Addr = XCHG->getPointerOperand();
321 } else if (auto *CI = dyn_cast<CallInst>(Val: I)) {
322 auto *F = CI->getCalledFunction();
323 if (F && (F->getIntrinsicID() == Intrinsic::masked_load ||
324 F->getIntrinsicID() == Intrinsic::masked_store)) {
325 unsigned OpOffset = 0;
326 if (F->getIntrinsicID() == Intrinsic::masked_store) {
327 if (!ClInstrumentWrites)
328 return std::nullopt;
329 // Masked store has an initial operand for the value.
330 OpOffset = 1;
331 Access.AccessTy = CI->getArgOperand(i: 0)->getType();
332 Access.IsWrite = true;
333 } else {
334 if (!ClInstrumentReads)
335 return std::nullopt;
336 Access.AccessTy = CI->getType();
337 Access.IsWrite = false;
338 }
339
340 auto *BasePtr = CI->getOperand(i_nocapture: 0 + OpOffset);
341 Access.MaybeMask = CI->getOperand(i_nocapture: 2 + OpOffset);
342 Access.Addr = BasePtr;
343 }
344 }
345
346 if (!Access.Addr)
347 return std::nullopt;
348
349 // Do not instrument accesses from different address spaces; we cannot deal
350 // with them.
351 Type *PtrTy = cast<PointerType>(Val: Access.Addr->getType()->getScalarType());
352 if (PtrTy->getPointerAddressSpace() != 0)
353 return std::nullopt;
354
355 // Ignore swifterror addresses.
356 // swifterror memory addresses are mem2reg promoted by instruction
357 // selection. As such they cannot have regular uses like an instrumentation
358 // function and it makes no sense to track them as memory.
359 if (Access.Addr->isSwiftError())
360 return std::nullopt;
361
362 // Peel off GEPs and BitCasts.
363 auto *Addr = Access.Addr->stripInBoundsOffsets();
364
365 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: Addr)) {
366 // Do not instrument PGO counter updates.
367 if (GV->hasSection()) {
368 StringRef SectionName = GV->getSection();
369 // Check if the global is in the PGO counters section.
370 auto OF = I->getModule()->getTargetTriple().getObjectFormat();
371 if (SectionName.ends_with(
372 Suffix: getInstrProfSectionName(IPSK: IPSK_cnts, OF, /*AddSegmentInfo=*/false)))
373 return std::nullopt;
374 }
375
376 // Do not instrument accesses to LLVM internal variables.
377 if (GV->getName().starts_with(Prefix: "__llvm"))
378 return std::nullopt;
379 }
380
381 return Access;
382}
383
384void MemProfiler::instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
385 Instruction *I, Value *Addr,
386 Type *AccessTy, bool IsWrite) {
387 auto *VTy = cast<FixedVectorType>(Val: AccessTy);
388 unsigned Num = VTy->getNumElements();
389 auto *Zero = ConstantInt::get(Ty: IntptrTy, V: 0);
390 for (unsigned Idx = 0; Idx < Num; ++Idx) {
391 Value *InstrumentedAddress = nullptr;
392 Instruction *InsertBefore = I;
393 if (auto *Vector = dyn_cast<ConstantVector>(Val: Mask)) {
394 // dyn_cast as we might get UndefValue
395 if (auto *Masked = dyn_cast<ConstantInt>(Val: Vector->getOperand(i_nocapture: Idx))) {
396 if (Masked->isZero())
397 // Mask is constant false, so no instrumentation needed.
398 continue;
399 // If we have a true or undef value, fall through to instrumentAddress.
400 // with InsertBefore == I
401 }
402 } else {
403 IRBuilder<> IRB(I);
404 Value *MaskElem = IRB.CreateExtractElement(Vec: Mask, Idx);
405 Instruction *ThenTerm = SplitBlockAndInsertIfThen(Cond: MaskElem, SplitBefore: I, Unreachable: false);
406 InsertBefore = ThenTerm;
407 }
408
409 IRBuilder<> IRB(InsertBefore);
410 InstrumentedAddress =
411 IRB.CreateGEP(Ty: VTy, Ptr: Addr, IdxList: {Zero, ConstantInt::get(Ty: IntptrTy, V: Idx)});
412 instrumentAddress(OrigIns: I, InsertBefore, Addr: InstrumentedAddress, IsWrite);
413 }
414}
415
416void MemProfiler::instrumentMop(Instruction *I, const DataLayout &DL,
417 InterestingMemoryAccess &Access) {
418 // Skip instrumentation of stack accesses unless requested.
419 if (!ClStack && isa<AllocaInst>(Val: getUnderlyingObject(V: Access.Addr))) {
420 if (Access.IsWrite)
421 ++NumSkippedStackWrites;
422 else
423 ++NumSkippedStackReads;
424 return;
425 }
426
427 if (Access.IsWrite)
428 NumInstrumentedWrites++;
429 else
430 NumInstrumentedReads++;
431
432 if (Access.MaybeMask) {
433 instrumentMaskedLoadOrStore(DL, Mask: Access.MaybeMask, I, Addr: Access.Addr,
434 AccessTy: Access.AccessTy, IsWrite: Access.IsWrite);
435 } else {
436 // Since the access counts will be accumulated across the entire allocation,
437 // we only update the shadow access count for the first location and thus
438 // don't need to worry about alignment and type size.
439 instrumentAddress(OrigIns: I, InsertBefore: I, Addr: Access.Addr, IsWrite: Access.IsWrite);
440 }
441}
442
443void MemProfiler::instrumentAddress(Instruction *OrigIns,
444 Instruction *InsertBefore, Value *Addr,
445 bool IsWrite) {
446 IRBuilder<> IRB(InsertBefore);
447 Value *AddrLong = IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy);
448
449 if (ClUseCalls) {
450 IRB.CreateCall(Callee: MemProfMemoryAccessCallback[IsWrite], Args: AddrLong);
451 return;
452 }
453
454 Type *ShadowTy = ClHistogram ? Type::getInt8Ty(C&: *C) : Type::getInt64Ty(C&: *C);
455 Type *ShadowPtrTy = PointerType::get(C&: *C, AddressSpace: 0);
456
457 Value *ShadowPtr = memToShadow(Shadow: AddrLong, IRB);
458 Value *ShadowAddr = IRB.CreateIntToPtr(V: ShadowPtr, DestTy: ShadowPtrTy);
459 Value *ShadowValue = IRB.CreateLoad(Ty: ShadowTy, Ptr: ShadowAddr);
460 // If we are profiling with histograms, add overflow protection at 255.
461 if (ClHistogram) {
462 Value *MaxCount = ConstantInt::get(Ty: Type::getInt8Ty(C&: *C), V: 255);
463 Value *Cmp = IRB.CreateICmpULT(LHS: ShadowValue, RHS: MaxCount);
464 Instruction *IncBlock =
465 SplitBlockAndInsertIfThen(Cond: Cmp, SplitBefore: InsertBefore, /*Unreachable=*/false);
466 IRB.SetInsertPoint(IncBlock);
467 }
468 Value *Inc = ConstantInt::get(Ty: ShadowTy, V: 1);
469 ShadowValue = IRB.CreateAdd(LHS: ShadowValue, RHS: Inc);
470 IRB.CreateStore(Val: ShadowValue, Ptr: ShadowAddr);
471}
472
473// Create the variable for the profile file name.
474void createProfileFileNameVar(Module &M) {
475 const MDString *MemProfFilename =
476 dyn_cast_or_null<MDString>(Val: M.getModuleFlag(Key: "MemProfProfileFilename"));
477 if (!MemProfFilename)
478 return;
479 assert(!MemProfFilename->getString().empty() &&
480 "Unexpected MemProfProfileFilename metadata with empty string");
481 Constant *ProfileNameConst = ConstantDataArray::getString(
482 Context&: M.getContext(), Initializer: MemProfFilename->getString(), AddNull: true);
483 GlobalVariable *ProfileNameVar = new GlobalVariable(
484 M, ProfileNameConst->getType(), /*isConstant=*/true,
485 GlobalValue::WeakAnyLinkage, ProfileNameConst, MemProfFilenameVar);
486 const Triple &TT = M.getTargetTriple();
487 if (TT.supportsCOMDAT()) {
488 ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
489 ProfileNameVar->setComdat(M.getOrInsertComdat(Name: MemProfFilenameVar));
490 }
491}
492
493// Set MemprofHistogramFlag as a Global veriable in IR. This makes it accessible
494// to the runtime, changing shadow count behavior.
495void createMemprofHistogramFlagVar(Module &M) {
496 const StringRef VarName(MemProfHistogramFlagVar);
497 Type *IntTy1 = Type::getInt1Ty(C&: M.getContext());
498 auto MemprofHistogramFlag = new GlobalVariable(
499 M, IntTy1, true, GlobalValue::WeakAnyLinkage,
500 Constant::getIntegerValue(Ty: IntTy1, V: APInt(1, ClHistogram)), VarName);
501 const Triple &TT = M.getTargetTriple();
502 if (TT.supportsCOMDAT()) {
503 MemprofHistogramFlag->setLinkage(GlobalValue::ExternalLinkage);
504 MemprofHistogramFlag->setComdat(M.getOrInsertComdat(Name: VarName));
505 }
506 appendToCompilerUsed(M, Values: MemprofHistogramFlag);
507}
508
509void createMemprofDefaultOptionsVar(Module &M) {
510 Constant *OptionsConst = ConstantDataArray::getString(
511 Context&: M.getContext(), Initializer: MemprofRuntimeDefaultOptions, /*AddNull=*/true);
512 GlobalVariable *OptionsVar =
513 new GlobalVariable(M, OptionsConst->getType(), /*isConstant=*/true,
514 GlobalValue::WeakAnyLinkage, OptionsConst,
515 memprof::getMemprofOptionsSymbolName());
516 const Triple &TT = M.getTargetTriple();
517 if (TT.supportsCOMDAT()) {
518 OptionsVar->setLinkage(GlobalValue::ExternalLinkage);
519 OptionsVar->setComdat(M.getOrInsertComdat(Name: OptionsVar->getName()));
520 }
521}
522
523bool ModuleMemProfiler::instrumentModule(Module &M) {
524
525 // Create a module constructor.
526 std::string MemProfVersion = std::to_string(val: LLVM_MEM_PROFILER_VERSION);
527 std::string VersionCheckName =
528 ClInsertVersionCheck ? (MemProfVersionCheckNamePrefix + MemProfVersion)
529 : "";
530 std::tie(args&: MemProfCtorFunction, args: std::ignore) =
531 createSanitizerCtorAndInitFunctions(M, CtorName: MemProfModuleCtorName,
532 InitName: MemProfInitName, /*InitArgTypes=*/{},
533 /*InitArgs=*/{}, VersionCheckName);
534
535 const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
536 appendToGlobalCtors(M, F: MemProfCtorFunction, Priority);
537
538 createProfileFileNameVar(M);
539
540 createMemprofHistogramFlagVar(M);
541
542 createMemprofDefaultOptionsVar(M);
543
544 return true;
545}
546
547void MemProfiler::initializeCallbacks(Module &M) {
548 IRBuilder<> IRB(*C);
549
550 for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
551 const std::string TypeStr = AccessIsWrite ? "store" : "load";
552 const std::string HistPrefix = ClHistogram ? "hist_" : "";
553
554 SmallVector<Type *, 2> Args1{1, IntptrTy};
555 MemProfMemoryAccessCallback[AccessIsWrite] = M.getOrInsertFunction(
556 Name: ClMemoryAccessCallbackPrefix + HistPrefix + TypeStr,
557 T: FunctionType::get(Result: IRB.getVoidTy(), Params: Args1, isVarArg: false));
558 }
559 MemProfMemmove = M.getOrInsertFunction(
560 Name: ClMemoryAccessCallbackPrefix + "memmove", RetTy: PtrTy, Args: PtrTy, Args: PtrTy, Args: IntptrTy);
561 MemProfMemcpy = M.getOrInsertFunction(Name: ClMemoryAccessCallbackPrefix + "memcpy",
562 RetTy: PtrTy, Args: PtrTy, Args: PtrTy, Args: IntptrTy);
563 MemProfMemset =
564 M.getOrInsertFunction(Name: ClMemoryAccessCallbackPrefix + "memset", RetTy: PtrTy,
565 Args: PtrTy, Args: IRB.getInt32Ty(), Args: IntptrTy);
566}
567
568bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) {
569 // For each NSObject descendant having a +load method, this method is invoked
570 // by the ObjC runtime before any of the static constructors is called.
571 // Therefore we need to instrument such methods with a call to __memprof_init
572 // at the beginning in order to initialize our runtime before any access to
573 // the shadow memory.
574 // We cannot just ignore these methods, because they may call other
575 // instrumented functions.
576 if (F.getName().contains(Other: " load]")) {
577 FunctionCallee MemProfInitFunction =
578 declareSanitizerInitFunction(M&: *F.getParent(), InitName: MemProfInitName, InitArgTypes: {});
579 IRBuilder<> IRB(&F.front(), F.front().begin());
580 IRB.CreateCall(Callee: MemProfInitFunction, Args: {});
581 return true;
582 }
583 return false;
584}
585
586bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {
587 IRBuilder<> IRB(&F.front().front());
588 Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
589 Name: MemProfShadowMemoryDynamicAddress, Ty: IntptrTy);
590 if (F.getParent()->getPICLevel() == PICLevel::NotPIC)
591 cast<GlobalVariable>(Val: GlobalDynamicAddress)->setDSOLocal(true);
592 DynamicShadowOffset = IRB.CreateLoad(Ty: IntptrTy, Ptr: GlobalDynamicAddress);
593 return true;
594}
595
596bool MemProfiler::instrumentFunction(Function &F) {
597 if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
598 return false;
599 if (ClDebugFunc == F.getName())
600 return false;
601 if (F.getName().starts_with(Prefix: "__memprof_"))
602 return false;
603
604 bool FunctionModified = false;
605
606 // If needed, insert __memprof_init.
607 // This function needs to be called even if the function body is not
608 // instrumented.
609 if (maybeInsertMemProfInitAtFunctionEntry(F))
610 FunctionModified = true;
611
612 LLVM_DEBUG(dbgs() << "MEMPROF instrumenting:\n" << F << "\n");
613
614 initializeCallbacks(M&: *F.getParent());
615
616 SmallVector<Instruction *, 16> ToInstrument;
617
618 // Fill the set of memory operations to instrument.
619 for (auto &BB : F) {
620 for (auto &Inst : BB) {
621 if (isInterestingMemoryAccess(I: &Inst) || isa<MemIntrinsic>(Val: Inst))
622 ToInstrument.push_back(Elt: &Inst);
623 }
624 }
625
626 if (ToInstrument.empty()) {
627 LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified
628 << " " << F << "\n");
629
630 return FunctionModified;
631 }
632
633 FunctionModified |= insertDynamicShadowAtFunctionEntry(F);
634
635 int NumInstrumented = 0;
636 for (auto *Inst : ToInstrument) {
637 if (ClDebugMin < 0 || ClDebugMax < 0 ||
638 (NumInstrumented >= ClDebugMin && NumInstrumented <= ClDebugMax)) {
639 std::optional<InterestingMemoryAccess> Access =
640 isInterestingMemoryAccess(I: Inst);
641 if (Access)
642 instrumentMop(I: Inst, DL: F.getDataLayout(), Access&: *Access);
643 else
644 instrumentMemIntrinsic(MI: cast<MemIntrinsic>(Val: Inst));
645 }
646 NumInstrumented++;
647 }
648
649 if (NumInstrumented > 0)
650 FunctionModified = true;
651
652 LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified << " "
653 << F << "\n");
654
655 return FunctionModified;
656}
657