1//===- MemProfInstrumentation.cpp - memory alloc and access instrumentation ==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of MemProf. Memory accesses are instrumented
10// to increment the access count held in a shadow memory location, or
11// alternatively to call into the runtime. Memory intrinsic calls (memmove,
12// memcpy, memset) are changed to call the memory profiling runtime version
13// instead.
14//
15//===----------------------------------------------------------------------===//
16
17#include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/ADT/Statistic.h"
20#include "llvm/ADT/StringRef.h"
21#include "llvm/Analysis/MemoryBuiltins.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/IR/Constant.h"
25#include "llvm/IR/DataLayout.h"
26#include "llvm/IR/DiagnosticInfo.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/GlobalValue.h"
29#include "llvm/IR/IRBuilder.h"
30#include "llvm/IR/Instruction.h"
31#include "llvm/IR/IntrinsicInst.h"
32#include "llvm/IR/Module.h"
33#include "llvm/IR/Type.h"
34#include "llvm/IR/Value.h"
35#include "llvm/ProfileData/InstrProf.h"
36#include "llvm/ProfileData/MemProf.h"
37#include "llvm/Support/CommandLine.h"
38#include "llvm/Support/Debug.h"
39#include "llvm/TargetParser/Triple.h"
40#include "llvm/Transforms/Utils/BasicBlockUtils.h"
41#include "llvm/Transforms/Utils/ModuleUtils.h"
42
43using namespace llvm;
44using namespace llvm::memprof;
45
46#define DEBUG_TYPE "memprof"
47
48constexpr int LLVM_MEM_PROFILER_VERSION = 1;
49
50// Size of memory mapped to a single shadow location.
51constexpr uint64_t DefaultMemGranularity = 64;
52
53// Size of memory mapped to a single histogram bucket.
54constexpr uint64_t HistogramGranularity = 8;
55
56// Scale from granularity down to shadow size.
57constexpr uint64_t DefaultShadowScale = 3;
58
59constexpr char MemProfModuleCtorName[] = "memprof.module_ctor";
60constexpr uint64_t MemProfCtorAndDtorPriority = 1;
61// On Emscripten, the system needs more than one priorities for constructors.
62constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50;
63constexpr char MemProfInitName[] = "__memprof_init";
64constexpr char MemProfVersionCheckNamePrefix[] =
65 "__memprof_version_mismatch_check_v";
66
67constexpr char MemProfShadowMemoryDynamicAddress[] =
68 "__memprof_shadow_memory_dynamic_address";
69
70constexpr char MemProfFilenameVar[] = "__memprof_profile_filename";
71
72constexpr char MemProfHistogramFlagVar[] = "__memprof_histogram";
73
74// Command-line flags.
75
76static cl::opt<bool> ClInsertVersionCheck(
77 "memprof-guard-against-version-mismatch",
78 cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,
79 cl::init(Val: true));
80
81// This flag may need to be replaced with -f[no-]memprof-reads.
82static cl::opt<bool> ClInstrumentReads("memprof-instrument-reads",
83 cl::desc("instrument read instructions"),
84 cl::Hidden, cl::init(Val: true));
85
86static cl::opt<bool>
87 ClInstrumentWrites("memprof-instrument-writes",
88 cl::desc("instrument write instructions"), cl::Hidden,
89 cl::init(Val: true));
90
91static cl::opt<bool> ClInstrumentAtomics(
92 "memprof-instrument-atomics",
93 cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
94 cl::init(Val: true));
95
96static cl::opt<bool> ClUseCalls(
97 "memprof-use-callbacks",
98 cl::desc("Use callbacks instead of inline instrumentation sequences."),
99 cl::Hidden, cl::init(Val: false));
100
101static cl::opt<std::string>
102 ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix",
103 cl::desc("Prefix for memory access callbacks"),
104 cl::Hidden, cl::init(Val: "__memprof_"));
105
106// These flags allow to change the shadow mapping.
107// The shadow mapping looks like
108// Shadow = ((Mem & mask) >> scale) + offset
109
110static cl::opt<int> ClMappingScale("memprof-mapping-scale",
111 cl::desc("scale of memprof shadow mapping"),
112 cl::Hidden, cl::init(Val: DefaultShadowScale));
113
114static cl::opt<int>
115 ClMappingGranularity("memprof-mapping-granularity",
116 cl::desc("granularity of memprof shadow mapping"),
117 cl::Hidden, cl::init(Val: DefaultMemGranularity));
118
119static cl::opt<bool> ClStack("memprof-instrument-stack",
120 cl::desc("Instrument scalar stack variables"),
121 cl::Hidden, cl::init(Val: false));
122
123// Debug flags.
124
125static cl::opt<int> ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden,
126 cl::init(Val: 0));
127
128static cl::opt<std::string> ClDebugFunc("memprof-debug-func", cl::Hidden,
129 cl::desc("Debug func"));
130
131static cl::opt<int> ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"),
132 cl::Hidden, cl::init(Val: -1));
133
134static cl::opt<int> ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"),
135 cl::Hidden, cl::init(Val: -1));
136
137static cl::opt<bool> ClHistogram("memprof-histogram",
138 cl::desc("Collect access count histograms"),
139 cl::Hidden, cl::init(Val: false));
140
141static cl::opt<std::string>
142 MemprofRuntimeDefaultOptions("memprof-runtime-default-options",
143 cl::desc("The default memprof options"),
144 cl::Hidden, cl::init(Val: ""));
145
146// Instrumentation statistics
147STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
148STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
149STATISTIC(NumSkippedStackReads, "Number of non-instrumented stack reads");
150STATISTIC(NumSkippedStackWrites, "Number of non-instrumented stack writes");
151
152namespace {
153
154/// This struct defines the shadow mapping using the rule:
155/// shadow = ((mem & mask) >> Scale) ADD DynamicShadowOffset.
156struct ShadowMapping {
157 ShadowMapping() {
158 Scale = ClMappingScale;
159 Granularity = ClHistogram ? HistogramGranularity : ClMappingGranularity;
160 Mask = ~(Granularity - 1);
161 }
162
163 int Scale;
164 int Granularity;
165 uint64_t Mask; // Computed as ~(Granularity-1)
166};
167
168static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) {
169 return TargetTriple.isOSEmscripten() ? MemProfEmscriptenCtorAndDtorPriority
170 : MemProfCtorAndDtorPriority;
171}
172
173struct InterestingMemoryAccess {
174 Value *Addr = nullptr;
175 bool IsWrite;
176 Type *AccessTy;
177 Value *MaybeMask = nullptr;
178};
179
180/// Instrument the code in module to profile memory accesses.
181class MemProfiler {
182public:
183 MemProfiler(Module &M) {
184 C = &(M.getContext());
185 LongSize = M.getDataLayout().getPointerSizeInBits();
186 IntptrTy = Type::getIntNTy(C&: *C, N: LongSize);
187 PtrTy = PointerType::getUnqual(C&: *C);
188 }
189
190 /// If it is an interesting memory access, populate information
191 /// about the access and return a InterestingMemoryAccess struct.
192 /// Otherwise return std::nullopt.
193 std::optional<InterestingMemoryAccess>
194 isInterestingMemoryAccess(Instruction *I) const;
195
196 void instrumentMop(Instruction *I, const DataLayout &DL,
197 InterestingMemoryAccess &Access);
198 void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
199 Value *Addr, bool IsWrite);
200 void instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
201 Instruction *I, Value *Addr, Type *AccessTy,
202 bool IsWrite);
203 void instrumentMemIntrinsic(MemIntrinsic *MI);
204 Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
205 bool instrumentFunction(Function &F);
206 bool maybeInsertMemProfInitAtFunctionEntry(Function &F);
207 bool insertDynamicShadowAtFunctionEntry(Function &F);
208
209private:
210 void initializeCallbacks(Module &M);
211
212 LLVMContext *C;
213 int LongSize;
214 Type *IntptrTy;
215 PointerType *PtrTy;
216 ShadowMapping Mapping;
217
218 // These arrays is indexed by AccessIsWrite
219 FunctionCallee MemProfMemoryAccessCallback[2];
220
221 FunctionCallee MemProfMemmove, MemProfMemcpy, MemProfMemset;
222 Value *DynamicShadowOffset = nullptr;
223};
224
225class ModuleMemProfiler {
226public:
227 ModuleMemProfiler(Module &M) { TargetTriple = M.getTargetTriple(); }
228
229 bool instrumentModule(Module &);
230
231private:
232 Triple TargetTriple;
233 ShadowMapping Mapping;
234 Function *MemProfCtorFunction = nullptr;
235};
236
237} // end anonymous namespace
238
239MemProfilerPass::MemProfilerPass() = default;
240
241PreservedAnalyses MemProfilerPass::run(Function &F,
242 AnalysisManager<Function> &AM) {
243 assert((!ClHistogram || ClMappingGranularity == DefaultMemGranularity) &&
244 "Memprof with histogram only supports default mapping granularity");
245 Module &M = *F.getParent();
246 MemProfiler Profiler(M);
247 if (Profiler.instrumentFunction(F))
248 return PreservedAnalyses::none();
249 return PreservedAnalyses::all();
250}
251
252ModuleMemProfilerPass::ModuleMemProfilerPass() = default;
253
254PreservedAnalyses ModuleMemProfilerPass::run(Module &M,
255 AnalysisManager<Module> &AM) {
256
257 ModuleMemProfiler Profiler(M);
258 if (Profiler.instrumentModule(M))
259 return PreservedAnalyses::none();
260 return PreservedAnalyses::all();
261}
262
263Value *MemProfiler::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
264 // (Shadow & mask) >> scale
265 Shadow = IRB.CreateAnd(LHS: Shadow, RHS: Mapping.Mask);
266 Shadow = IRB.CreateLShr(LHS: Shadow, RHS: Mapping.Scale);
267 // (Shadow >> scale) | offset
268 assert(DynamicShadowOffset);
269 return IRB.CreateAdd(LHS: Shadow, RHS: DynamicShadowOffset);
270}
271
272// Instrument memset/memmove/memcpy
273void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) {
274 IRBuilder<> IRB(MI);
275 if (isa<MemTransferInst>(Val: MI)) {
276 IRB.CreateCall(Callee: isa<MemMoveInst>(Val: MI) ? MemProfMemmove : MemProfMemcpy,
277 Args: {MI->getOperand(i_nocapture: 0), MI->getOperand(i_nocapture: 1),
278 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 2), DestTy: IntptrTy, isSigned: false)});
279 } else if (isa<MemSetInst>(Val: MI)) {
280 IRB.CreateCall(
281 Callee: MemProfMemset,
282 Args: {MI->getOperand(i_nocapture: 0),
283 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 1), DestTy: IRB.getInt32Ty(), isSigned: false),
284 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 2), DestTy: IntptrTy, isSigned: false)});
285 }
286 MI->eraseFromParent();
287}
288
289std::optional<InterestingMemoryAccess>
290MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
291 // Do not instrument the load fetching the dynamic shadow address.
292 if (DynamicShadowOffset == I)
293 return std::nullopt;
294
295 InterestingMemoryAccess Access;
296
297 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
298 if (!ClInstrumentReads)
299 return std::nullopt;
300 Access.IsWrite = false;
301 Access.AccessTy = LI->getType();
302 Access.Addr = LI->getPointerOperand();
303 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
304 if (!ClInstrumentWrites)
305 return std::nullopt;
306 Access.IsWrite = true;
307 Access.AccessTy = SI->getValueOperand()->getType();
308 Access.Addr = SI->getPointerOperand();
309 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: I)) {
310 if (!ClInstrumentAtomics)
311 return std::nullopt;
312 Access.IsWrite = true;
313 Access.AccessTy = RMW->getValOperand()->getType();
314 Access.Addr = RMW->getPointerOperand();
315 } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
316 if (!ClInstrumentAtomics)
317 return std::nullopt;
318 Access.IsWrite = true;
319 Access.AccessTy = XCHG->getCompareOperand()->getType();
320 Access.Addr = XCHG->getPointerOperand();
321 } else if (auto *CI = dyn_cast<CallInst>(Val: I)) {
322 auto *F = CI->getCalledFunction();
323 if (F && (F->getIntrinsicID() == Intrinsic::masked_load ||
324 F->getIntrinsicID() == Intrinsic::masked_store)) {
325 unsigned OpOffset = 0;
326 if (F->getIntrinsicID() == Intrinsic::masked_store) {
327 if (!ClInstrumentWrites)
328 return std::nullopt;
329 // Masked store has an initial operand for the value.
330 OpOffset = 1;
331 Access.AccessTy = CI->getArgOperand(i: 0)->getType();
332 Access.IsWrite = true;
333 } else {
334 if (!ClInstrumentReads)
335 return std::nullopt;
336 Access.AccessTy = CI->getType();
337 Access.IsWrite = false;
338 }
339
340 auto *BasePtr = CI->getOperand(i_nocapture: 0 + OpOffset);
341 Access.MaybeMask = CI->getOperand(i_nocapture: 1 + OpOffset);
342 Access.Addr = BasePtr;
343 }
344 }
345
346 if (!Access.Addr)
347 return std::nullopt;
348
349 // Do not instrument accesses from different address spaces; we cannot deal
350 // with them.
351 Type *PtrTy = cast<PointerType>(Val: Access.Addr->getType()->getScalarType());
352 if (PtrTy->getPointerAddressSpace() != 0)
353 return std::nullopt;
354
355 // Ignore swifterror addresses.
356 // swifterror memory addresses are mem2reg promoted by instruction
357 // selection. As such they cannot have regular uses like an instrumentation
358 // function and it makes no sense to track them as memory.
359 if (Access.Addr->isSwiftError())
360 return std::nullopt;
361
362 // Peel off GEPs and BitCasts.
363 auto *Addr = Access.Addr->stripInBoundsOffsets();
364
365 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: Addr)) {
366 // Do not instrument PGO counter updates.
367 if (GV->hasSection()) {
368 StringRef SectionName = GV->getSection();
369 // Check if the global is in the PGO counters section.
370 auto OF = I->getModule()->getTargetTriple().getObjectFormat();
371 if (SectionName.ends_with(
372 Suffix: getInstrProfSectionName(IPSK: IPSK_cnts, OF, /*AddSegmentInfo=*/false)))
373 return std::nullopt;
374 }
375
376 // Do not instrument accesses to LLVM internal variables.
377 if (GV->getName().starts_with(Prefix: "__llvm") ||
378 GV->getName().starts_with(Prefix: getInstrProfVarPrefix()))
379 return std::nullopt;
380 }
381
382 return Access;
383}
384
385void MemProfiler::instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
386 Instruction *I, Value *Addr,
387 Type *AccessTy, bool IsWrite) {
388 auto *VTy = cast<FixedVectorType>(Val: AccessTy);
389 unsigned Num = VTy->getNumElements();
390 auto *Zero = ConstantInt::get(Ty: IntptrTy, V: 0);
391 for (unsigned Idx = 0; Idx < Num; ++Idx) {
392 Value *InstrumentedAddress = nullptr;
393 Instruction *InsertBefore = I;
394 if (auto *Vector = dyn_cast<ConstantVector>(Val: Mask)) {
395 // dyn_cast as we might get UndefValue
396 if (auto *Masked = dyn_cast<ConstantInt>(Val: Vector->getOperand(i_nocapture: Idx))) {
397 if (Masked->isZero())
398 // Mask is constant false, so no instrumentation needed.
399 continue;
400 // If we have a true or undef value, fall through to instrumentAddress.
401 // with InsertBefore == I
402 }
403 } else {
404 IRBuilder<> IRB(I);
405 Value *MaskElem = IRB.CreateExtractElement(Vec: Mask, Idx);
406 Instruction *ThenTerm = SplitBlockAndInsertIfThen(Cond: MaskElem, SplitBefore: I, Unreachable: false);
407 InsertBefore = ThenTerm;
408 }
409
410 IRBuilder<> IRB(InsertBefore);
411 InstrumentedAddress =
412 IRB.CreateGEP(Ty: VTy, Ptr: Addr, IdxList: {Zero, ConstantInt::get(Ty: IntptrTy, V: Idx)});
413 instrumentAddress(OrigIns: I, InsertBefore, Addr: InstrumentedAddress, IsWrite);
414 }
415}
416
417void MemProfiler::instrumentMop(Instruction *I, const DataLayout &DL,
418 InterestingMemoryAccess &Access) {
419 // Skip instrumentation of stack accesses unless requested.
420 if (!ClStack && isa<AllocaInst>(Val: getUnderlyingObject(V: Access.Addr))) {
421 if (Access.IsWrite)
422 ++NumSkippedStackWrites;
423 else
424 ++NumSkippedStackReads;
425 return;
426 }
427
428 if (Access.IsWrite)
429 NumInstrumentedWrites++;
430 else
431 NumInstrumentedReads++;
432
433 if (Access.MaybeMask) {
434 instrumentMaskedLoadOrStore(DL, Mask: Access.MaybeMask, I, Addr: Access.Addr,
435 AccessTy: Access.AccessTy, IsWrite: Access.IsWrite);
436 } else {
437 // Since the access counts will be accumulated across the entire allocation,
438 // we only update the shadow access count for the first location and thus
439 // don't need to worry about alignment and type size.
440 instrumentAddress(OrigIns: I, InsertBefore: I, Addr: Access.Addr, IsWrite: Access.IsWrite);
441 }
442}
443
444void MemProfiler::instrumentAddress(Instruction *OrigIns,
445 Instruction *InsertBefore, Value *Addr,
446 bool IsWrite) {
447 IRBuilder<> IRB(InsertBefore);
448 Value *AddrLong = IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy);
449
450 if (ClUseCalls) {
451 IRB.CreateCall(Callee: MemProfMemoryAccessCallback[IsWrite], Args: AddrLong);
452 return;
453 }
454
455 Type *ShadowTy = ClHistogram ? Type::getInt8Ty(C&: *C) : Type::getInt64Ty(C&: *C);
456 Type *ShadowPtrTy = PointerType::get(C&: *C, AddressSpace: 0);
457
458 Value *ShadowPtr = memToShadow(Shadow: AddrLong, IRB);
459 Value *ShadowAddr = IRB.CreateIntToPtr(V: ShadowPtr, DestTy: ShadowPtrTy);
460 Value *ShadowValue = IRB.CreateLoad(Ty: ShadowTy, Ptr: ShadowAddr);
461 // If we are profiling with histograms, add overflow protection at 255.
462 if (ClHistogram) {
463 Value *MaxCount = ConstantInt::get(Ty: Type::getInt8Ty(C&: *C), V: 255);
464 Value *Cmp = IRB.CreateICmpULT(LHS: ShadowValue, RHS: MaxCount);
465 Instruction *IncBlock =
466 SplitBlockAndInsertIfThen(Cond: Cmp, SplitBefore: InsertBefore, /*Unreachable=*/false);
467 IRB.SetInsertPoint(IncBlock);
468 }
469 Value *Inc = ConstantInt::get(Ty: ShadowTy, V: 1);
470 ShadowValue = IRB.CreateAdd(LHS: ShadowValue, RHS: Inc);
471 IRB.CreateStore(Val: ShadowValue, Ptr: ShadowAddr);
472}
473
474// Create the variable for the profile file name.
475void createProfileFileNameVar(Module &M) {
476 const MDString *MemProfFilename =
477 dyn_cast_or_null<MDString>(Val: M.getModuleFlag(Key: "MemProfProfileFilename"));
478 if (!MemProfFilename)
479 return;
480 assert(!MemProfFilename->getString().empty() &&
481 "Unexpected MemProfProfileFilename metadata with empty string");
482 Constant *ProfileNameConst = ConstantDataArray::getString(
483 Context&: M.getContext(), Initializer: MemProfFilename->getString(), AddNull: true);
484 GlobalVariable *ProfileNameVar = new GlobalVariable(
485 M, ProfileNameConst->getType(), /*isConstant=*/true,
486 GlobalValue::WeakAnyLinkage, ProfileNameConst, MemProfFilenameVar);
487 const Triple &TT = M.getTargetTriple();
488 if (TT.supportsCOMDAT()) {
489 ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
490 ProfileNameVar->setComdat(M.getOrInsertComdat(Name: MemProfFilenameVar));
491 }
492}
493
494// Set MemprofHistogramFlag as a Global variable in IR. This makes it accessible
495// to the runtime, changing shadow count behavior.
496void createMemprofHistogramFlagVar(Module &M) {
497 const StringRef VarName(MemProfHistogramFlagVar);
498 Type *IntTy1 = Type::getInt1Ty(C&: M.getContext());
499 auto MemprofHistogramFlag = new GlobalVariable(
500 M, IntTy1, true, GlobalValue::WeakAnyLinkage,
501 Constant::getIntegerValue(Ty: IntTy1, V: APInt(1, ClHistogram)), VarName);
502 const Triple &TT = M.getTargetTriple();
503 if (TT.supportsCOMDAT()) {
504 MemprofHistogramFlag->setLinkage(GlobalValue::ExternalLinkage);
505 MemprofHistogramFlag->setComdat(M.getOrInsertComdat(Name: VarName));
506 }
507 appendToCompilerUsed(M, Values: MemprofHistogramFlag);
508}
509
510void createMemprofDefaultOptionsVar(Module &M) {
511 Constant *OptionsConst = ConstantDataArray::getString(
512 Context&: M.getContext(), Initializer: MemprofRuntimeDefaultOptions, /*AddNull=*/true);
513 GlobalVariable *OptionsVar =
514 new GlobalVariable(M, OptionsConst->getType(), /*isConstant=*/true,
515 GlobalValue::WeakAnyLinkage, OptionsConst,
516 memprof::getMemprofOptionsSymbolName());
517 const Triple &TT = M.getTargetTriple();
518 if (TT.supportsCOMDAT()) {
519 OptionsVar->setLinkage(GlobalValue::ExternalLinkage);
520 OptionsVar->setComdat(M.getOrInsertComdat(Name: OptionsVar->getName()));
521 }
522}
523
524bool ModuleMemProfiler::instrumentModule(Module &M) {
525
526 // Create a module constructor.
527 std::string MemProfVersion = std::to_string(val: LLVM_MEM_PROFILER_VERSION);
528 std::string VersionCheckName =
529 ClInsertVersionCheck ? (MemProfVersionCheckNamePrefix + MemProfVersion)
530 : "";
531 std::tie(args&: MemProfCtorFunction, args: std::ignore) =
532 createSanitizerCtorAndInitFunctions(M, CtorName: MemProfModuleCtorName,
533 InitName: MemProfInitName, /*InitArgTypes=*/{},
534 /*InitArgs=*/{}, VersionCheckName);
535
536 const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
537 appendToGlobalCtors(M, F: MemProfCtorFunction, Priority);
538
539 createProfileFileNameVar(M);
540
541 createMemprofHistogramFlagVar(M);
542
543 createMemprofDefaultOptionsVar(M);
544
545 return true;
546}
547
548void MemProfiler::initializeCallbacks(Module &M) {
549 IRBuilder<> IRB(*C);
550
551 for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
552 const std::string TypeStr = AccessIsWrite ? "store" : "load";
553 const std::string HistPrefix = ClHistogram ? "hist_" : "";
554
555 SmallVector<Type *, 2> Args1{1, IntptrTy};
556 MemProfMemoryAccessCallback[AccessIsWrite] = M.getOrInsertFunction(
557 Name: ClMemoryAccessCallbackPrefix + HistPrefix + TypeStr,
558 T: FunctionType::get(Result: IRB.getVoidTy(), Params: Args1, isVarArg: false));
559 }
560 MemProfMemmove = M.getOrInsertFunction(
561 Name: ClMemoryAccessCallbackPrefix + "memmove", RetTy: PtrTy, Args: PtrTy, Args: PtrTy, Args: IntptrTy);
562 MemProfMemcpy = M.getOrInsertFunction(Name: ClMemoryAccessCallbackPrefix + "memcpy",
563 RetTy: PtrTy, Args: PtrTy, Args: PtrTy, Args: IntptrTy);
564 MemProfMemset =
565 M.getOrInsertFunction(Name: ClMemoryAccessCallbackPrefix + "memset", RetTy: PtrTy,
566 Args: PtrTy, Args: IRB.getInt32Ty(), Args: IntptrTy);
567}
568
569bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) {
570 // For each NSObject descendant having a +load method, this method is invoked
571 // by the ObjC runtime before any of the static constructors is called.
572 // Therefore we need to instrument such methods with a call to __memprof_init
573 // at the beginning in order to initialize our runtime before any access to
574 // the shadow memory.
575 // We cannot just ignore these methods, because they may call other
576 // instrumented functions.
577 if (F.getName().contains(Other: " load]")) {
578 FunctionCallee MemProfInitFunction =
579 declareSanitizerInitFunction(M&: *F.getParent(), InitName: MemProfInitName, InitArgTypes: {});
580 IRBuilder<> IRB(&F.front(), F.front().begin());
581 IRB.CreateCall(Callee: MemProfInitFunction, Args: {});
582 return true;
583 }
584 return false;
585}
586
587bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {
588 IRBuilder<> IRB(&F.front().front());
589 Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
590 Name: MemProfShadowMemoryDynamicAddress, Ty: IntptrTy);
591 if (F.getParent()->getPICLevel() == PICLevel::NotPIC)
592 cast<GlobalVariable>(Val: GlobalDynamicAddress)->setDSOLocal(true);
593 DynamicShadowOffset = IRB.CreateLoad(Ty: IntptrTy, Ptr: GlobalDynamicAddress);
594 return true;
595}
596
597bool MemProfiler::instrumentFunction(Function &F) {
598 if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
599 return false;
600 if (ClDebugFunc == F.getName())
601 return false;
602 if (F.getName().starts_with(Prefix: "__memprof_"))
603 return false;
604
605 bool FunctionModified = false;
606
607 // If needed, insert __memprof_init.
608 // This function needs to be called even if the function body is not
609 // instrumented.
610 if (maybeInsertMemProfInitAtFunctionEntry(F))
611 FunctionModified = true;
612
613 LLVM_DEBUG(dbgs() << "MEMPROF instrumenting:\n" << F << "\n");
614
615 initializeCallbacks(M&: *F.getParent());
616
617 SmallVector<Instruction *, 16> ToInstrument;
618
619 // Fill the set of memory operations to instrument.
620 for (auto &BB : F) {
621 for (auto &Inst : BB) {
622 if (isInterestingMemoryAccess(I: &Inst) || isa<MemIntrinsic>(Val: Inst))
623 ToInstrument.push_back(Elt: &Inst);
624 }
625 }
626
627 if (ToInstrument.empty()) {
628 LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified
629 << " " << F << "\n");
630
631 return FunctionModified;
632 }
633
634 FunctionModified |= insertDynamicShadowAtFunctionEntry(F);
635
636 int NumInstrumented = 0;
637 for (auto *Inst : ToInstrument) {
638 if (ClDebugMin < 0 || ClDebugMax < 0 ||
639 (NumInstrumented >= ClDebugMin && NumInstrumented <= ClDebugMax)) {
640 std::optional<InterestingMemoryAccess> Access =
641 isInterestingMemoryAccess(I: Inst);
642 if (Access)
643 instrumentMop(I: Inst, DL: F.getDataLayout(), Access&: *Access);
644 else
645 instrumentMemIntrinsic(MI: cast<MemIntrinsic>(Val: Inst));
646 }
647 NumInstrumented++;
648 }
649
650 if (NumInstrumented > 0)
651 FunctionModified = true;
652
653 LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified << " "
654 << F << "\n");
655
656 return FunctionModified;
657}
658