1//===- MemProfInstrumentation.cpp - memory alloc and access instrumentation ==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of MemProf. Memory accesses are instrumented
10// to increment the access count held in a shadow memory location, or
11// alternatively to call into the runtime. Memory intrinsic calls (memmove,
12// memcpy, memset) are changed to call the memory profiling runtime version
13// instead.
14//
15//===----------------------------------------------------------------------===//
16
17#include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"
18#include "InstrumentationOptions.h"
19#include "llvm/ADT/SmallVector.h"
20#include "llvm/ADT/Statistic.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/Analysis/MemoryBuiltins.h"
23#include "llvm/Analysis/TargetLibraryInfo.h"
24#include "llvm/Analysis/ValueTracking.h"
25#include "llvm/IR/Constant.h"
26#include "llvm/IR/DataLayout.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/Function.h"
29#include "llvm/IR/GlobalValue.h"
30#include "llvm/IR/IRBuilder.h"
31#include "llvm/IR/Instruction.h"
32#include "llvm/IR/IntrinsicInst.h"
33#include "llvm/IR/Module.h"
34#include "llvm/IR/Type.h"
35#include "llvm/IR/Value.h"
36#include "llvm/ProfileData/InstrProf.h"
37#include "llvm/ProfileData/MemProf.h"
38#include "llvm/Support/Debug.h"
39#include "llvm/TargetParser/Triple.h"
40#include "llvm/Transforms/Utils/BasicBlockUtils.h"
41#include "llvm/Transforms/Utils/ModuleUtils.h"
42
43using namespace llvm;
44using namespace llvm::memprof;
45
46#define DEBUG_TYPE "memprof"
47
48constexpr int LLVM_MEM_PROFILER_VERSION = 1;
49
50// Size of memory mapped to a single shadow location.
51[[maybe_unused]] constexpr uint64_t DefaultMemGranularity = 64;
52
53// Size of memory mapped to a single histogram bucket.
54constexpr uint64_t HistogramGranularity = 8;
55
56constexpr char MemProfModuleCtorName[] = "memprof.module_ctor";
57constexpr uint64_t MemProfCtorAndDtorPriority = 1;
58// On Emscripten, the system needs more than one priorities for constructors.
59constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50;
60constexpr char MemProfInitName[] = "__memprof_init";
61constexpr char MemProfVersionCheckNamePrefix[] =
62 "__memprof_version_mismatch_check_v";
63
64constexpr char MemProfShadowMemoryDynamicAddress[] =
65 "__memprof_shadow_memory_dynamic_address";
66
67constexpr char MemProfFilenameVar[] = "__memprof_profile_filename";
68
69constexpr char MemProfHistogramFlagVar[] = "__memprof_histogram";
70
71// Instrumentation statistics
72STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
73STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
74STATISTIC(NumSkippedStackReads, "Number of non-instrumented stack reads");
75STATISTIC(NumSkippedStackWrites, "Number of non-instrumented stack writes");
76
77namespace {
78
79/// This struct defines the shadow mapping using the rule:
80/// shadow = ((mem & mask) >> Scale) ADD DynamicShadowOffset.
81struct ShadowMapping {
82 ShadowMapping(const InstrumentationOptions &Opts) {
83 Scale = Opts.memprof_mapping_scale;
84 Granularity = Opts.memprof_histogram ? HistogramGranularity
85 : Opts.memprof_mapping_granularity;
86 Mask = ~(Granularity - 1);
87 }
88
89 int Scale;
90 int Granularity;
91 uint64_t Mask; // Computed as ~(Granularity-1)
92};
93
94static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) {
95 return TargetTriple.isOSEmscripten() ? MemProfEmscriptenCtorAndDtorPriority
96 : MemProfCtorAndDtorPriority;
97}
98
99struct InterestingMemoryAccess {
100 Value *Addr = nullptr;
101 bool IsWrite;
102 Type *AccessTy;
103 Value *MaybeMask = nullptr;
104};
105
106/// Instrument the code in module to profile memory accesses.
107class MemProfiler {
108public:
109 MemProfiler(const InstrumentationOptions &Opts, Module &M)
110 : Opts(Opts), Mapping(Opts) {
111 C = &(M.getContext());
112 LongSize = M.getDataLayout().getPointerSizeInBits();
113 IntptrTy = Type::getIntNTy(C&: *C, N: LongSize);
114 PtrTy = PointerType::getUnqual(C&: *C);
115 }
116
117 /// If it is an interesting memory access, populate information
118 /// about the access and return a InterestingMemoryAccess struct.
119 /// Otherwise return std::nullopt.
120 std::optional<InterestingMemoryAccess>
121 isInterestingMemoryAccess(Instruction *I) const;
122
123 void instrumentMop(Instruction *I, const DataLayout &DL,
124 InterestingMemoryAccess &Access);
125 void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
126 Value *Addr, bool IsWrite);
127 void instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
128 Instruction *I, Value *Addr, Type *AccessTy,
129 bool IsWrite);
130 void instrumentMemIntrinsic(MemIntrinsic *MI);
131 Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
132 bool instrumentFunction(Function &F);
133 bool maybeInsertMemProfInitAtFunctionEntry(Function &F);
134 bool insertDynamicShadowAtFunctionEntry(Function &F);
135
136private:
137 void initializeCallbacks(Module &M);
138
139 const InstrumentationOptions &Opts;
140 LLVMContext *C;
141 int LongSize;
142 Type *IntptrTy;
143 PointerType *PtrTy;
144 ShadowMapping Mapping;
145
146 // These arrays is indexed by AccessIsWrite
147 FunctionCallee MemProfMemoryAccessCallback[2];
148
149 FunctionCallee MemProfMemmove, MemProfMemcpy, MemProfMemset;
150 Value *DynamicShadowOffset = nullptr;
151};
152
153class ModuleMemProfiler {
154public:
155 ModuleMemProfiler(const InstrumentationOptions &Opts, Module &M)
156 : Opts(Opts), Mapping(Opts) {
157 TargetTriple = M.getTargetTriple();
158 }
159
160 bool instrumentModule(Module &);
161
162private:
163 const InstrumentationOptions &Opts;
164 Triple TargetTriple;
165 ShadowMapping Mapping;
166 Function *MemProfCtorFunction = nullptr;
167};
168
169} // end anonymous namespace
170
171MemProfilerPass::MemProfilerPass() = default;
172
173PreservedAnalyses MemProfilerPass::run(Function &F,
174 AnalysisManager<Function> &AM) {
175 const InstrumentationOptions &Opts =
176 F.getContext().getLibraryOptions().get<InstrumentationOptions>();
177 assert((!Opts.memprof_histogram ||
178 Opts.memprof_mapping_granularity == DefaultMemGranularity) &&
179 "Memprof with histogram only supports default mapping granularity");
180 Module &M = *F.getParent();
181 MemProfiler Profiler(Opts, M);
182 if (Profiler.instrumentFunction(F))
183 return PreservedAnalyses::none();
184 return PreservedAnalyses::all();
185}
186
187ModuleMemProfilerPass::ModuleMemProfilerPass() = default;
188
189PreservedAnalyses ModuleMemProfilerPass::run(Module &M,
190 AnalysisManager<Module> &AM) {
191
192 const InstrumentationOptions &Opts =
193 M.getContext().getLibraryOptions().get<InstrumentationOptions>();
194 ModuleMemProfiler Profiler(Opts, M);
195 if (Profiler.instrumentModule(M))
196 return PreservedAnalyses::none();
197 return PreservedAnalyses::all();
198}
199
200Value *MemProfiler::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
201 // (Shadow & mask) >> scale
202 Shadow = IRB.CreateAnd(LHS: Shadow, RHS: Mapping.Mask);
203 Shadow = IRB.CreateLShr(LHS: Shadow, RHS: Mapping.Scale);
204 // (Shadow >> scale) | offset
205 assert(DynamicShadowOffset);
206 return IRB.CreateAdd(LHS: Shadow, RHS: DynamicShadowOffset);
207}
208
209// Instrument memset/memmove/memcpy
210void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) {
211 IRBuilder<> IRB(MI);
212 if (isa<MemTransferInst>(Val: MI)) {
213 IRB.CreateCall(Callee: isa<MemMoveInst>(Val: MI) ? MemProfMemmove : MemProfMemcpy,
214 Args: {MI->getOperand(i_nocapture: 0), MI->getOperand(i_nocapture: 1),
215 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 2), DestTy: IntptrTy, isSigned: false)});
216 } else if (isa<MemSetInst>(Val: MI)) {
217 IRB.CreateCall(
218 Callee: MemProfMemset,
219 Args: {MI->getOperand(i_nocapture: 0),
220 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 1), DestTy: IRB.getInt32Ty(), isSigned: false),
221 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 2), DestTy: IntptrTy, isSigned: false)});
222 }
223 MI->eraseFromParent();
224}
225
226std::optional<InterestingMemoryAccess>
227MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
228 // Do not instrument the load fetching the dynamic shadow address.
229 if (DynamicShadowOffset == I)
230 return std::nullopt;
231
232 InterestingMemoryAccess Access;
233
234 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
235 if (!Opts.memprof_instrument_reads)
236 return std::nullopt;
237 Access.IsWrite = false;
238 Access.AccessTy = LI->getType();
239 Access.Addr = LI->getPointerOperand();
240 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
241 if (!Opts.memprof_instrument_writes)
242 return std::nullopt;
243 Access.IsWrite = true;
244 Access.AccessTy = SI->getValueOperand()->getType();
245 Access.Addr = SI->getPointerOperand();
246 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: I)) {
247 if (!Opts.memprof_instrument_atomics)
248 return std::nullopt;
249 Access.IsWrite = true;
250 Access.AccessTy = RMW->getValOperand()->getType();
251 Access.Addr = RMW->getPointerOperand();
252 } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
253 if (!Opts.memprof_instrument_atomics)
254 return std::nullopt;
255 Access.IsWrite = true;
256 Access.AccessTy = XCHG->getCompareOperand()->getType();
257 Access.Addr = XCHG->getPointerOperand();
258 } else if (auto *CI = dyn_cast<CallInst>(Val: I)) {
259 auto *F = CI->getCalledFunction();
260 if (F && (F->getIntrinsicID() == Intrinsic::masked_load ||
261 F->getIntrinsicID() == Intrinsic::masked_store)) {
262 unsigned OpOffset = 0;
263 if (F->getIntrinsicID() == Intrinsic::masked_store) {
264 if (!Opts.memprof_instrument_writes)
265 return std::nullopt;
266 // Masked store has an initial operand for the value.
267 OpOffset = 1;
268 Access.AccessTy = CI->getArgOperand(i: 0)->getType();
269 Access.IsWrite = true;
270 } else {
271 if (!Opts.memprof_instrument_reads)
272 return std::nullopt;
273 Access.AccessTy = CI->getType();
274 Access.IsWrite = false;
275 }
276
277 auto *BasePtr = CI->getOperand(i_nocapture: 0 + OpOffset);
278 Access.MaybeMask = CI->getOperand(i_nocapture: 1 + OpOffset);
279 Access.Addr = BasePtr;
280 }
281 }
282
283 if (!Access.Addr)
284 return std::nullopt;
285
286 // Do not instrument accesses from different address spaces; we cannot deal
287 // with them.
288 Type *PtrTy = cast<PointerType>(Val: Access.Addr->getType()->getScalarType());
289 if (PtrTy->getPointerAddressSpace() != 0)
290 return std::nullopt;
291
292 // Ignore swifterror addresses.
293 // swifterror memory addresses are mem2reg promoted by instruction
294 // selection. As such they cannot have regular uses like an instrumentation
295 // function and it makes no sense to track them as memory.
296 if (Access.Addr->isSwiftError())
297 return std::nullopt;
298
299 // Peel off GEPs and BitCasts.
300 auto *Addr = Access.Addr->stripInBoundsOffsets();
301
302 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: Addr)) {
303 // Do not instrument PGO counter updates.
304 if (GV->hasSection()) {
305 StringRef SectionName = GV->getSection();
306 // Check if the global is in the PGO counters section.
307 auto OF = I->getModule()->getTargetTriple().getObjectFormat();
308 if (SectionName.ends_with(
309 Suffix: getInstrProfSectionName(IPSK: IPSK_cnts, OF, /*AddSegmentInfo=*/false)))
310 return std::nullopt;
311 }
312
313 // Do not instrument accesses to LLVM internal variables.
314 if (GV->getName().starts_with(Prefix: "__llvm") ||
315 GV->getName().starts_with(Prefix: getInstrProfVarPrefix()))
316 return std::nullopt;
317 }
318
319 return Access;
320}
321
322void MemProfiler::instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
323 Instruction *I, Value *Addr,
324 Type *AccessTy, bool IsWrite) {
325 auto *VTy = cast<FixedVectorType>(Val: AccessTy);
326 unsigned Num = VTy->getNumElements();
327 auto *Zero = ConstantInt::get(Ty: IntptrTy, V: 0);
328 for (unsigned Idx = 0; Idx < Num; ++Idx) {
329 Value *InstrumentedAddress = nullptr;
330 Instruction *InsertBefore = I;
331 if (auto *Vector = dyn_cast<ConstantVector>(Val: Mask)) {
332 // dyn_cast as we might get UndefValue
333 if (auto *Masked = dyn_cast<ConstantInt>(Val: Vector->getOperand(i_nocapture: Idx))) {
334 if (Masked->isZero())
335 // Mask is constant false, so no instrumentation needed.
336 continue;
337 // If we have a true or undef value, fall through to instrumentAddress.
338 // with InsertBefore == I
339 }
340 } else {
341 IRBuilder<> IRB(I);
342 Value *MaskElem = IRB.CreateExtractElement(Vec: Mask, Idx);
343 Instruction *ThenTerm = SplitBlockAndInsertIfThen(Cond: MaskElem, SplitBefore: I, Unreachable: false);
344 InsertBefore = ThenTerm;
345 }
346
347 IRBuilder<> IRB(InsertBefore);
348 InstrumentedAddress =
349 IRB.CreateGEP(Ty: VTy, Ptr: Addr, IdxList: {Zero, ConstantInt::get(Ty: IntptrTy, V: Idx)});
350 instrumentAddress(OrigIns: I, InsertBefore, Addr: InstrumentedAddress, IsWrite);
351 }
352}
353
354void MemProfiler::instrumentMop(Instruction *I, const DataLayout &DL,
355 InterestingMemoryAccess &Access) {
356 // Skip instrumentation of stack accesses unless requested.
357 if (!Opts.memprof_instrument_stack &&
358 isa<AllocaInst>(Val: getUnderlyingObject(V: Access.Addr))) {
359 if (Access.IsWrite)
360 ++NumSkippedStackWrites;
361 else
362 ++NumSkippedStackReads;
363 return;
364 }
365
366 if (Access.IsWrite)
367 NumInstrumentedWrites++;
368 else
369 NumInstrumentedReads++;
370
371 if (Access.MaybeMask) {
372 instrumentMaskedLoadOrStore(DL, Mask: Access.MaybeMask, I, Addr: Access.Addr,
373 AccessTy: Access.AccessTy, IsWrite: Access.IsWrite);
374 } else {
375 // Since the access counts will be accumulated across the entire allocation,
376 // we only update the shadow access count for the first location and thus
377 // don't need to worry about alignment and type size.
378 instrumentAddress(OrigIns: I, InsertBefore: I, Addr: Access.Addr, IsWrite: Access.IsWrite);
379 }
380}
381
382void MemProfiler::instrumentAddress(Instruction *OrigIns,
383 Instruction *InsertBefore, Value *Addr,
384 bool IsWrite) {
385 IRBuilder<> IRB(InsertBefore);
386 Value *AddrLong = IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy);
387
388 if (Opts.memprof_use_callbacks) {
389 IRB.CreateCall(Callee: MemProfMemoryAccessCallback[IsWrite], Args: AddrLong);
390 return;
391 }
392
393 Type *ShadowTy =
394 Opts.memprof_histogram ? Type::getInt8Ty(C&: *C) : Type::getInt64Ty(C&: *C);
395 Type *ShadowPtrTy = PointerType::get(C&: *C, AddressSpace: 0);
396
397 Value *ShadowPtr = memToShadow(Shadow: AddrLong, IRB);
398 Value *ShadowAddr = IRB.CreateIntToPtr(V: ShadowPtr, DestTy: ShadowPtrTy);
399 Value *ShadowValue = IRB.CreateLoad(Ty: ShadowTy, Ptr: ShadowAddr);
400 // If we are profiling with histograms, add overflow protection at 255.
401 if (Opts.memprof_histogram) {
402 Value *MaxCount = ConstantInt::get(Ty: Type::getInt8Ty(C&: *C), V: 255);
403 Value *Cmp = IRB.CreateICmpULT(LHS: ShadowValue, RHS: MaxCount);
404 Instruction *IncBlock =
405 SplitBlockAndInsertIfThen(Cond: Cmp, SplitBefore: InsertBefore, /*Unreachable=*/false);
406 IRB.SetInsertPoint(IncBlock);
407 }
408 Value *Inc = ConstantInt::get(Ty: ShadowTy, V: 1);
409 ShadowValue = IRB.CreateAdd(LHS: ShadowValue, RHS: Inc);
410 IRB.CreateStore(Val: ShadowValue, Ptr: ShadowAddr);
411}
412
413// Create the variable for the profile file name.
414void createProfileFileNameVar(Module &M) {
415 const MDString *MemProfFilename =
416 dyn_cast_or_null<MDString>(Val: M.getModuleFlag(Key: "MemProfProfileFilename"));
417 if (!MemProfFilename)
418 return;
419 assert(!MemProfFilename->getString().empty() &&
420 "Unexpected MemProfProfileFilename metadata with empty string");
421 Constant *ProfileNameConst = ConstantDataArray::getString(
422 Context&: M.getContext(), Initializer: MemProfFilename->getString(), AddNull: true);
423 GlobalVariable *ProfileNameVar = new GlobalVariable(
424 M, ProfileNameConst->getType(), /*isConstant=*/true,
425 GlobalValue::WeakAnyLinkage, ProfileNameConst, MemProfFilenameVar);
426 const Triple &TT = M.getTargetTriple();
427 if (TT.supportsCOMDAT()) {
428 ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
429 ProfileNameVar->setComdat(M.getOrInsertComdat(Name: MemProfFilenameVar));
430 }
431}
432
433// Set MemprofHistogramFlag as a Global variable in IR. This makes it accessible
434// to the runtime, changing shadow count behavior.
435void createMemprofHistogramFlagVar(const InstrumentationOptions &Opts,
436 Module &M) {
437 const StringRef VarName(MemProfHistogramFlagVar);
438 Type *IntTy1 = Type::getInt1Ty(C&: M.getContext());
439 auto MemprofHistogramFlag = new GlobalVariable(
440 M, IntTy1, true, GlobalValue::WeakAnyLinkage,
441 Constant::getIntegerValue(Ty: IntTy1, V: APInt(1, Opts.memprof_histogram)),
442 VarName);
443 const Triple &TT = M.getTargetTriple();
444 if (TT.supportsCOMDAT()) {
445 MemprofHistogramFlag->setLinkage(GlobalValue::ExternalLinkage);
446 MemprofHistogramFlag->setComdat(M.getOrInsertComdat(Name: VarName));
447 }
448 appendToCompilerUsed(M, Values: MemprofHistogramFlag);
449}
450
451void createMemprofDefaultOptionsVar(const InstrumentationOptions &Opts,
452 Module &M) {
453 Constant *OptionsConst = ConstantDataArray::getString(
454 Context&: M.getContext(), Initializer: Opts.memprof_runtime_default_options, /*AddNull=*/true);
455 GlobalVariable *OptionsVar =
456 new GlobalVariable(M, OptionsConst->getType(), /*isConstant=*/true,
457 GlobalValue::WeakAnyLinkage, OptionsConst,
458 memprof::getMemprofOptionsSymbolName());
459 const Triple &TT = M.getTargetTriple();
460 if (TT.supportsCOMDAT()) {
461 OptionsVar->setLinkage(GlobalValue::ExternalLinkage);
462 OptionsVar->setComdat(M.getOrInsertComdat(Name: OptionsVar->getName()));
463 }
464}
465
466bool ModuleMemProfiler::instrumentModule(Module &M) {
467
468 // Create a module constructor.
469 std::string MemProfVersion = std::to_string(val: LLVM_MEM_PROFILER_VERSION);
470 std::string VersionCheckName =
471 Opts.memprof_guard_against_version_mismatch
472 ? (MemProfVersionCheckNamePrefix + MemProfVersion)
473 : "";
474 std::tie(args&: MemProfCtorFunction, args: std::ignore) =
475 createSanitizerCtorAndInitFunctions(M, CtorName: MemProfModuleCtorName,
476 InitName: MemProfInitName, /*InitArgTypes=*/{},
477 /*InitArgs=*/{}, VersionCheckName);
478
479 const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
480 appendToGlobalCtors(M, F: MemProfCtorFunction, Priority);
481
482 createProfileFileNameVar(M);
483
484 createMemprofHistogramFlagVar(Opts, M);
485
486 createMemprofDefaultOptionsVar(Opts, M);
487
488 return true;
489}
490
491void MemProfiler::initializeCallbacks(Module &M) {
492 IRBuilder<> IRB(M);
493
494 for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
495 const std::string TypeStr = AccessIsWrite ? "store" : "load";
496 const std::string HistPrefix = Opts.memprof_histogram ? "hist_" : "";
497
498 SmallVector<Type *, 2> Args1{1, IntptrTy};
499 MemProfMemoryAccessCallback[AccessIsWrite] = M.getOrInsertFunction(
500 Name: (Opts.memprof_memory_access_callback_prefix + HistPrefix + TypeStr)
501 .str(),
502 T: FunctionType::get(Result: IRB.getVoidTy(), Params: Args1, isVarArg: false));
503 }
504 MemProfMemmove = M.getOrInsertFunction(
505 Name: (Opts.memprof_memory_access_callback_prefix + "memmove").str(), RetTy: PtrTy,
506 Args: PtrTy, Args: PtrTy, Args: IntptrTy);
507 MemProfMemcpy = M.getOrInsertFunction(
508 Name: (Opts.memprof_memory_access_callback_prefix + "memcpy").str(), RetTy: PtrTy,
509 Args: PtrTy, Args: PtrTy, Args: IntptrTy);
510 MemProfMemset = M.getOrInsertFunction(
511 Name: (Opts.memprof_memory_access_callback_prefix + "memset").str(), RetTy: PtrTy,
512 Args: PtrTy, Args: IRB.getInt32Ty(), Args: IntptrTy);
513}
514
515bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) {
516 // For each NSObject descendant having a +load method, this method is invoked
517 // by the ObjC runtime before any of the static constructors is called.
518 // Therefore we need to instrument such methods with a call to __memprof_init
519 // at the beginning in order to initialize our runtime before any access to
520 // the shadow memory.
521 // We cannot just ignore these methods, because they may call other
522 // instrumented functions.
523 if (F.getName().contains(Other: " load]")) {
524 FunctionCallee MemProfInitFunction =
525 declareSanitizerInitFunction(M&: *F.getParent(), InitName: MemProfInitName, InitArgTypes: {});
526 IRBuilder<> IRB(F.front().begin());
527 IRB.CreateCall(Callee: MemProfInitFunction, Args: {});
528 return true;
529 }
530 return false;
531}
532
533bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {
534 IRBuilder<> IRB(&F.front().front());
535 Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
536 Name: MemProfShadowMemoryDynamicAddress, Ty: IntptrTy);
537 if (F.getParent()->getPICLevel() == PICLevel::NotPIC)
538 cast<GlobalVariable>(Val: GlobalDynamicAddress)->setDSOLocal(true);
539 DynamicShadowOffset = IRB.CreateLoad(Ty: IntptrTy, Ptr: GlobalDynamicAddress);
540 return true;
541}
542
543bool MemProfiler::instrumentFunction(Function &F) {
544 if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
545 return false;
546 if (Opts.memprof_debug_func == F.getName())
547 return false;
548 if (F.getName().starts_with(Prefix: "__memprof_"))
549 return false;
550
551 bool FunctionModified = false;
552
553 // If needed, insert __memprof_init.
554 // This function needs to be called even if the function body is not
555 // instrumented.
556 if (maybeInsertMemProfInitAtFunctionEntry(F))
557 FunctionModified = true;
558
559 LLVM_DEBUG(dbgs() << "MEMPROF instrumenting:\n" << F << "\n");
560
561 initializeCallbacks(M&: *F.getParent());
562
563 SmallVector<Instruction *, 16> ToInstrument;
564
565 // Fill the set of memory operations to instrument.
566 for (auto &BB : F) {
567 for (auto &Inst : BB) {
568 if (isInterestingMemoryAccess(I: &Inst) || isa<MemIntrinsic>(Val: Inst))
569 ToInstrument.push_back(Elt: &Inst);
570 }
571 }
572
573 if (ToInstrument.empty()) {
574 LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified
575 << " " << F << "\n");
576
577 return FunctionModified;
578 }
579
580 FunctionModified |= insertDynamicShadowAtFunctionEntry(F);
581
582 int NumInstrumented = 0;
583 for (auto *Inst : ToInstrument) {
584 if (Opts.memprof_debug_min < 0 || Opts.memprof_debug_max < 0 ||
585 (NumInstrumented >= Opts.memprof_debug_min &&
586 NumInstrumented <= Opts.memprof_debug_max)) {
587 std::optional<InterestingMemoryAccess> Access =
588 isInterestingMemoryAccess(I: Inst);
589 if (Access)
590 instrumentMop(I: Inst, DL: F.getDataLayout(), Access&: *Access);
591 else
592 instrumentMemIntrinsic(MI: cast<MemIntrinsic>(Val: Inst));
593 }
594 NumInstrumented++;
595 }
596
597 if (NumInstrumented > 0)
598 FunctionModified = true;
599
600 LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified << " "
601 << F << "\n");
602
603 return FunctionModified;
604}
605