1//===- MemProfInstrumentation.cpp - memory alloc and access instrumentation ==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of MemProf. Memory accesses are instrumented
10// to increment the access count held in a shadow memory location, or
11// alternatively to call into the runtime. Memory intrinsic calls (memmove,
12// memcpy, memset) are changed to call the memory profiling runtime version
13// instead.
14//
15//===----------------------------------------------------------------------===//
16
17#include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"
18#include "InstrumentationOptions.h"
19#include "llvm/ADT/SmallVector.h"
20#include "llvm/ADT/Statistic.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/Analysis/MemoryBuiltins.h"
23#include "llvm/Analysis/TargetLibraryInfo.h"
24#include "llvm/Analysis/ValueTracking.h"
25#include "llvm/IR/Constant.h"
26#include "llvm/IR/DataLayout.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/Function.h"
29#include "llvm/IR/GlobalValue.h"
30#include "llvm/IR/IRBuilder.h"
31#include "llvm/IR/Instruction.h"
32#include "llvm/IR/IntrinsicInst.h"
33#include "llvm/IR/Module.h"
34#include "llvm/IR/Type.h"
35#include "llvm/IR/Value.h"
36#include "llvm/ProfileData/InstrProf.h"
37#include "llvm/ProfileData/MemProf.h"
38#include "llvm/Support/Debug.h"
39#include "llvm/TargetParser/Triple.h"
40#include "llvm/Transforms/Utils/BasicBlockUtils.h"
41#include "llvm/Transforms/Utils/ModuleUtils.h"
42
43using namespace llvm;
44using namespace llvm::memprof;
45
46#define DEBUG_TYPE "memprof"
47
48constexpr int LLVM_MEM_PROFILER_VERSION = 1;
49
50// Size of memory mapped to a single shadow location.
51[[maybe_unused]] constexpr uint64_t DefaultMemGranularity = 64;
52
53// Size of memory mapped to a single histogram bucket.
54constexpr uint64_t HistogramGranularity = 8;
55
56constexpr char MemProfModuleCtorName[] = "memprof.module_ctor";
57constexpr uint64_t MemProfCtorAndDtorPriority = 1;
58// On Emscripten, the system needs more than one priorities for constructors.
59constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50;
60constexpr char MemProfInitName[] = "__memprof_init";
61constexpr char MemProfVersionCheckNamePrefix[] =
62 "__memprof_version_mismatch_check_v";
63
64constexpr char MemProfShadowMemoryDynamicAddress[] =
65 "__memprof_shadow_memory_dynamic_address";
66
67constexpr char MemProfFilenameVar[] = "__memprof_profile_filename";
68
69constexpr char MemProfHistogramFlagVar[] = "__memprof_histogram";
70
71// Instrumentation statistics
72STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
73STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
74STATISTIC(NumSkippedStackReads, "Number of non-instrumented stack reads");
75STATISTIC(NumSkippedStackWrites, "Number of non-instrumented stack writes");
76
77namespace {
78
79/// This struct defines the shadow mapping using the rule:
80/// shadow = ((mem & mask) >> Scale) ADD DynamicShadowOffset.
81struct ShadowMapping {
82 ShadowMapping(const InstrumentationOptions &Opts) {
83 Scale = Opts.memprof_mapping_scale;
84 Granularity = Opts.memprof_histogram ? HistogramGranularity
85 : Opts.memprof_mapping_granularity;
86 Mask = ~(Granularity - 1);
87 }
88
89 int Scale;
90 int Granularity;
91 uint64_t Mask; // Computed as ~(Granularity-1)
92};
93
94static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) {
95 return TargetTriple.isOSEmscripten() ? MemProfEmscriptenCtorAndDtorPriority
96 : MemProfCtorAndDtorPriority;
97}
98
99struct InterestingMemoryAccess {
100 Value *Addr = nullptr;
101 bool IsWrite;
102 Type *AccessTy;
103 Value *MaybeMask = nullptr;
104};
105
106/// Instrument the code in module to profile memory accesses.
107class MemProfiler {
108public:
109 MemProfiler(const InstrumentationOptions &Opts, Module &M)
110 : Opts(Opts), Mapping(Opts) {
111 C = &(M.getContext());
112 LongSize = M.getDataLayout().getPointerSizeInBits();
113 IntptrTy = Type::getIntNTy(C&: *C, N: LongSize);
114 PtrTy = PointerType::getUnqual(C&: *C);
115 }
116
117 /// If it is an interesting memory access, populate information
118 /// about the access and return a InterestingMemoryAccess struct.
119 /// Otherwise return std::nullopt.
120 std::optional<InterestingMemoryAccess>
121 isInterestingMemoryAccess(Instruction *I) const;
122
123 void instrumentMop(Instruction *I, const DataLayout &DL,
124 InterestingMemoryAccess &Access);
125 void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
126 Value *Addr, bool IsWrite);
127 void instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
128 Instruction *I, Value *Addr, Type *AccessTy,
129 bool IsWrite);
130 void instrumentMemIntrinsic(MemIntrinsic *MI);
131 Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
132 bool instrumentFunction(Function &F);
133 bool maybeInsertMemProfInitAtFunctionEntry(Function &F);
134 bool insertDynamicShadowAtFunctionEntry(Function &F);
135
136private:
137 void initializeCallbacks(Module &M);
138
139 const InstrumentationOptions &Opts;
140 LLVMContext *C;
141 int LongSize;
142 Type *IntptrTy;
143 PointerType *PtrTy;
144 ShadowMapping Mapping;
145
146 // These arrays is indexed by AccessIsWrite
147 FunctionCallee MemProfMemoryAccessCallback[2];
148
149 FunctionCallee MemProfMemmove, MemProfMemcpy, MemProfMemset;
150 Value *DynamicShadowOffset = nullptr;
151};
152
153class ModuleMemProfiler {
154public:
155 ModuleMemProfiler(const InstrumentationOptions &Opts, Module &M)
156 : Opts(Opts), Mapping(Opts) {
157 TargetTriple = M.getTargetTriple();
158 }
159
160 bool instrumentModule(Module &);
161
162private:
163 const InstrumentationOptions &Opts;
164 Triple TargetTriple;
165 ShadowMapping Mapping;
166 Function *MemProfCtorFunction = nullptr;
167};
168
169} // end anonymous namespace
170
171MemProfilerPass::MemProfilerPass() = default;
172
173PreservedAnalyses MemProfilerPass::run(Function &F,
174 AnalysisManager<Function> &AM) {
175 const InstrumentationOptions &Opts = getInstrumentationOptions(F);
176 assert((!Opts.memprof_histogram ||
177 Opts.memprof_mapping_granularity == DefaultMemGranularity) &&
178 "Memprof with histogram only supports default mapping granularity");
179 Module &M = *F.getParent();
180 MemProfiler Profiler(Opts, M);
181 if (Profiler.instrumentFunction(F))
182 return PreservedAnalyses::none();
183 return PreservedAnalyses::all();
184}
185
186ModuleMemProfilerPass::ModuleMemProfilerPass() = default;
187
188PreservedAnalyses ModuleMemProfilerPass::run(Module &M,
189 AnalysisManager<Module> &AM) {
190
191 const InstrumentationOptions &Opts = getInstrumentationOptions(M);
192 ModuleMemProfiler Profiler(Opts, M);
193 if (Profiler.instrumentModule(M))
194 return PreservedAnalyses::none();
195 return PreservedAnalyses::all();
196}
197
198Value *MemProfiler::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
199 // (Shadow & mask) >> scale
200 Shadow = IRB.CreateAnd(LHS: Shadow, RHS: Mapping.Mask);
201 Shadow = IRB.CreateLShr(LHS: Shadow, RHS: Mapping.Scale);
202 // (Shadow >> scale) | offset
203 assert(DynamicShadowOffset);
204 return IRB.CreateAdd(LHS: Shadow, RHS: DynamicShadowOffset);
205}
206
207// Instrument memset/memmove/memcpy
208void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) {
209 IRBuilder<> IRB(MI);
210 if (isa<MemTransferInst>(Val: MI)) {
211 IRB.CreateCall(Callee: isa<MemMoveInst>(Val: MI) ? MemProfMemmove : MemProfMemcpy,
212 Args: {MI->getOperand(i_nocapture: 0), MI->getOperand(i_nocapture: 1),
213 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 2), DestTy: IntptrTy, isSigned: false)});
214 } else if (isa<MemSetInst>(Val: MI)) {
215 IRB.CreateCall(
216 Callee: MemProfMemset,
217 Args: {MI->getOperand(i_nocapture: 0),
218 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 1), DestTy: IRB.getInt32Ty(), isSigned: false),
219 IRB.CreateIntCast(V: MI->getOperand(i_nocapture: 2), DestTy: IntptrTy, isSigned: false)});
220 }
221 MI->eraseFromParent();
222}
223
224std::optional<InterestingMemoryAccess>
225MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
226 // Do not instrument the load fetching the dynamic shadow address.
227 if (DynamicShadowOffset == I)
228 return std::nullopt;
229
230 InterestingMemoryAccess Access;
231
232 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
233 if (!Opts.memprof_instrument_reads)
234 return std::nullopt;
235 Access.IsWrite = false;
236 Access.AccessTy = LI->getType();
237 Access.Addr = LI->getPointerOperand();
238 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
239 if (!Opts.memprof_instrument_writes)
240 return std::nullopt;
241 Access.IsWrite = true;
242 Access.AccessTy = SI->getValueOperand()->getType();
243 Access.Addr = SI->getPointerOperand();
244 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: I)) {
245 if (!Opts.memprof_instrument_atomics)
246 return std::nullopt;
247 Access.IsWrite = true;
248 Access.AccessTy = RMW->getValOperand()->getType();
249 Access.Addr = RMW->getPointerOperand();
250 } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
251 if (!Opts.memprof_instrument_atomics)
252 return std::nullopt;
253 Access.IsWrite = true;
254 Access.AccessTy = XCHG->getCompareOperand()->getType();
255 Access.Addr = XCHG->getPointerOperand();
256 } else if (auto *CI = dyn_cast<CallInst>(Val: I)) {
257 auto *F = CI->getCalledFunction();
258 if (F && (F->getIntrinsicID() == Intrinsic::masked_load ||
259 F->getIntrinsicID() == Intrinsic::masked_store)) {
260 unsigned OpOffset = 0;
261 if (F->getIntrinsicID() == Intrinsic::masked_store) {
262 if (!Opts.memprof_instrument_writes)
263 return std::nullopt;
264 // Masked store has an initial operand for the value.
265 OpOffset = 1;
266 Access.AccessTy = CI->getArgOperand(i: 0)->getType();
267 Access.IsWrite = true;
268 } else {
269 if (!Opts.memprof_instrument_reads)
270 return std::nullopt;
271 Access.AccessTy = CI->getType();
272 Access.IsWrite = false;
273 }
274
275 auto *BasePtr = CI->getOperand(i_nocapture: 0 + OpOffset);
276 Access.MaybeMask = CI->getOperand(i_nocapture: 1 + OpOffset);
277 Access.Addr = BasePtr;
278 }
279 }
280
281 if (!Access.Addr)
282 return std::nullopt;
283
284 // Do not instrument accesses from different address spaces; we cannot deal
285 // with them.
286 Type *PtrTy = cast<PointerType>(Val: Access.Addr->getType()->getScalarType());
287 if (PtrTy->getPointerAddressSpace() != 0)
288 return std::nullopt;
289
290 // Ignore swifterror addresses.
291 // swifterror memory addresses are mem2reg promoted by instruction
292 // selection. As such they cannot have regular uses like an instrumentation
293 // function and it makes no sense to track them as memory.
294 if (Access.Addr->isSwiftError())
295 return std::nullopt;
296
297 // Peel off GEPs and BitCasts.
298 auto *Addr = Access.Addr->stripInBoundsOffsets();
299
300 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: Addr)) {
301 // Do not instrument PGO counter updates.
302 if (GV->hasSection()) {
303 StringRef SectionName = GV->getSection();
304 // Check if the global is in the PGO counters section.
305 auto OF = I->getModule()->getTargetTriple().getObjectFormat();
306 if (SectionName.ends_with(
307 Suffix: getInstrProfSectionName(IPSK: IPSK_cnts, OF, /*AddSegmentInfo=*/false)))
308 return std::nullopt;
309 }
310
311 // Do not instrument accesses to LLVM internal variables.
312 if (GV->getName().starts_with(Prefix: "__llvm") ||
313 GV->getName().starts_with(Prefix: getInstrProfVarPrefix()))
314 return std::nullopt;
315 }
316
317 return Access;
318}
319
320void MemProfiler::instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
321 Instruction *I, Value *Addr,
322 Type *AccessTy, bool IsWrite) {
323 auto *VTy = cast<FixedVectorType>(Val: AccessTy);
324 unsigned Num = VTy->getNumElements();
325 auto *Zero = ConstantInt::get(Ty: IntptrTy, V: 0);
326 for (unsigned Idx = 0; Idx < Num; ++Idx) {
327 Value *InstrumentedAddress = nullptr;
328 Instruction *InsertBefore = I;
329 if (auto *Vector = dyn_cast<ConstantVector>(Val: Mask)) {
330 // dyn_cast as we might get UndefValue
331 if (auto *Masked = dyn_cast<ConstantInt>(Val: Vector->getOperand(i_nocapture: Idx))) {
332 if (Masked->isZero())
333 // Mask is constant false, so no instrumentation needed.
334 continue;
335 // If we have a true or undef value, fall through to instrumentAddress.
336 // with InsertBefore == I
337 }
338 } else {
339 IRBuilder<> IRB(I);
340 Value *MaskElem = IRB.CreateExtractElement(Vec: Mask, Idx);
341 Instruction *ThenTerm = SplitBlockAndInsertIfThen(Cond: MaskElem, SplitBefore: I, Unreachable: false);
342 InsertBefore = ThenTerm;
343 }
344
345 IRBuilder<> IRB(InsertBefore);
346 IRB.disableGEPCanonicalization();
347 InstrumentedAddress =
348 IRB.CreateGEP(Ty: VTy, Ptr: Addr, IdxList: {Zero, ConstantInt::get(Ty: IntptrTy, V: Idx)});
349 instrumentAddress(OrigIns: I, InsertBefore, Addr: InstrumentedAddress, IsWrite);
350 }
351}
352
353void MemProfiler::instrumentMop(Instruction *I, const DataLayout &DL,
354 InterestingMemoryAccess &Access) {
355 // Skip instrumentation of stack accesses unless requested.
356 if (!Opts.memprof_instrument_stack &&
357 isa<AllocaInst>(Val: getUnderlyingObject(V: Access.Addr))) {
358 if (Access.IsWrite)
359 ++NumSkippedStackWrites;
360 else
361 ++NumSkippedStackReads;
362 return;
363 }
364
365 if (Access.IsWrite)
366 NumInstrumentedWrites++;
367 else
368 NumInstrumentedReads++;
369
370 if (Access.MaybeMask) {
371 instrumentMaskedLoadOrStore(DL, Mask: Access.MaybeMask, I, Addr: Access.Addr,
372 AccessTy: Access.AccessTy, IsWrite: Access.IsWrite);
373 } else {
374 // Since the access counts will be accumulated across the entire allocation,
375 // we only update the shadow access count for the first location and thus
376 // don't need to worry about alignment and type size.
377 instrumentAddress(OrigIns: I, InsertBefore: I, Addr: Access.Addr, IsWrite: Access.IsWrite);
378 }
379}
380
381void MemProfiler::instrumentAddress(Instruction *OrigIns,
382 Instruction *InsertBefore, Value *Addr,
383 bool IsWrite) {
384 IRBuilder<> IRB(InsertBefore);
385 Value *AddrLong = IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy);
386
387 if (Opts.memprof_use_callbacks) {
388 IRB.CreateCall(Callee: MemProfMemoryAccessCallback[IsWrite], Args: AddrLong);
389 return;
390 }
391
392 Type *ShadowTy =
393 Opts.memprof_histogram ? Type::getInt8Ty(C&: *C) : Type::getInt64Ty(C&: *C);
394 Type *ShadowPtrTy = PointerType::get(C&: *C, AddressSpace: 0);
395
396 Value *ShadowPtr = memToShadow(Shadow: AddrLong, IRB);
397 Value *ShadowAddr = IRB.CreateIntToPtr(V: ShadowPtr, DestTy: ShadowPtrTy);
398 Value *ShadowValue = IRB.CreateLoad(Ty: ShadowTy, Ptr: ShadowAddr);
399 // If we are profiling with histograms, add overflow protection at 255.
400 if (Opts.memprof_histogram) {
401 Value *MaxCount = ConstantInt::get(Ty: Type::getInt8Ty(C&: *C), V: 255);
402 Value *Cmp = IRB.CreateICmpULT(LHS: ShadowValue, RHS: MaxCount);
403 Instruction *IncBlock =
404 SplitBlockAndInsertIfThen(Cond: Cmp, SplitBefore: InsertBefore, /*Unreachable=*/false);
405 IRB.SetInsertPoint(IncBlock);
406 }
407 Value *Inc = ConstantInt::get(Ty: ShadowTy, V: 1);
408 ShadowValue = IRB.CreateAdd(LHS: ShadowValue, RHS: Inc);
409 IRB.CreateStore(Val: ShadowValue, Ptr: ShadowAddr);
410}
411
412// Create the variable for the profile file name.
413void createProfileFileNameVar(Module &M) {
414 const MDString *MemProfFilename =
415 dyn_cast_or_null<MDString>(Val: M.getModuleFlag(Key: "MemProfProfileFilename"));
416 if (!MemProfFilename)
417 return;
418 assert(!MemProfFilename->getString().empty() &&
419 "Unexpected MemProfProfileFilename metadata with empty string");
420 Constant *ProfileNameConst = ConstantDataArray::getString(
421 Context&: M.getContext(), Initializer: MemProfFilename->getString(), AddNull: true);
422 GlobalVariable *ProfileNameVar = new GlobalVariable(
423 M, ProfileNameConst->getType(), /*isConstant=*/true,
424 GlobalValue::WeakAnyLinkage, ProfileNameConst, MemProfFilenameVar);
425 const Triple &TT = M.getTargetTriple();
426 if (TT.supportsCOMDAT()) {
427 ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
428 ProfileNameVar->setComdat(M.getOrInsertComdat(Name: MemProfFilenameVar));
429 }
430}
431
432// Set MemprofHistogramFlag as a Global variable in IR. This makes it accessible
433// to the runtime, changing shadow count behavior.
434void createMemprofHistogramFlagVar(const InstrumentationOptions &Opts,
435 Module &M) {
436 const StringRef VarName(MemProfHistogramFlagVar);
437 Type *IntTy1 = Type::getInt1Ty(C&: M.getContext());
438 auto MemprofHistogramFlag = new GlobalVariable(
439 M, IntTy1, true, GlobalValue::WeakAnyLinkage,
440 Constant::getIntegerValue(Ty: IntTy1, V: APInt(1, Opts.memprof_histogram)),
441 VarName);
442 const Triple &TT = M.getTargetTriple();
443 if (TT.supportsCOMDAT()) {
444 MemprofHistogramFlag->setLinkage(GlobalValue::ExternalLinkage);
445 MemprofHistogramFlag->setComdat(M.getOrInsertComdat(Name: VarName));
446 }
447 appendToCompilerUsed(M, Values: MemprofHistogramFlag);
448}
449
450void createMemprofDefaultOptionsVar(const InstrumentationOptions &Opts,
451 Module &M) {
452 Constant *OptionsConst = ConstantDataArray::getString(
453 Context&: M.getContext(), Initializer: Opts.memprof_runtime_default_options, /*AddNull=*/true);
454 GlobalVariable *OptionsVar =
455 new GlobalVariable(M, OptionsConst->getType(), /*isConstant=*/true,
456 GlobalValue::WeakAnyLinkage, OptionsConst,
457 memprof::getMemprofOptionsSymbolName());
458 const Triple &TT = M.getTargetTriple();
459 if (TT.supportsCOMDAT()) {
460 OptionsVar->setLinkage(GlobalValue::ExternalLinkage);
461 OptionsVar->setComdat(M.getOrInsertComdat(Name: OptionsVar->getName()));
462 }
463}
464
465bool ModuleMemProfiler::instrumentModule(Module &M) {
466
467 // Create a module constructor.
468 std::string MemProfVersion = std::to_string(val: LLVM_MEM_PROFILER_VERSION);
469 std::string VersionCheckName =
470 Opts.memprof_guard_against_version_mismatch
471 ? (MemProfVersionCheckNamePrefix + MemProfVersion)
472 : "";
473 std::tie(args&: MemProfCtorFunction, args: std::ignore) =
474 createSanitizerCtorAndInitFunctions(M, CtorName: MemProfModuleCtorName,
475 InitName: MemProfInitName, /*InitArgTypes=*/{},
476 /*InitArgs=*/{}, VersionCheckName);
477
478 const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
479 appendToGlobalCtors(M, F: MemProfCtorFunction, Priority);
480
481 createProfileFileNameVar(M);
482
483 createMemprofHistogramFlagVar(Opts, M);
484
485 createMemprofDefaultOptionsVar(Opts, M);
486
487 return true;
488}
489
490void MemProfiler::initializeCallbacks(Module &M) {
491 IRBuilder<> IRB(M);
492
493 for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
494 const std::string TypeStr = AccessIsWrite ? "store" : "load";
495 const std::string HistPrefix = Opts.memprof_histogram ? "hist_" : "";
496
497 SmallVector<Type *, 2> Args1{1, IntptrTy};
498 MemProfMemoryAccessCallback[AccessIsWrite] = M.getOrInsertFunction(
499 Name: (Opts.memprof_memory_access_callback_prefix + HistPrefix + TypeStr)
500 .str(),
501 T: FunctionType::get(Result: IRB.getVoidTy(), Params: Args1, isVarArg: false));
502 }
503 MemProfMemmove = M.getOrInsertFunction(
504 Name: (Opts.memprof_memory_access_callback_prefix + "memmove").str(), RetTy: PtrTy,
505 Args: PtrTy, Args: PtrTy, Args: IntptrTy);
506 MemProfMemcpy = M.getOrInsertFunction(
507 Name: (Opts.memprof_memory_access_callback_prefix + "memcpy").str(), RetTy: PtrTy,
508 Args: PtrTy, Args: PtrTy, Args: IntptrTy);
509 MemProfMemset = M.getOrInsertFunction(
510 Name: (Opts.memprof_memory_access_callback_prefix + "memset").str(), RetTy: PtrTy,
511 Args: PtrTy, Args: IRB.getInt32Ty(), Args: IntptrTy);
512}
513
514bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) {
515 // For each NSObject descendant having a +load method, this method is invoked
516 // by the ObjC runtime before any of the static constructors is called.
517 // Therefore we need to instrument such methods with a call to __memprof_init
518 // at the beginning in order to initialize our runtime before any access to
519 // the shadow memory.
520 // We cannot just ignore these methods, because they may call other
521 // instrumented functions.
522 if (F.getName().contains(Other: " load]")) {
523 FunctionCallee MemProfInitFunction =
524 declareSanitizerInitFunction(M&: *F.getParent(), InitName: MemProfInitName, InitArgTypes: {});
525 IRBuilder<> IRB(F.front().begin());
526 IRB.CreateCall(Callee: MemProfInitFunction, Args: {});
527 return true;
528 }
529 return false;
530}
531
532bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {
533 IRBuilder<> IRB(&F.front().front());
534 Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
535 Name: MemProfShadowMemoryDynamicAddress, Ty: IntptrTy);
536 if (F.getParent()->getPICLevel() == PICLevel::NotPIC)
537 cast<GlobalVariable>(Val: GlobalDynamicAddress)->setDSOLocal(true);
538 DynamicShadowOffset = IRB.CreateLoad(Ty: IntptrTy, Ptr: GlobalDynamicAddress);
539 return true;
540}
541
542bool MemProfiler::instrumentFunction(Function &F) {
543 if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
544 return false;
545 if (Opts.memprof_debug_func == F.getName())
546 return false;
547 if (F.getName().starts_with(Prefix: "__memprof_"))
548 return false;
549
550 bool FunctionModified = false;
551
552 // If needed, insert __memprof_init.
553 // This function needs to be called even if the function body is not
554 // instrumented.
555 if (maybeInsertMemProfInitAtFunctionEntry(F))
556 FunctionModified = true;
557
558 LLVM_DEBUG(dbgs() << "MEMPROF instrumenting:\n" << F << "\n");
559
560 initializeCallbacks(M&: *F.getParent());
561
562 SmallVector<Instruction *, 16> ToInstrument;
563
564 // Fill the set of memory operations to instrument.
565 for (auto &BB : F) {
566 for (auto &Inst : BB) {
567 if (isInterestingMemoryAccess(I: &Inst) || isa<MemIntrinsic>(Val: Inst))
568 ToInstrument.push_back(Elt: &Inst);
569 }
570 }
571
572 if (ToInstrument.empty()) {
573 LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified
574 << " " << F << "\n");
575
576 return FunctionModified;
577 }
578
579 FunctionModified |= insertDynamicShadowAtFunctionEntry(F);
580
581 int NumInstrumented = 0;
582 for (auto *Inst : ToInstrument) {
583 if (Opts.memprof_debug_min < 0 || Opts.memprof_debug_max < 0 ||
584 (NumInstrumented >= Opts.memprof_debug_min &&
585 NumInstrumented <= Opts.memprof_debug_max)) {
586 std::optional<InterestingMemoryAccess> Access =
587 isInterestingMemoryAccess(I: Inst);
588 if (Access)
589 instrumentMop(I: Inst, DL: F.getDataLayout(), Access&: *Access);
590 else
591 instrumentMemIntrinsic(MI: cast<MemIntrinsic>(Val: Inst));
592 }
593 NumInstrumented++;
594 }
595
596 if (NumInstrumented > 0)
597 FunctionModified = true;
598
599 LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified << " "
600 << F << "\n");
601
602 return FunctionModified;
603}
604