| 1 | //===- AllocToken.cpp - Allocation token instrumentation ------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements AllocToken, an instrumentation pass that |
| 10 | // replaces allocation calls with token-enabled versions. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "llvm/Transforms/Instrumentation/AllocToken.h" |
| 15 | #include "llvm/ADT/DenseMap.h" |
| 16 | #include "llvm/ADT/SmallVector.h" |
| 17 | #include "llvm/ADT/Statistic.h" |
| 18 | #include "llvm/ADT/StringExtras.h" |
| 19 | #include "llvm/ADT/StringRef.h" |
| 20 | #include "llvm/Analysis/MemoryBuiltins.h" |
| 21 | #include "llvm/Analysis/OptimizationRemarkEmitter.h" |
| 22 | #include "llvm/Analysis/TargetLibraryInfo.h" |
| 23 | #include "llvm/IR/Analysis.h" |
| 24 | #include "llvm/IR/Attributes.h" |
| 25 | #include "llvm/IR/Constants.h" |
| 26 | #include "llvm/IR/DerivedTypes.h" |
| 27 | #include "llvm/IR/Function.h" |
| 28 | #include "llvm/IR/GlobalValue.h" |
| 29 | #include "llvm/IR/IRBuilder.h" |
| 30 | #include "llvm/IR/InstIterator.h" |
| 31 | #include "llvm/IR/InstrTypes.h" |
| 32 | #include "llvm/IR/Instructions.h" |
| 33 | #include "llvm/IR/IntrinsicInst.h" |
| 34 | #include "llvm/IR/Metadata.h" |
| 35 | #include "llvm/IR/Module.h" |
| 36 | #include "llvm/IR/PassManager.h" |
| 37 | #include "llvm/IR/Type.h" |
| 38 | #include "llvm/Support/AllocToken.h" |
| 39 | #include "llvm/Support/Casting.h" |
| 40 | #include "llvm/Support/CommandLine.h" |
| 41 | #include "llvm/Support/Compiler.h" |
| 42 | #include "llvm/Support/ErrorHandling.h" |
| 43 | #include "llvm/Support/RandomNumberGenerator.h" |
| 44 | #include "llvm/Support/SipHash.h" |
| 45 | #include <cassert> |
| 46 | #include <cstdint> |
| 47 | #include <memory> |
| 48 | #include <optional> |
| 49 | #include <string> |
| 50 | #include <utility> |
| 51 | #include <variant> |
| 52 | |
| 53 | using namespace llvm; |
| 54 | using TokenMode = AllocTokenMode; |
| 55 | |
| 56 | #define DEBUG_TYPE "alloc-token" |
| 57 | |
| 58 | namespace { |
| 59 | |
| 60 | //===--- Command-line options ---------------------------------------------===// |
| 61 | |
| 62 | cl::opt<std::string> ClFuncPrefix("alloc-token-prefix" , |
| 63 | cl::desc("The allocation function prefix" ), |
| 64 | cl::Hidden, cl::init(Val: "__alloc_token_" )); |
| 65 | |
| 66 | cl::opt<uint64_t> |
| 67 | ClMaxTokens("alloc-token-max" , |
| 68 | cl::desc("Maximum number of tokens (0 = target SIZE_MAX)" ), |
| 69 | cl::Hidden, cl::init(Val: 0)); |
| 70 | |
| 71 | cl::opt<bool> |
| 72 | ClFastABI("alloc-token-fast-abi" , |
| 73 | cl::desc("The token ID is encoded in the function name" ), |
| 74 | cl::Hidden, cl::init(Val: false)); |
| 75 | |
| 76 | // Instrument libcalls only by default - compatible allocators only need to take |
| 77 | // care of providing standard allocation functions. With extended coverage, also |
| 78 | // instrument non-libcall allocation function calls with !alloc_token |
| 79 | // metadata. |
| 80 | cl::opt<bool> |
| 81 | ClExtended("alloc-token-extended" , |
| 82 | cl::desc("Extend coverage to custom allocation functions" ), |
| 83 | cl::Hidden, cl::init(Val: false)); |
| 84 | |
| 85 | // C++ defines ::operator new (and variants) as replaceable (vs. standard |
| 86 | // library versions), which are nobuiltin, and are therefore not covered by |
| 87 | // isAllocationFn(). Cover by default, as users of AllocToken are already |
| 88 | // required to provide token-aware allocation functions (no defaults). |
| 89 | cl::opt<bool> ClCoverReplaceableNew("alloc-token-cover-replaceable-new" , |
| 90 | cl::desc("Cover replaceable operator new" ), |
| 91 | cl::Hidden, cl::init(Val: true)); |
| 92 | |
| 93 | cl::opt<uint64_t> ClFallbackToken( |
| 94 | "alloc-token-fallback" , |
| 95 | cl::desc("The default fallback token where none could be determined" ), |
| 96 | cl::Hidden, cl::init(Val: 0)); |
| 97 | |
| 98 | //===--- Statistics -------------------------------------------------------===// |
| 99 | |
| 100 | STATISTIC(NumFunctionsModified, "Functions modified" ); |
| 101 | STATISTIC(NumAllocationsInstrumented, "Allocations instrumented" ); |
| 102 | |
| 103 | //===----------------------------------------------------------------------===// |
| 104 | |
| 105 | /// Returns the !alloc_token metadata if available. |
| 106 | /// |
| 107 | /// Expected format is: !{<type-name>, <contains-pointer>} |
| 108 | MDNode *getAllocTokenMetadata(const CallBase &CB) { |
| 109 | MDNode *Ret = nullptr; |
| 110 | if (auto *II = dyn_cast<IntrinsicInst>(Val: &CB); |
| 111 | II && II->getIntrinsicID() == Intrinsic::alloc_token_id) { |
| 112 | auto *MDV = cast<MetadataAsValue>(Val: II->getArgOperand(i: 0)); |
| 113 | Ret = cast<MDNode>(Val: MDV->getMetadata()); |
| 114 | // If the intrinsic has an empty MDNode, type inference failed. |
| 115 | if (Ret->getNumOperands() == 0) |
| 116 | return nullptr; |
| 117 | } else { |
| 118 | Ret = CB.getMetadata(KindID: LLVMContext::MD_alloc_token); |
| 119 | if (!Ret) |
| 120 | return nullptr; |
| 121 | } |
| 122 | assert(Ret->getNumOperands() == 2 && "bad !alloc_token" ); |
| 123 | assert(isa<MDString>(Ret->getOperand(0))); |
| 124 | assert(isa<ConstantAsMetadata>(Ret->getOperand(1))); |
| 125 | return Ret; |
| 126 | } |
| 127 | |
| 128 | bool containsPointer(const MDNode *MD) { |
| 129 | ConstantAsMetadata *C = cast<ConstantAsMetadata>(Val: MD->getOperand(I: 1)); |
| 130 | auto *CI = cast<ConstantInt>(Val: C->getValue()); |
| 131 | return CI->getValue().getBoolValue(); |
| 132 | } |
| 133 | |
| 134 | class ModeBase { |
| 135 | public: |
| 136 | explicit ModeBase(const IntegerType &TokenTy, uint64_t MaxTokens) |
| 137 | : MaxTokens(MaxTokens ? MaxTokens : TokenTy.getBitMask()) { |
| 138 | assert(MaxTokens <= TokenTy.getBitMask()); |
| 139 | } |
| 140 | |
| 141 | protected: |
| 142 | uint64_t boundedToken(uint64_t Val) const { |
| 143 | assert(MaxTokens != 0); |
| 144 | return Val % MaxTokens; |
| 145 | } |
| 146 | |
| 147 | const uint64_t MaxTokens; |
| 148 | }; |
| 149 | |
| 150 | /// Implementation for TokenMode::Increment. |
| 151 | class IncrementMode : public ModeBase { |
| 152 | public: |
| 153 | using ModeBase::ModeBase; |
| 154 | |
| 155 | uint64_t (const CallBase &CB, OptimizationRemarkEmitter &) { |
| 156 | return boundedToken(Val: Counter++); |
| 157 | } |
| 158 | |
| 159 | private: |
| 160 | uint64_t Counter = 0; |
| 161 | }; |
| 162 | |
| 163 | /// Implementation for TokenMode::Random. |
| 164 | class RandomMode : public ModeBase { |
| 165 | public: |
| 166 | RandomMode(const IntegerType &TokenTy, uint64_t MaxTokens, |
| 167 | std::unique_ptr<RandomNumberGenerator> RNG) |
| 168 | : ModeBase(TokenTy, MaxTokens), RNG(std::move(RNG)) {} |
| 169 | uint64_t operator()(const CallBase &CB, OptimizationRemarkEmitter &) { |
| 170 | return boundedToken(Val: (*RNG)()); |
| 171 | } |
| 172 | |
| 173 | private: |
| 174 | std::unique_ptr<RandomNumberGenerator> RNG; |
| 175 | }; |
| 176 | |
| 177 | /// Implementation for TokenMode::TypeHash. The implementation ensures |
| 178 | /// hashes are stable across different compiler invocations. Uses SipHash as the |
| 179 | /// hash function. |
| 180 | class TypeHashMode : public ModeBase { |
| 181 | public: |
| 182 | using ModeBase::ModeBase; |
| 183 | |
| 184 | uint64_t (const CallBase &CB, OptimizationRemarkEmitter &ORE) { |
| 185 | |
| 186 | if (MDNode *N = getAllocTokenMetadata(CB)) { |
| 187 | MDString *S = cast<MDString>(Val: N->getOperand(I: 0)); |
| 188 | AllocTokenMetadata Metadata{.TypeName: S->getString(), .ContainsPointer: containsPointer(MD: N)}; |
| 189 | if (auto Token = getAllocToken(Mode: TokenMode::TypeHash, Metadata, MaxTokens)) |
| 190 | return *Token; |
| 191 | } |
| 192 | // Fallback. |
| 193 | remarkNoMetadata(CB, ORE); |
| 194 | return ClFallbackToken; |
| 195 | } |
| 196 | |
| 197 | protected: |
| 198 | /// Remark that there was no precise type information. |
| 199 | static void (const CallBase &CB, |
| 200 | OptimizationRemarkEmitter &ORE) { |
| 201 | ORE.emit(RemarkBuilder: [&] { |
| 202 | ore::NV FuncNV("Function" , CB.getParent()->getParent()); |
| 203 | const Function *Callee = CB.getCalledFunction(); |
| 204 | ore::NV CalleeNV("Callee" , Callee ? Callee->getName() : "<unknown>" ); |
| 205 | return OptimizationRemark(DEBUG_TYPE, "NoAllocToken" , &CB) |
| 206 | << "Call to '" << CalleeNV << "' in '" << FuncNV |
| 207 | << "' without source-level type token" ; |
| 208 | }); |
| 209 | } |
| 210 | }; |
| 211 | |
| 212 | /// Implementation for TokenMode::TypeHashPointerSplit. |
| 213 | class TypeHashPointerSplitMode : public TypeHashMode { |
| 214 | public: |
| 215 | using TypeHashMode::TypeHashMode; |
| 216 | |
| 217 | uint64_t (const CallBase &CB, OptimizationRemarkEmitter &ORE) { |
| 218 | if (MDNode *N = getAllocTokenMetadata(CB)) { |
| 219 | MDString *S = cast<MDString>(Val: N->getOperand(I: 0)); |
| 220 | AllocTokenMetadata Metadata{.TypeName: S->getString(), .ContainsPointer: containsPointer(MD: N)}; |
| 221 | if (auto Token = getAllocToken(Mode: TokenMode::TypeHashPointerSplit, Metadata, |
| 222 | MaxTokens)) |
| 223 | return *Token; |
| 224 | } |
| 225 | // Pick the fallback token (ClFallbackToken), which by default is 0, meaning |
| 226 | // it'll fall into the pointer-less bucket. Override by setting |
| 227 | // -alloc-token-fallback if that is the wrong choice. |
| 228 | remarkNoMetadata(CB, ORE); |
| 229 | return ClFallbackToken; |
| 230 | } |
| 231 | }; |
| 232 | |
| 233 | // Apply opt overrides and module flags. |
| 234 | static AllocTokenOptions resolveOptions(AllocTokenOptions Opts, |
| 235 | const Module &M) { |
| 236 | auto IntModuleFlagOrNull = [&](StringRef Key) { |
| 237 | return mdconst::extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key)); |
| 238 | }; |
| 239 | |
| 240 | if (auto *S = dyn_cast_or_null<MDString>(Val: M.getModuleFlag(Key: "alloc-token-mode" ))) |
| 241 | if (auto Mode = getAllocTokenModeFromString(Name: S->getString())) |
| 242 | Opts.Mode = *Mode; |
| 243 | if (auto *Val = IntModuleFlagOrNull("alloc-token-max" )) |
| 244 | Opts.MaxTokens = Val->getZExtValue(); |
| 245 | if (auto *Val = IntModuleFlagOrNull("alloc-token-fast-abi" )) |
| 246 | Opts.FastABI |= Val->isOne(); |
| 247 | if (auto *Val = IntModuleFlagOrNull("alloc-token-extended" )) |
| 248 | Opts.Extended |= Val->isOne(); |
| 249 | |
| 250 | // Allow overriding options from command line options. |
| 251 | if (ClMaxTokens.getNumOccurrences()) |
| 252 | Opts.MaxTokens = ClMaxTokens; |
| 253 | if (ClFastABI.getNumOccurrences()) |
| 254 | Opts.FastABI = ClFastABI; |
| 255 | if (ClExtended.getNumOccurrences()) |
| 256 | Opts.Extended = ClExtended; |
| 257 | |
| 258 | return Opts; |
| 259 | } |
| 260 | |
| 261 | class AllocToken { |
| 262 | public: |
| 263 | explicit AllocToken(AllocTokenOptions Opts, Module &M, |
| 264 | ModuleAnalysisManager &MAM) |
| 265 | : Options(resolveOptions(Opts: std::move(Opts), M)), Mod(M), |
| 266 | FAM(MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager()), |
| 267 | Mode(IncrementMode(*IntPtrTy, Options.MaxTokens)) { |
| 268 | switch (Options.Mode) { |
| 269 | case TokenMode::Increment: |
| 270 | break; |
| 271 | case TokenMode::Random: |
| 272 | Mode.emplace<RandomMode>(args&: *IntPtrTy, args: Options.MaxTokens, |
| 273 | args: M.createRNG(DEBUG_TYPE)); |
| 274 | break; |
| 275 | case TokenMode::TypeHash: |
| 276 | Mode.emplace<TypeHashMode>(args&: *IntPtrTy, args: Options.MaxTokens); |
| 277 | break; |
| 278 | case TokenMode::TypeHashPointerSplit: |
| 279 | Mode.emplace<TypeHashPointerSplitMode>(args&: *IntPtrTy, args: Options.MaxTokens); |
| 280 | break; |
| 281 | } |
| 282 | } |
| 283 | |
| 284 | bool instrumentFunction(Function &F); |
| 285 | |
| 286 | private: |
| 287 | /// Returns the LibFunc (or NotLibFunc) if this call should be instrumented. |
| 288 | std::optional<LibFunc> |
| 289 | shouldInstrumentCall(const CallBase &CB, const TargetLibraryInfo &TLI) const; |
| 290 | |
| 291 | /// Returns true for functions that are eligible for instrumentation. |
| 292 | static bool isInstrumentableLibFunc(LibFunc Func, const CallBase &CB, |
| 293 | const TargetLibraryInfo &TLI); |
| 294 | |
| 295 | /// Returns true for isAllocationFn() functions that we should ignore. |
| 296 | static bool ignoreInstrumentableLibFunc(LibFunc Func); |
| 297 | |
| 298 | /// Replace a call/invoke with a call/invoke to the allocation function |
| 299 | /// with token ID. |
| 300 | bool replaceAllocationCall(CallBase *CB, LibFunc Func, |
| 301 | OptimizationRemarkEmitter &ORE, |
| 302 | const TargetLibraryInfo &TLI); |
| 303 | |
| 304 | /// Return replacement function for a LibFunc that takes a token ID. |
| 305 | FunctionCallee getTokenAllocFunction(const CallBase &CB, uint64_t TokenID, |
| 306 | LibFunc OriginalFunc); |
| 307 | |
| 308 | /// Lower alloc_token_* intrinsics. |
| 309 | void replaceIntrinsicInst(IntrinsicInst *II, OptimizationRemarkEmitter &ORE); |
| 310 | |
| 311 | /// Return the token ID from metadata in the call. |
| 312 | uint64_t (const CallBase &CB, OptimizationRemarkEmitter &ORE) { |
| 313 | return std::visit(visitor: [&](auto &&Mode) { return Mode(CB, ORE); }, variants&: Mode); |
| 314 | } |
| 315 | |
| 316 | const AllocTokenOptions Options; |
| 317 | Module &Mod; |
| 318 | IntegerType *IntPtrTy = Mod.getDataLayout().getIntPtrType(C&: Mod.getContext()); |
| 319 | FunctionAnalysisManager &FAM; |
| 320 | // Cache for replacement functions. |
| 321 | DenseMap<std::pair<LibFunc, uint64_t>, FunctionCallee> TokenAllocFunctions; |
| 322 | // Selected mode. |
| 323 | std::variant<IncrementMode, RandomMode, TypeHashMode, |
| 324 | TypeHashPointerSplitMode> |
| 325 | Mode; |
| 326 | }; |
| 327 | |
| 328 | bool AllocToken::instrumentFunction(Function &F) { |
| 329 | // Do not apply any instrumentation for naked functions. |
| 330 | if (F.hasFnAttribute(Kind: Attribute::Naked)) |
| 331 | return false; |
| 332 | // Don't touch available_externally functions, their actual body is elsewhere. |
| 333 | if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage) |
| 334 | return false; |
| 335 | |
| 336 | SmallVector<std::pair<CallBase *, LibFunc>, 4> AllocCalls; |
| 337 | SmallVector<IntrinsicInst *, 4> IntrinsicInsts; |
| 338 | |
| 339 | // Only instrument functions that have the sanitize_alloc_token attribute. |
| 340 | const bool InstrumentFunction = |
| 341 | F.hasFnAttribute(Kind: Attribute::SanitizeAllocToken) && |
| 342 | !F.hasFnAttribute(Kind: Attribute::DisableSanitizerInstrumentation); |
| 343 | |
| 344 | // Get TLI only when required. |
| 345 | const TargetLibraryInfo *TLI = |
| 346 | InstrumentFunction ? &FAM.getResult<TargetLibraryAnalysis>(IR&: F) : nullptr; |
| 347 | |
| 348 | // Collect all allocation calls to avoid iterator invalidation. |
| 349 | for (Instruction &I : instructions(F)) { |
| 350 | // Collect all alloc_token_* intrinsics. |
| 351 | if (auto *II = dyn_cast<IntrinsicInst>(Val: &I); |
| 352 | II && II->getIntrinsicID() == Intrinsic::alloc_token_id) { |
| 353 | IntrinsicInsts.emplace_back(Args&: II); |
| 354 | continue; |
| 355 | } |
| 356 | |
| 357 | if (!InstrumentFunction) |
| 358 | continue; |
| 359 | |
| 360 | auto *CB = dyn_cast<CallBase>(Val: &I); |
| 361 | if (!CB) |
| 362 | continue; |
| 363 | if (std::optional<LibFunc> Func = shouldInstrumentCall(CB: *CB, TLI: *TLI)) |
| 364 | AllocCalls.emplace_back(Args&: CB, Args&: Func.value()); |
| 365 | } |
| 366 | |
| 367 | // Return early to avoid unnecessarily instantiating the ORE. |
| 368 | if (AllocCalls.empty() && IntrinsicInsts.empty()) |
| 369 | return false; |
| 370 | |
| 371 | auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: F); |
| 372 | bool Modified = false; |
| 373 | |
| 374 | for (auto &[CB, Func] : AllocCalls) |
| 375 | Modified |= replaceAllocationCall(CB, Func, ORE, TLI: *TLI); |
| 376 | |
| 377 | for (auto *II : IntrinsicInsts) { |
| 378 | replaceIntrinsicInst(II, ORE); |
| 379 | Modified = true; |
| 380 | } |
| 381 | |
| 382 | if (Modified) |
| 383 | NumFunctionsModified++; |
| 384 | |
| 385 | return Modified; |
| 386 | } |
| 387 | |
| 388 | std::optional<LibFunc> |
| 389 | AllocToken::shouldInstrumentCall(const CallBase &CB, |
| 390 | const TargetLibraryInfo &TLI) const { |
| 391 | const Function *Callee = CB.getCalledFunction(); |
| 392 | if (!Callee) |
| 393 | return std::nullopt; |
| 394 | |
| 395 | // Ignore nobuiltin of the CallBase, so that we can cover nobuiltin libcalls |
| 396 | // if requested via isInstrumentableLibFunc(). Note that isAllocationFn() is |
| 397 | // returning false for nobuiltin calls. |
| 398 | LibFunc Func = TLI.getLibFunc(FDecl: *Callee); |
| 399 | if (Func != NotLibFunc) { |
| 400 | if (isInstrumentableLibFunc(Func, CB, TLI)) |
| 401 | return Func; |
| 402 | } else if (Options.Extended && CB.getMetadata(KindID: LLVMContext::MD_alloc_token)) { |
| 403 | return NotLibFunc; |
| 404 | } |
| 405 | |
| 406 | return std::nullopt; |
| 407 | } |
| 408 | |
| 409 | bool AllocToken::isInstrumentableLibFunc(LibFunc Func, const CallBase &CB, |
| 410 | const TargetLibraryInfo &TLI) { |
| 411 | if (ignoreInstrumentableLibFunc(Func)) |
| 412 | return false; |
| 413 | |
| 414 | if (isAllocationFn(V: &CB, TLI: &TLI)) |
| 415 | return true; |
| 416 | |
| 417 | switch (Func) { |
| 418 | // These libfuncs don't return normal pointers, and are therefore not handled |
| 419 | // by isAllocationFn(). |
| 420 | case LibFunc_posix_memalign: |
| 421 | case LibFunc_size_returning_new: |
| 422 | case LibFunc_size_returning_new_hot_cold: |
| 423 | case LibFunc_size_returning_new_aligned: |
| 424 | case LibFunc_size_returning_new_aligned_hot_cold: |
| 425 | return true; |
| 426 | |
| 427 | // See comment above ClCoverReplaceableNew. |
| 428 | case LibFunc_Znwj: |
| 429 | case LibFunc_ZnwjRKSt9nothrow_t: |
| 430 | case LibFunc_ZnwjSt11align_val_t: |
| 431 | case LibFunc_ZnwjSt11align_val_tRKSt9nothrow_t: |
| 432 | case LibFunc_Znwm: |
| 433 | case LibFunc_Znwm12__hot_cold_t: |
| 434 | case LibFunc_ZnwmRKSt9nothrow_t: |
| 435 | case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t: |
| 436 | case LibFunc_ZnwmSt11align_val_t: |
| 437 | case LibFunc_ZnwmSt11align_val_t12__hot_cold_t: |
| 438 | case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t: |
| 439 | case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t: |
| 440 | case LibFunc_Znaj: |
| 441 | case LibFunc_ZnajRKSt9nothrow_t: |
| 442 | case LibFunc_ZnajSt11align_val_t: |
| 443 | case LibFunc_ZnajSt11align_val_tRKSt9nothrow_t: |
| 444 | case LibFunc_Znam: |
| 445 | case LibFunc_Znam12__hot_cold_t: |
| 446 | case LibFunc_ZnamRKSt9nothrow_t: |
| 447 | case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t: |
| 448 | case LibFunc_ZnamSt11align_val_t: |
| 449 | case LibFunc_ZnamSt11align_val_t12__hot_cold_t: |
| 450 | case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t: |
| 451 | case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t: |
| 452 | return ClCoverReplaceableNew; |
| 453 | |
| 454 | default: |
| 455 | return false; |
| 456 | } |
| 457 | } |
| 458 | |
| 459 | bool AllocToken::ignoreInstrumentableLibFunc(LibFunc Func) { |
| 460 | switch (Func) { |
| 461 | case LibFunc_strdup: |
| 462 | case LibFunc_dunder_strdup: |
| 463 | case LibFunc_strndup: |
| 464 | case LibFunc_dunder_strndup: |
| 465 | return true; |
| 466 | default: |
| 467 | return false; |
| 468 | } |
| 469 | } |
| 470 | |
| 471 | bool AllocToken::(CallBase *CB, LibFunc Func, |
| 472 | OptimizationRemarkEmitter &ORE, |
| 473 | const TargetLibraryInfo &TLI) { |
| 474 | uint64_t TokenID = getToken(CB: *CB, ORE); |
| 475 | |
| 476 | FunctionCallee TokenAlloc = getTokenAllocFunction(CB: *CB, TokenID, OriginalFunc: Func); |
| 477 | if (!TokenAlloc) |
| 478 | return false; |
| 479 | NumAllocationsInstrumented++; |
| 480 | |
| 481 | if (Options.FastABI) { |
| 482 | assert(TokenAlloc.getFunctionType()->getNumParams() == CB->arg_size()); |
| 483 | CB->setCalledFunction(TokenAlloc); |
| 484 | return true; |
| 485 | } |
| 486 | |
| 487 | IRBuilder<> IRB(CB); |
| 488 | // Original args. |
| 489 | SmallVector<Value *, 4> NewArgs{CB->args()}; |
| 490 | // Add token ID, truncated to IntPtrTy width. |
| 491 | NewArgs.push_back(Elt: ConstantInt::get(Ty: IntPtrTy, V: TokenID)); |
| 492 | assert(TokenAlloc.getFunctionType()->getNumParams() == NewArgs.size()); |
| 493 | |
| 494 | // Preserve invoke vs call semantics for exception handling. |
| 495 | CallBase *NewCall; |
| 496 | if (auto *II = dyn_cast<InvokeInst>(Val: CB)) { |
| 497 | NewCall = IRB.CreateInvoke(Callee: TokenAlloc, NormalDest: II->getNormalDest(), |
| 498 | UnwindDest: II->getUnwindDest(), Args: NewArgs); |
| 499 | } else { |
| 500 | NewCall = IRB.CreateCall(Callee: TokenAlloc, Args: NewArgs); |
| 501 | cast<CallInst>(Val: NewCall)->setTailCall(CB->isTailCall()); |
| 502 | } |
| 503 | NewCall->setCallingConv(CB->getCallingConv()); |
| 504 | NewCall->copyMetadata(SrcInst: *CB); |
| 505 | NewCall->setAttributes(CB->getAttributes()); |
| 506 | |
| 507 | // Replace all uses and delete the old call. |
| 508 | CB->replaceAllUsesWith(V: NewCall); |
| 509 | CB->eraseFromParent(); |
| 510 | return true; |
| 511 | } |
| 512 | |
| 513 | FunctionCallee AllocToken::getTokenAllocFunction(const CallBase &CB, |
| 514 | uint64_t TokenID, |
| 515 | LibFunc OriginalFunc) { |
| 516 | std::optional<std::pair<LibFunc, uint64_t>> Key; |
| 517 | if (OriginalFunc != NotLibFunc) { |
| 518 | Key = std::make_pair(x&: OriginalFunc, y: Options.FastABI ? TokenID : 0); |
| 519 | auto It = TokenAllocFunctions.find(Val: *Key); |
| 520 | if (It != TokenAllocFunctions.end()) |
| 521 | return It->second; |
| 522 | } |
| 523 | |
| 524 | const Function *Callee = CB.getCalledFunction(); |
| 525 | if (!Callee) |
| 526 | return FunctionCallee(); |
| 527 | const FunctionType *OldFTy = Callee->getFunctionType(); |
| 528 | if (OldFTy->isVarArg()) |
| 529 | return FunctionCallee(); |
| 530 | // Copy params, and append token ID type. |
| 531 | Type *RetTy = OldFTy->getReturnType(); |
| 532 | SmallVector<Type *, 4> NewParams{OldFTy->params()}; |
| 533 | std::string TokenAllocName = ClFuncPrefix; |
| 534 | if (Options.FastABI) |
| 535 | TokenAllocName += utostr(X: TokenID) + "_" ; |
| 536 | else |
| 537 | NewParams.push_back(Elt: IntPtrTy); // token ID |
| 538 | TokenAllocName += Callee->getName(); |
| 539 | FunctionType *NewFTy = FunctionType::get(Result: RetTy, Params: NewParams, isVarArg: false); |
| 540 | AttributeList NewAttrs = Callee->getAttributes(); |
| 541 | FunctionCallee TokenAlloc = |
| 542 | Mod.getOrInsertFunction(Name: TokenAllocName, T: NewFTy, AttributeList: NewAttrs); |
| 543 | |
| 544 | if (Key.has_value()) |
| 545 | TokenAllocFunctions[*Key] = TokenAlloc; |
| 546 | return TokenAlloc; |
| 547 | } |
| 548 | |
| 549 | void AllocToken::(IntrinsicInst *II, |
| 550 | OptimizationRemarkEmitter &ORE) { |
| 551 | assert(II->getIntrinsicID() == Intrinsic::alloc_token_id); |
| 552 | |
| 553 | uint64_t TokenID = getToken(CB: *II, ORE); |
| 554 | Value *V = ConstantInt::get(Ty: IntPtrTy, V: TokenID); |
| 555 | II->replaceAllUsesWith(V); |
| 556 | II->eraseFromParent(); |
| 557 | } |
| 558 | |
| 559 | } // namespace |
| 560 | |
| 561 | AllocTokenPass::AllocTokenPass(AllocTokenOptions Opts) |
| 562 | : Options(std::move(Opts)) {} |
| 563 | |
| 564 | PreservedAnalyses AllocTokenPass::run(Module &M, ModuleAnalysisManager &MAM) { |
| 565 | AllocToken Pass(Options, M, MAM); |
| 566 | bool Modified = false; |
| 567 | |
| 568 | for (Function &F : M) { |
| 569 | if (F.empty()) |
| 570 | continue; // declaration |
| 571 | Modified |= Pass.instrumentFunction(F); |
| 572 | } |
| 573 | |
| 574 | return Modified ? PreservedAnalyses::none().preserveSet<CFGAnalyses>() |
| 575 | : PreservedAnalyses::all(); |
| 576 | } |
| 577 | |