| 1 | //===---- GlobalMergeFunctions.cpp - Global merge functions -------*- C++ -===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This pass implements the global merge function pass. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "llvm/CodeGen/GlobalMergeFunctions.h" |
| 14 | #include "llvm/ADT/Statistic.h" |
| 15 | #include "llvm/Analysis/ModuleSummaryAnalysis.h" |
| 16 | #include "llvm/CGData/CodeGenData.h" |
| 17 | #include "llvm/CGData/CodeGenDataWriter.h" |
| 18 | #include "llvm/CodeGen/Passes.h" |
| 19 | #include "llvm/IR/IRBuilder.h" |
| 20 | #include "llvm/IR/StructuralHash.h" |
| 21 | #include "llvm/InitializePasses.h" |
| 22 | #include "llvm/Support/CommandLine.h" |
| 23 | #include "llvm/Transforms/Utils/ModuleUtils.h" |
| 24 | |
| 25 | #define DEBUG_TYPE "global-merge-func" |
| 26 | |
| 27 | using namespace llvm; |
| 28 | using namespace llvm::support; |
| 29 | |
| 30 | static cl::opt<bool> DisableCGDataForMerging( |
| 31 | "disable-cgdata-for-merging" , cl::Hidden, |
| 32 | cl::desc("Disable codegen data for function merging. Local " |
| 33 | "merging is still enabled within a module." ), |
| 34 | cl::init(Val: false)); |
| 35 | |
| 36 | STATISTIC(NumMergedFunctions, |
| 37 | "Number of functions that are actually merged using function hash" ); |
| 38 | STATISTIC(NumAnalyzedModues, "Number of modules that are analyzed" ); |
| 39 | STATISTIC(NumAnalyzedFunctions, "Number of functions that are analyzed" ); |
| 40 | STATISTIC(NumEligibleFunctions, "Number of functions that are eligible" ); |
| 41 | |
| 42 | /// Returns true if the \OpIdx operand of \p CI is the callee operand. |
| 43 | static bool isCalleeOperand(const CallBase *CI, unsigned OpIdx) { |
| 44 | return &CI->getCalledOperandUse() == &CI->getOperandUse(i: OpIdx); |
| 45 | } |
| 46 | |
| 47 | static bool canParameterizeCallOperand(const CallBase *CI, unsigned OpIdx) { |
| 48 | if (CI->isInlineAsm()) |
| 49 | return false; |
| 50 | Function *Callee = CI->getCalledOperand() |
| 51 | ? dyn_cast_or_null<Function>( |
| 52 | Val: CI->getCalledOperand()->stripPointerCasts()) |
| 53 | : nullptr; |
| 54 | if (Callee) { |
| 55 | if (Callee->isIntrinsic()) |
| 56 | return false; |
| 57 | auto Name = Callee->getName(); |
| 58 | // objc_msgSend stubs must be called, and can't have their address taken. |
| 59 | if (Name.starts_with(Prefix: "objc_msgSend$" )) |
| 60 | return false; |
| 61 | // Calls to dtrace probes must generate unique patchpoints. |
| 62 | if (Name.starts_with(Prefix: "__dtrace" )) |
| 63 | return false; |
| 64 | } |
| 65 | if (isCalleeOperand(CI, OpIdx)) { |
| 66 | // The operand is the callee and it has already been signed. Ignore this |
| 67 | // because we cannot add another ptrauth bundle to the call instruction. |
| 68 | if (CI->getOperandBundle(ID: LLVMContext::OB_ptrauth).has_value()) |
| 69 | return false; |
| 70 | } else { |
| 71 | // The target of the arc-attached call must be a constant and cannot be |
| 72 | // parameterized. |
| 73 | if (CI->isOperandBundleOfType(ID: LLVMContext::OB_clang_arc_attachedcall, |
| 74 | Idx: OpIdx)) |
| 75 | return false; |
| 76 | } |
| 77 | return true; |
| 78 | } |
| 79 | |
| 80 | /// Returns true if function \p F is eligible for merging. |
| 81 | bool isEligibleFunction(Function *F) { |
| 82 | if (F->isDeclaration()) |
| 83 | return false; |
| 84 | |
| 85 | if (F->hasFnAttribute(Kind: llvm::Attribute::NoMerge) || |
| 86 | F->hasFnAttribute(Kind: llvm::Attribute::AlwaysInline)) |
| 87 | return false; |
| 88 | |
| 89 | if (F->hasAvailableExternallyLinkage()) |
| 90 | return false; |
| 91 | |
| 92 | if (F->getFunctionType()->isVarArg()) |
| 93 | return false; |
| 94 | |
| 95 | if (F->getCallingConv() == CallingConv::SwiftTail) |
| 96 | return false; |
| 97 | |
| 98 | // If function contains callsites with musttail, if we merge |
| 99 | // it, the merged function will have the musttail callsite, but |
| 100 | // the number of parameters can change, thus the parameter count |
| 101 | // of the callsite will mismatch with the function itself. |
| 102 | for (const BasicBlock &BB : *F) { |
| 103 | for (const Instruction &I : BB) { |
| 104 | const auto *CB = dyn_cast<CallBase>(Val: &I); |
| 105 | if (CB && CB->isMustTailCall()) |
| 106 | return false; |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | return true; |
| 111 | } |
| 112 | |
| 113 | static bool isEligibleInstructionForConstantSharing(const Instruction *I) { |
| 114 | switch (I->getOpcode()) { |
| 115 | case Instruction::Load: |
| 116 | case Instruction::Store: |
| 117 | case Instruction::Call: |
| 118 | case Instruction::Invoke: |
| 119 | return true; |
| 120 | default: |
| 121 | return false; |
| 122 | } |
| 123 | } |
| 124 | |
| 125 | // This function takes an instruction, \p I, and an operand index, \p OpIdx. |
| 126 | // It returns true if the operand should be ignored in the hash computation. |
| 127 | // If \p OpIdx is out of range based on the other instruction context, it cannot |
| 128 | // be ignored. |
| 129 | static bool ignoreOp(const Instruction *I, unsigned OpIdx) { |
| 130 | if (OpIdx >= I->getNumOperands()) |
| 131 | return false; |
| 132 | |
| 133 | if (!isEligibleInstructionForConstantSharing(I)) |
| 134 | return false; |
| 135 | |
| 136 | if (!isa<Constant>(Val: I->getOperand(i: OpIdx))) |
| 137 | return false; |
| 138 | |
| 139 | if (const auto *CI = dyn_cast<CallBase>(Val: I)) |
| 140 | return canParameterizeCallOperand(CI, OpIdx); |
| 141 | |
| 142 | return true; |
| 143 | } |
| 144 | |
| 145 | void GlobalMergeFunc::analyze(Module &M) { |
| 146 | ++NumAnalyzedModues; |
| 147 | for (Function &Func : M) { |
| 148 | ++NumAnalyzedFunctions; |
| 149 | if (isEligibleFunction(F: &Func)) { |
| 150 | ++NumEligibleFunctions; |
| 151 | |
| 152 | auto FI = llvm::StructuralHashWithDifferences(F: Func, IgnoreOp: ignoreOp); |
| 153 | |
| 154 | // Convert the operand map to a vector for a serialization-friendly |
| 155 | // format. |
| 156 | IndexOperandHashVecType IndexOperandHashes; |
| 157 | for (auto &Pair : *FI.IndexOperandHashMap) |
| 158 | IndexOperandHashes.emplace_back(Args&: Pair); |
| 159 | |
| 160 | StableFunction SF(FI.FunctionHash, get_stable_name(Name: Func.getName()).str(), |
| 161 | M.getModuleIdentifier(), FI.IndexInstruction->size(), |
| 162 | std::move(IndexOperandHashes)); |
| 163 | |
| 164 | LocalFunctionMap->insert(Func: SF); |
| 165 | } |
| 166 | } |
| 167 | } |
| 168 | |
| 169 | /// Tuple to hold function info to process merging. |
| 170 | struct FuncMergeInfo { |
| 171 | StableFunctionMap::StableFunctionEntry *SF; |
| 172 | Function *F; |
| 173 | IndexInstrMap *IndexInstruction; |
| 174 | FuncMergeInfo(StableFunctionMap::StableFunctionEntry *SF, Function *F, |
| 175 | IndexInstrMap *IndexInstruction) |
| 176 | : SF(SF), F(F), IndexInstruction(std::move(IndexInstruction)) {} |
| 177 | }; |
| 178 | |
| 179 | // Given the func info, and the parameterized locations, create and return |
| 180 | // a new merged function by replacing the original constants with the new |
| 181 | // parameters. |
| 182 | static Function *createMergedFunction(FuncMergeInfo &FI, |
| 183 | ArrayRef<Type *> ConstParamTypes, |
| 184 | const ParamLocsVecTy &ParamLocsVec) { |
| 185 | // Synthesize a new merged function name by appending ".Tgm" to the root |
| 186 | // function's name. |
| 187 | auto *MergedFunc = FI.F; |
| 188 | std::string NewFunctionName = |
| 189 | MergedFunc->getName().str() + GlobalMergeFunc::MergingInstanceSuffix; |
| 190 | auto *M = MergedFunc->getParent(); |
| 191 | assert(!M->getFunction(NewFunctionName)); |
| 192 | |
| 193 | FunctionType *OrigTy = MergedFunc->getFunctionType(); |
| 194 | // Get the original params' types. |
| 195 | SmallVector<Type *> ParamTypes(OrigTy->param_begin(), OrigTy->param_end()); |
| 196 | // Append const parameter types that are passed in. |
| 197 | ParamTypes.append(in_start: ConstParamTypes.begin(), in_end: ConstParamTypes.end()); |
| 198 | FunctionType *FuncType = FunctionType::get(Result: OrigTy->getReturnType(), |
| 199 | Params: ParamTypes, /*isVarArg=*/false); |
| 200 | |
| 201 | // Declare a new function |
| 202 | Function *NewFunction = |
| 203 | Function::Create(Ty: FuncType, Linkage: MergedFunc->getLinkage(), N: NewFunctionName); |
| 204 | if (auto *SP = MergedFunc->getSubprogram()) |
| 205 | NewFunction->setSubprogram(SP); |
| 206 | NewFunction->copyAttributesFrom(Src: MergedFunc); |
| 207 | NewFunction->setDLLStorageClass(GlobalValue::DefaultStorageClass); |
| 208 | |
| 209 | NewFunction->setLinkage(GlobalValue::InternalLinkage); |
| 210 | NewFunction->addFnAttr(Kind: Attribute::NoInline); |
| 211 | |
| 212 | // Add the new function before the root function. |
| 213 | M->getFunctionList().insert(where: MergedFunc->getIterator(), New: NewFunction); |
| 214 | |
| 215 | // Move the body of MergedFunc into the NewFunction. |
| 216 | NewFunction->splice(ToIt: NewFunction->begin(), FromF: MergedFunc); |
| 217 | |
| 218 | // Update the original args by the new args. |
| 219 | auto NewArgIter = NewFunction->arg_begin(); |
| 220 | for (Argument &OrigArg : MergedFunc->args()) { |
| 221 | Argument &NewArg = *NewArgIter++; |
| 222 | OrigArg.replaceAllUsesWith(V: &NewArg); |
| 223 | } |
| 224 | |
| 225 | // Replace the original Constants by the new args. |
| 226 | unsigned NumOrigArgs = MergedFunc->arg_size(); |
| 227 | for (unsigned ParamIdx = 0; ParamIdx < ParamLocsVec.size(); ++ParamIdx) { |
| 228 | Argument *NewArg = NewFunction->getArg(i: NumOrigArgs + ParamIdx); |
| 229 | for (auto [InstIndex, OpndIndex] : ParamLocsVec[ParamIdx]) { |
| 230 | auto *Inst = FI.IndexInstruction->lookup(Key: InstIndex); |
| 231 | auto *OrigC = Inst->getOperand(i: OpndIndex); |
| 232 | if (OrigC->getType() != NewArg->getType()) { |
| 233 | IRBuilder<> Builder(Inst->getParent(), Inst->getIterator()); |
| 234 | Inst->setOperand(i: OpndIndex, |
| 235 | Val: Builder.CreateAggregateCast(V: NewArg, DestTy: OrigC->getType())); |
| 236 | } else { |
| 237 | Inst->setOperand(i: OpndIndex, Val: NewArg); |
| 238 | } |
| 239 | } |
| 240 | } |
| 241 | |
| 242 | return NewFunction; |
| 243 | } |
| 244 | |
| 245 | // Given the original function (Thunk) and the merged function (ToFunc), create |
| 246 | // a thunk to the merged function. |
| 247 | static void createThunk(FuncMergeInfo &FI, ArrayRef<Constant *> Params, |
| 248 | Function *ToFunc) { |
| 249 | auto *Thunk = FI.F; |
| 250 | |
| 251 | assert(Thunk->arg_size() + Params.size() == |
| 252 | ToFunc->getFunctionType()->getNumParams()); |
| 253 | Thunk->dropAllReferences(); |
| 254 | |
| 255 | BasicBlock *BB = BasicBlock::Create(Context&: Thunk->getContext(), Name: "" , Parent: Thunk); |
| 256 | IRBuilder<> Builder(BB); |
| 257 | |
| 258 | SmallVector<Value *> Args; |
| 259 | unsigned ParamIdx = 0; |
| 260 | FunctionType *ToFuncTy = ToFunc->getFunctionType(); |
| 261 | |
| 262 | // Add arguments which are passed through Thunk. |
| 263 | for (Argument &AI : Thunk->args()) { |
| 264 | Args.push_back( |
| 265 | Elt: Builder.CreateAggregateCast(V: &AI, DestTy: ToFuncTy->getParamType(i: ParamIdx))); |
| 266 | ++ParamIdx; |
| 267 | } |
| 268 | |
| 269 | // Add new arguments defined by Params. |
| 270 | for (auto *Param : Params) { |
| 271 | assert(ParamIdx < ToFuncTy->getNumParams()); |
| 272 | Args.push_back( |
| 273 | Elt: Builder.CreateAggregateCast(V: Param, DestTy: ToFuncTy->getParamType(i: ParamIdx))); |
| 274 | ++ParamIdx; |
| 275 | } |
| 276 | |
| 277 | CallInst *CI = Builder.CreateCall(Callee: ToFunc, Args); |
| 278 | bool isSwiftTailCall = ToFunc->getCallingConv() == CallingConv::SwiftTail && |
| 279 | Thunk->getCallingConv() == CallingConv::SwiftTail; |
| 280 | CI->setTailCallKind(isSwiftTailCall ? llvm::CallInst::TCK_MustTail |
| 281 | : llvm::CallInst::TCK_Tail); |
| 282 | CI->setCallingConv(ToFunc->getCallingConv()); |
| 283 | CI->setAttributes(ToFunc->getAttributes()); |
| 284 | if (Thunk->getReturnType()->isVoidTy()) |
| 285 | Builder.CreateRetVoid(); |
| 286 | else |
| 287 | Builder.CreateRet(V: Builder.CreateAggregateCast(V: CI, DestTy: Thunk->getReturnType())); |
| 288 | } |
| 289 | |
| 290 | // Check if the old merged/optimized IndexOperandHashMap is compatible with |
| 291 | // the current IndexOperandHashMap. An operand hash may not be stable across |
| 292 | // different builds due to varying modules combined. To address this, we relax |
| 293 | // the hash check condition by comparing Const hash patterns instead of absolute |
| 294 | // hash values. For example, let's assume we have three Consts located at idx1, |
| 295 | // idx3, and idx6, where their corresponding hashes are hash1, hash2, and hash1 |
| 296 | // in the old merged map below: |
| 297 | // Old (Merged): [(idx1, hash1), (idx3, hash2), (idx6, hash1)] |
| 298 | // Current: [(idx1, hash1'), (idx3, hash2'), (idx6, hash1')] |
| 299 | // If the current function also has three Consts in the same locations, |
| 300 | // with hash sequences hash1', hash2', and hash1' where the first and third |
| 301 | // are the same as the old hash sequences, we consider them matched. |
| 302 | static bool checkConstHashCompatible( |
| 303 | const DenseMap<IndexPair, stable_hash> &OldInstOpndIndexToConstHash, |
| 304 | const DenseMap<IndexPair, stable_hash> &CurrInstOpndIndexToConstHash) { |
| 305 | |
| 306 | DenseMap<stable_hash, stable_hash> OldHashToCurrHash; |
| 307 | for (const auto &[Index, OldHash] : OldInstOpndIndexToConstHash) { |
| 308 | auto It = CurrInstOpndIndexToConstHash.find(Val: Index); |
| 309 | if (It == CurrInstOpndIndexToConstHash.end()) |
| 310 | return false; |
| 311 | |
| 312 | auto CurrHash = It->second; |
| 313 | auto J = OldHashToCurrHash.find(Val: OldHash); |
| 314 | if (J == OldHashToCurrHash.end()) |
| 315 | OldHashToCurrHash.insert(KV: {OldHash, CurrHash}); |
| 316 | else if (J->second != CurrHash) |
| 317 | return false; |
| 318 | } |
| 319 | |
| 320 | return true; |
| 321 | } |
| 322 | |
| 323 | // Validate the locations pointed by a param has the same hash and Constant. |
| 324 | static bool |
| 325 | checkConstLocationCompatible(const StableFunctionMap::StableFunctionEntry &SF, |
| 326 | const IndexInstrMap &IndexInstruction, |
| 327 | const ParamLocsVecTy &ParamLocsVec) { |
| 328 | for (auto &ParamLocs : ParamLocsVec) { |
| 329 | std::optional<stable_hash> OldHash; |
| 330 | std::optional<Constant *> OldConst; |
| 331 | for (auto &Loc : ParamLocs) { |
| 332 | assert(SF.IndexOperandHashMap->count(Loc)); |
| 333 | auto CurrHash = SF.IndexOperandHashMap->at(Val: Loc); |
| 334 | auto [InstIndex, OpndIndex] = Loc; |
| 335 | assert(InstIndex < IndexInstruction.size()); |
| 336 | const auto *Inst = IndexInstruction.lookup(Key: InstIndex); |
| 337 | auto *CurrConst = cast<Constant>(Val: Inst->getOperand(i: OpndIndex)); |
| 338 | if (!OldHash) { |
| 339 | OldHash = CurrHash; |
| 340 | OldConst = CurrConst; |
| 341 | } else if (CurrConst != *OldConst || CurrHash != *OldHash) { |
| 342 | return false; |
| 343 | } |
| 344 | } |
| 345 | } |
| 346 | return true; |
| 347 | } |
| 348 | |
| 349 | static ParamLocsVecTy computeParamInfo( |
| 350 | const SmallVector<std::unique_ptr<StableFunctionMap::StableFunctionEntry>> |
| 351 | &SFS) { |
| 352 | std::map<std::vector<stable_hash>, ParamLocs> HashSeqToLocs; |
| 353 | auto &RSF = *SFS[0]; |
| 354 | unsigned StableFunctionCount = SFS.size(); |
| 355 | |
| 356 | for (auto &[IndexPair, Hash] : *RSF.IndexOperandHashMap) { |
| 357 | // Const hash sequence across stable functions. |
| 358 | // We will allocate a parameter per unique hash squence. |
| 359 | // can't use SmallVector as key |
| 360 | std::vector<stable_hash> ConstHashSeq; |
| 361 | ConstHashSeq.push_back(x: Hash); |
| 362 | bool Identical = true; |
| 363 | for (unsigned J = 1; J < StableFunctionCount; ++J) { |
| 364 | auto &SF = SFS[J]; |
| 365 | auto SHash = SF->IndexOperandHashMap->at(Val: IndexPair); |
| 366 | if (Hash != SHash) |
| 367 | Identical = false; |
| 368 | ConstHashSeq.push_back(x: SHash); |
| 369 | } |
| 370 | |
| 371 | if (Identical) |
| 372 | continue; |
| 373 | |
| 374 | // For each unique Const hash sequence (parameter), add the locations. |
| 375 | HashSeqToLocs[ConstHashSeq].push_back(Elt: IndexPair); |
| 376 | } |
| 377 | |
| 378 | ParamLocsVecTy ParamLocsVec; |
| 379 | for (auto &[HashSeq, Locs] : HashSeqToLocs) |
| 380 | ParamLocsVec.push_back(Elt: std::move(Locs)); |
| 381 | |
| 382 | llvm::sort(C&: ParamLocsVec, Comp: [&](const ParamLocs &L, const ParamLocs &R) { |
| 383 | return L[0] < R[0]; |
| 384 | }); |
| 385 | |
| 386 | return ParamLocsVec; |
| 387 | } |
| 388 | |
| 389 | bool GlobalMergeFunc::merge(Module &M, const StableFunctionMap *FunctionMap) { |
| 390 | bool Changed = false; |
| 391 | |
| 392 | // Collect stable functions related to the current module. |
| 393 | DenseMap<stable_hash, SmallVector<std::pair<Function *, FunctionHashInfo>>> |
| 394 | HashToFuncs; |
| 395 | auto &Maps = FunctionMap->getFunctionMap(); |
| 396 | for (auto &F : M) { |
| 397 | if (!isEligibleFunction(F: &F)) |
| 398 | continue; |
| 399 | auto FI = llvm::StructuralHashWithDifferences(F, IgnoreOp: ignoreOp); |
| 400 | if (Maps.contains(Val: FI.FunctionHash)) |
| 401 | HashToFuncs[FI.FunctionHash].emplace_back(Args: &F, Args: std::move(FI)); |
| 402 | } |
| 403 | |
| 404 | for (auto &[Hash, Funcs] : HashToFuncs) { |
| 405 | std::optional<ParamLocsVecTy> ParamLocsVec; |
| 406 | SmallVector<FuncMergeInfo> FuncMergeInfos; |
| 407 | auto &SFS = Maps.at(Val: Hash); |
| 408 | assert(!SFS.empty()); |
| 409 | auto &RFS = SFS[0]; |
| 410 | |
| 411 | // Iterate functions with the same hash. |
| 412 | for (auto &[F, FI] : Funcs) { |
| 413 | // Check if the function is compatible with any stable function |
| 414 | // in terms of the number of instructions and ignored operands. |
| 415 | if (RFS->InstCount != FI.IndexInstruction->size()) |
| 416 | continue; |
| 417 | |
| 418 | auto hasValidSharedConst = [&](StableFunctionMap::StableFunctionEntry *SF, |
| 419 | FunctionHashInfo &FHI) { |
| 420 | for (auto &[Index, Hash] : *SF->IndexOperandHashMap) { |
| 421 | auto [InstIndex, OpndIndex] = Index; |
| 422 | assert(InstIndex < FHI.IndexInstruction->size()); |
| 423 | auto *Inst = FHI.IndexInstruction->lookup(Key: InstIndex); |
| 424 | if (!ignoreOp(I: Inst, OpIdx: OpndIndex)) |
| 425 | return false; |
| 426 | } |
| 427 | return true; |
| 428 | }; |
| 429 | if (!hasValidSharedConst(RFS.get(), FI)) |
| 430 | continue; |
| 431 | |
| 432 | for (auto &SF : SFS) { |
| 433 | assert(SF->InstCount == FI.IndexInstruction->size()); |
| 434 | assert(hasValidSharedConst(SF.get(), FI)); |
| 435 | // Check if there is any stable function that is compatiable with the |
| 436 | // current one. |
| 437 | if (!checkConstHashCompatible(OldInstOpndIndexToConstHash: *SF->IndexOperandHashMap, |
| 438 | CurrInstOpndIndexToConstHash: *FI.IndexOperandHashMap)) |
| 439 | continue; |
| 440 | if (!ParamLocsVec.has_value()) { |
| 441 | ParamLocsVec = computeParamInfo(SFS); |
| 442 | LLVM_DEBUG(dbgs() << "[GlobalMergeFunc] Merging hash: " << Hash |
| 443 | << " with Params " << ParamLocsVec->size() << "\n" ); |
| 444 | } |
| 445 | if (!checkConstLocationCompatible(SF: *SF, IndexInstruction: *FI.IndexInstruction, |
| 446 | ParamLocsVec: *ParamLocsVec)) |
| 447 | continue; |
| 448 | |
| 449 | // If a stable function matching the current one is found, |
| 450 | // create a candidate for merging and proceed to the next function. |
| 451 | FuncMergeInfos.emplace_back(Args: SF.get(), Args&: F, Args: FI.IndexInstruction.get()); |
| 452 | break; |
| 453 | } |
| 454 | } |
| 455 | unsigned FuncMergeInfoSize = FuncMergeInfos.size(); |
| 456 | if (FuncMergeInfoSize == 0) |
| 457 | continue; |
| 458 | |
| 459 | LLVM_DEBUG(dbgs() << "[GlobalMergeFunc] Merging function count " |
| 460 | << FuncMergeInfoSize << " for hash: " << Hash << "\n" ); |
| 461 | |
| 462 | for (auto &FMI : FuncMergeInfos) { |
| 463 | Changed = true; |
| 464 | |
| 465 | // We've already validated all locations of constant operands pointed by |
| 466 | // the parameters. Populate parameters pointing to the original constants. |
| 467 | SmallVector<Constant *> Params; |
| 468 | SmallVector<Type *> ParamTypes; |
| 469 | for (auto &ParamLocs : *ParamLocsVec) { |
| 470 | assert(!ParamLocs.empty()); |
| 471 | auto &[InstIndex, OpndIndex] = ParamLocs[0]; |
| 472 | auto *Inst = FMI.IndexInstruction->lookup(Key: InstIndex); |
| 473 | auto *Opnd = cast<Constant>(Val: Inst->getOperand(i: OpndIndex)); |
| 474 | Params.push_back(Elt: Opnd); |
| 475 | ParamTypes.push_back(Elt: Opnd->getType()); |
| 476 | } |
| 477 | |
| 478 | // Create a merged function derived from the current function. |
| 479 | Function *MergedFunc = |
| 480 | createMergedFunction(FI&: FMI, ConstParamTypes: ParamTypes, ParamLocsVec: *ParamLocsVec); |
| 481 | |
| 482 | LLVM_DEBUG({ |
| 483 | dbgs() << "[GlobalMergeFunc] Merged function (hash:" << FMI.SF->Hash |
| 484 | << ") " << MergedFunc->getName() << " generated from " |
| 485 | << FMI.F->getName() << ":\n" ; |
| 486 | MergedFunc->dump(); |
| 487 | }); |
| 488 | |
| 489 | // Transform the current function into a thunk that calls the merged |
| 490 | // function. |
| 491 | createThunk(FI&: FMI, Params, ToFunc: MergedFunc); |
| 492 | LLVM_DEBUG({ |
| 493 | dbgs() << "[GlobalMergeFunc] Thunk generated: \n" ; |
| 494 | FMI.F->dump(); |
| 495 | }); |
| 496 | ++NumMergedFunctions; |
| 497 | } |
| 498 | } |
| 499 | |
| 500 | return Changed; |
| 501 | } |
| 502 | |
| 503 | void GlobalMergeFunc::initializeMergerMode(const Module &M) { |
| 504 | // Initialize the local function map regardless of the merger mode. |
| 505 | LocalFunctionMap = std::make_unique<StableFunctionMap>(); |
| 506 | |
| 507 | // Disable codegen data for merging. The local merge is still enabled. |
| 508 | if (DisableCGDataForMerging) |
| 509 | return; |
| 510 | |
| 511 | // (Full)LTO module does not have functions added to the index. |
| 512 | // In this case, we run a local merger without using codegen data. |
| 513 | if (Index && !Index->hasExportedFunctions(M)) |
| 514 | return; |
| 515 | |
| 516 | if (cgdata::emitCGData()) |
| 517 | MergerMode = HashFunctionMode::BuildingHashFuncion; |
| 518 | else if (cgdata::hasStableFunctionMap()) |
| 519 | MergerMode = HashFunctionMode::UsingHashFunction; |
| 520 | } |
| 521 | |
| 522 | void GlobalMergeFunc::emitFunctionMap(Module &M) { |
| 523 | LLVM_DEBUG(dbgs() << "Emit function map. Size: " << LocalFunctionMap->size() |
| 524 | << "\n" ); |
| 525 | // No need to emit the function map if it is empty. |
| 526 | if (LocalFunctionMap->empty()) |
| 527 | return; |
| 528 | SmallVector<char> Buf; |
| 529 | raw_svector_ostream OS(Buf); |
| 530 | |
| 531 | std::vector<CGDataPatchItem> PatchItems; |
| 532 | StableFunctionMapRecord::serialize(OS, FunctionMap: LocalFunctionMap.get(), PatchItems); |
| 533 | CGDataOStream COS(OS); |
| 534 | COS.patch(P: PatchItems); |
| 535 | |
| 536 | std::unique_ptr<MemoryBuffer> Buffer = MemoryBuffer::getMemBuffer( |
| 537 | InputData: OS.str(), BufferName: "in-memory stable function map" , RequiresNullTerminator: false); |
| 538 | |
| 539 | Triple TT(M.getTargetTriple()); |
| 540 | embedBufferInModule(M, Buf: *Buffer, |
| 541 | SectionName: getCodeGenDataSectionName(CGSK: CG_merge, OF: TT.getObjectFormat()), |
| 542 | Alignment: Align(4)); |
| 543 | } |
| 544 | |
| 545 | bool GlobalMergeFunc::run(Module &M) { |
| 546 | initializeMergerMode(M); |
| 547 | |
| 548 | const StableFunctionMap *FuncMap; |
| 549 | if (MergerMode == HashFunctionMode::UsingHashFunction) { |
| 550 | // Use the prior CG data to optimistically create global merge candidates. |
| 551 | FuncMap = cgdata::getStableFunctionMap(); |
| 552 | } else { |
| 553 | analyze(M); |
| 554 | // Emit the local function map to the custom section, __llvm_merge before |
| 555 | // finalizing it. |
| 556 | if (MergerMode == HashFunctionMode::BuildingHashFuncion) |
| 557 | emitFunctionMap(M); |
| 558 | LocalFunctionMap->finalize(); |
| 559 | FuncMap = LocalFunctionMap.get(); |
| 560 | } |
| 561 | |
| 562 | return merge(M, FunctionMap: FuncMap); |
| 563 | } |
| 564 | |
| 565 | namespace { |
| 566 | |
| 567 | class GlobalMergeFuncPassWrapper : public ModulePass { |
| 568 | |
| 569 | public: |
| 570 | static char ID; |
| 571 | |
| 572 | GlobalMergeFuncPassWrapper(); |
| 573 | |
| 574 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
| 575 | AU.addUsedIfAvailable<ImmutableModuleSummaryIndexWrapperPass>(); |
| 576 | AU.setPreservesAll(); |
| 577 | ModulePass::getAnalysisUsage(AU); |
| 578 | } |
| 579 | |
| 580 | StringRef getPassName() const override { return "Global Merge Functions" ; } |
| 581 | |
| 582 | bool runOnModule(Module &M) override; |
| 583 | }; |
| 584 | |
| 585 | } // namespace |
| 586 | |
| 587 | char GlobalMergeFuncPassWrapper::ID = 0; |
| 588 | INITIALIZE_PASS_BEGIN(GlobalMergeFuncPassWrapper, "global-merge-func" , |
| 589 | "Global merge function pass" , false, false) |
| 590 | INITIALIZE_PASS_END(GlobalMergeFuncPassWrapper, "global-merge-func" , |
| 591 | "Global merge function pass" , false, false) |
| 592 | |
| 593 | namespace llvm { |
| 594 | ModulePass *createGlobalMergeFuncPass() { |
| 595 | return new GlobalMergeFuncPassWrapper(); |
| 596 | } |
| 597 | } // namespace llvm |
| 598 | |
| 599 | GlobalMergeFuncPassWrapper::GlobalMergeFuncPassWrapper() : ModulePass(ID) { |
| 600 | initializeGlobalMergeFuncPassWrapperPass( |
| 601 | Registry&: *llvm::PassRegistry::getPassRegistry()); |
| 602 | } |
| 603 | |
| 604 | bool GlobalMergeFuncPassWrapper::runOnModule(Module &M) { |
| 605 | const ModuleSummaryIndex *Index = nullptr; |
| 606 | if (auto *IndexWrapperPass = |
| 607 | getAnalysisIfAvailable<ImmutableModuleSummaryIndexWrapperPass>()) |
| 608 | Index = IndexWrapperPass->getIndex(); |
| 609 | |
| 610 | return GlobalMergeFunc(Index).run(M); |
| 611 | } |
| 612 | |
| 613 | PreservedAnalyses GlobalMergeFuncPass::run(Module &M, |
| 614 | AnalysisManager<Module> &AM) { |
| 615 | bool Changed = GlobalMergeFunc(ImportSummary).run(M); |
| 616 | return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); |
| 617 | } |
| 618 | |