1 | //===---- GlobalMergeFunctions.cpp - Global merge functions -------*- C++ -===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This pass implements the global merge function pass. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "llvm/CodeGen/GlobalMergeFunctions.h" |
14 | #include "llvm/ADT/Statistic.h" |
15 | #include "llvm/Analysis/ModuleSummaryAnalysis.h" |
16 | #include "llvm/CGData/CodeGenData.h" |
17 | #include "llvm/CGData/CodeGenDataWriter.h" |
18 | #include "llvm/CodeGen/Passes.h" |
19 | #include "llvm/IR/IRBuilder.h" |
20 | #include "llvm/IR/StructuralHash.h" |
21 | #include "llvm/InitializePasses.h" |
22 | #include "llvm/Support/CommandLine.h" |
23 | #include "llvm/Transforms/Utils/ModuleUtils.h" |
24 | |
25 | #define DEBUG_TYPE "global-merge-func" |
26 | |
27 | using namespace llvm; |
28 | using namespace llvm::support; |
29 | |
30 | static cl::opt<bool> DisableCGDataForMerging( |
31 | "disable-cgdata-for-merging" , cl::Hidden, |
32 | cl::desc("Disable codegen data for function merging. Local " |
33 | "merging is still enabled within a module." ), |
34 | cl::init(Val: false)); |
35 | |
36 | STATISTIC(NumMergedFunctions, |
37 | "Number of functions that are actually merged using function hash" ); |
38 | STATISTIC(NumAnalyzedModues, "Number of modules that are analyzed" ); |
39 | STATISTIC(NumAnalyzedFunctions, "Number of functions that are analyzed" ); |
40 | STATISTIC(NumEligibleFunctions, "Number of functions that are eligible" ); |
41 | |
42 | /// Returns true if the \OpIdx operand of \p CI is the callee operand. |
43 | static bool isCalleeOperand(const CallBase *CI, unsigned OpIdx) { |
44 | return &CI->getCalledOperandUse() == &CI->getOperandUse(i: OpIdx); |
45 | } |
46 | |
47 | static bool canParameterizeCallOperand(const CallBase *CI, unsigned OpIdx) { |
48 | if (CI->isInlineAsm()) |
49 | return false; |
50 | Function *Callee = CI->getCalledOperand() |
51 | ? dyn_cast_or_null<Function>( |
52 | Val: CI->getCalledOperand()->stripPointerCasts()) |
53 | : nullptr; |
54 | if (Callee) { |
55 | if (Callee->isIntrinsic()) |
56 | return false; |
57 | auto Name = Callee->getName(); |
58 | // objc_msgSend stubs must be called, and can't have their address taken. |
59 | if (Name.starts_with(Prefix: "objc_msgSend$" )) |
60 | return false; |
61 | // Calls to dtrace probes must generate unique patchpoints. |
62 | if (Name.starts_with(Prefix: "__dtrace" )) |
63 | return false; |
64 | } |
65 | if (isCalleeOperand(CI, OpIdx)) { |
66 | // The operand is the callee and it has already been signed. Ignore this |
67 | // because we cannot add another ptrauth bundle to the call instruction. |
68 | if (CI->getOperandBundle(ID: LLVMContext::OB_ptrauth).has_value()) |
69 | return false; |
70 | } else { |
71 | // The target of the arc-attached call must be a constant and cannot be |
72 | // parameterized. |
73 | if (CI->isOperandBundleOfType(ID: LLVMContext::OB_clang_arc_attachedcall, |
74 | Idx: OpIdx)) |
75 | return false; |
76 | } |
77 | return true; |
78 | } |
79 | |
80 | /// Returns true if function \p F is eligible for merging. |
81 | bool isEligibleFunction(Function *F) { |
82 | if (F->isDeclaration()) |
83 | return false; |
84 | |
85 | if (F->hasFnAttribute(Kind: llvm::Attribute::NoMerge) || |
86 | F->hasFnAttribute(Kind: llvm::Attribute::AlwaysInline)) |
87 | return false; |
88 | |
89 | if (F->hasAvailableExternallyLinkage()) |
90 | return false; |
91 | |
92 | if (F->getFunctionType()->isVarArg()) |
93 | return false; |
94 | |
95 | if (F->getCallingConv() == CallingConv::SwiftTail) |
96 | return false; |
97 | |
98 | // If function contains callsites with musttail, if we merge |
99 | // it, the merged function will have the musttail callsite, but |
100 | // the number of parameters can change, thus the parameter count |
101 | // of the callsite will mismatch with the function itself. |
102 | for (const BasicBlock &BB : *F) { |
103 | for (const Instruction &I : BB) { |
104 | const auto *CB = dyn_cast<CallBase>(Val: &I); |
105 | if (CB && CB->isMustTailCall()) |
106 | return false; |
107 | } |
108 | } |
109 | |
110 | return true; |
111 | } |
112 | |
113 | static bool isEligibleInstructionForConstantSharing(const Instruction *I) { |
114 | switch (I->getOpcode()) { |
115 | case Instruction::Load: |
116 | case Instruction::Store: |
117 | case Instruction::Call: |
118 | case Instruction::Invoke: |
119 | return true; |
120 | default: |
121 | return false; |
122 | } |
123 | } |
124 | |
125 | // This function takes an instruction, \p I, and an operand index, \p OpIdx. |
126 | // It returns true if the operand should be ignored in the hash computation. |
127 | // If \p OpIdx is out of range based on the other instruction context, it cannot |
128 | // be ignored. |
129 | static bool ignoreOp(const Instruction *I, unsigned OpIdx) { |
130 | if (OpIdx >= I->getNumOperands()) |
131 | return false; |
132 | |
133 | if (!isEligibleInstructionForConstantSharing(I)) |
134 | return false; |
135 | |
136 | if (!isa<Constant>(Val: I->getOperand(i: OpIdx))) |
137 | return false; |
138 | |
139 | if (const auto *CI = dyn_cast<CallBase>(Val: I)) |
140 | return canParameterizeCallOperand(CI, OpIdx); |
141 | |
142 | return true; |
143 | } |
144 | |
145 | void GlobalMergeFunc::analyze(Module &M) { |
146 | ++NumAnalyzedModues; |
147 | for (Function &Func : M) { |
148 | ++NumAnalyzedFunctions; |
149 | if (isEligibleFunction(F: &Func)) { |
150 | ++NumEligibleFunctions; |
151 | |
152 | auto FI = llvm::StructuralHashWithDifferences(F: Func, IgnoreOp: ignoreOp); |
153 | |
154 | // Convert the operand map to a vector for a serialization-friendly |
155 | // format. |
156 | IndexOperandHashVecType IndexOperandHashes; |
157 | for (auto &Pair : *FI.IndexOperandHashMap) |
158 | IndexOperandHashes.emplace_back(Args&: Pair); |
159 | |
160 | StableFunction SF(FI.FunctionHash, get_stable_name(Name: Func.getName()).str(), |
161 | M.getModuleIdentifier(), FI.IndexInstruction->size(), |
162 | std::move(IndexOperandHashes)); |
163 | |
164 | LocalFunctionMap->insert(Func: SF); |
165 | } |
166 | } |
167 | } |
168 | |
169 | /// Tuple to hold function info to process merging. |
170 | struct FuncMergeInfo { |
171 | StableFunctionMap::StableFunctionEntry *SF; |
172 | Function *F; |
173 | IndexInstrMap *IndexInstruction; |
174 | FuncMergeInfo(StableFunctionMap::StableFunctionEntry *SF, Function *F, |
175 | IndexInstrMap *IndexInstruction) |
176 | : SF(SF), F(F), IndexInstruction(std::move(IndexInstruction)) {} |
177 | }; |
178 | |
179 | // Given the func info, and the parameterized locations, create and return |
180 | // a new merged function by replacing the original constants with the new |
181 | // parameters. |
182 | static Function *createMergedFunction(FuncMergeInfo &FI, |
183 | ArrayRef<Type *> ConstParamTypes, |
184 | const ParamLocsVecTy &ParamLocsVec) { |
185 | // Synthesize a new merged function name by appending ".Tgm" to the root |
186 | // function's name. |
187 | auto *MergedFunc = FI.F; |
188 | std::string NewFunctionName = |
189 | MergedFunc->getName().str() + GlobalMergeFunc::MergingInstanceSuffix; |
190 | auto *M = MergedFunc->getParent(); |
191 | assert(!M->getFunction(NewFunctionName)); |
192 | |
193 | FunctionType *OrigTy = MergedFunc->getFunctionType(); |
194 | // Get the original params' types. |
195 | SmallVector<Type *> ParamTypes(OrigTy->param_begin(), OrigTy->param_end()); |
196 | // Append const parameter types that are passed in. |
197 | ParamTypes.append(in_start: ConstParamTypes.begin(), in_end: ConstParamTypes.end()); |
198 | FunctionType *FuncType = FunctionType::get(Result: OrigTy->getReturnType(), |
199 | Params: ParamTypes, /*isVarArg=*/false); |
200 | |
201 | // Declare a new function |
202 | Function *NewFunction = |
203 | Function::Create(Ty: FuncType, Linkage: MergedFunc->getLinkage(), N: NewFunctionName); |
204 | if (auto *SP = MergedFunc->getSubprogram()) |
205 | NewFunction->setSubprogram(SP); |
206 | NewFunction->copyAttributesFrom(Src: MergedFunc); |
207 | NewFunction->setDLLStorageClass(GlobalValue::DefaultStorageClass); |
208 | |
209 | NewFunction->setLinkage(GlobalValue::InternalLinkage); |
210 | NewFunction->addFnAttr(Kind: Attribute::NoInline); |
211 | |
212 | // Add the new function before the root function. |
213 | M->getFunctionList().insert(where: MergedFunc->getIterator(), New: NewFunction); |
214 | |
215 | // Move the body of MergedFunc into the NewFunction. |
216 | NewFunction->splice(ToIt: NewFunction->begin(), FromF: MergedFunc); |
217 | |
218 | // Update the original args by the new args. |
219 | auto NewArgIter = NewFunction->arg_begin(); |
220 | for (Argument &OrigArg : MergedFunc->args()) { |
221 | Argument &NewArg = *NewArgIter++; |
222 | OrigArg.replaceAllUsesWith(V: &NewArg); |
223 | } |
224 | |
225 | // Replace the original Constants by the new args. |
226 | unsigned NumOrigArgs = MergedFunc->arg_size(); |
227 | for (unsigned ParamIdx = 0; ParamIdx < ParamLocsVec.size(); ++ParamIdx) { |
228 | Argument *NewArg = NewFunction->getArg(i: NumOrigArgs + ParamIdx); |
229 | for (auto [InstIndex, OpndIndex] : ParamLocsVec[ParamIdx]) { |
230 | auto *Inst = FI.IndexInstruction->lookup(Key: InstIndex); |
231 | auto *OrigC = Inst->getOperand(i: OpndIndex); |
232 | if (OrigC->getType() != NewArg->getType()) { |
233 | IRBuilder<> Builder(Inst->getParent(), Inst->getIterator()); |
234 | Inst->setOperand(i: OpndIndex, |
235 | Val: Builder.CreateAggregateCast(V: NewArg, DestTy: OrigC->getType())); |
236 | } else { |
237 | Inst->setOperand(i: OpndIndex, Val: NewArg); |
238 | } |
239 | } |
240 | } |
241 | |
242 | return NewFunction; |
243 | } |
244 | |
245 | // Given the original function (Thunk) and the merged function (ToFunc), create |
246 | // a thunk to the merged function. |
247 | static void createThunk(FuncMergeInfo &FI, ArrayRef<Constant *> Params, |
248 | Function *ToFunc) { |
249 | auto *Thunk = FI.F; |
250 | |
251 | assert(Thunk->arg_size() + Params.size() == |
252 | ToFunc->getFunctionType()->getNumParams()); |
253 | Thunk->dropAllReferences(); |
254 | |
255 | BasicBlock *BB = BasicBlock::Create(Context&: Thunk->getContext(), Name: "" , Parent: Thunk); |
256 | IRBuilder<> Builder(BB); |
257 | |
258 | SmallVector<Value *> Args; |
259 | unsigned ParamIdx = 0; |
260 | FunctionType *ToFuncTy = ToFunc->getFunctionType(); |
261 | |
262 | // Add arguments which are passed through Thunk. |
263 | for (Argument &AI : Thunk->args()) { |
264 | Args.push_back( |
265 | Elt: Builder.CreateAggregateCast(V: &AI, DestTy: ToFuncTy->getParamType(i: ParamIdx))); |
266 | ++ParamIdx; |
267 | } |
268 | |
269 | // Add new arguments defined by Params. |
270 | for (auto *Param : Params) { |
271 | assert(ParamIdx < ToFuncTy->getNumParams()); |
272 | Args.push_back( |
273 | Elt: Builder.CreateAggregateCast(V: Param, DestTy: ToFuncTy->getParamType(i: ParamIdx))); |
274 | ++ParamIdx; |
275 | } |
276 | |
277 | CallInst *CI = Builder.CreateCall(Callee: ToFunc, Args); |
278 | bool isSwiftTailCall = ToFunc->getCallingConv() == CallingConv::SwiftTail && |
279 | Thunk->getCallingConv() == CallingConv::SwiftTail; |
280 | CI->setTailCallKind(isSwiftTailCall ? llvm::CallInst::TCK_MustTail |
281 | : llvm::CallInst::TCK_Tail); |
282 | CI->setCallingConv(ToFunc->getCallingConv()); |
283 | CI->setAttributes(ToFunc->getAttributes()); |
284 | if (Thunk->getReturnType()->isVoidTy()) |
285 | Builder.CreateRetVoid(); |
286 | else |
287 | Builder.CreateRet(V: Builder.CreateAggregateCast(V: CI, DestTy: Thunk->getReturnType())); |
288 | } |
289 | |
290 | // Check if the old merged/optimized IndexOperandHashMap is compatible with |
291 | // the current IndexOperandHashMap. An operand hash may not be stable across |
292 | // different builds due to varying modules combined. To address this, we relax |
293 | // the hash check condition by comparing Const hash patterns instead of absolute |
294 | // hash values. For example, let's assume we have three Consts located at idx1, |
295 | // idx3, and idx6, where their corresponding hashes are hash1, hash2, and hash1 |
296 | // in the old merged map below: |
297 | // Old (Merged): [(idx1, hash1), (idx3, hash2), (idx6, hash1)] |
298 | // Current: [(idx1, hash1'), (idx3, hash2'), (idx6, hash1')] |
299 | // If the current function also has three Consts in the same locations, |
300 | // with hash sequences hash1', hash2', and hash1' where the first and third |
301 | // are the same as the old hash sequences, we consider them matched. |
302 | static bool checkConstHashCompatible( |
303 | const DenseMap<IndexPair, stable_hash> &OldInstOpndIndexToConstHash, |
304 | const DenseMap<IndexPair, stable_hash> &CurrInstOpndIndexToConstHash) { |
305 | |
306 | DenseMap<stable_hash, stable_hash> OldHashToCurrHash; |
307 | for (const auto &[Index, OldHash] : OldInstOpndIndexToConstHash) { |
308 | auto It = CurrInstOpndIndexToConstHash.find(Val: Index); |
309 | if (It == CurrInstOpndIndexToConstHash.end()) |
310 | return false; |
311 | |
312 | auto CurrHash = It->second; |
313 | auto J = OldHashToCurrHash.find(Val: OldHash); |
314 | if (J == OldHashToCurrHash.end()) |
315 | OldHashToCurrHash.insert(KV: {OldHash, CurrHash}); |
316 | else if (J->second != CurrHash) |
317 | return false; |
318 | } |
319 | |
320 | return true; |
321 | } |
322 | |
323 | // Validate the locations pointed by a param has the same hash and Constant. |
324 | static bool |
325 | checkConstLocationCompatible(const StableFunctionMap::StableFunctionEntry &SF, |
326 | const IndexInstrMap &IndexInstruction, |
327 | const ParamLocsVecTy &ParamLocsVec) { |
328 | for (auto &ParamLocs : ParamLocsVec) { |
329 | std::optional<stable_hash> OldHash; |
330 | std::optional<Constant *> OldConst; |
331 | for (auto &Loc : ParamLocs) { |
332 | assert(SF.IndexOperandHashMap->count(Loc)); |
333 | auto CurrHash = SF.IndexOperandHashMap->at(Val: Loc); |
334 | auto [InstIndex, OpndIndex] = Loc; |
335 | assert(InstIndex < IndexInstruction.size()); |
336 | const auto *Inst = IndexInstruction.lookup(Key: InstIndex); |
337 | auto *CurrConst = cast<Constant>(Val: Inst->getOperand(i: OpndIndex)); |
338 | if (!OldHash) { |
339 | OldHash = CurrHash; |
340 | OldConst = CurrConst; |
341 | } else if (CurrConst != *OldConst || CurrHash != *OldHash) { |
342 | return false; |
343 | } |
344 | } |
345 | } |
346 | return true; |
347 | } |
348 | |
349 | static ParamLocsVecTy computeParamInfo( |
350 | const SmallVector<std::unique_ptr<StableFunctionMap::StableFunctionEntry>> |
351 | &SFS) { |
352 | std::map<std::vector<stable_hash>, ParamLocs> HashSeqToLocs; |
353 | auto &RSF = *SFS[0]; |
354 | unsigned StableFunctionCount = SFS.size(); |
355 | |
356 | for (auto &[IndexPair, Hash] : *RSF.IndexOperandHashMap) { |
357 | // Const hash sequence across stable functions. |
358 | // We will allocate a parameter per unique hash squence. |
359 | // can't use SmallVector as key |
360 | std::vector<stable_hash> ConstHashSeq; |
361 | ConstHashSeq.push_back(x: Hash); |
362 | bool Identical = true; |
363 | for (unsigned J = 1; J < StableFunctionCount; ++J) { |
364 | auto &SF = SFS[J]; |
365 | auto SHash = SF->IndexOperandHashMap->at(Val: IndexPair); |
366 | if (Hash != SHash) |
367 | Identical = false; |
368 | ConstHashSeq.push_back(x: SHash); |
369 | } |
370 | |
371 | if (Identical) |
372 | continue; |
373 | |
374 | // For each unique Const hash sequence (parameter), add the locations. |
375 | HashSeqToLocs[ConstHashSeq].push_back(Elt: IndexPair); |
376 | } |
377 | |
378 | ParamLocsVecTy ParamLocsVec; |
379 | for (auto &[HashSeq, Locs] : HashSeqToLocs) |
380 | ParamLocsVec.push_back(Elt: std::move(Locs)); |
381 | |
382 | llvm::sort(C&: ParamLocsVec, Comp: [&](const ParamLocs &L, const ParamLocs &R) { |
383 | return L[0] < R[0]; |
384 | }); |
385 | |
386 | return ParamLocsVec; |
387 | } |
388 | |
389 | bool GlobalMergeFunc::merge(Module &M, const StableFunctionMap *FunctionMap) { |
390 | bool Changed = false; |
391 | |
392 | // Collect stable functions related to the current module. |
393 | DenseMap<stable_hash, SmallVector<std::pair<Function *, FunctionHashInfo>>> |
394 | HashToFuncs; |
395 | auto &Maps = FunctionMap->getFunctionMap(); |
396 | for (auto &F : M) { |
397 | if (!isEligibleFunction(F: &F)) |
398 | continue; |
399 | auto FI = llvm::StructuralHashWithDifferences(F, IgnoreOp: ignoreOp); |
400 | if (Maps.contains(Val: FI.FunctionHash)) |
401 | HashToFuncs[FI.FunctionHash].emplace_back(Args: &F, Args: std::move(FI)); |
402 | } |
403 | |
404 | for (auto &[Hash, Funcs] : HashToFuncs) { |
405 | std::optional<ParamLocsVecTy> ParamLocsVec; |
406 | SmallVector<FuncMergeInfo> FuncMergeInfos; |
407 | auto &SFS = Maps.at(Val: Hash); |
408 | assert(!SFS.empty()); |
409 | auto &RFS = SFS[0]; |
410 | |
411 | // Iterate functions with the same hash. |
412 | for (auto &[F, FI] : Funcs) { |
413 | // Check if the function is compatible with any stable function |
414 | // in terms of the number of instructions and ignored operands. |
415 | if (RFS->InstCount != FI.IndexInstruction->size()) |
416 | continue; |
417 | |
418 | auto hasValidSharedConst = [&](StableFunctionMap::StableFunctionEntry *SF, |
419 | FunctionHashInfo &FHI) { |
420 | for (auto &[Index, Hash] : *SF->IndexOperandHashMap) { |
421 | auto [InstIndex, OpndIndex] = Index; |
422 | assert(InstIndex < FHI.IndexInstruction->size()); |
423 | auto *Inst = FHI.IndexInstruction->lookup(Key: InstIndex); |
424 | if (!ignoreOp(I: Inst, OpIdx: OpndIndex)) |
425 | return false; |
426 | } |
427 | return true; |
428 | }; |
429 | if (!hasValidSharedConst(RFS.get(), FI)) |
430 | continue; |
431 | |
432 | for (auto &SF : SFS) { |
433 | assert(SF->InstCount == FI.IndexInstruction->size()); |
434 | assert(hasValidSharedConst(SF.get(), FI)); |
435 | // Check if there is any stable function that is compatiable with the |
436 | // current one. |
437 | if (!checkConstHashCompatible(OldInstOpndIndexToConstHash: *SF->IndexOperandHashMap, |
438 | CurrInstOpndIndexToConstHash: *FI.IndexOperandHashMap)) |
439 | continue; |
440 | if (!ParamLocsVec.has_value()) { |
441 | ParamLocsVec = computeParamInfo(SFS); |
442 | LLVM_DEBUG(dbgs() << "[GlobalMergeFunc] Merging hash: " << Hash |
443 | << " with Params " << ParamLocsVec->size() << "\n" ); |
444 | } |
445 | if (!checkConstLocationCompatible(SF: *SF, IndexInstruction: *FI.IndexInstruction, |
446 | ParamLocsVec: *ParamLocsVec)) |
447 | continue; |
448 | |
449 | // If a stable function matching the current one is found, |
450 | // create a candidate for merging and proceed to the next function. |
451 | FuncMergeInfos.emplace_back(Args: SF.get(), Args&: F, Args: FI.IndexInstruction.get()); |
452 | break; |
453 | } |
454 | } |
455 | unsigned FuncMergeInfoSize = FuncMergeInfos.size(); |
456 | if (FuncMergeInfoSize == 0) |
457 | continue; |
458 | |
459 | LLVM_DEBUG(dbgs() << "[GlobalMergeFunc] Merging function count " |
460 | << FuncMergeInfoSize << " for hash: " << Hash << "\n" ); |
461 | |
462 | for (auto &FMI : FuncMergeInfos) { |
463 | Changed = true; |
464 | |
465 | // We've already validated all locations of constant operands pointed by |
466 | // the parameters. Populate parameters pointing to the original constants. |
467 | SmallVector<Constant *> Params; |
468 | SmallVector<Type *> ParamTypes; |
469 | for (auto &ParamLocs : *ParamLocsVec) { |
470 | assert(!ParamLocs.empty()); |
471 | auto &[InstIndex, OpndIndex] = ParamLocs[0]; |
472 | auto *Inst = FMI.IndexInstruction->lookup(Key: InstIndex); |
473 | auto *Opnd = cast<Constant>(Val: Inst->getOperand(i: OpndIndex)); |
474 | Params.push_back(Elt: Opnd); |
475 | ParamTypes.push_back(Elt: Opnd->getType()); |
476 | } |
477 | |
478 | // Create a merged function derived from the current function. |
479 | Function *MergedFunc = |
480 | createMergedFunction(FI&: FMI, ConstParamTypes: ParamTypes, ParamLocsVec: *ParamLocsVec); |
481 | |
482 | LLVM_DEBUG({ |
483 | dbgs() << "[GlobalMergeFunc] Merged function (hash:" << FMI.SF->Hash |
484 | << ") " << MergedFunc->getName() << " generated from " |
485 | << FMI.F->getName() << ":\n" ; |
486 | MergedFunc->dump(); |
487 | }); |
488 | |
489 | // Transform the current function into a thunk that calls the merged |
490 | // function. |
491 | createThunk(FI&: FMI, Params, ToFunc: MergedFunc); |
492 | LLVM_DEBUG({ |
493 | dbgs() << "[GlobalMergeFunc] Thunk generated: \n" ; |
494 | FMI.F->dump(); |
495 | }); |
496 | ++NumMergedFunctions; |
497 | } |
498 | } |
499 | |
500 | return Changed; |
501 | } |
502 | |
503 | void GlobalMergeFunc::initializeMergerMode(const Module &M) { |
504 | // Initialize the local function map regardless of the merger mode. |
505 | LocalFunctionMap = std::make_unique<StableFunctionMap>(); |
506 | |
507 | // Disable codegen data for merging. The local merge is still enabled. |
508 | if (DisableCGDataForMerging) |
509 | return; |
510 | |
511 | // (Full)LTO module does not have functions added to the index. |
512 | // In this case, we run a local merger without using codegen data. |
513 | if (Index && !Index->hasExportedFunctions(M)) |
514 | return; |
515 | |
516 | if (cgdata::emitCGData()) |
517 | MergerMode = HashFunctionMode::BuildingHashFuncion; |
518 | else if (cgdata::hasStableFunctionMap()) |
519 | MergerMode = HashFunctionMode::UsingHashFunction; |
520 | } |
521 | |
522 | void GlobalMergeFunc::emitFunctionMap(Module &M) { |
523 | LLVM_DEBUG(dbgs() << "Emit function map. Size: " << LocalFunctionMap->size() |
524 | << "\n" ); |
525 | // No need to emit the function map if it is empty. |
526 | if (LocalFunctionMap->empty()) |
527 | return; |
528 | SmallVector<char> Buf; |
529 | raw_svector_ostream OS(Buf); |
530 | |
531 | std::vector<CGDataPatchItem> PatchItems; |
532 | StableFunctionMapRecord::serialize(OS, FunctionMap: LocalFunctionMap.get(), PatchItems); |
533 | CGDataOStream COS(OS); |
534 | COS.patch(P: PatchItems); |
535 | |
536 | std::unique_ptr<MemoryBuffer> Buffer = MemoryBuffer::getMemBuffer( |
537 | InputData: OS.str(), BufferName: "in-memory stable function map" , RequiresNullTerminator: false); |
538 | |
539 | Triple TT(M.getTargetTriple()); |
540 | embedBufferInModule(M, Buf: *Buffer, |
541 | SectionName: getCodeGenDataSectionName(CGSK: CG_merge, OF: TT.getObjectFormat()), |
542 | Alignment: Align(4)); |
543 | } |
544 | |
545 | bool GlobalMergeFunc::run(Module &M) { |
546 | initializeMergerMode(M); |
547 | |
548 | const StableFunctionMap *FuncMap; |
549 | if (MergerMode == HashFunctionMode::UsingHashFunction) { |
550 | // Use the prior CG data to optimistically create global merge candidates. |
551 | FuncMap = cgdata::getStableFunctionMap(); |
552 | } else { |
553 | analyze(M); |
554 | // Emit the local function map to the custom section, __llvm_merge before |
555 | // finalizing it. |
556 | if (MergerMode == HashFunctionMode::BuildingHashFuncion) |
557 | emitFunctionMap(M); |
558 | LocalFunctionMap->finalize(); |
559 | FuncMap = LocalFunctionMap.get(); |
560 | } |
561 | |
562 | return merge(M, FunctionMap: FuncMap); |
563 | } |
564 | |
565 | namespace { |
566 | |
567 | class GlobalMergeFuncPassWrapper : public ModulePass { |
568 | |
569 | public: |
570 | static char ID; |
571 | |
572 | GlobalMergeFuncPassWrapper(); |
573 | |
574 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
575 | AU.addUsedIfAvailable<ImmutableModuleSummaryIndexWrapperPass>(); |
576 | AU.setPreservesAll(); |
577 | ModulePass::getAnalysisUsage(AU); |
578 | } |
579 | |
580 | StringRef getPassName() const override { return "Global Merge Functions" ; } |
581 | |
582 | bool runOnModule(Module &M) override; |
583 | }; |
584 | |
585 | } // namespace |
586 | |
587 | char GlobalMergeFuncPassWrapper::ID = 0; |
588 | INITIALIZE_PASS_BEGIN(GlobalMergeFuncPassWrapper, "global-merge-func" , |
589 | "Global merge function pass" , false, false) |
590 | INITIALIZE_PASS_END(GlobalMergeFuncPassWrapper, "global-merge-func" , |
591 | "Global merge function pass" , false, false) |
592 | |
593 | namespace llvm { |
594 | ModulePass *createGlobalMergeFuncPass() { |
595 | return new GlobalMergeFuncPassWrapper(); |
596 | } |
597 | } // namespace llvm |
598 | |
599 | GlobalMergeFuncPassWrapper::GlobalMergeFuncPassWrapper() : ModulePass(ID) { |
600 | initializeGlobalMergeFuncPassWrapperPass( |
601 | Registry&: *llvm::PassRegistry::getPassRegistry()); |
602 | } |
603 | |
604 | bool GlobalMergeFuncPassWrapper::runOnModule(Module &M) { |
605 | const ModuleSummaryIndex *Index = nullptr; |
606 | if (auto *IndexWrapperPass = |
607 | getAnalysisIfAvailable<ImmutableModuleSummaryIndexWrapperPass>()) |
608 | Index = IndexWrapperPass->getIndex(); |
609 | |
610 | return GlobalMergeFunc(Index).run(M); |
611 | } |
612 | |
613 | PreservedAnalyses GlobalMergeFuncPass::run(Module &M, |
614 | AnalysisManager<Module> &AM) { |
615 | bool Changed = GlobalMergeFunc(ImportSummary).run(M); |
616 | return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); |
617 | } |
618 | |