1//==-- MemProfContextDisambiguation.cpp - Disambiguate contexts -------------=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements support for context disambiguation of allocation
10// calls for profile-guided memory optimization (MemPGO). Specifically, it uses
11// MemProf profiles which indicate context specific allocation behavior
12// (currently distinguishing cold vs hot memory allocations). Cloning is
13// performed to expose the cold allocation call contexts, and the allocation
14// calls are subsequently annotated with an attribute for later transformation.
15//
16// The transformations can be performed either directly on IR (regular LTO), or
17// on a ThinLTO index (and later applied to the IR during the ThinLTO backend).
18// Both types of LTO operate on a the same base graph representation, which
19// uses CRTP to support either IR or Index formats.
20//
21//===----------------------------------------------------------------------===//
22
23#include "llvm/Transforms/IPO/MemProfContextDisambiguation.h"
24#include "llvm/ADT/DenseMap.h"
25#include "llvm/ADT/DenseSet.h"
26#include "llvm/ADT/MapVector.h"
27#include "llvm/ADT/SetOperations.h"
28#include "llvm/ADT/SmallPtrSet.h"
29#include "llvm/ADT/SmallSet.h"
30#include "llvm/ADT/SmallVector.h"
31#include "llvm/ADT/Statistic.h"
32#include "llvm/ADT/StringExtras.h"
33#include "llvm/Analysis/MemoryProfileInfo.h"
34#include "llvm/Analysis/ModuleSummaryAnalysis.h"
35#include "llvm/Analysis/OptimizationRemarkEmitter.h"
36#include "llvm/Bitcode/BitcodeReader.h"
37#include "llvm/IR/Instructions.h"
38#include "llvm/IR/Module.h"
39#include "llvm/IR/ModuleSummaryIndex.h"
40#include "llvm/Pass.h"
41#include "llvm/Support/CommandLine.h"
42#include "llvm/Support/GraphWriter.h"
43#include "llvm/Support/SHA1.h"
44#include "llvm/Support/raw_ostream.h"
45#include "llvm/Transforms/IPO.h"
46#include "llvm/Transforms/Utils/CallPromotionUtils.h"
47#include "llvm/Transforms/Utils/Cloning.h"
48#include "llvm/Transforms/Utils/Instrumentation.h"
49#include <deque>
50#include <sstream>
51#include <vector>
52using namespace llvm;
53using namespace llvm::memprof;
54
55#define DEBUG_TYPE "memprof-context-disambiguation"
56
57STATISTIC(FunctionClonesAnalysis,
58 "Number of function clones created during whole program analysis");
59STATISTIC(FunctionClonesThinBackend,
60 "Number of function clones created during ThinLTO backend");
61STATISTIC(FunctionsClonedThinBackend,
62 "Number of functions that had clones created during ThinLTO backend");
63STATISTIC(
64 FunctionCloneDuplicatesThinBackend,
65 "Number of function clone duplicates detected during ThinLTO backend");
66STATISTIC(AllocTypeNotCold, "Number of not cold static allocations (possibly "
67 "cloned) during whole program analysis");
68STATISTIC(AllocTypeCold, "Number of cold static allocations (possibly cloned) "
69 "during whole program analysis");
70STATISTIC(AllocTypeAmbiguous,
71 "Number of ambiguous static allocations (possibly cloned) during "
72 "whole program analysis");
73STATISTIC(AllocTypeNotColdThinBackend,
74 "Number of not cold static allocations (possibly cloned) during "
75 "ThinLTO backend");
76STATISTIC(AllocTypeColdThinBackend, "Number of cold static allocations "
77 "(possibly cloned) during ThinLTO backend");
78STATISTIC(AllocTypeAmbiguousThinBackend,
79 "Number of ambiguous static allocations (possibly cloned) during "
80 "ThinLTO backend");
81STATISTIC(OrigAllocsThinBackend,
82 "Number of original (not cloned) allocations with memprof profiles "
83 "during ThinLTO backend");
84STATISTIC(
85 AllocVersionsThinBackend,
86 "Number of allocation versions (including clones) during ThinLTO backend");
87STATISTIC(MaxAllocVersionsThinBackend,
88 "Maximum number of allocation versions created for an original "
89 "allocation during ThinLTO backend");
90STATISTIC(UnclonableAllocsThinBackend,
91 "Number of unclonable ambigous allocations during ThinLTO backend");
92STATISTIC(RemovedEdgesWithMismatchedCallees,
93 "Number of edges removed due to mismatched callees (profiled vs IR)");
94STATISTIC(FoundProfiledCalleeCount,
95 "Number of profiled callees found via tail calls");
96STATISTIC(FoundProfiledCalleeDepth,
97 "Aggregate depth of profiled callees found via tail calls");
98STATISTIC(FoundProfiledCalleeMaxDepth,
99 "Maximum depth of profiled callees found via tail calls");
100STATISTIC(FoundProfiledCalleeNonUniquelyCount,
101 "Number of profiled callees found via multiple tail call chains");
102STATISTIC(FoundProfiledCalleeIndirectTailCallCount,
103 "Number of tail calls in chains to profiled callees that are "
104 "indirect (thin link)");
105STATISTIC(IndirectTailCallsRecordedForICPThinBackend,
106 "Number of indirect tail calls recorded for ICP for cloning during "
107 "ThinLTO backend");
108STATISTIC(DeferredBackedges, "Number of backedges with deferred cloning");
109STATISTIC(NewMergedNodes, "Number of new nodes created during merging");
110STATISTIC(NonNewMergedNodes, "Number of non new nodes used during merging");
111STATISTIC(MissingAllocForContextId,
112 "Number of missing alloc nodes for context ids");
113STATISTIC(SkippedCallsCloning,
114 "Number of calls skipped during cloning due to unexpected operand");
115STATISTIC(MismatchedCloneAssignments,
116 "Number of callsites assigned to call multiple non-matching clones");
117STATISTIC(TotalMergeInvokes, "Number of merge invocations for nodes");
118STATISTIC(TotalMergeIters, "Number of merge iterations for nodes");
119STATISTIC(MaxMergeIters, "Max merge iterations for nodes");
120STATISTIC(NumImportantContextIds, "Number of important context ids");
121STATISTIC(NumFixupEdgeIdsInserted, "Number of fixup edge ids inserted");
122STATISTIC(NumFixupEdgesAdded, "Number of fixup edges added");
123STATISTIC(NumFixedContexts, "Number of contexts with fixed edges");
124STATISTIC(NumImplicitCallersAssigned,
125 "Number of callsite clones without an edge to a callee's clones "
126 "assigned the callee function clone of their original node");
127STATISTIC(NumImplicitCallersSourced,
128 "Number of calls in function clones whose callee was taken from a "
129 "callsite clone an implicit caller requires");
130STATISTIC(NumTrimmedContextIds,
131 "Number of context ids with call stacks trimmed at a callsite");
132STATISTIC(NumImplicitCallersUnresolved,
133 "Number of callsite clones without an edge to a callee's clones "
134 "left in a function clone calling a conflicting callee clone");
135STATISTIC(AliaseesPrevailingInDiffModuleFromAlias,
136 "Number of aliasees prevailing in a different module than its alias");
137STATISTIC(NumClonedCalleeCallsites,
138 "Number of summary callsites assigned to call a cloned callee (thin "
139 "link)");
140STATISTIC(NumClonedCalleeIndirectCallsites,
141 "Number of summary callsites assigned to call a cloned callee with "
142 "an indirect call edge (thin link)");
143STATISTIC(NumClonedCalleeEdgesMarkedRequiresDeclImport,
144 "Number of summary indirect call edges to cloned callees marked as "
145 "requiring a declaration import (thin link)");
146STATISTIC(NumCallsAssignedTrimmedInlinedContexts,
147 "Number of calls assigned contexts trimmed within their inlined "
148 "stack id sequence");
149STATISTIC(NumTrimmedInlinedContextIds,
150 "Number of (duplicated) context ids for contexts trimmed within a "
151 "call's inlined stack id sequence");
152
153static cl::opt<std::string> DotFilePathPrefix(
154 "memprof-dot-file-path-prefix", cl::init(Val: ""), cl::Hidden,
155 cl::value_desc("filename"),
156 cl::desc("Specify the path prefix of the MemProf dot files."));
157
158static cl::opt<bool> ExportToDot("memprof-export-to-dot", cl::init(Val: false),
159 cl::Hidden,
160 cl::desc("Export graph to dot files."));
161
162// TODO: Remove this option once new handling is validated more widely.
163static cl::opt<bool> DoMergeIteration(
164 "memprof-merge-iteration", cl::init(Val: true), cl::Hidden,
165 cl::desc("Iteratively apply merging on a node to catch new callers"));
166
167// How much of the graph to export to dot.
168enum DotScope {
169 All, // The full CCG graph.
170 Alloc, // Only contexts for the specified allocation.
171 Context, // Only the specified context.
172};
173
174static cl::opt<DotScope> DotGraphScope(
175 "memprof-dot-scope", cl::desc("Scope of graph to export to dot"),
176 cl::Hidden, cl::init(Val: DotScope::All),
177 cl::values(
178 clEnumValN(DotScope::All, "all", "Export full callsite graph"),
179 clEnumValN(DotScope::Alloc, "alloc",
180 "Export only nodes with contexts feeding given "
181 "-memprof-dot-alloc-id"),
182 clEnumValN(DotScope::Context, "context",
183 "Export only nodes with given -memprof-dot-context-id")));
184
185static cl::opt<unsigned>
186 AllocIdForDot("memprof-dot-alloc-id", cl::init(Val: 0), cl::Hidden,
187 cl::desc("Id of alloc to export if -memprof-dot-scope=alloc "
188 "or to highlight if -memprof-dot-scope=all"));
189
190static cl::opt<unsigned> ContextIdForDot(
191 "memprof-dot-context-id", cl::init(Val: 0), cl::Hidden,
192 cl::desc("Id of context to export if -memprof-dot-scope=context or to "
193 "highlight otherwise"));
194
195static cl::opt<bool>
196 DumpCCG("memprof-dump-ccg", cl::init(Val: false), cl::Hidden,
197 cl::desc("Dump CallingContextGraph to stdout after each stage."));
198
199static cl::opt<bool>
200 VerifyCCG("memprof-verify-ccg", cl::init(Val: false), cl::Hidden,
201 cl::desc("Perform verification checks on CallingContextGraph."));
202
203static cl::opt<bool>
204 VerifyNodes("memprof-verify-nodes", cl::init(Val: false), cl::Hidden,
205 cl::desc("Perform frequent verification checks on nodes."));
206
207static cl::opt<std::string> MemProfImportSummary(
208 "memprof-import-summary",
209 cl::desc("Import summary to use for testing the ThinLTO backend via opt"),
210 cl::Hidden);
211
212static cl::opt<unsigned>
213 TailCallSearchDepth("memprof-tail-call-search-depth", cl::init(Val: 5),
214 cl::Hidden,
215 cl::desc("Max depth to recursively search for missing "
216 "frames through tail calls."));
217
218// Whether the ThinLTO search for frames missing due to tail calls should look
219// through indirect tail calls, via the summary call edges to their profiled
220// targets.
221static cl::opt<bool> IndirectTailCallSearch(
222 "memprof-indirect-tail-call-search", cl::init(Val: true), cl::Hidden,
223 cl::desc("Search for missing frames through indirect tail calls to their "
224 "profiled targets (ThinLTO)."));
225
226// Optionally enable cloning of callsites involved with recursive cycles
227static cl::opt<bool> AllowRecursiveCallsites(
228 "memprof-allow-recursive-callsites", cl::init(Val: true), cl::Hidden,
229 cl::desc("Allow cloning of callsites involved in recursive cycles"));
230
231static cl::opt<bool> CloneRecursiveContexts(
232 "memprof-clone-recursive-contexts", cl::init(Val: true), cl::Hidden,
233 cl::desc("Allow cloning of contexts through recursive cycles"));
234
235// Generally this is needed for correct assignment of allocation clones to
236// function clones, however, allow it to be disabled for debugging while the
237// functionality is new and being tested more widely.
238static cl::opt<bool>
239 MergeClones("memprof-merge-clones", cl::init(Val: true), cl::Hidden,
240 cl::desc("Merge clones before assigning functions"));
241
242// When disabled, try to detect and prevent cloning of recursive contexts.
243// This is only necessary until we support cloning through recursive cycles.
244// Leave on by default for now, as disabling requires a little bit of compile
245// time overhead and doesn't affect correctness, it will just inflate the cold
246// hinted bytes reporting a bit when -memprof-report-hinted-sizes is enabled.
247static cl::opt<bool> AllowRecursiveContexts(
248 "memprof-allow-recursive-contexts", cl::init(Val: true), cl::Hidden,
249 cl::desc("Allow cloning of contexts having recursive cycles"));
250
251// When updating the stack nodes for callsites, a call with inlined frames is
252// assigned the contexts that include all of its stack ids that have context
253// nodes. The profile may have trimmed some contexts at an earlier stack id in
254// that sequence (because they had a single allocation type from that point
255// on); when a later stack id has a node due to other contexts, those trimmed
256// contexts are not assigned to the call, only to calls whose last stack id
257// with a node is the earlier id (e.g. an out-of-line copy of the inlined
258// callee). When enabled, also assign them to the call with the inlined frames,
259// via duplicates of their context ids (the other calls keep the originals).
260// TODO: Remove this option once the new handling is validated more widely.
261static cl::opt<bool> AssignTrimmedInlinedContexts(
262 "memprof-assign-trimmed-inlined-contexts", cl::init(Val: true), cl::Hidden,
263 cl::desc("Assign contexts trimmed within a call's inlined stack id "
264 "sequence to that call"));
265
266// Set the minimum absolute count threshold for allowing inlining of indirect
267// calls promoted during cloning.
268static cl::opt<unsigned> MemProfICPNoInlineThreshold(
269 "memprof-icp-noinline-threshold", cl::init(Val: 0), cl::Hidden,
270 cl::desc("Minimum absolute count for promoted target to be inlinable"));
271
272namespace llvm {
273cl::opt<bool> EnableMemProfContextDisambiguation(
274 "enable-memprof-context-disambiguation", cl::Hidden,
275 cl::desc("Enable MemProf context disambiguation"));
276
277// Indicate we are linking with an allocator that supports hot/cold operator
278// new interfaces.
279cl::opt<bool> SupportsHotColdNew(
280 "supports-hot-cold-new", cl::init(Val: false), cl::Hidden,
281 cl::desc("Linking with hot/cold operator new interfaces"));
282
283static cl::opt<bool> MemProfRequireDefinitionForPromotion(
284 "memprof-require-definition-for-promotion", cl::init(Val: false), cl::Hidden,
285 cl::desc(
286 "Require target function definition when promoting indirect calls"));
287
288// Applying the thin link cloning decisions to indirect calls requires the
289// ThinLTO backend to perform ICP to the profiled target, which requires that
290// the target be available in the caller's module (or in any module the caller
291// is imported into). When the target is defined in a different module it must
292// be imported, but the profiled targets are often synthesized from the MemProf
293// profile with a small count, so their summary edges are not hot and large
294// targets are often not imported. Since the thin link cloning runs before the
295// function importing analysis, we can mark the indirect call edges to callees
296// of callsites that were assigned to call a cloned callee as requiring a
297// declaration import, in which case the function importer imports at least a
298// declaration of the callee into any module containing the caller that
299// doesn't define the callee. Direct calls don't require importing the callee,
300// since the ThinLTO backend can simply call a declaration of the callee clone,
301// so we only do this for edges marked as being from indirect calls in the
302// summary. Note that unlike regular ICP, which by default requires the
303// target's definition (see -icp-allow-decls), MemProf ICP can promote to a
304// declaration of the target (see -memprof-require-definition-for-promotion),
305// since the purpose is to call the assigned clone rather than to inline. This
306// is therefore not done under -memprof-require-definition-for-promotion.
307static cl::opt<bool> MemProfImportClonedIndirectTargets(
308 "memprof-import-cloned-indirect-targets", cl::init(Val: true), cl::Hidden,
309 cl::desc("Import at least a declaration of the indirect call targets of "
310 "callsites assigned to call a cloned callee (needed for MemProf "
311 "ICP in the ThinLTO backend)"));
312
313extern cl::opt<bool> MemProfReportHintedSizes;
314extern cl::opt<unsigned> MinClonedColdBytePercent;
315extern cl::opt<bool> MemProfUseAmbiguousAttributes;
316
317// Appends a per-context diagnostic suffix to each -memprof-report-hinted-sizes
318// line: the allocation function clone, how far the context matched, and the
319// post-cloning call paths from the outermost matched caller to the reached
320// allocation clone(s) and their final hints. Requires
321// -memprof-report-hinted-sizes, and adds some memory and time overhead.
322static cl::opt<bool> MemProfReportHintedSizesDetail(
323 "memprof-report-hinted-sizes-detail", cl::init(Val: false), cl::Hidden,
324 cl::desc("Append post-cloning call path and matching details to each "
325 "-memprof-report-hinted-sizes line"));
326
327// Post-cloning call paths in the detail suffix longer than this (in functions)
328// have runs of uncloned (or cloned) functions in their middle elided to
329// "...", keeping the ends and every cloned/uncloned transition.
330static cl::opt<unsigned> MemProfReportHintedSizesDetailMaxPathLen(
331 "memprof-report-hinted-sizes-detail-max-path-len", cl::init(Val: 7), cl::Hidden,
332 cl::desc("Maximum number of functions printed in full for a post-cloning "
333 "call path in the -memprof-report-hinted-sizes-detail suffix; "
334 "longer paths are elided in the middle"));
335
336cl::opt<unsigned> MemProfTopNImportant(
337 "memprof-top-n-important", cl::init(Val: 10), cl::Hidden,
338 cl::desc("Number of largest cold contexts to consider important"));
339
340cl::opt<bool> MemProfFixupImportant(
341 "memprof-fixup-important", cl::init(Val: true), cl::Hidden,
342 cl::desc("Enables edge fixup for important contexts"));
343
344} // namespace llvm
345
346namespace {
347
348/// CRTP base for graphs built from either IR or ThinLTO summary index.
349///
350/// The graph represents the call contexts in all memprof metadata on allocation
351/// calls, with nodes for the allocations themselves, as well as for the calls
352/// in each context. The graph is initially built from the allocation memprof
353/// metadata (or summary) MIBs. It is then updated to match calls with callsite
354/// metadata onto the nodes, updating it to reflect any inlining performed on
355/// those calls.
356///
357/// Each MIB (representing an allocation's call context with allocation
358/// behavior) is assigned a unique context id during the graph build. The edges
359/// and nodes in the graph are decorated with the context ids they carry. This
360/// is used to correctly update the graph when cloning is performed so that we
361/// can uniquify the context for a single (possibly cloned) allocation.
362template <typename DerivedCCG, typename FuncTy, typename CallTy>
363class CallsiteContextGraph {
364public:
365 CallsiteContextGraph() = default;
366 CallsiteContextGraph(const CallsiteContextGraph &) = default;
367 CallsiteContextGraph(CallsiteContextGraph &&) = default;
368
369 /// Main entry point to perform analysis and transformations on graph.
370 bool process(function_ref<void(StringRef, StringRef, const Twine &)>
371 EmitRemark = nullptr,
372 bool AllowExtraAnalysis = false);
373
374 /// Perform cloning on the graph necessary to uniquely identify the allocation
375 /// behavior of an allocation based on its context.
376 void identifyClones();
377
378 /// Assign callsite clones to functions, cloning functions as needed to
379 /// accommodate the combinations of their callsite clones reached by callers.
380 /// For regular LTO this clones functions and callsites in the IR, but for
381 /// ThinLTO the cloning decisions are noted in the summaries and later applied
382 /// in applyImport.
383 bool assignFunctions();
384
385 void dump() const;
386 void print(raw_ostream &OS) const;
387 void printTotalSizes(raw_ostream &OS,
388 function_ref<void(StringRef, StringRef, const Twine &)>
389 EmitRemark = nullptr) const;
390
391 friend raw_ostream &operator<<(raw_ostream &OS,
392 const CallsiteContextGraph &CCG) {
393 CCG.print(OS);
394 return OS;
395 }
396
397 friend struct GraphTraits<
398 const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>;
399 friend struct DOTGraphTraits<
400 const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>;
401
402 void exportToDot(std::string Label) const;
403
404 /// Represents a function clone via FuncTy pointer and clone number pair.
405 struct FuncInfo final
406 : public std::pair<FuncTy *, unsigned /*Clone number*/> {
407 using Base = std::pair<FuncTy *, unsigned>;
408 FuncInfo(const Base &B) : Base(B) {}
409 FuncInfo(FuncTy *F = nullptr, unsigned CloneNo = 0) : Base(F, CloneNo) {}
410 explicit operator bool() const { return this->first != nullptr; }
411 FuncTy *func() const { return this->first; }
412 unsigned cloneNo() const { return this->second; }
413 };
414
415 /// Represents a callsite clone via CallTy and clone number pair.
416 struct CallInfo final : public std::pair<CallTy, unsigned /*Clone number*/> {
417 using Base = std::pair<CallTy, unsigned>;
418 CallInfo(const Base &B) : Base(B) {}
419 CallInfo(CallTy Call = nullptr, unsigned CloneNo = 0)
420 : Base(Call, CloneNo) {}
421 explicit operator bool() const { return (bool)this->first; }
422 CallTy call() const { return this->first; }
423 unsigned cloneNo() const { return this->second; }
424 void setCloneNo(unsigned N) { this->second = N; }
425 void print(raw_ostream &OS) const {
426 if (!operator bool()) {
427 assert(!cloneNo());
428 OS << "null Call";
429 return;
430 }
431 call()->print(OS);
432 OS << "\t(clone " << cloneNo() << ")";
433 }
434 void dump() const {
435 print(OS&: dbgs());
436 dbgs() << "\n";
437 }
438 friend raw_ostream &operator<<(raw_ostream &OS, const CallInfo &Call) {
439 Call.print(OS);
440 return OS;
441 }
442 };
443
444 struct ContextEdge;
445
446 /// Node in the Callsite Context Graph
447 struct ContextNode {
448 // Assigned to nodes as they are created, useful for debugging.
449 unsigned NodeId = 0;
450
451 // Keep this for now since in the IR case where we have an Instruction* it
452 // is not as immediately discoverable. Used for printing richer information
453 // when dumping graph.
454 bool IsAllocation;
455
456 // Keeps track of when the Call was reset to null because there was
457 // recursion.
458 bool Recursive = false;
459
460 // This will be formed by ORing together the AllocationType enum values
461 // for contexts including this node.
462 uint8_t AllocTypes = 0;
463
464 // The corresponding allocation or interior call. This is the primary call
465 // for which we have created this node.
466 CallInfo Call;
467
468 // List of other calls that can be treated the same as the primary call
469 // through cloning. I.e. located in the same function and have the same
470 // (possibly trimmed) stack ids. They will be updated the same way as the
471 // primary call when assigning to function clones.
472 SmallVector<CallInfo, 0> MatchingCalls;
473
474 // For alloc nodes this is a unique id assigned when constructed, and for
475 // callsite stack nodes it is the original stack id when the node is
476 // constructed from the memprof MIB metadata on the alloc nodes. Note that
477 // this is only used when matching callsite metadata onto the stack nodes
478 // created when processing the allocation memprof MIBs, and for labeling
479 // nodes in the dot graph. Therefore we don't bother to assign a value for
480 // clones.
481 uint64_t OrigStackOrAllocId = 0;
482
483 // Edges to all callees in the profiled call stacks.
484 // TODO: Should this be a map (from Callee node) for more efficient lookup?
485 std::vector<std::shared_ptr<ContextEdge>> CalleeEdges;
486
487 // Edges to all callers in the profiled call stacks.
488 // TODO: Should this be a map (from Caller node) for more efficient lookup?
489 std::vector<std::shared_ptr<ContextEdge>> CallerEdges;
490
491 // Returns true if we need to look at the callee edges for determining the
492 // node context ids and allocation type.
493 bool useCallerEdgesForContextInfo() const {
494 // Typically if the callee edges are empty either the caller edges are
495 // also empty, or this is an allocation (leaf node). However, if we are
496 // allowing recursive callsites and contexts this will be violated for
497 // incompletely cloned recursive cycles.
498 assert(!CalleeEdges.empty() || CallerEdges.empty() || IsAllocation ||
499 (AllowRecursiveCallsites && AllowRecursiveContexts));
500 // When cloning for a recursive context, during cloning we might be in the
501 // midst of cloning for a recurrence and have moved context ids off of a
502 // caller edge onto the clone but not yet off of the incoming caller
503 // (back) edge. If we don't look at those we miss the fact that this node
504 // still has context ids of interest.
505 return IsAllocation || CloneRecursiveContexts;
506 }
507
508 // Compute the context ids for this node from the union of its edge context
509 // ids.
510 DenseSet<uint32_t> getContextIds() const {
511 unsigned Count = 0;
512 // Compute the number of ids for reserve below. In general we only need to
513 // look at one set of edges, typically the callee edges, since other than
514 // allocations and in some cases during recursion cloning, all the context
515 // ids on the callers should also flow out via callee edges.
516 for (auto &Edge : CalleeEdges.empty() ? CallerEdges : CalleeEdges)
517 Count += Edge->getContextIds().size();
518 DenseSet<uint32_t> ContextIds;
519 ContextIds.reserve(Size: Count);
520 auto Edges = llvm::concat<const std::shared_ptr<ContextEdge>>(
521 CalleeEdges, useCallerEdgesForContextInfo()
522 ? CallerEdges
523 : std::vector<std::shared_ptr<ContextEdge>>());
524 for (const auto &Edge : Edges)
525 ContextIds.insert_range(Edge->getContextIds());
526 return ContextIds;
527 }
528
529 // Compute the allocation type for this node from the OR of its edge
530 // allocation types.
531 uint8_t computeAllocType() const {
532 uint8_t BothTypes =
533 (uint8_t)AllocationType::Cold | (uint8_t)AllocationType::NotCold;
534 uint8_t AllocType = (uint8_t)AllocationType::None;
535 auto Edges = llvm::concat<const std::shared_ptr<ContextEdge>>(
536 CalleeEdges, useCallerEdgesForContextInfo()
537 ? CallerEdges
538 : std::vector<std::shared_ptr<ContextEdge>>());
539 for (const auto &Edge : Edges) {
540 AllocType |= Edge->AllocTypes;
541 // Bail early if alloc type reached both, no further refinement.
542 if (AllocType == BothTypes)
543 return AllocType;
544 }
545 return AllocType;
546 }
547
548 // The context ids set for this node is empty if its edge context ids are
549 // also all empty.
550 bool emptyContextIds() const {
551 auto Edges = llvm::concat<const std::shared_ptr<ContextEdge>>(
552 CalleeEdges, useCallerEdgesForContextInfo()
553 ? CallerEdges
554 : std::vector<std::shared_ptr<ContextEdge>>());
555 for (const auto &Edge : Edges) {
556 if (!Edge->getContextIds().empty())
557 return false;
558 }
559 return true;
560 }
561
562 // Whether the given context id is among this node's context ids.
563 bool hasContextId(uint32_t Id) const {
564 auto Edges = llvm::concat<const std::shared_ptr<ContextEdge>>(
565 CalleeEdges, useCallerEdgesForContextInfo()
566 ? CallerEdges
567 : std::vector<std::shared_ptr<ContextEdge>>());
568 for (const auto &Edge : Edges) {
569 if (Edge->getContextIds().contains(Id))
570 return true;
571 }
572 return false;
573 }
574
575 // Removes from Ids any that are not on this node (i.e. on none of the
576 // edges its context ids are computed from). Walks the edges once for all
577 // ids, stopping early once every id has been found.
578 void removeContextIdsNotPresent(DenseSet<uint32_t> &Ids) const {
579 DenseSet<uint32_t> NotFound = Ids;
580 auto Edges = llvm::concat<const std::shared_ptr<ContextEdge>>(
581 CalleeEdges, useCallerEdgesForContextInfo()
582 ? CallerEdges
583 : std::vector<std::shared_ptr<ContextEdge>>());
584 for (const auto &Edge : Edges) {
585 const auto &EdgeIds = Edge->getContextIds();
586 // Iterate over the smaller of the two sets.
587 if (EdgeIds.size() < NotFound.size()) {
588 for (auto Id : EdgeIds)
589 NotFound.erase(Id);
590 } else {
591 NotFound.remove_if(
592 [&EdgeIds](uint32_t Id) { return EdgeIds.contains(Id); });
593 }
594 if (NotFound.empty())
595 return;
596 }
597 set_subtract(S1&: Ids, S2: NotFound);
598 }
599
600 // List of clones of this ContextNode, initially empty.
601 std::vector<ContextNode *> Clones;
602
603 // If a clone, points to the original uncloned node.
604 ContextNode *CloneOf = nullptr;
605
606 ContextNode(bool IsAllocation) : IsAllocation(IsAllocation), Call() {}
607
608 ContextNode(bool IsAllocation, CallInfo C)
609 : IsAllocation(IsAllocation), Call(C) {}
610
611 void addClone(ContextNode *Clone) {
612 if (CloneOf) {
613 CloneOf->Clones.push_back(Clone);
614 Clone->CloneOf = CloneOf;
615 } else {
616 Clones.push_back(Clone);
617 assert(!Clone->CloneOf);
618 Clone->CloneOf = this;
619 }
620 }
621
622 ContextNode *getOrigNode() {
623 if (!CloneOf)
624 return this;
625 return CloneOf;
626 }
627
628 const ContextNode *getOrigNode() const {
629 if (!CloneOf)
630 return this;
631 return CloneOf;
632 }
633
634 void addOrUpdateCallerEdge(ContextNode *Caller, AllocationType AllocType,
635 unsigned int ContextId);
636
637 ContextEdge *findEdgeFromCallee(const ContextNode *Callee);
638 ContextEdge *findEdgeFromCaller(const ContextNode *Caller);
639 void eraseCalleeEdge(const ContextEdge *Edge);
640 void eraseCallerEdge(const ContextEdge *Edge);
641
642 void setCall(CallInfo C) { Call = std::move(C); }
643
644 bool hasCall() const { return (bool)Call.call(); }
645
646 void printCall(raw_ostream &OS) const { Call.print(OS); }
647
648 // True if this node was effectively removed from the graph, in which case
649 // it should have an allocation type of None and empty context ids.
650 bool isRemoved() const {
651 // Typically if the callee edges are empty either the caller edges are
652 // also empty, or this is an allocation (leaf node). However, if we are
653 // allowing recursive callsites and contexts this will be violated for
654 // incompletely cloned recursive cycles.
655 assert((AllowRecursiveCallsites && AllowRecursiveContexts) ||
656 (AllocTypes == (uint8_t)AllocationType::None) ==
657 emptyContextIds());
658 return AllocTypes == (uint8_t)AllocationType::None;
659 }
660
661 void dump() const;
662 void print(raw_ostream &OS) const;
663
664 friend raw_ostream &operator<<(raw_ostream &OS, const ContextNode &Node) {
665 Node.print(OS);
666 return OS;
667 }
668 };
669
670 /// Edge in the Callsite Context Graph from a ContextNode N to a caller or
671 /// callee.
672 struct ContextEdge {
673 ContextNode *Callee;
674 ContextNode *Caller;
675
676 // This will be formed by ORing together the AllocationType enum values
677 // for contexts including this edge.
678 uint8_t AllocTypes = 0;
679
680 // Set just before initiating cloning when cloning of recursive contexts is
681 // enabled. Used to defer cloning of backedges until we have done cloning of
682 // the callee node for non-backedge caller edges. This exposes cloning
683 // opportunities through the backedge of the cycle.
684 // TODO: Note that this is not updated during cloning, and it is unclear
685 // whether that would be needed.
686 bool IsBackedge = false;
687
688 // The set of IDs for contexts including this edge.
689 DenseSet<uint32_t> ContextIds;
690
691 ContextEdge(ContextNode *Callee, ContextNode *Caller, uint8_t AllocType,
692 DenseSet<uint32_t> ContextIds)
693 : Callee(Callee), Caller(Caller), AllocTypes(AllocType),
694 ContextIds(std::move(ContextIds)) {}
695
696 DenseSet<uint32_t> &getContextIds() { return ContextIds; }
697
698 // Helper to clear the fields of this edge when we are removing it from the
699 // graph.
700 inline void clear() {
701 ContextIds.clear();
702 AllocTypes = (uint8_t)AllocationType::None;
703 Caller = nullptr;
704 Callee = nullptr;
705 }
706
707 // Check if edge was removed from the graph. This is useful while iterating
708 // over a copy of edge lists when performing operations that mutate the
709 // graph in ways that might remove one of the edges.
710 inline bool isRemoved() const {
711 if (Callee || Caller)
712 return false;
713 // Any edges that have been removed from the graph but are still in a
714 // shared_ptr somewhere should have all fields null'ed out by clear()
715 // above.
716 assert(AllocTypes == (uint8_t)AllocationType::None);
717 assert(ContextIds.empty());
718 return true;
719 }
720
721 void dump() const;
722 void print(raw_ostream &OS) const;
723
724 friend raw_ostream &operator<<(raw_ostream &OS, const ContextEdge &Edge) {
725 Edge.print(OS);
726 return OS;
727 }
728 };
729
730 /// Helpers to remove edges that have allocation type None (due to not
731 /// carrying any context ids) after transformations.
732 void removeNoneTypeCalleeEdges(ContextNode *Node);
733 void removeNoneTypeCallerEdges(ContextNode *Node);
734 void
735 recursivelyRemoveNoneTypeCalleeEdges(ContextNode *Node,
736 DenseSet<const ContextNode *> &Visited);
737
738protected:
739 /// Get the list of context nodes for the stack ids in the given callsite
740 /// context, stopping at the first stack id that does not have a node.
741 template <class NodeT, class IteratorT>
742 std::vector<ContextNode *>
743 getStackNodesForCallsite(CallStack<NodeT, IteratorT> &CallsiteContext);
744
745 /// Adds nodes for the given allocation and any stack ids on its memprof MIB
746 /// metadata (or summary).
747 ContextNode *addAllocNode(CallInfo Call, const FuncTy *F);
748
749 /// Adds nodes for the given MIB stack ids.
750 template <class NodeT, class IteratorT>
751 void addStackNodesForMIB(
752 ContextNode *AllocNode, CallStack<NodeT, IteratorT> &StackContext,
753 CallStack<NodeT, IteratorT> &CallsiteContext, AllocationType AllocType,
754 ArrayRef<ContextTotalSize> ContextSizeInfo,
755 std::map<uint64_t, uint32_t> &TotalSizeToContextIdTopNCold);
756
757 /// Matches all callsite metadata (or summary) to the nodes created for
758 /// allocation memprof MIB metadata, synthesizing new nodes to reflect any
759 /// inlining performed on those callsite instructions.
760 void updateStackNodes();
761
762 /// Optionally fixup edges for the N largest cold contexts to better enable
763 /// cloning. This is particularly helpful if the context includes recursion
764 /// as well as inlining, resulting in a single stack node for multiple stack
765 /// ids in the context. With recursion it is particularly difficult to get the
766 /// edge updates correct as in the general case we have lost the original
767 /// stack id ordering for the context. Do more expensive fixup for the largest
768 /// contexts, controlled by MemProfTopNImportant and MemProfFixupImportant.
769 void fixupImportantContexts();
770
771 /// Update graph to conservatively handle any callsite stack nodes that target
772 /// multiple different callee target functions.
773 void handleCallsitesWithMultipleTargets();
774
775 /// Mark backedges via the standard DFS based backedge algorithm.
776 void markBackedges();
777
778 /// Merge clones generated during cloning for different allocations but that
779 /// are called by the same caller node, to ensure proper function assignment.
780 void mergeClones();
781
782 // Try to partition calls on the given node (already placed into the AllCalls
783 // array) by callee function, creating new copies of Node as needed to hold
784 // calls with different callees, and moving the callee edges appropriately.
785 // Returns true if partitioning was successful.
786 bool partitionCallsByCallee(
787 ContextNode *Node, ArrayRef<CallInfo> AllCalls,
788 std::vector<std::pair<CallInfo, ContextNode *>> &NewCallToNode);
789
790 /// Save lists of calls with MemProf metadata in each function, for faster
791 /// iteration.
792 MapVector<FuncTy *, std::vector<CallInfo>> FuncToCallsWithMetadata;
793
794 /// Map from callsite node to the enclosing caller function.
795 std::map<const ContextNode *, const FuncTy *> NodeToCallingFunc;
796
797 const FuncTy *getCallingFunc(const ContextNode *N) const {
798 auto It = NodeToCallingFunc.find(N);
799 return It != NodeToCallingFunc.end() ? It->second : nullptr;
800 }
801
802 /// Functions that are referenced (called directly, or address-taken) in the
803 /// module or summary index. Only populated, at the start of reporting, when
804 /// MemProfReportHintedSizesDetail is enabled; used to distinguish allocation
805 /// functions with no references at all from those whose callers simply did
806 /// not match the profile.
807 DenseSet<const FuncTy *> FunctionsWithCallers;
808
809 // When exporting to dot, and an allocation id is specified, contains the
810 // context ids on that allocation.
811 DenseSet<uint32_t> DotAllocContextIds;
812
813private:
814 using EdgeIter = typename std::vector<std::shared_ptr<ContextEdge>>::iterator;
815
816 // Structure to keep track of information for each call as we are matching
817 // non-allocation callsites onto context nodes created from the allocation
818 // call metadata / summary contexts.
819 struct CallContextInfo {
820 // The callsite we're trying to match.
821 CallTy Call;
822 // The context nodes for the callsite's stack ids that have one, saved to
823 // avoid repeatedly looking them up. The stack ids are available via the
824 // nodes' OrigStackOrAllocId. Since each stack id has its own node, and the
825 // stack id to node map is not modified while matching calls, two calls
826 // have the same stack ids iff they have the same nodes.
827 std::vector<ContextNode *> StackNodes;
828 // The function containing this callsite.
829 const FuncTy *Func;
830 // Initially empty, if needed this will be updated to contain the context
831 // ids for use in a new context node created for this callsite.
832 DenseSet<uint32_t> ContextIds;
833 // Duplicates, created for this callsite, of the ids of contexts trimmed at
834 // a stack id before the last one in StackNodes (see
835 // -memprof-assign-trimmed-inlined-contexts). Any new node created for this
836 // callsite gets these in addition to ContextIds.
837 DenseSet<uint32_t> TrimmedInlinedDupIds;
838 };
839
840 /// Helper to remove edge from graph, updating edge iterator if it is provided
841 /// (in which case CalleeIter indicates which edge list is being iterated).
842 /// This will also perform the necessary clearing of the ContextEdge members
843 /// to enable later checking if the edge has been removed (since we may have
844 /// other copies of the shared_ptr in existence, and in fact rely on this to
845 /// enable removal while iterating over a copy of a node's edge list).
846 void removeEdgeFromGraph(ContextEdge *Edge, EdgeIter *EI = nullptr,
847 bool CalleeIter = true);
848
849 /// Assigns the given Node to calls at or inlined into the location with
850 /// the Node's stack id, after post order traversing and processing its
851 /// caller nodes. Uses the call information recorded in the given
852 /// StackIdToMatchingCalls map, and creates new nodes for inlined sequences
853 /// as needed. Called by updateStackNodes which sets up the given
854 /// StackIdToMatchingCalls map.
855 void assignStackNodesPostOrder(
856 ContextNode *Node, DenseSet<const ContextNode *> &Visited,
857 DenseMap<uint64_t, std::vector<CallContextInfo>> &StackIdToMatchingCalls,
858 DenseMap<CallInfo, CallInfo> &CallToMatchingCall,
859 const DenseSet<uint32_t> &ImportantContextIds);
860
861 /// Duplicates the given set of context ids, updating the provided
862 /// map from each original id with the newly generated context ids,
863 /// and returning the new duplicated id set. If provided, OnDuplicate is
864 /// invoked with each original id and its new duplicate.
865 DenseSet<uint32_t> duplicateContextIds(
866 const DenseSet<uint32_t> &StackSequenceContextIds,
867 DenseMap<uint32_t, DenseSet<uint32_t>> &OldToNewContextIds,
868 function_ref<void(uint32_t OldId, uint32_t NewId)> OnDuplicate = nullptr);
869
870 /// Propagates all duplicated context ids across the graph.
871 void propagateDuplicateContextIds(
872 const DenseMap<uint32_t, DenseSet<uint32_t>> &OldToNewContextIds);
873
874 /// Cache of the ids of contexts trimmed at each node (i.e. that are on the
875 /// node but not on any of its caller edges). The sets are stored in a vector
876 /// and looked up by index, so that references to them are not invalidated
877 /// by adding more entries.
878 struct TrimmedAtNodeCache {
879 DenseMap<const ContextNode *, unsigned> NodeToIndex;
880 std::vector<DenseSet<uint32_t>> TrimmedAtNodeIds;
881 };
882
883 /// Returns the ids of contexts that were trimmed at a stack node before the
884 /// last one in the given call's inlined stack node sequence (see the
885 /// definition for details).
886 DenseSet<uint32_t>
887 computeTrimmedInlinedContextIds(ArrayRef<ContextNode *> StackNodes,
888 TrimmedAtNodeCache &Cache);
889
890 /// Connect the NewNode to OrigNode's callees if TowardsCallee is true,
891 /// else to its callers. Also updates OrigNode's edges to remove any context
892 /// ids moved to the newly created edge.
893 void connectNewNode(ContextNode *NewNode, ContextNode *OrigNode,
894 bool TowardsCallee, const DenseSet<uint32_t> &ContextIds);
895
896 /// Get the stack id corresponding to the given Id or Index (for IR this will
897 /// return itself, for a summary index this will return the id recorded in the
898 /// index for that stack id index value).
899 uint64_t getStackId(uint64_t IdOrIndex) const {
900 return static_cast<const DerivedCCG *>(this)->getStackId(IdOrIndex);
901 }
902
903 /// Returns true if the given call targets the callee of the given edge, or if
904 /// we were able to identify the call chain through intermediate tail calls.
905 /// In the latter case new context nodes are added to the graph for the
906 /// identified tail calls, and their synthesized nodes are added to
907 /// TailCallToContextNodeMap. The EdgeIter is updated in the latter case for
908 /// the updated edges and to prepare it for an increment in the caller.
909 bool
910 calleesMatch(CallTy Call, EdgeIter &EI,
911 MapVector<CallInfo, ContextNode *> &TailCallToContextNodeMap);
912
913 // Return the callee function of the given call, or nullptr if it can't be
914 // determined
915 const FuncTy *getCalleeFunc(CallTy Call) {
916 return static_cast<DerivedCCG *>(this)->getCalleeFunc(Call);
917 }
918
919 /// Returns true if the given call targets the given function, or if we were
920 /// able to identify the call chain through intermediate tail calls (in which
921 /// case FoundCalleeChain will be populated).
922 bool calleeMatchesFunc(
923 CallTy Call, const FuncTy *Func, const FuncTy *CallerFunc,
924 std::vector<std::pair<CallTy, FuncTy *>> &FoundCalleeChain) {
925 return static_cast<DerivedCCG *>(this)->calleeMatchesFunc(
926 Call, Func, CallerFunc, FoundCalleeChain);
927 }
928
929 /// Returns true if both call instructions have the same callee.
930 bool sameCallee(CallTy Call1, CallTy Call2) {
931 return static_cast<DerivedCCG *>(this)->sameCallee(Call1, Call2);
932 }
933
934 /// Get the list of context nodes for the stack ids in the given callsite's
935 /// context, stopping at the first stack id that does not have a node.
936 std::vector<ContextNode *> getStackNodesForCall(CallTy Call) {
937 return static_cast<DerivedCCG *>(this)->getStackNodesForCall(Call);
938 }
939
940 /// Get the last stack id in the context for callsite.
941 uint64_t getLastStackId(CallTy Call) {
942 return static_cast<DerivedCCG *>(this)->getLastStackId(Call);
943 }
944
945 /// Update the allocation call to record type of allocated memory.
946 void updateAllocationCall(CallInfo &Call, AllocationType AllocType) {
947 if (AllocType == AllocationType::Cold)
948 AllocTypeCold++;
949 else if (isAmbiguousAllocType(AllocTypes: (uint8_t)AllocType))
950 AllocTypeAmbiguous++;
951 else
952 AllocTypeNotCold++;
953 static_cast<DerivedCCG *>(this)->updateAllocationCall(Call, AllocType);
954 }
955
956 /// Get the AllocationType assigned to the given allocation instruction clone.
957 AllocationType getAllocationCallType(const CallInfo &Call) const {
958 return static_cast<const DerivedCCG *>(this)->getAllocationCallType(Call);
959 }
960
961 /// Update non-allocation call to invoke (possibly cloned) function
962 /// CalleeFunc.
963 void updateCall(CallInfo &CallerCall, FuncInfo CalleeFunc) {
964 static_cast<DerivedCCG *>(this)->updateCall(CallerCall, CalleeFunc);
965 }
966
967 /// Clone the given function for the given callsite, recording mapping of all
968 /// of the functions tracked calls to their new versions in the CallMap.
969 /// Assigns new clones to clone number CloneNo.
970 FuncInfo cloneFunctionForCallsite(
971 FuncInfo &Func, CallInfo &Call, DenseMap<CallInfo, CallInfo> &CallMap,
972 std::vector<CallInfo> &CallsWithMetadataInFunc, unsigned CloneNo) {
973 return static_cast<DerivedCCG *>(this)->cloneFunctionForCallsite(
974 Func, Call, CallMap, CallsWithMetadataInFunc, CloneNo);
975 }
976
977 /// Gets a label to use in the dot graph for the given call clone in the given
978 /// function.
979 std::string getLabel(const FuncTy *Func, const CallTy Call,
980 unsigned CloneNo) const {
981 return static_cast<const DerivedCCG *>(this)->getLabel(Func, Call, CloneNo);
982 }
983
984 /// Gets the function name (with optional clone suffix) for reporting.
985 std::string getFuncName(const FuncTy *Func, unsigned CloneNo = 0) const {
986 return static_cast<const DerivedCCG *>(this)->getFuncName(Func, CloneNo);
987 }
988
989 /// Returns true if the given function was marked dead by ThinLTO dead symbol
990 /// analysis.
991 bool isFuncDead(const FuncTy *Func) const {
992 return static_cast<const DerivedCCG *>(this)->isFuncDead(Func);
993 }
994
995 /// Populates FunctionsWithCallers (MemProfReportHintedSizesDetail only).
996 void populateFunctionsWithCallers() {
997 static_cast<DerivedCCG *>(this)->populateFunctionsWithCallers();
998 }
999
1000 /// Records, before identifyClones(), the chain of (uncloned) nodes of every
1001 /// context and how far it matched (MemProfReportHintedSizesDetail only).
1002 void capturePreCloningContextInfo();
1003
1004 /// Formats the detail suffix for the report line of context Id, whose
1005 /// (possibly cloned) allocation node is Node (MemProfReportHintedSizesDetail
1006 /// only). See the option description for the format.
1007 std::string formatContextReportSuffix(const ContextNode *Node,
1008 uint32_t Id) const;
1009
1010 // Create and return a new ContextNode.
1011 ContextNode *createNewNode(bool IsAllocation, const FuncTy *F = nullptr,
1012 CallInfo C = CallInfo()) {
1013 NodeOwner.push_back(std::make_unique<ContextNode>(IsAllocation, C));
1014 auto *NewNode = NodeOwner.back().get();
1015 if (F)
1016 NodeToCallingFunc[NewNode] = F;
1017 NewNode->NodeId = NodeOwner.size();
1018 return NewNode;
1019 }
1020
1021 /// Helpers to find the node corresponding to the given call or stackid.
1022 ContextNode *getNodeForInst(const CallInfo &C);
1023 ContextNode *getNodeForAlloc(const CallInfo &C);
1024 ContextNode *getNodeForStackId(uint64_t StackId);
1025
1026 /// Computes the alloc type corresponding to the given context ids, by
1027 /// unioning their recorded alloc types.
1028 uint8_t computeAllocType(DenseSet<uint32_t> &ContextIds) const;
1029
1030 /// Returns the allocation type of the intersection of the contexts of two
1031 /// nodes (based on their provided context id sets), optimized for the case
1032 /// when Node1Ids is smaller than Node2Ids.
1033 uint8_t intersectAllocTypesImpl(const DenseSet<uint32_t> &Node1Ids,
1034 const DenseSet<uint32_t> &Node2Ids) const;
1035
1036 /// Returns the allocation type of the intersection of the contexts of two
1037 /// nodes (based on their provided context id sets).
1038 uint8_t intersectAllocTypes(const DenseSet<uint32_t> &Node1Ids,
1039 const DenseSet<uint32_t> &Node2Ids) const;
1040
1041 /// Create a clone of Edge's callee and move Edge to that new callee node,
1042 /// performing the necessary context id and allocation type updates.
1043 /// If ContextIdsToMove is non-empty, only that subset of Edge's ids are
1044 /// moved to an edge to the new callee.
1045 ContextNode *
1046 moveEdgeToNewCalleeClone(const std::shared_ptr<ContextEdge> &Edge,
1047 DenseSet<uint32_t> ContextIdsToMove = {});
1048
1049 /// Change the callee of Edge to existing callee clone NewCallee, performing
1050 /// the necessary context id and allocation type updates.
1051 /// If ContextIdsToMove is non-empty, only that subset of Edge's ids are
1052 /// moved to an edge to the new callee.
1053 void moveEdgeToExistingCalleeClone(const std::shared_ptr<ContextEdge> &Edge,
1054 ContextNode *NewCallee,
1055 bool NewClone = false,
1056 DenseSet<uint32_t> ContextIdsToMove = {});
1057
1058 /// Change the caller of the edge at the given callee edge iterator to be
1059 /// NewCaller, performing the necessary context id and allocation type
1060 /// updates. This is similar to the above moveEdgeToExistingCalleeClone, but
1061 /// a simplified version of it as we always move the given edge and all of its
1062 /// context ids.
1063 void moveCalleeEdgeToNewCaller(const std::shared_ptr<ContextEdge> &Edge,
1064 ContextNode *NewCaller);
1065
1066 /// Recursive helper for marking backedges via DFS.
1067 void markBackedges(ContextNode *Node, DenseSet<const ContextNode *> &Visited,
1068 DenseSet<const ContextNode *> &CurrentStack);
1069
1070 /// Recursive helper for merging clones.
1071 void
1072 mergeClones(ContextNode *Node, DenseSet<const ContextNode *> &Visited,
1073 DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode);
1074 /// Main worker for merging callee clones for a given node.
1075 void mergeNodeCalleeClones(
1076 ContextNode *Node, DenseSet<const ContextNode *> &Visited,
1077 DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode);
1078 /// Helper to find other callers of the given set of callee edges that can
1079 /// share the same callee merge node.
1080 void findOtherCallersToShareMerge(
1081 ContextNode *Node, std::vector<std::shared_ptr<ContextEdge>> &CalleeEdges,
1082 DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode,
1083 DenseSet<ContextNode *> &OtherCallersToShareMerge);
1084
1085 /// Recursively perform cloning on the graph for the given Node and its
1086 /// callers, in order to uniquely identify the allocation behavior of an
1087 /// allocation given its context. The context ids of the allocation being
1088 /// processed are given in AllocContextIds.
1089 void identifyClones(ContextNode *Node, DenseSet<const ContextNode *> &Visited,
1090 const DenseSet<uint32_t> &AllocContextIds);
1091
1092 /// Records which clone of a callsite node an implicit caller requires (see
1093 /// assignImplicitCallers). The call to that callsite in a given function
1094 /// clone (the map key below) must call the same callee version as Source.
1095 struct ImplicitSource {
1096 /// The required clone of the callsite node.
1097 const ContextNode *Source;
1098 /// One of the trimmed context ids that made Source required. If Source is
1099 /// later emptied by further cloning, this id is used to find the clone
1100 /// that now carries the trimmed contexts (see resolveImplicitSource).
1101 uint32_t ContextId;
1102 };
1103 /// Map from (original callsite node, function clone number) to the clone of
1104 /// that callsite node that the call in that function clone must follow.
1105 using ImplicitSourceMap =
1106 DenseMap<std::pair<const ContextNode *, unsigned>, ImplicitSource>;
1107
1108 /// Assign the implicit callers of the clones of Node (callsite clones without
1109 /// an edge to any clone of Node which contexts trimmed at Node's frame would
1110 /// have reached) the function clone called by the caller carrying those
1111 /// contexts, and record the clone of Node their calls must follow. See
1112 /// assignFunctions for the maps.
1113 void assignImplicitCallers(
1114 ContextNode *Node, const DenseSet<uint32_t> &TrimmedContextIds,
1115 DenseMap<ContextNode *, FuncInfo> &CallsiteToCalleeFuncCloneMap,
1116 const std::map<FuncInfo, ContextNode *> &FuncCloneToCurNodeCloneMap,
1117 ImplicitSourceMap &ImplicitSources);
1118
1119 /// Returns the clone of Node (the original callsite node) that the implicit
1120 /// source Src refers to: Src.Source unless it has since been emptied, in
1121 /// which case the clone that now carries Src.ContextId (null if none does,
1122 /// which is not expected).
1123 static const ContextNode *resolveImplicitSource(const ContextNode *Node,
1124 const ImplicitSource &Src);
1125
1126 /// Map from each context ID to the AllocationType assigned to that context.
1127 DenseMap<uint32_t, AllocationType> ContextIdToAllocationType;
1128
1129 /// Map from each contextID to the profiled full contexts and their total
1130 /// sizes (there may be more than one due to context trimming),
1131 /// optionally populated when requested (via MemProfReportHintedSizes or
1132 /// MinClonedColdBytePercent).
1133 DenseMap<uint32_t, std::vector<ContextTotalSize>> ContextIdToContextSizeInfos;
1134
1135 /// Identifies the context node created for a stack id when adding the MIB
1136 /// contexts to the graph. This is used to locate the context nodes when
1137 /// trying to assign the corresponding callsites with those stack ids to these
1138 /// nodes.
1139 DenseMap<uint64_t, ContextNode *> StackEntryIdToContextNodeMap;
1140
1141 /// Saves information for the contexts identified as important (the largest
1142 /// cold contexts up to MemProfTopNImportant).
1143 struct ImportantContextInfo {
1144 // The original list of leaf first stack ids corresponding to this context.
1145 std::vector<uint64_t> StackIds;
1146 // Max length of stack ids corresponding to a single stack ContextNode for
1147 // this context (i.e. the max length of a key in StackIdsToNode below).
1148 unsigned MaxLength = 0;
1149 // Mapping of slices of the stack ids to the corresponding ContextNode
1150 // (there can be multiple stack ids due to inlining). Populated when
1151 // updating stack nodes while matching them to the IR or summary.
1152 std::map<std::vector<uint64_t>, ContextNode *> StackIdsToNode;
1153 };
1154
1155 // Map of important full context ids to information about each.
1156 DenseMap<uint32_t, ImportantContextInfo> ImportantContextIdInfo;
1157
1158 // For each important context id found in Node (if any), records the list of
1159 // stack ids (those of the given StackNodes) that corresponded to the given
1160 // callsite Node. There can be more than one in the case of inlining.
1161 void recordStackNode(ArrayRef<ContextNode *> StackNodes, ContextNode *Node,
1162 // We pass in the Node's context ids to avoid the
1163 // overhead of computing them as the caller already has
1164 // them in some cases.
1165 const DenseSet<uint32_t> &NodeContextIds,
1166 const DenseSet<uint32_t> &ImportantContextIds) {
1167 if (!MemProfTopNImportant) {
1168 assert(ImportantContextIds.empty());
1169 return;
1170 }
1171 DenseSet<uint32_t> Ids =
1172 set_intersection(S1: NodeContextIds, S2: ImportantContextIds);
1173 if (Ids.empty())
1174 return;
1175 std::vector<uint64_t> StackIds;
1176 StackIds.reserve(n: StackNodes.size());
1177 for (const ContextNode *N : StackNodes)
1178 StackIds.push_back(N->OrigStackOrAllocId);
1179 auto Size = StackIds.size();
1180 for (auto Id : Ids) {
1181 auto &Entry = ImportantContextIdInfo[Id];
1182 Entry.StackIdsToNode[StackIds] = Node;
1183 // Keep track of the max to simplify later analysis.
1184 if (Size > Entry.MaxLength)
1185 Entry.MaxLength = Size;
1186 }
1187 }
1188
1189 /// Maps to track the calls to their corresponding nodes in the graph.
1190 MapVector<CallInfo, ContextNode *> AllocationCallToContextNodeMap;
1191 MapVector<CallInfo, ContextNode *> NonAllocationCallToContextNodeMap;
1192
1193 /// Owner of all ContextNode unique_ptrs.
1194 std::vector<std::unique_ptr<ContextNode>> NodeOwner;
1195
1196 /// Perform sanity checks on graph when requested.
1197 void check() const;
1198
1199 /// Keeps track of the last unique context id assigned.
1200 unsigned int LastContextId = 0;
1201
1202 /// Per-context information captured before identifyClones() when
1203 /// MemProfReportHintedSizesDetail is enabled, so that the report can relate
1204 /// the final (cloned) state back to the original context. "Frames" below are
1205 /// CCG nodes, not profile stack frames: a node matched to a call covers all
1206 /// of that call's inlined stack frames, so a chain can be shorter than the
1207 /// profiled stack.
1208 struct PreCloningContextInfo {
1209 // Ordered chain of uncloned ContextNodes for this context ID from the leaf
1210 // allocation node (Chain[0]) up to the outermost node in the CCG for this
1211 // context.
1212 std::vector<const ContextNode *> Chain;
1213 // Index in Chain of the outermost matched callsite node, i.e. the
1214 // outermost node such that it and all nodes below it
1215 // (Chain[1..MaxMatchedFrame]) were matched to a call in the IR/summary
1216 // (hasCall() == true). If the whole context matched, this is the
1217 // context's outermost node (possibly where the context was trimmed);
1218 // otherwise it is the last matched node below the first unmatched one. The
1219 // function containing it (the "outermost matched caller") is where the
1220 // reported post-cloning paths start. 0 if no caller of the allocation
1221 // matched.
1222 unsigned MaxMatchedFrame = 0;
1223 // True if the outermost matched node had caller edges for other contexts,
1224 // i.e. the context was trimmed below callers that other contexts reach.
1225 bool OutermostMatchedNodeHasOtherCallers = false;
1226 // If MaxMatchedFrame + 1 < Chain.size(), why did Chain[MaxMatchedFrame + 1]
1227 // not have a call: it was a recursive cycle node, or its calling function
1228 // was found but the call there did not match the callee (mismatched
1229 // callee); otherwise the caller had no !callsite for this stack id.
1230 bool StoppedAtRecursive = false;
1231 bool StoppedAtMismatchedCallee = false;
1232 // Stack id and (when known) calling function of the first unmatched node.
1233 uint64_t UnmatchedStackId = 0;
1234 const FuncTy *UnmatchedCallingFunc = nullptr;
1235 };
1236 /// Map from context id to its pre-cloning information.
1237 DenseMap<uint32_t, PreCloningContextInfo> PreCloningContextInfoMap;
1238 /// Allocation nodes (uncloned) that had at least one matched caller before
1239 /// cloning.
1240 DenseSet<const ContextNode *> PreCloningAllocHasMatchedCaller;
1241 /// Map from function to the number of function clones assigned to it
1242 /// (including the original), recorded during assignFunctions().
1243 DenseMap<const FuncTy *, unsigned> FuncToNumClones;
1244 /// Map from (original callsite node, clone number of the function clone
1245 /// containing the call) to the clone number of the callee function the call
1246 /// was updated to in assignFunctions(). Used to follow the post-cloning call
1247 /// path from a caller clone down to the allocation clone it reaches.
1248 DenseMap<std::pair<const ContextNode *, unsigned>, unsigned>
1249 OrigNodeAndCloneToCalleeClone;
1250 /// Map from (original allocation node, clone number of the function clone
1251 /// containing the allocation) to the allocation node in that clone, whose
1252 /// call carries the final hint.
1253 DenseMap<std::pair<const ContextNode *, unsigned>, const ContextNode *>
1254 OrigAllocAndCloneToNode;
1255};
1256
1257template <typename DerivedCCG, typename FuncTy, typename CallTy>
1258using ContextNode =
1259 typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode;
1260template <typename DerivedCCG, typename FuncTy, typename CallTy>
1261using ContextEdge =
1262 typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge;
1263template <typename DerivedCCG, typename FuncTy, typename CallTy>
1264using FuncInfo =
1265 typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::FuncInfo;
1266template <typename DerivedCCG, typename FuncTy, typename CallTy>
1267using CallInfo =
1268 typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::CallInfo;
1269
1270/// CRTP derived class for graphs built from IR (regular LTO).
1271class ModuleCallsiteContextGraph
1272 : public CallsiteContextGraph<ModuleCallsiteContextGraph, Function,
1273 Instruction *> {
1274public:
1275 ModuleCallsiteContextGraph(
1276 Module &M,
1277 llvm::function_ref<OptimizationRemarkEmitter &(Function *)> OREGetter);
1278
1279private:
1280 friend CallsiteContextGraph<ModuleCallsiteContextGraph, Function,
1281 Instruction *>;
1282
1283 uint64_t getStackId(uint64_t IdOrIndex) const;
1284 const Function *getCalleeFunc(Instruction *Call);
1285 bool calleeMatchesFunc(
1286 Instruction *Call, const Function *Func, const Function *CallerFunc,
1287 std::vector<std::pair<Instruction *, Function *>> &FoundCalleeChain);
1288 bool sameCallee(Instruction *Call1, Instruction *Call2);
1289 bool findProfiledCalleeThroughTailCalls(
1290 const Function *ProfiledCallee, Value *CurCallee, unsigned Depth,
1291 std::vector<std::pair<Instruction *, Function *>> &FoundCalleeChain,
1292 bool &FoundMultipleCalleeChains);
1293 uint64_t getLastStackId(Instruction *Call);
1294 std::vector<ContextNode *> getStackNodesForCall(Instruction *Call);
1295 void updateAllocationCall(CallInfo &Call, AllocationType AllocType);
1296 AllocationType getAllocationCallType(const CallInfo &Call) const;
1297 void updateCall(CallInfo &CallerCall, FuncInfo CalleeFunc);
1298 CallsiteContextGraph<ModuleCallsiteContextGraph, Function,
1299 Instruction *>::FuncInfo
1300 cloneFunctionForCallsite(FuncInfo &Func, CallInfo &Call,
1301 DenseMap<CallInfo, CallInfo> &CallMap,
1302 std::vector<CallInfo> &CallsWithMetadataInFunc,
1303 unsigned CloneNo);
1304 std::string getLabel(const Function *Func, const Instruction *Call,
1305 unsigned CloneNo) const;
1306 std::string getFuncName(const Function *Func, unsigned CloneNo = 0) const;
1307 bool isFuncDead(const Function *Func) const { return false; }
1308 void populateFunctionsWithCallers();
1309
1310 const Module &Mod;
1311 llvm::function_ref<OptimizationRemarkEmitter &(Function *)> OREGetter;
1312};
1313
1314/// Represents a call in the summary index graph, which can either be an
1315/// allocation or an interior callsite node in an allocation's context.
1316/// Holds a pointer to the corresponding data structure in the index.
1317struct IndexCall : public PointerUnion<CallsiteInfo *, AllocInfo *> {
1318 IndexCall() : PointerUnion() {}
1319 IndexCall(std::nullptr_t) : IndexCall() {}
1320 IndexCall(CallsiteInfo *StackNode) : PointerUnion(StackNode) {}
1321 IndexCall(AllocInfo *AllocNode) : PointerUnion(AllocNode) {}
1322 IndexCall(PointerUnion PT) : PointerUnion(PT) {}
1323
1324 IndexCall *operator->() { return this; }
1325
1326 void print(raw_ostream &OS) const {
1327 PointerUnion<CallsiteInfo *, AllocInfo *> Base = *this;
1328 if (auto *AI = llvm::dyn_cast_if_present<AllocInfo *>(Val&: Base)) {
1329 OS << *AI;
1330 } else {
1331 auto *CI = llvm::dyn_cast_if_present<CallsiteInfo *>(Val&: Base);
1332 assert(CI);
1333 OS << *CI;
1334 }
1335 }
1336};
1337} // namespace
1338
1339namespace llvm {
1340template <> struct simplify_type<IndexCall> {
1341 using SimpleType = PointerUnion<CallsiteInfo *, AllocInfo *>;
1342 static SimpleType getSimplifiedValue(IndexCall &Val) { return Val; }
1343};
1344template <> struct simplify_type<const IndexCall> {
1345 using SimpleType = const PointerUnion<CallsiteInfo *, AllocInfo *>;
1346 static SimpleType getSimplifiedValue(const IndexCall &Val) { return Val; }
1347};
1348} // namespace llvm
1349
1350namespace {
1351/// CRTP derived class for graphs built from summary index (ThinLTO).
1352class IndexCallsiteContextGraph
1353 : public CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
1354 IndexCall> {
1355public:
1356 IndexCallsiteContextGraph(
1357 ModuleSummaryIndex &Index,
1358 llvm::function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1359 isPrevailing);
1360
1361 ~IndexCallsiteContextGraph() {
1362 // Now that we are done with the graph it is safe to add the new
1363 // CallsiteInfo structs to the function summary vectors. The graph nodes
1364 // point into locations within these vectors, so we don't want to add them
1365 // any earlier.
1366 for (auto &I : FunctionCalleesToSynthesizedCallsiteInfos) {
1367 auto *FS = I.first;
1368 for (auto &Callsite : I.second)
1369 FS->addCallsite(Callsite: std::move(*Callsite.second));
1370 }
1371 }
1372
1373private:
1374 friend CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
1375 IndexCall>;
1376
1377 uint64_t getStackId(uint64_t IdOrIndex) const;
1378 const FunctionSummary *getCalleeFunc(IndexCall &Call);
1379 bool calleeMatchesFunc(
1380 IndexCall &Call, const FunctionSummary *Func,
1381 const FunctionSummary *CallerFunc,
1382 std::vector<std::pair<IndexCall, FunctionSummary *>> &FoundCalleeChain);
1383 bool sameCallee(IndexCall &Call1, IndexCall &Call2);
1384 bool findProfiledCalleeThroughTailCalls(
1385 ValueInfo ProfiledCallee, ValueInfo CurCallee, unsigned Depth,
1386 std::vector<std::pair<ValueInfo, FunctionSummary *>> &FoundCalleeChain,
1387 bool &FoundMultipleCalleeChains);
1388 uint64_t getLastStackId(IndexCall &Call);
1389 std::vector<ContextNode *> getStackNodesForCall(IndexCall &Call);
1390 void updateAllocationCall(CallInfo &Call, AllocationType AllocType);
1391 AllocationType getAllocationCallType(const CallInfo &Call) const;
1392 void updateCall(CallInfo &CallerCall, FuncInfo CalleeFunc);
1393 CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
1394 IndexCall>::FuncInfo
1395 cloneFunctionForCallsite(FuncInfo &Func, CallInfo &Call,
1396 DenseMap<CallInfo, CallInfo> &CallMap,
1397 std::vector<CallInfo> &CallsWithMetadataInFunc,
1398 unsigned CloneNo);
1399 std::string getLabel(const FunctionSummary *Func, const IndexCall &Call,
1400 unsigned CloneNo) const;
1401 std::string getFuncName(const FunctionSummary *Func,
1402 unsigned CloneNo = 0) const;
1403 bool isFuncDead(const FunctionSummary *Func) const;
1404 void populateFunctionsWithCallers();
1405 DenseSet<GlobalValue::GUID> findAliaseeGUIDsPrevailingInDifferentModule();
1406
1407 // Saves mapping from function summaries containing memprof records back to
1408 // its VI, for use in checking and debugging.
1409 std::map<const FunctionSummary *, ValueInfo> FSToVIMap;
1410
1411 const ModuleSummaryIndex &Index;
1412 llvm::function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1413 isPrevailing;
1414
1415 // Saves/owns the callsite info structures synthesized for missing tail call
1416 // frames that we discover while building the graph.
1417 // It maps from the summary of the function making the tail call, to a map
1418 // of callee ValueInfo to corresponding synthesized callsite info.
1419 DenseMap<FunctionSummary *,
1420 std::map<ValueInfo, std::unique_ptr<CallsiteInfo>>>
1421 FunctionCalleesToSynthesizedCallsiteInfos;
1422};
1423} // namespace
1424
1425template <>
1426struct llvm::DenseMapInfo<CallsiteContextGraph<
1427 ModuleCallsiteContextGraph, Function, Instruction *>::CallInfo>
1428 : public DenseMapInfo<std::pair<Instruction *, unsigned>> {};
1429template <>
1430struct llvm::DenseMapInfo<CallsiteContextGraph<
1431 IndexCallsiteContextGraph, FunctionSummary, IndexCall>::CallInfo>
1432 : public DenseMapInfo<std::pair<IndexCall, unsigned>> {};
1433template <>
1434struct llvm::DenseMapInfo<IndexCall>
1435 : public DenseMapInfo<PointerUnion<CallsiteInfo *, AllocInfo *>> {};
1436
1437namespace {
1438
1439// Map the uint8_t alloc types (which may contain NotCold|Cold) to the alloc
1440// type we should actually use on the corresponding allocation.
1441// If we can't clone a node that has NotCold+Cold alloc type, we will fall
1442// back to using NotCold. So don't bother cloning to distinguish NotCold+Cold
1443// from NotCold.
1444AllocationType allocTypeToUse(uint8_t AllocTypes) {
1445 assert(AllocTypes != (uint8_t)AllocationType::None);
1446 if (AllocTypes ==
1447 ((uint8_t)AllocationType::NotCold | (uint8_t)AllocationType::Cold))
1448 return AllocationType::NotCold;
1449 else
1450 return (AllocationType)AllocTypes;
1451}
1452
1453// Helper to check if the alloc types for all edges recorded in the
1454// InAllocTypes vector match the alloc types for all edges in the Edges
1455// vector.
1456template <typename DerivedCCG, typename FuncTy, typename CallTy>
1457bool allocTypesMatch(
1458 const std::vector<uint8_t> &InAllocTypes,
1459 const std::vector<std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>>>
1460 &Edges) {
1461 // This should be called only when the InAllocTypes vector was computed for
1462 // this set of Edges. Make sure the sizes are the same.
1463 assert(InAllocTypes.size() == Edges.size());
1464 return std::equal(
1465 InAllocTypes.begin(), InAllocTypes.end(), Edges.begin(), Edges.end(),
1466 [](const uint8_t &l,
1467 const std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>> &r) {
1468 // Can share if one of the edges is None type - don't
1469 // care about the type along that edge as it doesn't
1470 // exist for those context ids.
1471 if (l == (uint8_t)AllocationType::None ||
1472 r->AllocTypes == (uint8_t)AllocationType::None)
1473 return true;
1474 return allocTypeToUse(AllocTypes: l) == allocTypeToUse(r->AllocTypes);
1475 });
1476}
1477
1478// Helper to check if the alloc types for all edges recorded in the
1479// InAllocTypes vector match the alloc types for callee edges in the given
1480// clone. Because the InAllocTypes were computed from the original node's callee
1481// edges, and other cloning could have happened after this clone was created, we
1482// need to find the matching clone callee edge, which may or may not exist.
1483template <typename DerivedCCG, typename FuncTy, typename CallTy>
1484bool allocTypesMatchClone(
1485 const std::vector<uint8_t> &InAllocTypes,
1486 const ContextNode<DerivedCCG, FuncTy, CallTy> *Clone) {
1487 const ContextNode<DerivedCCG, FuncTy, CallTy> *Node = Clone->CloneOf;
1488 assert(Node);
1489 // InAllocTypes should have been computed for the original node's callee
1490 // edges.
1491 assert(InAllocTypes.size() == Node->CalleeEdges.size());
1492 // First create a map of the clone callee edge callees to the edge alloc type.
1493 DenseMap<const ContextNode<DerivedCCG, FuncTy, CallTy> *, uint8_t>
1494 EdgeCalleeMap;
1495 for (const auto &E : Clone->CalleeEdges) {
1496 assert(!EdgeCalleeMap.contains(E->Callee));
1497 EdgeCalleeMap[E->Callee] = E->AllocTypes;
1498 }
1499 // Next, walk the original node's callees, and look for the corresponding
1500 // clone edge to that callee.
1501 for (unsigned I = 0; I < Node->CalleeEdges.size(); I++) {
1502 auto Iter = EdgeCalleeMap.find(Node->CalleeEdges[I]->Callee);
1503 // Not found is ok, we will simply add an edge if we use this clone.
1504 if (Iter == EdgeCalleeMap.end())
1505 continue;
1506 // Can share if one of the edges is None type - don't
1507 // care about the type along that edge as it doesn't
1508 // exist for those context ids.
1509 if (InAllocTypes[I] == (uint8_t)AllocationType::None ||
1510 Iter->second == (uint8_t)AllocationType::None)
1511 continue;
1512 if (allocTypeToUse(Iter->second) != allocTypeToUse(AllocTypes: InAllocTypes[I]))
1513 return false;
1514 }
1515 return true;
1516}
1517
1518} // end anonymous namespace
1519
1520template <typename DerivedCCG, typename FuncTy, typename CallTy>
1521typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
1522CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::getNodeForInst(
1523 const CallInfo &C) {
1524 ContextNode *Node = getNodeForAlloc(C);
1525 if (Node)
1526 return Node;
1527
1528 return NonAllocationCallToContextNodeMap.lookup(C);
1529}
1530
1531template <typename DerivedCCG, typename FuncTy, typename CallTy>
1532typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
1533CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::getNodeForAlloc(
1534 const CallInfo &C) {
1535 return AllocationCallToContextNodeMap.lookup(C);
1536}
1537
1538template <typename DerivedCCG, typename FuncTy, typename CallTy>
1539typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
1540CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::getNodeForStackId(
1541 uint64_t StackId) {
1542 auto StackEntryNode = StackEntryIdToContextNodeMap.find(StackId);
1543 if (StackEntryNode != StackEntryIdToContextNodeMap.end())
1544 return StackEntryNode->second;
1545 return nullptr;
1546}
1547
1548template <typename DerivedCCG, typename FuncTy, typename CallTy>
1549void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1550 addOrUpdateCallerEdge(ContextNode *Caller, AllocationType AllocType,
1551 unsigned int ContextId) {
1552 for (auto &Edge : CallerEdges) {
1553 if (Edge->Caller == Caller) {
1554 Edge->AllocTypes |= (uint8_t)AllocType;
1555 Edge->getContextIds().insert(ContextId);
1556 return;
1557 }
1558 }
1559 std::shared_ptr<ContextEdge> Edge = std::make_shared<ContextEdge>(
1560 this, Caller, (uint8_t)AllocType, DenseSet<uint32_t>({ContextId}));
1561 CallerEdges.push_back(Edge);
1562 Caller->CalleeEdges.push_back(Edge);
1563}
1564
1565template <typename DerivedCCG, typename FuncTy, typename CallTy>
1566void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::removeEdgeFromGraph(
1567 ContextEdge *Edge, EdgeIter *EI, bool CalleeIter) {
1568 assert(!EI || (*EI)->get() == Edge);
1569 assert(!Edge->isRemoved());
1570 // Save the Caller and Callee pointers so we can erase Edge from their edge
1571 // lists after clearing Edge below. We do the clearing first in case it is
1572 // destructed after removing from the edge lists (if those were the last
1573 // shared_ptr references to Edge).
1574 auto *Callee = Edge->Callee;
1575 auto *Caller = Edge->Caller;
1576
1577 // Make sure the edge fields are cleared out so we can properly detect
1578 // removed edges if Edge is not destructed because there is still a shared_ptr
1579 // reference.
1580 Edge->clear();
1581
1582#ifndef NDEBUG
1583 auto CalleeCallerCount = Callee->CallerEdges.size();
1584 auto CallerCalleeCount = Caller->CalleeEdges.size();
1585#endif
1586 if (!EI) {
1587 Callee->eraseCallerEdge(Edge);
1588 Caller->eraseCalleeEdge(Edge);
1589 } else if (CalleeIter) {
1590 Callee->eraseCallerEdge(Edge);
1591 *EI = Caller->CalleeEdges.erase(*EI);
1592 } else {
1593 Caller->eraseCalleeEdge(Edge);
1594 *EI = Callee->CallerEdges.erase(*EI);
1595 }
1596 assert(Callee->CallerEdges.size() < CalleeCallerCount);
1597 assert(Caller->CalleeEdges.size() < CallerCalleeCount);
1598}
1599
1600template <typename DerivedCCG, typename FuncTy, typename CallTy>
1601void CallsiteContextGraph<
1602 DerivedCCG, FuncTy, CallTy>::removeNoneTypeCalleeEdges(ContextNode *Node) {
1603 for (auto EI = Node->CalleeEdges.begin(); EI != Node->CalleeEdges.end();) {
1604 auto Edge = *EI;
1605 if (Edge->AllocTypes == (uint8_t)AllocationType::None) {
1606 assert(Edge->ContextIds.empty());
1607 removeEdgeFromGraph(Edge: Edge.get(), EI: &EI, /*CalleeIter=*/true);
1608 } else
1609 ++EI;
1610 }
1611}
1612
1613template <typename DerivedCCG, typename FuncTy, typename CallTy>
1614void CallsiteContextGraph<
1615 DerivedCCG, FuncTy, CallTy>::removeNoneTypeCallerEdges(ContextNode *Node) {
1616 for (auto EI = Node->CallerEdges.begin(); EI != Node->CallerEdges.end();) {
1617 auto Edge = *EI;
1618 if (Edge->AllocTypes == (uint8_t)AllocationType::None) {
1619 assert(Edge->ContextIds.empty());
1620 Edge->Caller->eraseCalleeEdge(Edge.get());
1621 EI = Node->CallerEdges.erase(EI);
1622 } else
1623 ++EI;
1624 }
1625}
1626
1627template <typename DerivedCCG, typename FuncTy, typename CallTy>
1628typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge *
1629CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1630 findEdgeFromCallee(const ContextNode *Callee) {
1631 for (const auto &Edge : CalleeEdges)
1632 if (Edge->Callee == Callee)
1633 return Edge.get();
1634 return nullptr;
1635}
1636
1637template <typename DerivedCCG, typename FuncTy, typename CallTy>
1638typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge *
1639CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1640 findEdgeFromCaller(const ContextNode *Caller) {
1641 for (const auto &Edge : CallerEdges)
1642 if (Edge->Caller == Caller)
1643 return Edge.get();
1644 return nullptr;
1645}
1646
1647template <typename DerivedCCG, typename FuncTy, typename CallTy>
1648void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1649 eraseCalleeEdge(const ContextEdge *Edge) {
1650 auto EI = llvm::find_if(
1651 CalleeEdges, [Edge](const std::shared_ptr<ContextEdge> &CalleeEdge) {
1652 return CalleeEdge.get() == Edge;
1653 });
1654 assert(EI != CalleeEdges.end());
1655 CalleeEdges.erase(EI);
1656}
1657
1658template <typename DerivedCCG, typename FuncTy, typename CallTy>
1659void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1660 eraseCallerEdge(const ContextEdge *Edge) {
1661 auto EI = llvm::find_if(
1662 CallerEdges, [Edge](const std::shared_ptr<ContextEdge> &CallerEdge) {
1663 return CallerEdge.get() == Edge;
1664 });
1665 assert(EI != CallerEdges.end());
1666 CallerEdges.erase(EI);
1667}
1668
1669template <typename DerivedCCG, typename FuncTy, typename CallTy>
1670uint8_t CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::computeAllocType(
1671 DenseSet<uint32_t> &ContextIds) const {
1672 uint8_t BothTypes =
1673 (uint8_t)AllocationType::Cold | (uint8_t)AllocationType::NotCold;
1674 uint8_t AllocType = (uint8_t)AllocationType::None;
1675 for (auto Id : ContextIds) {
1676 AllocType |= (uint8_t)ContextIdToAllocationType.at(Val: Id);
1677 // Bail early if alloc type reached both, no further refinement.
1678 if (AllocType == BothTypes)
1679 return AllocType;
1680 }
1681 return AllocType;
1682}
1683
1684template <typename DerivedCCG, typename FuncTy, typename CallTy>
1685uint8_t
1686CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::intersectAllocTypesImpl(
1687 const DenseSet<uint32_t> &Node1Ids,
1688 const DenseSet<uint32_t> &Node2Ids) const {
1689 uint8_t BothTypes =
1690 (uint8_t)AllocationType::Cold | (uint8_t)AllocationType::NotCold;
1691 uint8_t AllocType = (uint8_t)AllocationType::None;
1692 for (auto Id : Node1Ids) {
1693 if (!Node2Ids.count(V: Id))
1694 continue;
1695 AllocType |= (uint8_t)ContextIdToAllocationType.at(Val: Id);
1696 // Bail early if alloc type reached both, no further refinement.
1697 if (AllocType == BothTypes)
1698 return AllocType;
1699 }
1700 return AllocType;
1701}
1702
1703template <typename DerivedCCG, typename FuncTy, typename CallTy>
1704uint8_t CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::intersectAllocTypes(
1705 const DenseSet<uint32_t> &Node1Ids,
1706 const DenseSet<uint32_t> &Node2Ids) const {
1707 if (Node1Ids.size() < Node2Ids.size())
1708 return intersectAllocTypesImpl(Node1Ids, Node2Ids);
1709 else
1710 return intersectAllocTypesImpl(Node1Ids: Node2Ids, Node2Ids: Node1Ids);
1711}
1712
1713template <typename DerivedCCG, typename FuncTy, typename CallTy>
1714typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
1715CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::addAllocNode(
1716 CallInfo Call, const FuncTy *F) {
1717 assert(!getNodeForAlloc(Call));
1718 ContextNode *AllocNode = createNewNode(/*IsAllocation=*/true, F, C: Call);
1719 AllocationCallToContextNodeMap[Call] = AllocNode;
1720 // Use LastContextId as a uniq id for MIB allocation nodes.
1721 AllocNode->OrigStackOrAllocId = LastContextId;
1722 // Alloc type should be updated as we add in the MIBs. We should assert
1723 // afterwards that it is not still None.
1724 AllocNode->AllocTypes = (uint8_t)AllocationType::None;
1725
1726 return AllocNode;
1727}
1728
1729static std::string getAllocTypeString(uint8_t AllocTypes) {
1730 if (!AllocTypes)
1731 return "None";
1732 std::string Str;
1733 if (AllocTypes & (uint8_t)AllocationType::NotCold)
1734 Str += "NotCold";
1735 if (AllocTypes & (uint8_t)AllocationType::Cold)
1736 Str += "Cold";
1737 return Str;
1738}
1739
1740template <typename DerivedCCG, typename FuncTy, typename CallTy>
1741template <class NodeT, class IteratorT>
1742void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::addStackNodesForMIB(
1743 ContextNode *AllocNode, CallStack<NodeT, IteratorT> &StackContext,
1744 CallStack<NodeT, IteratorT> &CallsiteContext, AllocationType AllocType,
1745 ArrayRef<ContextTotalSize> ContextSizeInfo,
1746 std::map<uint64_t, uint32_t> &TotalSizeToContextIdTopNCold) {
1747 // Treating the hot alloc type as NotCold before the disambiguation for "hot"
1748 // is done.
1749 if (AllocType == AllocationType::Hot)
1750 AllocType = AllocationType::NotCold;
1751
1752 ContextIdToAllocationType[++LastContextId] = AllocType;
1753
1754 bool IsImportant = false;
1755 if (!ContextSizeInfo.empty()) {
1756 auto &Entry = ContextIdToContextSizeInfos[LastContextId];
1757 // If this is a cold allocation, and we are collecting non-zero largest
1758 // contexts, see if this is a candidate.
1759 if (AllocType == AllocationType::Cold && MemProfTopNImportant > 0) {
1760 uint64_t TotalCold = 0;
1761 for (auto &CSI : ContextSizeInfo)
1762 TotalCold += CSI.TotalSize;
1763 // Record this context if either we haven't found the first top-n largest
1764 // yet, or if it is larger than the smallest already recorded.
1765 if (TotalSizeToContextIdTopNCold.size() < MemProfTopNImportant ||
1766 // Since TotalSizeToContextIdTopNCold is a std::map, it is implicitly
1767 // sorted in ascending size of its key which is the size.
1768 TotalCold > TotalSizeToContextIdTopNCold.begin()->first) {
1769 if (TotalSizeToContextIdTopNCold.size() == MemProfTopNImportant) {
1770 // Remove old one and its associated entries.
1771 auto IdToRemove = TotalSizeToContextIdTopNCold.begin()->second;
1772 TotalSizeToContextIdTopNCold.erase(
1773 position: TotalSizeToContextIdTopNCold.begin());
1774 assert(ImportantContextIdInfo.count(IdToRemove));
1775 ImportantContextIdInfo.erase(IdToRemove);
1776 }
1777 TotalSizeToContextIdTopNCold[TotalCold] = LastContextId;
1778 IsImportant = true;
1779 }
1780 }
1781 Entry.insert(position: Entry.begin(), first: ContextSizeInfo.begin(), last: ContextSizeInfo.end());
1782 }
1783
1784 // Update alloc type and context ids for this MIB.
1785 AllocNode->AllocTypes |= (uint8_t)AllocType;
1786
1787 // Now add or update nodes for each stack id in alloc's context.
1788 // Later when processing the stack ids on non-alloc callsites we will adjust
1789 // for any inlining in the context.
1790 ContextNode *PrevNode = AllocNode;
1791 // Look for recursion (direct recursion should have been collapsed by
1792 // module summary analysis, here we should just be detecting mutual
1793 // recursion). Mark these nodes so we don't try to clone.
1794 SmallSet<uint64_t, 8> StackIdSet;
1795 // Skip any on the allocation call (inlining).
1796 for (auto ContextIter = StackContext.beginAfterSharedPrefix(CallsiteContext);
1797 ContextIter != StackContext.end(); ++ContextIter) {
1798 auto StackId = getStackId(IdOrIndex: *ContextIter);
1799 if (IsImportant)
1800 ImportantContextIdInfo[LastContextId].StackIds.push_back(StackId);
1801 ContextNode *StackNode = getNodeForStackId(StackId);
1802 if (!StackNode) {
1803 StackNode = createNewNode(/*IsAllocation=*/false);
1804 StackEntryIdToContextNodeMap[StackId] = StackNode;
1805 StackNode->OrigStackOrAllocId = StackId;
1806 }
1807 // Marking a node recursive will prevent its cloning completely, even for
1808 // non-recursive contexts flowing through it.
1809 if (!AllowRecursiveCallsites) {
1810 auto Ins = StackIdSet.insert(StackId);
1811 if (!Ins.second)
1812 StackNode->Recursive = true;
1813 }
1814 StackNode->AllocTypes |= (uint8_t)AllocType;
1815 PrevNode->addOrUpdateCallerEdge(StackNode, AllocType, LastContextId);
1816 PrevNode = StackNode;
1817 }
1818}
1819
1820template <typename DerivedCCG, typename FuncTy, typename CallTy>
1821DenseSet<uint32_t>
1822CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::duplicateContextIds(
1823 const DenseSet<uint32_t> &StackSequenceContextIds,
1824 DenseMap<uint32_t, DenseSet<uint32_t>> &OldToNewContextIds,
1825 function_ref<void(uint32_t OldId, uint32_t NewId)> OnDuplicate) {
1826 DenseSet<uint32_t> NewContextIds;
1827 NewContextIds.reserve(Size: StackSequenceContextIds.size());
1828 for (auto OldId : StackSequenceContextIds) {
1829 NewContextIds.insert(V: ++LastContextId);
1830 OldToNewContextIds[OldId].insert(V: LastContextId);
1831 if (OnDuplicate)
1832 OnDuplicate(OldId, LastContextId);
1833 assert(ContextIdToAllocationType.count(OldId));
1834 // The new context has the same allocation type and size info as original.
1835 ContextIdToAllocationType[LastContextId] = ContextIdToAllocationType[OldId];
1836 auto CSI = ContextIdToContextSizeInfos.find(Val: OldId);
1837 if (CSI != ContextIdToContextSizeInfos.end())
1838 ContextIdToContextSizeInfos[LastContextId] = CSI->second;
1839 if (DotAllocContextIds.contains(V: OldId))
1840 DotAllocContextIds.insert(V: LastContextId);
1841 }
1842 return NewContextIds;
1843}
1844
1845template <typename DerivedCCG, typename FuncTy, typename CallTy>
1846void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
1847 propagateDuplicateContextIds(
1848 const DenseMap<uint32_t, DenseSet<uint32_t>> &OldToNewContextIds) {
1849 // Build a set of duplicated context ids corresponding to the input id set.
1850 auto GetNewIds = [&OldToNewContextIds](const DenseSet<uint32_t> &ContextIds) {
1851 DenseSet<uint32_t> NewIds;
1852 for (auto Id : ContextIds)
1853 if (auto NewId = OldToNewContextIds.find(Val: Id);
1854 NewId != OldToNewContextIds.end())
1855 NewIds.insert_range(R: NewId->second);
1856 return NewIds;
1857 };
1858
1859 // Recursively update context ids sets along caller edges.
1860 auto UpdateCallers = [&](ContextNode *Node,
1861 DenseSet<const ContextEdge *> &Visited,
1862 auto &&UpdateCallers) -> void {
1863 for (const auto &Edge : Node->CallerEdges) {
1864 auto Inserted = Visited.insert(Edge.get());
1865 if (!Inserted.second)
1866 continue;
1867 ContextNode *NextNode = Edge->Caller;
1868 DenseSet<uint32_t> NewIdsToAdd = GetNewIds(Edge->getContextIds());
1869 // Only need to recursively iterate to NextNode via this caller edge if
1870 // it resulted in any added ids to NextNode.
1871 if (!NewIdsToAdd.empty()) {
1872 Edge->getContextIds().insert_range(NewIdsToAdd);
1873 UpdateCallers(NextNode, Visited, UpdateCallers);
1874 }
1875 }
1876 };
1877
1878 DenseSet<const ContextEdge *> Visited;
1879 for (auto &Entry : AllocationCallToContextNodeMap) {
1880 auto *Node = Entry.second;
1881 UpdateCallers(Node, Visited, UpdateCallers);
1882 }
1883}
1884
1885template <typename DerivedCCG, typename FuncTy, typename CallTy>
1886void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::connectNewNode(
1887 ContextNode *NewNode, ContextNode *OrigNode, bool TowardsCallee,
1888 const DenseSet<uint32_t> &ContextIds) {
1889 auto &OrigEdges =
1890 TowardsCallee ? OrigNode->CalleeEdges : OrigNode->CallerEdges;
1891 // Increment iterator in loop so that we can remove edges as needed.
1892 for (auto EI = OrigEdges.begin(); EI != OrigEdges.end();) {
1893 auto Edge = *EI;
1894 DenseSet<uint32_t> NewEdgeContextIds;
1895 // Remove any matching context ids from Edge, return set that were found and
1896 // removed, these are the new edge's context ids.
1897 set_subtract(Edge->getContextIds(), ContextIds, NewEdgeContextIds);
1898 // If no matching context ids for this edge, skip it.
1899 if (NewEdgeContextIds.empty()) {
1900 ++EI;
1901 continue;
1902 }
1903 if (TowardsCallee) {
1904 uint8_t NewAllocType = computeAllocType(ContextIds&: NewEdgeContextIds);
1905 auto NewEdge = std::make_shared<ContextEdge>(
1906 Edge->Callee, NewNode, NewAllocType, std::move(NewEdgeContextIds));
1907 NewNode->CalleeEdges.push_back(NewEdge);
1908 NewEdge->Callee->CallerEdges.push_back(NewEdge);
1909 } else {
1910 uint8_t NewAllocType = computeAllocType(ContextIds&: NewEdgeContextIds);
1911 auto NewEdge = std::make_shared<ContextEdge>(
1912 NewNode, Edge->Caller, NewAllocType, std::move(NewEdgeContextIds));
1913 NewNode->CallerEdges.push_back(NewEdge);
1914 NewEdge->Caller->CalleeEdges.push_back(NewEdge);
1915 }
1916 // Remove old edge if context ids empty, otherwise update its alloc type to
1917 // reflect the removed context ids.
1918 if (Edge->getContextIds().empty()) {
1919 removeEdgeFromGraph(Edge: Edge.get(), EI: &EI, CalleeIter: TowardsCallee);
1920 continue;
1921 }
1922 Edge->AllocTypes = computeAllocType(ContextIds&: Edge->getContextIds());
1923 ++EI;
1924 }
1925}
1926
1927template <typename DerivedCCG, typename FuncTy, typename CallTy>
1928static void checkEdge(
1929 const std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>> &Edge) {
1930 // Confirm that alloc type is not None and that we have at least one context
1931 // id.
1932 assert(Edge->AllocTypes != (uint8_t)AllocationType::None);
1933 assert(!Edge->ContextIds.empty());
1934}
1935
1936template <typename DerivedCCG, typename FuncTy, typename CallTy>
1937static void checkNode(const ContextNode<DerivedCCG, FuncTy, CallTy> *Node,
1938 bool CheckEdges = true) {
1939 if (Node->isRemoved())
1940 return;
1941#ifndef NDEBUG
1942 // Compute node's context ids once for use in asserts.
1943 auto NodeContextIds = Node->getContextIds();
1944#endif
1945 // Node's context ids should be the union of both its callee and caller edge
1946 // context ids.
1947 if (Node->CallerEdges.size()) {
1948 DenseSet<uint32_t> CallerEdgeContextIds(
1949 Node->CallerEdges.front()->ContextIds);
1950 for (const auto &Edge : llvm::drop_begin(Node->CallerEdges)) {
1951 if (CheckEdges)
1952 checkEdge<DerivedCCG, FuncTy, CallTy>(Edge);
1953 set_union(CallerEdgeContextIds, Edge->ContextIds);
1954 }
1955 // Node can have more context ids than callers if some contexts terminate at
1956 // node and some are longer. If we are allowing recursive callsites and
1957 // contexts this will be violated for incompletely cloned recursive cycles,
1958 // so skip the checking in that case.
1959 assert((AllowRecursiveCallsites && AllowRecursiveContexts) ||
1960 NodeContextIds == CallerEdgeContextIds ||
1961 set_is_subset(CallerEdgeContextIds, NodeContextIds));
1962 }
1963 if (Node->CalleeEdges.size()) {
1964 DenseSet<uint32_t> CalleeEdgeContextIds(
1965 Node->CalleeEdges.front()->ContextIds);
1966 for (const auto &Edge : llvm::drop_begin(Node->CalleeEdges)) {
1967 if (CheckEdges)
1968 checkEdge<DerivedCCG, FuncTy, CallTy>(Edge);
1969 set_union(CalleeEdgeContextIds, Edge->getContextIds());
1970 }
1971 // If we are allowing recursive callsites and contexts this will be violated
1972 // for incompletely cloned recursive cycles, so skip the checking in that
1973 // case.
1974 assert((AllowRecursiveCallsites && AllowRecursiveContexts) ||
1975 NodeContextIds == CalleeEdgeContextIds);
1976 }
1977 // FIXME: Since this checking is only invoked under an option, we should
1978 // change the error checking from using assert to something that will trigger
1979 // an error on a release build.
1980#ifndef NDEBUG
1981 // Make sure we don't end up with duplicate edges between the same caller and
1982 // callee.
1983 DenseSet<ContextNode<DerivedCCG, FuncTy, CallTy> *> NodeSet;
1984 for (const auto &E : Node->CalleeEdges)
1985 NodeSet.insert(E->Callee);
1986 assert(NodeSet.size() == Node->CalleeEdges.size());
1987#endif
1988}
1989
1990template <typename DerivedCCG, typename FuncTy, typename CallTy>
1991void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
1992 assignStackNodesPostOrder(ContextNode *Node,
1993 DenseSet<const ContextNode *> &Visited,
1994 DenseMap<uint64_t, std::vector<CallContextInfo>>
1995 &StackIdToMatchingCalls,
1996 DenseMap<CallInfo, CallInfo> &CallToMatchingCall,
1997 const DenseSet<uint32_t> &ImportantContextIds) {
1998 auto Inserted = Visited.insert(Node);
1999 if (!Inserted.second)
2000 return;
2001 // Post order traversal. Iterate over a copy since we may add nodes and
2002 // therefore new callers during the recursive call, invalidating any
2003 // iterator over the original edge vector. We don't need to process these
2004 // new nodes as they were already processed on creation.
2005 auto CallerEdges = Node->CallerEdges;
2006 for (auto &Edge : CallerEdges) {
2007 // Skip any that have been removed during the recursion.
2008 if (Edge->isRemoved()) {
2009 assert(!is_contained(Node->CallerEdges, Edge));
2010 continue;
2011 }
2012 assignStackNodesPostOrder(Node: Edge->Caller, Visited, StackIdToMatchingCalls,
2013 CallToMatchingCall, ImportantContextIds);
2014 }
2015
2016 // If this node's stack id is in the map, update the graph to contain new
2017 // nodes representing any inlining at interior callsites. Note we move the
2018 // associated context ids over to the new nodes.
2019
2020 // Ignore this node if it is for an allocation or we didn't record any
2021 // stack id lists ending at it.
2022 if (Node->IsAllocation ||
2023 !StackIdToMatchingCalls.count(Node->OrigStackOrAllocId))
2024 return;
2025
2026 auto &Calls = StackIdToMatchingCalls[Node->OrigStackOrAllocId];
2027 // Handle the simple case first. A single call with a single stack id.
2028 // In this case there is no need to create any new context nodes, simply
2029 // assign the context node for stack id to this Call.
2030 if (Calls.size() == 1) {
2031 auto &[Call, StackNodes, Func, SavedContextIds, TrimmedInlinedDupIds] =
2032 Calls[0];
2033 if (StackNodes.size() == 1) {
2034 assert(SavedContextIds.empty());
2035 // Only calls with inlined frames get these.
2036 assert(TrimmedInlinedDupIds.empty());
2037 // It should be this Node
2038 assert(Node == StackNodes[0]);
2039 if (Node->Recursive)
2040 return;
2041 Node->setCall(Call);
2042 NonAllocationCallToContextNodeMap[Call] = Node;
2043 NodeToCallingFunc[Node] = Func;
2044 recordStackNode(StackNodes, Node, NodeContextIds: Node->getContextIds(),
2045 ImportantContextIds);
2046 return;
2047 }
2048 }
2049
2050#ifndef NDEBUG
2051 // Find the node for the last stack id, which should be the same
2052 // across all calls recorded for this id, and is this node's id.
2053 uint64_t LastId = Node->OrigStackOrAllocId;
2054 ContextNode *LastNode = getNodeForStackId(LastId);
2055 // We should only have kept stack ids that had nodes.
2056 assert(LastNode);
2057 assert(LastNode == Node);
2058#else
2059 ContextNode *LastNode = Node;
2060#endif
2061
2062 // Compute the last node's context ids once, as it is shared by all calls in
2063 // this entry.
2064 DenseSet<uint32_t> LastNodeContextIds = LastNode->getContextIds();
2065
2066 [[maybe_unused]] bool PrevIterCreatedNode = false;
2067 bool CreatedNode = false;
2068 for (unsigned I = 0; I < Calls.size();
2069 I++, PrevIterCreatedNode = CreatedNode) {
2070 CreatedNode = false;
2071 auto &[Call, StackNodes, Func, SavedContextIds, TrimmedInlinedDupIds] =
2072 Calls[I];
2073 // Skip any for which we didn't assign any ids, these don't get a node in
2074 // the graph.
2075 if (SavedContextIds.empty() && TrimmedInlinedDupIds.empty()) {
2076 // If this call has a matching call (located in the same function and
2077 // having the same stack ids), simply add it to the context node created
2078 // for its matching call earlier. These can be treated the same through
2079 // cloning and get updated at the same time.
2080 if (!CallToMatchingCall.contains(Call))
2081 continue;
2082 auto MatchingCall = CallToMatchingCall[Call];
2083 if (!NonAllocationCallToContextNodeMap.contains(MatchingCall)) {
2084 // This should only happen if we had a prior iteration, and it didn't
2085 // create a node because of the below recomputation of context ids
2086 // finding none remaining and continuing early.
2087 assert(I > 0 && !PrevIterCreatedNode);
2088 continue;
2089 }
2090 NonAllocationCallToContextNodeMap[MatchingCall]->MatchingCalls.push_back(
2091 Call);
2092 continue;
2093 }
2094
2095 assert(LastNode == StackNodes.back());
2096
2097 // Recompute the context ids for this stack id sequence (the
2098 // intersection of the context ids of the corresponding nodes).
2099 // Start with the ids we saved in the map for this call, which could be
2100 // duplicated context ids. We have to recompute as we might have overlap
2101 // overlap between the saved context ids for different last nodes, and
2102 // removed them already during the post order traversal.
2103 set_intersect(SavedContextIds, LastNodeContextIds);
2104 ContextNode *PrevNode = LastNode;
2105 bool Skip = SavedContextIds.empty();
2106 // Iterate backwards through the stack nodes, starting after the last one
2107 // in the list, which was handled once outside for all Calls.
2108 for (auto NodeIter = StackNodes.rbegin() + 1;
2109 !Skip && NodeIter != StackNodes.rend(); NodeIter++) {
2110 ContextNode *CurNode = *NodeIter;
2111 // We should only have kept stack ids that had nodes and weren't
2112 // recursive.
2113 assert(CurNode);
2114 assert(!CurNode->Recursive);
2115
2116 auto *Edge = CurNode->findEdgeFromCaller(PrevNode);
2117 if (!Edge) {
2118 Skip = true;
2119 break;
2120 }
2121 PrevNode = CurNode;
2122
2123 // Update the context ids, which is the intersection of the ids along
2124 // all edges in the sequence.
2125 set_intersect(SavedContextIds, Edge->getContextIds());
2126
2127 // If we now have no context ids for clone, skip this call.
2128 if (SavedContextIds.empty()) {
2129 Skip = true;
2130 break;
2131 }
2132 }
2133 if (Skip)
2134 SavedContextIds.clear();
2135 // Add the duplicates created for this call (see updateStackNodes) that are
2136 // still on its first node. Other calls given them end below that node and
2137 // leave them on it, but a recursive context may traverse this call's
2138 // initial nodes more than once with different continuations, in which
2139 // case a call assigned one of the other traversals may have been processed
2140 // first and claimed them (as happens to the original ids too).
2141 if (!TrimmedInlinedDupIds.empty()) {
2142 StackNodes[0]->removeContextIdsNotPresent(TrimmedInlinedDupIds);
2143 SavedContextIds.insert_range(TrimmedInlinedDupIds);
2144 }
2145 if (SavedContextIds.empty())
2146 continue;
2147
2148 // Create new context node.
2149 ContextNode *NewNode = createNewNode(/*IsAllocation=*/false, F: Func, C: Call);
2150 NonAllocationCallToContextNodeMap[Call] = NewNode;
2151 CreatedNode = true;
2152 NewNode->AllocTypes = computeAllocType(ContextIds&: SavedContextIds);
2153
2154 ContextNode *FirstNode = StackNodes[0];
2155 assert(FirstNode);
2156
2157 // Connect to callees of innermost stack frame in inlined call chain.
2158 // This updates context ids for FirstNode's callee's to reflect those
2159 // moved to NewNode.
2160 connectNewNode(NewNode, OrigNode: FirstNode, /*TowardsCallee=*/true, ContextIds: SavedContextIds);
2161
2162 // Connect to callers of outermost stack frame in inlined call chain.
2163 // This updates context ids for FirstNode's caller's to reflect those
2164 // moved to NewNode.
2165 connectNewNode(NewNode, OrigNode: LastNode, /*TowardsCallee=*/false, ContextIds: SavedContextIds);
2166
2167 // Now we need to remove context ids from edges/nodes between First and
2168 // Last Node.
2169 PrevNode = nullptr;
2170 for (ContextNode *CurNode : StackNodes) {
2171 // We should only have kept stack ids that had nodes.
2172 assert(CurNode);
2173
2174 // Remove the context ids moved to NewNode from CurNode, and the
2175 // edge from the prior node.
2176 if (PrevNode) {
2177 auto *PrevEdge = CurNode->findEdgeFromCallee(PrevNode);
2178 // If the sequence contained recursion, we might have already removed
2179 // some edges during the connectNewNode calls above.
2180 if (!PrevEdge) {
2181 PrevNode = CurNode;
2182 continue;
2183 }
2184 set_subtract(PrevEdge->getContextIds(), SavedContextIds);
2185 if (PrevEdge->getContextIds().empty())
2186 removeEdgeFromGraph(Edge: PrevEdge);
2187 else
2188 PrevEdge->AllocTypes = computeAllocType(ContextIds&: PrevEdge->getContextIds());
2189 }
2190 // Since we update the edges from leaf to tail, only look at the callee
2191 // edges. This isn't an alloc node, so if there are no callee edges, the
2192 // alloc type is None.
2193 CurNode->AllocTypes = CurNode->CalleeEdges.empty()
2194 ? (uint8_t)AllocationType::None
2195 : CurNode->computeAllocType();
2196 PrevNode = CurNode;
2197 }
2198
2199 recordStackNode(StackNodes, Node: NewNode, NodeContextIds: SavedContextIds, ImportantContextIds);
2200
2201 if (VerifyNodes) {
2202 checkNode<DerivedCCG, FuncTy, CallTy>(NewNode, /*CheckEdges=*/true);
2203 for (ContextNode *CurNode : StackNodes) {
2204 // We should only have kept stack ids that had nodes.
2205 assert(CurNode);
2206 checkNode<DerivedCCG, FuncTy, CallTy>(CurNode, /*CheckEdges=*/true);
2207 }
2208 }
2209 }
2210}
2211
2212template <typename DerivedCCG, typename FuncTy, typename CallTy>
2213void CallsiteContextGraph<DerivedCCG, FuncTy,
2214 CallTy>::fixupImportantContexts() {
2215 if (ImportantContextIdInfo.empty())
2216 return;
2217
2218 // Update statistics as we are done building this map at this point.
2219 NumImportantContextIds = ImportantContextIdInfo.size();
2220
2221 if (!MemProfFixupImportant)
2222 return;
2223
2224 if (ExportToDot)
2225 exportToDot(Label: "beforestackfixup");
2226
2227 // For each context we identified as important, walk through the saved context
2228 // stack ids in order from leaf upwards, and make sure all edges are correct.
2229 // These can be difficult to get right when updating the graph while mapping
2230 // nodes onto summary or IR, especially when there is recursion. In
2231 // particular, when we have created new nodes to reflect inlining, it is
2232 // sometimes impossible to know exactly how to update the edges in the face of
2233 // recursion, as we have lost the original ordering of the stack ids in the
2234 // contexts.
2235 // TODO: Consider only doing this if we detect the context has recursive
2236 // cycles.
2237 //
2238 // I.e. assume we have a context with stack ids like: {A B A C A D E}
2239 // and let's say A was inlined into B, C, and D. The original graph will have
2240 // multiple recursive cycles through A. When we match the original context
2241 // nodes onto the IR or summary, we will merge {A B} into one context node,
2242 // {A C} onto another, and {A D} onto another. Looking at the stack sequence
2243 // above, we should end up with a non-cyclic set of edges like:
2244 // {AB} <- {AC} <- {AD} <- E. However, because we normally have lost the
2245 // original ordering, we won't get the edges correct initially (it's
2246 // impossible without the original ordering). Here we do the fixup (add and
2247 // removing edges where necessary) for this context. In the
2248 // ImportantContextInfo struct in this case we should have a MaxLength = 2,
2249 // and map entries for {A B}, {A C}, {A D}, and {E}.
2250 for (auto &[CurContextId, Info] : ImportantContextIdInfo) {
2251 if (Info.StackIdsToNode.empty())
2252 continue;
2253 bool Changed = false;
2254 ContextNode *PrevNode = nullptr;
2255 ContextNode *CurNode = nullptr;
2256 DenseSet<const ContextEdge *> VisitedEdges;
2257 ArrayRef<uint64_t> AllStackIds(Info.StackIds);
2258 // Try to identify what callsite ContextNode maps to which slice of the
2259 // context's ordered stack ids.
2260 for (unsigned I = 0; I < AllStackIds.size(); I++, PrevNode = CurNode) {
2261 // We will do this greedily, trying up to MaxLength stack ids in a row, to
2262 // see if we recorded a context node for that sequence.
2263 auto Len = Info.MaxLength;
2264 auto LenToEnd = AllStackIds.size() - I;
2265 if (Len > LenToEnd)
2266 Len = LenToEnd;
2267 CurNode = nullptr;
2268 // Try to find a recorded context node starting with the longest length
2269 // recorded, and on down until we check for just a single stack node.
2270 for (; Len > 0; Len--) {
2271 // Get the slice of the original stack id sequence to check.
2272 auto CheckStackIds = AllStackIds.slice(I, Len);
2273 auto EntryIt = Info.StackIdsToNode.find(CheckStackIds);
2274 if (EntryIt == Info.StackIdsToNode.end())
2275 continue;
2276 CurNode = EntryIt->second;
2277 // Skip forward so we don't try to look for the ones we just matched.
2278 // We increment by Len - 1, because the outer for loop will increment I.
2279 I += Len - 1;
2280 break;
2281 }
2282 // Give up if we couldn't find a node. Since we need to clone from the
2283 // leaf allocation upwards, no sense in doing anymore fixup further up
2284 // the context if we couldn't match part of the original stack context
2285 // onto a callsite node.
2286 if (!CurNode)
2287 break;
2288 // No edges to fix up until we have a pair of nodes that should be
2289 // adjacent in the graph.
2290 if (!PrevNode)
2291 continue;
2292 // See if we already have a call edge from CurNode to PrevNode.
2293 auto *CurEdge = PrevNode->findEdgeFromCaller(CurNode);
2294 if (CurEdge) {
2295 // We already have an edge. Make sure it contains this context id.
2296 if (CurEdge->getContextIds().insert(CurContextId).second) {
2297 NumFixupEdgeIdsInserted++;
2298 Changed = true;
2299 }
2300 } else {
2301 // No edge exists - add one.
2302 NumFixupEdgesAdded++;
2303 DenseSet<uint32_t> ContextIds({CurContextId});
2304 auto AllocType = computeAllocType(ContextIds);
2305 auto NewEdge = std::make_shared<ContextEdge>(
2306 PrevNode, CurNode, AllocType, std::move(ContextIds));
2307 PrevNode->CallerEdges.push_back(NewEdge);
2308 CurNode->CalleeEdges.push_back(NewEdge);
2309 // Save the new edge for the below handling.
2310 CurEdge = NewEdge.get();
2311 Changed = true;
2312 }
2313 VisitedEdges.insert(CurEdge);
2314 // Now remove this context id from any other caller edges calling
2315 // PrevNode.
2316 for (auto &Edge : PrevNode->CallerEdges) {
2317 // Skip the edge updating/created above and edges we have already
2318 // visited (due to recursion).
2319 if (Edge.get() != CurEdge && !VisitedEdges.contains(Edge.get()))
2320 Edge->getContextIds().erase(CurContextId);
2321 }
2322 }
2323 if (Changed)
2324 NumFixedContexts++;
2325 }
2326}
2327
2328// Returns the ids of contexts whose stacks end partway through the given
2329// call's inlined sequence of stack nodes: those that follow the sequence from
2330// StackNodes[0] (the innermost inlined callee frame) through some node before
2331// the last, and were trimmed at that node, i.e. have no caller edge out of it.
2332// For example, given nodes [A, B, C], a context whose stack ends at A, or that
2333// follows A->B and ends at B, is returned, but one that follows A->B->C (which
2334// is assigned to the call normally) or A->D is not.
2335//
2336// Nothing is returned if the sequence contains a recursive node before the
2337// last, as updateStackNodes does not assign such calls a node at all.
2338template <typename DerivedCCG, typename FuncTy, typename CallTy>
2339DenseSet<uint32_t> CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
2340 computeTrimmedInlinedContextIds(ArrayRef<ContextNode *> StackNodes,
2341 TrimmedAtNodeCache &Cache) {
2342 assert(StackNodes.size() >= 2);
2343 DenseSet<uint32_t> Result;
2344
2345 // Returns the index in the cache of the ids of contexts trimmed at the given
2346 // node. This only depends on the node, which is often shared by many calls'
2347 // inlined sequences, so it is computed once per node.
2348 auto GetTrimmedAtNodeIndex = [&Cache](const ContextNode *Node) {
2349 auto [It, Inserted] =
2350 Cache.NodeToIndex.try_emplace(Node, Cache.TrimmedAtNodeIds.size());
2351 unsigned Index = It->second;
2352 if (!Inserted)
2353 return Index;
2354 // All of the node's context ids flow in through its callee edges. Those
2355 // not flowing out through a caller edge were trimmed here.
2356 DenseSet<uint32_t> &Trimmed = Cache.TrimmedAtNodeIds.emplace_back();
2357 for (const auto &CalleeEdge : Node->CalleeEdges)
2358 Trimmed.insert_range(CalleeEdge->getContextIds());
2359 for (const auto &CallerEdge : Node->CallerEdges) {
2360 set_subtract(Trimmed, CallerEdge->getContextIds());
2361 if (Trimmed.empty())
2362 break;
2363 }
2364 return Index;
2365 };
2366
2367 // Collect the cache index for each node but the last. Skip the walk
2368 // entirely if no contexts were trimmed at any of them (the common case).
2369 SmallVector<unsigned, 8> TrimmedAtNodeIdsIndices;
2370 bool AnyTrimmed = false;
2371 for (ContextNode *Node : StackNodes.drop_back()) {
2372 assert(Node);
2373 if (Node->Recursive)
2374 return Result;
2375 unsigned Index = GetTrimmedAtNodeIndex(Node);
2376 AnyTrimmed |= !Cache.TrimmedAtNodeIds[Index].empty();
2377 TrimmedAtNodeIdsIndices.push_back(Elt: Index);
2378 }
2379 if (!AnyTrimmed)
2380 return Result;
2381
2382 // Walk the sequence from the innermost node outward, tracking in
2383 // PrefixContextIds the contexts that follow the call's stack nodes from the
2384 // first through the current one. Any of those contexts that were trimmed at
2385 // the current node end partway through the call's sequence, so add them to
2386 // Result. All contexts on the first node trivially follow the sequence up to
2387 // it, so start with all those trimmed there, and start PrefixContextIds with
2388 // the ids on the edge out of the first node (rather than the possibly very
2389 // large set of ids on the first node itself).
2390 //
2391 // If an edge is missing between two of the nodes, the contexts trimmed at
2392 // the nodes before it (in particular all those trimmed at the first node)
2393 // are deliberately kept, even though updateStackNodes does not assign such
2394 // a call any contexts otherwise: a trimmed context is unambiguous for every
2395 // continuation, including an inlined sequence never observed in the profile.
2396 Result = Cache.TrimmedAtNodeIds[TrimmedAtNodeIdsIndices[0]];
2397 DenseSet<uint32_t> PrefixContextIds;
2398 for (unsigned I = 1; I < TrimmedAtNodeIdsIndices.size(); I++) {
2399 auto *Edge = StackNodes[I]->findEdgeFromCallee(StackNodes[I - 1]);
2400 if (!Edge)
2401 break;
2402 if (I == 1)
2403 PrefixContextIds = Edge->getContextIds();
2404 else
2405 set_intersect(PrefixContextIds, Edge->getContextIds());
2406 if (PrefixContextIds.empty())
2407 break;
2408 Result.insert_range(set_intersection(
2409 PrefixContextIds, Cache.TrimmedAtNodeIds[TrimmedAtNodeIdsIndices[I]]));
2410 }
2411 return Result;
2412}
2413
2414template <typename DerivedCCG, typename FuncTy, typename CallTy>
2415void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::updateStackNodes() {
2416 // Map of stack id to all calls with that as the last (outermost caller)
2417 // callsite id that has a context node (some might not due to trimming
2418 // performed during matching of the allocation profile contexts).
2419 // The CallContextInfo contains the Call and a list of the context nodes for
2420 // its stack ids, the function containing Call, and the set of context ids
2421 // the analysis will eventually identify for use in any new node created
2422 // for that callsite.
2423 DenseMap<uint64_t, std::vector<CallContextInfo>> StackIdToMatchingCalls;
2424 for (auto &[Func, CallsWithMetadata] : FuncToCallsWithMetadata) {
2425 for (auto &Call : CallsWithMetadata) {
2426 // Ignore allocations, already handled.
2427 if (AllocationCallToContextNodeMap.count(Call))
2428 continue;
2429 std::vector<ContextNode *> StackNodes = getStackNodesForCall(Call: Call.call());
2430 // If there were no nodes created for MIBs on allocs (maybe this was in
2431 // the unambiguous part of the MIB stack that was trimmed), ignore.
2432 if (StackNodes.empty())
2433 continue;
2434 // Otherwise, record this Call along with the list of nodes for the last
2435 // (outermost caller) stack id with a node.
2436 uint64_t LastId = StackNodes.back()->OrigStackOrAllocId;
2437 StackIdToMatchingCalls[LastId].push_back(
2438 {Call.call(), std::move(StackNodes), Func, {}, {}});
2439 }
2440 }
2441
2442 // In order to do the best and maximal matching of inlined calls to context
2443 // node sequences, sort the calls with each last stack id by the length of
2444 // their stack node sequences in descending order, and within each length,
2445 // lexicographically by stack id. The latter is so that we can specially
2446 // handle calls that have identical stack id sequences (either due to cloning
2447 // or artificially because of the MIB context trimming). Those with the same
2448 // stack ids are then sorted by function to facilitate efficiently mapping
2449 // them to the same context node. Because the functions are pointers, to
2450 // ensure a stable sort first assign each function pointer to its first index
2451 // in the Calls array, and then use that to sort by. The stack nodes are
2452 // compared by stack id rather than by pointer, for a deterministic order.
2453 auto StackIdsLess = [](ArrayRef<ContextNode *> A, ArrayRef<ContextNode *> B) {
2454 return std::lexicographical_compare(
2455 A.begin(), A.end(), B.begin(), B.end(),
2456 [](const ContextNode *X, const ContextNode *Y) {
2457 return X->OrigStackOrAllocId < Y->OrigStackOrAllocId;
2458 });
2459 };
2460 // Save a map from each call to any that are found to match it. I.e. located
2461 // in the same function and have the same (possibly trimmed) stack ids. We use
2462 // this to avoid creating extra graph nodes as they can be treated the same.
2463 DenseMap<CallInfo, CallInfo> CallToMatchingCall;
2464 for (auto &Entry : StackIdToMatchingCalls) {
2465 auto &Calls = Entry.second;
2466 if (Calls.size() == 1)
2467 continue;
2468 DenseMap<const FuncTy *, unsigned> FuncToIndex;
2469 for (const auto &[Idx, CallCtxInfo] : enumerate(Calls))
2470 FuncToIndex.insert({CallCtxInfo.Func, Idx});
2471 auto CallLess = [&FuncToIndex, &StackIdsLess](const CallContextInfo &A,
2472 const CallContextInfo &B) {
2473 return A.StackNodes.size() > B.StackNodes.size() ||
2474 (A.StackNodes.size() == B.StackNodes.size() &&
2475 (StackIdsLess(A.StackNodes, B.StackNodes) ||
2476 (A.StackNodes == B.StackNodes &&
2477 FuncToIndex[A.Func] < FuncToIndex[B.Func])));
2478 };
2479 llvm::stable_sort(Calls, CallLess);
2480 // Matching calls are now adjacent. Record each run's calls after the first
2481 // as matching it. Comparing the stack nodes is equivalent to comparing the
2482 // stack ids, as each stack id has its own node and the stack id to node
2483 // map is not modified here.
2484 unsigned FirstInRun = 0;
2485 for (unsigned I = 1; I < Calls.size(); I++) {
2486 if (Calls[I].Func != Calls[FirstInRun].Func ||
2487 Calls[I].StackNodes != Calls[FirstInRun].StackNodes) {
2488 FirstInRun = I;
2489 continue;
2490 }
2491 CallToMatchingCall[Calls[I].Call] = Calls[FirstInRun].Call;
2492 }
2493 }
2494
2495 // First make a pass through all stack ids that correspond to a call,
2496 // as identified in the above loop. Compute the context ids corresponding to
2497 // each of these calls when they correspond to multiple stack ids due to
2498 // due to inlining. Perform any duplication of context ids required when
2499 // there is more than one call with the same stack ids. Their (possibly newly
2500 // duplicated) context ids are saved in the StackIdToMatchingCalls map.
2501 DenseMap<uint32_t, DenseSet<uint32_t>> OldToNewContextIds;
2502
2503 // First, give each call with inlined frames duplicates of the ids of any
2504 // contexts trimmed at a stack id before the last one in its sequence (see
2505 // computeTrimmedInlinedContextIds), saved in its TrimmedInlinedDupIds.
2506 // Those contexts are also assigned, with their original ids, to calls whose
2507 // last stack id with a node is the earlier id, so this call gets its own
2508 // copies. This pass only reads the graph, as does the main loop below (the
2509 // duplicates are only added to the graph afterwards, by
2510 // propagateDuplicateContextIds), so it can run first; that lets the main
2511 // loop hand these duplicates to the other calls that should also get them
2512 // (see AddTrimmedInlinedDups).
2513
2514 // The stack nodes of each call that duplicates were created for. These
2515 // reference the vectors in StackIdToMatchingCalls, which is not modified
2516 // after this point.
2517 std::vector<ArrayRef<ContextNode *>> TrimmedInlinedDupCreatorNodes;
2518 // Map from each original trimmed context id to its duplicates, each paired
2519 // with the index in TrimmedInlinedDupCreatorNodes of the creating call.
2520 DenseMap<uint32_t, SmallVector<std::pair<uint32_t, unsigned>, 1>>
2521 OrigToTrimmedInlinedDups;
2522 if (AssignTrimmedInlinedContexts) {
2523 TrimmedAtNodeCache Cache;
2524 for (auto &Entry : StackIdToMatchingCalls) {
2525 auto &Calls = Entry.second;
2526 // The main loop below skips these, see the comments there. All calls in
2527 // this entry share the same last node.
2528 if (Calls[0].StackNodes.back()->Recursive)
2529 continue;
2530 for (unsigned I = 0; I < Calls.size(); I++) {
2531 auto &CallCtxInfo = Calls[I];
2532 auto &StackNodes = CallCtxInfo.StackNodes;
2533 // Calls are sorted longest sequence first, so no later call has
2534 // inlined frames either.
2535 if (StackNodes.size() < 2)
2536 break;
2537 // Calls matching an earlier call in the same function will share its
2538 // context node, so only give duplicates to the first.
2539 if (CallToMatchingCall.contains(CallCtxInfo.Call))
2540 continue;
2541 auto Trimmed = computeTrimmedInlinedContextIds(StackNodes, Cache);
2542 if (Trimmed.empty())
2543 continue;
2544 unsigned CreatorIdx = TrimmedInlinedDupCreatorNodes.size();
2545 TrimmedInlinedDupCreatorNodes.push_back(StackNodes);
2546 CallCtxInfo.TrimmedInlinedDupIds = duplicateContextIds(
2547 StackSequenceContextIds: Trimmed, OldToNewContextIds, OnDuplicate: [&](uint32_t OldId, uint32_t NewId) {
2548 OrigToTrimmedInlinedDups[OldId].push_back(Elt: {NewId, CreatorIdx});
2549 });
2550 NumCallsAssignedTrimmedInlinedContexts++;
2551 NumTrimmedInlinedContextIds += Trimmed.size();
2552 }
2553 }
2554 }
2555
2556 // Adds to the given saved context ids, for a call with the given last stack
2557 // node, the trimmed inlined duplicates of any of those ids created for other
2558 // calls. The saved context ids are computed from the graph, which does not
2559 // include the duplicates until propagateDuplicateContextIds below adds them
2560 // to all of the same nodes and edges as the originals. A call assigned an
2561 // original should also be assigned its duplicates; otherwise, when that
2562 // call's new node takes the originals, the duplicates would be left behind
2563 // on the original stack nodes, which have no call to clone.
2564 //
2565 // TODO: The saved context ids may themselves be duplicates (created for a
2566 // call with the same stack ids in a different function), in which case the
2567 // original context id is not found here and the trimmed inlined duplicates
2568 // are not added. In the future, handle that by looking up the originals of
2569 // the saved ids.
2570 auto AddTrimmedInlinedDups = [&](DenseSet<uint32_t> &SavedContextIds,
2571 const ContextNode *LastNode) {
2572 SmallVector<uint32_t> DupIdsToAdd;
2573 // Collects the duplicates of one original id that this call should get.
2574 auto AddDups = [&](ArrayRef<std::pair<uint32_t, unsigned>> Dups) {
2575 for (auto [DupId, CreatorIdx] : Dups) {
2576 ArrayRef<ContextNode *> CreatorNodes =
2577 TrimmedInlinedDupCreatorNodes[CreatorIdx];
2578 // Skip the duplicate if this call ends at any node of the creating
2579 // call's sequence. Such a call overlaps the stretch of the context
2580 // the creating call needs the duplicate on, and could claim the
2581 // duplicate first (the post order processes the creating call first
2582 // only when all edges along its sequence exist); being assigned the
2583 // original there is enough for it. Calls ending below the creating
2584 // call's first node only cover the callee side of the context, and
2585 // leave the duplicate on that first node for the creating call.
2586 if (is_contained(CreatorNodes, LastNode))
2587 continue;
2588 DupIdsToAdd.push_back(Elt: DupId);
2589 }
2590 };
2591 // Iterate over the smaller of the two.
2592 if (OrigToTrimmedInlinedDups.size() < SavedContextIds.size()) {
2593 for (auto &[OrigId, Dups] : OrigToTrimmedInlinedDups)
2594 if (SavedContextIds.contains(V: OrigId))
2595 AddDups(Dups);
2596 } else {
2597 for (auto Id : SavedContextIds) {
2598 auto It = OrigToTrimmedInlinedDups.find(Val: Id);
2599 if (It != OrigToTrimmedInlinedDups.end())
2600 AddDups(It->second);
2601 }
2602 }
2603 SavedContextIds.insert_range(R&: DupIdsToAdd);
2604 };
2605 for (auto &It : StackIdToMatchingCalls) {
2606 auto &Calls = It.getSecond();
2607 // Skip single calls with a single stack id. These don't need a new node.
2608 if (Calls.size() == 1) {
2609 auto &StackNodes = Calls[0].StackNodes;
2610 if (StackNodes.size() == 1)
2611 continue;
2612 }
2613
2614 // Find the node for the last stack id, which should be the same
2615 // across all calls recorded for this id, and is the id for this
2616 // entry in the StackIdToMatchingCalls map.
2617 uint64_t LastId = It.getFirst();
2618 // All calls in this entry share the same last node.
2619 ContextNode *LastNode = Calls[0].StackNodes.back();
2620 // We should only have kept stack ids that had nodes.
2621 assert(LastNode);
2622 assert(LastNode == getNodeForStackId(LastId));
2623
2624 if (LastNode->Recursive)
2625 continue;
2626
2627 // Initialize the context ids with the last node's. We will subsequently
2628 // refine the context ids by computing the intersection along all edges.
2629 DenseSet<uint32_t> LastNodeContextIds = LastNode->getContextIds();
2630 assert(!LastNodeContextIds.empty());
2631
2632#ifndef NDEBUG
2633 // Save the set of functions seen for a particular set of the same stack
2634 // ids. This is used to ensure that they have been correctly sorted to be
2635 // adjacent in the Calls list, since we rely on that to efficiently place
2636 // all such matching calls onto the same context node.
2637 DenseSet<const FuncTy *> MatchingIdsFuncSet;
2638#endif
2639
2640 for (unsigned I = 0; I < Calls.size(); I++) {
2641 auto &[Call, StackNodes, Func, SavedContextIds, TrimmedInlinedDupIds] =
2642 Calls[I];
2643 assert(SavedContextIds.empty());
2644 assert(LastNode == StackNodes.back());
2645
2646#ifndef NDEBUG
2647 // If this call has a different set of ids than the last one, clear the
2648 // set used to ensure they are sorted properly.
2649 if (I > 0 && StackNodes != Calls[I - 1].StackNodes)
2650 MatchingIdsFuncSet.clear();
2651#endif
2652
2653 // Skip the calls matching this one (recorded in CallToMatchingCall when
2654 // sorting above), which are adjacent to it, so that they get assigned to
2655 // the same context node. Also note whether a call in a different
2656 // function has the same stack ids.
2657 bool DuplicateContextIds = false;
2658 for (unsigned J = I + 1; J < Calls.size(); J++) {
2659 auto &CallCtxInfo = Calls[J];
2660 if (CallCtxInfo.StackNodes != StackNodes)
2661 break;
2662 auto *NextFunc = CallCtxInfo.Func;
2663 if (NextFunc != Func) {
2664 // We have another Call with the same ids but that cannot share this
2665 // node, must duplicate ids for it.
2666 DuplicateContextIds = true;
2667 break;
2668 }
2669 assert(CallToMatchingCall.lookup(CallCtxInfo.Call).call() == Call);
2670 // Update I so that it gets incremented correctly to skip this call.
2671 I = J;
2672 }
2673
2674 // First compute the context ids for this stack id sequence (the
2675 // intersection of the context ids of the corresponding nodes).
2676 // Start with the remaining saved ids for the last node.
2677 assert(!LastNodeContextIds.empty());
2678 DenseSet<uint32_t> StackSequenceContextIds = LastNodeContextIds;
2679
2680 ContextNode *PrevNode = LastNode;
2681 ContextNode *CurNode = LastNode;
2682 bool Skip = false;
2683
2684 // Iterate backwards through the stack nodes, starting after the last
2685 // one in the list, which was handled once outside for all Calls.
2686 for (auto NodeIter = StackNodes.rbegin() + 1;
2687 NodeIter != StackNodes.rend(); NodeIter++) {
2688 CurNode = *NodeIter;
2689 // We should only have kept stack ids that had nodes.
2690 assert(CurNode);
2691
2692 if (CurNode->Recursive) {
2693 Skip = true;
2694 break;
2695 }
2696
2697 auto *Edge = CurNode->findEdgeFromCaller(PrevNode);
2698 // If there is no edge then the nodes belong to different MIB contexts,
2699 // and we should skip this inlined context sequence. For example, this
2700 // particular inlined context may include stack ids A->B, and we may
2701 // indeed have nodes for both A and B, but it is possible that they were
2702 // never profiled in sequence in a single MIB for any allocation (i.e.
2703 // we might have profiled an allocation that involves the callsite A,
2704 // but through a different one of its callee callsites, and we might
2705 // have profiled an allocation that involves callsite B, but reached
2706 // from a different caller callsite).
2707 if (!Edge) {
2708 Skip = true;
2709 break;
2710 }
2711 PrevNode = CurNode;
2712
2713 // Update the context ids, which is the intersection of the ids along
2714 // all edges in the sequence.
2715 set_intersect(StackSequenceContextIds, Edge->getContextIds());
2716
2717 // If we now have no context ids for clone, skip this call.
2718 if (StackSequenceContextIds.empty()) {
2719 Skip = true;
2720 break;
2721 }
2722 }
2723 if (Skip)
2724 continue;
2725
2726 // If some of this call's stack ids did not have corresponding nodes (due
2727 // to trimming), don't include any context ids for contexts that extend
2728 // beyond these nodes. Otherwise we would be matching part of unrelated /
2729 // not fully matching stack contexts. To do this, subtract any context ids
2730 // found in caller nodes of the last node found above.
2731 if (LastId != getLastStackId(Call)) {
2732 for (const auto &PE : LastNode->CallerEdges) {
2733 set_subtract(StackSequenceContextIds, PE->getContextIds());
2734 if (StackSequenceContextIds.empty())
2735 break;
2736 }
2737 // If we now have no context ids for clone, skip this call.
2738 if (StackSequenceContextIds.empty())
2739 continue;
2740 }
2741
2742#ifndef NDEBUG
2743 // If the prior call had the same stack ids this set would not be empty.
2744 // Check if we already have a call that "matches" because it is located
2745 // in the same function. If the Calls list was sorted properly we should
2746 // not encounter this situation as all such entries should be adjacent
2747 // and processed in bulk further below.
2748 assert(!MatchingIdsFuncSet.contains(Func));
2749
2750 MatchingIdsFuncSet.insert(Func);
2751#endif
2752
2753 // If we don't have duplicate context ids, then we can assign all the
2754 // context ids computed for the original node sequence to this call.
2755 // If there are duplicate calls with the same stack ids then we synthesize
2756 // new context ids that are duplicates of the originals. These are
2757 // assigned to SavedContextIds, which is a reference into the map entry
2758 // for this call, allowing us to access these ids later on.
2759 OldToNewContextIds.reserve(NumEntries: OldToNewContextIds.size() +
2760 StackSequenceContextIds.size());
2761 SavedContextIds =
2762 DuplicateContextIds
2763 ? duplicateContextIds(StackSequenceContextIds, OldToNewContextIds)
2764 : StackSequenceContextIds;
2765 assert(!SavedContextIds.empty());
2766 if (!OrigToTrimmedInlinedDups.empty())
2767 AddTrimmedInlinedDups(SavedContextIds, LastNode);
2768
2769 if (!DuplicateContextIds) {
2770 // Update saved last node's context ids to remove those that are
2771 // assigned to other calls, so that it is ready for the next call at
2772 // this stack id.
2773 set_subtract(S1&: LastNodeContextIds, S2: StackSequenceContextIds);
2774 if (LastNodeContextIds.empty())
2775 break;
2776 }
2777 }
2778 }
2779
2780 // Propagate the duplicate context ids over the graph.
2781 propagateDuplicateContextIds(OldToNewContextIds);
2782
2783 if (VerifyCCG)
2784 check();
2785
2786 // Now perform a post-order traversal over the graph, starting with the
2787 // allocation nodes, essentially processing nodes from callers to callees.
2788 // For any that contains an id in the map, update the graph to contain new
2789 // nodes representing any inlining at interior callsites. Note we move the
2790 // associated context ids over to the new nodes.
2791 DenseSet<const ContextNode *> Visited;
2792 DenseSet<uint32_t> ImportantContextIds(llvm::from_range,
2793 ImportantContextIdInfo.keys());
2794 for (auto &Entry : AllocationCallToContextNodeMap)
2795 assignStackNodesPostOrder(Node: Entry.second, Visited, StackIdToMatchingCalls,
2796 CallToMatchingCall, ImportantContextIds);
2797
2798 fixupImportantContexts();
2799
2800 if (VerifyCCG)
2801 check();
2802}
2803
2804uint64_t ModuleCallsiteContextGraph::getLastStackId(Instruction *Call) {
2805 CallStack<MDNode, MDNode::op_iterator> CallsiteContext(
2806 Call->getMetadata(KindID: LLVMContext::MD_callsite));
2807 return CallsiteContext.back();
2808}
2809
2810uint64_t IndexCallsiteContextGraph::getLastStackId(IndexCall &Call) {
2811 assert(isa<CallsiteInfo *>(Call));
2812 CallStack<CallsiteInfo, SmallVector<unsigned>::const_iterator>
2813 CallsiteContext(dyn_cast_if_present<CallsiteInfo *>(Val&: Call));
2814 // Need to convert index into stack id.
2815 return Index.getStackIdAtIndex(Index: CallsiteContext.back());
2816}
2817
2818static std::string getMemProfFuncName(Twine Base, unsigned CloneNo) {
2819 // We use CloneNo == 0 to refer to the original version, which doesn't get
2820 // renamed with a suffix.
2821 if (!CloneNo)
2822 return Base.str();
2823 return (Base + MemProfCloneSuffix + Twine(CloneNo)).str();
2824}
2825
2826// Return the clone number of the given function by extracting it from the
2827// memprof suffix. Assumes the caller has already confirmed it is a memprof
2828// clone.
2829static unsigned getMemProfCloneNum(const Function &F) {
2830 assert(isMemProfClone(F));
2831 auto Pos = F.getName().find_last_of(C: '.');
2832 assert(Pos > 0);
2833 unsigned CloneNo;
2834 bool Err = F.getName().drop_front(N: Pos + 1).getAsInteger(Radix: 10, Result&: CloneNo);
2835 assert(!Err);
2836 (void)Err;
2837 return CloneNo;
2838}
2839
2840std::string ModuleCallsiteContextGraph::getLabel(const Function *Func,
2841 const Instruction *Call,
2842 unsigned CloneNo) const {
2843 return (Twine(Call->getFunction()->getName()) + " -> " +
2844 cast<CallBase>(Val: Call)->getCalledFunction()->getName())
2845 .str();
2846}
2847
2848std::string IndexCallsiteContextGraph::getLabel(const FunctionSummary *Func,
2849 const IndexCall &Call,
2850 unsigned CloneNo) const {
2851 auto VI = FSToVIMap.find(x: Func);
2852 assert(VI != FSToVIMap.end());
2853 std::string CallerName = getMemProfFuncName(Base: VI->second.name(), CloneNo);
2854 if (isa<AllocInfo *>(Val: Call))
2855 return CallerName + " -> alloc";
2856 else {
2857 auto *Callsite = dyn_cast_if_present<CallsiteInfo *>(Val: Call);
2858 return CallerName + " -> " +
2859 getMemProfFuncName(Base: Callsite->Callee.name(),
2860 CloneNo: Callsite->Clones[CloneNo]);
2861 }
2862}
2863
2864std::string ModuleCallsiteContextGraph::getFuncName(const Function *Func,
2865 unsigned CloneNo) const {
2866 if (!Func)
2867 return "<unknown>";
2868 return getMemProfFuncName(Base: Func->getName(), CloneNo);
2869}
2870
2871void ModuleCallsiteContextGraph::populateFunctionsWithCallers() {
2872 // Any use (direct call, address taken, alias, etc.) means the function is
2873 // referenced and may be reached by callers above the profiled context.
2874 for (const auto &F : Mod)
2875 if (!F.use_empty())
2876 FunctionsWithCallers.insert(V: &F);
2877}
2878
2879std::string IndexCallsiteContextGraph::getFuncName(const FunctionSummary *Func,
2880 unsigned CloneNo) const {
2881 if (!Func)
2882 return "<unknown>";
2883 auto VI = FSToVIMap.find(x: Func);
2884 if (VI != FSToVIMap.end()) {
2885 if (!VI->second.name().empty())
2886 return getMemProfFuncName(Base: VI->second.name(), CloneNo);
2887 return getMemProfFuncName(Base: "GUID:" + std::to_string(val: VI->second.getGUID()),
2888 CloneNo);
2889 }
2890 return getMemProfFuncName(Base: "<unknown>", CloneNo);
2891}
2892
2893bool IndexCallsiteContextGraph::isFuncDead(const FunctionSummary *Func) const {
2894 return Index.withGlobalValueDeadStripping() && Func && !Func->isLive();
2895}
2896
2897void IndexCallsiteContextGraph::populateFunctionsWithCallers() {
2898 for (const auto &I : Index) {
2899 auto VI = Index.getValueInfo(R: I);
2900 for (const auto &S : VI.getSummaryList()) {
2901 if (!GlobalValue::isLocalLinkage(Linkage: S->linkage()) &&
2902 !isPrevailing(VI.getGUID(), S.get()))
2903 continue;
2904 if (Index.withGlobalValueDeadStripping() && !S->isLive())
2905 continue;
2906 auto AddReferenced = [&](ValueInfo RefVI) {
2907 for (const auto &RefS : RefVI.getSummaryList()) {
2908 const GlobalValueSummary *Base = RefS->getBaseObject();
2909 if (const auto *RefFS = dyn_cast<FunctionSummary>(Val: Base))
2910 FunctionsWithCallers.insert(V: RefFS);
2911 }
2912 };
2913 // Address-taken references (e.g. from vtables or function pointers).
2914 for (const auto &RefVI : S->refs())
2915 AddReferenced(RefVI);
2916 // Direct calls.
2917 if (const auto *FS = dyn_cast<FunctionSummary>(Val: S.get()))
2918 for (const auto &CallEdge : FS->calls())
2919 AddReferenced(CallEdge.first);
2920 }
2921 }
2922}
2923
2924std::vector<ModuleCallsiteContextGraph::ContextNode *>
2925ModuleCallsiteContextGraph::getStackNodesForCall(Instruction *Call) {
2926 CallStack<MDNode, MDNode::op_iterator> CallsiteContext(
2927 Call->getMetadata(KindID: LLVMContext::MD_callsite));
2928 return getStackNodesForCallsite<MDNode, MDNode::op_iterator>(CallsiteContext);
2929}
2930
2931std::vector<IndexCallsiteContextGraph::ContextNode *>
2932IndexCallsiteContextGraph::getStackNodesForCall(IndexCall &Call) {
2933 assert(isa<CallsiteInfo *>(Call));
2934 CallStack<CallsiteInfo, SmallVector<unsigned>::const_iterator>
2935 CallsiteContext(dyn_cast_if_present<CallsiteInfo *>(Val&: Call));
2936 return getStackNodesForCallsite<CallsiteInfo,
2937 SmallVector<unsigned>::const_iterator>(
2938 CallsiteContext);
2939}
2940
2941template <typename DerivedCCG, typename FuncTy, typename CallTy>
2942template <class NodeT, class IteratorT>
2943std::vector<
2944 typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *>
2945CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::getStackNodesForCallsite(
2946 CallStack<NodeT, IteratorT> &CallsiteContext) {
2947 std::vector<ContextNode *> StackNodes;
2948 for (auto IdOrIndex : CallsiteContext) {
2949 auto StackId = getStackId(IdOrIndex);
2950 ContextNode *Node = getNodeForStackId(StackId);
2951 if (!Node)
2952 break;
2953 StackNodes.push_back(Node);
2954 }
2955 return StackNodes;
2956}
2957
2958ModuleCallsiteContextGraph::ModuleCallsiteContextGraph(
2959 Module &M,
2960 llvm::function_ref<OptimizationRemarkEmitter &(Function *)> OREGetter)
2961 : Mod(M), OREGetter(OREGetter) {
2962 // Map for keeping track of the largest cold contexts up to the number given
2963 // by MemProfTopNImportant. Must be a std::map (not DenseMap) because keys
2964 // must be sorted.
2965 std::map<uint64_t, uint32_t> TotalSizeToContextIdTopNCold;
2966 for (auto &F : M) {
2967 std::vector<CallInfo> CallsWithMetadata;
2968 for (auto &BB : F) {
2969 for (auto &I : BB) {
2970 if (!isa<CallBase>(Val: I))
2971 continue;
2972 if (auto *MemProfMD = I.getMetadata(KindID: LLVMContext::MD_memprof)) {
2973 CallsWithMetadata.push_back(x: &I);
2974 auto *AllocNode = addAllocNode(Call: &I, F: &F);
2975 auto *CallsiteMD = I.getMetadata(KindID: LLVMContext::MD_callsite);
2976 assert(CallsiteMD);
2977 CallStack<MDNode, MDNode::op_iterator> CallsiteContext(CallsiteMD);
2978 // Add all of the MIBs and their stack nodes.
2979 for (auto &MDOp : MemProfMD->operands()) {
2980 auto *MIBMD = cast<const MDNode>(Val: MDOp);
2981 std::vector<ContextTotalSize> ContextSizeInfo;
2982 // Collect the context size information if it exists.
2983 if (MIBMD->getNumOperands() > 2) {
2984 for (unsigned I = 2; I < MIBMD->getNumOperands(); I++) {
2985 MDNode *ContextSizePair =
2986 dyn_cast<MDNode>(Val: MIBMD->getOperand(I));
2987 assert(ContextSizePair->getNumOperands() == 2);
2988 uint64_t FullStackId = mdconst::dyn_extract<ConstantInt>(
2989 MD: ContextSizePair->getOperand(I: 0))
2990 ->getZExtValue();
2991 uint64_t TotalSize = mdconst::dyn_extract<ConstantInt>(
2992 MD: ContextSizePair->getOperand(I: 1))
2993 ->getZExtValue();
2994 ContextSizeInfo.push_back(x: {.FullStackId: FullStackId, .TotalSize: TotalSize});
2995 }
2996 }
2997 MDNode *StackNode = getMIBStackNode(MIB: MIBMD);
2998 assert(StackNode);
2999 CallStack<MDNode, MDNode::op_iterator> StackContext(StackNode);
3000 addStackNodesForMIB<MDNode, MDNode::op_iterator>(
3001 AllocNode, StackContext, CallsiteContext,
3002 AllocType: getMIBAllocType(MIB: MIBMD), ContextSizeInfo,
3003 TotalSizeToContextIdTopNCold);
3004 }
3005 // If exporting the graph to dot and an allocation id of interest was
3006 // specified, record all the context ids for this allocation node.
3007 if (ExportToDot && AllocNode->OrigStackOrAllocId == AllocIdForDot)
3008 DotAllocContextIds = AllocNode->getContextIds();
3009 assert(AllocNode->AllocTypes != (uint8_t)AllocationType::None);
3010 // Memprof and callsite metadata on memory allocations no longer
3011 // needed.
3012 I.setMetadata(KindID: LLVMContext::MD_memprof, Node: nullptr);
3013 I.setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
3014 }
3015 // For callsite metadata, add to list for this function for later use.
3016 else if (I.getMetadata(KindID: LLVMContext::MD_callsite)) {
3017 CallsWithMetadata.push_back(x: &I);
3018 }
3019 }
3020 }
3021 if (!CallsWithMetadata.empty())
3022 FuncToCallsWithMetadata[&F] = CallsWithMetadata;
3023 }
3024
3025 if (DumpCCG) {
3026 dbgs() << "CCG before updating call stack chains:\n";
3027 dbgs() << *this;
3028 }
3029
3030 if (ExportToDot)
3031 exportToDot(Label: "prestackupdate");
3032
3033 updateStackNodes();
3034
3035 if (ExportToDot)
3036 exportToDot(Label: "poststackupdate");
3037
3038 handleCallsitesWithMultipleTargets();
3039
3040 markBackedges();
3041
3042 // Strip off remaining callsite metadata, no longer needed.
3043 for (auto &FuncEntry : FuncToCallsWithMetadata)
3044 for (auto &Call : FuncEntry.second)
3045 Call.call()->setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
3046}
3047
3048// Finds the set of GUIDs for weak aliasees that are prevailing in different
3049// modules than any of their aliases. We need to handle these specially.
3050DenseSet<GlobalValue::GUID>
3051IndexCallsiteContextGraph::findAliaseeGUIDsPrevailingInDifferentModule() {
3052 DenseSet<GlobalValue::GUID> AliaseeGUIDs;
3053 for (auto &I : Index) {
3054 auto VI = Index.getValueInfo(R: I);
3055 for (auto &S : VI.getSummaryList()) {
3056 // We only care about aliases to functions.
3057 auto *AS = dyn_cast<AliasSummary>(Val: S.get());
3058 if (!AS)
3059 continue;
3060 auto *AliaseeSummary = &AS->getAliasee();
3061 auto *AliaseeFS = dyn_cast<FunctionSummary>(Val: AliaseeSummary);
3062 if (!AliaseeFS)
3063 continue;
3064 // Skip this summary if it is not for the prevailing symbol for this GUID.
3065 // The linker doesn't resolve local linkage values so don't check whether
3066 // those are prevailing.
3067 if (!GlobalValue::isLocalLinkage(Linkage: S->linkage()) &&
3068 !isPrevailing(VI.getGUID(), S.get()))
3069 continue;
3070 // Prevailing aliasee could be in a different module only if it is weak.
3071 if (!GlobalValue::isWeakForLinker(Linkage: AliaseeSummary->linkage()))
3072 continue;
3073 auto AliaseeGUID = AS->getAliaseeGUID();
3074 // If the aliasee copy in this module is not prevailing, record it.
3075 if (!isPrevailing(AliaseeGUID, AliaseeSummary))
3076 AliaseeGUIDs.insert(V: AliaseeGUID);
3077 }
3078 }
3079 AliaseesPrevailingInDiffModuleFromAlias += AliaseeGUIDs.size();
3080 return AliaseeGUIDs;
3081}
3082
3083IndexCallsiteContextGraph::IndexCallsiteContextGraph(
3084 ModuleSummaryIndex &Index,
3085 llvm::function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
3086 isPrevailing)
3087 : Index(Index), isPrevailing(isPrevailing) {
3088 // Since we use the aliasee summary info to create the necessary clones for
3089 // its aliases, conservatively skip recording the aliasee function's callsites
3090 // in the CCG for any that are prevailing in a different module than one of
3091 // its aliases. We could record the necessary information to do this in the
3092 // summary, but this case should not be common.
3093 DenseSet<GlobalValue::GUID> GUIDsToSkip =
3094 findAliaseeGUIDsPrevailingInDifferentModule();
3095 // Map for keeping track of the largest cold contexts up to the number given
3096 // by MemProfTopNImportant. Must be a std::map (not DenseMap) because keys
3097 // must be sorted.
3098 std::map<uint64_t, uint32_t> TotalSizeToContextIdTopNCold;
3099 // Sort by GUID for deterministic graph construction order.
3100 // TODO: This sort has a measurable cost on the thin link when memprof is
3101 // enabled. Investigate gating it behind an option that is only enabled for
3102 // tests that check internal state.
3103 for (const auto &I : Index.sortedGlobalValueSummariesRange()) {
3104 auto VI = Index.getValueInfo(R: I);
3105 if (GUIDsToSkip.contains(V: VI.getGUID()))
3106 continue;
3107 for (auto &S : VI.getSummaryList()) {
3108 // We should only add the prevailing nodes. Otherwise we may try to clone
3109 // in a weak copy that won't be linked (and may be different than the
3110 // prevailing version).
3111 // We only keep the memprof summary on the prevailing copy now when
3112 // building the combined index, as a space optimization, however don't
3113 // rely on this optimization. The linker doesn't resolve local linkage
3114 // values so don't check whether those are prevailing.
3115 if (!GlobalValue::isLocalLinkage(Linkage: S->linkage()) &&
3116 !isPrevailing(VI.getGUID(), S.get()))
3117 continue;
3118 auto *FS = dyn_cast<FunctionSummary>(Val: S.get());
3119 if (!FS)
3120 continue;
3121 std::vector<CallInfo> CallsWithMetadata;
3122 if (!FS->allocs().empty()) {
3123 for (auto &AN : FS->mutableAllocs()) {
3124 // This can happen because of recursion elimination handling that
3125 // currently exists in ModuleSummaryAnalysis. Skip these for now.
3126 // We still added them to the summary because we need to be able to
3127 // correlate properly in applyImport in the backends.
3128 if (AN.MIBs.empty())
3129 continue;
3130 IndexCall AllocCall(&AN);
3131 CallsWithMetadata.push_back(x: AllocCall);
3132 auto *AllocNode = addAllocNode(Call: AllocCall, F: FS);
3133 // Pass an empty CallStack to the CallsiteContext (second)
3134 // parameter, since for ThinLTO we already collapsed out the inlined
3135 // stack ids on the allocation call during ModuleSummaryAnalysis.
3136 CallStack<MIBInfo, SmallVector<unsigned>::const_iterator>
3137 EmptyContext;
3138 unsigned I = 0;
3139 assert(!metadataMayIncludeContextSizeInfo() ||
3140 AN.ContextSizeInfos.size() == AN.MIBs.size());
3141 // Now add all of the MIBs and their stack nodes.
3142 for (auto &MIB : AN.MIBs) {
3143 CallStack<MIBInfo, SmallVector<unsigned>::const_iterator>
3144 StackContext(&MIB);
3145 std::vector<ContextTotalSize> ContextSizeInfo;
3146 if (!AN.ContextSizeInfos.empty()) {
3147 for (auto [FullStackId, TotalSize] : AN.ContextSizeInfos[I])
3148 ContextSizeInfo.push_back(x: {.FullStackId: FullStackId, .TotalSize: TotalSize});
3149 }
3150 addStackNodesForMIB<MIBInfo, SmallVector<unsigned>::const_iterator>(
3151 AllocNode, StackContext, CallsiteContext&: EmptyContext, AllocType: MIB.AllocType,
3152 ContextSizeInfo, TotalSizeToContextIdTopNCold);
3153 I++;
3154 }
3155 // If exporting the graph to dot and an allocation id of interest was
3156 // specified, record all the context ids for this allocation node.
3157 if (ExportToDot && AllocNode->OrigStackOrAllocId == AllocIdForDot)
3158 DotAllocContextIds = AllocNode->getContextIds();
3159 assert(AllocNode->AllocTypes != (uint8_t)AllocationType::None);
3160 // Initialize version 0 on the summary alloc node to the current alloc
3161 // type, unless it has both types in which case make it default, so
3162 // that in the case where we aren't able to clone the original version
3163 // always ends up with the default allocation behavior.
3164 AN.Versions[0] = (uint8_t)allocTypeToUse(AllocTypes: AllocNode->AllocTypes);
3165 }
3166 }
3167 // For callsite metadata, add to list for this function for later use.
3168 if (!FS->callsites().empty())
3169 for (auto &SN : FS->mutableCallsites()) {
3170 IndexCall StackNodeCall(&SN);
3171 CallsWithMetadata.push_back(x: StackNodeCall);
3172 }
3173
3174 if (!CallsWithMetadata.empty())
3175 FuncToCallsWithMetadata[FS] = CallsWithMetadata;
3176
3177 if (!FS->allocs().empty() || !FS->callsites().empty())
3178 FSToVIMap[FS] = VI;
3179 }
3180 }
3181
3182 if (DumpCCG) {
3183 dbgs() << "CCG before updating call stack chains:\n";
3184 dbgs() << *this;
3185 }
3186
3187 if (ExportToDot)
3188 exportToDot(Label: "prestackupdate");
3189
3190 updateStackNodes();
3191
3192 if (ExportToDot)
3193 exportToDot(Label: "poststackupdate");
3194
3195 handleCallsitesWithMultipleTargets();
3196
3197 markBackedges();
3198}
3199
3200template <typename DerivedCCG, typename FuncTy, typename CallTy>
3201void CallsiteContextGraph<DerivedCCG, FuncTy,
3202 CallTy>::handleCallsitesWithMultipleTargets() {
3203 // Look for and workaround callsites that call multiple functions.
3204 // This can happen for indirect calls, which needs better handling, and in
3205 // more rare cases (e.g. macro expansion).
3206 // TODO: To fix this for indirect calls we will want to perform speculative
3207 // devirtualization using either the normal PGO info with ICP, or using the
3208 // information in the profiled MemProf contexts. We can do this prior to
3209 // this transformation for regular LTO, and for ThinLTO we can simulate that
3210 // effect in the summary and perform the actual speculative devirtualization
3211 // while cloning in the ThinLTO backend.
3212
3213 // Keep track of the new nodes synthesized for discovered tail calls missing
3214 // from the profiled contexts.
3215 MapVector<CallInfo, ContextNode *> TailCallToContextNodeMap;
3216
3217 std::vector<std::pair<CallInfo, ContextNode *>> NewCallToNode;
3218 for (auto &Entry : NonAllocationCallToContextNodeMap) {
3219 auto *Node = Entry.second;
3220 assert(Node->Clones.empty());
3221 // Check all node callees and see if in the same function.
3222 // We need to check all of the calls recorded in this Node, because in some
3223 // cases we may have had multiple calls with the same debug info calling
3224 // different callees. This can happen, for example, when an object is
3225 // constructed in the paramter list - the destructor call of the object has
3226 // the same debug info (line/col) as the call the object was passed to.
3227 // Here we will prune any that don't match all callee nodes.
3228 std::vector<CallInfo> AllCalls;
3229 AllCalls.reserve(Node->MatchingCalls.size() + 1);
3230 AllCalls.push_back(Node->Call);
3231 llvm::append_range(AllCalls, Node->MatchingCalls);
3232
3233 // First see if we can partition the calls by callee function, creating new
3234 // nodes to host each set of calls calling the same callees. This is
3235 // necessary for support indirect calls with ThinLTO, for which we
3236 // synthesized CallsiteInfo records for each target. They will all have the
3237 // same callsite stack ids and would be sharing a context node at this
3238 // point. We need to perform separate cloning for each, which will be
3239 // applied along with speculative devirtualization in the ThinLTO backends
3240 // as needed. Note this does not currently support looking through tail
3241 // calls, it is unclear if we need that for indirect call targets.
3242 // First partition calls by callee func. Map indexed by func, value is
3243 // struct with list of matching calls, assigned node.
3244 if (partitionCallsByCallee(Node, AllCalls, NewCallToNode))
3245 continue;
3246
3247 auto It = AllCalls.begin();
3248 // Iterate through the calls until we find the first that matches.
3249 for (; It != AllCalls.end(); ++It) {
3250 auto ThisCall = *It;
3251 bool Match = true;
3252 for (auto EI = Node->CalleeEdges.begin(); EI != Node->CalleeEdges.end();
3253 ++EI) {
3254 auto Edge = *EI;
3255 if (!Edge->Callee->hasCall())
3256 continue;
3257 assert(NodeToCallingFunc.count(Edge->Callee));
3258 // Check if the called function matches that of the callee node.
3259 if (!calleesMatch(Call: ThisCall.call(), EI, TailCallToContextNodeMap)) {
3260 Match = false;
3261 break;
3262 }
3263 }
3264 // Found a call that matches the callee nodes, we can quit now.
3265 if (Match) {
3266 // If the first match is not the primary call on the Node, update it
3267 // now. We will update the list of matching calls further below.
3268 if (Node->Call != ThisCall) {
3269 Node->setCall(ThisCall);
3270 // We need to update the NonAllocationCallToContextNodeMap, but don't
3271 // want to do this during iteration over that map, so save the calls
3272 // that need updated entries.
3273 NewCallToNode.push_back({ThisCall, Node});
3274 }
3275 break;
3276 }
3277 }
3278 // We will update this list below (or leave it cleared if there was no
3279 // match found above).
3280 Node->MatchingCalls.clear();
3281 // If we hit the end of the AllCalls vector, no call matching the callee
3282 // nodes was found, clear the call information in the node.
3283 if (It == AllCalls.end()) {
3284 RemovedEdgesWithMismatchedCallees++;
3285 // Work around by setting Node to have a null call, so it gets
3286 // skipped during cloning. Otherwise assignFunctions will assert
3287 // because its data structures are not designed to handle this case.
3288 Node->setCall(CallInfo());
3289 continue;
3290 }
3291 // Now add back any matching calls that call the same function as the
3292 // matching primary call on Node.
3293 for (++It; It != AllCalls.end(); ++It) {
3294 auto ThisCall = *It;
3295 if (!sameCallee(Call1: Node->Call.call(), Call2: ThisCall.call()))
3296 continue;
3297 Node->MatchingCalls.push_back(ThisCall);
3298 }
3299 }
3300
3301 // Remove all mismatched nodes identified in the above loop from the node map
3302 // (checking whether they have a null call which is set above). For a
3303 // MapVector like NonAllocationCallToContextNodeMap it is much more efficient
3304 // to do the removal via remove_if than by individually erasing entries above.
3305 // Also remove any entries if we updated the node's primary call above.
3306 NonAllocationCallToContextNodeMap.remove_if([](const auto &it) {
3307 return !it.second->hasCall() || it.second->Call != it.first;
3308 });
3309
3310 // Add entries for any new primary calls recorded above.
3311 for (auto &[Call, Node] : NewCallToNode)
3312 NonAllocationCallToContextNodeMap[Call] = Node;
3313
3314 // Add the new nodes after the above loop so that the iteration is not
3315 // invalidated.
3316 for (auto &[Call, Node] : TailCallToContextNodeMap)
3317 NonAllocationCallToContextNodeMap[Call] = Node;
3318}
3319
3320template <typename DerivedCCG, typename FuncTy, typename CallTy>
3321bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::partitionCallsByCallee(
3322 ContextNode *Node, ArrayRef<CallInfo> AllCalls,
3323 std::vector<std::pair<CallInfo, ContextNode *>> &NewCallToNode) {
3324 // Struct to keep track of all the calls having the same callee function,
3325 // and the node we eventually assign to them. Eventually we will record the
3326 // context node assigned to this group of calls.
3327 struct CallsWithSameCallee {
3328 std::vector<CallInfo> Calls;
3329 ContextNode *Node = nullptr;
3330 };
3331
3332 // First partition calls by callee function. Build map from each function
3333 // to the list of matching calls.
3334 DenseMap<const FuncTy *, CallsWithSameCallee> CalleeFuncToCallInfo;
3335 for (auto ThisCall : AllCalls) {
3336 auto *F = getCalleeFunc(Call: ThisCall.call());
3337 if (F)
3338 CalleeFuncToCallInfo[F].Calls.push_back(ThisCall);
3339 }
3340
3341 // Next, walk through all callee edges. For each callee node, get its
3342 // containing function and see if it was recorded in the above map (meaning we
3343 // have at least one matching call). Build another map from each callee node
3344 // with a matching call to the structure instance created above containing all
3345 // the calls.
3346 DenseMap<ContextNode *, CallsWithSameCallee *> CalleeNodeToCallInfo;
3347 for (const auto &Edge : Node->CalleeEdges) {
3348 if (!Edge->Callee->hasCall())
3349 continue;
3350 const FuncTy *ProfiledCalleeFunc = NodeToCallingFunc[Edge->Callee];
3351 if (CalleeFuncToCallInfo.contains(ProfiledCalleeFunc))
3352 CalleeNodeToCallInfo[Edge->Callee] =
3353 &CalleeFuncToCallInfo[ProfiledCalleeFunc];
3354 }
3355
3356 // If there are entries in the second map, then there were no matching
3357 // calls/callees, nothing to do here. Return so we can go to the handling that
3358 // looks through tail calls.
3359 if (CalleeNodeToCallInfo.empty())
3360 return false;
3361
3362 // Walk through all callee edges again. Any and all callee edges that didn't
3363 // match any calls (callee not in the CalleeNodeToCallInfo map) are moved to a
3364 // new caller node (UnmatchedCalleesNode) which gets a null call so that it is
3365 // ignored during cloning. If it is in the map, then we use the node recorded
3366 // in that entry (creating it if needed), and move the callee edge to it.
3367 // The first callee will use the original node instead of creating a new one.
3368 // Note that any of the original calls on this node (in AllCalls) that didn't
3369 // have a callee function automatically get dropped from the node as part of
3370 // this process.
3371 ContextNode *UnmatchedCalleesNode = nullptr;
3372 // Track whether we already assigned original node to a callee.
3373 bool UsedOrigNode = false;
3374 assert(NodeToCallingFunc[Node]);
3375 // Iterate over a copy of Node's callee edges, since we may need to remove
3376 // edges in moveCalleeEdgeToNewCaller, and this simplifies the handling and
3377 // makes it less error-prone.
3378 auto CalleeEdges = Node->CalleeEdges;
3379 for (auto &Edge : CalleeEdges) {
3380 if (!Edge->Callee->hasCall())
3381 continue;
3382
3383 // Will be updated below to point to whatever (caller) node this callee edge
3384 // should be moved to.
3385 ContextNode *CallerNodeToUse = nullptr;
3386
3387 // Handle the case where there were no matching calls first. Move this
3388 // callee edge to the UnmatchedCalleesNode, creating it if needed.
3389 if (!CalleeNodeToCallInfo.contains(Edge->Callee)) {
3390 if (!UnmatchedCalleesNode)
3391 UnmatchedCalleesNode =
3392 createNewNode(/*IsAllocation=*/false, F: NodeToCallingFunc[Node]);
3393 CallerNodeToUse = UnmatchedCalleesNode;
3394 } else {
3395 // Look up the information recorded for this callee node, and use the
3396 // recorded caller node (creating it if needed).
3397 auto *Info = CalleeNodeToCallInfo[Edge->Callee];
3398 if (!Info->Node) {
3399 // If we haven't assigned any callees to the original node use it.
3400 if (!UsedOrigNode) {
3401 Info->Node = Node;
3402 // Clear the set of matching calls which will be updated below.
3403 Node->MatchingCalls.clear();
3404 UsedOrigNode = true;
3405 } else
3406 Info->Node =
3407 createNewNode(/*IsAllocation=*/false, F: NodeToCallingFunc[Node]);
3408 assert(!Info->Calls.empty());
3409 // The first call becomes the primary call for this caller node, and the
3410 // rest go in the matching calls list.
3411 Info->Node->setCall(Info->Calls.front());
3412 llvm::append_range(Info->Node->MatchingCalls,
3413 llvm::drop_begin(Info->Calls));
3414 // Save the primary call to node correspondence so that we can update
3415 // the NonAllocationCallToContextNodeMap, which is being iterated in the
3416 // caller of this function.
3417 NewCallToNode.push_back({Info->Node->Call, Info->Node});
3418 }
3419 CallerNodeToUse = Info->Node;
3420 }
3421
3422 // Don't need to move edge if we are using the original node;
3423 if (CallerNodeToUse == Node)
3424 continue;
3425
3426 moveCalleeEdgeToNewCaller(Edge, NewCaller: CallerNodeToUse);
3427 }
3428 // Now that we are done moving edges, clean up any caller edges that ended
3429 // up with no type or context ids. During moveCalleeEdgeToNewCaller all
3430 // caller edges from Node are replicated onto the new callers, and it
3431 // simplifies the handling to leave them until we have moved all
3432 // edges/context ids.
3433 for (auto &I : CalleeNodeToCallInfo)
3434 removeNoneTypeCallerEdges(Node: I.second->Node);
3435 if (UnmatchedCalleesNode)
3436 removeNoneTypeCallerEdges(Node: UnmatchedCalleesNode);
3437 removeNoneTypeCallerEdges(Node);
3438
3439 return true;
3440}
3441
3442uint64_t ModuleCallsiteContextGraph::getStackId(uint64_t IdOrIndex) const {
3443 // In the Module (IR) case this is already the Id.
3444 return IdOrIndex;
3445}
3446
3447uint64_t IndexCallsiteContextGraph::getStackId(uint64_t IdOrIndex) const {
3448 // In the Index case this is an index into the stack id list in the summary
3449 // index, convert it to an Id.
3450 return Index.getStackIdAtIndex(Index: IdOrIndex);
3451}
3452
3453template <typename DerivedCCG, typename FuncTy, typename CallTy>
3454bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::calleesMatch(
3455 CallTy Call, EdgeIter &EI,
3456 MapVector<CallInfo, ContextNode *> &TailCallToContextNodeMap) {
3457 auto Edge = *EI;
3458 const FuncTy *ProfiledCalleeFunc = NodeToCallingFunc[Edge->Callee];
3459 const FuncTy *CallerFunc = NodeToCallingFunc[Edge->Caller];
3460 // Will be populated in order of callee to caller if we find a chain of tail
3461 // calls between the profiled caller and callee.
3462 std::vector<std::pair<CallTy, FuncTy *>> FoundCalleeChain;
3463 if (!calleeMatchesFunc(Call, Func: ProfiledCalleeFunc, CallerFunc,
3464 FoundCalleeChain))
3465 return false;
3466
3467 // The usual case where the profiled callee matches that of the IR/summary.
3468 if (FoundCalleeChain.empty())
3469 return true;
3470
3471 auto AddEdge = [Edge, &EI](ContextNode *Caller, ContextNode *Callee) {
3472 auto *CurEdge = Callee->findEdgeFromCaller(Caller);
3473 // If there is already an edge between these nodes, simply update it and
3474 // return.
3475 if (CurEdge) {
3476 CurEdge->ContextIds.insert_range(Edge->ContextIds);
3477 CurEdge->AllocTypes |= Edge->AllocTypes;
3478 return;
3479 }
3480 // Otherwise, create a new edge and insert it into the caller and callee
3481 // lists.
3482 auto NewEdge = std::make_shared<ContextEdge>(
3483 Callee, Caller, Edge->AllocTypes, Edge->ContextIds);
3484 Callee->CallerEdges.push_back(NewEdge);
3485 if (Caller == Edge->Caller) {
3486 // If we are inserting the new edge into the current edge's caller, insert
3487 // the new edge before the current iterator position, and then increment
3488 // back to the current edge.
3489 EI = Caller->CalleeEdges.insert(EI, NewEdge);
3490 ++EI;
3491 assert(*EI == Edge &&
3492 "Iterator position not restored after insert and increment");
3493 } else
3494 Caller->CalleeEdges.push_back(NewEdge);
3495 };
3496
3497 // Create new nodes for each found callee and connect in between the profiled
3498 // caller and callee.
3499 auto *CurCalleeNode = Edge->Callee;
3500 for (auto &[NewCall, Func] : FoundCalleeChain) {
3501 ContextNode *NewNode = nullptr;
3502 // First check if we have already synthesized a node for this tail call.
3503 if (TailCallToContextNodeMap.count(NewCall)) {
3504 NewNode = TailCallToContextNodeMap[NewCall];
3505 NewNode->AllocTypes |= Edge->AllocTypes;
3506 } else {
3507 FuncToCallsWithMetadata[Func].push_back({NewCall});
3508 // Create Node and record node info.
3509 NewNode = createNewNode(/*IsAllocation=*/false, F: Func, C: NewCall);
3510 TailCallToContextNodeMap[NewCall] = NewNode;
3511 NewNode->AllocTypes = Edge->AllocTypes;
3512 }
3513
3514 // Hook up node to its callee node
3515 AddEdge(NewNode, CurCalleeNode);
3516
3517 CurCalleeNode = NewNode;
3518 }
3519
3520 // Hook up edge's original caller to new callee node.
3521 AddEdge(Edge->Caller, CurCalleeNode);
3522
3523#ifndef NDEBUG
3524 // Save this because Edge's fields get cleared below when removed.
3525 auto *Caller = Edge->Caller;
3526#endif
3527
3528 // Remove old edge
3529 removeEdgeFromGraph(Edge: Edge.get(), EI: &EI, /*CalleeIter=*/true);
3530
3531 // To simplify the increment of EI in the caller, subtract one from EI.
3532 // In the final AddEdge call we would have either added a new callee edge,
3533 // to Edge->Caller, or found an existing one. Either way we are guaranteed
3534 // that there is at least one callee edge.
3535 assert(!Caller->CalleeEdges.empty());
3536 --EI;
3537
3538 return true;
3539}
3540
3541bool ModuleCallsiteContextGraph::findProfiledCalleeThroughTailCalls(
3542 const Function *ProfiledCallee, Value *CurCallee, unsigned Depth,
3543 std::vector<std::pair<Instruction *, Function *>> &FoundCalleeChain,
3544 bool &FoundMultipleCalleeChains) {
3545 // Stop recursive search if we have already explored the maximum specified
3546 // depth.
3547 if (Depth > TailCallSearchDepth)
3548 return false;
3549
3550 auto SaveCallsiteInfo = [&](Instruction *Callsite, Function *F) {
3551 FoundCalleeChain.push_back(x: {Callsite, F});
3552 };
3553
3554 auto *CalleeFunc = dyn_cast<Function>(Val: CurCallee);
3555 if (!CalleeFunc) {
3556 auto *Alias = dyn_cast<GlobalAlias>(Val: CurCallee);
3557 assert(Alias);
3558 CalleeFunc = dyn_cast<Function>(Val: Alias->getAliasee());
3559 assert(CalleeFunc);
3560 }
3561
3562 // Look for tail calls in this function, and check if they either call the
3563 // profiled callee directly, or indirectly (via a recursive search).
3564 // Only succeed if there is a single unique tail call chain found between the
3565 // profiled caller and callee, otherwise we could perform incorrect cloning.
3566 bool FoundSingleCalleeChain = false;
3567 for (auto &BB : *CalleeFunc) {
3568 for (auto &I : BB) {
3569 auto *CB = dyn_cast<CallBase>(Val: &I);
3570 if (!CB || !CB->isTailCall())
3571 continue;
3572 auto *CalledValue = CB->getCalledOperand();
3573 auto *CalledFunction = CB->getCalledFunction();
3574 if (CalledValue && !CalledFunction) {
3575 CalledValue = CalledValue->stripPointerCasts();
3576 // Stripping pointer casts can reveal a called function.
3577 CalledFunction = dyn_cast<Function>(Val: CalledValue);
3578 }
3579 // Check if this is an alias to a function. If so, get the
3580 // called aliasee for the checks below.
3581 if (auto *GA = dyn_cast<GlobalAlias>(Val: CalledValue)) {
3582 assert(!CalledFunction &&
3583 "Expected null called function in callsite for alias");
3584 CalledFunction = dyn_cast<Function>(Val: GA->getAliaseeObject());
3585 }
3586 if (!CalledFunction)
3587 continue;
3588 if (CalledFunction == ProfiledCallee) {
3589 if (FoundSingleCalleeChain) {
3590 FoundMultipleCalleeChains = true;
3591 return false;
3592 }
3593 FoundSingleCalleeChain = true;
3594 FoundProfiledCalleeCount++;
3595 FoundProfiledCalleeDepth += Depth;
3596 if (Depth > FoundProfiledCalleeMaxDepth)
3597 FoundProfiledCalleeMaxDepth = Depth;
3598 SaveCallsiteInfo(&I, CalleeFunc);
3599 } else if (findProfiledCalleeThroughTailCalls(
3600 ProfiledCallee, CurCallee: CalledFunction, Depth: Depth + 1,
3601 FoundCalleeChain, FoundMultipleCalleeChains)) {
3602 // findProfiledCalleeThroughTailCalls should not have returned
3603 // true if FoundMultipleCalleeChains.
3604 assert(!FoundMultipleCalleeChains);
3605 if (FoundSingleCalleeChain) {
3606 FoundMultipleCalleeChains = true;
3607 return false;
3608 }
3609 FoundSingleCalleeChain = true;
3610 SaveCallsiteInfo(&I, CalleeFunc);
3611 } else if (FoundMultipleCalleeChains)
3612 return false;
3613 }
3614 }
3615
3616 return FoundSingleCalleeChain;
3617}
3618
3619const Function *ModuleCallsiteContextGraph::getCalleeFunc(Instruction *Call) {
3620 auto *CB = dyn_cast<CallBase>(Val: Call);
3621 if (!CB->getCalledOperand() || CB->isIndirectCall())
3622 return nullptr;
3623 auto *CalleeVal = CB->getCalledOperand()->stripPointerCasts();
3624 auto *Alias = dyn_cast<GlobalAlias>(Val: CalleeVal);
3625 if (Alias)
3626 return dyn_cast<Function>(Val: Alias->getAliasee());
3627 return dyn_cast<Function>(Val: CalleeVal);
3628}
3629
3630bool ModuleCallsiteContextGraph::calleeMatchesFunc(
3631 Instruction *Call, const Function *Func, const Function *CallerFunc,
3632 std::vector<std::pair<Instruction *, Function *>> &FoundCalleeChain) {
3633 auto *CB = dyn_cast<CallBase>(Val: Call);
3634 if (!CB->getCalledOperand() || CB->isIndirectCall())
3635 return false;
3636 auto *CalleeVal = CB->getCalledOperand()->stripPointerCasts();
3637 auto *CalleeFunc = dyn_cast<Function>(Val: CalleeVal);
3638 if (CalleeFunc == Func)
3639 return true;
3640 auto *Alias = dyn_cast<GlobalAlias>(Val: CalleeVal);
3641 if (Alias && Alias->getAliasee() == Func)
3642 return true;
3643
3644 // Recursively search for the profiled callee through tail calls starting with
3645 // the actual Callee. The discovered tail call chain is saved in
3646 // FoundCalleeChain, and we will fixup the graph to include these callsites
3647 // after returning.
3648 // FIXME: We will currently redo the same recursive walk if we find the same
3649 // mismatched callee from another callsite. We can improve this with more
3650 // bookkeeping of the created chain of new nodes for each mismatch.
3651 unsigned Depth = 1;
3652 bool FoundMultipleCalleeChains = false;
3653 if (!findProfiledCalleeThroughTailCalls(ProfiledCallee: Func, CurCallee: CalleeVal, Depth,
3654 FoundCalleeChain,
3655 FoundMultipleCalleeChains)) {
3656 LLVM_DEBUG(dbgs() << "Not found through unique tail call chain: "
3657 << Func->getName() << " from " << CallerFunc->getName()
3658 << " that actually called " << CalleeVal->getName()
3659 << (FoundMultipleCalleeChains
3660 ? " (found multiple possible chains)"
3661 : "")
3662 << "\n");
3663 if (FoundMultipleCalleeChains)
3664 FoundProfiledCalleeNonUniquelyCount++;
3665 return false;
3666 }
3667
3668 return true;
3669}
3670
3671bool ModuleCallsiteContextGraph::sameCallee(Instruction *Call1,
3672 Instruction *Call2) {
3673 auto *CB1 = cast<CallBase>(Val: Call1);
3674 if (!CB1->getCalledOperand() || CB1->isIndirectCall())
3675 return false;
3676 auto *CalleeVal1 = CB1->getCalledOperand()->stripPointerCasts();
3677 auto *CalleeFunc1 = dyn_cast<Function>(Val: CalleeVal1);
3678 auto *CB2 = cast<CallBase>(Val: Call2);
3679 if (!CB2->getCalledOperand() || CB2->isIndirectCall())
3680 return false;
3681 auto *CalleeVal2 = CB2->getCalledOperand()->stripPointerCasts();
3682 auto *CalleeFunc2 = dyn_cast<Function>(Val: CalleeVal2);
3683 return CalleeFunc1 == CalleeFunc2;
3684}
3685
3686bool IndexCallsiteContextGraph::findProfiledCalleeThroughTailCalls(
3687 ValueInfo ProfiledCallee, ValueInfo CurCallee, unsigned Depth,
3688 std::vector<std::pair<ValueInfo, FunctionSummary *>> &FoundCalleeChain,
3689 bool &FoundMultipleCalleeChains) {
3690 // Stop recursive search if we have already explored the maximum specified
3691 // depth.
3692 if (Depth > TailCallSearchDepth)
3693 return false;
3694
3695 // Record each discovered tail call (callee and calling function) in the
3696 // chain. The CallsiteInfo records are only synthesized by the caller once
3697 // the search succeeds, since the chain is discarded if the profiled callee
3698 // is found through multiple chains.
3699 auto SaveTailCall = [&](ValueInfo Callee, FunctionSummary *FS) {
3700 FoundCalleeChain.push_back(x: {Callee, FS});
3701 };
3702
3703 // Returns true if the given tail call callee is the profiled callee, or is
3704 // an alias of it. Only the aliasee's function summary gets a context node
3705 // (see calleeMatchesFunc), so the profiled callee is always the aliasee. The
3706 // synthesized callsite record is still keyed by the alias (the callee of the
3707 // call edge), which is what the ThinLTO backend looks up for the tail call
3708 // (see findValueInfoForGV).
3709 auto IsProfiledCallee = [&](ValueInfo Callee) {
3710 if (Callee == ProfiledCallee)
3711 return true;
3712 for (auto &S : Callee.getSummaryList()) {
3713 if (!GlobalValue::isLocalLinkage(Linkage: S->linkage()) &&
3714 !isPrevailing(Callee.getGUID(), S.get()))
3715 continue;
3716 auto *AS = dyn_cast<AliasSummary>(Val: S.get());
3717 return AS && AS->getAliaseeVI() == ProfiledCallee;
3718 }
3719 return false;
3720 };
3721
3722 // Look for tail calls in this function, and check if they either call the
3723 // profiled callee directly, or indirectly (via a recursive search).
3724 // Only succeed if there is a single unique tail call chain found between the
3725 // profiled caller and callee, otherwise we could perform incorrect cloning.
3726 bool FoundSingleCalleeChain = false;
3727 for (auto &S : CurCallee.getSummaryList()) {
3728 if (!GlobalValue::isLocalLinkage(Linkage: S->linkage()) &&
3729 !isPrevailing(CurCallee.getGUID(), S.get()))
3730 continue;
3731 auto *FS = dyn_cast<FunctionSummary>(Val: S->getBaseObject());
3732 if (!FS)
3733 continue;
3734 auto FSVI = CurCallee;
3735 auto *AS = dyn_cast<AliasSummary>(Val: S.get());
3736 if (AS)
3737 FSVI = AS->getAliaseeVI();
3738 for (auto &CallEdge : FS->calls()) {
3739 if (!CallEdge.second.hasTailCall())
3740 continue;
3741 // Edges marked indirect are the profiled targets of indirect calls.
3742 // If the tail call is indirect, the ThinLTO backend will perform
3743 // ICP to the target in order to call a clone of it.
3744 if (CallEdge.second.hasIndirectCall() && !IndirectTailCallSearch)
3745 continue;
3746 if (IsProfiledCallee(CallEdge.first)) {
3747 if (FoundSingleCalleeChain) {
3748 FoundMultipleCalleeChains = true;
3749 return false;
3750 }
3751 FoundSingleCalleeChain = true;
3752 FoundProfiledCalleeCount++;
3753 FoundProfiledCalleeDepth += Depth;
3754 if (Depth > FoundProfiledCalleeMaxDepth)
3755 FoundProfiledCalleeMaxDepth = Depth;
3756 if (CallEdge.second.hasIndirectCall())
3757 FoundProfiledCalleeIndirectTailCallCount++;
3758 SaveTailCall(CallEdge.first, FS);
3759 // Add FS to FSToVIMap in case it isn't already there.
3760 assert(!FSToVIMap.count(FS) || FSToVIMap[FS] == FSVI);
3761 FSToVIMap[FS] = FSVI;
3762 } else if (findProfiledCalleeThroughTailCalls(
3763 ProfiledCallee, CurCallee: CallEdge.first, Depth: Depth + 1,
3764 FoundCalleeChain, FoundMultipleCalleeChains)) {
3765 // findProfiledCalleeThroughTailCalls should not have returned
3766 // true if FoundMultipleCalleeChains.
3767 assert(!FoundMultipleCalleeChains);
3768 if (FoundSingleCalleeChain) {
3769 FoundMultipleCalleeChains = true;
3770 return false;
3771 }
3772 FoundSingleCalleeChain = true;
3773 if (CallEdge.second.hasIndirectCall())
3774 FoundProfiledCalleeIndirectTailCallCount++;
3775 SaveTailCall(CallEdge.first, FS);
3776 // Add FS to FSToVIMap in case it isn't already there.
3777 assert(!FSToVIMap.count(FS) || FSToVIMap[FS] == FSVI);
3778 FSToVIMap[FS] = FSVI;
3779 } else if (FoundMultipleCalleeChains)
3780 return false;
3781 }
3782 }
3783
3784 return FoundSingleCalleeChain;
3785}
3786
3787const FunctionSummary *
3788IndexCallsiteContextGraph::getCalleeFunc(IndexCall &Call) {
3789 ValueInfo Callee = dyn_cast_if_present<CallsiteInfo *>(Val&: Call)->Callee;
3790 if (Callee.getSummaryList().empty())
3791 return nullptr;
3792 return dyn_cast<FunctionSummary>(Val: Callee.getSummaryList()[0]->getBaseObject());
3793}
3794
3795bool IndexCallsiteContextGraph::calleeMatchesFunc(
3796 IndexCall &Call, const FunctionSummary *Func,
3797 const FunctionSummary *CallerFunc,
3798 std::vector<std::pair<IndexCall, FunctionSummary *>> &FoundCalleeChain) {
3799 ValueInfo Callee = dyn_cast_if_present<CallsiteInfo *>(Val&: Call)->Callee;
3800 // If there is no summary list then this is a call to an externally defined
3801 // symbol.
3802 AliasSummary *Alias =
3803 Callee.getSummaryList().empty()
3804 ? nullptr
3805 : dyn_cast<AliasSummary>(Val: Callee.getSummaryList()[0].get());
3806 assert(FSToVIMap.count(Func));
3807 auto FuncVI = FSToVIMap[Func];
3808 if (Callee == FuncVI ||
3809 // If callee is an alias, check the aliasee, since only function
3810 // summary base objects will contain the stack node summaries and thus
3811 // get a context node.
3812 (Alias && Alias->getAliaseeVI() == FuncVI))
3813 return true;
3814
3815 // Recursively search for the profiled callee through tail calls starting with
3816 // the actual Callee. The discovered tail call chain is saved in
3817 // FoundCalleeChain, and we will fixup the graph to include these callsites
3818 // after returning.
3819 // FIXME: We will currently redo the same recursive walk if we find the same
3820 // mismatched callee from another callsite. We can improve this with more
3821 // bookkeeping of the created chain of new nodes for each mismatch.
3822 unsigned Depth = 1;
3823 bool FoundMultipleCalleeChains = false;
3824 std::vector<std::pair<ValueInfo, FunctionSummary *>> FoundCalleeVIChain;
3825 if (!findProfiledCalleeThroughTailCalls(ProfiledCallee: FuncVI, CurCallee: Callee, Depth,
3826 FoundCalleeChain&: FoundCalleeVIChain,
3827 FoundMultipleCalleeChains)) {
3828 LLVM_DEBUG(dbgs() << "Not found through unique tail call chain: " << FuncVI
3829 << " from " << FSToVIMap[CallerFunc]
3830 << " that actually called " << Callee
3831 << (FoundMultipleCalleeChains
3832 ? " (found multiple possible chains)"
3833 : "")
3834 << "\n");
3835 if (FoundMultipleCalleeChains)
3836 FoundProfiledCalleeNonUniquelyCount++;
3837 return false;
3838 }
3839
3840 // Now that we know the chain is unique, synthesize a CallsiteInfo for each
3841 // tail call in it, if one hasn't already been synthesized (e.g. for a chain
3842 // to a different profiled callee through the same tail call). These are only
3843 // added to the function summaries once the graph is destroyed (see
3844 // ~IndexCallsiteContextGraph).
3845 for (auto &[CalleeVI, FS] : FoundCalleeVIChain) {
3846 auto &CallsiteInfoPtr =
3847 FunctionCalleesToSynthesizedCallsiteInfos[FS][CalleeVI];
3848 if (!CallsiteInfoPtr)
3849 // StackIds is empty (we don't have debug info available in the index for
3850 // these callsites)
3851 CallsiteInfoPtr =
3852 std::make_unique<CallsiteInfo>(args&: CalleeVI, args: SmallVector<unsigned>());
3853 FoundCalleeChain.push_back(x: {CallsiteInfoPtr.get(), FS});
3854 }
3855
3856 return true;
3857}
3858
3859bool IndexCallsiteContextGraph::sameCallee(IndexCall &Call1, IndexCall &Call2) {
3860 ValueInfo Callee1 = dyn_cast_if_present<CallsiteInfo *>(Val&: Call1)->Callee;
3861 ValueInfo Callee2 = dyn_cast_if_present<CallsiteInfo *>(Val&: Call2)->Callee;
3862 return Callee1 == Callee2;
3863}
3864
3865template <typename DerivedCCG, typename FuncTy, typename CallTy>
3866void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::dump()
3867 const {
3868 print(OS&: dbgs());
3869 dbgs() << "\n";
3870}
3871
3872template <typename DerivedCCG, typename FuncTy, typename CallTy>
3873void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::print(
3874 raw_ostream &OS) const {
3875 OS << "Node " << this << "\n";
3876 OS << "\t";
3877 printCall(OS);
3878 if (Recursive)
3879 OS << " (recursive)";
3880 OS << "\n";
3881 if (!MatchingCalls.empty()) {
3882 OS << "\tMatchingCalls:\n";
3883 for (auto &MatchingCall : MatchingCalls) {
3884 OS << "\t";
3885 MatchingCall.print(OS);
3886 OS << "\n";
3887 }
3888 }
3889 OS << "\tNodeId: " << NodeId << "\n";
3890 OS << "\tAllocTypes: " << getAllocTypeString(AllocTypes) << "\n";
3891 OS << "\tContextIds:";
3892 // Make a copy of the computed context ids that we can sort for stability.
3893 auto ContextIds = getContextIds();
3894 std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
3895 std::sort(first: SortedIds.begin(), last: SortedIds.end());
3896 for (auto Id : SortedIds)
3897 OS << " " << Id;
3898 OS << "\n";
3899 OS << "\tCalleeEdges:\n";
3900 for (auto &Edge : CalleeEdges)
3901 OS << "\t\t" << *Edge << " (Callee NodeId: " << Edge->Callee->NodeId
3902 << ")\n";
3903 OS << "\tCallerEdges:\n";
3904 for (auto &Edge : CallerEdges)
3905 OS << "\t\t" << *Edge << " (Caller NodeId: " << Edge->Caller->NodeId
3906 << ")\n";
3907 if (!Clones.empty()) {
3908 OS << "\tClones: ";
3909 ListSeparator LS;
3910 for (auto *C : Clones)
3911 OS << LS << C << " NodeId: " << C->NodeId;
3912 OS << "\n";
3913 } else if (CloneOf) {
3914 OS << "\tClone of " << CloneOf << " NodeId: " << CloneOf->NodeId << "\n";
3915 }
3916}
3917
3918template <typename DerivedCCG, typename FuncTy, typename CallTy>
3919void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge::dump()
3920 const {
3921 print(OS&: dbgs());
3922 dbgs() << "\n";
3923}
3924
3925template <typename DerivedCCG, typename FuncTy, typename CallTy>
3926void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge::print(
3927 raw_ostream &OS) const {
3928 OS << "Edge from Callee " << Callee << " to Caller: " << Caller
3929 << (IsBackedge ? " (BE)" : "")
3930 << " AllocTypes: " << getAllocTypeString(AllocTypes);
3931 OS << " ContextIds:";
3932 std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
3933 std::sort(first: SortedIds.begin(), last: SortedIds.end());
3934 for (auto Id : SortedIds)
3935 OS << " " << Id;
3936}
3937
3938template <typename DerivedCCG, typename FuncTy, typename CallTy>
3939void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::dump() const {
3940 print(OS&: dbgs());
3941}
3942
3943template <typename DerivedCCG, typename FuncTy, typename CallTy>
3944void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::print(
3945 raw_ostream &OS) const {
3946 OS << "Callsite Context Graph:\n";
3947 using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
3948 for (const auto Node : nodes<GraphType>(this)) {
3949 if (Node->isRemoved())
3950 continue;
3951 Node->print(OS);
3952 OS << "\n";
3953 }
3954}
3955
3956template <typename DerivedCCG, typename FuncTy, typename CallTy>
3957void CallsiteContextGraph<DerivedCCG, FuncTy,
3958 CallTy>::capturePreCloningContextInfo() {
3959 for (const auto &Entry : AllocationCallToContextNodeMap) {
3960 const ContextNode *AllocNode = Entry.second;
3961 // Remember whether any caller of this allocation matched a call, so that
3962 // the report can distinguish "no matched callers" from an allocation whose
3963 // callers are simply not part of this context.
3964 for (const auto &Edge : AllocNode->CallerEdges) {
3965 if (!Edge->isRemoved() && Edge->Caller->hasCall()) {
3966 PreCloningAllocHasMatchedCaller.insert(AllocNode);
3967 break;
3968 }
3969 }
3970 DenseSet<uint32_t> ContextIds = AllocNode->getContextIds();
3971 for (uint32_t Id : ContextIds) {
3972 PreCloningContextInfo Info;
3973 // Walk the caller edges carrying this context id from the allocation
3974 // node outwards to build the chain of nodes for this context. Each node
3975 // is visited once; backedges (recursive cycles) are only followed when
3976 // no other caller edge carries the id, so that the chain reflects the
3977 // context's stack order rather than looping.
3978 Info.Chain.push_back(AllocNode);
3979 SmallPtrSet<const ContextNode *, 16> Visited;
3980 Visited.insert(AllocNode);
3981 const ContextNode *Cur = AllocNode;
3982 while (true) {
3983 const ContextNode *Next = nullptr;
3984 for (const auto &Edge : Cur->CallerEdges) {
3985 if (Edge->isRemoved() || Edge->IsBackedge)
3986 continue;
3987 if (Edge->getContextIds().contains(Id) &&
3988 !Visited.contains(Edge->Caller)) {
3989 Next = Edge->Caller;
3990 break;
3991 }
3992 }
3993 if (!Next) {
3994 for (const auto &Edge : Cur->CallerEdges) {
3995 if (Edge->isRemoved())
3996 continue;
3997 if (Edge->getContextIds().contains(Id) &&
3998 !Visited.contains(Edge->Caller)) {
3999 Next = Edge->Caller;
4000 break;
4001 }
4002 }
4003 }
4004 if (!Next)
4005 break;
4006 Visited.insert(Next);
4007 Info.Chain.push_back(Next);
4008 Cur = Next;
4009 }
4010
4011 // Find the outermost node (walking from the allocation) such that it
4012 // and all nodes below it matched a call, and record why the first
4013 // unmatched node (if any) did not match.
4014 Info.MaxMatchedFrame = 0;
4015 for (unsigned I = 1; I < Info.Chain.size(); ++I) {
4016 if (Info.Chain[I]->hasCall()) {
4017 Info.MaxMatchedFrame = I;
4018 } else {
4019 const ContextNode *Unmatched = Info.Chain[I];
4020 Info.UnmatchedStackId = Unmatched->OrigStackOrAllocId;
4021 Info.StoppedAtRecursive = Unmatched->Recursive;
4022 // A calling function is known for the unmatched node only when a
4023 // call in that function was considered and rejected because its
4024 // callee did not match the node's callee.
4025 if (auto It = NodeToCallingFunc.find(Unmatched);
4026 It != NodeToCallingFunc.end()) {
4027 Info.StoppedAtMismatchedCallee = true;
4028 Info.UnmatchedCallingFunc = It->second;
4029 }
4030 break;
4031 }
4032 }
4033
4034 // If the whole context matched, check whether its outermost node has
4035 // matched callers reached by other contexts: then this context was
4036 // trimmed below them, and clones of those callers can change which
4037 // allocation clone it reaches.
4038 if (Info.MaxMatchedFrame > 0 &&
4039 Info.MaxMatchedFrame + 1 == Info.Chain.size()) {
4040 const ContextNode *OutermostMatchedNode =
4041 Info.Chain[Info.MaxMatchedFrame];
4042 for (const auto &Edge : OutermostMatchedNode->CallerEdges) {
4043 if (!Edge->isRemoved() && Edge->Caller->hasCall()) {
4044 Info.OutermostMatchedNodeHasOtherCallers = true;
4045 break;
4046 }
4047 }
4048 }
4049
4050 PreCloningContextInfoMap[Id] = std::move(Info);
4051 }
4052 }
4053}
4054
4055template <typename DerivedCCG, typename FuncTy, typename CallTy>
4056std::string
4057CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::formatContextReportSuffix(
4058 const ContextNode *Node, uint32_t Id) const {
4059 auto InfoIt = PreCloningContextInfoMap.find(Id);
4060 if (InfoIt == PreCloningContextInfoMap.end())
4061 return "";
4062 const PreCloningContextInfo &Info = InfoIt->second;
4063
4064 // 1. The allocation function (clone) the context's allocation node ended up
4065 // in, with a qualifier when it is dead, or when the context has callers in
4066 // the profile but none of the allocation's callers matched (either because
4067 // nothing references the function at all, or because the references did not
4068 // match the profiled callsites).
4069 const ContextNode *OrigAlloc = Node->getOrigNode();
4070 const FuncTy *AllocFunc = getCallingFunc(N: OrigAlloc);
4071 unsigned AllocCloneNo = Node->Call.cloneNo();
4072 std::string AllocFuncStr = getFuncName(Func: AllocFunc, CloneNo: AllocCloneNo);
4073 if (isFuncDead(Func: AllocFunc)) {
4074 AllocFuncStr += " (dead)";
4075 } else if (!PreCloningAllocHasMatchedCaller.contains(OrigAlloc) &&
4076 Info.Chain.size() > 1) {
4077 // Distinguish a function with no references in the IR/summary (it may
4078 // still be referenced from native objects) from one whose references did
4079 // not match.
4080 AllocFuncStr += AllocFunc && !FunctionsWithCallers.contains(AllocFunc)
4081 ? " (no references in IR/summary)"
4082 : " (no matched callers)";
4083 }
4084
4085 // 2. How far the context matched: all of it ("matched N frames to F", or
4086 // "trimmed at frame N in F" when F has callers above that the context does
4087 // not cover), or up to an unmatched node (with the reason and stack id).
4088 unsigned TotalFrames = Info.Chain.size() > 0 ? Info.Chain.size() - 1 : 0;
4089 unsigned Matched = Info.MaxMatchedFrame;
4090 std::string StatusStr;
4091 if (TotalFrames == 0) {
4092 StatusStr = "inlined into leaf";
4093 } else if (Matched < TotalFrames) {
4094 // Where and why matching stopped: the first unmatched node was either a
4095 // recursive callsite (not matched when recursive callsites are disabled),
4096 // a callsite whose IR callee did not match the profiled callee, or a
4097 // callsite with no matching call at all.
4098 if (Info.StoppedAtRecursive)
4099 StatusStr = "unmatched recursive callsite";
4100 else if (Info.StoppedAtMismatchedCallee)
4101 StatusStr = "unmatched callsite (mismatched callee in " +
4102 getFuncName(Func: Info.UnmatchedCallingFunc, CloneNo: 0) + ")";
4103 else
4104 StatusStr = "unmatched callsite";
4105 StatusStr += " at frame " + std::to_string(val: Matched + 1) + "/" +
4106 std::to_string(val: TotalFrames) + " (stack id " +
4107 std::to_string(Info.UnmatchedStackId) + ")";
4108 } else {
4109 const ContextNode *OutermostMatchedNode = Info.Chain[Matched];
4110 const FuncTy *OutermostMatchedFunc = getCallingFunc(N: OutermostMatchedNode);
4111 unsigned NumOutermostMatchedFuncClones =
4112 std::max<unsigned>(1, FuncToNumClones.lookup(OutermostMatchedFunc));
4113 bool HasCallersAbove =
4114 NumOutermostMatchedFuncClones > 1 ||
4115 Info.OutermostMatchedNodeHasOtherCallers ||
4116 (OutermostMatchedFunc &&
4117 FunctionsWithCallers.contains(OutermostMatchedFunc));
4118 if (HasCallersAbove) {
4119 StatusStr = "trimmed at frame " + std::to_string(val: Matched) + " in " +
4120 getFuncName(Func: OutermostMatchedFunc, CloneNo: 0);
4121 } else {
4122 StatusStr = "matched " + std::to_string(val: Matched) + " frames to " +
4123 getFuncName(Func: OutermostMatchedFunc, CloneNo: 0);
4124 }
4125 }
4126
4127 // 3. The post-cloning call paths: starting from each function clone of the
4128 // outermost matched caller, follow the callee clones assigned to the
4129 // context's callsites down to the allocation clone reached, and report the
4130 // hint that clone ended up with. Paths reaching the same allocation clone
4131 // are reported once.
4132 std::string ReachesStr;
4133 if (Matched >= 1) {
4134 const ContextNode *OutermostMatchedNode = Info.Chain[Matched];
4135 const FuncTy *OutermostMatchedFunc = getCallingFunc(N: OutermostMatchedNode);
4136 unsigned NumOutermostMatchedFuncClones =
4137 std::max<unsigned>(1, FuncToNumClones.lookup(OutermostMatchedFunc));
4138 ReachesStr = "; reaches ";
4139 // Allocation clones (by clone number) already reported for this context.
4140 // Different function clones of the outermost matched caller often lead to
4141 // the same allocation clone; such paths are reported only once, for the
4142 // first caller clone reaching that allocation clone.
4143 SmallSet<unsigned, 4> ReportedAllocClones;
4144 bool FirstPath = true;
4145 for (unsigned OutermostMatchedFuncClone = 0;
4146 OutermostMatchedFuncClone < NumOutermostMatchedFuncClones;
4147 ++OutermostMatchedFuncClone) {
4148 // PathFuncs/PathClones[S] is the function and clone number at step S of
4149 // the path, from the outermost matched caller (S = 0) down to the
4150 // allocation function (S = Matched).
4151 std::vector<const FuncTy *> PathFuncs(Matched + 1);
4152 std::vector<unsigned> PathClones(Matched + 1);
4153 PathFuncs[0] = OutermostMatchedFunc;
4154 PathClones[0] = OutermostMatchedFuncClone;
4155 unsigned CurClone = OutermostMatchedFuncClone;
4156 for (unsigned Step = 0; Step < Matched; ++Step) {
4157 unsigned FrameIdx = Matched - Step;
4158 const ContextNode *CallNode = Info.Chain[FrameIdx];
4159 unsigned NextClone =
4160 OrigNodeAndCloneToCalleeClone.lookup({CallNode, CurClone});
4161 const FuncTy *CalleeFunc = getCallingFunc(N: Info.Chain[FrameIdx - 1]);
4162 PathFuncs[Step + 1] = CalleeFunc;
4163 PathClones[Step + 1] = NextClone;
4164 CurClone = NextClone;
4165 }
4166
4167 if (!ReportedAllocClones.insert(V: CurClone).second)
4168 continue;
4169 if (!FirstPath)
4170 ReachesStr += ", ";
4171 FirstPath = false;
4172
4173 const ContextNode *ReachedAlloc =
4174 OrigAllocAndCloneToNode.lookup({OrigAlloc, CurClone});
4175 // Report the hint actually applied to the reached allocation clone
4176 // (which accounts for allocTypeToUse and MinClonedColdBytePercent),
4177 // rather than the graph node's (possibly ambiguous) AllocTypes.
4178 std::string ReachedTypeStr;
4179 if (ReachedAlloc)
4180 ReachedTypeStr = getAllocTypeString(
4181 AllocTypes: (uint8_t)getAllocationCallType(Call: ReachedAlloc->Call));
4182 else if (CurClone == 0)
4183 ReachedTypeStr =
4184 getAllocTypeString(AllocTypes: (uint8_t)getAllocationCallType(Call: OrigAlloc->Call));
4185 else
4186 ReachedTypeStr = "None";
4187
4188 // Print the path, eliding runs of uncloned (or cloned) functions in the
4189 // middle of paths longer than MemProfReportHintedSizesDetailMaxPathLen;
4190 // the ends and every transition between cloned and uncloned functions
4191 // are kept.
4192 std::string PathStr;
4193 bool SkippedPrev = false;
4194 bool PrintFullPath =
4195 Matched + 1 <= MemProfReportHintedSizesDetailMaxPathLen;
4196 for (unsigned S = 0; S <= Matched; ++S) {
4197 bool IsCloned = PathClones[S] > 0;
4198 bool PrevCloned = S > 0 ? (PathClones[S - 1] > 0) : IsCloned;
4199 bool NextCloned = S + 1 <= Matched ? (PathClones[S + 1] > 0) : IsCloned;
4200 bool Include = PrintFullPath || (S <= 1) || (S + 1 >= Matched) ||
4201 (IsCloned != PrevCloned) || (IsCloned != NextCloned);
4202 if (!Include) {
4203 if (!SkippedPrev) {
4204 PathStr += " -> ...";
4205 SkippedPrev = true;
4206 }
4207 continue;
4208 }
4209 SkippedPrev = false;
4210 if (!PathStr.empty())
4211 PathStr += " -> ";
4212 PathStr += getFuncName(Func: PathFuncs[S], CloneNo: PathClones[S]);
4213 }
4214 ReachesStr += ReachedTypeStr + " via " + PathStr;
4215 }
4216 }
4217
4218 return " [alloc in " + AllocFuncStr + "; " + StatusStr + ReachesStr + "]";
4219}
4220
4221template <typename DerivedCCG, typename FuncTy, typename CallTy>
4222void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::printTotalSizes(
4223 raw_ostream &OS,
4224 function_ref<void(StringRef, StringRef, const Twine &)> EmitRemark) const {
4225 using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
4226 for (const auto Node : nodes<GraphType>(this)) {
4227 if (Node->isRemoved())
4228 continue;
4229 if (!Node->IsAllocation)
4230 continue;
4231 DenseSet<uint32_t> ContextIds = Node->getContextIds();
4232 auto AllocTypeFromCall = getAllocationCallType(Call: Node->Call);
4233 std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
4234 std::sort(first: SortedIds.begin(), last: SortedIds.end());
4235 for (auto Id : SortedIds) {
4236 auto TypeI = ContextIdToAllocationType.find(Val: Id);
4237 assert(TypeI != ContextIdToAllocationType.end());
4238 std::string Suffix;
4239 bool ComputedSuffix = false;
4240 auto GetSuffix = [&]() -> const std::string & {
4241 if (!ComputedSuffix) {
4242 Suffix = MemProfReportHintedSizesDetail
4243 ? formatContextReportSuffix(Node, Id)
4244 : "";
4245 ComputedSuffix = true;
4246 }
4247 return Suffix;
4248 };
4249 auto CSI = ContextIdToContextSizeInfos.find(Val: Id);
4250 if (CSI != ContextIdToContextSizeInfos.end()) {
4251 for (auto &Info : CSI->second) {
4252 std::string Msg =
4253 "MemProf hinting: " + getAllocTypeString(AllocTypes: (uint8_t)TypeI->second) +
4254 " full allocation context " + std::to_string(val: Info.FullStackId) +
4255 " with total size " + std::to_string(val: Info.TotalSize) + " is " +
4256 getAllocTypeString(Node->AllocTypes) + " after cloning";
4257 if (allocTypeToUse(Node->AllocTypes) !=
4258 allocTypeToUse(AllocTypes: (uint8_t)AllocTypeFromCall))
4259 Msg += " marked " + getAllocTypeString(AllocTypes: (uint8_t)AllocTypeFromCall) +
4260 " due to cold byte percent";
4261 // Print the internal context id to aid debugging and visualization.
4262 Msg += " (internal context id " + std::to_string(val: Id) + ")";
4263 if (MemProfReportHintedSizes)
4264 OS << Msg << GetSuffix() << "\n";
4265 if (EmitRemark)
4266 EmitRemark(DEBUG_TYPE, "MemProfReport", Msg);
4267 }
4268 } else {
4269 // This is only emitted if the context size info is not present.
4270 std::string Msg =
4271 "MemProf hinting: " + getAllocTypeString(AllocTypes: (uint8_t)TypeI->second) +
4272 " context is " + getAllocTypeString(Node->AllocTypes) +
4273 " after cloning";
4274 if (allocTypeToUse(Node->AllocTypes) !=
4275 allocTypeToUse(AllocTypes: (uint8_t)AllocTypeFromCall))
4276 Msg += " marked " + getAllocTypeString(AllocTypes: (uint8_t)AllocTypeFromCall) +
4277 " due to cold byte percent";
4278 // Print the internal context id to aid debugging and visualization.
4279 Msg += " (internal context id " + std::to_string(val: Id) + ")";
4280 if (MemProfReportHintedSizes)
4281 OS << Msg << GetSuffix() << "\n";
4282 if (EmitRemark)
4283 EmitRemark(DEBUG_TYPE, "MemProfReport", Msg);
4284 }
4285 }
4286 }
4287}
4288
4289template <typename DerivedCCG, typename FuncTy, typename CallTy>
4290void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::check() const {
4291 using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
4292 for (const auto Node : nodes<GraphType>(this)) {
4293 checkNode<DerivedCCG, FuncTy, CallTy>(Node, /*CheckEdges=*/false);
4294 for (auto &Edge : Node->CallerEdges)
4295 checkEdge<DerivedCCG, FuncTy, CallTy>(Edge);
4296 }
4297}
4298
4299template <typename DerivedCCG, typename FuncTy, typename CallTy>
4300struct GraphTraits<const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *> {
4301 using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
4302 using NodeRef = const ContextNode<DerivedCCG, FuncTy, CallTy> *;
4303
4304 using NodePtrTy = std::unique_ptr<ContextNode<DerivedCCG, FuncTy, CallTy>>;
4305 static NodeRef getNode(const NodePtrTy &P) { return P.get(); }
4306
4307 using nodes_iterator =
4308 mapped_iterator<typename std::vector<NodePtrTy>::const_iterator,
4309 decltype(&getNode)>;
4310
4311 static nodes_iterator nodes_begin(GraphType G) {
4312 return nodes_iterator(G->NodeOwner.begin(), &getNode);
4313 }
4314
4315 static nodes_iterator nodes_end(GraphType G) {
4316 return nodes_iterator(G->NodeOwner.end(), &getNode);
4317 }
4318
4319 static NodeRef getEntryNode(GraphType G) {
4320 return G->NodeOwner.begin()->get();
4321 }
4322
4323 using EdgePtrTy = std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>>;
4324 static const ContextNode<DerivedCCG, FuncTy, CallTy> *
4325 GetCallee(const EdgePtrTy &P) {
4326 return P->Callee;
4327 }
4328
4329 using ChildIteratorType =
4330 mapped_iterator<typename std::vector<std::shared_ptr<ContextEdge<
4331 DerivedCCG, FuncTy, CallTy>>>::const_iterator,
4332 decltype(&GetCallee)>;
4333
4334 static ChildIteratorType child_begin(NodeRef N) {
4335 return ChildIteratorType(N->CalleeEdges.begin(), &GetCallee);
4336 }
4337
4338 static ChildIteratorType child_end(NodeRef N) {
4339 return ChildIteratorType(N->CalleeEdges.end(), &GetCallee);
4340 }
4341};
4342
4343template <typename DerivedCCG, typename FuncTy, typename CallTy>
4344struct DOTGraphTraits<const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>
4345 : public DefaultDOTGraphTraits {
4346 DOTGraphTraits(bool IsSimple = false) : DefaultDOTGraphTraits(IsSimple) {
4347 // If the user requested the full graph to be exported, but provided an
4348 // allocation id, or if the user gave a context id and requested more than
4349 // just a specific context to be exported, note that highlighting is
4350 // enabled.
4351 DoHighlight =
4352 (AllocIdForDot.getNumOccurrences() && DotGraphScope == DotScope::All) ||
4353 (ContextIdForDot.getNumOccurrences() &&
4354 DotGraphScope != DotScope::Context);
4355 }
4356
4357 using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
4358 using GTraits = GraphTraits<GraphType>;
4359 using NodeRef = typename GTraits::NodeRef;
4360 using ChildIteratorType = typename GTraits::ChildIteratorType;
4361
4362 static std::string getNodeLabel(NodeRef Node, GraphType G) {
4363 std::string LabelString =
4364 (Twine("OrigId: ") + (Node->IsAllocation ? "Alloc" : "") +
4365 Twine(Node->OrigStackOrAllocId) + " NodeId: " + Twine(Node->NodeId))
4366 .str();
4367 LabelString += "\n";
4368 if (Node->hasCall()) {
4369 auto Func = G->NodeToCallingFunc.find(Node);
4370 assert(Func != G->NodeToCallingFunc.end());
4371 LabelString +=
4372 G->getLabel(Func->second, Node->Call.call(), Node->Call.cloneNo());
4373 for (auto &MatchingCall : Node->MatchingCalls) {
4374 LabelString += "\n";
4375 LabelString += G->getLabel(Func->second, MatchingCall.call(),
4376 MatchingCall.cloneNo());
4377 }
4378 } else {
4379 LabelString += "null call";
4380 if (Node->Recursive)
4381 LabelString += " (recursive)";
4382 else
4383 LabelString += " (external)";
4384 }
4385 return LabelString;
4386 }
4387
4388 static std::string getNodeAttributes(NodeRef Node, GraphType G) {
4389 auto ContextIds = Node->getContextIds();
4390 // If highlighting enabled, see if this node contains any of the context ids
4391 // of interest. If so, it will use a different color and a larger fontsize
4392 // (which makes the node larger as well).
4393 bool Highlight = false;
4394 if (DoHighlight) {
4395 assert(ContextIdForDot.getNumOccurrences() ||
4396 AllocIdForDot.getNumOccurrences());
4397 if (ContextIdForDot.getNumOccurrences())
4398 Highlight = ContextIds.contains(ContextIdForDot);
4399 else
4400 Highlight = set_intersects(ContextIds, G->DotAllocContextIds);
4401 }
4402 std::string AttributeString = (Twine("tooltip=\"") + getNodeId(Node) + " " +
4403 getContextIds(ContextIds) + "\"")
4404 .str();
4405 // Default fontsize is 14
4406 if (Highlight)
4407 AttributeString += ",fontsize=\"30\"";
4408 AttributeString +=
4409 (Twine(",fillcolor=\"") + getColor(AllocTypes: Node->AllocTypes, Highlight) + "\"")
4410 .str();
4411 if (Node->CloneOf) {
4412 AttributeString += ",color=\"blue\"";
4413 AttributeString += ",style=\"filled,bold,dashed\"";
4414 } else
4415 AttributeString += ",style=\"filled\"";
4416 return AttributeString;
4417 }
4418
4419 static std::string getEdgeAttributes(NodeRef, ChildIteratorType ChildIter,
4420 GraphType G) {
4421 auto &Edge = *(ChildIter.getCurrent());
4422 // If highlighting enabled, see if this edge contains any of the context ids
4423 // of interest. If so, it will use a different color and a heavier arrow
4424 // size and weight (the larger weight makes the highlighted path
4425 // straighter).
4426 bool Highlight = false;
4427 if (DoHighlight) {
4428 assert(ContextIdForDot.getNumOccurrences() ||
4429 AllocIdForDot.getNumOccurrences());
4430 if (ContextIdForDot.getNumOccurrences())
4431 Highlight = Edge->ContextIds.contains(ContextIdForDot);
4432 else
4433 Highlight = set_intersects(Edge->ContextIds, G->DotAllocContextIds);
4434 }
4435 auto Color = getColor(AllocTypes: Edge->AllocTypes, Highlight);
4436 std::string AttributeString =
4437 (Twine("tooltip=\"") + getContextIds(ContextIds: Edge->ContextIds) + "\"" +
4438 // fillcolor is the arrow head and color is the line
4439 Twine(",fillcolor=\"") + Color + "\"" + Twine(",color=\"") + Color +
4440 "\"")
4441 .str();
4442 if (Edge->IsBackedge)
4443 AttributeString += ",style=\"dotted\"";
4444 // Default penwidth and weight are both 1.
4445 if (Highlight)
4446 AttributeString += ",penwidth=\"2.0\",weight=\"2\"";
4447 return AttributeString;
4448 }
4449
4450 // Since the NodeOwners list includes nodes that are no longer connected to
4451 // the graph, skip them here.
4452 static bool isNodeHidden(NodeRef Node, GraphType G) {
4453 if (Node->isRemoved())
4454 return true;
4455 // If a scope smaller than the full graph was requested, see if this node
4456 // contains any of the context ids of interest.
4457 if (DotGraphScope == DotScope::Alloc)
4458 return !set_intersects(Node->getContextIds(), G->DotAllocContextIds);
4459 if (DotGraphScope == DotScope::Context)
4460 return !Node->getContextIds().contains(ContextIdForDot);
4461 return false;
4462 }
4463
4464private:
4465 static std::string getContextIds(const DenseSet<uint32_t> &ContextIds) {
4466 std::string IdString = "ContextIds:";
4467 if (ContextIds.size() < 100) {
4468 std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
4469 std::sort(first: SortedIds.begin(), last: SortedIds.end());
4470 for (auto Id : SortedIds)
4471 IdString += (" " + Twine(Id)).str();
4472 } else {
4473 IdString += (" (" + Twine(ContextIds.size()) + " ids)").str();
4474 }
4475 return IdString;
4476 }
4477
4478 static std::string getColor(uint8_t AllocTypes, bool Highlight) {
4479 // If DoHighlight is not enabled, we want to use the highlight colors for
4480 // NotCold and Cold, and the non-highlight color for NotCold+Cold. This is
4481 // both compatible with the color scheme before highlighting was supported,
4482 // and for the NotCold+Cold color the non-highlight color is a bit more
4483 // readable.
4484 if (AllocTypes == (uint8_t)AllocationType::NotCold)
4485 // Color "brown1" actually looks like a lighter red.
4486 return !DoHighlight || Highlight ? "brown1" : "lightpink";
4487 if (AllocTypes == (uint8_t)AllocationType::Cold)
4488 return !DoHighlight || Highlight ? "cyan" : "lightskyblue";
4489 if (AllocTypes ==
4490 ((uint8_t)AllocationType::NotCold | (uint8_t)AllocationType::Cold))
4491 return Highlight ? "magenta" : "mediumorchid1";
4492 return "gray";
4493 }
4494
4495 static std::string getNodeId(NodeRef Node) {
4496 std::stringstream SStream;
4497 SStream << std::hex << "N0x" << (unsigned long long)Node;
4498 std::string Result = SStream.str();
4499 return Result;
4500 }
4501
4502 // True if we should highlight a specific context or allocation's contexts in
4503 // the emitted graph.
4504 static bool DoHighlight;
4505};
4506
4507template <typename DerivedCCG, typename FuncTy, typename CallTy>
4508bool DOTGraphTraits<
4509 const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>::DoHighlight =
4510 false;
4511
4512template <typename DerivedCCG, typename FuncTy, typename CallTy>
4513void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::exportToDot(
4514 std::string Label) const {
4515 WriteGraph(this, "", false, Label,
4516 DotFilePathPrefix + "ccg." + Label + ".dot");
4517}
4518
4519template <typename DerivedCCG, typename FuncTy, typename CallTy>
4520typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
4521CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::moveEdgeToNewCalleeClone(
4522 const std::shared_ptr<ContextEdge> &Edge,
4523 DenseSet<uint32_t> ContextIdsToMove) {
4524 ContextNode *Node = Edge->Callee;
4525 assert(NodeToCallingFunc.count(Node));
4526 ContextNode *Clone =
4527 createNewNode(IsAllocation: Node->IsAllocation, F: NodeToCallingFunc[Node], C: Node->Call);
4528 Node->addClone(Clone);
4529 Clone->MatchingCalls = Node->MatchingCalls;
4530 moveEdgeToExistingCalleeClone(Edge, NewCallee: Clone, /*NewClone=*/true,
4531 ContextIdsToMove);
4532 return Clone;
4533}
4534
4535template <typename DerivedCCG, typename FuncTy, typename CallTy>
4536void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
4537 moveEdgeToExistingCalleeClone(const std::shared_ptr<ContextEdge> &Edge,
4538 ContextNode *NewCallee, bool NewClone,
4539 DenseSet<uint32_t> ContextIdsToMove) {
4540 // NewCallee and Edge's current callee must be clones of the same original
4541 // node (Edge's current callee may be the original node too).
4542 assert(NewCallee->getOrigNode() == Edge->Callee->getOrigNode());
4543
4544 bool EdgeIsRecursive = Edge->Callee == Edge->Caller;
4545
4546 ContextNode *OldCallee = Edge->Callee;
4547
4548 // We might already have an edge to the new callee from earlier cloning for a
4549 // different allocation. If one exists we will reuse it.
4550 auto ExistingEdgeToNewCallee = NewCallee->findEdgeFromCaller(Edge->Caller);
4551
4552 // Callers will pass an empty ContextIdsToMove set when they want to move the
4553 // edge. Copy in Edge's ids for simplicity.
4554 if (ContextIdsToMove.empty())
4555 ContextIdsToMove = Edge->getContextIds();
4556
4557 // If we are moving all of Edge's ids, then just move the whole Edge.
4558 // Otherwise only move the specified subset, to a new edge if needed.
4559 if (Edge->getContextIds().size() == ContextIdsToMove.size()) {
4560 // First, update the alloc types on New Callee from Edge.
4561 // Do this before we potentially clear Edge's fields below!
4562 NewCallee->AllocTypes |= Edge->AllocTypes;
4563 // Moving the whole Edge.
4564 if (ExistingEdgeToNewCallee) {
4565 // Since we already have an edge to NewCallee, simply move the ids
4566 // onto it, and remove the existing Edge.
4567 ExistingEdgeToNewCallee->getContextIds().insert_range(ContextIdsToMove);
4568 ExistingEdgeToNewCallee->AllocTypes |= Edge->AllocTypes;
4569 assert(Edge->ContextIds == ContextIdsToMove);
4570 removeEdgeFromGraph(Edge: Edge.get());
4571 } else {
4572 // Otherwise just reconnect Edge to NewCallee.
4573 Edge->Callee = NewCallee;
4574 NewCallee->CallerEdges.push_back(Edge);
4575 // Remove it from callee where it was previously connected.
4576 OldCallee->eraseCallerEdge(Edge.get());
4577 // Don't need to update Edge's context ids since we are simply
4578 // reconnecting it.
4579 }
4580 } else {
4581 // Only moving a subset of Edge's ids.
4582 // Compute the alloc type of the subset of ids being moved.
4583 auto CallerEdgeAllocType = computeAllocType(ContextIds&: ContextIdsToMove);
4584 if (ExistingEdgeToNewCallee) {
4585 // Since we already have an edge to NewCallee, simply move the ids
4586 // onto it.
4587 ExistingEdgeToNewCallee->getContextIds().insert_range(ContextIdsToMove);
4588 ExistingEdgeToNewCallee->AllocTypes |= CallerEdgeAllocType;
4589 } else {
4590 // Otherwise, create a new edge to NewCallee for the ids being moved.
4591 auto NewEdge = std::make_shared<ContextEdge>(
4592 NewCallee, Edge->Caller, CallerEdgeAllocType, ContextIdsToMove);
4593 Edge->Caller->CalleeEdges.push_back(NewEdge);
4594 NewCallee->CallerEdges.push_back(NewEdge);
4595 }
4596 // In either case, need to update the alloc types on NewCallee, and remove
4597 // those ids and update the alloc type on the original Edge.
4598 NewCallee->AllocTypes |= CallerEdgeAllocType;
4599 set_subtract(Edge->ContextIds, ContextIdsToMove);
4600 Edge->AllocTypes = computeAllocType(ContextIds&: Edge->ContextIds);
4601 }
4602 // Now walk the old callee node's callee edges and move Edge's context ids
4603 // over to the corresponding edge into the clone (which is created here if
4604 // this is a newly created clone).
4605 for (auto &OldCalleeEdge : OldCallee->CalleeEdges) {
4606 ContextNode *CalleeToUse = OldCalleeEdge->Callee;
4607 // If this is a direct recursion edge, use NewCallee (the clone) as the
4608 // callee as well, so that any edge updated/created here is also direct
4609 // recursive.
4610 if (CalleeToUse == OldCallee) {
4611 // If this is a recursive edge, see if we already moved a recursive edge
4612 // (which would have to have been this one) - if we were only moving a
4613 // subset of context ids it would still be on OldCallee.
4614 if (EdgeIsRecursive) {
4615 assert(OldCalleeEdge == Edge);
4616 continue;
4617 }
4618 CalleeToUse = NewCallee;
4619 }
4620 // The context ids moving to the new callee are the subset of this edge's
4621 // context ids and the context ids on the caller edge being moved.
4622 DenseSet<uint32_t> EdgeContextIdsToMove =
4623 set_intersection(OldCalleeEdge->getContextIds(), ContextIdsToMove);
4624 set_subtract(OldCalleeEdge->getContextIds(), EdgeContextIdsToMove);
4625 OldCalleeEdge->AllocTypes =
4626 computeAllocType(ContextIds&: OldCalleeEdge->getContextIds());
4627 if (!NewClone) {
4628 // Update context ids / alloc type on corresponding edge to NewCallee.
4629 // There is a chance this may not exist if we are reusing an existing
4630 // clone, specifically during function assignment, where we would have
4631 // removed none type edges after creating the clone. If we can't find
4632 // a corresponding edge there, fall through to the cloning below.
4633 if (auto *NewCalleeEdge = NewCallee->findEdgeFromCallee(CalleeToUse)) {
4634 NewCalleeEdge->getContextIds().insert_range(EdgeContextIdsToMove);
4635 NewCalleeEdge->AllocTypes |= computeAllocType(ContextIds&: EdgeContextIdsToMove);
4636 continue;
4637 }
4638 }
4639 auto NewEdge = std::make_shared<ContextEdge>(
4640 CalleeToUse, NewCallee, computeAllocType(ContextIds&: EdgeContextIdsToMove),
4641 EdgeContextIdsToMove);
4642 NewCallee->CalleeEdges.push_back(NewEdge);
4643 NewEdge->Callee->CallerEdges.push_back(NewEdge);
4644 }
4645 // Recompute the node alloc type now that its callee edges have been
4646 // updated (since we will compute from those edges).
4647 OldCallee->AllocTypes = OldCallee->computeAllocType();
4648 // OldCallee alloc type should be None iff its context id set is now empty.
4649 assert((OldCallee->AllocTypes == (uint8_t)AllocationType::None) ==
4650 OldCallee->emptyContextIds());
4651 if (VerifyCCG) {
4652 checkNode<DerivedCCG, FuncTy, CallTy>(OldCallee, /*CheckEdges=*/false);
4653 checkNode<DerivedCCG, FuncTy, CallTy>(NewCallee, /*CheckEdges=*/false);
4654 for (const auto &OldCalleeEdge : OldCallee->CalleeEdges)
4655 checkNode<DerivedCCG, FuncTy, CallTy>(OldCalleeEdge->Callee,
4656 /*CheckEdges=*/false);
4657 for (const auto &NewCalleeEdge : NewCallee->CalleeEdges)
4658 checkNode<DerivedCCG, FuncTy, CallTy>(NewCalleeEdge->Callee,
4659 /*CheckEdges=*/false);
4660 }
4661}
4662
4663template <typename DerivedCCG, typename FuncTy, typename CallTy>
4664void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
4665 moveCalleeEdgeToNewCaller(const std::shared_ptr<ContextEdge> &Edge,
4666 ContextNode *NewCaller) {
4667 auto *OldCallee = Edge->Callee;
4668 auto *NewCallee = OldCallee;
4669 // If this edge was direct recursive, make any new/updated edge also direct
4670 // recursive to NewCaller.
4671 bool Recursive = Edge->Caller == Edge->Callee;
4672 if (Recursive)
4673 NewCallee = NewCaller;
4674
4675 ContextNode *OldCaller = Edge->Caller;
4676 OldCaller->eraseCalleeEdge(Edge.get());
4677
4678 // We might already have an edge to the new caller. If one exists we will
4679 // reuse it.
4680 auto ExistingEdgeToNewCaller = NewCaller->findEdgeFromCallee(NewCallee);
4681
4682 if (ExistingEdgeToNewCaller) {
4683 // Since we already have an edge to NewCaller, simply move the ids
4684 // onto it, and remove the existing Edge.
4685 ExistingEdgeToNewCaller->getContextIds().insert_range(
4686 Edge->getContextIds());
4687 ExistingEdgeToNewCaller->AllocTypes |= Edge->AllocTypes;
4688 Edge->ContextIds.clear();
4689 Edge->AllocTypes = (uint8_t)AllocationType::None;
4690 OldCallee->eraseCallerEdge(Edge.get());
4691 } else {
4692 // Otherwise just reconnect Edge to NewCaller.
4693 Edge->Caller = NewCaller;
4694 NewCaller->CalleeEdges.push_back(Edge);
4695 if (Recursive) {
4696 assert(NewCallee == NewCaller);
4697 // In the case of (direct) recursive edges, we update the callee as well
4698 // so that it becomes recursive on the new caller.
4699 Edge->Callee = NewCallee;
4700 NewCallee->CallerEdges.push_back(Edge);
4701 OldCallee->eraseCallerEdge(Edge.get());
4702 }
4703 // Don't need to update Edge's context ids since we are simply
4704 // reconnecting it.
4705 }
4706 // In either case, need to update the alloc types on New Caller.
4707 NewCaller->AllocTypes |= Edge->AllocTypes;
4708
4709 // Now walk the old caller node's caller edges and move Edge's context ids
4710 // over to the corresponding edge into the node (which is created here if
4711 // this is a newly created node). We can tell whether this is a newly created
4712 // node by seeing if it has any caller edges yet.
4713#ifndef NDEBUG
4714 bool IsNewNode = NewCaller->CallerEdges.empty();
4715#endif
4716 // If we just moved a direct recursive edge, presumably its context ids should
4717 // also flow out of OldCaller via some other non-recursive callee edge. We
4718 // don't want to remove the recursive context ids from other caller edges yet,
4719 // otherwise the context ids get into an inconsistent state on OldCaller.
4720 // We will update these context ids on the non-recursive caller edge when and
4721 // if they are updated on the non-recursive callee.
4722 if (!Recursive) {
4723 for (auto &OldCallerEdge : OldCaller->CallerEdges) {
4724 auto OldCallerCaller = OldCallerEdge->Caller;
4725 // The context ids moving to the new caller are the subset of this edge's
4726 // context ids and the context ids on the callee edge being moved.
4727 DenseSet<uint32_t> EdgeContextIdsToMove = set_intersection(
4728 OldCallerEdge->getContextIds(), Edge->getContextIds());
4729 if (OldCaller == OldCallerCaller) {
4730 OldCallerCaller = NewCaller;
4731 // Don't actually move this one. The caller will move it directly via a
4732 // call to this function with this as the Edge if it is appropriate to
4733 // move to a diff node that has a matching callee (itself).
4734 continue;
4735 }
4736 set_subtract(OldCallerEdge->getContextIds(), EdgeContextIdsToMove);
4737 OldCallerEdge->AllocTypes =
4738 computeAllocType(ContextIds&: OldCallerEdge->getContextIds());
4739 // In this function we expect that any pre-existing node already has edges
4740 // from the same callers as the old node. That should be true in the
4741 // current use case, where we will remove None-type edges after copying
4742 // over all caller edges from the callee.
4743 auto *ExistingCallerEdge = NewCaller->findEdgeFromCaller(OldCallerCaller);
4744 // Since we would have skipped caller edges when moving a direct recursive
4745 // edge, this may not hold true when recursive handling enabled.
4746 assert(IsNewNode || ExistingCallerEdge || AllowRecursiveCallsites);
4747 if (ExistingCallerEdge) {
4748 ExistingCallerEdge->getContextIds().insert_range(EdgeContextIdsToMove);
4749 ExistingCallerEdge->AllocTypes |=
4750 computeAllocType(ContextIds&: EdgeContextIdsToMove);
4751 continue;
4752 }
4753 auto NewEdge = std::make_shared<ContextEdge>(
4754 NewCaller, OldCallerCaller, computeAllocType(ContextIds&: EdgeContextIdsToMove),
4755 EdgeContextIdsToMove);
4756 NewCaller->CallerEdges.push_back(NewEdge);
4757 NewEdge->Caller->CalleeEdges.push_back(NewEdge);
4758 }
4759 }
4760 // Recompute the node alloc type now that its caller edges have been
4761 // updated (since we will compute from those edges).
4762 OldCaller->AllocTypes = OldCaller->computeAllocType();
4763 // OldCaller alloc type should be None iff its context id set is now empty.
4764 assert((OldCaller->AllocTypes == (uint8_t)AllocationType::None) ==
4765 OldCaller->emptyContextIds());
4766 if (VerifyCCG) {
4767 checkNode<DerivedCCG, FuncTy, CallTy>(OldCaller, /*CheckEdges=*/false);
4768 checkNode<DerivedCCG, FuncTy, CallTy>(NewCaller, /*CheckEdges=*/false);
4769 for (const auto &OldCallerEdge : OldCaller->CallerEdges)
4770 checkNode<DerivedCCG, FuncTy, CallTy>(OldCallerEdge->Caller,
4771 /*CheckEdges=*/false);
4772 for (const auto &NewCallerEdge : NewCaller->CallerEdges)
4773 checkNode<DerivedCCG, FuncTy, CallTy>(NewCallerEdge->Caller,
4774 /*CheckEdges=*/false);
4775 }
4776}
4777
4778template <typename DerivedCCG, typename FuncTy, typename CallTy>
4779void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
4780 recursivelyRemoveNoneTypeCalleeEdges(
4781 ContextNode *Node, DenseSet<const ContextNode *> &Visited) {
4782 auto Inserted = Visited.insert(Node);
4783 if (!Inserted.second)
4784 return;
4785
4786 removeNoneTypeCalleeEdges(Node);
4787
4788 for (auto *Clone : Node->Clones)
4789 recursivelyRemoveNoneTypeCalleeEdges(Node: Clone, Visited);
4790
4791 // The recursive call may remove some of this Node's caller edges.
4792 // Iterate over a copy and skip any that were removed.
4793 auto CallerEdges = Node->CallerEdges;
4794 for (auto &Edge : CallerEdges) {
4795 // Skip any that have been removed by an earlier recursive call.
4796 if (Edge->isRemoved()) {
4797 assert(!is_contained(Node->CallerEdges, Edge));
4798 continue;
4799 }
4800 recursivelyRemoveNoneTypeCalleeEdges(Node: Edge->Caller, Visited);
4801 }
4802}
4803
4804// This is the standard DFS based backedge discovery algorithm.
4805template <typename DerivedCCG, typename FuncTy, typename CallTy>
4806void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::markBackedges() {
4807 // If we are cloning recursive contexts, find and mark backedges from all root
4808 // callers, using the typical DFS based backedge analysis.
4809 if (!CloneRecursiveContexts)
4810 return;
4811 DenseSet<const ContextNode *> Visited;
4812 DenseSet<const ContextNode *> CurrentStack;
4813 for (auto &Entry : NonAllocationCallToContextNodeMap) {
4814 auto *Node = Entry.second;
4815 if (Node->isRemoved())
4816 continue;
4817 // It is a root if it doesn't have callers.
4818 if (!Node->CallerEdges.empty())
4819 continue;
4820 markBackedges(Node, Visited, CurrentStack);
4821 assert(CurrentStack.empty());
4822 }
4823}
4824
4825// Recursive helper for above markBackedges method.
4826template <typename DerivedCCG, typename FuncTy, typename CallTy>
4827void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::markBackedges(
4828 ContextNode *Node, DenseSet<const ContextNode *> &Visited,
4829 DenseSet<const ContextNode *> &CurrentStack) {
4830 auto I = Visited.insert(Node);
4831 // We should only call this for unvisited nodes.
4832 assert(I.second);
4833 (void)I;
4834 for (auto &CalleeEdge : Node->CalleeEdges) {
4835 auto *Callee = CalleeEdge->Callee;
4836 if (Visited.count(Callee)) {
4837 // Since this was already visited we need to check if it is currently on
4838 // the recursive stack in which case it is a backedge.
4839 if (CurrentStack.count(Callee))
4840 CalleeEdge->IsBackedge = true;
4841 continue;
4842 }
4843 CurrentStack.insert(Callee);
4844 markBackedges(Callee, Visited, CurrentStack);
4845 CurrentStack.erase(Callee);
4846 }
4847}
4848
4849template <typename DerivedCCG, typename FuncTy, typename CallTy>
4850void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::identifyClones() {
4851 DenseSet<const ContextNode *> Visited;
4852 for (auto &Entry : AllocationCallToContextNodeMap) {
4853 Visited.clear();
4854 identifyClones(Entry.second, Visited, Entry.second->getContextIds());
4855 }
4856 Visited.clear();
4857 for (auto &Entry : AllocationCallToContextNodeMap)
4858 recursivelyRemoveNoneTypeCalleeEdges(Node: Entry.second, Visited);
4859 if (VerifyCCG)
4860 check();
4861}
4862
4863// helper function to check an AllocType is cold or notcold or both.
4864bool checkColdOrNotCold(uint8_t AllocType) {
4865 return (AllocType == (uint8_t)AllocationType::Cold) ||
4866 (AllocType == (uint8_t)AllocationType::NotCold) ||
4867 (AllocType ==
4868 ((uint8_t)AllocationType::Cold | (uint8_t)AllocationType::NotCold));
4869}
4870
4871template <typename DerivedCCG, typename FuncTy, typename CallTy>
4872void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::identifyClones(
4873 ContextNode *Node, DenseSet<const ContextNode *> &Visited,
4874 const DenseSet<uint32_t> &AllocContextIds) {
4875 if (VerifyNodes)
4876 checkNode<DerivedCCG, FuncTy, CallTy>(Node, /*CheckEdges=*/false);
4877 assert(!Node->CloneOf);
4878
4879 // If Node as a null call, then either it wasn't found in the module (regular
4880 // LTO) or summary index (ThinLTO), or there were other conditions blocking
4881 // cloning (e.g. recursion, calls multiple targets, etc).
4882 // Do this here so that we don't try to recursively clone callers below, which
4883 // isn't useful at least for this node.
4884 if (!Node->hasCall())
4885 return;
4886
4887 // No need to look at any callers if allocation type already unambiguous.
4888 if (hasSingleAllocType(Node->AllocTypes))
4889 return;
4890
4891#ifndef NDEBUG
4892 auto Insert =
4893#endif
4894 Visited.insert(Node);
4895 // We should not have visited this node yet.
4896 assert(Insert.second);
4897 // The recursive call to identifyClones may delete the current edge from the
4898 // CallerEdges vector. Make a copy and iterate on that, simpler than passing
4899 // in an iterator and having recursive call erase from it. Other edges may
4900 // also get removed during the recursion, which will have null Callee and
4901 // Caller pointers (and are deleted later), so we skip those below.
4902 {
4903 auto CallerEdges = Node->CallerEdges;
4904 for (auto &Edge : CallerEdges) {
4905 // Skip any that have been removed by an earlier recursive call.
4906 if (Edge->isRemoved()) {
4907 assert(!is_contained(Node->CallerEdges, Edge));
4908 continue;
4909 }
4910 // Defer backedges. See comments further below where these edges are
4911 // handled during the cloning of this Node.
4912 if (Edge->IsBackedge) {
4913 // We should only mark these if cloning recursive contexts, where we
4914 // need to do this deferral.
4915 assert(CloneRecursiveContexts);
4916 continue;
4917 }
4918 // Ignore any caller we previously visited via another edge.
4919 if (!Visited.count(Edge->Caller) && !Edge->Caller->CloneOf) {
4920 identifyClones(Edge->Caller, Visited, AllocContextIds);
4921 }
4922 }
4923 }
4924
4925 // Check if we reached an unambiguous call or have have only a single caller.
4926 if (hasSingleAllocType(Node->AllocTypes) || Node->CallerEdges.size() <= 1)
4927 return;
4928
4929 // We need to clone.
4930
4931 // Try to keep the original version as alloc type NotCold. This will make
4932 // cases with indirect calls or any other situation with an unknown call to
4933 // the original function get the default behavior. We do this by sorting the
4934 // CallerEdges of the Node we will clone by alloc type.
4935 //
4936 // Give NotCold edge the lowest sort priority so those edges are at the end of
4937 // the caller edges vector, and stay on the original version (since the below
4938 // code clones greedily until it finds all remaining edges have the same type
4939 // and leaves the remaining ones on the original Node).
4940 //
4941 // We shouldn't actually have any None type edges, so the sorting priority for
4942 // that is arbitrary, and we assert in that case below.
4943 const unsigned AllocTypeCloningPriority[] = {/*None*/ 3, /*NotCold*/ 4,
4944 /*Cold*/ 1,
4945 /*NotColdCold*/ 2};
4946 llvm::stable_sort(Node->CallerEdges,
4947 [&](const std::shared_ptr<ContextEdge> &A,
4948 const std::shared_ptr<ContextEdge> &B) {
4949 // Nodes with non-empty context ids should be sorted
4950 // before those with empty context ids.
4951 if (A->ContextIds.empty())
4952 // Either B ContextIds are non-empty (in which case we
4953 // should return false because B < A), or B ContextIds
4954 // are empty, in which case they are equal, and we
4955 // should maintain the original relative ordering.
4956 return false;
4957 if (B->ContextIds.empty())
4958 return true;
4959
4960 if (A->AllocTypes == B->AllocTypes)
4961 // Use the caller node id as a deterministic
4962 // tie-breaker.
4963 return A->Caller->NodeId < B->Caller->NodeId;
4964 return AllocTypeCloningPriority[A->AllocTypes] <
4965 AllocTypeCloningPriority[B->AllocTypes];
4966 });
4967
4968 assert(Node->AllocTypes != (uint8_t)AllocationType::None);
4969
4970 DenseSet<uint32_t> RecursiveContextIds;
4971 assert(AllowRecursiveContexts || !CloneRecursiveContexts);
4972 // If we are allowing recursive callsites, but have also disabled recursive
4973 // contexts, look for context ids that show up in multiple caller edges.
4974 if (AllowRecursiveCallsites && !AllowRecursiveContexts) {
4975 DenseSet<uint32_t> AllCallerContextIds;
4976 for (auto &CE : Node->CallerEdges) {
4977 // Resize to the largest set of caller context ids, since we know the
4978 // final set will be at least that large.
4979 AllCallerContextIds.reserve(Size: CE->getContextIds().size());
4980 for (auto Id : CE->getContextIds())
4981 if (!AllCallerContextIds.insert(Id).second)
4982 RecursiveContextIds.insert(Id);
4983 }
4984 }
4985
4986 // Iterate until we find no more opportunities for disambiguating the alloc
4987 // types via cloning. In most cases this loop will terminate once the Node
4988 // has a single allocation type, in which case no more cloning is needed.
4989 // Iterate over a copy of Node's caller edges, since we may need to remove
4990 // edges in the moveEdgeTo* methods, and this simplifies the handling and
4991 // makes it less error-prone.
4992 auto CallerEdges = Node->CallerEdges;
4993 for (auto &CallerEdge : CallerEdges) {
4994 // Skip any that have been removed by an earlier recursive call.
4995 if (CallerEdge->isRemoved()) {
4996 assert(!is_contained(Node->CallerEdges, CallerEdge));
4997 continue;
4998 }
4999 assert(CallerEdge->Callee == Node);
5000
5001 // See if cloning the prior caller edge left this node with a single alloc
5002 // type or a single caller. In that case no more cloning of Node is needed.
5003 if (hasSingleAllocType(Node->AllocTypes) || Node->CallerEdges.size() <= 1)
5004 break;
5005
5006 // If the caller was not successfully matched to a call in the IR/summary,
5007 // there is no point in trying to clone for it as we can't update that call.
5008 if (!CallerEdge->Caller->hasCall())
5009 continue;
5010
5011 // Only need to process the ids along this edge pertaining to the given
5012 // allocation.
5013 auto CallerEdgeContextsForAlloc =
5014 set_intersection(CallerEdge->getContextIds(), AllocContextIds);
5015 if (!RecursiveContextIds.empty())
5016 CallerEdgeContextsForAlloc =
5017 set_difference(CallerEdgeContextsForAlloc, RecursiveContextIds);
5018 if (CallerEdgeContextsForAlloc.empty())
5019 continue;
5020
5021 auto CallerAllocTypeForAlloc = computeAllocType(ContextIds&: CallerEdgeContextsForAlloc);
5022
5023 // Compute the node callee edge alloc types corresponding to the context ids
5024 // for this caller edge.
5025 std::vector<uint8_t> CalleeEdgeAllocTypesForCallerEdge;
5026 CalleeEdgeAllocTypesForCallerEdge.reserve(n: Node->CalleeEdges.size());
5027 for (auto &CalleeEdge : Node->CalleeEdges)
5028 CalleeEdgeAllocTypesForCallerEdge.push_back(intersectAllocTypes(
5029 Node1Ids: CalleeEdge->getContextIds(), Node2Ids: CallerEdgeContextsForAlloc));
5030
5031 // Don't clone if doing so will not disambiguate any alloc types amongst
5032 // caller edges (including the callee edges that would be cloned).
5033 // Otherwise we will simply move all edges to the clone.
5034 //
5035 // First check if by cloning we will disambiguate the caller allocation
5036 // type from node's allocation type. Query allocTypeToUse so that we don't
5037 // bother cloning to distinguish NotCold+Cold from NotCold. Note that
5038 // neither of these should be None type.
5039 //
5040 // Then check if by cloning node at least one of the callee edges will be
5041 // disambiguated by splitting out different context ids.
5042 //
5043 // However, always do the cloning if this is a backedge, in which case we
5044 // have not yet cloned along this caller edge.
5045 assert(CallerEdge->AllocTypes != (uint8_t)AllocationType::None);
5046 assert(Node->AllocTypes != (uint8_t)AllocationType::None);
5047 if (!CallerEdge->IsBackedge &&
5048 allocTypeToUse(CallerAllocTypeForAlloc) ==
5049 allocTypeToUse(Node->AllocTypes) &&
5050 allocTypesMatch<DerivedCCG, FuncTy, CallTy>(
5051 CalleeEdgeAllocTypesForCallerEdge, Node->CalleeEdges)) {
5052 continue;
5053 }
5054
5055 if (CallerEdge->IsBackedge) {
5056 // We should only mark these if cloning recursive contexts, where we
5057 // need to do this deferral.
5058 assert(CloneRecursiveContexts);
5059 DeferredBackedges++;
5060 }
5061
5062 // If this is a backedge, we now do recursive cloning starting from its
5063 // caller since we may have moved unambiguous caller contexts to a clone
5064 // of this Node in a previous iteration of the current loop, giving more
5065 // opportunity for cloning through the backedge. Because we sorted the
5066 // caller edges earlier so that cold caller edges are first, we would have
5067 // visited and cloned this node for any unamibiguously cold non-recursive
5068 // callers before any ambiguous backedge callers. Note that we don't do this
5069 // if the caller is already cloned or visited during cloning (e.g. via a
5070 // different context path from the allocation).
5071 // TODO: Can we do better in the case where the caller was already visited?
5072 if (CallerEdge->IsBackedge && !CallerEdge->Caller->CloneOf &&
5073 !Visited.count(CallerEdge->Caller)) {
5074 const auto OrigIdCount = CallerEdge->getContextIds().size();
5075 // Now do the recursive cloning of this backedge's caller, which was
5076 // deferred earlier.
5077 identifyClones(CallerEdge->Caller, Visited, CallerEdgeContextsForAlloc);
5078 removeNoneTypeCalleeEdges(Node: CallerEdge->Caller);
5079 // See if the recursive call to identifyClones moved the context ids to a
5080 // new edge from this node to a clone of caller, and switch to looking at
5081 // that new edge so that we clone Node for the new caller clone.
5082 bool UpdatedEdge = false;
5083 if (OrigIdCount > CallerEdge->getContextIds().size()) {
5084 for (auto E : Node->CallerEdges) {
5085 // Only interested in clones of the current edges caller.
5086 if (E->Caller->CloneOf != CallerEdge->Caller)
5087 continue;
5088 // See if this edge contains any of the context ids originally on the
5089 // current caller edge.
5090 auto CallerEdgeContextsForAllocNew =
5091 set_intersection(CallerEdgeContextsForAlloc, E->getContextIds());
5092 if (CallerEdgeContextsForAllocNew.empty())
5093 continue;
5094 // Make sure we don't pick a previously existing caller edge of this
5095 // Node, which would be processed on a different iteration of the
5096 // outer loop over the saved CallerEdges.
5097 if (llvm::is_contained(CallerEdges, E))
5098 continue;
5099 // The CallerAllocTypeForAlloc and CalleeEdgeAllocTypesForCallerEdge
5100 // are updated further below for all cases where we just invoked
5101 // identifyClones recursively.
5102 CallerEdgeContextsForAlloc.swap(CallerEdgeContextsForAllocNew);
5103 CallerEdge = E;
5104 UpdatedEdge = true;
5105 break;
5106 }
5107 }
5108 // If cloning removed this edge (and we didn't update it to a new edge
5109 // above), we're done with this edge. It's possible we moved all of the
5110 // context ids to an existing clone, in which case there's no need to do
5111 // further processing for them.
5112 if (CallerEdge->isRemoved())
5113 continue;
5114
5115 // Now we need to update the information used for the cloning decisions
5116 // further below, as we may have modified edges and their context ids.
5117
5118 // Note if we changed the CallerEdge above we would have already updated
5119 // the context ids.
5120 if (!UpdatedEdge) {
5121 CallerEdgeContextsForAlloc = set_intersection(
5122 CallerEdgeContextsForAlloc, CallerEdge->getContextIds());
5123 if (CallerEdgeContextsForAlloc.empty())
5124 continue;
5125 }
5126 // Update the other information that depends on the edges and on the now
5127 // updated CallerEdgeContextsForAlloc.
5128 CallerAllocTypeForAlloc = computeAllocType(ContextIds&: CallerEdgeContextsForAlloc);
5129 CalleeEdgeAllocTypesForCallerEdge.clear();
5130 for (auto &CalleeEdge : Node->CalleeEdges) {
5131 CalleeEdgeAllocTypesForCallerEdge.push_back(intersectAllocTypes(
5132 Node1Ids: CalleeEdge->getContextIds(), Node2Ids: CallerEdgeContextsForAlloc));
5133 }
5134 }
5135
5136 // First see if we can use an existing clone. Check each clone and its
5137 // callee edges for matching alloc types.
5138 ContextNode *Clone = nullptr;
5139 for (auto *CurClone : Node->Clones) {
5140 if (allocTypeToUse(CurClone->AllocTypes) !=
5141 allocTypeToUse(CallerAllocTypeForAlloc))
5142 continue;
5143
5144 bool BothSingleAlloc = hasSingleAllocType(CurClone->AllocTypes) &&
5145 hasSingleAllocType(CallerAllocTypeForAlloc);
5146 // The above check should mean that if both have single alloc types that
5147 // they should be equal.
5148 assert(!BothSingleAlloc ||
5149 CurClone->AllocTypes == CallerAllocTypeForAlloc);
5150
5151 // If either both have a single alloc type (which are the same), or if the
5152 // clone's callee edges have the same alloc types as those for the current
5153 // allocation on Node's callee edges (CalleeEdgeAllocTypesForCallerEdge),
5154 // then we can reuse this clone.
5155 if (BothSingleAlloc || allocTypesMatchClone<DerivedCCG, FuncTy, CallTy>(
5156 CalleeEdgeAllocTypesForCallerEdge, CurClone)) {
5157 Clone = CurClone;
5158 break;
5159 }
5160 }
5161
5162 // The edge iterator is adjusted when we move the CallerEdge to the clone.
5163 if (Clone)
5164 moveEdgeToExistingCalleeClone(Edge: CallerEdge, NewCallee: Clone, /*NewClone=*/false,
5165 ContextIdsToMove: CallerEdgeContextsForAlloc);
5166 else
5167 Clone = moveEdgeToNewCalleeClone(Edge: CallerEdge, ContextIdsToMove: CallerEdgeContextsForAlloc);
5168
5169 // Sanity check that no alloc types on clone or its edges are None.
5170 assert(Clone->AllocTypes != (uint8_t)AllocationType::None);
5171 }
5172
5173 // We should still have some context ids on the original Node.
5174 assert(!Node->emptyContextIds());
5175
5176 // Sanity check that no alloc types on node or edges are None.
5177 assert(Node->AllocTypes != (uint8_t)AllocationType::None);
5178
5179 if (VerifyNodes)
5180 checkNode<DerivedCCG, FuncTy, CallTy>(Node, /*CheckEdges=*/false);
5181}
5182
5183void ModuleCallsiteContextGraph::updateAllocationCall(
5184 CallInfo &Call, AllocationType AllocType) {
5185 std::string AllocTypeString = getAllocTypeAttributeString(Type: AllocType);
5186 auto *CB = cast<CallBase>(Val: Call.call());
5187 if (isAmbiguousAllocType(AllocTypes: (uint8_t)AllocType)) {
5188 // The contexts reaching this allocation could not be disambiguated, so it
5189 // keeps (or gets) the ambiguous attribute added during matching rather than
5190 // the default not cold attribute.
5191 addAmbiguousAttribute(CB);
5192 } else {
5193 removeAnyExistingAmbiguousAttribute(CB);
5194 auto A = llvm::Attribute::get(Context&: CB->getFunction()->getContext(), Kind: "memprof",
5195 Val: AllocTypeString);
5196 CB->addFnAttr(Attr: A);
5197 }
5198 OREGetter(Call.call()->getFunction())
5199 .emit(OptDiag: OptimizationRemark(DEBUG_TYPE, "MemprofAttribute", Call.call())
5200 << ore::NV("AllocationCall", Call.call()) << " in clone "
5201 << ore::NV("Caller", Call.call()->getFunction())
5202 << " marked with memprof allocation attribute "
5203 << ore::NV("Attribute", AllocTypeString));
5204}
5205
5206void IndexCallsiteContextGraph::updateAllocationCall(CallInfo &Call,
5207 AllocationType AllocType) {
5208 auto *AI = cast<AllocInfo *>(Val: Call.call());
5209 assert(AI);
5210 assert(AI->Versions.size() > Call.cloneNo());
5211 AI->Versions[Call.cloneNo()] = (uint8_t)AllocType;
5212}
5213
5214AllocationType
5215ModuleCallsiteContextGraph::getAllocationCallType(const CallInfo &Call) const {
5216 const auto *CB = cast<CallBase>(Val: Call.call());
5217 if (!CB->getAttributes().hasFnAttr(Kind: "memprof"))
5218 return AllocationType::None;
5219 StringRef AllocTypeString =
5220 CB->getAttributes().getFnAttr(Kind: "memprof").getValueAsString();
5221 if (AllocTypeString == "cold")
5222 return AllocationType::Cold;
5223 if (AllocTypeString == "ambiguous")
5224 return (AllocationType)((uint8_t)AllocationType::NotCold |
5225 (uint8_t)AllocationType::Cold);
5226 return AllocationType::NotCold;
5227}
5228
5229AllocationType
5230IndexCallsiteContextGraph::getAllocationCallType(const CallInfo &Call) const {
5231 const auto *AI = cast<AllocInfo *>(Val: Call.call());
5232 assert(AI->Versions.size() > Call.cloneNo());
5233 return (AllocationType)AI->Versions[Call.cloneNo()];
5234}
5235
5236void ModuleCallsiteContextGraph::updateCall(CallInfo &CallerCall,
5237 FuncInfo CalleeFunc) {
5238 auto *CurF = getCalleeFunc(Call: CallerCall.call());
5239 auto NewCalleeCloneNo = CalleeFunc.cloneNo();
5240 if (isMemProfClone(GV: *CurF)) {
5241 // If we already assigned this callsite to call a specific non-default
5242 // clone (i.e. not the original function which is clone 0), ensure that we
5243 // aren't trying to now update it to call a different clone, which is
5244 // indicative of a bug in the graph or function assignment.
5245 auto CurCalleeCloneNo = getMemProfCloneNum(F: *CurF);
5246 if (CurCalleeCloneNo != NewCalleeCloneNo) {
5247 LLVM_DEBUG(dbgs() << "Mismatch in call clone assignment: was "
5248 << CurCalleeCloneNo << " now " << NewCalleeCloneNo
5249 << "\n");
5250 MismatchedCloneAssignments++;
5251 }
5252 }
5253 if (NewCalleeCloneNo > 0)
5254 cast<CallBase>(Val: CallerCall.call())->setCalledFunction(CalleeFunc.func());
5255 OREGetter(CallerCall.call()->getFunction())
5256 .emit(OptDiag: OptimizationRemark(DEBUG_TYPE, "MemprofCall", CallerCall.call())
5257 << ore::NV("Call", CallerCall.call()) << " in clone "
5258 << ore::NV("Caller", CallerCall.call()->getFunction())
5259 << " assigned to call function clone "
5260 << ore::NV("Callee", CalleeFunc.func()));
5261}
5262
5263void IndexCallsiteContextGraph::updateCall(CallInfo &CallerCall,
5264 FuncInfo CalleeFunc) {
5265 auto *CI = cast<CallsiteInfo *>(Val: CallerCall.call());
5266 assert(CI &&
5267 "Caller cannot be an allocation which should not have profiled calls");
5268 assert(CI->Clones.size() > CallerCall.cloneNo());
5269 auto NewCalleeCloneNo = CalleeFunc.cloneNo();
5270 auto &CurCalleeCloneNo = CI->Clones[CallerCall.cloneNo()];
5271 // If we already assigned this callsite to call a specific non-default
5272 // clone (i.e. not the original function which is clone 0), ensure that we
5273 // aren't trying to now update it to call a different clone, which is
5274 // indicative of a bug in the graph or function assignment.
5275 if (CurCalleeCloneNo != 0 && CurCalleeCloneNo != NewCalleeCloneNo) {
5276 LLVM_DEBUG(dbgs() << "Mismatch in call clone assignment: was "
5277 << CurCalleeCloneNo << " now " << NewCalleeCloneNo
5278 << "\n");
5279 MismatchedCloneAssignments++;
5280 }
5281 CurCalleeCloneNo = NewCalleeCloneNo;
5282}
5283
5284// Update the debug information attached to NewFunc to use the clone Name. Note
5285// this needs to be done for both any existing DISubprogram for the definition,
5286// as well as any separate declaration DISubprogram.
5287static void updateSubprogramLinkageName(Function *NewFunc, StringRef Name) {
5288 assert(Name == NewFunc->getName());
5289 auto *SP = NewFunc->getSubprogram();
5290 if (!SP)
5291 return;
5292 auto *MDName = MDString::get(Context&: NewFunc->getParent()->getContext(), Str: Name);
5293 SP->replaceLinkageName(LN: MDName);
5294 DISubprogram *Decl = SP->getDeclaration();
5295 if (!Decl)
5296 return;
5297 TempDISubprogram NewDecl = Decl->clone();
5298 NewDecl->replaceLinkageName(LN: MDName);
5299 SP->replaceDeclaration(Decl: MDNode::replaceWithUniqued(N: std::move(NewDecl)));
5300}
5301
5302CallsiteContextGraph<ModuleCallsiteContextGraph, Function,
5303 Instruction *>::FuncInfo
5304ModuleCallsiteContextGraph::cloneFunctionForCallsite(
5305 FuncInfo &Func, CallInfo &Call, DenseMap<CallInfo, CallInfo> &CallMap,
5306 std::vector<CallInfo> &CallsWithMetadataInFunc, unsigned CloneNo) {
5307 // Use existing LLVM facilities for cloning and obtaining Call in clone
5308 ValueToValueMapTy VMap;
5309 auto *NewFunc = CloneFunction(F: Func.func(), VMap);
5310 // Clones are only called directly and never have their address taken, so
5311 // strip !kcfi_type to avoid emitting KCFI type preambles/symbols on them or
5312 // making them valid KCFI indirect-call targets.
5313 NewFunc->setMetadata(KindID: LLVMContext::MD_kcfi_type, Node: nullptr);
5314 std::string Name = getMemProfFuncName(Base: Func.func()->getName(), CloneNo);
5315 assert(!Func.func()->getParent()->getFunction(Name));
5316 NewFunc->setName(Name);
5317 updateSubprogramLinkageName(NewFunc, Name);
5318 for (auto &Inst : CallsWithMetadataInFunc) {
5319 // This map always has the initial version in it.
5320 assert(Inst.cloneNo() == 0);
5321 CallMap[Inst] = {cast<Instruction>(Val&: VMap[Inst.call()]), CloneNo};
5322 }
5323 OREGetter(Func.func())
5324 .emit(OptDiag: OptimizationRemark(DEBUG_TYPE, "MemprofClone", Func.func())
5325 << "created clone " << ore::NV("NewFunction", NewFunc));
5326 return {NewFunc, CloneNo};
5327}
5328
5329CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
5330 IndexCall>::FuncInfo
5331IndexCallsiteContextGraph::cloneFunctionForCallsite(
5332 FuncInfo &Func, CallInfo &Call, DenseMap<CallInfo, CallInfo> &CallMap,
5333 std::vector<CallInfo> &CallsWithMetadataInFunc, unsigned CloneNo) {
5334 // Check how many clones we have of Call (and therefore function).
5335 // The next clone number is the current size of versions array.
5336 // Confirm this matches the CloneNo provided by the caller, which is based on
5337 // the number of function clones we have.
5338 assert(CloneNo == (isa<AllocInfo *>(Call.call())
5339 ? cast<AllocInfo *>(Call.call())->Versions.size()
5340 : cast<CallsiteInfo *>(Call.call())->Clones.size()));
5341 // Walk all the instructions in this function. Create a new version for
5342 // each (by adding an entry to the Versions/Clones summary array), and copy
5343 // over the version being called for the function clone being cloned here.
5344 // Additionally, add an entry to the CallMap for the new function clone,
5345 // mapping the original call (clone 0, what is in CallsWithMetadataInFunc)
5346 // to the new call clone.
5347 for (auto &Inst : CallsWithMetadataInFunc) {
5348 // This map always has the initial version in it.
5349 assert(Inst.cloneNo() == 0);
5350 if (auto *AI = dyn_cast<AllocInfo *>(Val: Inst.call())) {
5351 assert(AI->Versions.size() == CloneNo);
5352 // We assign the allocation type later (in updateAllocationCall), just add
5353 // an entry for it here.
5354 AI->Versions.push_back(Elt: 0);
5355 } else {
5356 auto *CI = cast<CallsiteInfo *>(Val: Inst.call());
5357 assert(CI && CI->Clones.size() == CloneNo);
5358 // We assign the clone number later (in updateCall), just add an entry for
5359 // it here.
5360 CI->Clones.push_back(Elt: 0);
5361 }
5362 CallMap[Inst] = {Inst.call(), CloneNo};
5363 }
5364 return {Func.func(), CloneNo};
5365}
5366
5367// We perform cloning for each allocation node separately. However, this
5368// sometimes results in a situation where the same node calls multiple
5369// clones of the same callee, created for different allocations. This
5370// causes issues when assigning functions to these clones, as each node can
5371// in reality only call a single callee clone.
5372//
5373// To address this, before assigning functions, merge callee clone nodes as
5374// needed using a post order traversal from the allocations. We attempt to
5375// use existing clones as the merge node when legal, and to share them
5376// among callers with the same properties (callers calling the same set of
5377// callee clone nodes for the same allocations).
5378//
5379// Without this fix, in some cases incorrect function assignment will lead
5380// to calling the wrong allocation clone.
5381template <typename DerivedCCG, typename FuncTy, typename CallTy>
5382void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::mergeClones() {
5383 if (!MergeClones)
5384 return;
5385
5386 // Generate a map from context id to the associated allocation node for use
5387 // when merging clones.
5388 DenseMap<uint32_t, ContextNode *> ContextIdToAllocationNode;
5389 for (auto &Entry : AllocationCallToContextNodeMap) {
5390 auto *Node = Entry.second;
5391 for (auto Id : Node->getContextIds())
5392 ContextIdToAllocationNode[Id] = Node->getOrigNode();
5393 for (auto *Clone : Node->Clones) {
5394 for (auto Id : Clone->getContextIds())
5395 ContextIdToAllocationNode[Id] = Clone->getOrigNode();
5396 }
5397 }
5398
5399 // Post order traversal starting from allocations to ensure each callsite
5400 // calls a single clone of its callee. Callee nodes that are clones of each
5401 // other are merged (via new merge nodes if needed) to achieve this.
5402 DenseSet<const ContextNode *> Visited;
5403 for (auto &Entry : AllocationCallToContextNodeMap) {
5404 auto *Node = Entry.second;
5405
5406 mergeClones(Node, Visited, ContextIdToAllocationNode);
5407
5408 // Make a copy so the recursive post order traversal that may create new
5409 // clones doesn't mess up iteration. Note that the recursive traversal
5410 // itself does not call mergeClones on any of these nodes, which are all
5411 // (clones of) allocations.
5412 auto Clones = Node->Clones;
5413 for (auto *Clone : Clones)
5414 mergeClones(Clone, Visited, ContextIdToAllocationNode);
5415 }
5416
5417 if (DumpCCG) {
5418 dbgs() << "CCG after merging:\n";
5419 dbgs() << *this;
5420 }
5421 if (ExportToDot)
5422 exportToDot(Label: "aftermerge");
5423
5424 if (VerifyCCG) {
5425 check();
5426 }
5427}
5428
5429// Recursive helper for above mergeClones method.
5430template <typename DerivedCCG, typename FuncTy, typename CallTy>
5431void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::mergeClones(
5432 ContextNode *Node, DenseSet<const ContextNode *> &Visited,
5433 DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode) {
5434 auto Inserted = Visited.insert(Node);
5435 if (!Inserted.second)
5436 return;
5437
5438 // Iteratively perform merging on this node to handle new caller nodes created
5439 // during the recursive traversal. We could do something more elegant such as
5440 // maintain a worklist, but this is a simple approach that doesn't cause a
5441 // measureable compile time effect, as most nodes don't have many caller
5442 // edges to check.
5443 bool FoundUnvisited = true;
5444 unsigned Iters = 0;
5445 while (FoundUnvisited) {
5446 Iters++;
5447 FoundUnvisited = false;
5448 // Make a copy since the recursive call may move a caller edge to a new
5449 // callee, messing up the iterator.
5450 auto CallerEdges = Node->CallerEdges;
5451 for (auto CallerEdge : CallerEdges) {
5452 // Skip any caller edge moved onto a different callee during recursion.
5453 if (CallerEdge->Callee != Node)
5454 continue;
5455 // If we found an unvisited caller, note that we should check the caller
5456 // edges again as mergeClones may add or change caller nodes.
5457 if (DoMergeIteration && !Visited.contains(CallerEdge->Caller))
5458 FoundUnvisited = true;
5459 mergeClones(CallerEdge->Caller, Visited, ContextIdToAllocationNode);
5460 }
5461 }
5462
5463 TotalMergeInvokes++;
5464 TotalMergeIters += Iters;
5465 if (Iters > MaxMergeIters)
5466 MaxMergeIters = Iters;
5467
5468 // Merge for this node after we handle its callers.
5469 mergeNodeCalleeClones(Node, Visited, ContextIdToAllocationNode);
5470}
5471
5472template <typename DerivedCCG, typename FuncTy, typename CallTy>
5473void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::mergeNodeCalleeClones(
5474 ContextNode *Node, DenseSet<const ContextNode *> &Visited,
5475 DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode) {
5476 // Ignore Node if we moved all of its contexts to clones.
5477 if (Node->emptyContextIds())
5478 return;
5479
5480 // First identify groups of clones among Node's callee edges, by building
5481 // a map from each callee base node to the associated callee edges from Node.
5482 MapVector<ContextNode *, std::vector<std::shared_ptr<ContextEdge>>>
5483 OrigNodeToCloneEdges;
5484 for (const auto &E : Node->CalleeEdges) {
5485 auto *Callee = E->Callee;
5486 if (!Callee->CloneOf && Callee->Clones.empty())
5487 continue;
5488 ContextNode *Base = Callee->getOrigNode();
5489 OrigNodeToCloneEdges[Base].push_back(E);
5490 }
5491
5492 // Helper for callee edge sorting below. Return true if A's callee has fewer
5493 // caller edges than B, or if A is a clone and B is not, or if A's callee
5494 // node id is smaller than B's.
5495 auto CalleeCallerEdgeLessThan = [](const std::shared_ptr<ContextEdge> &A,
5496 const std::shared_ptr<ContextEdge> &B) {
5497 if (A->Callee->CallerEdges.size() != B->Callee->CallerEdges.size())
5498 return A->Callee->CallerEdges.size() < B->Callee->CallerEdges.size();
5499 if (A->Callee->CloneOf && !B->Callee->CloneOf)
5500 return true;
5501 else if (!A->Callee->CloneOf && B->Callee->CloneOf)
5502 return false;
5503 // Use the callee node id as a deterministic tie-breaker.
5504 return A->Callee->NodeId < B->Callee->NodeId;
5505 };
5506
5507 // Process each set of callee clones called by Node, performing the needed
5508 // merging.
5509 for (auto Entry : OrigNodeToCloneEdges) {
5510 // CalleeEdges is the set of edges from Node reaching callees that are
5511 // mutual clones of each other.
5512 auto &CalleeEdges = Entry.second;
5513 auto NumCalleeClones = CalleeEdges.size();
5514 // A single edge means there is no merging needed.
5515 if (NumCalleeClones == 1)
5516 continue;
5517 // Sort the CalleeEdges calling this group of clones in ascending order of
5518 // their caller edge counts, putting the original non-clone node first in
5519 // cases of a tie. This simplifies finding an existing node to use as the
5520 // merge node.
5521 llvm::stable_sort(CalleeEdges, CalleeCallerEdgeLessThan);
5522
5523 /// Find other callers of the given set of callee edges that can
5524 /// share the same callee merge node. See the comments at this method
5525 /// definition for details.
5526 DenseSet<ContextNode *> OtherCallersToShareMerge;
5527 findOtherCallersToShareMerge(Node, CalleeEdges, ContextIdToAllocationNode,
5528 OtherCallersToShareMerge);
5529
5530 // Now do the actual merging. Identify existing or create a new MergeNode
5531 // during the first iteration. Move each callee over, along with edges from
5532 // other callers we've determined above can share the same merge node.
5533 ContextNode *MergeNode = nullptr;
5534 DenseMap<ContextNode *, unsigned> CallerToMoveCount;
5535 for (auto CalleeEdge : CalleeEdges) {
5536 auto *OrigCallee = CalleeEdge->Callee;
5537 // If we don't have a MergeNode yet (only happens on the first iteration,
5538 // as a new one will be created when we go to move the first callee edge
5539 // over as needed), see if we can use this callee.
5540 if (!MergeNode) {
5541 // If there are no other callers, simply use this callee.
5542 if (CalleeEdge->Callee->CallerEdges.size() == 1) {
5543 MergeNode = OrigCallee;
5544 NonNewMergedNodes++;
5545 continue;
5546 }
5547 // Otherwise, if we have identified other caller nodes that can share
5548 // the merge node with Node, see if all of OrigCallee's callers are
5549 // going to share the same merge node. In that case we can use callee
5550 // (since all of its callers would move to the new merge node).
5551 if (!OtherCallersToShareMerge.empty()) {
5552 bool MoveAllCallerEdges = true;
5553 for (auto CalleeCallerE : OrigCallee->CallerEdges) {
5554 if (CalleeCallerE == CalleeEdge)
5555 continue;
5556 if (!OtherCallersToShareMerge.contains(CalleeCallerE->Caller)) {
5557 MoveAllCallerEdges = false;
5558 break;
5559 }
5560 }
5561 // If we are going to move all callers over, we can use this callee as
5562 // the MergeNode.
5563 if (MoveAllCallerEdges) {
5564 MergeNode = OrigCallee;
5565 NonNewMergedNodes++;
5566 continue;
5567 }
5568 }
5569 }
5570 // Move this callee edge, creating a new merge node if necessary.
5571 if (MergeNode) {
5572 assert(MergeNode != OrigCallee);
5573 moveEdgeToExistingCalleeClone(Edge: CalleeEdge, NewCallee: MergeNode,
5574 /*NewClone*/ false);
5575 } else {
5576 MergeNode = moveEdgeToNewCalleeClone(Edge: CalleeEdge);
5577 NewMergedNodes++;
5578 }
5579 // Now move all identified edges from other callers over to the merge node
5580 // as well.
5581 if (!OtherCallersToShareMerge.empty()) {
5582 // Make and iterate over a copy of OrigCallee's caller edges because
5583 // some of these will be moved off of the OrigCallee and that would mess
5584 // up the iteration from OrigCallee.
5585 auto OrigCalleeCallerEdges = OrigCallee->CallerEdges;
5586 for (auto &CalleeCallerE : OrigCalleeCallerEdges) {
5587 if (CalleeCallerE == CalleeEdge)
5588 continue;
5589 if (!OtherCallersToShareMerge.contains(CalleeCallerE->Caller))
5590 continue;
5591 CallerToMoveCount[CalleeCallerE->Caller]++;
5592 moveEdgeToExistingCalleeClone(Edge: CalleeCallerE, NewCallee: MergeNode,
5593 /*NewClone*/ false);
5594 }
5595 }
5596 removeNoneTypeCalleeEdges(Node: OrigCallee);
5597 removeNoneTypeCalleeEdges(Node: MergeNode);
5598 }
5599 }
5600}
5601
5602// Look for other nodes that have edges to the same set of callee
5603// clones as the current Node. Those can share the eventual merge node
5604// (reducing cloning and binary size overhead) iff:
5605// - they have edges to the same set of callee clones
5606// - each callee edge reaches a subset of the same allocations as Node's
5607// corresponding edge to the same callee clone.
5608// The second requirement is to ensure that we don't undo any of the
5609// necessary cloning to distinguish contexts with different allocation
5610// behavior.
5611// FIXME: This is somewhat conservative, as we really just need to ensure
5612// that they don't reach the same allocations as contexts on edges from Node
5613// going to any of the *other* callee clones being merged. However, that
5614// requires more tracking and checking to get right.
5615template <typename DerivedCCG, typename FuncTy, typename CallTy>
5616void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
5617 findOtherCallersToShareMerge(
5618 ContextNode *Node,
5619 std::vector<std::shared_ptr<ContextEdge>> &CalleeEdges,
5620 DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode,
5621 DenseSet<ContextNode *> &OtherCallersToShareMerge) {
5622 auto NumCalleeClones = CalleeEdges.size();
5623 // This map counts how many edges to the same callee clone exist for other
5624 // caller nodes of each callee clone.
5625 DenseMap<ContextNode *, unsigned> OtherCallersToSharedCalleeEdgeCount;
5626 // Counts the number of other caller nodes that have edges to all callee
5627 // clones that don't violate the allocation context checking.
5628 unsigned PossibleOtherCallerNodes = 0;
5629
5630 // We only need to look at other Caller nodes if the first callee edge has
5631 // multiple callers (recall they are sorted in ascending order above).
5632 if (CalleeEdges[0]->Callee->CallerEdges.size() < 2)
5633 return;
5634
5635 // For each callee edge:
5636 // - Collect the count of other caller nodes calling the same callees.
5637 // - Collect the alloc nodes reached by contexts on each callee edge.
5638 DenseMap<ContextEdge *, DenseSet<ContextNode *>> CalleeEdgeToAllocNodes;
5639 for (auto CalleeEdge : CalleeEdges) {
5640 assert(CalleeEdge->Callee->CallerEdges.size() > 1);
5641 // For each other caller of the same callee, increment the count of
5642 // edges reaching the same callee clone.
5643 for (auto CalleeCallerEdges : CalleeEdge->Callee->CallerEdges) {
5644 if (CalleeCallerEdges->Caller == Node) {
5645 assert(CalleeCallerEdges == CalleeEdge);
5646 continue;
5647 }
5648 OtherCallersToSharedCalleeEdgeCount[CalleeCallerEdges->Caller]++;
5649 // If this caller edge now reaches all of the same callee clones,
5650 // increment the count of candidate other caller nodes.
5651 if (OtherCallersToSharedCalleeEdgeCount[CalleeCallerEdges->Caller] ==
5652 NumCalleeClones)
5653 PossibleOtherCallerNodes++;
5654 }
5655 // Collect the alloc nodes reached by contexts on each callee edge, for
5656 // later analysis.
5657 for (auto Id : CalleeEdge->getContextIds()) {
5658 auto *Alloc = ContextIdToAllocationNode.lookup(Id);
5659 if (!Alloc) {
5660 // FIXME: unclear why this happens occasionally, presumably
5661 // imperfect graph updates possibly with recursion.
5662 MissingAllocForContextId++;
5663 continue;
5664 }
5665 CalleeEdgeToAllocNodes[CalleeEdge.get()].insert(Alloc);
5666 }
5667 }
5668
5669 // Now walk the callee edges again, and make sure that for each candidate
5670 // caller node all of its edges to the callees reach the same allocs (or
5671 // a subset) as those along the corresponding callee edge from Node.
5672 for (auto CalleeEdge : CalleeEdges) {
5673 assert(CalleeEdge->Callee->CallerEdges.size() > 1);
5674 // Stop if we do not have any (more) candidate other caller nodes.
5675 if (!PossibleOtherCallerNodes)
5676 break;
5677 auto &CurCalleeAllocNodes = CalleeEdgeToAllocNodes[CalleeEdge.get()];
5678 // Check each other caller of this callee clone.
5679 for (auto &CalleeCallerE : CalleeEdge->Callee->CallerEdges) {
5680 // Not interested in the callee edge from Node itself.
5681 if (CalleeCallerE == CalleeEdge)
5682 continue;
5683 // Skip any callers that didn't have callee edges to all the same
5684 // callee clones.
5685 if (OtherCallersToSharedCalleeEdgeCount[CalleeCallerE->Caller] !=
5686 NumCalleeClones)
5687 continue;
5688 // Make sure that each context along edge from candidate caller node
5689 // reaches an allocation also reached by this callee edge from Node.
5690 for (auto Id : CalleeCallerE->getContextIds()) {
5691 auto *Alloc = ContextIdToAllocationNode.lookup(Id);
5692 if (!Alloc)
5693 continue;
5694 // If not, simply reset the map entry to 0 so caller is ignored, and
5695 // reduce the count of candidate other caller nodes.
5696 if (!CurCalleeAllocNodes.contains(Alloc)) {
5697 OtherCallersToSharedCalleeEdgeCount[CalleeCallerE->Caller] = 0;
5698 PossibleOtherCallerNodes--;
5699 break;
5700 }
5701 }
5702 }
5703 }
5704
5705 if (!PossibleOtherCallerNodes)
5706 return;
5707
5708 // Build the set of other caller nodes that can use the same callee merge
5709 // node.
5710 for (auto &[OtherCaller, Count] : OtherCallersToSharedCalleeEdgeCount) {
5711 if (Count != NumCalleeClones)
5712 continue;
5713 OtherCallersToShareMerge.insert(OtherCaller);
5714 }
5715}
5716
5717template <typename DerivedCCG, typename FuncTy, typename CallTy>
5718const typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
5719CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::resolveImplicitSource(
5720 const ContextNode *Node, const ImplicitSource &Src) {
5721 if (!Src.Source->emptyContextIds())
5722 return Src.Source;
5723 // Src.Source lost its contexts to another clone (e.g. its last caller was
5724 // moved to a new clone when specializing a function clone further). The
5725 // trimmed contexts stay together, so the clone carrying one of them is the
5726 // version the implicit caller requires.
5727 if (Node->hasContextId(Src.ContextId))
5728 return Node;
5729 for (const ContextNode *C : Node->Clones)
5730 if (C->hasContextId(Src.ContextId))
5731 return C;
5732 // Context ids only move between the clones of a node during function
5733 // assignment, so the id should still be on one of them. Return null, which
5734 // makes the call follow the original node, rather than an emptied clone.
5735 assert(false && "trimmed context id on no clone of its callsite node");
5736 return nullptr;
5737}
5738
5739// Contexts whose call stacks were trimmed at a callsite node's frame are on
5740// its callee edges but on none of its caller edges. When that node, or one
5741// below it on the path of such a context, is cloned for the contexts of
5742// another allocation reaching it through a caller edge, the trimmed contexts
5743// stay with the node that was cloned, and the clone gets no edge to the callee
5744// nodes the trimmed contexts flow through. The clone is therefore not a caller
5745// of any clone of such a callee node, and would otherwise be assigned a
5746// function clone of the callee without regard to it, possibly one calling the
5747// wrong version of the callee for the trimmed contexts, which are agnostic to
5748// the callers of the node they were trimmed at and apply to the clone equally.
5749//
5750// This is called once the clones of callsite Node have been assigned to
5751// function clones. For each caller Y of a clone of Node whose edge carries
5752// trimmed contexts, each clone K of Y's original node that has no edge to any
5753// clone of Node, and which some of those trimmed contexts would reach (see
5754// ImpliesContextIds), is an implicit caller of Node. For example, with a
5755// context t trimmed at the frame of Y's callsite:
5756//
5757// (callers of Y) (callers of K)
5758// | |
5759// Y K clones of the same callsite node
5760// | \
5761// t | \ K has no edge to any clone of Node
5762// v v
5763// Required (other callsites in Node's function)
5764// (clone of Node called
5765// by Y, assigned to
5766// Y's function clone)
5767//
5768// Had t not been trimmed, it would also be on an edge from K, so K should get
5769// the same version of Node's function as Y:
5770// - If K was not yet assigned a function clone, assign it Y's, so that the
5771// assignment of the callsites it does have edges to takes it into account
5772// (creating further function clones derived from it if needed).
5773// - If K was already assigned to a function clone without a clone of Node,
5774// record that the Node call in that function clone should follow Required.
5775// - Otherwise K's function clone calls a conflicting clone of Node, which is
5776// left as is (and counted).
5777template <typename DerivedCCG, typename FuncTy, typename CallTy>
5778void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::assignImplicitCallers(
5779 ContextNode *Node, const DenseSet<uint32_t> &TrimmedContextIds,
5780 DenseMap<ContextNode *, FuncInfo> &CallsiteToCalleeFuncCloneMap,
5781 const std::map<FuncInfo, ContextNode *> &FuncCloneToCurNodeCloneMap,
5782 ImplicitSourceMap &ImplicitSources) {
5783 auto HasEdgeToNodeFamily = [&](const ContextNode *N) {
5784 return llvm::any_of(N->CalleeEdges,
5785 [&](const std::shared_ptr<ContextEdge> &E) {
5786 return E->Callee->getOrigNode() == Node;
5787 });
5788 };
5789 // Returns true if any of the trimmed context ids Ids, which are on (the
5790 // clone) Y but not on K, a clone of the same callsite node, would reach K had
5791 // they not been trimmed:
5792 // - An id terminating at Y applies to all clones of Y's original node (see
5793 // the example above assignImplicitCallers).
5794 // - An id reaching Y from a caller Z applies to K only if it would equally
5795 // apply to some caller C of K that is a clone of Z's original node, i.e.
5796 // it is not on C (if it were, C explicitly directs it to a callee other
5797 // than K) and, recursively, it applies to C the way it applies to Z (e.g.
5798 // because it terminates at Z, so it applies to all clones of Z's original
5799 // node). The id would then flow from C to K just as it flows from Z to Y,
5800 // had it not been trimmed:
5801 //
5802 // ... ... t terminates at Z or one of its callers
5803 // |
5804 // Z C clones of the same callsite node
5805 // | :
5806 // t | : (t) t is not on C or K; it applies to them
5807 // v v because it was trimmed above
5808 // Y K clones of the same callsite node
5809 // |
5810 // t v
5811 // Required
5812 //
5813 // This is checked recursively on (C, Z) with the ids not on C. If C == Z,
5814 // Z explicitly directs the ids to Y rather than K, so they don't apply to
5815 // K.
5816 //
5817 // Visited records the ids already examined for each (K, Y) pair, which
5818 // guards against cycles from recursion while still examining ids first seen
5819 // for the pair on a reconverging caller chain.
5820 auto ImpliesContextIds =
5821 [&](ContextNode *K, ContextNode *Y, const DenseSet<uint32_t> &Ids,
5822 DenseMap<std::pair<ContextNode *, ContextNode *>, DenseSet<uint32_t>>
5823 &Visited,
5824 auto &&ImpliesContextIds) -> bool {
5825 assert(K->getOrigNode() == Y->getOrigNode());
5826 DenseSet<uint32_t> &VisitedIds = Visited[{K, Y}];
5827 DenseSet<uint32_t> NewIds;
5828 for (auto Id : Ids)
5829 if (VisitedIds.insert(V: Id).second)
5830 NewIds.insert(V: Id);
5831 if (NewIds.empty())
5832 return false;
5833 // The ids found on each caller edge of Y, with that edge's caller Z, and
5834 // their union (a set rather than a count, as a recursive context can put
5835 // the same id on several caller edges).
5836 SmallVector<std::pair<ContextNode *, DenseSet<uint32_t>>> IdsFromCallers;
5837 DenseSet<uint32_t> IdsOnCallerEdges;
5838 for (auto &YEdge : Y->CallerEdges) {
5839 DenseSet<uint32_t> IdsFromZ =
5840 set_intersection(NewIds, YEdge->getContextIds());
5841 if (IdsFromZ.empty())
5842 continue;
5843 set_union(S1&: IdsOnCallerEdges, S2: IdsFromZ);
5844 IdsFromCallers.push_back({YEdge->Caller, std::move(IdsFromZ)});
5845 }
5846 // Any ids on none of Y's caller edges terminate at Y, and so apply to all
5847 // clones of Y's original node, including K.
5848 if (IdsOnCallerEdges.size() < NewIds.size())
5849 return true;
5850 // Otherwise check whether the ids reaching Y from a caller Z also apply to
5851 // K through a caller C of K that is a clone of Z's original node.
5852 for (auto &[Z, IdsFromZ] : IdsFromCallers) {
5853 for (auto &KEdge : K->CallerEdges) {
5854 ContextNode *C = KEdge->Caller;
5855 if (C == Z || C->getOrigNode() != Z->getOrigNode())
5856 continue;
5857 // Only the ids not on C can apply to C implicitly (C directs the ones
5858 // it has elsewhere); recursively check whether they do.
5859 DenseSet<uint32_t> IdsNotOnC;
5860 for (auto Id : IdsFromZ)
5861 if (!C->hasContextId(Id))
5862 IdsNotOnC.insert(Id);
5863 if (!IdsNotOnC.empty() &&
5864 ImpliesContextIds(C, Z, IdsNotOnC, Visited, ImpliesContextIds))
5865 return true;
5866 }
5867 }
5868 return false;
5869 };
5870 // Returns true if the given clones of Node are interchangeable for an
5871 // implicit caller: the same clone, allocations with the same hint, or
5872 // callsites already assigned to call the same function clone.
5873 auto SameVersion = [&](const ContextNode *A, const ContextNode *B) {
5874 if (A == B)
5875 return true;
5876 if (Node->IsAllocation)
5877 return allocTypeToUse(A->AllocTypes) == allocTypeToUse(B->AllocTypes);
5878 auto AIt = CallsiteToCalleeFuncCloneMap.find(A);
5879 auto BIt = CallsiteToCalleeFuncCloneMap.find(B);
5880 return AIt != CallsiteToCalleeFuncCloneMap.end() &&
5881 BIt != CallsiteToCalleeFuncCloneMap.end() &&
5882 AIt->second == BIt->second;
5883 };
5884 // The clones already identified as implicit callers of Node, so that each is
5885 // handled once (for the first caller edge whose trimmed contexts reach it).
5886 DenseSet<const ContextNode *> ImplicitCallers;
5887 // Handles the caller edge Edge of a clone of Node (or of Node itself): finds
5888 // the implicit callers among the clones of its caller Y's original node, and
5889 // assigns them, or records the clone of Node their calls must follow, as
5890 // described above.
5891 auto ProcessCaller = [&](const std::shared_ptr<ContextEdge> &Edge) {
5892 ContextNode *Y = Edge->Caller;
5893 ContextNode *Orig = Y->getOrigNode();
5894 if (!Y->hasCall() || Orig->Clones.empty())
5895 return;
5896 DenseSet<uint32_t> TrimmedIds =
5897 set_intersection(Edge->getContextIds(), TrimmedContextIds);
5898 if (TrimmedIds.empty())
5899 return;
5900 auto YIt = CallsiteToCalleeFuncCloneMap.find(Y);
5901 if (YIt == CallsiteToCalleeFuncCloneMap.end())
5902 return;
5903 // Copy, as recording assignments below may grow the map.
5904 const FuncInfo YFuncClone = YIt->second;
5905 // The clone of Node called by Y, which must be assigned to the same
5906 // function clone.
5907 auto RequiredIt = FuncCloneToCurNodeCloneMap.find(YFuncClone);
5908 assert(RequiredIt != FuncCloneToCurNodeCloneMap.end());
5909 const ContextNode *Required = RequiredIt->second;
5910 // Record in ImplicitSources that the Node call in function clone F must
5911 // follow the callsite clone Src. This is consulted when the calls in
5912 // function clones without their own clone of Node are collected (see
5913 // UnassignedCallClones in assignFunctions), and makes the requirement
5914 // survive if Src is later moved to another function clone derived from F
5915 // (when one of its callers conflicts at another callsite), after which the
5916 // call in F would otherwise follow the function clone F was derived from.
5917 auto RecordImplicitSource = [&](const FuncInfo &F, const ContextNode *Src) {
5918 // The calls in the original function (clone 0) are not collected there,
5919 // so nothing can be recorded for it.
5920 if (F.cloneNo() != 0)
5921 ImplicitSources.try_emplace({Node, F.cloneNo()},
5922 ImplicitSource{Src, *TrimmedIds.begin()});
5923 };
5924 // Check each clone of Y's original node (and the original itself) without
5925 // an edge to any clone of Node.
5926 for (ContextNode *K : llvm::concat<ContextNode *const>(
5927 Orig->Clones, ArrayRef<ContextNode *>(Orig))) {
5928 if (K == Y || !K->hasCall() || K->emptyContextIds() ||
5929 HasEdgeToNodeFamily(K) || ImplicitCallers.contains(K))
5930 continue;
5931 DenseMap<std::pair<ContextNode *, ContextNode *>, DenseSet<uint32_t>>
5932 Visited;
5933 if (!ImpliesContextIds(K, Y, TrimmedIds, Visited, ImpliesContextIds))
5934 continue;
5935 // K is an implicit caller of Node.
5936 ImplicitCallers.insert(K);
5937 auto KIt = CallsiteToCalleeFuncCloneMap.find(K);
5938 if (KIt == CallsiteToCalleeFuncCloneMap.end()) {
5939 // Not yet assigned: give it Y's function clone, in which the Node call
5940 // then follows Required.
5941 CallsiteToCalleeFuncCloneMap[K] = YFuncClone;
5942 RecordImplicitSource(YFuncClone, Required);
5943 NumImplicitCallersAssigned++;
5944 continue;
5945 }
5946 const FuncInfo KFuncClone = KIt->second;
5947 if (KFuncClone == YFuncClone) {
5948 // Already in Y's function clone.
5949 RecordImplicitSource(YFuncClone, Required);
5950 continue;
5951 }
5952 unsigned KCloneNo = KFuncClone.cloneNo();
5953 auto KNodeCloneIt = FuncCloneToCurNodeCloneMap.find(KFuncClone);
5954 if (KNodeCloneIt == FuncCloneToCurNodeCloneMap.end()) {
5955 // K's function clone has no clone of Node.
5956 if (KCloneNo == 0) {
5957 // The call in the original function keeps calling the original
5958 // callee function, or stays unhinted for an allocation. That
5959 // satisfies the requirement only if Required is that version.
5960 if (Node->IsAllocation) {
5961 if (allocTypeToUse(Required->AllocTypes) == AllocationType::NotCold)
5962 continue;
5963 } else if (auto It = CallsiteToCalleeFuncCloneMap.find(Required);
5964 It != CallsiteToCalleeFuncCloneMap.end() &&
5965 It->second.cloneNo() == 0)
5966 continue;
5967 } else {
5968 // Record that the Node call in K's function clone must follow
5969 // Required, unless an earlier implicit caller already recorded a
5970 // conflicting clone.
5971 auto [It, Inserted] = ImplicitSources.try_emplace(
5972 {Node, KCloneNo}, ImplicitSource{Required, *TrimmedIds.begin()});
5973 if (Inserted)
5974 NumImplicitCallersSourced++;
5975 if (SameVersion(It->second.Source, Required))
5976 continue;
5977 }
5978 } else if (SameVersion(KNodeCloneIt->second, Required)) {
5979 // K's function clone has a clone of Node interchangeable with
5980 // Required.
5981 RecordImplicitSource(KFuncClone, KNodeCloneIt->second);
5982 continue;
5983 }
5984 // K's function clone calls a conflicting clone of Node.
5985 NumImplicitCallersUnresolved++;
5986 }
5987 };
5988 // Process the caller edges of Node itself and of each of its clones: any of
5989 // them may carry trimmed contexts (a trimmed context stays with the clone
5990 // that was cloned, which may be Node or a clone made earlier).
5991 for (auto &Edge : Node->CallerEdges)
5992 ProcessCaller(Edge);
5993 for (auto *Clone : Node->Clones)
5994 for (auto &Edge : Clone->CallerEdges)
5995 ProcessCaller(Edge);
5996}
5997
5998// This method assigns cloned callsites to functions, cloning the functions as
5999// needed. The assignment is greedy and proceeds roughly as follows:
6000//
6001// For each function Func:
6002// For each call with graph Node having clones:
6003// Initialize ClonesWorklist to Node and its clones
6004// Initialize NodeCloneCount to 0
6005// While ClonesWorklist is not empty:
6006// Clone = pop front ClonesWorklist
6007// NodeCloneCount++
6008// If Func has been cloned less than NodeCloneCount times:
6009// If NodeCloneCount is 1:
6010// Assign Clone to original Func
6011// Continue
6012// Create a new function clone
6013// If other callers not assigned to call a function clone yet:
6014// Assign them to call new function clone
6015// Continue
6016// Assign any other caller calling the cloned version to new clone
6017//
6018// For each caller of Clone:
6019// If caller is assigned to call a specific function clone:
6020// If we cannot assign Clone to that function clone:
6021// Create new callsite Clone NewClone
6022// Add NewClone to ClonesWorklist
6023// Continue
6024// Assign Clone to existing caller's called function clone
6025// Else:
6026// If Clone not already assigned to a function clone:
6027// Assign to first function clone without assignment
6028// Assign caller to selected function clone
6029// For each call with graph Node having clones:
6030// If number func clones > number call's callsite Node clones:
6031// Record func CallInfo clones without Node clone in UnassignedCallClones
6032// For callsite Nodes in DFS order from allocations:
6033// If IsAllocation:
6034// Update allocation with alloc type
6035// Else:
6036// For Call, all MatchingCalls, and associated UnnassignedCallClones:
6037// Update call to call recorded callee clone
6038//
6039template <typename DerivedCCG, typename FuncTy, typename CallTy>
6040bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::assignFunctions() {
6041 bool Changed = false;
6042
6043 mergeClones();
6044
6045 // Context ids terminating at a non-allocation node which has or had callers
6046 // (i.e. is not a root), meaning their call stacks were trimmed there. Used
6047 // to identify implicit callers (see assignImplicitCallers).
6048 DenseSet<uint32_t> TrimmedContextIds;
6049 for (auto &NodePtr : NodeOwner) {
6050 ContextNode *N = NodePtr.get();
6051 if (N->IsAllocation || N->CalleeEdges.empty() ||
6052 (N->CallerEdges.empty() && N->Clones.empty() && !N->CloneOf))
6053 continue;
6054 // A context id trimmed at N is on a callee edge but on no caller edge.
6055 // Compare distinct ids rather than edge sizes: a recursive context can put
6056 // the same id on several callee and caller edges of N.
6057 DenseSet<uint32_t> CallerIds;
6058 for (auto &E : N->CallerEdges)
6059 set_union(CallerIds, E->getContextIds());
6060 for (auto &CalleeEdge : N->CalleeEdges)
6061 for (auto Id : CalleeEdge->getContextIds())
6062 if (!CallerIds.contains(V: Id))
6063 TrimmedContextIds.insert(Id);
6064 }
6065 NumTrimmedContextIds += TrimmedContextIds.size();
6066
6067 // Keep track of the assignment of nodes (callsites) to function clones they
6068 // call.
6069 DenseMap<ContextNode *, FuncInfo> CallsiteToCalleeFuncCloneMap;
6070
6071 // Update caller node to call function version CalleeFunc, by recording the
6072 // assignment in CallsiteToCalleeFuncCloneMap.
6073 auto RecordCalleeFuncOfCallsite = [&](ContextNode *Caller,
6074 const FuncInfo &CalleeFunc) {
6075 assert(Caller->hasCall());
6076 CallsiteToCalleeFuncCloneMap[Caller] = CalleeFunc;
6077 };
6078
6079 // Information for a single clone of this Func.
6080 struct FuncCloneInfo {
6081 // The function clone.
6082 FuncInfo FuncClone;
6083 // Remappings of each call of interest (from original uncloned call to the
6084 // corresponding cloned call in this function clone).
6085 DenseMap<CallInfo, CallInfo> CallMap;
6086 };
6087
6088 // Map to keep track of information needed to update calls in function clones
6089 // when their corresponding callsite node was not itself cloned for that
6090 // function clone. Because of call context trimming (i.e. we only keep as much
6091 // caller information as needed to distinguish hot vs cold), we may not have
6092 // caller edges coming to each callsite node from all possible function
6093 // callers. A function clone may get created for other callsites in the
6094 // function for which there are caller edges that were not trimmed. Any other
6095 // callsites in that function clone, which were not themselved cloned for
6096 // that function clone, should get updated the same way as the corresponding
6097 // callsite in the original function (which may call a clone of its callee).
6098 //
6099 // We build this map after completing function cloning for each function, so
6100 // that we can record the information from its call maps before they are
6101 // destructed. The map will be used as we update calls to update any still
6102 // unassigned call clones. Note that we may create new node clones as we clone
6103 // other functions, so later on we check which node clones were still not
6104 // created. To this end, the inner map is a map from function clone number to
6105 // the list of calls cloned for that function (can be more than one due to the
6106 // Node's MatchingCalls array).
6107 //
6108 // The alternative is creating new callsite clone nodes below as we clone the
6109 // function, but that is tricker to get right and likely more overhead.
6110 //
6111 // Each entry also records the callsite node whose callee assignment the
6112 // calls should follow (Source). By default that is the callsite clone (if
6113 // any) in the function clone the function clone was derived from, following
6114 // the chain of derivations to the original function, since a function clone
6115 // created to specialize another one should otherwise behave like it. It can
6116 // also be the callsite clone required by an implicit caller (see
6117 // assignImplicitCallers). A null Source means the original node.
6118 //
6119 // Inner map is a std::map so sorted by key (clone number), in order to get
6120 // ordered remarks in the full LTO case.
6121 struct UnassignedCallCloneInfo {
6122 SmallVector<CallInfo, 0> Calls;
6123 const ContextNode *Source = nullptr;
6124 };
6125 DenseMap<const ContextNode *, std::map<unsigned, UnassignedCallCloneInfo>>
6126 UnassignedCallClones;
6127
6128 // Walk all functions for which we saw calls with memprof metadata, and handle
6129 // cloning for each of its calls.
6130 for (auto &[Func, CallsWithMetadata] : FuncToCallsWithMetadata) {
6131 FuncInfo OrigFunc(Func);
6132 // Map from each clone number of OrigFunc to information about that function
6133 // clone (the function clone FuncInfo and call remappings). The index into
6134 // the vector is the clone number, as function clones are created and
6135 // numbered sequentially.
6136 std::vector<FuncCloneInfo> FuncCloneInfos;
6137 // For each function clone number, the clone number of the function clone
6138 // it was derived from (created to further specialize, when a caller
6139 // already assigned to it needed a different clone of some callsite), or 0
6140 // for clones created from the original function. Used to decide which
6141 // callsite clone the calls not themselves cloned for a function clone
6142 // should follow (see UnassignedCallClones).
6143 std::vector<unsigned> FuncCloneParent;
6144 // Callsite clones required for calls in specific function clones by
6145 // implicit callers (see assignImplicitCallers).
6146 ImplicitSourceMap ImplicitSources;
6147 // Map from each original callsite node in this function to its original
6148 // call and matching calls (clone 0). The calls recorded on an original node
6149 // are updated to the calls in a function clone if that node is assigned to
6150 // a function clone, after which they can no longer be used to index the
6151 // function clone CallMaps (which are keyed by the original calls). No node
6152 // in this function has been assigned yet at this point.
6153 struct OrigCallsInfo {
6154 CallInfo Call;
6155 SmallVector<CallInfo, 0> MatchingCalls;
6156 };
6157 DenseMap<const ContextNode *, OrigCallsInfo> OrigNodeToOrigCalls;
6158 for (auto &Call : CallsWithMetadata) {
6159 ContextNode *Node = getNodeForInst(C: Call);
6160 if (Node && Node->Call == Call)
6161 OrigNodeToOrigCalls[Node] = {Call, Node->MatchingCalls};
6162 }
6163 for (auto &Call : CallsWithMetadata) {
6164 ContextNode *Node = getNodeForInst(C: Call);
6165 // Skip call if we do not have a node for it (all uses of its stack ids
6166 // were either on inlined chains or trimmed from the MIBs), or if we did
6167 // not create any clones for it.
6168 if (!Node || Node->Clones.empty())
6169 continue;
6170 assert(Node->hasCall() &&
6171 "Not having a call should have prevented cloning");
6172
6173 // Track the assignment of function clones to clones of the current
6174 // callsite Node being handled.
6175 std::map<FuncInfo, ContextNode *> FuncCloneToCurNodeCloneMap;
6176
6177 // Assign callsite version CallsiteClone to function version FuncClone,
6178 // and also assign (possibly cloned) Call to CallsiteClone.
6179 auto AssignCallsiteCloneToFuncClone = [&](const FuncInfo &FuncClone,
6180 CallInfo &Call,
6181 ContextNode *CallsiteClone,
6182 bool IsAlloc) {
6183 // Record the clone of callsite node assigned to this function clone.
6184 FuncCloneToCurNodeCloneMap[FuncClone] = CallsiteClone;
6185
6186 assert(FuncCloneInfos.size() > FuncClone.cloneNo());
6187 DenseMap<CallInfo, CallInfo> &CallMap =
6188 FuncCloneInfos[FuncClone.cloneNo()].CallMap;
6189 CallInfo CallClone(Call);
6190 if (auto It = CallMap.find(Call); It != CallMap.end())
6191 CallClone = It->second;
6192 CallsiteClone->setCall(CallClone);
6193 // Need to do the same for all matching calls. Rebuild the list of
6194 // CallsiteClone from the original matching calls (it may have been
6195 // copied from a node whose calls were already remapped), using the
6196 // same mapping as for Call.
6197 assert(OrigNodeToOrigCalls.count(Node));
6198 CallsiteClone->MatchingCalls.clear();
6199 for (const CallInfo &OrigMatchingCall :
6200 OrigNodeToOrigCalls[Node].MatchingCalls) {
6201 CallInfo CallClone(OrigMatchingCall);
6202 if (auto It = CallMap.find(OrigMatchingCall); It != CallMap.end())
6203 CallClone = It->second;
6204 CallsiteClone->MatchingCalls.push_back(CallClone);
6205 }
6206 };
6207
6208 // Invokes moveEdgeToNewCalleeClone which creates a new clone, and then
6209 // performs the necessary fixups (removing none type edges, and
6210 // importantly, propagating any function call assignment of the original
6211 // node to the new clone).
6212 auto MoveEdgeToNewCalleeCloneAndSetUp =
6213 [&](const std::shared_ptr<ContextEdge> &Edge) {
6214 ContextNode *OrigCallee = Edge->Callee;
6215 ContextNode *NewClone = moveEdgeToNewCalleeClone(Edge);
6216 removeNoneTypeCalleeEdges(Node: NewClone);
6217 assert(NewClone->AllocTypes != (uint8_t)AllocationType::None);
6218 // If the original Callee was already assigned to call a specific
6219 // function version, make sure its new clone is assigned to call
6220 // that same function clone.
6221 if (CallsiteToCalleeFuncCloneMap.count(OrigCallee))
6222 RecordCalleeFuncOfCallsite(
6223 NewClone, CallsiteToCalleeFuncCloneMap[OrigCallee]);
6224 return NewClone;
6225 };
6226
6227 // Keep track of the clones of callsite Node that need to be assigned to
6228 // function clones. This list may be expanded in the loop body below if we
6229 // find additional cloning is required.
6230 std::deque<ContextNode *> ClonesWorklist;
6231 // Ignore original Node if we moved all of its contexts to clones.
6232 if (!Node->emptyContextIds())
6233 ClonesWorklist.push_back(Node);
6234 llvm::append_range(ClonesWorklist, Node->Clones);
6235
6236 // Now walk through all of the clones of this callsite Node that we need,
6237 // and determine the assignment to a corresponding clone of the current
6238 // function (creating new function clones as needed).
6239 unsigned NodeCloneCount = 0;
6240 while (!ClonesWorklist.empty()) {
6241 ContextNode *Clone = ClonesWorklist.front();
6242 ClonesWorklist.pop_front();
6243 NodeCloneCount++;
6244 if (VerifyNodes)
6245 checkNode<DerivedCCG, FuncTy, CallTy>(Clone);
6246
6247 // Need to create a new function clone if we have more callsite clones
6248 // than existing function clones, which would have been assigned to an
6249 // earlier clone in the list (we assign callsite clones to function
6250 // clones greedily).
6251 if (FuncCloneInfos.size() < NodeCloneCount) {
6252 // If this is the first callsite copy, assign to original function.
6253 if (NodeCloneCount == 1) {
6254 // Since FuncCloneInfos is empty in this case, no clones have
6255 // been created for this function yet, and no callers should have
6256 // been assigned a function clone for this callee node yet.
6257 assert(llvm::none_of(
6258 Clone->CallerEdges, [&](const std::shared_ptr<ContextEdge> &E) {
6259 return CallsiteToCalleeFuncCloneMap.count(E->Caller);
6260 }));
6261 // Initialize with empty call map, assign Clone to original function
6262 // and its callers, and skip to the next clone.
6263 FuncCloneInfos.push_back(
6264 {OrigFunc, DenseMap<CallInfo, CallInfo>()});
6265 FuncCloneParent.push_back(x: 0);
6266 AssignCallsiteCloneToFuncClone(
6267 OrigFunc, Call, Clone,
6268 AllocationCallToContextNodeMap.count(Call));
6269 for (auto &CE : Clone->CallerEdges) {
6270 // Ignore any caller that does not have a recorded callsite Call.
6271 if (!CE->Caller->hasCall())
6272 continue;
6273 RecordCalleeFuncOfCallsite(CE->Caller, OrigFunc);
6274 }
6275 continue;
6276 }
6277
6278 // First locate which copy of OrigFunc to clone again. If a caller
6279 // of this callsite clone was already assigned to call a particular
6280 // function clone, we need to redirect all of those callers to the
6281 // new function clone, and update their other callees within this
6282 // function.
6283 FuncInfo PreviousAssignedFuncClone;
6284 auto EI = llvm::find_if(
6285 Clone->CallerEdges, [&](const std::shared_ptr<ContextEdge> &E) {
6286 return CallsiteToCalleeFuncCloneMap.count(E->Caller);
6287 });
6288 bool CallerAssignedToCloneOfFunc = false;
6289 if (EI != Clone->CallerEdges.end()) {
6290 const std::shared_ptr<ContextEdge> &Edge = *EI;
6291 PreviousAssignedFuncClone =
6292 CallsiteToCalleeFuncCloneMap[Edge->Caller];
6293 CallerAssignedToCloneOfFunc = true;
6294 }
6295
6296 // Clone function and save it along with the CallInfo map created
6297 // during cloning in the FuncCloneInfos.
6298 DenseMap<CallInfo, CallInfo> NewCallMap;
6299 unsigned CloneNo = FuncCloneInfos.size();
6300 assert(CloneNo > 0 && "Clone 0 is the original function, which "
6301 "should already exist in the map");
6302 FuncInfo NewFuncClone = cloneFunctionForCallsite(
6303 Func&: OrigFunc, Call, CallMap&: NewCallMap, CallsWithMetadataInFunc&: CallsWithMetadata, CloneNo);
6304 FuncCloneInfos.push_back({NewFuncClone, std::move(NewCallMap)});
6305 // Record the function clone it is derived from (if any), whose
6306 // callers it will take over below.
6307 FuncCloneParent.push_back(CallerAssignedToCloneOfFunc
6308 ? PreviousAssignedFuncClone.cloneNo()
6309 : 0);
6310 FunctionClonesAnalysis++;
6311 Changed = true;
6312
6313 // If no caller callsites were already assigned to a clone of this
6314 // function, we can simply assign this clone to the new func clone
6315 // and update all callers to it, then skip to the next clone.
6316 if (!CallerAssignedToCloneOfFunc) {
6317 AssignCallsiteCloneToFuncClone(
6318 NewFuncClone, Call, Clone,
6319 AllocationCallToContextNodeMap.count(Call));
6320 for (auto &CE : Clone->CallerEdges) {
6321 // Ignore any caller that does not have a recorded callsite Call.
6322 if (!CE->Caller->hasCall())
6323 continue;
6324 RecordCalleeFuncOfCallsite(CE->Caller, NewFuncClone);
6325 }
6326 continue;
6327 }
6328
6329 // We may need to do additional node cloning in this case.
6330 // Reset the CallsiteToCalleeFuncCloneMap entry for any callers
6331 // that were previously assigned to call PreviousAssignedFuncClone,
6332 // to record that they now call NewFuncClone.
6333 // The none type edge removal may remove some of this Clone's caller
6334 // edges, if it is reached via another of its caller's callees.
6335 // Iterate over a copy and skip any that were removed.
6336 auto CallerEdges = Clone->CallerEdges;
6337 for (auto CE : CallerEdges) {
6338 // Skip any that have been removed on an earlier iteration.
6339 if (CE->isRemoved()) {
6340 assert(!is_contained(Clone->CallerEdges, CE));
6341 continue;
6342 }
6343 assert(CE);
6344 // Ignore any caller that does not have a recorded callsite Call.
6345 if (!CE->Caller->hasCall())
6346 continue;
6347
6348 if (!CallsiteToCalleeFuncCloneMap.count(CE->Caller) ||
6349 // We subsequently fall through to later handling that
6350 // will perform any additional cloning required for
6351 // callers that were calling other function clones.
6352 CallsiteToCalleeFuncCloneMap[CE->Caller] !=
6353 PreviousAssignedFuncClone)
6354 continue;
6355
6356 RecordCalleeFuncOfCallsite(CE->Caller, NewFuncClone);
6357
6358 // If we are cloning a function that was already assigned to some
6359 // callers, then essentially we are creating new callsite clones
6360 // of the other callsites in that function that are reached by those
6361 // callers. Clone the other callees of the current callsite's caller
6362 // that were already assigned to PreviousAssignedFuncClone
6363 // accordingly. This is important since we subsequently update the
6364 // calls from the nodes in the graph and their assignments to callee
6365 // functions recorded in CallsiteToCalleeFuncCloneMap.
6366 // The none type edge removal may remove some of this caller's
6367 // callee edges, if it is reached via another of its callees.
6368 // Iterate over a copy and skip any that were removed.
6369 auto CalleeEdges = CE->Caller->CalleeEdges;
6370 for (auto CalleeEdge : CalleeEdges) {
6371 // Skip any that have been removed on an earlier iteration when
6372 // cleaning up newly None type callee edges.
6373 if (CalleeEdge->isRemoved()) {
6374 assert(!is_contained(CE->Caller->CalleeEdges, CalleeEdge));
6375 continue;
6376 }
6377 assert(CalleeEdge);
6378 ContextNode *Callee = CalleeEdge->Callee;
6379 // Skip the current callsite, we are looking for other
6380 // callsites Caller calls, as well as any that does not have a
6381 // recorded callsite Call.
6382 if (Callee == Clone || !Callee->hasCall())
6383 continue;
6384 // Skip direct recursive calls. We don't need/want to clone the
6385 // caller node again, and this loop will not behave as expected if
6386 // we tried.
6387 if (Callee == CalleeEdge->Caller)
6388 continue;
6389 ContextNode *NewClone =
6390 MoveEdgeToNewCalleeCloneAndSetUp(CalleeEdge);
6391 // Moving the edge may have resulted in some none type
6392 // callee edges on the original Callee.
6393 removeNoneTypeCalleeEdges(Node: Callee);
6394 // Update NewClone with the new Call clone of this callsite's Call
6395 // created for the new function clone created earlier.
6396 // Recall that we have already ensured when building the graph
6397 // that each caller can only call callsites within the same
6398 // function, so we are guaranteed that Callee Call is in the
6399 // current OrigFunc.
6400 // CallMap is set up as indexed by original calls at clone 0. Note
6401 // that we cannot simply use the calls recorded on Callee's
6402 // original node, as those are updated to the calls in a function
6403 // clone if the original node was itself assigned to a function
6404 // clone.
6405 assert(OrigNodeToOrigCalls.count(Callee->getOrigNode()));
6406 auto &OrigCalls = OrigNodeToOrigCalls[Callee->getOrigNode()];
6407 DenseMap<CallInfo, CallInfo> &CallMap =
6408 FuncCloneInfos[NewFuncClone.cloneNo()].CallMap;
6409 assert(CallMap.count(OrigCalls.Call));
6410 CallInfo NewCall(CallMap[OrigCalls.Call]);
6411 assert(NewCall);
6412 NewClone->setCall(NewCall);
6413 // Need to do the same for all matching calls.
6414 NewClone->MatchingCalls.clear();
6415 for (auto &OrigMatchingCall : OrigCalls.MatchingCalls) {
6416 assert(CallMap.count(OrigMatchingCall));
6417 CallInfo NewCall(CallMap[OrigMatchingCall]);
6418 assert(NewCall);
6419 NewClone->MatchingCalls.push_back(NewCall);
6420 }
6421 }
6422 }
6423 // Fall through to handling below to perform the recording of the
6424 // function for this callsite clone. This enables handling of cases
6425 // where the callers were assigned to different clones of a function.
6426 }
6427
6428 auto FindFirstAvailFuncClone = [&]() {
6429 // Find first function in FuncCloneInfos without an assigned
6430 // clone of this callsite Node. We should always have one
6431 // available at this point due to the earlier cloning when the
6432 // FuncCloneInfos size was smaller than the clone number.
6433 for (auto &CF : FuncCloneInfos) {
6434 if (!FuncCloneToCurNodeCloneMap.count(CF.FuncClone))
6435 return CF.FuncClone;
6436 }
6437 llvm_unreachable(
6438 "Expected an available func clone for this callsite clone");
6439 };
6440
6441 // See if we can use existing function clone. Walk through
6442 // all caller edges to see if any have already been assigned to
6443 // a clone of this callsite's function. If we can use it, do so. If not,
6444 // because that function clone is already assigned to a different clone
6445 // of this callsite, then we need to clone again.
6446 // Basically, this checking is needed to handle the case where different
6447 // caller functions/callsites may need versions of this function
6448 // containing different mixes of callsite clones across the different
6449 // callsites within the function. If that happens, we need to create
6450 // additional function clones to handle the various combinations.
6451 //
6452 // Keep track of any new clones of this callsite created by the
6453 // following loop, as well as any existing clone that we decided to
6454 // assign this clone to.
6455 std::map<FuncInfo, ContextNode *> FuncCloneToNewCallsiteCloneMap;
6456 FuncInfo FuncCloneAssignedToCurCallsiteClone;
6457 // Iterate over a copy of Clone's caller edges, since we may need to
6458 // remove edges in the moveEdgeTo* methods, and this simplifies the
6459 // handling and makes it less error-prone.
6460 auto CloneCallerEdges = Clone->CallerEdges;
6461 for (auto &Edge : CloneCallerEdges) {
6462 // Skip removed edges (due to direct recursive edges updated when
6463 // updating callee edges when moving an edge and subsequently
6464 // removed by call to removeNoneTypeCalleeEdges on the Clone).
6465 if (Edge->isRemoved())
6466 continue;
6467 // Ignore any caller that does not have a recorded callsite Call.
6468 if (!Edge->Caller->hasCall())
6469 continue;
6470 // If this caller already assigned to call a version of OrigFunc, need
6471 // to ensure we can assign this callsite clone to that function clone.
6472 if (CallsiteToCalleeFuncCloneMap.count(Edge->Caller)) {
6473 FuncInfo FuncCloneCalledByCaller =
6474 CallsiteToCalleeFuncCloneMap[Edge->Caller];
6475 // First we need to confirm that this function clone is available
6476 // for use by this callsite node clone.
6477 //
6478 // While FuncCloneToCurNodeCloneMap is built only for this Node and
6479 // its callsite clones, one of those callsite clones X could have
6480 // been assigned to the same function clone called by Edge's caller
6481 // - if Edge's caller calls another callsite within Node's original
6482 // function, and that callsite has another caller reaching clone X.
6483 // We need to clone Node again in this case.
6484 if ((FuncCloneToCurNodeCloneMap.count(FuncCloneCalledByCaller) &&
6485 FuncCloneToCurNodeCloneMap[FuncCloneCalledByCaller] !=
6486 Clone) ||
6487 // Detect when we have multiple callers of this callsite that
6488 // have already been assigned to specific, and different, clones
6489 // of OrigFunc (due to other unrelated callsites in Func they
6490 // reach via call contexts). Is this Clone of callsite Node
6491 // assigned to a different clone of OrigFunc? If so, clone Node
6492 // again.
6493 (FuncCloneAssignedToCurCallsiteClone &&
6494 FuncCloneAssignedToCurCallsiteClone !=
6495 FuncCloneCalledByCaller)) {
6496 // We need to use a different newly created callsite clone, in
6497 // order to assign it to another new function clone on a
6498 // subsequent iteration over the Clones array (adjusted below).
6499 // Note we specifically do not reset the
6500 // CallsiteToCalleeFuncCloneMap entry for this caller, so that
6501 // when this new clone is processed later we know which version of
6502 // the function to copy (so that other callsite clones we have
6503 // assigned to that function clone are properly cloned over). See
6504 // comments in the function cloning handling earlier.
6505
6506 // Check if we already have cloned this callsite again while
6507 // walking through caller edges, for a caller calling the same
6508 // function clone. If so, we can move this edge to that new clone
6509 // rather than creating yet another new clone.
6510 if (FuncCloneToNewCallsiteCloneMap.count(
6511 FuncCloneCalledByCaller)) {
6512 ContextNode *NewClone =
6513 FuncCloneToNewCallsiteCloneMap[FuncCloneCalledByCaller];
6514 moveEdgeToExistingCalleeClone(Edge, NewCallee: NewClone);
6515 // Cleanup any none type edges cloned over.
6516 removeNoneTypeCalleeEdges(Node: NewClone);
6517 } else {
6518 // Create a new callsite clone.
6519 ContextNode *NewClone = MoveEdgeToNewCalleeCloneAndSetUp(Edge);
6520 FuncCloneToNewCallsiteCloneMap[FuncCloneCalledByCaller] =
6521 NewClone;
6522 // Add to list of clones and process later.
6523 ClonesWorklist.push_back(NewClone);
6524 }
6525 // Moving the caller edge may have resulted in some none type
6526 // callee edges.
6527 removeNoneTypeCalleeEdges(Node: Clone);
6528 // We will handle the newly created callsite clone in a subsequent
6529 // iteration over this Node's Clones.
6530 continue;
6531 }
6532
6533 // Otherwise, we can use the function clone already assigned to this
6534 // caller.
6535 if (!FuncCloneAssignedToCurCallsiteClone) {
6536 FuncCloneAssignedToCurCallsiteClone = FuncCloneCalledByCaller;
6537 // Assign Clone to FuncCloneCalledByCaller
6538 AssignCallsiteCloneToFuncClone(
6539 FuncCloneCalledByCaller, Call, Clone,
6540 AllocationCallToContextNodeMap.count(Call));
6541 } else
6542 // Don't need to do anything - callsite is already calling this
6543 // function clone.
6544 assert(FuncCloneAssignedToCurCallsiteClone ==
6545 FuncCloneCalledByCaller);
6546
6547 } else {
6548 // We have not already assigned this caller to a version of
6549 // OrigFunc. Do the assignment now.
6550
6551 // First check if we have already assigned this callsite clone to a
6552 // clone of OrigFunc for another caller during this iteration over
6553 // its caller edges.
6554 if (!FuncCloneAssignedToCurCallsiteClone) {
6555 FuncCloneAssignedToCurCallsiteClone = FindFirstAvailFuncClone();
6556 assert(FuncCloneAssignedToCurCallsiteClone);
6557 // Assign Clone to FuncCloneAssignedToCurCallsiteClone
6558 AssignCallsiteCloneToFuncClone(
6559 FuncCloneAssignedToCurCallsiteClone, Call, Clone,
6560 AllocationCallToContextNodeMap.count(Call));
6561 } else
6562 assert(FuncCloneToCurNodeCloneMap
6563 [FuncCloneAssignedToCurCallsiteClone] == Clone);
6564 // Update callers to record function version called.
6565 RecordCalleeFuncOfCallsite(Edge->Caller,
6566 FuncCloneAssignedToCurCallsiteClone);
6567 }
6568 }
6569 // If we didn't assign a function clone to this callsite clone yet, e.g.
6570 // none of its callers has a non-null call, do the assignment here.
6571 // We want to ensure that every callsite clone is assigned to some
6572 // function clone, so that the call updates below work as expected.
6573 // In particular if this is the original callsite, we want to ensure it
6574 // is assigned to the original function, otherwise the original function
6575 // will appear available for assignment to other callsite clones,
6576 // leading to unintended effects. For one, the unknown and not updated
6577 // callers will call into cloned paths leading to the wrong hints,
6578 // because they still call the original function (clone 0). Also,
6579 // because all callsites start out as being clone 0 by default, we can't
6580 // easily distinguish between callsites explicitly assigned to clone 0
6581 // vs those never assigned, which can lead to multiple updates of the
6582 // calls when invoking updateCall below, with mismatched clone values.
6583 // TODO: Add a flag to the callsite nodes or some other mechanism to
6584 // better distinguish and identify callsite clones that are not getting
6585 // assigned to function clones as expected.
6586 if (!FuncCloneAssignedToCurCallsiteClone) {
6587 FuncCloneAssignedToCurCallsiteClone = FindFirstAvailFuncClone();
6588 assert(FuncCloneAssignedToCurCallsiteClone &&
6589 "No available func clone for this callsite clone");
6590 AssignCallsiteCloneToFuncClone(
6591 FuncCloneAssignedToCurCallsiteClone, Call, Clone,
6592 /*IsAlloc=*/AllocationCallToContextNodeMap.contains(Call));
6593 }
6594 }
6595 assignImplicitCallers(Node, TrimmedContextIds,
6596 CallsiteToCalleeFuncCloneMap,
6597 FuncCloneToCurNodeCloneMap, ImplicitSources);
6598 if (VerifyCCG) {
6599 checkNode<DerivedCCG, FuncTy, CallTy>(Node);
6600 for (const auto &PE : Node->CalleeEdges)
6601 checkNode<DerivedCCG, FuncTy, CallTy>(PE->Callee);
6602 for (const auto &CE : Node->CallerEdges)
6603 checkNode<DerivedCCG, FuncTy, CallTy>(CE->Caller);
6604 for (auto *Clone : Node->Clones) {
6605 checkNode<DerivedCCG, FuncTy, CallTy>(Clone);
6606 for (const auto &PE : Clone->CalleeEdges)
6607 checkNode<DerivedCCG, FuncTy, CallTy>(PE->Callee);
6608 for (const auto &CE : Clone->CallerEdges)
6609 checkNode<DerivedCCG, FuncTy, CallTy>(CE->Caller);
6610 }
6611 }
6612 }
6613
6614 if (MemProfReportHintedSizesDetail)
6615 FuncToNumClones[Func] = std::max<unsigned>(1, FuncCloneInfos.size());
6616 if (FuncCloneInfos.size() < 2)
6617 continue;
6618
6619 // In this case there is more than just the original function copy.
6620 // Record call clones of any callsite nodes in the function that did not
6621 // themselves get cloned for all of the function clones.
6622 for (auto &Call : CallsWithMetadata) {
6623 ContextNode *Node = getNodeForInst(C: Call);
6624 if (!Node || !Node->hasCall() || Node->emptyContextIds())
6625 continue;
6626 // First collect the function clones that have a clone of this callsite
6627 // node (or the node itself). They may not be sequential due to empty
6628 // clones e.g.
6629 DenseMap<unsigned, const ContextNode *> FuncCloneToNodeClone;
6630 FuncCloneToNodeClone[Node->Call.cloneNo()] = Node;
6631 // A clone left without context ids (its callers were all moved to other
6632 // clones while assigning function clones) has no edges and is not
6633 // reached by the call updates below, so the call in its function clone
6634 // keeps the original callee. If an implicit caller recorded the callsite
6635 // clone that call must follow, treat the function clone as having no
6636 // clone of this node, so that the call follows the recorded source like
6637 // any other call not cloned for its function clone.
6638 for (auto *C : Node->Clones)
6639 if (!C->emptyContextIds() ||
6640 !ImplicitSources.contains({Node, C->Call.cloneNo()}))
6641 FuncCloneToNodeClone[C->Call.cloneNo()] = C;
6642 // If Node has enough clones already to cover all function clones, we can
6643 // skip it.
6644 if (FuncCloneToNodeClone.size() >= FuncCloneInfos.size())
6645 continue;
6646 // Now check all the function clones.
6647 for (auto &FC : FuncCloneInfos) {
6648 unsigned I = FC.FuncClone.cloneNo();
6649 // Skip the first clone which got the original call.
6650 // Also skip any other clones created for this Node.
6651 if (I == 0 || FuncCloneToNodeClone.contains(I)) {
6652 continue;
6653 }
6654 // Record the call clones created for this callsite in this function
6655 // clone, and the callsite node whose callee they should follow: the
6656 // one required by an implicit caller, if any, else the clone of this
6657 // callsite node in the nearest function clone in the chain of function
6658 // clones this one was derived from (null for the original node).
6659 auto &Info = UnassignedCallClones[Node][I];
6660 auto &CallVector = Info.Calls;
6661 for (unsigned P = I;; P = FuncCloneParent[P]) {
6662 if (auto It = ImplicitSources.find({Node, P});
6663 It != ImplicitSources.end()) {
6664 Info.Source = resolveImplicitSource(Node, Src: It->second);
6665 break;
6666 }
6667 if (auto It = FuncCloneToNodeClone.find(P);
6668 It != FuncCloneToNodeClone.end()) {
6669 Info.Source = It->second;
6670 break;
6671 }
6672 if (P == 0)
6673 break;
6674 }
6675 DenseMap<CallInfo, CallInfo> &CallMap = FC.CallMap;
6676 if (auto It = CallMap.find(Call); It != CallMap.end()) {
6677 CallInfo CallClone = It->second;
6678 CallVector.push_back(CallClone);
6679 } else {
6680 // All but the original clone (skipped earlier) should have an entry
6681 // for all calls.
6682 assert(false && "Expected to find call in CallMap");
6683 }
6684 // Need to do the same for all matching calls. Use the original
6685 // matching calls, as those on Node may have been remapped to the calls
6686 // of a function clone if Node was assigned to one.
6687 assert(OrigNodeToOrigCalls.count(Node));
6688 for (const CallInfo &OrigMatchingCall :
6689 OrigNodeToOrigCalls[Node].MatchingCalls) {
6690 if (auto It = CallMap.find(OrigMatchingCall); It != CallMap.end()) {
6691 CallInfo CallClone = It->second;
6692 CallVector.push_back(CallClone);
6693 } else {
6694 // All but the original clone (skipped earlier) should have an entry
6695 // for all calls.
6696 assert(false && "Expected to find call in CallMap");
6697 }
6698 }
6699 }
6700 }
6701 }
6702
6703 uint8_t BothTypes =
6704 (uint8_t)AllocationType::Cold | (uint8_t)AllocationType::NotCold;
6705
6706 auto UpdateCalls = [&](ContextNode *Node,
6707 DenseSet<const ContextNode *> &Visited,
6708 auto &&UpdateCalls) {
6709 auto Inserted = Visited.insert(Node);
6710 if (!Inserted.second)
6711 return;
6712
6713 for (auto *Clone : Node->Clones)
6714 UpdateCalls(Clone, Visited, UpdateCalls);
6715
6716 for (auto &Edge : Node->CallerEdges)
6717 UpdateCalls(Edge->Caller, Visited, UpdateCalls);
6718
6719 // Skip if either no call to update, or if we ended up with no context ids
6720 // (we moved all edges onto other clones).
6721 if (!Node->hasCall() || Node->emptyContextIds())
6722 return;
6723
6724 // Returns the recorded call clones in function clones which don't have a
6725 // clone of this callsite node, along with the node whose decision each
6726 // should follow (the Source recorded earlier, falling back to Node).
6727 auto GetUnassignedCallClones = [&](ContextNode *Node) {
6728 SmallVector<std::pair<const ContextNode *, MutableArrayRef<CallInfo>>>
6729 Unassigned;
6730 auto UnassignedIt = UnassignedCallClones.find(Node);
6731 if (UnassignedIt == UnassignedCallClones.end())
6732 return Unassigned;
6733 DenseSet<unsigned> NodeCallClones;
6734 for (auto *C : Node->Clones)
6735 if (!C->emptyContextIds())
6736 NodeCallClones.insert(C->Call.cloneNo());
6737 for (auto &[CloneNo, Info] : UnassignedIt->second) {
6738 // Should start at 1 as we never create an entry for original node.
6739 assert(CloneNo > 0);
6740 // If we subsequently created a clone, skip this one.
6741 if (NodeCallClones.contains(V: CloneNo))
6742 continue;
6743 Unassigned.push_back({Info.Source ? Info.Source : Node, Info.Calls});
6744 }
6745 return Unassigned;
6746 };
6747
6748 if (Node->IsAllocation) {
6749 auto GetAllocTypeToUse = [&](const ContextNode *Node) {
6750 auto AT = allocTypeToUse(Node->AllocTypes);
6751 // If the allocation type is still ambiguous after cloning, record
6752 // that as the allocation type (the attribute stays "ambiguous") rather
6753 // than falling back to the default not cold, unless ambiguous
6754 // attributes are disabled.
6755 if (Node->AllocTypes == BothTypes && MemProfUseAmbiguousAttributes)
6756 AT = (AllocationType)BothTypes;
6757 // If the allocation type is ambiguous, and more aggressive hinting
6758 // has been enabled via the MinClonedColdBytePercent flag, see if this
6759 // allocation should be hinted cold anyway because its fraction cold
6760 // bytes allocated is at least the given threshold.
6761 if (Node->AllocTypes == BothTypes && MinClonedColdBytePercent < 100 &&
6762 !ContextIdToContextSizeInfos.empty()) {
6763 uint64_t TotalCold = 0;
6764 uint64_t Total = 0;
6765 for (auto Id : Node->getContextIds()) {
6766 auto TypeI = ContextIdToAllocationType.find(Id);
6767 assert(TypeI != ContextIdToAllocationType.end());
6768 auto CSI = ContextIdToContextSizeInfos.find(Id);
6769 if (CSI != ContextIdToContextSizeInfos.end()) {
6770 for (auto &Info : CSI->second) {
6771 Total += Info.TotalSize;
6772 if (TypeI->second == AllocationType::Cold)
6773 TotalCold += Info.TotalSize;
6774 }
6775 }
6776 }
6777 if (TotalCold * 100 >= Total * MinClonedColdBytePercent)
6778 AT = AllocationType::Cold;
6779 }
6780 return AT;
6781 };
6782 updateAllocationCall(Call&: Node->Call, AllocType: GetAllocTypeToUse(Node));
6783 if (MemProfReportHintedSizesDetail)
6784 OrigAllocAndCloneToNode[{Node->getOrigNode(), Node->Call.cloneNo()}] =
6785 Node;
6786 assert(Node->MatchingCalls.empty());
6787 // Update the allocation calls in function clones without a clone of this
6788 // node the same way as the node they should follow.
6789 for (auto &[Source, Calls] : GetUnassignedCallClones(Node)) {
6790 auto AT = GetAllocTypeToUse(Source);
6791 for (auto &Call : Calls) {
6792 updateAllocationCall(Call, AllocType: AT);
6793 // For the report, such an allocation clone follows Source's hint.
6794 if (MemProfReportHintedSizesDetail)
6795 OrigAllocAndCloneToNode[{Node->getOrigNode(), Call.cloneNo()}] =
6796 Source;
6797 }
6798 }
6799 return;
6800 }
6801
6802 auto CalleeFuncIt = CallsiteToCalleeFuncCloneMap.find(Node);
6803 if (CalleeFuncIt != CallsiteToCalleeFuncCloneMap.end()) {
6804 auto &CalleeFunc = CalleeFuncIt->second;
6805 updateCall(CallerCall&: Node->Call, CalleeFunc);
6806 if (MemProfReportHintedSizesDetail)
6807 OrigNodeAndCloneToCalleeClone[{
6808 Node->getOrigNode(), Node->Call.cloneNo()}] = CalleeFunc.cloneNo();
6809 // Update all the matching calls as well.
6810 for (auto &Call : Node->MatchingCalls)
6811 updateCall(CallerCall&: Call, CalleeFunc);
6812 }
6813
6814 // Now update all calls recorded earlier that are still in function clones
6815 // which don't have a clone of this callsite node, the same way as the
6816 // node they should follow.
6817 for (auto &[Source, Calls] : GetUnassignedCallClones(Node)) {
6818 auto It = CallsiteToCalleeFuncCloneMap.find(Source);
6819 // Fall back to the original Node's callee function.
6820 if (It == CallsiteToCalleeFuncCloneMap.end())
6821 It = CalleeFuncIt;
6822 if (It == CallsiteToCalleeFuncCloneMap.end())
6823 continue;
6824 for (auto &Call : Calls) {
6825 updateCall(CallerCall&: Call, CalleeFunc: It->second);
6826 if (MemProfReportHintedSizesDetail)
6827 OrigNodeAndCloneToCalleeClone[{Node->getOrigNode(), Call.cloneNo()}] =
6828 It->second.cloneNo();
6829 }
6830 }
6831 };
6832
6833 // Performs DFS traversal starting from allocation nodes to update calls to
6834 // reflect cloning decisions recorded earlier. For regular LTO this will
6835 // update the actual calls in the IR to call the appropriate function clone
6836 // (and add attributes to allocation calls), whereas for ThinLTO the decisions
6837 // are recorded in the summary entries.
6838 DenseSet<const ContextNode *> Visited;
6839 for (auto &Entry : AllocationCallToContextNodeMap)
6840 UpdateCalls(Entry.second, Visited, UpdateCalls);
6841
6842 return Changed;
6843}
6844
6845// Compute a SHA1 hash of the callsite and alloc version information of clone I
6846// in the summary, to use in detection of duplicate clones.
6847uint64_t ComputeHash(const FunctionSummary *FS, unsigned I) {
6848 SHA1 Hasher;
6849 // Add a record for the callsite or alloc at the given index in the summary,
6850 // with the given version. The index must be included, since two clones that
6851 // assign the same versions to different callsites or allocs are not
6852 // duplicates.
6853 auto AddRecord = [&](char Kind, uint32_t Index, uint32_t Version) {
6854 uint8_t Data[1 + 2 * sizeof(uint32_t)];
6855 Data[0] = Kind;
6856 support::endian::write32le(P: &Data[1], V: Index);
6857 support::endian::write32le(P: &Data[1 + sizeof(uint32_t)], V: Version);
6858 Hasher.update(Data);
6859 };
6860 // Update hash with any callsites that call non-default (non-zero) callee
6861 // versions.
6862 for (auto [Index, SN] : enumerate(First: FS->callsites())) {
6863 // In theory all callsites and allocs in this function should have the same
6864 // number of clone entries, but handle any discrepancies gracefully below
6865 // for NDEBUG builds.
6866 assert(
6867 SN.Clones.size() > I &&
6868 "Callsite summary has fewer entries than other summaries in function");
6869 if (SN.Clones.size() <= I || !SN.Clones[I])
6870 continue;
6871 AddRecord('c', Index, SN.Clones[I]);
6872 }
6873 // Update hash with any allocs that have non-default (non-None) hints.
6874 for (auto [Index, AN] : enumerate(First: FS->allocs())) {
6875 // In theory all callsites and allocs in this function should have the same
6876 // number of clone entries, but handle any discrepancies gracefully below
6877 // for NDEBUG builds.
6878 assert(AN.Versions.size() > I &&
6879 "Alloc summary has fewer entries than other summaries in function");
6880 if (AN.Versions.size() <= I ||
6881 (AllocationType)AN.Versions[I] == AllocationType::None)
6882 continue;
6883 AddRecord('a', Index, AN.Versions[I]);
6884 }
6885 return support::endian::read64le(P: Hasher.result().data());
6886}
6887
6888#ifndef NDEBUG
6889// Check that clones I and J of FS have identical callsite and alloc versions.
6890static bool clonesHaveSameVersions(const FunctionSummary *FS, unsigned I,
6891 unsigned J) {
6892 for (auto &SN : FS->callsites()) {
6893 assert(SN.Clones.size() > I && SN.Clones.size() > J &&
6894 "Callsite summary has fewer entries than other summaries in "
6895 "function");
6896 if (SN.Clones[I] != SN.Clones[J])
6897 return false;
6898 }
6899 for (auto &AN : FS->allocs()) {
6900 assert(AN.Versions.size() > I && AN.Versions.size() > J &&
6901 "Alloc summary has fewer entries than other summaries in function");
6902 if (AN.Versions[I] != AN.Versions[J])
6903 return false;
6904 }
6905 return true;
6906}
6907#endif
6908
6909static SmallVector<std::unique_ptr<ValueToValueMapTy>, 4> createFunctionClones(
6910 Function &F, unsigned NumClones, Module &M, OptimizationRemarkEmitter &ORE,
6911 std::map<const Function *, SmallPtrSet<const GlobalAlias *, 1>>
6912 &FuncToAliasMap,
6913 FunctionSummary *FS) {
6914 auto TakeDeclNameAndReplace = [](GlobalValue *DeclGV, GlobalValue *NewGV) {
6915 // We might have created this when adjusting callsite in another
6916 // function. It should be a declaration.
6917 assert(DeclGV->isDeclaration());
6918 NewGV->takeName(V: DeclGV);
6919 DeclGV->replaceAllUsesWith(V: NewGV);
6920 DeclGV->eraseFromParent();
6921 };
6922
6923 // Handle aliases to this function, and create analogous alias clones to the
6924 // provided clone of this function.
6925 auto CloneFuncAliases = [&](Function *NewF, unsigned I) {
6926 if (!FuncToAliasMap.count(x: &F))
6927 return;
6928 for (auto *A : FuncToAliasMap[&F]) {
6929 std::string AliasName = getMemProfFuncName(Base: A->getName(), CloneNo: I);
6930 auto *PrevA = M.getNamedAlias(Name: AliasName);
6931 auto *NewA = GlobalAlias::create(Ty: A->getValueType(),
6932 AddressSpace: A->getType()->getPointerAddressSpace(),
6933 Linkage: A->getLinkage(), Name: AliasName, Aliasee: NewF);
6934 NewA->copyAttributesFrom(Src: A);
6935 if (PrevA)
6936 TakeDeclNameAndReplace(PrevA, NewA);
6937 }
6938 };
6939
6940 // The first "clone" is the original copy, we should only call this if we
6941 // needed to create new clones.
6942 assert(NumClones > 1);
6943 SmallVector<std::unique_ptr<ValueToValueMapTy>, 4> VMaps;
6944 VMaps.reserve(N: NumClones - 1);
6945 FunctionsClonedThinBackend++;
6946
6947 // Map of hash of callsite/alloc versions to the instantiated function clone
6948 // (possibly the original) implementing those calls and its clone number. Used
6949 // to avoid instantiating duplicate function clones.
6950 // FIXME: Ideally the thin link would not generate such duplicate clones to
6951 // start with, but right now it happens due to phase ordering in the function
6952 // assignment and possible new clones that produces. We simply make each
6953 // duplicate an alias to the matching instantiated clone recorded in the map
6954 // (except for available_externally which are made declarations as they would
6955 // be aliases in the prevailing module, and available_externally aliases are
6956 // not well supported right now).
6957 DenseMap<uint64_t, std::pair<Function *, unsigned>> HashToFunc;
6958
6959 // Save the hash of the original function version.
6960 HashToFunc[ComputeHash(FS, I: 0)] = {&F, 0};
6961
6962 for (unsigned I = 1; I < NumClones; I++) {
6963 VMaps.emplace_back(Args: std::make_unique<ValueToValueMapTy>());
6964 std::string Name = getMemProfFuncName(Base: F.getName(), CloneNo: I);
6965 auto Hash = ComputeHash(FS, I);
6966 // If this clone would duplicate a previously seen clone, don't generate the
6967 // duplicate clone body, just make an alias to satisfy any (potentially
6968 // cross-module) references.
6969 if (auto It = HashToFunc.find(Val: Hash); It != HashToFunc.end()) {
6970 FunctionCloneDuplicatesThinBackend++;
6971 auto [Func, OrigI] = It->second;
6972 assert(clonesHaveSameVersions(FS, I, OrigI) &&
6973 "Duplicate clone hash collision");
6974 if (Func->hasAvailableExternallyLinkage()) {
6975 // Skip these as EliminateAvailableExternallyPass does not handle
6976 // available_externally aliases correctly and we end up with an
6977 // available_externally alias to a declaration. Just create a
6978 // declaration for now as we know we will have a definition in another
6979 // module.
6980 auto Decl = M.getOrInsertFunction(Name, T: Func->getFunctionType());
6981 ORE.emit(OptDiag: OptimizationRemark(DEBUG_TYPE, "MemprofClone", &F)
6982 << "created clone decl " << ore::NV("Decl", Decl.getCallee()));
6983 continue;
6984 }
6985 auto *PrevF = M.getFunction(Name);
6986 auto *Alias = GlobalAlias::create(Name, Aliasee: Func);
6987 if (PrevF)
6988 TakeDeclNameAndReplace(PrevF, Alias);
6989 ORE.emit(OptDiag: OptimizationRemark(DEBUG_TYPE, "MemprofClone", &F)
6990 << "created clone alias " << ore::NV("Alias", Alias));
6991
6992 // Now handle aliases to this function, and clone those as well.
6993 CloneFuncAliases(Func, I);
6994 continue;
6995 }
6996 auto *NewF = CloneFunction(F: &F, VMap&: *VMaps.back());
6997 HashToFunc[Hash] = {NewF, I};
6998 FunctionClonesThinBackend++;
6999 // Strip !kcfi_type (see comment in cloneFunctionForCallsite).
7000 NewF->setMetadata(KindID: LLVMContext::MD_kcfi_type, Node: nullptr);
7001 // Strip memprof and callsite metadata from clone as they are no longer
7002 // needed.
7003 for (auto &BB : *NewF) {
7004 for (auto &Inst : BB) {
7005 Inst.setMetadata(KindID: LLVMContext::MD_memprof, Node: nullptr);
7006 Inst.setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
7007 }
7008 }
7009 auto *PrevF = M.getFunction(Name);
7010 if (PrevF)
7011 TakeDeclNameAndReplace(PrevF, NewF);
7012 else
7013 NewF->setName(Name);
7014 updateSubprogramLinkageName(NewFunc: NewF, Name);
7015 ORE.emit(OptDiag: OptimizationRemark(DEBUG_TYPE, "MemprofClone", &F)
7016 << "created clone " << ore::NV("NewFunction", NewF));
7017
7018 // Now handle aliases to this function, and clone those as well.
7019 CloneFuncAliases(NewF, I);
7020 }
7021 return VMaps;
7022}
7023
7024// If F is an imported copy of an alias (materialized as a Function by
7025// replaceAliasWithAliasee during FunctionImport while preserving the aliasee's
7026// !guid metadata for body-level passes), return the original alias GUID
7027// recorded in its thinlto_alias_guid metadata.
7028static std::optional<GlobalValue::GUID>
7029getImportedAliasGUID(const Function &F) {
7030 if (auto *MD = F.getMetadata(Kind: "thinlto_alias_guid"))
7031 return mdconst::extract<ConstantInt>(MD: MD->getOperand(I: 0))->getZExtValue();
7032 return std::nullopt;
7033}
7034
7035// Locate the summary for GV. This is complicated by the fact that it might
7036// have been internalized, promoted, or imported as a copy of an alias.
7037static ValueInfo findValueInfoForGV(const GlobalValue &GV, const Module &M,
7038 const ModuleSummaryIndex *ImportSummary,
7039 const Function *CallingFunc = nullptr) {
7040 if (const auto *F = dyn_cast<Function>(Val: &GV))
7041 if (auto AliasGUID = getImportedAliasGUID(F: *F)) {
7042 ValueInfo AliasVI = ImportSummary->getValueInfo(GUID: *AliasGUID);
7043 assert(AliasVI && "Expected ValueInfo for imported alias GUID");
7044 return AliasVI;
7045 }
7046
7047 // FIXME: Ideally we would retain the original GUID in some fashion on the
7048 // function (e.g. as metadata), but for now do our best to locate the
7049 // summary without that information.
7050 ValueInfo TheFnVI = ImportSummary->getValueInfo(GUID: GV.getGUID());
7051 if (!TheFnVI)
7052 // See if GV was internalized, by checking index directly with
7053 // original name (this avoids the name adjustment done by getGUID() for
7054 // internal symbols).
7055 TheFnVI = ImportSummary->getValueInfo(
7056 GUID: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: GV.getName()));
7057 if (TheFnVI)
7058 return TheFnVI;
7059 // Now query with the original name before any promotion was performed.
7060 StringRef OrigName =
7061 ModuleSummaryIndex::getOriginalNameBeforePromote(Name: GV.getName());
7062 // When this pass is enabled, we always add thinlto_src_file provenance
7063 // metadata to imported function definitions, which allows us to recreate the
7064 // original internal symbol's GUID.
7065 const auto *GO = dyn_cast<GlobalObject>(Val: &GV);
7066 MDNode *SrcFileMD = GO ? GO->getMetadata(Kind: "thinlto_src_file") : nullptr;
7067 // If this is a call to an imported/promoted local for which we didn't import
7068 // the definition, the metadata will not exist on the declaration. However,
7069 // since we are doing this early, before any inlining in the LTO backend, we
7070 // can simply look at the metadata on the calling function which must have
7071 // been from the same module if GV was an internal symbol originally.
7072 if (!SrcFileMD && GV.isDeclaration()) {
7073 // We would only call this for a declaration for a direct callsite, in which
7074 // case the caller would have provided the calling function pointer.
7075 assert(CallingFunc);
7076 SrcFileMD = CallingFunc->getMetadata(Kind: "thinlto_src_file");
7077 // If this is a promoted local (OrigName != GV.getName()), since this is a
7078 // declaration, it must be imported from a different module and therefore we
7079 // should always find the metadata on its calling function. Any call to a
7080 // promoted local that came from this module should still be a definition.
7081 assert(SrcFileMD || OrigName == GV.getName());
7082 }
7083 StringRef SrcFile = M.getSourceFileName();
7084 if (SrcFileMD)
7085 SrcFile = dyn_cast<MDString>(Val: SrcFileMD->getOperand(I: 0))->getString();
7086 std::string OrigId = GlobalValue::getGlobalIdentifier(
7087 Name: OrigName, Linkage: GlobalValue::InternalLinkage, FileName: SrcFile);
7088 TheFnVI = ImportSummary->getValueInfo(
7089 GUID: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigId));
7090 // Internal func in original module may have gotten a numbered suffix if we
7091 // imported an external function with the same name. This happens
7092 // automatically during IR linking for naming conflicts. It would have to
7093 // still be internal in that case (otherwise it would have been renamed on
7094 // promotion in which case we wouldn't have a naming conflict).
7095 if (!TheFnVI && OrigName == GV.getName() && GV.hasLocalLinkage() &&
7096 GV.getName().contains(C: '.')) {
7097 OrigName = GV.getName().rsplit(Separator: '.').first;
7098 OrigId = GlobalValue::getGlobalIdentifier(
7099 Name: OrigName, Linkage: GlobalValue::InternalLinkage, FileName: SrcFile);
7100 TheFnVI = ImportSummary->getValueInfo(
7101 GUID: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigId));
7102 }
7103 // The only way we may not have a VI is if this is a declaration created for
7104 // an imported reference. For distributed ThinLTO we may not have a VI for
7105 // such declarations in the distributed summary.
7106 assert(TheFnVI || GV.isDeclaration());
7107 return TheFnVI;
7108}
7109
7110// Find the CallsiteInfo record synthesized by the thin link for profiled target
7111// CandidateGUID of an indirect tail call, in MapTailCallCalleeVIToCallsite
7112// (which maps each synthesized record's callee to the record). Returns null if
7113// there is none. OrigNameGUIDToTailCallCallsite caches a map from the original
7114// name GUID of each record's callee to the record, and is built on first use
7115// (along with AliasVIToGV, which caches alias ValueInfos in M).
7116static const CallsiteInfo *findIndirectTailCallCallsite(
7117 uint64_t CandidateGUID,
7118 const DenseMap<ValueInfo, CallsiteInfo> &MapTailCallCalleeVIToCallsite,
7119 std::optional<DenseMap<uint64_t, const CallsiteInfo *>>
7120 &OrigNameGUIDToTailCallCallsite,
7121 std::optional<DenseMap<ValueInfo, const GlobalAlias *>> &AliasVIToGV,
7122 const Module &M, const ModuleSummaryIndex *ImportSummary,
7123 InstrProfSymtab &Symtab) {
7124 auto End = MapTailCallCalleeVIToCallsite.end();
7125 // The callee of the synthesized record is normally the VP metadata GUID, so
7126 // check that first.
7127 if (ValueInfo VI = ImportSummary->getValueInfo(GUID: CandidateGUID)) {
7128 auto It = MapTailCallCalleeVIToCallsite.find(Val: VI);
7129 if (It != End)
7130 return &It->second;
7131 }
7132 // For local targets the VP metadata may contain the GUID of the original
7133 // name without the source file prefix, which the thin link translates to the
7134 // actual GUID of the local function (see updateValueInfoForIndirectCalls).
7135 if (auto LocalGUID = ImportSummary->getGUIDFromOriginalID(OriginalID: CandidateGUID)) {
7136 if (ValueInfo LocalVI = ImportSummary->getValueInfo(GUID: LocalGUID)) {
7137 auto It = MapTailCallCalleeVIToCallsite.find(Val: LocalVI);
7138 if (It != End)
7139 return &It->second;
7140 }
7141 }
7142 // The index for a distributed ThinLTO backend does not record the original
7143 // names of local functions, in which case the above lookup via
7144 // getGUIDFromOriginalID fails. Instead, compute the original name GUID of
7145 // each record's callee from its function or alias in this module, using its
7146 // name before any promotion.
7147 if (!OrigNameGUIDToTailCallCallsite) {
7148 // Construct the (initially empty) map so that it is marked as built, even
7149 // if no entries are added below, to avoid rebuilding it on later calls.
7150 OrigNameGUIDToTailCallCallsite.emplace();
7151 for (const auto &[CalleeVI, Callsite] : MapTailCallCalleeVIToCallsite) {
7152 const GlobalValue *Callee = Symtab.getFunction(FuncMD5Hash: CalleeVI.getGUID());
7153 if (!Callee) {
7154 // InstrProfSymtab only indexes Functions, not GlobalAliases (imported
7155 // alias copies are materialized as Functions with thinlto_alias_guid
7156 // metadata and are found via Symtab above). Lazily build a module-wide
7157 // map for same-module function GlobalAliases once per module so we can
7158 // recover the alias's pre-promotion name.
7159 if (!AliasVIToGV) {
7160 AliasVIToGV.emplace();
7161 for (const GlobalAlias &A : M.aliases()) {
7162 if (isMemProfClone(GV: A) ||
7163 !isa_and_nonnull<Function>(Val: A.getAliaseeObject()))
7164 continue;
7165 if (ValueInfo VI = findValueInfoForGV(GV: A, M, ImportSummary))
7166 AliasVIToGV->try_emplace(Key: VI, Args: &A);
7167 }
7168 }
7169 Callee = AliasVIToGV->lookup(Val: CalleeVI);
7170 if (!Callee)
7171 continue;
7172 }
7173 StringRef OrigName =
7174 ModuleSummaryIndex::getOriginalNameBeforePromote(Name: Callee->getName());
7175 auto [It, Inserted] = OrigNameGUIDToTailCallCallsite->try_emplace(
7176 Key: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigName), Args: &Callsite);
7177 // If multiple callees have the same original name, we can't tell which
7178 // corresponds to the profiled target.
7179 if (!Inserted && It->second != &Callsite)
7180 It->second = nullptr;
7181 }
7182 }
7183 return OrigNameGUIDToTailCallCallsite->lookup(Val: CandidateGUID);
7184}
7185
7186bool MemProfContextDisambiguation::initializeIndirectCallPromotionInfo(
7187 Module &M) {
7188 ICallAnalysis = std::make_unique<ICallPromotionAnalysis>();
7189 Symtab = std::make_unique<InstrProfSymtab>();
7190 // Don't add canonical names, to avoid multiple functions to the symtab
7191 // when they both have the same root name with "." suffixes stripped.
7192 // If we pick the wrong one then this could lead to incorrect ICP and calling
7193 // a memprof clone that we don't actually create (resulting in linker unsats).
7194 // What this means is that the GUID of the function *must* match that in the
7195 // VP metadata to allow promotion. In practice this should not be a
7196 // limitation, since local functions have a GUID from before they were
7197 // promoted and global function names shouldn't need any special handling
7198 // (they should not get the ".llvm.*" suffix that the canonicalization
7199 // handling is attempting to strip).
7200 // Key imported copies of aliases by the alias's GUID rather than the
7201 // aliasee's GUID retained in their !guid metadata, since we rely on the
7202 // looked up function's name (to compute its original name GUID and the names
7203 // of its clones), and so that they don't shadow the aliasee's own function
7204 // when both are in the module. See AliaseeGUIDToImportedAlias for how
7205 // performICP handles profiled targets that are the aliasee of an imported
7206 // alias copy when the aliasee itself is not in the module.
7207 AliaseeGUIDToImportedAlias.reset();
7208 if (Error E = Symtab->create(M, /*InLTO=*/true, /*AddCanonical=*/false,
7209 /*KeyImportedAliasesByAliasGUID=*/true)) {
7210 std::string SymtabFailure = toString(E: std::move(E));
7211 M.getContext().emitError(ErrorStr: "Failed to create symtab: " + SymtabFailure);
7212 return false;
7213 }
7214 return true;
7215}
7216
7217#ifndef NDEBUG
7218// Sanity check that the MIB stack ids match between the summary and
7219// instruction metadata.
7220static void checkAllocContextIds(
7221 const AllocInfo &AllocNode, const MDNode *MemProfMD,
7222 const CallStack<MDNode, MDNode::op_iterator> &CallsiteContext,
7223 const ModuleSummaryIndex *ImportSummary) {
7224 auto MIBIter = AllocNode.MIBs.begin();
7225 for (auto &MDOp : MemProfMD->operands()) {
7226 assert(MIBIter != AllocNode.MIBs.end());
7227 // The combined index may have been written without the MIB contexts (see
7228 // -combined-index-memprof-context, which defaults to false in NDEBUG
7229 // builds), e.g. if the thin link was performed by a release build while
7230 // this backend has assertions enabled. Since the MIB context in the summary
7231 // can never be empty after trimming the allocation's callsite context, an
7232 // empty list means the context was not serialized, and there is nothing to
7233 // check.
7234 if (MIBIter->StackIdIndices.empty()) {
7235 MIBIter++;
7236 continue;
7237 }
7238 auto StackIdIndexIter = MIBIter->StackIdIndices.begin();
7239 auto *MIBMD = cast<const MDNode>(MDOp);
7240 MDNode *StackMDNode = getMIBStackNode(MIBMD);
7241 assert(StackMDNode);
7242 CallStack<MDNode, MDNode::op_iterator> StackContext(StackMDNode);
7243 auto ContextIterBegin =
7244 StackContext.beginAfterSharedPrefix(CallsiteContext);
7245 // Skip the checking on the first iteration.
7246 uint64_t LastStackContextId =
7247 (ContextIterBegin != StackContext.end() && *ContextIterBegin == 0) ? 1
7248 : 0;
7249 for (auto ContextIter = ContextIterBegin; ContextIter != StackContext.end();
7250 ++ContextIter) {
7251 // If this is a direct recursion, simply skip the duplicate
7252 // entries, to be consistent with how the summary ids were
7253 // generated during ModuleSummaryAnalysis.
7254 if (LastStackContextId == *ContextIter)
7255 continue;
7256 LastStackContextId = *ContextIter;
7257 assert(StackIdIndexIter != MIBIter->StackIdIndices.end());
7258 assert(ImportSummary->getStackIdAtIndex(*StackIdIndexIter) ==
7259 *ContextIter);
7260 StackIdIndexIter++;
7261 }
7262 MIBIter++;
7263 }
7264}
7265#endif
7266
7267bool MemProfContextDisambiguation::applyImport(Module &M) {
7268 assert(ImportSummary);
7269 bool Changed = false;
7270
7271 // We also need to clone any aliases that reference cloned functions, because
7272 // the modified callsites may invoke via the alias. Keep track of the aliases
7273 // for each function.
7274 std::map<const Function *, SmallPtrSet<const GlobalAlias *, 1>>
7275 FuncToAliasMap;
7276 for (auto &A : M.aliases()) {
7277 auto *Aliasee = A.getAliaseeObject();
7278 if (auto *F = dyn_cast<Function>(Val: Aliasee))
7279 FuncToAliasMap[F].insert(Ptr: &A);
7280 }
7281
7282 if (!initializeIndirectCallPromotionInfo(M))
7283 return false;
7284
7285 // Lazily populated map from alias ValueInfo to the corresponding same-module
7286 // GlobalAlias, used by findIndirectTailCallCallsite when a local alias target
7287 // is not in Symtab.
7288 std::optional<DenseMap<ValueInfo, const GlobalAlias *>> AliasVIToGV;
7289
7290 for (auto &F : M) {
7291 if (F.isDeclaration() || isMemProfClone(GV: F))
7292 continue;
7293
7294 OptimizationRemarkEmitter ORE(&F);
7295
7296 SmallVector<std::unique_ptr<ValueToValueMapTy>, 4> VMaps;
7297 bool ClonesCreated = false;
7298 unsigned NumClonesCreated = 0;
7299 auto CloneFuncIfNeeded = [&](unsigned NumClones, FunctionSummary *FS) {
7300 // We should at least have version 0 which is the original copy.
7301 assert(NumClones > 0);
7302 // If only one copy needed use original.
7303 if (NumClones == 1)
7304 return;
7305 // If we already performed cloning of this function, confirm that the
7306 // requested number of clones matches (the thin link should ensure the
7307 // number of clones for each constituent callsite is consistent within
7308 // each function), before returning.
7309 if (ClonesCreated) {
7310 assert(NumClonesCreated == NumClones);
7311 return;
7312 }
7313 VMaps = createFunctionClones(F, NumClones, M, ORE, FuncToAliasMap, FS);
7314 // The first "clone" is the original copy, which doesn't have a VMap.
7315 assert(VMaps.size() == NumClones - 1);
7316 Changed = true;
7317 ClonesCreated = true;
7318 NumClonesCreated = NumClones;
7319 };
7320
7321 auto CloneCallsite = [&](const CallsiteInfo &StackNode, CallBase *CB,
7322 Function *CalledFunction, FunctionSummary *FS) {
7323 // Perform cloning if not yet done.
7324 CloneFuncIfNeeded(/*NumClones=*/StackNode.Clones.size(), FS);
7325
7326 assert(!isMemProfClone(*CalledFunction));
7327
7328 // Because we update the cloned calls by calling setCalledOperand (see
7329 // comment below), out of an abundance of caution make sure the called
7330 // function was actually the called operand (or its aliasee). We also
7331 // strip pointer casts when looking for calls (to match behavior during
7332 // summary generation), however, with opaque pointers in theory this
7333 // should not be an issue. Note we still clone the current function
7334 // (containing this call) above, as that could be needed for its callers.
7335 auto *GA = dyn_cast_or_null<GlobalAlias>(Val: CB->getCalledOperand());
7336 if (CalledFunction != CB->getCalledOperand() &&
7337 (!GA || CalledFunction != GA->getAliaseeObject())) {
7338 SkippedCallsCloning++;
7339 return;
7340 }
7341 // Update the calls per the summary info.
7342 // Save orig name since it gets updated in the first iteration
7343 // below.
7344 auto CalleeOrigName = CalledFunction->getName();
7345 for (unsigned J = 0; J < StackNode.Clones.size(); J++) {
7346 // If the VMap is empty, this clone was a duplicate of another and was
7347 // created as an alias or a declaration.
7348 if (J > 0 && VMaps[J - 1]->empty())
7349 continue;
7350 // Do nothing if this version calls the original version of its
7351 // callee.
7352 if (!StackNode.Clones[J])
7353 continue;
7354 auto NewF = M.getOrInsertFunction(
7355 Name: getMemProfFuncName(Base: CalleeOrigName, CloneNo: StackNode.Clones[J]),
7356 T: CalledFunction->getFunctionType());
7357 CallBase *CBClone;
7358 // Copy 0 is the original function.
7359 if (!J)
7360 CBClone = CB;
7361 else
7362 CBClone = cast<CallBase>(Val&: (*VMaps[J - 1])[CB]);
7363 // Set the called operand directly instead of calling setCalledFunction,
7364 // as the latter mutates the function type on the call. In rare cases
7365 // we may have a slightly different type on a callee function
7366 // declaration due to it being imported from a different module with
7367 // incomplete types. We really just want to change the name of the
7368 // function to the clone, and not make any type changes.
7369 CBClone->setCalledOperand(NewF.getCallee());
7370 ORE.emit(OptDiag: OptimizationRemark(DEBUG_TYPE, "MemprofCall", CBClone)
7371 << ore::NV("Call", CBClone) << " in clone "
7372 << ore::NV("Caller", CBClone->getFunction())
7373 << " assigned to call function clone "
7374 << ore::NV("Callee", NewF.getCallee()));
7375 }
7376 };
7377
7378 // Locate the summary for F.
7379 ValueInfo TheFnVI = findValueInfoForGV(GV: F, M, ImportSummary);
7380 // If not found, this could be an imported local (see comment in
7381 // findValueInfoForGV). Skip for now as it will be cloned in its original
7382 // module (where it would have been promoted to global scope so should
7383 // satisfy any reference in this module).
7384 if (!TheFnVI)
7385 continue;
7386
7387 auto *GVSummary =
7388 ImportSummary->findSummaryInModule(VI: TheFnVI, ModuleId: M.getModuleIdentifier());
7389 if (!GVSummary) {
7390 // Must have been imported, use the summary which matches the definition。
7391 // (might be multiple if this was a linkonce_odr).
7392 auto SrcModuleMD = F.getMetadata(Kind: "thinlto_src_module");
7393 assert(SrcModuleMD &&
7394 "enable-import-metadata is needed to emit thinlto_src_module");
7395 StringRef SrcModule =
7396 dyn_cast<MDString>(Val: SrcModuleMD->getOperand(I: 0))->getString();
7397 for (auto &GVS : TheFnVI.getSummaryList()) {
7398 if (GVS->modulePath() == SrcModule) {
7399 GVSummary = GVS.get();
7400 break;
7401 }
7402 }
7403 assert(GVSummary && GVSummary->modulePath() == SrcModule);
7404 }
7405
7406 // If this was an imported alias skip it as we won't have the function
7407 // summary, and it should be cloned in the original module.
7408 if (isa<AliasSummary>(Val: GVSummary))
7409 continue;
7410
7411 auto *FS = cast<FunctionSummary>(Val: GVSummary->getBaseObject());
7412
7413 if (FS->allocs().empty() && FS->callsites().empty())
7414 continue;
7415
7416 auto SI = FS->callsites().begin();
7417 auto AI = FS->allocs().begin();
7418
7419 // To handle callsite infos synthesized for tail calls which have missing
7420 // frames in the profiled context, map callee VI to the synthesized callsite
7421 // info.
7422 DenseMap<ValueInfo, CallsiteInfo> MapTailCallCalleeVIToCallsite;
7423 // Iterate the callsites for this function in reverse, since we place all
7424 // those synthesized for tail calls at the end.
7425 for (auto CallsiteIt = FS->callsites().rbegin();
7426 CallsiteIt != FS->callsites().rend(); CallsiteIt++) {
7427 auto &Callsite = *CallsiteIt;
7428 // Stop as soon as we see a non-synthesized callsite info (see comment
7429 // above loop). All the entries added for discovered tail calls have empty
7430 // stack ids.
7431 if (!Callsite.StackIdIndices.empty())
7432 break;
7433 MapTailCallCalleeVIToCallsite.insert(KV: {Callsite.Callee, Callsite});
7434 }
7435 // Maps the original name GUID of each synthesized callsite info's callee to
7436 // the callsite info, built on first use by findIndirectTailCallCallsite.
7437 std::optional<DenseMap<uint64_t, const CallsiteInfo *>>
7438 OrigNameGUIDToTailCallCallsite;
7439
7440 // Keeps track of needed ICP for the function.
7441 SmallVector<ICallAnalysisData> ICallAnalysisInfo;
7442
7443 // Assume for now that the instructions are in the exact same order
7444 // as when the summary was created, but confirm this is correct by
7445 // matching the stack ids.
7446 for (auto &BB : F) {
7447 for (auto &I : BB) {
7448 auto *CB = dyn_cast<CallBase>(Val: &I);
7449 // Same handling as when creating module summary.
7450 if (!mayHaveMemprofSummary(CB))
7451 continue;
7452
7453 auto *CalledValue = CB->getCalledOperand();
7454 auto *CalledFunction = CB->getCalledFunction();
7455 if (CalledValue && !CalledFunction) {
7456 CalledValue = CalledValue->stripPointerCasts();
7457 // Stripping pointer casts can reveal a called function.
7458 CalledFunction = dyn_cast<Function>(Val: CalledValue);
7459 }
7460 // Check if this is an alias to a function. If so, get the
7461 // called aliasee for the checks below.
7462 if (auto *GA = dyn_cast<GlobalAlias>(Val: CalledValue)) {
7463 assert(!CalledFunction &&
7464 "Expected null called function in callsite for alias");
7465 CalledFunction = dyn_cast<Function>(Val: GA->getAliaseeObject());
7466 }
7467
7468 CallStack<MDNode, MDNode::op_iterator> CallsiteContext(
7469 I.getMetadata(KindID: LLVMContext::MD_callsite));
7470 auto *MemProfMD = I.getMetadata(KindID: LLVMContext::MD_memprof);
7471
7472 // Include allocs that were already assigned a memprof function
7473 // attribute in the statistics. Only do this for those that do not have
7474 // memprof metadata, since we add an "ambiguous" memprof attribute by
7475 // default.
7476 if (CB->getAttributes().hasFnAttr(Kind: "memprof") && !MemProfMD) {
7477 CB->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "cold"
7478 ? AllocTypeColdThinBackend++
7479 : AllocTypeNotColdThinBackend++;
7480 OrigAllocsThinBackend++;
7481 AllocVersionsThinBackend++;
7482 if (!MaxAllocVersionsThinBackend)
7483 MaxAllocVersionsThinBackend = 1;
7484 continue;
7485 }
7486
7487 if (MemProfMD) {
7488 // Consult the next alloc node.
7489 assert(AI != FS->allocs().end());
7490 auto &AllocNode = *(AI++);
7491
7492#ifndef NDEBUG
7493 checkAllocContextIds(AllocNode, MemProfMD, CallsiteContext,
7494 ImportSummary);
7495#endif
7496
7497 // Perform cloning if not yet done.
7498 CloneFuncIfNeeded(/*NumClones=*/AllocNode.Versions.size(), FS);
7499
7500 OrigAllocsThinBackend++;
7501 AllocVersionsThinBackend += AllocNode.Versions.size();
7502 if (MaxAllocVersionsThinBackend < AllocNode.Versions.size())
7503 MaxAllocVersionsThinBackend = AllocNode.Versions.size();
7504
7505 // If there is only one version that means we didn't end up
7506 // considering this function for cloning. Count that as unclonable
7507 // unless the single version is cold due to the
7508 // MinClonedColdBytePercent heuristic. The alloc may still be none
7509 // type (not reached during whole program analysis), in which case
7510 // there is nothing to apply; otherwise it got the type the whole
7511 // program analysis resolved it to (not cold, or both types when the
7512 // contexts could not be disambiguated), which is applied below like
7513 // for cloned functions so that the attribute matches the regular
7514 // LTO result.
7515 if (AllocNode.Versions.size() == 1 &&
7516 (AllocationType)AllocNode.Versions[0] != AllocationType::Cold)
7517 UnclonableAllocsThinBackend++;
7518
7519 // Update the allocation types per the summary info.
7520 for (unsigned J = 0; J < AllocNode.Versions.size(); J++) {
7521 // If the VMap is empty, this clone was a duplicate of another and
7522 // was created as an alias or a declaration.
7523 if (J > 0 && VMaps[J - 1]->empty())
7524 continue;
7525 // Ignore any that didn't get an assigned allocation type.
7526 if (AllocNode.Versions[J] == (uint8_t)AllocationType::None)
7527 continue;
7528 AllocationType AllocTy = (AllocationType)AllocNode.Versions[J];
7529 // The whole program analysis only records both types when
7530 // ambiguous attributes are enabled; if the backend runs without
7531 // them, fall back to the not cold default.
7532 if (isAmbiguousAllocType(AllocTypes: (uint8_t)AllocTy) &&
7533 !MemProfUseAmbiguousAttributes)
7534 AllocTy = AllocationType::NotCold;
7535 const bool IsAmbiguous = isAmbiguousAllocType(AllocTypes: (uint8_t)AllocTy);
7536 if (AllocTy == AllocationType::Cold)
7537 AllocTypeColdThinBackend++;
7538 else if (IsAmbiguous)
7539 AllocTypeAmbiguousThinBackend++;
7540 else
7541 AllocTypeNotColdThinBackend++;
7542 std::string AllocTypeString = getAllocTypeAttributeString(Type: AllocTy);
7543 CallBase *CBClone;
7544 // Copy 0 is the original function.
7545 if (!J)
7546 CBClone = CB;
7547 else
7548 // Since VMaps are only created for new clones, we index with
7549 // clone J-1 (J==0 is the original clone and does not have a VMaps
7550 // entry).
7551 CBClone = cast<CallBase>(Val&: (*VMaps[J - 1])[CB]);
7552 if (IsAmbiguous) {
7553 // The contexts could not be disambiguated: keep (or add) the
7554 // ambiguous attribute from matching instead of a specific type.
7555 addAmbiguousAttribute(CB: CBClone);
7556 } else {
7557 removeAnyExistingAmbiguousAttribute(CB: CBClone);
7558 CBClone->addFnAttr(Attr: llvm::Attribute::get(Context&: F.getContext(), Kind: "memprof",
7559 Val: AllocTypeString));
7560 }
7561 ORE.emit(OptDiag: OptimizationRemark(DEBUG_TYPE, "MemprofAttribute", CBClone)
7562 << ore::NV("AllocationCall", CBClone) << " in clone "
7563 << ore::NV("Caller", CBClone->getFunction())
7564 << " marked with memprof allocation attribute "
7565 << ore::NV("Attribute", AllocTypeString));
7566 }
7567 } else if (!CallsiteContext.empty()) {
7568 if (!CalledFunction) {
7569#ifndef NDEBUG
7570 // We should have skipped inline assembly calls.
7571 auto *CI = dyn_cast<CallInst>(CB);
7572 assert(!CI || !CI->isInlineAsm());
7573#endif
7574 // We should have skipped direct calls via a Constant.
7575 assert(CalledValue && !isa<Constant>(CalledValue));
7576
7577 // This is an indirect call, see if we have profile information and
7578 // whether any clones were recorded for the profiled targets (that
7579 // we synthesized CallsiteInfo summary records for when building the
7580 // index). These records are the next ones in the function's
7581 // callsites, one per profiled target.
7582 auto NumClones = recordICPInfo(
7583 CB,
7584 GetCallsite: [&](const InstrProfValueData &Candidate) {
7585 assert(SI != FS->callsites().end());
7586#ifndef NDEBUG
7587 auto CalleeValueInfo =
7588 ImportSummary->getValueInfo(Candidate.Value);
7589 auto LocalGUID =
7590 ImportSummary->getGUIDFromOriginalID(Candidate.Value);
7591#endif
7592 // We might not have a ValueInfo if this is a distributed
7593 // ThinLTO backend and decided not to import that function.
7594 // The thin link may also have updated the callee from the
7595 // original name GUID recorded in the VP metadata to the GUID
7596 // of the local target function (see
7597 // updateValueInfoForIndirectCalls).
7598 assert(!CalleeValueInfo || SI->Callee == CalleeValueInfo ||
7599 (SI->Callee && SI->Callee.getGUID() == LocalGUID));
7600 return &*(SI++);
7601 },
7602 ICallAnalysisInfo);
7603
7604 // Perform cloning if not yet done. This is done here in case
7605 // we don't need to do ICP, but might need to clone this
7606 // function as it is the target of other cloned calls.
7607 if (NumClones)
7608 CloneFuncIfNeeded(NumClones, FS);
7609 }
7610
7611 else {
7612 // Consult the next callsite node.
7613 assert(SI != FS->callsites().end());
7614 auto &StackNode = *(SI++);
7615
7616#ifndef NDEBUG
7617 // Sanity check that the stack ids match between the summary and
7618 // instruction metadata.
7619 auto StackIdIndexIter = StackNode.StackIdIndices.begin();
7620 for (auto StackId : CallsiteContext) {
7621 assert(StackIdIndexIter != StackNode.StackIdIndices.end());
7622 assert(ImportSummary->getStackIdAtIndex(*StackIdIndexIter) ==
7623 StackId);
7624 StackIdIndexIter++;
7625 }
7626#endif
7627
7628 CloneCallsite(StackNode, CB, CalledFunction, FS);
7629 }
7630 } else if (CB->isTailCall() && CalledFunction) {
7631 // Locate the synthesized callsite info for the callee VI, if any was
7632 // created, and use that for cloning.
7633 ValueInfo CalleeVI = findValueInfoForGV(
7634 GV: *cast<GlobalValue>(Val: CalledValue), M, ImportSummary, CallingFunc: &F);
7635 if (CalleeVI && MapTailCallCalleeVIToCallsite.count(Val: CalleeVI)) {
7636 auto Callsite = MapTailCallCalleeVIToCallsite.find(Val: CalleeVI);
7637 assert(Callsite != MapTailCallCalleeVIToCallsite.end());
7638 CloneCallsite(Callsite->second, CB, CalledFunction, FS);
7639 }
7640 } else if (CB->isTailCall() && !CalledFunction &&
7641 !MapTailCallCalleeVIToCallsite.empty()) {
7642 // An indirect tail call, which the thin link may have identified as
7643 // being on profiled contexts with a missing frame, via the profiled
7644 // targets in its VP metadata. If so, the callsite infos synthesized
7645 // for those targets are used to perform ICP so that each clone of
7646 // this function calls the assigned clone of the target.
7647 auto PrevNumICallAnalysisInfo = ICallAnalysisInfo.size();
7648 auto NumClones = recordICPInfo(
7649 CB,
7650 GetCallsite: [&](const InstrProfValueData &Candidate) {
7651 return findIndirectTailCallCallsite(
7652 CandidateGUID: Candidate.Value, MapTailCallCalleeVIToCallsite,
7653 OrigNameGUIDToTailCallCallsite, AliasVIToGV, M,
7654 ImportSummary, Symtab&: *Symtab);
7655 },
7656 ICallAnalysisInfo);
7657 if (ICallAnalysisInfo.size() > PrevNumICallAnalysisInfo)
7658 IndirectTailCallsRecordedForICPThinBackend++;
7659 if (NumClones)
7660 CloneFuncIfNeeded(NumClones, FS);
7661 }
7662 }
7663 }
7664
7665 // Now do any promotion required for cloning.
7666 performICP(M, VMaps, ICallAnalysisInfo, ORE);
7667 }
7668
7669 // We skip some of the functions and instructions above, so remove all the
7670 // metadata in a single sweep here.
7671 for (auto &F : M) {
7672 // We can skip memprof clones because createFunctionClones already strips
7673 // the metadata from the newly created clones.
7674 if (F.isDeclaration() || isMemProfClone(GV: F))
7675 continue;
7676 for (auto &BB : F) {
7677 for (auto &I : BB) {
7678 if (!isa<CallBase>(Val: I))
7679 continue;
7680 I.setMetadata(KindID: LLVMContext::MD_memprof, Node: nullptr);
7681 I.setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
7682 }
7683 }
7684 }
7685
7686 return Changed;
7687}
7688
7689unsigned MemProfContextDisambiguation::recordICPInfo(
7690 CallBase *CB,
7691 function_ref<const CallsiteInfo *(const InstrProfValueData &)> GetCallsite,
7692 SmallVector<ICallAnalysisData> &ICallAnalysisInfo) {
7693 // First see if we have profile information for this indirect call.
7694 uint32_t NumCandidates;
7695 uint64_t TotalCount;
7696 auto CandidateProfileData =
7697 ICallAnalysis->getPromotionCandidatesForInstruction(
7698 I: CB, TotalCount, NumCandidates,
7699 MaxNumValueData: getMaxSummaryIndirectEdges(C: CB->getContext()));
7700 if (CandidateProfileData.empty())
7701 return 0;
7702
7703 // Iterate through all of the candidate profiled targets along with the
7704 // CallsiteInfo summary records synthesized for them, and see if any are
7705 // cloned and/or refer to clones.
7706 bool ICPNeeded = false;
7707 unsigned NumClones = 0;
7708 SmallVector<const CallsiteInfo *> Callsites;
7709 for (const auto &Candidate : CandidateProfileData) {
7710 const CallsiteInfo *Callsite = GetCallsite(Candidate);
7711 Callsites.push_back(Elt: Callsite);
7712 if (!Callsite)
7713 continue;
7714 // See if any of the clones of the indirect callsite for this
7715 // profiled target should call a cloned version of the profiled
7716 // target. We only need to do the ICP here if so.
7717 ICPNeeded |= llvm::any_of(Range: Callsite->Clones,
7718 P: [](unsigned CloneNo) { return CloneNo != 0; });
7719 // Every callsite in the same function should have been cloned the same
7720 // number of times.
7721 assert(!NumClones || NumClones == Callsite->Clones.size());
7722 NumClones = Callsite->Clones.size();
7723 }
7724 if (!ICPNeeded)
7725 return NumClones;
7726 // Save information for ICP, which is performed later to avoid messing up the
7727 // current function traversal.
7728 ICallAnalysisInfo.push_back(Elt: {.CB: CB, .CandidateProfileData: CandidateProfileData.vec(), .NumCandidates: NumCandidates,
7729 .TotalCount: TotalCount, .NumClones: NumClones, .Callsites: std::move(Callsites)});
7730 return NumClones;
7731}
7732
7733void MemProfContextDisambiguation::performICP(
7734 Module &M, ArrayRef<std::unique_ptr<ValueToValueMapTy>> VMaps,
7735 ArrayRef<ICallAnalysisData> ICallAnalysisInfo,
7736 OptimizationRemarkEmitter &ORE) {
7737 // Now do any promotion required for cloning. Specifically, for each
7738 // recorded ICP candidate (which was only recorded because one clone of that
7739 // candidate should call a cloned target), we perform ICP (speculative
7740 // devirtualization) for each clone of the callsite, and update its callee
7741 // to the appropriate clone. Note that the ICP compares against the original
7742 // version of the target, which is what is in the vtable.
7743 for (auto &Info : ICallAnalysisInfo) {
7744 auto *CB = Info.CB;
7745 auto TotalCount = Info.TotalCount;
7746 unsigned NumClones = Info.NumClones;
7747 SmallVector<InstrProfValueData, 8> RemainingCandidates;
7748
7749 assert(Info.CandidateProfileData.size() == Info.Callsites.size());
7750 for (unsigned I = 0; I < Info.CandidateProfileData.size(); I++) {
7751 auto &Candidate = Info.CandidateProfileData[I];
7752 const CallsiteInfo *StackNode = Info.Callsites[I];
7753 // All calls in the same function must have the same number of clones.
7754 assert(!StackNode || NumClones == StackNode->Clones.size());
7755
7756 // See if the target is in the module. If it wasn't imported, it is
7757 // possible that this profile could have been collected on a different
7758 // target (or version of the code), and we need to be conservative
7759 // (similar to what is done in the ICP pass).
7760 Function *TargetFunction = Symtab->getFunction(FuncMD5Hash: Candidate.Value);
7761 // If not found, try the GUID of the callee recorded in the summary. For
7762 // local targets the VP metadata may contain the GUID of the original
7763 // name without the source file prefix (e.g. if it was synthesized from
7764 // the MemProf profile's callee GUIDs or from a SamplePGO profile), which
7765 // the thin link translates to the actual GUID of the local function
7766 // (see updateValueInfoForIndirectCalls). The latter is recorded in the
7767 // function's !guid metadata, which is preserved across promotion and
7768 // importing, and is added to the Symtab. If there is no summary record,
7769 // the target was not cloned, so we conservatively only promote it if
7770 // found directly via the VP metadata GUID.
7771 if (!TargetFunction && StackNode && StackNode->Callee &&
7772 StackNode->Callee.getGUID() != Candidate.Value)
7773 TargetFunction = Symtab->getFunction(FuncMD5Hash: StackNode->Callee.getGUID());
7774 // The target may be the aliasee of an alias imported as a copy of the
7775 // aliasee, without the aliasee itself being imported (e.g. an IRPGO value
7776 // profile records the aliasee for calls through the alias, as the alias
7777 // has no profile data of its own). The Symtab keys such a copy by the
7778 // alias GUID (see initializeIndirectCallPromotionInfo), so look for one
7779 // with the target's GUID as its aliasee GUID. Promoting to the copy is
7780 // correct as it has the same address, and its clones are aliases of the
7781 // aliasee's clones in the defining module.
7782 if (!TargetFunction) {
7783 if (!AliaseeGUIDToImportedAlias) {
7784 AliaseeGUIDToImportedAlias.emplace();
7785 for (Function &Fn : M) {
7786 if (isMemProfClone(GV: Fn) || !getImportedAliasGUID(F: Fn))
7787 continue;
7788 if (auto AliaseeGUID = Fn.getGUIDIfAssigned())
7789 AliaseeGUIDToImportedAlias->try_emplace(Key: *AliaseeGUID, Args: &Fn);
7790 }
7791 }
7792 TargetFunction = AliaseeGUIDToImportedAlias->lookup(Val: Candidate.Value);
7793 if (!TargetFunction && StackNode && StackNode->Callee &&
7794 StackNode->Callee.getGUID() != Candidate.Value)
7795 TargetFunction =
7796 AliaseeGUIDToImportedAlias->lookup(Val: StackNode->Callee.getGUID());
7797 }
7798 if (TargetFunction == nullptr ||
7799 // Any ThinLTO global dead symbol removal should have already
7800 // occurred, so it should be safe to promote when the target is a
7801 // declaration.
7802 // TODO: Remove internal option once more fully tested.
7803 (MemProfRequireDefinitionForPromotion &&
7804 TargetFunction->isDeclaration())) {
7805 ORE.emit(RemarkBuilder: [&]() {
7806 return OptimizationRemarkMissed(DEBUG_TYPE, "UnableToFindTarget", CB)
7807 << "Memprof cannot promote indirect call: target with md5sum "
7808 << ore::NV("target md5sum", Candidate.Value) << " not found";
7809 });
7810 // FIXME: See if we can use the new declaration importing support to
7811 // at least get the declarations imported for this case. Hot indirect
7812 // targets should have been imported normally, however.
7813 RemainingCandidates.push_back(Elt: Candidate);
7814 continue;
7815 }
7816
7817 // Check if legal to promote
7818 const char *Reason = nullptr;
7819 if (!isLegalToPromote(CB: *CB, Callee: TargetFunction, FailureReason: &Reason)) {
7820 ORE.emit(RemarkBuilder: [&]() {
7821 return OptimizationRemarkMissed(DEBUG_TYPE, "UnableToPromote", CB)
7822 << "Memprof cannot promote indirect call to "
7823 << ore::NV("TargetFunction", TargetFunction)
7824 << " with count of " << ore::NV("TotalCount", TotalCount)
7825 << ": " << Reason;
7826 });
7827 RemainingCandidates.push_back(Elt: Candidate);
7828 continue;
7829 }
7830
7831 assert(!isMemProfClone(*TargetFunction));
7832
7833 // Handle each call clone, applying ICP so that each clone directly
7834 // calls the specified callee clone, guarded by the appropriate ICP
7835 // check.
7836 CallBase *CBClone = CB;
7837 for (unsigned J = 0; J < NumClones; J++) {
7838 // If the VMap is empty, this clone was a duplicate of another and was
7839 // created as an alias or a declaration.
7840 if (J > 0 && VMaps[J - 1]->empty())
7841 continue;
7842 // Copy 0 is the original function.
7843 if (J > 0)
7844 CBClone = cast<CallBase>(Val&: (*VMaps[J - 1])[CB]);
7845 // We do the promotion using the original name, so that the comparison
7846 // is against the name in the vtable. Then just below, change the new
7847 // direct call to call the cloned function.
7848 auto &DirectCall =
7849 pgo::promoteIndirectCall(CB&: *CBClone, F: TargetFunction, Count: Candidate.Count,
7850 TotalCount, AttachProfToDirectCall: isSamplePGO, ORE: &ORE);
7851 auto *TargetToUse = TargetFunction;
7852 // Call original if this version calls the original version of its
7853 // callee (always the case if there is no summary record).
7854 if (StackNode && StackNode->Clones[J]) {
7855 TargetToUse =
7856 cast<Function>(Val: M.getOrInsertFunction(
7857 Name: getMemProfFuncName(Base: TargetFunction->getName(),
7858 CloneNo: StackNode->Clones[J]),
7859 T: TargetFunction->getFunctionType())
7860 .getCallee());
7861 }
7862 DirectCall.setCalledFunction(TargetToUse);
7863 // During matching we generate synthetic VP metadata for indirect calls
7864 // not already having any, from the memprof profile's callee GUIDs. If
7865 // we subsequently promote and inline those callees, we currently lose
7866 // the ability to generate this synthetic VP metadata. Optionally apply
7867 // a noinline attribute to promoted direct calls, where the threshold is
7868 // set to capture synthetic VP metadata targets which get a count of 1.
7869 if (MemProfICPNoInlineThreshold &&
7870 Candidate.Count < MemProfICPNoInlineThreshold)
7871 DirectCall.setIsNoInline();
7872 ORE.emit(OptDiag: OptimizationRemark(DEBUG_TYPE, "MemprofCall", CBClone)
7873 << ore::NV("Call", CBClone) << " in clone "
7874 << ore::NV("Caller", CBClone->getFunction())
7875 << " promoted and assigned to call function clone "
7876 << ore::NV("Callee", TargetToUse));
7877 }
7878
7879 // Update TotalCount (all clones should get same count above)
7880 TotalCount -= Candidate.Count;
7881 }
7882 // Adjust the MD.prof metadata for all clones, now that we have the new
7883 // TotalCount and the remaining candidates.
7884 CallBase *CBClone = CB;
7885 for (unsigned J = 0; J < NumClones; J++) {
7886 // If the VMap is empty, this clone was a duplicate of another and was
7887 // created as an alias or a declaration.
7888 if (J > 0 && VMaps[J - 1]->empty())
7889 continue;
7890 // Copy 0 is the original function.
7891 if (J > 0)
7892 CBClone = cast<CallBase>(Val&: (*VMaps[J - 1])[CB]);
7893 // First delete the old one.
7894 CBClone->setMetadata(KindID: LLVMContext::MD_prof, Node: nullptr);
7895 // If all promoted, we don't need the MD.prof metadata.
7896 // Otherwise we need update with the un-promoted records back.
7897 if (TotalCount != 0)
7898 annotateValueSite(M, Inst&: *CBClone, VDs: RemainingCandidates, Sum: TotalCount,
7899 ValueKind: IPVK_IndirectCallTarget, MaxMDCount: Info.NumCandidates);
7900 }
7901 }
7902}
7903
7904template <typename DerivedCCG, typename FuncTy, typename CallTy>
7905bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::process(
7906 function_ref<void(StringRef, StringRef, const Twine &)> EmitRemark,
7907 bool AllowExtraAnalysis) {
7908 if (DumpCCG) {
7909 dbgs() << "CCG before cloning:\n";
7910 dbgs() << *this;
7911 }
7912 if (ExportToDot)
7913 exportToDot(Label: "postbuild");
7914
7915 if (VerifyCCG) {
7916 check();
7917 }
7918
7919 // The detail suffix needs the pre-cloning state of each context; only
7920 // capture it (extra memory and time) when it will be printed.
7921 if (MemProfReportHintedSizes && MemProfReportHintedSizesDetail)
7922 capturePreCloningContextInfo();
7923
7924 identifyClones();
7925
7926 if (VerifyCCG) {
7927 check();
7928 }
7929
7930 if (DumpCCG) {
7931 dbgs() << "CCG after cloning:\n";
7932 dbgs() << *this;
7933 }
7934 if (ExportToDot)
7935 exportToDot(Label: "cloned");
7936
7937 bool Changed = assignFunctions();
7938
7939 if (DumpCCG) {
7940 dbgs() << "CCG after assigning function clones:\n";
7941 dbgs() << *this;
7942 }
7943 if (ExportToDot)
7944 exportToDot(Label: "clonefuncassign");
7945
7946 // AllowExtraAnalysis is set when the hinted size messages were enabled via
7947 // remarks (-pass-remarks=memprof-context-disambiguation).
7948 // TODO: Consider also supporting the detail suffix for the remark output.
7949 if (MemProfReportHintedSizes || AllowExtraAnalysis) {
7950 // The referenced-function set can be large; build it only now, when it is
7951 // used by the detail suffix, rather than keeping it across cloning.
7952 if (MemProfReportHintedSizes && MemProfReportHintedSizesDetail)
7953 populateFunctionsWithCallers();
7954 printTotalSizes(OS&: errs(), EmitRemark);
7955 }
7956
7957 return Changed;
7958}
7959
7960bool MemProfContextDisambiguation::processModule(
7961 Module &M,
7962 llvm::function_ref<OptimizationRemarkEmitter &(Function *)> OREGetter) {
7963
7964 // If we have an import summary, then the cloning decisions were made during
7965 // the thin link on the index. Apply them and return.
7966 if (ImportSummary)
7967 return applyImport(M);
7968
7969 // TODO: If/when other types of memprof cloning are enabled beyond just for
7970 // hot and cold, we will need to change this to individually control the
7971 // AllocationType passed to addStackNodesForMIB during CCG construction.
7972 // Note that we specifically check this after applying imports above, so that
7973 // the option isn't needed to be passed to distributed ThinLTO backend
7974 // clang processes, which won't necessarily have visibility into the linker
7975 // dependences. Instead the information is communicated from the LTO link to
7976 // the backends via the combined summary index.
7977 if (!SupportsHotColdNew)
7978 return false;
7979
7980 ModuleCallsiteContextGraph CCG(M, OREGetter);
7981 // TODO: Set up remarks for regular LTO. We need to decide what function to
7982 // use in the callback.
7983 return CCG.process();
7984}
7985
7986MemProfContextDisambiguation::MemProfContextDisambiguation(
7987 const ModuleSummaryIndex *Summary, bool isSamplePGO)
7988 : ImportSummary(Summary), isSamplePGO(isSamplePGO) {
7989 // Check the dot graph printing options once here, to make sure we have valid
7990 // and expected combinations.
7991 if (DotGraphScope == DotScope::Alloc && !AllocIdForDot.getNumOccurrences())
7992 llvm::report_fatal_error(
7993 reason: "-memprof-dot-scope=alloc requires -memprof-dot-alloc-id");
7994 if (DotGraphScope == DotScope::Context &&
7995 !ContextIdForDot.getNumOccurrences())
7996 llvm::report_fatal_error(
7997 reason: "-memprof-dot-scope=context requires -memprof-dot-context-id");
7998 if (DotGraphScope == DotScope::All && AllocIdForDot.getNumOccurrences() &&
7999 ContextIdForDot.getNumOccurrences())
8000 llvm::report_fatal_error(
8001 reason: "-memprof-dot-scope=all can't have both -memprof-dot-alloc-id and "
8002 "-memprof-dot-context-id");
8003 if (ImportSummary) {
8004 // The MemProfImportSummary should only be used for testing ThinLTO
8005 // distributed backend handling via opt, in which case we don't have a
8006 // summary from the pass pipeline.
8007 assert(MemProfImportSummary.empty());
8008 return;
8009 }
8010 if (MemProfImportSummary.empty())
8011 return;
8012
8013 auto ReadSummaryFile =
8014 errorOrToExpected(EO: MemoryBuffer::getFile(Filename: MemProfImportSummary));
8015 if (!ReadSummaryFile) {
8016 logAllUnhandledErrors(E: ReadSummaryFile.takeError(), OS&: errs(),
8017 ErrorBanner: "Error loading file '" + MemProfImportSummary +
8018 "': ");
8019 return;
8020 }
8021 auto ImportSummaryForTestingOrErr = getModuleSummaryIndex(Buffer: **ReadSummaryFile);
8022 if (!ImportSummaryForTestingOrErr) {
8023 logAllUnhandledErrors(E: ImportSummaryForTestingOrErr.takeError(), OS&: errs(),
8024 ErrorBanner: "Error parsing file '" + MemProfImportSummary +
8025 "': ");
8026 return;
8027 }
8028 ImportSummaryForTesting = std::move(*ImportSummaryForTestingOrErr);
8029 ImportSummary = ImportSummaryForTesting.get();
8030}
8031
8032PreservedAnalyses MemProfContextDisambiguation::run(Module &M,
8033 ModuleAnalysisManager &AM) {
8034 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
8035 auto OREGetter = [&](Function *F) -> OptimizationRemarkEmitter & {
8036 return FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: *F);
8037 };
8038 if (!processModule(M, OREGetter))
8039 return PreservedAnalyses::all();
8040 return PreservedAnalyses::none();
8041}
8042
8043// After the thin link cloning decisions have been recorded in the summary
8044// callsite records, mark the summary indirect call edges to any callee that a
8045// callsite was assigned to call a (non-original) clone of as requiring a
8046// declaration import.
8047// See the comments for the MemProfImportClonedIndirectTargets option.
8048static void updateClonedCalleeEdgesForImport(
8049 ModuleSummaryIndex &Index,
8050 llvm::function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
8051 isPrevailing) {
8052 // Map from callee to the corresponding call edge in the current function
8053 // summary, so that we don't need to walk the function's call edges for each
8054 // callsite assigned to call a cloned callee. Note that there is a single call
8055 // edge per callee. The map is built lazily, the first time we encounter such
8056 // a callsite in a given function summary, since most functions have none
8057 // (the map is cleared before processing each function summary). The
8058 // pointers into the call edge vector remain valid since it isn't modified
8059 // while the map is in use.
8060 SmallDenseMap<ValueInfo, CalleeInfo *> CalleeToEdge;
8061 for (auto &I : Index) {
8062 auto VI = Index.getValueInfo(R: I);
8063 for (auto &S : VI.getSummaryList()) {
8064 // Only the prevailing copies were added to the CCG (see the
8065 // IndexCallsiteContextGraph constructor).
8066 if (!GlobalValue::isLocalLinkage(Linkage: S->linkage()) &&
8067 !isPrevailing(VI.getGUID(), S.get()))
8068 continue;
8069 auto *FS = dyn_cast<FunctionSummary>(Val: S.get());
8070 if (!FS || FS->callsites().empty())
8071 continue;
8072 CalleeToEdge.clear();
8073 for (auto &CI : FS->callsites()) {
8074 if (!CI.Callee || llvm::all_of(Range: CI.Clones, P: [](unsigned CloneNo) {
8075 return CloneNo == 0;
8076 }))
8077 continue;
8078 // Note that we mark the edge even if the callee is defined in the
8079 // same module as the caller, since the caller may be imported into
8080 // other modules. The function importer will skip importing the callee
8081 // into any module that already defines it.
8082 NumClonedCalleeCallsites++;
8083 // Lazily build the map on the first such callsite in this function.
8084 if (CalleeToEdge.empty())
8085 for (auto &[CalleeVI, Edge] : FS->mutableCalls())
8086 CalleeToEdge[CalleeVI] = &Edge;
8087 auto It = CalleeToEdge.find(Val: CI.Callee);
8088 if (It == CalleeToEdge.end())
8089 continue;
8090 CalleeInfo *Edge = It->second;
8091 // Direct calls don't require importing the callee. Note that the
8092 // edge is marked indirect if any call to the callee in this function
8093 // is indirect, in which case we can't distinguish whether this
8094 // callsite is direct and conservatively treat it as indirect.
8095 if (!Edge->hasIndirectCall())
8096 continue;
8097 NumClonedCalleeIndirectCallsites++;
8098 if (!Edge->requiresDeclImport()) {
8099 Edge->setRequiresDeclImport(true);
8100 NumClonedCalleeEdgesMarkedRequiresDeclImport++;
8101 }
8102 }
8103 }
8104 }
8105}
8106
8107void MemProfContextDisambiguation::run(
8108 ModuleSummaryIndex &Index,
8109 llvm::function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
8110 isPrevailing,
8111 LLVMContext &Ctx,
8112 function_ref<void(StringRef, StringRef, const Twine &)> EmitRemark) {
8113 // TODO: If/when other types of memprof cloning are enabled beyond just for
8114 // hot and cold, we will need to change this to individually control the
8115 // AllocationType passed to addStackNodesForMIB during CCG construction.
8116 // The index was set from the option, so these should be in sync.
8117 assert(Index.withSupportsHotColdNew() == SupportsHotColdNew);
8118 if (!SupportsHotColdNew)
8119 return;
8120
8121 bool AllowExtraAnalysis =
8122 OptimizationRemarkEmitter::allowExtraAnalysis(Ctx, DEBUG_TYPE);
8123
8124 bool Changed = false;
8125 {
8126 IndexCallsiteContextGraph CCG(Index, isPrevailing);
8127 Changed = CCG.process(EmitRemark, AllowExtraAnalysis);
8128 }
8129 // Note this must be done after the CCG is destroyed, which is when the
8130 // callsite records synthesized for missing tail call frames are added to the
8131 // function summaries. Importing declarations is only useful if MemProf ICP
8132 // is allowed to promote to a declaration of the target.
8133 if (Changed && MemProfImportClonedIndirectTargets &&
8134 !MemProfRequireDefinitionForPromotion)
8135 updateClonedCalleeEdgesForImport(Index, isPrevailing);
8136}
8137
8138// Strips MemProf attributes and metadata. Can be invoked by the pass pipeline
8139// when we don't have an index that has recorded that we are linking with
8140// allocation libraries containing the necessary APIs for downstream
8141// transformations.
8142PreservedAnalyses MemProfRemoveInfo::run(Module &M, ModuleAnalysisManager &AM) {
8143 // The profile matcher applies hotness attributes directly for allocations,
8144 // and those will cause us to generate calls to the hot/cold interfaces
8145 // unconditionally. If supports-hot-cold-new was not enabled in the LTO
8146 // link then assume we don't want these calls (e.g. not linking with
8147 // the appropriate library, or otherwise trying to disable this behavior).
8148 bool Changed = false;
8149 for (auto &F : M) {
8150 for (auto &BB : F) {
8151 for (auto &I : BB) {
8152 auto *CI = dyn_cast<CallBase>(Val: &I);
8153 if (!CI)
8154 continue;
8155 if (CI->hasFnAttr(Kind: "memprof")) {
8156 CI->removeFnAttr(Kind: "memprof");
8157 Changed = true;
8158 }
8159 if (!CI->hasMetadata(KindID: LLVMContext::MD_callsite)) {
8160 assert(!CI->hasMetadata(LLVMContext::MD_memprof));
8161 continue;
8162 }
8163 // Strip off all memprof metadata as it is no longer needed.
8164 // Importantly, this avoids the addition of new memprof attributes
8165 // after inlining propagation.
8166 CI->setMetadata(KindID: LLVMContext::MD_memprof, Node: nullptr);
8167 CI->setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
8168 Changed = true;
8169 }
8170 }
8171 }
8172 if (!Changed)
8173 return PreservedAnalyses::all();
8174 return PreservedAnalyses::none();
8175}
8176