MemProfContextDisambiguation.cpp source code [llvm_projects/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp]

1	//==-- MemProfContextDisambiguation.cpp - Disambiguate contexts -------------=//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file implements support for context disambiguation of allocation
10	// calls for profile guided heap optimization. Specifically, it uses Memprof
11	// profiles which indicate context specific allocation behavior (currently
12	// distinguishing cold vs hot memory allocations). Cloning is performed to
13	// expose the cold allocation call contexts, and the allocation calls are
14	// subsequently annotated with an attribute for later transformation.
15	//
16	// The transformations can be performed either directly on IR (regular LTO), or
17	// on a ThinLTO index (and later applied to the IR during the ThinLTO backend).
18	// Both types of LTO operate on a the same base graph representation, which
19	// uses CRTP to support either IR or Index formats.
20	//
21	//===----------------------------------------------------------------------===//
22
23	#include "llvm/Transforms/IPO/MemProfContextDisambiguation.h"
24	#include "llvm/ADT/DenseMap.h"
25	#include "llvm/ADT/DenseSet.h"
26	#include "llvm/ADT/MapVector.h"
27	#include "llvm/ADT/SetOperations.h"
28	#include "llvm/ADT/SmallPtrSet.h"
29	#include "llvm/ADT/SmallSet.h"
30	#include "llvm/ADT/SmallVector.h"
31	#include "llvm/ADT/Statistic.h"
32	#include "llvm/ADT/StringExtras.h"
33	#include "llvm/Analysis/MemoryProfileInfo.h"
34	#include "llvm/Analysis/ModuleSummaryAnalysis.h"
35	#include "llvm/Analysis/OptimizationRemarkEmitter.h"
36	#include "llvm/Bitcode/BitcodeReader.h"
37	#include "llvm/IR/Instructions.h"
38	#include "llvm/IR/Module.h"
39	#include "llvm/IR/ModuleSummaryIndex.h"
40	#include "llvm/Pass.h"
41	#include "llvm/Support/CommandLine.h"
42	#include "llvm/Support/GraphWriter.h"
43	#include "llvm/Support/SHA1.h"
44	#include "llvm/Support/raw_ostream.h"
45	#include "llvm/Transforms/IPO.h"
46	#include "llvm/Transforms/Utils/CallPromotionUtils.h"
47	#include "llvm/Transforms/Utils/Cloning.h"
48	#include "llvm/Transforms/Utils/Instrumentation.h"
49	#include <deque>
50	#include <sstream>
51	#include <vector>
52	using namespace llvm;
53	using namespace llvm::memprof;
54
55	#define DEBUG_TYPE "memprof-context-disambiguation"
56
57	STATISTIC(FunctionClonesAnalysis,
58	"Number of function clones created during whole program analysis");
59	STATISTIC(FunctionClonesThinBackend,
60	"Number of function clones created during ThinLTO backend");
61	STATISTIC(FunctionsClonedThinBackend,
62	"Number of functions that had clones created during ThinLTO backend");
63	STATISTIC(
64	FunctionCloneDuplicatesThinBackend,
65	"Number of function clone duplicates detected during ThinLTO backend");
66	STATISTIC(AllocTypeNotCold, "Number of not cold static allocations (possibly "
67	"cloned) during whole program analysis");
68	STATISTIC(AllocTypeCold, "Number of cold static allocations (possibly cloned) "
69	"during whole program analysis");
70	STATISTIC(AllocTypeNotColdThinBackend,
71	"Number of not cold static allocations (possibly cloned) during "
72	"ThinLTO backend");
73	STATISTIC(AllocTypeColdThinBackend, "Number of cold static allocations "
74	"(possibly cloned) during ThinLTO backend");
75	STATISTIC(OrigAllocsThinBackend,
76	"Number of original (not cloned) allocations with memprof profiles "
77	"during ThinLTO backend");
78	STATISTIC(
79	AllocVersionsThinBackend,
80	"Number of allocation versions (including clones) during ThinLTO backend");
81	STATISTIC(MaxAllocVersionsThinBackend,
82	"Maximum number of allocation versions created for an original "
83	"allocation during ThinLTO backend");
84	STATISTIC(UnclonableAllocsThinBackend,
85	"Number of unclonable ambigous allocations during ThinLTO backend");
86	STATISTIC(RemovedEdgesWithMismatchedCallees,
87	"Number of edges removed due to mismatched callees (profiled vs IR)");
88	STATISTIC(FoundProfiledCalleeCount,
89	"Number of profiled callees found via tail calls");
90	STATISTIC(FoundProfiledCalleeDepth,
91	"Aggregate depth of profiled callees found via tail calls");
92	STATISTIC(FoundProfiledCalleeMaxDepth,
93	"Maximum depth of profiled callees found via tail calls");
94	STATISTIC(FoundProfiledCalleeNonUniquelyCount,
95	"Number of profiled callees found via multiple tail call chains");
96	STATISTIC(DeferredBackedges, "Number of backedges with deferred cloning");
97	STATISTIC(NewMergedNodes, "Number of new nodes created during merging");
98	STATISTIC(NonNewMergedNodes, "Number of non new nodes used during merging");
99	STATISTIC(MissingAllocForContextId,
100	"Number of missing alloc nodes for context ids");
101	STATISTIC(SkippedCallsCloning,
102	"Number of calls skipped during cloning due to unexpected operand");
103	STATISTIC(MismatchedCloneAssignments,
104	"Number of callsites assigned to call multiple non-matching clones");
105	STATISTIC(TotalMergeInvokes, "Number of merge invocations for nodes");
106	STATISTIC(TotalMergeIters, "Number of merge iterations for nodes");
107	STATISTIC(MaxMergeIters, "Max merge iterations for nodes");
108	STATISTIC(NumImportantContextIds, "Number of important context ids");
109	STATISTIC(NumFixupEdgeIdsInserted, "Number of fixup edge ids inserted");
110	STATISTIC(NumFixupEdgesAdded, "Number of fixup edges added");
111	STATISTIC(NumFixedContexts, "Number of contexts with fixed edges");
112	STATISTIC(AliaseesPrevailingInDiffModuleFromAlias,
113	"Number of aliasees prevailing in a different module than its alias");
114
115	static cl::opt<std::string> DotFilePathPrefix(
116	"memprof-dot-file-path-prefix", cl::init(Val: ""), cl::Hidden,
117	cl::value_desc("filename"),
118	cl::desc("Specify the path prefix of the MemProf dot files."));
119
120	static cl::opt<bool> ExportToDot("memprof-export-to-dot", cl::init(Val: false),
121	cl::Hidden,
122	cl::desc("Export graph to dot files."));
123
124	// TODO: Remove this option once new handling is validated more widely.
125	static cl::opt<bool> DoMergeIteration(
126	"memprof-merge-iteration", cl::init(Val: true), cl::Hidden,
127	cl::desc("Iteratively apply merging on a node to catch new callers"));
128
129	// How much of the graph to export to dot.
130	enum DotScope {
131	All, // The full CCG graph.
132	Alloc, // Only contexts for the specified allocation.
133	Context, // Only the specified context.
134	};
135
136	static cl::opt<DotScope> DotGraphScope(
137	"memprof-dot-scope", cl::desc("Scope of graph to export to dot"),
138	cl::Hidden, cl::init(Val: DotScope::All),
139	cl::values(
140	clEnumValN(DotScope::All, "all", "Export full callsite graph"),
141	clEnumValN(DotScope::Alloc, "alloc",
142	"Export only nodes with contexts feeding given "
143	"-memprof-dot-alloc-id"),
144	clEnumValN(DotScope::Context, "context",
145	"Export only nodes with given -memprof-dot-context-id")));
146
147	static cl::opt<unsigned>
148	AllocIdForDot("memprof-dot-alloc-id", cl::init(Val: `0`), cl::Hidden,
149	cl::desc("Id of alloc to export if -memprof-dot-scope=alloc "
150	"or to highlight if -memprof-dot-scope=all"));
151
152	static cl::opt<unsigned> ContextIdForDot(
153	"memprof-dot-context-id", cl::init(Val: `0`), cl::Hidden,
154	cl::desc("Id of context to export if -memprof-dot-scope=context or to "
155	"highlight otherwise"));
156
157	static cl::opt<bool>
158	DumpCCG("memprof-dump-ccg", cl::init(Val: false), cl::Hidden,
159	cl::desc("Dump CallingContextGraph to stdout after each stage."));
160
161	static cl::opt<bool>
162	VerifyCCG("memprof-verify-ccg", cl::init(Val: false), cl::Hidden,
163	cl::desc("Perform verification checks on CallingContextGraph."));
164
165	static cl::opt<bool>
166	VerifyNodes("memprof-verify-nodes", cl::init(Val: false), cl::Hidden,
167	cl::desc("Perform frequent verification checks on nodes."));
168
169	static cl::opt<std::string> MemProfImportSummary(
170	"memprof-import-summary",
171	cl::desc("Import summary to use for testing the ThinLTO backend via opt"),
172	cl::Hidden);
173
174	static cl::opt<unsigned>
175	TailCallSearchDepth("memprof-tail-call-search-depth", cl::init(Val: `5`),
176	cl::Hidden,
177	cl::desc("Max depth to recursively search for missing "
178	"frames through tail calls."));
179
180	// Optionally enable cloning of callsites involved with recursive cycles
181	static cl::opt<bool> AllowRecursiveCallsites(
182	"memprof-allow-recursive-callsites", cl::init(Val: true), cl::Hidden,
183	cl::desc("Allow cloning of callsites involved in recursive cycles"));
184
185	static cl::opt<bool> CloneRecursiveContexts(
186	"memprof-clone-recursive-contexts", cl::init(Val: true), cl::Hidden,
187	cl::desc("Allow cloning of contexts through recursive cycles"));
188
189	// Generally this is needed for correct assignment of allocation clones to
190	// function clones, however, allow it to be disabled for debugging while the
191	// functionality is new and being tested more widely.
192	static cl::opt<bool>
193	MergeClones("memprof-merge-clones", cl::init(Val: true), cl::Hidden,
194	cl::desc("Merge clones before assigning functions"));
195
196	// When disabled, try to detect and prevent cloning of recursive contexts.
197	// This is only necessary until we support cloning through recursive cycles.
198	// Leave on by default for now, as disabling requires a little bit of compile
199	// time overhead and doesn't affect correctness, it will just inflate the cold
200	// hinted bytes reporting a bit when -memprof-report-hinted-sizes is enabled.
201	static cl::opt<bool> AllowRecursiveContexts(
202	"memprof-allow-recursive-contexts", cl::init(Val: true), cl::Hidden,
203	cl::desc("Allow cloning of contexts having recursive cycles"));
204
205	// Set the minimum absolute count threshold for allowing inlining of indirect
206	// calls promoted during cloning.
207	static cl::opt<unsigned> MemProfICPNoInlineThreshold(
208	"memprof-icp-noinline-threshold", cl::init(Val: `0`), cl::Hidden,
209	cl::desc("Minimum absolute count for promoted target to be inlinable"));
210
211	namespace llvm {
212	cl::opt<bool> EnableMemProfContextDisambiguation(
213	"enable-memprof-context-disambiguation", cl::Hidden,
214	cl::desc("Enable MemProf context disambiguation"));
215
216	// Indicate we are linking with an allocator that supports hot/cold operator
217	// new interfaces.
218	cl::opt<bool> SupportsHotColdNew(
219	"supports-hot-cold-new", cl::init(Val: false), cl::Hidden,
220	cl::desc("Linking with hot/cold operator new interfaces"));
221
222	static cl::opt<bool> MemProfRequireDefinitionForPromotion(
223	"memprof-require-definition-for-promotion", cl::init(Val: false), cl::Hidden,
224	cl::desc(
225	"Require target function definition when promoting indirect calls"));
226
227	extern cl::opt<bool> MemProfReportHintedSizes;
228	extern cl::opt<unsigned> MinClonedColdBytePercent;
229
230	cl::opt<unsigned> MemProfTopNImportant(
231	"memprof-top-n-important", cl::init(Val: `10`), cl::Hidden,
232	cl::desc("Number of largest cold contexts to consider important"));
233
234	cl::opt<bool> MemProfFixupImportant(
235	"memprof-fixup-important", cl::init(Val: true), cl::Hidden,
236	cl::desc("Enables edge fixup for important contexts"));
237
238	extern cl::opt<unsigned> MaxSummaryIndirectEdges;
239
240	} // namespace llvm
241
242	namespace {
243
244	/// CRTP base for graphs built from either IR or ThinLTO summary index.
245	///
246	/// The graph represents the call contexts in all memprof metadata on allocation
247	/// calls, with nodes for the allocations themselves, as well as for the calls
248	/// in each context. The graph is initially built from the allocation memprof
249	/// metadata (or summary) MIBs. It is then updated to match calls with callsite
250	/// metadata onto the nodes, updating it to reflect any inlining performed on
251	/// those calls.
252	///
253	/// Each MIB (representing an allocation's call context with allocation
254	/// behavior) is assigned a unique context id during the graph build. The edges
255	/// and nodes in the graph are decorated with the context ids they carry. This
256	/// is used to correctly update the graph when cloning is performed so that we
257	/// can uniquify the context for a single (possibly cloned) allocation.
258	template <typename DerivedCCG, typename FuncTy, typename CallTy>
259	class CallsiteContextGraph {
260	public:
261	CallsiteContextGraph() = default;
262	CallsiteContextGraph(const CallsiteContextGraph &) = default;
263	CallsiteContextGraph(CallsiteContextGraph &&) = default;
264
265	/// Main entry point to perform analysis and transformations on graph.
266	bool process(function_ref<void(StringRef, StringRef, const Twine &)>
267	EmitRemark = nullptr,
268	bool AllowExtraAnalysis = false);
269
270	/// Perform cloning on the graph necessary to uniquely identify the allocation
271	/// behavior of an allocation based on its context.
272	void identifyClones();
273
274	/// Assign callsite clones to functions, cloning functions as needed to
275	/// accommodate the combinations of their callsite clones reached by callers.
276	/// For regular LTO this clones functions and callsites in the IR, but for
277	/// ThinLTO the cloning decisions are noted in the summaries and later applied
278	/// in applyImport.
279	bool assignFunctions();
280
281	void dump() const;
282	void print(raw_ostream &OS) const;
283	void printTotalSizes(raw_ostream &OS,
284	function_ref<void(StringRef, StringRef, const Twine &)>
285	EmitRemark = nullptr) const;
286
287	friend raw_ostream &operator<<(raw_ostream &OS,
288	const CallsiteContextGraph &CCG) {
289	CCG.print(OS);
290	return OS;
291	}
292
293	friend struct GraphTraits<
294	const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>;
295	friend struct DOTGraphTraits<
296	const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>;
297
298	void exportToDot(std::string Label) const;
299
300	/// Represents a function clone via FuncTy pointer and clone number pair.
301	struct FuncInfo final
302	: public std::pair<FuncTy , unsigned* /Clone number/> {
303	using Base = std::pair<FuncTy , unsigned*>;
304	FuncInfo(const Base &B) : Base(B) {}
305	FuncInfo(FuncTy F = nullptr, unsigned* CloneNo = `0`) : Base(F, CloneNo) {}
306	explicit operator bool() const { return this->first != nullptr; }
307	FuncTy func() const* { return this->first; }
308	unsigned cloneNo() const { return this->second; }
309	};
310
311	/// Represents a callsite clone via CallTy and clone number pair.
312	struct CallInfo final : public std::pair<CallTy, unsigned /Clone number/> {
313	using Base = std::pair<CallTy, unsigned>;
314	CallInfo(const Base &B) : Base(B) {}
315	CallInfo(CallTy Call = nullptr, unsigned CloneNo = `0`)
316	: Base(Call, CloneNo) {}
317	explicit operator bool() const { return (bool)this->first; }
318	CallTy call() const { return this->first; }
319	unsigned cloneNo() const { return this->second; }
320	void setCloneNo(unsigned N) { this->second = N; }
321	void print(raw_ostream &OS) const {
322	if (!operator bool()) {
323	assert(!cloneNo());
324	OS << "null Call";
325	return;
326	}
327	call()->print(OS);
328	OS << "\t(clone " << cloneNo() << ")";
329	}
330	void dump() const {
331	print(OS&: dbgs());
332	dbgs() << "\n";
333	}
334	friend raw_ostream &operator<<(raw_ostream &OS, const CallInfo &Call) {
335	Call.print(OS);
336	return OS;
337	}
338	};
339
340	struct ContextEdge;
341
342	/// Node in the Callsite Context Graph
343	struct ContextNode {
344	// Assigned to nodes as they are created, useful for debugging.
345	unsigned NodeId = `0`;
346
347	// Keep this for now since in the IR case where we have an Instruction it*
348	// is not as immediately discoverable. Used for printing richer information
349	// when dumping graph.
350	bool IsAllocation;
351
352	// Keeps track of when the Call was reset to null because there was
353	// recursion.
354	bool Recursive = false;
355
356	// This will be formed by ORing together the AllocationType enum values
357	// for contexts including this node.
358	uint8_t AllocTypes = `0`;
359
360	// The corresponding allocation or interior call. This is the primary call
361	// for which we have created this node.
362	CallInfo Call;
363
364	// List of other calls that can be treated the same as the primary call
365	// through cloning. I.e. located in the same function and have the same
366	// (possibly pruned) stack ids. They will be updated the same way as the
367	// primary call when assigning to function clones.
368	SmallVector<CallInfo, `0`> MatchingCalls;
369
370	// For alloc nodes this is a unique id assigned when constructed, and for
371	// callsite stack nodes it is the original stack id when the node is
372	// constructed from the memprof MIB metadata on the alloc nodes. Note that
373	// this is only used when matching callsite metadata onto the stack nodes
374	// created when processing the allocation memprof MIBs, and for labeling
375	// nodes in the dot graph. Therefore we don't bother to assign a value for
376	// clones.
377	uint64_t OrigStackOrAllocId = `0`;
378
379	// Edges to all callees in the profiled call stacks.
380	// TODO: Should this be a map (from Callee node) for more efficient lookup?
381	std::vector<std::shared_ptr<ContextEdge>> CalleeEdges;
382
383	// Edges to all callers in the profiled call stacks.
384	// TODO: Should this be a map (from Caller node) for more efficient lookup?
385	std::vector<std::shared_ptr<ContextEdge>> CallerEdges;
386
387	// Returns true if we need to look at the callee edges for determining the
388	// node context ids and allocation type.
389	bool useCallerEdgesForContextInfo() const {
390	// Typically if the callee edges are empty either the caller edges are
391	// also empty, or this is an allocation (leaf node). However, if we are
392	// allowing recursive callsites and contexts this will be violated for
393	// incompletely cloned recursive cycles.
394	assert(!CalleeEdges.empty() \|\| CallerEdges.empty() \|\| IsAllocation \|\|
395	(AllowRecursiveCallsites && AllowRecursiveContexts));
396	// When cloning for a recursive context, during cloning we might be in the
397	// midst of cloning for a recurrence and have moved context ids off of a
398	// caller edge onto the clone but not yet off of the incoming caller
399	// (back) edge. If we don't look at those we miss the fact that this node
400	// still has context ids of interest.
401	return IsAllocation \|\| CloneRecursiveContexts;
402	}
403
404	// Compute the context ids for this node from the union of its edge context
405	// ids.
406	DenseSet<uint32_t> getContextIds() const {
407	unsigned Count = `0`;
408	// Compute the number of ids for reserve below. In general we only need to
409	// look at one set of edges, typically the callee edges, since other than
410	// allocations and in some cases during recursion cloning, all the context
411	// ids on the callers should also flow out via callee edges.
412	for (auto &Edge : CalleeEdges.empty() ? CallerEdges : CalleeEdges)
413	Count += Edge->getContextIds().size();
414	DenseSet<uint32_t> ContextIds;
415	ContextIds.reserve(Size: Count);
416	auto Edges = llvm::concat<const std::shared_ptr<ContextEdge>>(
417	CalleeEdges, useCallerEdgesForContextInfo()
418	? CallerEdges
419	: std::vector<std::shared_ptr<ContextEdge>>());
420	for (const auto &Edge : Edges)
421	ContextIds.insert_range(Edge->getContextIds());
422	return ContextIds;
423	}
424
425	// Compute the allocation type for this node from the OR of its edge
426	// allocation types.
427	uint8_t computeAllocType() const {
428	uint8_t BothTypes =
429	(uint8_t)AllocationType::Cold \| (uint8_t)AllocationType::NotCold;
430	uint8_t AllocType = (uint8_t)AllocationType::None;
431	auto Edges = llvm::concat<const std::shared_ptr<ContextEdge>>(
432	CalleeEdges, useCallerEdgesForContextInfo()
433	? CallerEdges
434	: std::vector<std::shared_ptr<ContextEdge>>());
435	for (const auto &Edge : Edges) {
436	AllocType \|= Edge->AllocTypes;
437	// Bail early if alloc type reached both, no further refinement.
438	if (AllocType == BothTypes)
439	return AllocType;
440	}
441	return AllocType;
442	}
443
444	// The context ids set for this node is empty if its edge context ids are
445	// also all empty.
446	bool emptyContextIds() const {
447	auto Edges = llvm::concat<const std::shared_ptr<ContextEdge>>(
448	CalleeEdges, useCallerEdgesForContextInfo()
449	? CallerEdges
450	: std::vector<std::shared_ptr<ContextEdge>>());
451	for (const auto &Edge : Edges) {
452	if (!Edge->getContextIds().empty())
453	return false;
454	}
455	return true;
456	}
457
458	// List of clones of this ContextNode, initially empty.
459	std::vector<ContextNode *> Clones;
460
461	// If a clone, points to the original uncloned node.
462	ContextNode CloneOf = nullptr*;
463
464	ContextNode(bool IsAllocation) : IsAllocation(IsAllocation), Call() {}
465
466	ContextNode(bool IsAllocation, CallInfo C)
467	: IsAllocation(IsAllocation), Call(C) {}
468
469	void addClone(ContextNode *Clone) {
470	if (CloneOf) {
471	CloneOf->Clones.push_back(Clone);
472	Clone->CloneOf = CloneOf;
473	} else {
474	Clones.push_back(Clone);
475	assert(!Clone->CloneOf);
476	Clone->CloneOf = this;
477	}
478	}
479
480	ContextNode *getOrigNode() {
481	if (!CloneOf)
482	return this;
483	return CloneOf;
484	}
485
486	void addOrUpdateCallerEdge(ContextNode *Caller, AllocationType AllocType,
487	unsigned int ContextId);
488
489	ContextEdge findEdgeFromCallee(const* ContextNode *Callee);
490	ContextEdge findEdgeFromCaller(const* ContextNode *Caller);
491	void eraseCalleeEdge(const ContextEdge *Edge);
492	void eraseCallerEdge(const ContextEdge *Edge);
493
494	void setCall(CallInfo C) { Call = std::move(C); }
495
496	bool hasCall() const { return (bool)Call.call(); }
497
498	void printCall(raw_ostream &OS) const { Call.print(OS); }
499
500	// True if this node was effectively removed from the graph, in which case
501	// it should have an allocation type of None and empty context ids.
502	bool isRemoved() const {
503	// Typically if the callee edges are empty either the caller edges are
504	// also empty, or this is an allocation (leaf node). However, if we are
505	// allowing recursive callsites and contexts this will be violated for
506	// incompletely cloned recursive cycles.
507	assert((AllowRecursiveCallsites && AllowRecursiveContexts) \|\|
508	(AllocTypes == (uint8_t)AllocationType::None) ==
509	emptyContextIds());
510	return AllocTypes == (uint8_t)AllocationType::None;
511	}
512
513	void dump() const;
514	void print(raw_ostream &OS) const;
515
516	friend raw_ostream &operator<<(raw_ostream &OS, const ContextNode &Node) {
517	Node.print(OS);
518	return OS;
519	}
520	};
521
522	/// Edge in the Callsite Context Graph from a ContextNode N to a caller or
523	/// callee.
524	struct ContextEdge {
525	ContextNode *Callee;
526	ContextNode *Caller;
527
528	// This will be formed by ORing together the AllocationType enum values
529	// for contexts including this edge.
530	uint8_t AllocTypes = `0`;
531
532	// Set just before initiating cloning when cloning of recursive contexts is
533	// enabled. Used to defer cloning of backedges until we have done cloning of
534	// the callee node for non-backedge caller edges. This exposes cloning
535	// opportunities through the backedge of the cycle.
536	// TODO: Note that this is not updated during cloning, and it is unclear
537	// whether that would be needed.
538	bool IsBackedge = false;
539
540	// The set of IDs for contexts including this edge.
541	DenseSet<uint32_t> ContextIds;
542
543	ContextEdge(ContextNode Callee, ContextNode Caller, uint8_t AllocType,
544	DenseSet<uint32_t> ContextIds)
545	: Callee(Callee), Caller(Caller), AllocTypes(AllocType),
546	ContextIds (std::move(ContextIds)) {}
547
548	DenseSet<uint32_t> &getContextIds() { return ContextIds; }
549
550	// Helper to clear the fields of this edge when we are removing it from the
551	// graph.
552	inline void clear() {
553	ContextIds.clear();
554	AllocTypes = (uint8_t)AllocationType::None;
555	Caller = nullptr;
556	Callee = nullptr;
557	}
558
559	// Check if edge was removed from the graph. This is useful while iterating
560	// over a copy of edge lists when performing operations that mutate the
561	// graph in ways that might remove one of the edges.
562	inline bool isRemoved() const {
563	if (Callee \|\| Caller)
564	return false;
565	// Any edges that have been removed from the graph but are still in a
566	// shared_ptr somewhere should have all fields null'ed out by clear()
567	// above.
568	assert(AllocTypes == (uint8_t)AllocationType::None);
569	assert(ContextIds.empty());
570	return true;
571	}
572
573	void dump() const;
574	void print(raw_ostream &OS) const;
575
576	friend raw_ostream &operator<<(raw_ostream &OS, const ContextEdge &Edge) {
577	Edge.print(OS);
578	return OS;
579	}
580	};
581
582	/// Helpers to remove edges that have allocation type None (due to not
583	/// carrying any context ids) after transformations.
584	void removeNoneTypeCalleeEdges(ContextNode *Node);
585	void removeNoneTypeCallerEdges(ContextNode *Node);
586	void
587	recursivelyRemoveNoneTypeCalleeEdges(ContextNode *Node,
588	DenseSet<const ContextNode *> &Visited);
589
590	protected:
591	/// Get a list of nodes corresponding to the stack ids in the given callsite
592	/// context.
593	template <class NodeT, class IteratorT>
594	std::vector<uint64_t>
595	getStackIdsWithContextNodes(CallStack<NodeT, IteratorT> &CallsiteContext);
596
597	/// Adds nodes for the given allocation and any stack ids on its memprof MIB
598	/// metadata (or summary).
599	ContextNode addAllocNode(CallInfo Call, const* FuncTy *F);
600
601	/// Adds nodes for the given MIB stack ids.
602	template <class NodeT, class IteratorT>
603	void addStackNodesForMIB(
604	ContextNode *AllocNode, CallStack<NodeT, IteratorT> &StackContext,
605	CallStack<NodeT, IteratorT> &CallsiteContext, AllocationType AllocType,
606	ArrayRef<ContextTotalSize> ContextSizeInfo,
607	std::map<uint64_t, uint32_t> &TotalSizeToContextIdTopNCold);
608
609	/// Matches all callsite metadata (or summary) to the nodes created for
610	/// allocation memprof MIB metadata, synthesizing new nodes to reflect any
611	/// inlining performed on those callsite instructions.
612	void updateStackNodes();
613
614	/// Optionally fixup edges for the N largest cold contexts to better enable
615	/// cloning. This is particularly helpful if the context includes recursion
616	/// as well as inlining, resulting in a single stack node for multiple stack
617	/// ids in the context. With recursion it is particularly difficult to get the
618	/// edge updates correct as in the general case we have lost the original
619	/// stack id ordering for the context. Do more expensive fixup for the largest
620	/// contexts, controlled by MemProfTopNImportant and MemProfFixupImportant.
621	void fixupImportantContexts();
622
623	/// Update graph to conservatively handle any callsite stack nodes that target
624	/// multiple different callee target functions.
625	void handleCallsitesWithMultipleTargets();
626
627	/// Mark backedges via the standard DFS based backedge algorithm.
628	void markBackedges();
629
630	/// Merge clones generated during cloning for different allocations but that
631	/// are called by the same caller node, to ensure proper function assignment.
632	void mergeClones();
633
634	// Try to partition calls on the given node (already placed into the AllCalls
635	// array) by callee function, creating new copies of Node as needed to hold
636	// calls with different callees, and moving the callee edges appropriately.
637	// Returns true if partitioning was successful.
638	bool partitionCallsByCallee(
639	ContextNode *Node, ArrayRef<CallInfo> AllCalls,
640	std::vector<std::pair<CallInfo, ContextNode *>> &NewCallToNode);
641
642	/// Save lists of calls with MemProf metadata in each function, for faster
643	/// iteration.
644	MapVector<FuncTy *, std::vector<CallInfo>> FuncToCallsWithMetadata;
645
646	/// Map from callsite node to the enclosing caller function.
647	std::map<const ContextNode , const* FuncTy *> NodeToCallingFunc;
648
649	// When exporting to dot, and an allocation id is specified, contains the
650	// context ids on that allocation.
651	DenseSet<uint32_t> DotAllocContextIds;
652
653	private:
654	using EdgeIter = typename std::vector<std::shared_ptr<ContextEdge>>::iterator;
655
656	// Structure to keep track of information for each call as we are matching
657	// non-allocation callsites onto context nodes created from the allocation
658	// call metadata / summary contexts.
659	struct CallContextInfo {
660	// The callsite we're trying to match.
661	CallTy Call;
662	// The callsites stack ids that have a context node in the graph.
663	std::vector<uint64_t> StackIds;
664	// The function containing this callsite.
665	const FuncTy *Func;
666	// Initially empty, if needed this will be updated to contain the context
667	// ids for use in a new context node created for this callsite.
668	DenseSet<uint32_t> ContextIds;
669	};
670
671	/// Helper to remove edge from graph, updating edge iterator if it is provided
672	/// (in which case CalleeIter indicates which edge list is being iterated).
673	/// This will also perform the necessary clearing of the ContextEdge members
674	/// to enable later checking if the edge has been removed (since we may have
675	/// other copies of the shared_ptr in existence, and in fact rely on this to
676	/// enable removal while iterating over a copy of a node's edge list).
677	void removeEdgeFromGraph(ContextEdge Edge, EdgeIter EI = nullptr,
678	bool CalleeIter = true);
679
680	/// Assigns the given Node to calls at or inlined into the location with
681	/// the Node's stack id, after post order traversing and processing its
682	/// caller nodes. Uses the call information recorded in the given
683	/// StackIdToMatchingCalls map, and creates new nodes for inlined sequences
684	/// as needed. Called by updateStackNodes which sets up the given
685	/// StackIdToMatchingCalls map.
686	void assignStackNodesPostOrder(
687	ContextNode Node, DenseSet<const* ContextNode *> &Visited,
688	DenseMap<uint64_t, std::vector<CallContextInfo>> &StackIdToMatchingCalls,
689	DenseMap<CallInfo, CallInfo> &CallToMatchingCall,
690	const DenseSet<uint32_t> &ImportantContextIds);
691
692	/// Duplicates the given set of context ids, updating the provided
693	/// map from each original id with the newly generated context ids,
694	/// and returning the new duplicated id set.
695	DenseSet<uint32_t> duplicateContextIds(
696	const DenseSet<uint32_t> &StackSequenceContextIds,
697	DenseMap<uint32_t, DenseSet<uint32_t>> &OldToNewContextIds);
698
699	/// Propagates all duplicated context ids across the graph.
700	void propagateDuplicateContextIds(
701	const DenseMap<uint32_t, DenseSet<uint32_t>> &OldToNewContextIds);
702
703	/// Connect the NewNode to OrigNode's callees if TowardsCallee is true,
704	/// else to its callers. Also updates OrigNode's edges to remove any context
705	/// ids moved to the newly created edge.
706	void connectNewNode(ContextNode NewNode, ContextNode OrigNode,
707	bool TowardsCallee,
708	DenseSet<uint32_t> RemainingContextIds);
709
710	/// Get the stack id corresponding to the given Id or Index (for IR this will
711	/// return itself, for a summary index this will return the id recorded in the
712	/// index for that stack id index value).
713	uint64_t getStackId(uint64_t IdOrIndex) const {
714	return static_cast<const DerivedCCG >(this*)->getStackId(IdOrIndex);
715	}
716
717	/// Returns true if the given call targets the callee of the given edge, or if
718	/// we were able to identify the call chain through intermediate tail calls.
719	/// In the latter case new context nodes are added to the graph for the
720	/// identified tail calls, and their synthesized nodes are added to
721	/// TailCallToContextNodeMap. The EdgeIter is updated in the latter case for
722	/// the updated edges and to prepare it for an increment in the caller.
723	bool
724	calleesMatch(CallTy Call, EdgeIter &EI,
725	MapVector<CallInfo, ContextNode *> &TailCallToContextNodeMap);
726
727	// Return the callee function of the given call, or nullptr if it can't be
728	// determined
729	const FuncTy *getCalleeFunc(CallTy Call) {
730	return static_cast<DerivedCCG >(this*)->getCalleeFunc(Call);
731	}
732
733	/// Returns true if the given call targets the given function, or if we were
734	/// able to identify the call chain through intermediate tail calls (in which
735	/// case FoundCalleeChain will be populated).
736	bool calleeMatchesFunc(
737	CallTy Call, const FuncTy Func, const* FuncTy *CallerFunc,
738	std::vector<std::pair<CallTy, FuncTy *>> &FoundCalleeChain) {
739	return static_cast<DerivedCCG >(this*)->calleeMatchesFunc(
740	Call, Func, CallerFunc, FoundCalleeChain);
741	}
742
743	/// Returns true if both call instructions have the same callee.
744	bool sameCallee(CallTy Call1, CallTy Call2) {
745	return static_cast<DerivedCCG >(this*)->sameCallee(Call1, Call2);
746	}
747
748	/// Get a list of nodes corresponding to the stack ids in the given
749	/// callsite's context.
750	std::vector<uint64_t> getStackIdsWithContextNodesForCall(CallTy Call) {
751	return static_cast<DerivedCCG >(this*)->getStackIdsWithContextNodesForCall(
752	Call);
753	}
754
755	/// Get the last stack id in the context for callsite.
756	uint64_t getLastStackId(CallTy Call) {
757	return static_cast<DerivedCCG >(this*)->getLastStackId(Call);
758	}
759
760	/// Update the allocation call to record type of allocated memory.
761	void updateAllocationCall(CallInfo &Call, AllocationType AllocType) {
762	AllocType == AllocationType::Cold ? AllocTypeCold ++ : AllocTypeNotCold ++;
763	static_cast<DerivedCCG >(this*)->updateAllocationCall(Call, AllocType);
764	}
765
766	/// Get the AllocationType assigned to the given allocation instruction clone.
767	AllocationType getAllocationCallType(const CallInfo &Call) const {
768	return static_cast<const DerivedCCG >(this*)->getAllocationCallType(Call);
769	}
770
771	/// Update non-allocation call to invoke (possibly cloned) function
772	/// CalleeFunc.
773	void updateCall(CallInfo &CallerCall, FuncInfo CalleeFunc) {
774	static_cast<DerivedCCG >(this*)->updateCall(CallerCall, CalleeFunc);
775	}
776
777	/// Clone the given function for the given callsite, recording mapping of all
778	/// of the functions tracked calls to their new versions in the CallMap.
779	/// Assigns new clones to clone number CloneNo.
780	FuncInfo cloneFunctionForCallsite(
781	FuncInfo &Func, CallInfo &Call, DenseMap<CallInfo, CallInfo> &CallMap,
782	std::vector<CallInfo> &CallsWithMetadataInFunc, unsigned CloneNo) {
783	return static_cast<DerivedCCG >(this*)->cloneFunctionForCallsite(
784	Func, Call, CallMap, CallsWithMetadataInFunc, CloneNo);
785	}
786
787	/// Gets a label to use in the dot graph for the given call clone in the given
788	/// function.
789	std::string getLabel(const FuncTy Func, const* CallTy Call,
790	unsigned CloneNo) const {
791	return static_cast<const DerivedCCG >(this*)->getLabel(Func, Call, CloneNo);
792	}
793
794	// Create and return a new ContextNode.
795	ContextNode createNewNode(bool* IsAllocation, const FuncTy F = nullptr*,
796	CallInfo C = CallInfo()) {
797	NodeOwner.push_back(std::make_unique<ContextNode>(IsAllocation, C));
798	auto *NewNode = NodeOwner.back().get();
799	if (F)
800	NodeToCallingFunc[NewNode] = F;
801	NewNode->NodeId = NodeOwner.size();
802	return NewNode;
803	}
804
805	/// Helpers to find the node corresponding to the given call or stackid.
806	ContextNode getNodeForInst(const* CallInfo &C);
807	ContextNode getNodeForAlloc(const* CallInfo &C);
808	ContextNode *getNodeForStackId(uint64_t StackId);
809
810	/// Computes the alloc type corresponding to the given context ids, by
811	/// unioning their recorded alloc types.
812	uint8_t computeAllocType(DenseSet<uint32_t> &ContextIds) const;
813
814	/// Returns the allocation type of the intersection of the contexts of two
815	/// nodes (based on their provided context id sets), optimized for the case
816	/// when Node1Ids is smaller than Node2Ids.
817	uint8_t intersectAllocTypesImpl(const DenseSet<uint32_t> &Node1Ids,
818	const DenseSet<uint32_t> &Node2Ids) const;
819
820	/// Returns the allocation type of the intersection of the contexts of two
821	/// nodes (based on their provided context id sets).
822	uint8_t intersectAllocTypes(const DenseSet<uint32_t> &Node1Ids,
823	const DenseSet<uint32_t> &Node2Ids) const;
824
825	/// Create a clone of Edge's callee and move Edge to that new callee node,
826	/// performing the necessary context id and allocation type updates.
827	/// If ContextIdsToMove is non-empty, only that subset of Edge's ids are
828	/// moved to an edge to the new callee.
829	ContextNode *
830	moveEdgeToNewCalleeClone(const std::shared_ptr<ContextEdge> &Edge,
831	DenseSet<uint32_t> ContextIdsToMove = {});
832
833	/// Change the callee of Edge to existing callee clone NewCallee, performing
834	/// the necessary context id and allocation type updates.
835	/// If ContextIdsToMove is non-empty, only that subset of Edge's ids are
836	/// moved to an edge to the new callee.
837	void moveEdgeToExistingCalleeClone(const std::shared_ptr<ContextEdge> &Edge,
838	ContextNode *NewCallee,
839	bool NewClone = false,
840	DenseSet<uint32_t> ContextIdsToMove = {});
841
842	/// Change the caller of the edge at the given callee edge iterator to be
843	/// NewCaller, performing the necessary context id and allocation type
844	/// updates. This is similar to the above moveEdgeToExistingCalleeClone, but
845	/// a simplified version of it as we always move the given edge and all of its
846	/// context ids.
847	void moveCalleeEdgeToNewCaller(const std::shared_ptr<ContextEdge> &Edge,
848	ContextNode *NewCaller);
849
850	/// Recursive helper for marking backedges via DFS.
851	void markBackedges(ContextNode Node, DenseSet<const* ContextNode *> &Visited,
852	DenseSet<const ContextNode *> &CurrentStack);
853
854	/// Recursive helper for merging clones.
855	void
856	mergeClones(ContextNode Node, DenseSet<const* ContextNode *> &Visited,
857	DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode);
858	/// Main worker for merging callee clones for a given node.
859	void mergeNodeCalleeClones(
860	ContextNode Node, DenseSet<const* ContextNode *> &Visited,
861	DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode);
862	/// Helper to find other callers of the given set of callee edges that can
863	/// share the same callee merge node.
864	void findOtherCallersToShareMerge(
865	ContextNode *Node, std::vector<std::shared_ptr<ContextEdge>> &CalleeEdges,
866	DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode,
867	DenseSet<ContextNode *> &OtherCallersToShareMerge);
868
869	/// Recursively perform cloning on the graph for the given Node and its
870	/// callers, in order to uniquely identify the allocation behavior of an
871	/// allocation given its context. The context ids of the allocation being
872	/// processed are given in AllocContextIds.
873	void identifyClones(ContextNode Node, DenseSet<const* ContextNode *> &Visited,
874	const DenseSet<uint32_t> &AllocContextIds);
875
876	/// Map from each context ID to the AllocationType assigned to that context.
877	DenseMap<uint32_t, AllocationType> ContextIdToAllocationType;
878
879	/// Map from each contextID to the profiled full contexts and their total
880	/// sizes (there may be more than one due to context trimming),
881	/// optionally populated when requested (via MemProfReportHintedSizes or
882	/// MinClonedColdBytePercent).
883	DenseMap<uint32_t, std::vector<ContextTotalSize>> ContextIdToContextSizeInfos;
884
885	/// Identifies the context node created for a stack id when adding the MIB
886	/// contexts to the graph. This is used to locate the context nodes when
887	/// trying to assign the corresponding callsites with those stack ids to these
888	/// nodes.
889	DenseMap<uint64_t, ContextNode *> StackEntryIdToContextNodeMap;
890
891	/// Saves information for the contexts identified as important (the largest
892	/// cold contexts up to MemProfTopNImportant).
893	struct ImportantContextInfo {
894	// The original list of leaf first stack ids corresponding to this context.
895	std::vector<uint64_t> StackIds;
896	// Max length of stack ids corresponding to a single stack ContextNode for
897	// this context (i.e. the max length of a key in StackIdsToNode below).
898	unsigned MaxLength = `0`;
899	// Mapping of slices of the stack ids to the corresponding ContextNode
900	// (there can be multiple stack ids due to inlining). Populated when
901	// updating stack nodes while matching them to the IR or summary.
902	std::map<std::vector<uint64_t>, ContextNode *> StackIdsToNode;
903	};
904
905	// Map of important full context ids to information about each.
906	DenseMap<uint32_t, ImportantContextInfo> ImportantContextIdInfo;
907
908	// For each important context id found in Node (if any), records the list of
909	// stack ids that corresponded to the given callsite Node. There can be more
910	// than one in the case of inlining.
911	void recordStackNode(std::vector<uint64_t> &StackIds, ContextNode *Node,
912	// We pass in the Node's context ids to avoid the
913	// overhead of computing them as the caller already has
914	// them in some cases.
915	const DenseSet<uint32_t> &NodeContextIds,
916	const DenseSet<uint32_t> &ImportantContextIds) {
917	if (!MemProfTopNImportant) {
918	assert(ImportantContextIds.empty());
919	return;
920	}
921	DenseSet<uint32_t> Ids =
922	set_intersection(S1: NodeContextIds, S2: ImportantContextIds);
923	if (Ids.empty())
924	return;
925	auto Size = StackIds.size();
926	for (auto Id : Ids) {
927	auto &Entry = ImportantContextIdInfo[Id];
928	Entry.StackIdsToNode[StackIds] = Node;
929	// Keep track of the max to simplify later analysis.
930	if (Size > Entry.MaxLength)
931	Entry.MaxLength = Size;
932	}
933	}
934
935	/// Maps to track the calls to their corresponding nodes in the graph.
936	MapVector<CallInfo, ContextNode *> AllocationCallToContextNodeMap;
937	MapVector<CallInfo, ContextNode *> NonAllocationCallToContextNodeMap;
938
939	/// Owner of all ContextNode unique_ptrs.
940	std::vector<std::unique_ptr<ContextNode>> NodeOwner;
941
942	/// Perform sanity checks on graph when requested.
943	void check() const;
944
945	/// Keeps track of the last unique context id assigned.
946	unsigned int LastContextId = `0`;
947	};
948
949	template <typename DerivedCCG, typename FuncTy, typename CallTy>
950	using ContextNode =
951	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode;
952	template <typename DerivedCCG, typename FuncTy, typename CallTy>
953	using ContextEdge =
954	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge;
955	template <typename DerivedCCG, typename FuncTy, typename CallTy>
956	using FuncInfo =
957	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::FuncInfo;
958	template <typename DerivedCCG, typename FuncTy, typename CallTy>
959	using CallInfo =
960	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::CallInfo;
961
962	/// CRTP derived class for graphs built from IR (regular LTO).
963	class ModuleCallsiteContextGraph
964	: public CallsiteContextGraph<ModuleCallsiteContextGraph, Function,
965	Instruction *> {
966	public:
967	ModuleCallsiteContextGraph(
968	Module &M,
969	llvm::function_ref<OptimizationRemarkEmitter &(Function *)> OREGetter);
970
971	private:
972	friend CallsiteContextGraph<ModuleCallsiteContextGraph, Function,
973	Instruction *>;
974
975	uint64_t getStackId(uint64_t IdOrIndex) const;
976	const Function getCalleeFunc(Instruction Call);
977	bool calleeMatchesFunc(
978	Instruction Call, const* Function Func, const* Function *CallerFunc,
979	std::vector<std::pair<Instruction , Function >> &FoundCalleeChain);
980	bool sameCallee(Instruction Call1, Instruction Call2);
981	bool findProfiledCalleeThroughTailCalls(
982	const Function ProfiledCallee, Value CurCallee, unsigned Depth,
983	std::vector<std::pair<Instruction , Function >> &FoundCalleeChain,
984	bool &FoundMultipleCalleeChains);
985	uint64_t getLastStackId(Instruction *Call);
986	std::vector<uint64_t> getStackIdsWithContextNodesForCall(Instruction *Call);
987	void updateAllocationCall(CallInfo &Call, AllocationType AllocType);
988	AllocationType getAllocationCallType(const CallInfo &Call) const;
989	void updateCall(CallInfo &CallerCall, FuncInfo CalleeFunc);
990	CallsiteContextGraph<ModuleCallsiteContextGraph, Function,
991	Instruction *>::FuncInfo
992	cloneFunctionForCallsite(FuncInfo &Func, CallInfo &Call,
993	DenseMap<CallInfo, CallInfo> &CallMap,
994	std::vector<CallInfo> &CallsWithMetadataInFunc,
995	unsigned CloneNo);
996	std::string getLabel(const Function Func, const* Instruction *Call,
997	unsigned CloneNo) const;
998
999	const Module &Mod;
1000	llvm::function_ref<OptimizationRemarkEmitter &(Function *)> OREGetter;
1001	};
1002
1003	/// Represents a call in the summary index graph, which can either be an
1004	/// allocation or an interior callsite node in an allocation's context.
1005	/// Holds a pointer to the corresponding data structure in the index.
1006	struct IndexCall : public PointerUnion<CallsiteInfo , AllocInfo > {
1007	IndexCall() : PointerUnion () {}
1008	IndexCall(std::nullptr_t) : IndexCall () {}
1009	IndexCall(CallsiteInfo *StackNode) : PointerUnion (StackNode) {}
1010	IndexCall(AllocInfo *AllocNode) : PointerUnion (AllocNode) {}
1011	IndexCall(PointerUnion PT) : PointerUnion (PT) {}
1012
1013	IndexCall *operator->() { return this; }
1014
1015	void print(raw_ostream &OS) const {
1016	PointerUnion<CallsiteInfo , AllocInfo > Base = *this;
1017	if (auto AI = llvm::dyn_cast_if_present<AllocInfo >(Val&: Base)) {
1018	OS << *AI;
1019	} else {
1020	auto CI = llvm::dyn_cast_if_present<CallsiteInfo >(Val&: Base);
1021	assert(CI);
1022	OS << *CI;
1023	}
1024	}
1025	};
1026	} // namespace
1027
1028	namespace llvm {
1029	template <> struct simplify_type<IndexCall> {
1030	using SimpleType = PointerUnion<CallsiteInfo , AllocInfo >;
1031	static SimpleType getSimplifiedValue(IndexCall &Val) { return Val; }
1032	};
1033	template <> struct simplify_type<const IndexCall> {
1034	using SimpleType = const PointerUnion<CallsiteInfo , AllocInfo >;
1035	static SimpleType getSimplifiedValue(const IndexCall &Val) { return Val; }
1036	};
1037	} // namespace llvm
1038
1039	namespace {
1040	/// CRTP derived class for graphs built from summary index (ThinLTO).
1041	class IndexCallsiteContextGraph
1042	: public CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
1043	IndexCall> {
1044	public:
1045	IndexCallsiteContextGraph(
1046	ModuleSummaryIndex &Index,
1047	llvm::function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1048	isPrevailing);
1049
1050	~IndexCallsiteContextGraph() {
1051	// Now that we are done with the graph it is safe to add the new
1052	// CallsiteInfo structs to the function summary vectors. The graph nodes
1053	// point into locations within these vectors, so we don't want to add them
1054	// any earlier.
1055	for (auto &I : FunctionCalleesToSynthesizedCallsiteInfos) {
1056	auto *FS = I.first;
1057	for (auto &Callsite : I.second)
1058	FS->addCallsite(Callsite: std::move(*Callsite.second));
1059	}
1060	}
1061
1062	private:
1063	friend CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
1064	IndexCall>;
1065
1066	uint64_t getStackId(uint64_t IdOrIndex) const;
1067	const FunctionSummary *getCalleeFunc(IndexCall &Call);
1068	bool calleeMatchesFunc(
1069	IndexCall &Call, const FunctionSummary *Func,
1070	const FunctionSummary *CallerFunc,
1071	std::vector<std::pair<IndexCall, FunctionSummary *>> &FoundCalleeChain);
1072	bool sameCallee(IndexCall &Call1, IndexCall &Call2);
1073	bool findProfiledCalleeThroughTailCalls(
1074	ValueInfo ProfiledCallee, ValueInfo CurCallee, unsigned Depth,
1075	std::vector<std::pair<IndexCall, FunctionSummary *>> &FoundCalleeChain,
1076	bool &FoundMultipleCalleeChains);
1077	uint64_t getLastStackId(IndexCall &Call);
1078	std::vector<uint64_t> getStackIdsWithContextNodesForCall(IndexCall &Call);
1079	void updateAllocationCall(CallInfo &Call, AllocationType AllocType);
1080	AllocationType getAllocationCallType(const CallInfo &Call) const;
1081	void updateCall(CallInfo &CallerCall, FuncInfo CalleeFunc);
1082	CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
1083	IndexCall>::FuncInfo
1084	cloneFunctionForCallsite(FuncInfo &Func, CallInfo &Call,
1085	DenseMap<CallInfo, CallInfo> &CallMap,
1086	std::vector<CallInfo> &CallsWithMetadataInFunc,
1087	unsigned CloneNo);
1088	std::string getLabel(const FunctionSummary Func, const* IndexCall &Call,
1089	unsigned CloneNo) const;
1090	DenseSet<GlobalValue::GUID> findAliaseeGUIDsPrevailingInDifferentModule();
1091
1092	// Saves mapping from function summaries containing memprof records back to
1093	// its VI, for use in checking and debugging.
1094	std::map<const FunctionSummary *, ValueInfo> FSToVIMap;
1095
1096	const ModuleSummaryIndex &Index;
1097	llvm::function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1098	isPrevailing;
1099
1100	// Saves/owns the callsite info structures synthesized for missing tail call
1101	// frames that we discover while building the graph.
1102	// It maps from the summary of the function making the tail call, to a map
1103	// of callee ValueInfo to corresponding synthesized callsite info.
1104	DenseMap<FunctionSummary *,
1105	std::map<ValueInfo, std::unique_ptr<CallsiteInfo>>>
1106	FunctionCalleesToSynthesizedCallsiteInfos;
1107	};
1108	} // namespace
1109
1110	template <>
1111	struct llvm::DenseMapInfo<CallsiteContextGraph<
1112	ModuleCallsiteContextGraph, Function, Instruction *>::CallInfo>
1113	: public DenseMapInfo<std::pair<Instruction , unsigned*>> {};
1114	template <>
1115	struct llvm::DenseMapInfo<CallsiteContextGraph<
1116	IndexCallsiteContextGraph, FunctionSummary, IndexCall>::CallInfo>
1117	: public DenseMapInfo<std::pair<IndexCall, unsigned>> {};
1118	template <>
1119	struct llvm::DenseMapInfo<IndexCall>
1120	: public DenseMapInfo<PointerUnion<CallsiteInfo , AllocInfo >> {};
1121
1122	namespace {
1123
1124	// Map the uint8_t alloc types (which may contain NotCold\|Cold) to the alloc
1125	// type we should actually use on the corresponding allocation.
1126	// If we can't clone a node that has NotCold+Cold alloc type, we will fall
1127	// back to using NotCold. So don't bother cloning to distinguish NotCold+Cold
1128	// from NotCold.
1129	AllocationType allocTypeToUse(uint8_t AllocTypes) {
1130	assert(AllocTypes != (uint8_t)AllocationType::None);
1131	if (AllocTypes ==
1132	((uint8_t)AllocationType::NotCold \| (uint8_t)AllocationType::Cold))
1133	return AllocationType::NotCold;
1134	else
1135	return (AllocationType)AllocTypes;
1136	}
1137
1138	// Helper to check if the alloc types for all edges recorded in the
1139	// InAllocTypes vector match the alloc types for all edges in the Edges
1140	// vector.
1141	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1142	bool allocTypesMatch(
1143	const std::vector<uint8_t> &InAllocTypes,
1144	const std::vector<std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>>>
1145	&Edges) {
1146	// This should be called only when the InAllocTypes vector was computed for
1147	// this set of Edges. Make sure the sizes are the same.
1148	assert(InAllocTypes.size() == Edges.size());
1149	return std::equal(
1150	InAllocTypes.begin(), InAllocTypes.end(), Edges.begin(), Edges.end(),
1151	[](const uint8_t &l,
1152	const std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>> &r) {
1153	// Can share if one of the edges is None type - don't
1154	// care about the type along that edge as it doesn't
1155	// exist for those context ids.
1156	if (l == (uint8_t)AllocationType::None \|\|
1157	r->AllocTypes == (uint8_t)AllocationType::None)
1158	return true;
1159	return allocTypeToUse(AllocTypes: l) == allocTypeToUse(r->AllocTypes);
1160	});
1161	}
1162
1163	// Helper to check if the alloc types for all edges recorded in the
1164	// InAllocTypes vector match the alloc types for callee edges in the given
1165	// clone. Because the InAllocTypes were computed from the original node's callee
1166	// edges, and other cloning could have happened after this clone was created, we
1167	// need to find the matching clone callee edge, which may or may not exist.
1168	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1169	bool allocTypesMatchClone(
1170	const std::vector<uint8_t> &InAllocTypes,
1171	const ContextNode<DerivedCCG, FuncTy, CallTy> *Clone) {
1172	const ContextNode<DerivedCCG, FuncTy, CallTy> *Node = Clone->CloneOf;
1173	assert(Node);
1174	// InAllocTypes should have been computed for the original node's callee
1175	// edges.
1176	assert(InAllocTypes.size() == Node->CalleeEdges.size());
1177	// First create a map of the clone callee edge callees to the edge alloc type.
1178	DenseMap<const ContextNode<DerivedCCG, FuncTy, CallTy> *, uint8_t>
1179	EdgeCalleeMap;
1180	for (const auto &E : Clone->CalleeEdges) {
1181	assert(!EdgeCalleeMap.contains(E->Callee));
1182	EdgeCalleeMap[E->Callee] = E->AllocTypes;
1183	}
1184	// Next, walk the original node's callees, and look for the corresponding
1185	// clone edge to that callee.
1186	for (unsigned I = `0`; I < Node->CalleeEdges.size(); I++) {
1187	auto Iter = EdgeCalleeMap.find(Node->CalleeEdges[I]->Callee);
1188	// Not found is ok, we will simply add an edge if we use this clone.
1189	if (Iter == EdgeCalleeMap.end())
1190	continue;
1191	// Can share if one of the edges is None type - don't
1192	// care about the type along that edge as it doesn't
1193	// exist for those context ids.
1194	if (InAllocTypes [I] == (uint8_t)AllocationType::None \|\|
1195	Iter->second == (uint8_t)AllocationType::None)
1196	continue;
1197	if (allocTypeToUse(Iter->second) != allocTypeToUse(AllocTypes: InAllocTypes [I]))
1198	return false;
1199	}
1200	return true;
1201	}
1202
1203	} // end anonymous namespace
1204
1205	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1206	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
1207	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::getNodeForInst(
1208	const CallInfo &C) {
1209	ContextNode *Node = getNodeForAlloc(C);
1210	if (Node)
1211	return Node;
1212
1213	return NonAllocationCallToContextNodeMap.lookup(C);
1214	}
1215
1216	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1217	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
1218	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::getNodeForAlloc(
1219	const CallInfo &C) {
1220	return AllocationCallToContextNodeMap.lookup(C);
1221	}
1222
1223	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1224	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
1225	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::getNodeForStackId(
1226	uint64_t StackId) {
1227	auto StackEntryNode = StackEntryIdToContextNodeMap.find(StackId);
1228	if (StackEntryNode != StackEntryIdToContextNodeMap.end())
1229	return StackEntryNode->second;
1230	return nullptr;
1231	}
1232
1233	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1234	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1235	addOrUpdateCallerEdge(ContextNode *Caller, AllocationType AllocType,
1236	unsigned int ContextId) {
1237	for (auto &Edge : CallerEdges) {
1238	if (Edge->Caller == Caller) {
1239	Edge->AllocTypes \|= (uint8_t)AllocType;
1240	Edge->getContextIds().insert(ContextId);
1241	return;
1242	}
1243	}
1244	std::shared_ptr<ContextEdge> Edge = std::make_shared<ContextEdge>(
1245	this, Caller, (uint8_t)AllocType, DenseSet<uint32_t>({ContextId}));
1246	CallerEdges.push_back(Edge);
1247	Caller->CalleeEdges.push_back(Edge);
1248	}
1249
1250	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1251	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::removeEdgeFromGraph(
1252	ContextEdge Edge, EdgeIter EI, bool CalleeIter) {
1253	assert(!EI \|\| (*EI)->get() == Edge);
1254	assert(!Edge->isRemoved());
1255	// Save the Caller and Callee pointers so we can erase Edge from their edge
1256	// lists after clearing Edge below. We do the clearing first in case it is
1257	// destructed after removing from the edge lists (if those were the last
1258	// shared_ptr references to Edge).
1259	auto *Callee = Edge->Callee;
1260	auto *Caller = Edge->Caller;
1261
1262	// Make sure the edge fields are cleared out so we can properly detect
1263	// removed edges if Edge is not destructed because there is still a shared_ptr
1264	// reference.
1265	Edge->clear();
1266
1267	#ifndef NDEBUG
1268	auto CalleeCallerCount = Callee->CallerEdges.size();
1269	auto CallerCalleeCount = Caller->CalleeEdges.size();
1270	#endif
1271	if (!EI) {
1272	Callee->eraseCallerEdge(Edge);
1273	Caller->eraseCalleeEdge(Edge);
1274	} else if (CalleeIter) {
1275	Callee->eraseCallerEdge(Edge);
1276	EI = Caller->CalleeEdges.erase(EI);
1277	} else {
1278	Caller->eraseCalleeEdge(Edge);
1279	EI = Callee->CallerEdges.erase(EI);
1280	}
1281	assert(Callee->CallerEdges.size() < CalleeCallerCount);
1282	assert(Caller->CalleeEdges.size() < CallerCalleeCount);
1283	}
1284
1285	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1286	void CallsiteContextGraph<
1287	DerivedCCG, FuncTy, CallTy>::removeNoneTypeCalleeEdges(ContextNode *Node) {
1288	for (auto EI = Node->CalleeEdges.begin(); EI != Node->CalleeEdges.end();) {
1289	auto Edge = *EI;
1290	if (Edge->AllocTypes == (uint8_t)AllocationType::None) {
1291	assert(Edge->ContextIds.empty());
1292	removeEdgeFromGraph(Edge: Edge.get(), EI: &EI, /CalleeIter=/true);
1293	} else
1294	++EI;
1295	}
1296	}
1297
1298	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1299	void CallsiteContextGraph<
1300	DerivedCCG, FuncTy, CallTy>::removeNoneTypeCallerEdges(ContextNode *Node) {
1301	for (auto EI = Node->CallerEdges.begin(); EI != Node->CallerEdges.end();) {
1302	auto Edge = *EI;
1303	if (Edge->AllocTypes == (uint8_t)AllocationType::None) {
1304	assert(Edge->ContextIds.empty());
1305	Edge->Caller->eraseCalleeEdge(Edge.get());
1306	EI = Node->CallerEdges.erase(EI);
1307	} else
1308	++EI;
1309	}
1310	}
1311
1312	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1313	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge *
1314	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1315	findEdgeFromCallee(const ContextNode *Callee) {
1316	for (const auto &Edge : CalleeEdges)
1317	if (Edge->Callee == Callee)
1318	return Edge.get();
1319	return nullptr;
1320	}
1321
1322	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1323	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge *
1324	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1325	findEdgeFromCaller(const ContextNode *Caller) {
1326	for (const auto &Edge : CallerEdges)
1327	if (Edge->Caller == Caller)
1328	return Edge.get();
1329	return nullptr;
1330	}
1331
1332	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1333	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1334	eraseCalleeEdge(const ContextEdge *Edge) {
1335	auto EI = llvm::find_if(
1336	CalleeEdges, [Edge](const std::shared_ptr<ContextEdge> &CalleeEdge) {
1337	return CalleeEdge.get() == Edge;
1338	});
1339	assert(EI != CalleeEdges.end());
1340	CalleeEdges.erase(EI);
1341	}
1342
1343	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1344	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1345	eraseCallerEdge(const ContextEdge *Edge) {
1346	auto EI = llvm::find_if(
1347	CallerEdges, [Edge](const std::shared_ptr<ContextEdge> &CallerEdge) {
1348	return CallerEdge.get() == Edge;
1349	});
1350	assert(EI != CallerEdges.end());
1351	CallerEdges.erase(EI);
1352	}
1353
1354	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1355	uint8_t CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::computeAllocType(
1356	DenseSet<uint32_t> &ContextIds) const {
1357	uint8_t BothTypes =
1358	(uint8_t)AllocationType::Cold \| (uint8_t)AllocationType::NotCold;
1359	uint8_t AllocType = (uint8_t)AllocationType::None;
1360	for (auto Id : ContextIds) {
1361	AllocType \|= (uint8_t)ContextIdToAllocationType.at(Val: Id);
1362	// Bail early if alloc type reached both, no further refinement.
1363	if (AllocType == BothTypes)
1364	return AllocType;
1365	}
1366	return AllocType;
1367	}
1368
1369	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1370	uint8_t
1371	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::intersectAllocTypesImpl(
1372	const DenseSet<uint32_t> &Node1Ids,
1373	const DenseSet<uint32_t> &Node2Ids) const {
1374	uint8_t BothTypes =
1375	(uint8_t)AllocationType::Cold \| (uint8_t)AllocationType::NotCold;
1376	uint8_t AllocType = (uint8_t)AllocationType::None;
1377	for (auto Id : Node1Ids) {
1378	if (!Node2Ids.count(V: Id))
1379	continue;
1380	AllocType \|= (uint8_t)ContextIdToAllocationType.at(Val: Id);
1381	// Bail early if alloc type reached both, no further refinement.
1382	if (AllocType == BothTypes)
1383	return AllocType;
1384	}
1385	return AllocType;
1386	}
1387
1388	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1389	uint8_t CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::intersectAllocTypes(
1390	const DenseSet<uint32_t> &Node1Ids,
1391	const DenseSet<uint32_t> &Node2Ids) const {
1392	if (Node1Ids.size() < Node2Ids.size())
1393	return intersectAllocTypesImpl(Node1Ids, Node2Ids);
1394	else
1395	return intersectAllocTypesImpl(Node1Ids: Node2Ids, Node2Ids: Node1Ids);
1396	}
1397
1398	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1399	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
1400	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::addAllocNode(
1401	CallInfo Call, const FuncTy *F) {
1402	assert(!getNodeForAlloc(Call));
1403	ContextNode AllocNode = createNewNode(/IsAllocation=/*true, F, C: Call);
1404	AllocationCallToContextNodeMap[Call] = AllocNode;
1405	// Use LastContextId as a uniq id for MIB allocation nodes.
1406	AllocNode->OrigStackOrAllocId = LastContextId;
1407	// Alloc type should be updated as we add in the MIBs. We should assert
1408	// afterwards that it is not still None.
1409	AllocNode->AllocTypes = (uint8_t)AllocationType::None;
1410
1411	return AllocNode;
1412	}
1413
1414	static std::string getAllocTypeString(uint8_t AllocTypes) {
1415	if (!AllocTypes)
1416	return "None";
1417	std::string Str;
1418	if (AllocTypes & (uint8_t)AllocationType::NotCold)
1419	Str += "NotCold";
1420	if (AllocTypes & (uint8_t)AllocationType::Cold)
1421	Str += "Cold";
1422	return Str;
1423	}
1424
1425	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1426	template <class NodeT, class IteratorT>
1427	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::addStackNodesForMIB(
1428	ContextNode *AllocNode, CallStack<NodeT, IteratorT> &StackContext,
1429	CallStack<NodeT, IteratorT> &CallsiteContext, AllocationType AllocType,
1430	ArrayRef<ContextTotalSize> ContextSizeInfo,
1431	std::map<uint64_t, uint32_t> &TotalSizeToContextIdTopNCold) {
1432	// Treating the hot alloc type as NotCold before the disambiguation for "hot"
1433	// is done.
1434	if (AllocType == AllocationType::Hot)
1435	AllocType = AllocationType::NotCold;
1436
1437	ContextIdToAllocationType [++LastContextId] = AllocType;
1438
1439	bool IsImportant = false;
1440	if (!ContextSizeInfo.empty()) {
1441	auto &Entry = ContextIdToContextSizeInfos [LastContextId];
1442	// If this is a cold allocation, and we are collecting non-zero largest
1443	// contexts, see if this is a candidate.
1444	if (AllocType == AllocationType::Cold && MemProfTopNImportant > `0`) {
1445	uint64_t TotalCold = `0`;
1446	for (auto &CSI : ContextSizeInfo)
1447	TotalCold += CSI.TotalSize;
1448	// Record this context if either we haven't found the first top-n largest
1449	// yet, or if it is larger than the smallest already recorded.
1450	if (TotalSizeToContextIdTopNCold.size() < MemProfTopNImportant \|\|
1451	// Since TotalSizeToContextIdTopNCold is a std::map, it is implicitly
1452	// sorted in ascending size of its key which is the size.
1453	TotalCold > TotalSizeToContextIdTopNCold.begin()->first) {
1454	if (TotalSizeToContextIdTopNCold.size() == MemProfTopNImportant) {
1455	// Remove old one and its associated entries.
1456	auto IdToRemove = TotalSizeToContextIdTopNCold.begin()->second;
1457	TotalSizeToContextIdTopNCold.erase(
1458	position: TotalSizeToContextIdTopNCold.begin());
1459	assert(ImportantContextIdInfo.count(IdToRemove));
1460	ImportantContextIdInfo.erase(IdToRemove);
1461	}
1462	TotalSizeToContextIdTopNCold [TotalCold] = LastContextId;
1463	IsImportant = true;
1464	}
1465	}
1466	Entry.insert(position: Entry.begin(), first: ContextSizeInfo.begin(), last: ContextSizeInfo.end());
1467	}
1468
1469	// Update alloc type and context ids for this MIB.
1470	AllocNode->AllocTypes \|= (uint8_t)AllocType;
1471
1472	// Now add or update nodes for each stack id in alloc's context.
1473	// Later when processing the stack ids on non-alloc callsites we will adjust
1474	// for any inlining in the context.
1475	ContextNode *PrevNode = AllocNode;
1476	// Look for recursion (direct recursion should have been collapsed by
1477	// module summary analysis, here we should just be detecting mutual
1478	// recursion). Mark these nodes so we don't try to clone.
1479	SmallSet<uint64_t, `8`> StackIdSet;
1480	// Skip any on the allocation call (inlining).
1481	for (auto ContextIter = StackContext.beginAfterSharedPrefix(CallsiteContext);
1482	ContextIter != StackContext.end(); ++ContextIter) {
1483	auto StackId = getStackId(IdOrIndex: *ContextIter);
1484	if (IsImportant)
1485	ImportantContextIdInfo[LastContextId].StackIds.push_back(StackId);
1486	ContextNode *StackNode = getNodeForStackId(StackId);
1487	if (!StackNode) {
1488	StackNode = createNewNode(/IsAllocation=/false);
1489	StackEntryIdToContextNodeMap[StackId] = StackNode;
1490	StackNode->OrigStackOrAllocId = StackId;
1491	}
1492	// Marking a node recursive will prevent its cloning completely, even for
1493	// non-recursive contexts flowing through it.
1494	if (!AllowRecursiveCallsites) {
1495	auto Ins = StackIdSet.insert(StackId);
1496	if (!Ins.second)
1497	StackNode->Recursive = true;
1498	}
1499	StackNode->AllocTypes \|= (uint8_t)AllocType;
1500	PrevNode->addOrUpdateCallerEdge(StackNode, AllocType, LastContextId);
1501	PrevNode = StackNode;
1502	}
1503	}
1504
1505	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1506	DenseSet<uint32_t>
1507	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::duplicateContextIds(
1508	const DenseSet<uint32_t> &StackSequenceContextIds,
1509	DenseMap<uint32_t, DenseSet<uint32_t>> &OldToNewContextIds) {
1510	DenseSet<uint32_t> NewContextIds;
1511	for (auto OldId : StackSequenceContextIds) {
1512	NewContextIds.insert(V: ++LastContextId);
1513	OldToNewContextIds [OldId].insert(V: LastContextId);
1514	assert(ContextIdToAllocationType.count(OldId));
1515	// The new context has the same allocation type and size info as original.
1516	ContextIdToAllocationType [LastContextId] = ContextIdToAllocationType [OldId];
1517	auto CSI = ContextIdToContextSizeInfos.find(Val: OldId);
1518	if (CSI != ContextIdToContextSizeInfos.end())
1519	ContextIdToContextSizeInfos [LastContextId] = CSI ->second;
1520	if (DotAllocContextIds.contains(V: OldId))
1521	DotAllocContextIds.insert(V: LastContextId);
1522	}
1523	return NewContextIds;
1524	}
1525
1526	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1527	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
1528	propagateDuplicateContextIds(
1529	const DenseMap<uint32_t, DenseSet<uint32_t>> &OldToNewContextIds) {
1530	// Build a set of duplicated context ids corresponding to the input id set.
1531	auto GetNewIds = [&OldToNewContextIds](const DenseSet<uint32_t> &ContextIds) {
1532	DenseSet<uint32_t> NewIds;
1533	for (auto Id : ContextIds)
1534	if (auto NewId = OldToNewContextIds.find(Val: Id);
1535	NewId != OldToNewContextIds.end())
1536	NewIds.insert_range(R: NewId ->second);
1537	return NewIds;
1538	};
1539
1540	// Recursively update context ids sets along caller edges.
1541	auto UpdateCallers = [&](ContextNode *Node,
1542	DenseSet<const ContextEdge *> &Visited,
1543	auto &&UpdateCallers) -> void {
1544	for (const auto &Edge : Node->CallerEdges) {
1545	auto Inserted = Visited.insert(Edge.get());
1546	if (!Inserted.second)
1547	continue;
1548	ContextNode *NextNode = Edge->Caller;
1549	DenseSet<uint32_t> NewIdsToAdd = GetNewIds(Edge->getContextIds());
1550	// Only need to recursively iterate to NextNode via this caller edge if
1551	// it resulted in any added ids to NextNode.
1552	if (!NewIdsToAdd.empty()) {
1553	Edge->getContextIds().insert_range(NewIdsToAdd);
1554	UpdateCallers(NextNode, Visited, UpdateCallers);
1555	}
1556	}
1557	};
1558
1559	DenseSet<const ContextEdge *> Visited;
1560	for (auto &Entry : AllocationCallToContextNodeMap) {
1561	auto *Node = Entry.second;
1562	UpdateCallers(Node, Visited, UpdateCallers);
1563	}
1564	}
1565
1566	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1567	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::connectNewNode(
1568	ContextNode NewNode, ContextNode OrigNode, bool TowardsCallee,
1569	// This must be passed by value to make a copy since it will be adjusted
1570	// as ids are moved.
1571	DenseSet<uint32_t> RemainingContextIds) {
1572	auto &OrigEdges =
1573	TowardsCallee ? OrigNode->CalleeEdges : OrigNode->CallerEdges;
1574	DenseSet<uint32_t> RecursiveContextIds;
1575	DenseSet<uint32_t> AllCallerContextIds;
1576	if (AllowRecursiveCallsites) {
1577	// Identify which context ids are recursive which is needed to properly
1578	// update the RemainingContextIds set. The relevant recursive context ids
1579	// are those that are in multiple edges.
1580	for (auto &CE : OrigEdges) {
1581	AllCallerContextIds.reserve(Size: CE->getContextIds().size());
1582	for (auto Id : CE->getContextIds())
1583	if (!AllCallerContextIds.insert(Id).second)
1584	RecursiveContextIds.insert(Id);
1585	}
1586	}
1587	// Increment iterator in loop so that we can remove edges as needed.
1588	for (auto EI = OrigEdges.begin(); EI != OrigEdges.end();) {
1589	auto Edge = *EI;
1590	DenseSet<uint32_t> NewEdgeContextIds;
1591	DenseSet<uint32_t> NotFoundContextIds;
1592	// Remove any matching context ids from Edge, return set that were found and
1593	// removed, these are the new edge's context ids. Also update the remaining
1594	// (not found ids).
1595	set_subtract(Edge->getContextIds(), RemainingContextIds, NewEdgeContextIds,
1596	NotFoundContextIds);
1597	// Update the remaining context ids set for the later edges. This is a
1598	// compile time optimization.
1599	if (RecursiveContextIds.empty()) {
1600	// No recursive ids, so all of the previously remaining context ids that
1601	// were not seen on this edge are the new remaining set.
1602	RemainingContextIds.swap(RHS&: NotFoundContextIds);
1603	} else {
1604	// Keep the recursive ids in the remaining set as we expect to see those
1605	// on another edge. We can remove the non-recursive remaining ids that
1606	// were seen on this edge, however. We already have the set of remaining
1607	// ids that were on this edge (in NewEdgeContextIds). Figure out which are
1608	// non-recursive and only remove those. Note that despite the higher
1609	// overhead of updating the remaining context ids set when recursion
1610	// handling is enabled, it was found to be at worst performance neutral
1611	// and in one case a clear win.
1612	DenseSet<uint32_t> NonRecursiveRemainingCurEdgeIds =
1613	set_difference(S1: NewEdgeContextIds, S2: RecursiveContextIds);
1614	set_subtract(S1&: RemainingContextIds, S2: NonRecursiveRemainingCurEdgeIds);
1615	}
1616	// If no matching context ids for this edge, skip it.
1617	if (NewEdgeContextIds.empty()) {
1618	++EI;
1619	continue;
1620	}
1621	if (TowardsCallee) {
1622	uint8_t NewAllocType = computeAllocType(ContextIds&: NewEdgeContextIds);
1623	auto NewEdge = std::make_shared<ContextEdge>(
1624	Edge->Callee, NewNode, NewAllocType, std::move(NewEdgeContextIds));
1625	NewNode->CalleeEdges.push_back(NewEdge);
1626	NewEdge->Callee->CallerEdges.push_back(NewEdge);
1627	} else {
1628	uint8_t NewAllocType = computeAllocType(ContextIds&: NewEdgeContextIds);
1629	auto NewEdge = std::make_shared<ContextEdge>(
1630	NewNode, Edge->Caller, NewAllocType, std::move(NewEdgeContextIds));
1631	NewNode->CallerEdges.push_back(NewEdge);
1632	NewEdge->Caller->CalleeEdges.push_back(NewEdge);
1633	}
1634	// Remove old edge if context ids empty.
1635	if (Edge->getContextIds().empty()) {
1636	removeEdgeFromGraph(Edge: Edge.get(), EI: &EI, CalleeIter: TowardsCallee);
1637	continue;
1638	}
1639	++EI;
1640	}
1641	}
1642
1643	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1644	static void checkEdge(
1645	const std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>> &Edge) {
1646	// Confirm that alloc type is not None and that we have at least one context
1647	// id.
1648	assert(Edge->AllocTypes != (uint8_t)AllocationType::None);
1649	assert(!Edge->ContextIds.empty());
1650	}
1651
1652	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1653	static void checkNode(const ContextNode<DerivedCCG, FuncTy, CallTy> *Node,
1654	bool CheckEdges = true) {
1655	if (Node->isRemoved())
1656	return;
1657	#ifndef NDEBUG
1658	// Compute node's context ids once for use in asserts.
1659	auto NodeContextIds = Node->getContextIds();
1660	#endif
1661	// Node's context ids should be the union of both its callee and caller edge
1662	// context ids.
1663	if (Node->CallerEdges.size()) {
1664	DenseSet<uint32_t> CallerEdgeContextIds(
1665	Node->CallerEdges.front()->ContextIds);
1666	for (const auto &Edge : llvm::drop_begin(Node->CallerEdges)) {
1667	if (CheckEdges)
1668	checkEdge<DerivedCCG, FuncTy, CallTy>(Edge);
1669	set_union(CallerEdgeContextIds, Edge->ContextIds);
1670	}
1671	// Node can have more context ids than callers if some contexts terminate at
1672	// node and some are longer. If we are allowing recursive callsites and
1673	// contexts this will be violated for incompletely cloned recursive cycles,
1674	// so skip the checking in that case.
1675	assert((AllowRecursiveCallsites && AllowRecursiveContexts) \|\|
1676	NodeContextIds == CallerEdgeContextIds \|\|
1677	set_is_subset(CallerEdgeContextIds, NodeContextIds));
1678	}
1679	if (Node->CalleeEdges.size()) {
1680	DenseSet<uint32_t> CalleeEdgeContextIds(
1681	Node->CalleeEdges.front()->ContextIds);
1682	for (const auto &Edge : llvm::drop_begin(Node->CalleeEdges)) {
1683	if (CheckEdges)
1684	checkEdge<DerivedCCG, FuncTy, CallTy>(Edge);
1685	set_union(CalleeEdgeContextIds, Edge->getContextIds());
1686	}
1687	// If we are allowing recursive callsites and contexts this will be violated
1688	// for incompletely cloned recursive cycles, so skip the checking in that
1689	// case.
1690	assert((AllowRecursiveCallsites && AllowRecursiveContexts) \|\|
1691	NodeContextIds == CalleeEdgeContextIds);
1692	}
1693	// FIXME: Since this checking is only invoked under an option, we should
1694	// change the error checking from using assert to something that will trigger
1695	// an error on a release build.
1696	#ifndef NDEBUG
1697	// Make sure we don't end up with duplicate edges between the same caller and
1698	// callee.
1699	DenseSet<ContextNode<DerivedCCG, FuncTy, CallTy> *> NodeSet;
1700	for (const auto &E : Node->CalleeEdges)
1701	NodeSet.insert(E->Callee);
1702	assert(NodeSet.size() == Node->CalleeEdges.size());
1703	#endif
1704	}
1705
1706	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1707	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
1708	assignStackNodesPostOrder(ContextNode *Node,
1709	DenseSet<const ContextNode *> &Visited,
1710	DenseMap<uint64_t, std::vector<CallContextInfo>>
1711	&StackIdToMatchingCalls,
1712	DenseMap<CallInfo, CallInfo> &CallToMatchingCall,
1713	const DenseSet<uint32_t> &ImportantContextIds) {
1714	auto Inserted = Visited.insert(Node);
1715	if (!Inserted.second)
1716	return;
1717	// Post order traversal. Iterate over a copy since we may add nodes and
1718	// therefore new callers during the recursive call, invalidating any
1719	// iterator over the original edge vector. We don't need to process these
1720	// new nodes as they were already processed on creation.
1721	auto CallerEdges = Node->CallerEdges;
1722	for (auto &Edge : CallerEdges) {
1723	// Skip any that have been removed during the recursion.
1724	if (Edge->isRemoved()) {
1725	assert(!is_contained(Node->CallerEdges, Edge));
1726	continue;
1727	}
1728	assignStackNodesPostOrder(Node: Edge->Caller, Visited, StackIdToMatchingCalls,
1729	CallToMatchingCall, ImportantContextIds);
1730	}
1731
1732	// If this node's stack id is in the map, update the graph to contain new
1733	// nodes representing any inlining at interior callsites. Note we move the
1734	// associated context ids over to the new nodes.
1735
1736	// Ignore this node if it is for an allocation or we didn't record any
1737	// stack id lists ending at it.
1738	if (Node->IsAllocation \|\|
1739	!StackIdToMatchingCalls.count(Node->OrigStackOrAllocId))
1740	return;
1741
1742	auto &Calls = StackIdToMatchingCalls[Node->OrigStackOrAllocId];
1743	// Handle the simple case first. A single call with a single stack id.
1744	// In this case there is no need to create any new context nodes, simply
1745	// assign the context node for stack id to this Call.
1746	if (Calls.size() == `1`) {
1747	auto &[Call, Ids, Func, SavedContextIds] = Calls[`0`];
1748	if (Ids.size() == `1`) {
1749	assert(SavedContextIds.empty());
1750	// It should be this Node
1751	assert(Node == getNodeForStackId(Ids[`0`]));
1752	if (Node->Recursive)
1753	return;
1754	Node->setCall(Call);
1755	NonAllocationCallToContextNodeMap[Call] = Node;
1756	NodeToCallingFunc[Node] = Func;
1757	recordStackNode(StackIds&: Ids, Node, NodeContextIds: Node->getContextIds(), ImportantContextIds);
1758	return;
1759	}
1760	}
1761
1762	#ifndef NDEBUG
1763	// Find the node for the last stack id, which should be the same
1764	// across all calls recorded for this id, and is this node's id.
1765	uint64_t LastId = Node->OrigStackOrAllocId;
1766	ContextNode *LastNode = getNodeForStackId(LastId);
1767	// We should only have kept stack ids that had nodes.
1768	assert(LastNode);
1769	assert(LastNode == Node);
1770	#else
1771	ContextNode *LastNode = Node;
1772	#endif
1773
1774	// Compute the last node's context ids once, as it is shared by all calls in
1775	// this entry.
1776	DenseSet<uint32_t> LastNodeContextIds = LastNode->getContextIds();
1777
1778	[[maybe_unused]] bool PrevIterCreatedNode = false;
1779	bool CreatedNode = false;
1780	for (unsigned I = `0`; I < Calls.size();
1781	I++, PrevIterCreatedNode = CreatedNode) {
1782	CreatedNode = false;
1783	auto &[Call, Ids, Func, SavedContextIds] = Calls[I];
1784	// Skip any for which we didn't assign any ids, these don't get a node in
1785	// the graph.
1786	if (SavedContextIds.empty()) {
1787	// If this call has a matching call (located in the same function and
1788	// having the same stack ids), simply add it to the context node created
1789	// for its matching call earlier. These can be treated the same through
1790	// cloning and get updated at the same time.
1791	if (!CallToMatchingCall.contains(Call))
1792	continue;
1793	auto MatchingCall = CallToMatchingCall[Call];
1794	if (!NonAllocationCallToContextNodeMap.contains(MatchingCall)) {
1795	// This should only happen if we had a prior iteration, and it didn't
1796	// create a node because of the below recomputation of context ids
1797	// finding none remaining and continuing early.
1798	assert(I > `0` && !PrevIterCreatedNode);
1799	continue;
1800	}
1801	NonAllocationCallToContextNodeMap[MatchingCall]->MatchingCalls.push_back(
1802	Call);
1803	continue;
1804	}
1805
1806	assert(LastId == Ids.back());
1807
1808	// Recompute the context ids for this stack id sequence (the
1809	// intersection of the context ids of the corresponding nodes).
1810	// Start with the ids we saved in the map for this call, which could be
1811	// duplicated context ids. We have to recompute as we might have overlap
1812	// overlap between the saved context ids for different last nodes, and
1813	// removed them already during the post order traversal.
1814	set_intersect(SavedContextIds, LastNodeContextIds);
1815	ContextNode *PrevNode = LastNode;
1816	bool Skip = false;
1817	// Iterate backwards through the stack Ids, starting after the last Id
1818	// in the list, which was handled once outside for all Calls.
1819	for (auto IdIter = Ids.rbegin() + `1`; IdIter != Ids.rend(); IdIter++) {
1820	auto Id = *IdIter;
1821	ContextNode *CurNode = getNodeForStackId(StackId: Id);
1822	// We should only have kept stack ids that had nodes and weren't
1823	// recursive.
1824	assert(CurNode);
1825	assert(!CurNode->Recursive);
1826
1827	auto *Edge = CurNode->findEdgeFromCaller(PrevNode);
1828	if (!Edge) {
1829	Skip = true;
1830	break;
1831	}
1832	PrevNode = CurNode;
1833
1834	// Update the context ids, which is the intersection of the ids along
1835	// all edges in the sequence.
1836	set_intersect(SavedContextIds, Edge->getContextIds());
1837
1838	// If we now have no context ids for clone, skip this call.
1839	if (SavedContextIds.empty()) {
1840	Skip = true;
1841	break;
1842	}
1843	}
1844	if (Skip)
1845	continue;
1846
1847	// Create new context node.
1848	ContextNode NewNode = createNewNode(/IsAllocation=/*false, F: Func, C: Call);
1849	NonAllocationCallToContextNodeMap[Call] = NewNode;
1850	CreatedNode = true;
1851	NewNode->AllocTypes = computeAllocType(ContextIds&: SavedContextIds);
1852
1853	ContextNode *FirstNode = getNodeForStackId(StackId: Ids[`0`]);
1854	assert(FirstNode);
1855
1856	// Connect to callees of innermost stack frame in inlined call chain.
1857	// This updates context ids for FirstNode's callee's to reflect those
1858	// moved to NewNode.
1859	connectNewNode(NewNode, OrigNode: FirstNode, /TowardsCallee=/true, RemainingContextIds: SavedContextIds);
1860
1861	// Connect to callers of outermost stack frame in inlined call chain.
1862	// This updates context ids for FirstNode's caller's to reflect those
1863	// moved to NewNode.
1864	connectNewNode(NewNode, OrigNode: LastNode, /TowardsCallee=/false, RemainingContextIds: SavedContextIds);
1865
1866	// Now we need to remove context ids from edges/nodes between First and
1867	// Last Node.
1868	PrevNode = nullptr;
1869	for (auto Id : Ids) {
1870	ContextNode *CurNode = getNodeForStackId(StackId: Id);
1871	// We should only have kept stack ids that had nodes.
1872	assert(CurNode);
1873
1874	// Remove the context ids moved to NewNode from CurNode, and the
1875	// edge from the prior node.
1876	if (PrevNode) {
1877	auto *PrevEdge = CurNode->findEdgeFromCallee(PrevNode);
1878	// If the sequence contained recursion, we might have already removed
1879	// some edges during the connectNewNode calls above.
1880	if (!PrevEdge) {
1881	PrevNode = CurNode;
1882	continue;
1883	}
1884	set_subtract(PrevEdge->getContextIds(), SavedContextIds);
1885	if (PrevEdge->getContextIds().empty())
1886	removeEdgeFromGraph(Edge: PrevEdge);
1887	}
1888	// Since we update the edges from leaf to tail, only look at the callee
1889	// edges. This isn't an alloc node, so if there are no callee edges, the
1890	// alloc type is None.
1891	CurNode->AllocTypes = CurNode->CalleeEdges.empty()
1892	? (uint8_t)AllocationType::None
1893	: CurNode->computeAllocType();
1894	PrevNode = CurNode;
1895	}
1896
1897	recordStackNode(StackIds&: Ids, Node: NewNode, NodeContextIds: SavedContextIds, ImportantContextIds);
1898
1899	if (VerifyNodes) {
1900	checkNode<DerivedCCG, FuncTy, CallTy>(NewNode, /CheckEdges=/true);
1901	for (auto Id : Ids) {
1902	ContextNode *CurNode = getNodeForStackId(StackId: Id);
1903	// We should only have kept stack ids that had nodes.
1904	assert(CurNode);
1905	checkNode<DerivedCCG, FuncTy, CallTy>(CurNode, /CheckEdges=/true);
1906	}
1907	}
1908	}
1909	}
1910
1911	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1912	void CallsiteContextGraph<DerivedCCG, FuncTy,
1913	CallTy>::fixupImportantContexts() {
1914	if (ImportantContextIdInfo.empty())
1915	return;
1916
1917	// Update statistics as we are done building this map at this point.
1918	NumImportantContextIds = ImportantContextIdInfo.size();
1919
1920	if (!MemProfFixupImportant)
1921	return;
1922
1923	if (ExportToDot)
1924	exportToDot(Label: "beforestackfixup");
1925
1926	// For each context we identified as important, walk through the saved context
1927	// stack ids in order from leaf upwards, and make sure all edges are correct.
1928	// These can be difficult to get right when updating the graph while mapping
1929	// nodes onto summary or IR, especially when there is recursion. In
1930	// particular, when we have created new nodes to reflect inlining, it is
1931	// sometimes impossible to know exactly how to update the edges in the face of
1932	// recursion, as we have lost the original ordering of the stack ids in the
1933	// contexts.
1934	// TODO: Consider only doing this if we detect the context has recursive
1935	// cycles.
1936	//
1937	// I.e. assume we have a context with stack ids like: {A B A C A D E}
1938	// and let's say A was inlined into B, C, and D. The original graph will have
1939	// multiple recursive cycles through A. When we match the original context
1940	// nodes onto the IR or summary, we will merge {A B} into one context node,
1941	// {A C} onto another, and {A D} onto another. Looking at the stack sequence
1942	// above, we should end up with a non-cyclic set of edges like:
1943	// {AB} <- {AC} <- {AD} <- E. However, because we normally have lost the
1944	// original ordering, we won't get the edges correct initially (it's
1945	// impossible without the original ordering). Here we do the fixup (add and
1946	// removing edges where necessary) for this context. In the
1947	// ImportantContextInfo struct in this case we should have a MaxLength = 2,
1948	// and map entries for {A B}, {A C}, {A D}, and {E}.
1949	for (auto &[CurContextId, Info] : ImportantContextIdInfo) {
1950	if (Info.StackIdsToNode.empty())
1951	continue;
1952	bool Changed = false;
1953	ContextNode PrevNode = nullptr*;
1954	ContextNode CurNode = nullptr*;
1955	DenseSet<const ContextEdge *> VisitedEdges;
1956	ArrayRef<uint64_t> AllStackIds(Info.StackIds);
1957	// Try to identify what callsite ContextNode maps to which slice of the
1958	// context's ordered stack ids.
1959	for (unsigned I = `0`; I < AllStackIds.size(); I++, PrevNode = CurNode) {
1960	// We will do this greedily, trying up to MaxLength stack ids in a row, to
1961	// see if we recorded a context node for that sequence.
1962	auto Len = Info.MaxLength;
1963	auto LenToEnd = AllStackIds.size() - I;
1964	if (Len > LenToEnd)
1965	Len = LenToEnd;
1966	CurNode = nullptr;
1967	// Try to find a recorded context node starting with the longest length
1968	// recorded, and on down until we check for just a single stack node.
1969	for (; Len > `0`; Len--) {
1970	// Get the slice of the original stack id sequence to check.
1971	auto CheckStackIds = AllStackIds.slice(I, Len);
1972	auto EntryIt = Info.StackIdsToNode.find(CheckStackIds);
1973	if (EntryIt == Info.StackIdsToNode.end())
1974	continue;
1975	CurNode = EntryIt->second;
1976	// Skip forward so we don't try to look for the ones we just matched.
1977	// We increment by Len - 1, because the outer for loop will increment I.
1978	I += Len - `1`;
1979	break;
1980	}
1981	// Give up if we couldn't find a node. Since we need to clone from the
1982	// leaf allocation upwards, no sense in doing anymore fixup further up
1983	// the context if we couldn't match part of the original stack context
1984	// onto a callsite node.
1985	if (!CurNode)
1986	break;
1987	// No edges to fix up until we have a pair of nodes that should be
1988	// adjacent in the graph.
1989	if (!PrevNode)
1990	continue;
1991	// See if we already have a call edge from CurNode to PrevNode.
1992	auto *CurEdge = PrevNode->findEdgeFromCaller(CurNode);
1993	if (CurEdge) {
1994	// We already have an edge. Make sure it contains this context id.
1995	if (CurEdge->getContextIds().insert(CurContextId).second) {
1996	NumFixupEdgeIdsInserted ++;
1997	Changed = true;
1998	}
1999	} else {
2000	// No edge exists - add one.
2001	NumFixupEdgesAdded ++;
2002	DenseSet<uint32_t> ContextIds({CurContextId});
2003	auto AllocType = computeAllocType(ContextIds);
2004	auto NewEdge = std::make_shared<ContextEdge>(
2005	PrevNode, CurNode, AllocType, std::move(ContextIds));
2006	PrevNode->CallerEdges.push_back(NewEdge);
2007	CurNode->CalleeEdges.push_back(NewEdge);
2008	// Save the new edge for the below handling.
2009	CurEdge = NewEdge.get();
2010	Changed = true;
2011	}
2012	VisitedEdges.insert(CurEdge);
2013	// Now remove this context id from any other caller edges calling
2014	// PrevNode.
2015	for (auto &Edge : PrevNode->CallerEdges) {
2016	// Skip the edge updating/created above and edges we have already
2017	// visited (due to recursion).
2018	if (Edge.get() != CurEdge && !VisitedEdges.contains(Edge.get()))
2019	Edge->getContextIds().erase(CurContextId);
2020	}
2021	}
2022	if (Changed)
2023	NumFixedContexts ++;
2024	}
2025	}
2026
2027	template <typename DerivedCCG, typename FuncTy, typename CallTy>
2028	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::updateStackNodes() {
2029	// Map of stack id to all calls with that as the last (outermost caller)
2030	// callsite id that has a context node (some might not due to pruning
2031	// performed during matching of the allocation profile contexts).
2032	// The CallContextInfo contains the Call and a list of its stack ids with
2033	// ContextNodes, the function containing Call, and the set of context ids
2034	// the analysis will eventually identify for use in any new node created
2035	// for that callsite.
2036	DenseMap<uint64_t, std::vector<CallContextInfo>> StackIdToMatchingCalls;
2037	for (auto &[Func, CallsWithMetadata] : FuncToCallsWithMetadata) {
2038	for (auto &Call : CallsWithMetadata) {
2039	// Ignore allocations, already handled.
2040	if (AllocationCallToContextNodeMap.count(Call))
2041	continue;
2042	auto StackIdsWithContextNodes =
2043	getStackIdsWithContextNodesForCall(Call: Call.call());
2044	// If there were no nodes created for MIBs on allocs (maybe this was in
2045	// the unambiguous part of the MIB stack that was pruned), ignore.
2046	if (StackIdsWithContextNodes.empty())
2047	continue;
2048	// Otherwise, record this Call along with the list of ids for the last
2049	// (outermost caller) stack id with a node.
2050	StackIdToMatchingCalls[StackIdsWithContextNodes.back()].push_back(
2051	{Call.call(), StackIdsWithContextNodes, Func, {}});
2052	}
2053	}
2054
2055	// First make a pass through all stack ids that correspond to a call,
2056	// as identified in the above loop. Compute the context ids corresponding to
2057	// each of these calls when they correspond to multiple stack ids due to
2058	// due to inlining. Perform any duplication of context ids required when
2059	// there is more than one call with the same stack ids. Their (possibly newly
2060	// duplicated) context ids are saved in the StackIdToMatchingCalls map.
2061	DenseMap<uint32_t, DenseSet<uint32_t>> OldToNewContextIds;
2062	// Save a map from each call to any that are found to match it. I.e. located
2063	// in the same function and have the same (possibly pruned) stack ids. We use
2064	// this to avoid creating extra graph nodes as they can be treated the same.
2065	DenseMap<CallInfo, CallInfo> CallToMatchingCall;
2066	for (auto &It : StackIdToMatchingCalls) {
2067	auto &Calls = It.getSecond();
2068	// Skip single calls with a single stack id. These don't need a new node.
2069	if (Calls.size() == `1`) {
2070	auto &Ids = Calls[`0`].StackIds;
2071	if (Ids.size() == `1`)
2072	continue;
2073	}
2074	// In order to do the best and maximal matching of inlined calls to context
2075	// node sequences we will sort the vectors of stack ids in descending order
2076	// of length, and within each length, lexicographically by stack id. The
2077	// latter is so that we can specially handle calls that have identical stack
2078	// id sequences (either due to cloning or artificially because of the MIB
2079	// context pruning). Those with the same Ids are then sorted by function to
2080	// facilitate efficiently mapping them to the same context node.
2081	// Because the functions are pointers, to ensure a stable sort first assign
2082	// each function pointer to its first index in the Calls array, and then use
2083	// that to sort by.
2084	DenseMap<const FuncTy , unsigned*> FuncToIndex;
2085	for (const auto &[Idx, CallCtxInfo] : enumerate(Calls))
2086	FuncToIndex.insert({CallCtxInfo.Func, Idx});
2087	llvm::stable_sort(
2088	Calls,
2089	[&FuncToIndex](const CallContextInfo &A, const CallContextInfo &B) {
2090	return A.StackIds.size() > B.StackIds.size() \|\|
2091	(A.StackIds.size() == B.StackIds.size() &&
2092	(A.StackIds < B.StackIds \|\|
2093	(A.StackIds == B.StackIds &&
2094	FuncToIndex[A.Func] < FuncToIndex[B.Func])));
2095	});
2096
2097	// Find the node for the last stack id, which should be the same
2098	// across all calls recorded for this id, and is the id for this
2099	// entry in the StackIdToMatchingCalls map.
2100	uint64_t LastId = It.getFirst();
2101	ContextNode *LastNode = getNodeForStackId(StackId: LastId);
2102	// We should only have kept stack ids that had nodes.
2103	assert(LastNode);
2104
2105	if (LastNode->Recursive)
2106	continue;
2107
2108	// Initialize the context ids with the last node's. We will subsequently
2109	// refine the context ids by computing the intersection along all edges.
2110	DenseSet<uint32_t> LastNodeContextIds = LastNode->getContextIds();
2111	assert(!LastNodeContextIds.empty());
2112
2113	#ifndef NDEBUG
2114	// Save the set of functions seen for a particular set of the same stack
2115	// ids. This is used to ensure that they have been correctly sorted to be
2116	// adjacent in the Calls list, since we rely on that to efficiently place
2117	// all such matching calls onto the same context node.
2118	DenseSet<const FuncTy *> MatchingIdsFuncSet;
2119	#endif
2120
2121	for (unsigned I = `0`; I < Calls.size(); I++) {
2122	auto &[Call, Ids, Func, SavedContextIds] = Calls[I];
2123	assert(SavedContextIds.empty());
2124	assert(LastId == Ids.back());
2125
2126	#ifndef NDEBUG
2127	// If this call has a different set of ids than the last one, clear the
2128	// set used to ensure they are sorted properly.
2129	if (I > `0` && Ids != Calls[I - `1`].StackIds)
2130	MatchingIdsFuncSet.clear();
2131	#endif
2132
2133	// First compute the context ids for this stack id sequence (the
2134	// intersection of the context ids of the corresponding nodes).
2135	// Start with the remaining saved ids for the last node.
2136	assert(!LastNodeContextIds.empty());
2137	DenseSet<uint32_t> StackSequenceContextIds = LastNodeContextIds;
2138
2139	ContextNode *PrevNode = LastNode;
2140	ContextNode *CurNode = LastNode;
2141	bool Skip = false;
2142
2143	// Iterate backwards through the stack Ids, starting after the last Id
2144	// in the list, which was handled once outside for all Calls.
2145	for (auto IdIter = Ids.rbegin() + `1`; IdIter != Ids.rend(); IdIter++) {
2146	auto Id = *IdIter;
2147	CurNode = getNodeForStackId(StackId: Id);
2148	// We should only have kept stack ids that had nodes.
2149	assert(CurNode);
2150
2151	if (CurNode->Recursive) {
2152	Skip = true;
2153	break;
2154	}
2155
2156	auto *Edge = CurNode->findEdgeFromCaller(PrevNode);
2157	// If there is no edge then the nodes belong to different MIB contexts,
2158	// and we should skip this inlined context sequence. For example, this
2159	// particular inlined context may include stack ids A->B, and we may
2160	// indeed have nodes for both A and B, but it is possible that they were
2161	// never profiled in sequence in a single MIB for any allocation (i.e.
2162	// we might have profiled an allocation that involves the callsite A,
2163	// but through a different one of its callee callsites, and we might
2164	// have profiled an allocation that involves callsite B, but reached
2165	// from a different caller callsite).
2166	if (!Edge) {
2167	Skip = true;
2168	break;
2169	}
2170	PrevNode = CurNode;
2171
2172	// Update the context ids, which is the intersection of the ids along
2173	// all edges in the sequence.
2174	set_intersect(StackSequenceContextIds, Edge->getContextIds());
2175
2176	// If we now have no context ids for clone, skip this call.
2177	if (StackSequenceContextIds.empty()) {
2178	Skip = true;
2179	break;
2180	}
2181	}
2182	if (Skip)
2183	continue;
2184
2185	// If some of this call's stack ids did not have corresponding nodes (due
2186	// to pruning), don't include any context ids for contexts that extend
2187	// beyond these nodes. Otherwise we would be matching part of unrelated /
2188	// not fully matching stack contexts. To do this, subtract any context ids
2189	// found in caller nodes of the last node found above.
2190	if (Ids.back() != getLastStackId(Call)) {
2191	for (const auto &PE : LastNode->CallerEdges) {
2192	set_subtract(StackSequenceContextIds, PE->getContextIds());
2193	if (StackSequenceContextIds.empty())
2194	break;
2195	}
2196	// If we now have no context ids for clone, skip this call.
2197	if (StackSequenceContextIds.empty())
2198	continue;
2199	}
2200
2201	#ifndef NDEBUG
2202	// If the prior call had the same stack ids this set would not be empty.
2203	// Check if we already have a call that "matches" because it is located
2204	// in the same function. If the Calls list was sorted properly we should
2205	// not encounter this situation as all such entries should be adjacent
2206	// and processed in bulk further below.
2207	assert(!MatchingIdsFuncSet.contains(Func));
2208
2209	MatchingIdsFuncSet.insert(Func);
2210	#endif
2211
2212	// Check if the next set of stack ids is the same (since the Calls vector
2213	// of tuples is sorted by the stack ids we can just look at the next one).
2214	// If so, save them in the CallToMatchingCall map so that they get
2215	// assigned to the same context node, and skip them.
2216	bool DuplicateContextIds = false;
2217	for (unsigned J = I + `1`; J < Calls.size(); J++) {
2218	auto &CallCtxInfo = Calls[J];
2219	auto &NextIds = CallCtxInfo.StackIds;
2220	if (NextIds != Ids)
2221	break;
2222	auto *NextFunc = CallCtxInfo.Func;
2223	if (NextFunc != Func) {
2224	// We have another Call with the same ids but that cannot share this
2225	// node, must duplicate ids for it.
2226	DuplicateContextIds = true;
2227	break;
2228	}
2229	auto &NextCall = CallCtxInfo.Call;
2230	CallToMatchingCall[NextCall] = Call;
2231	// Update I so that it gets incremented correctly to skip this call.
2232	I = J;
2233	}
2234
2235	// If we don't have duplicate context ids, then we can assign all the
2236	// context ids computed for the original node sequence to this call.
2237	// If there are duplicate calls with the same stack ids then we synthesize
2238	// new context ids that are duplicates of the originals. These are
2239	// assigned to SavedContextIds, which is a reference into the map entry
2240	// for this call, allowing us to access these ids later on.
2241	OldToNewContextIds.reserve(NumEntries: OldToNewContextIds.size() +
2242	StackSequenceContextIds.size());
2243	SavedContextIds =
2244	DuplicateContextIds
2245	? duplicateContextIds(StackSequenceContextIds, OldToNewContextIds)
2246	: StackSequenceContextIds;
2247	assert(!SavedContextIds.empty());
2248
2249	if (!DuplicateContextIds) {
2250	// Update saved last node's context ids to remove those that are
2251	// assigned to other calls, so that it is ready for the next call at
2252	// this stack id.
2253	set_subtract(S1&: LastNodeContextIds, S2: StackSequenceContextIds);
2254	if (LastNodeContextIds.empty())
2255	break;
2256	}
2257	}
2258	}
2259
2260	// Propagate the duplicate context ids over the graph.
2261	propagateDuplicateContextIds(OldToNewContextIds);
2262
2263	if (VerifyCCG)
2264	check();
2265
2266	// Now perform a post-order traversal over the graph, starting with the
2267	// allocation nodes, essentially processing nodes from callers to callees.
2268	// For any that contains an id in the map, update the graph to contain new
2269	// nodes representing any inlining at interior callsites. Note we move the
2270	// associated context ids over to the new nodes.
2271	DenseSet<const ContextNode *> Visited;
2272	DenseSet<uint32_t> ImportantContextIds(llvm::from_range,
2273	ImportantContextIdInfo.keys());
2274	for (auto &Entry : AllocationCallToContextNodeMap)
2275	assignStackNodesPostOrder(Node: Entry.second, Visited, StackIdToMatchingCalls,
2276	CallToMatchingCall, ImportantContextIds);
2277
2278	fixupImportantContexts();
2279
2280	if (VerifyCCG)
2281	check();
2282	}
2283
2284	uint64_t ModuleCallsiteContextGraph::getLastStackId(Instruction *Call) {
2285	CallStack<MDNode, MDNode::op_iterator> CallsiteContext(
2286	Call->getMetadata(KindID: LLVMContext::MD_callsite));
2287	return CallsiteContext.back();
2288	}
2289
2290	uint64_t IndexCallsiteContextGraph::getLastStackId(IndexCall &Call) {
2291	assert(isa<CallsiteInfo *>(Call));
2292	CallStack<CallsiteInfo, SmallVector<unsigned>::const_iterator>
2293	CallsiteContext(dyn_cast_if_present<CallsiteInfo *>(Val&: Call));
2294	// Need to convert index into stack id.
2295	return Index.getStackIdAtIndex(Index: CallsiteContext.back());
2296	}
2297
2298	static const std::string MemProfCloneSuffix = ".memprof.";
2299
2300	static std::string getMemProfFuncName(Twine Base, unsigned CloneNo) {
2301	// We use CloneNo == 0 to refer to the original version, which doesn't get
2302	// renamed with a suffix.
2303	if (!CloneNo)
2304	return Base.str();
2305	return (Base + MemProfCloneSuffix + Twine (CloneNo)).str();
2306	}
2307
2308	static bool isMemProfClone(const Function &F) {
2309	return F.getName().contains(Other: MemProfCloneSuffix);
2310	}
2311
2312	// Return the clone number of the given function by extracting it from the
2313	// memprof suffix. Assumes the caller has already confirmed it is a memprof
2314	// clone.
2315	static unsigned getMemProfCloneNum(const Function &F) {
2316	assert(isMemProfClone(F));
2317	auto Pos = F.getName().find_last_of(C: `'.'`);
2318	assert(Pos > `0`);
2319	unsigned CloneNo;
2320	bool Err = F.getName().drop_front(N: Pos + `1`).getAsInteger(Radix: `10`, Result&: CloneNo);
2321	assert(!Err);
2322	(void)Err;
2323	return CloneNo;
2324	}
2325
2326	std::string ModuleCallsiteContextGraph::getLabel(const Function *Func,
2327	const Instruction *Call,
2328	unsigned CloneNo) const {
2329	return (Twine (Call->getFunction()->getName()) + " -> " +
2330	cast<CallBase>(Val: Call)->getCalledFunction()->getName())
2331	.str();
2332	}
2333
2334	std::string IndexCallsiteContextGraph::getLabel(const FunctionSummary *Func,
2335	const IndexCall &Call,
2336	unsigned CloneNo) const {
2337	auto VI = FSToVIMap.find(x: Func);
2338	assert(VI != FSToVIMap.end());
2339	std::string CallerName = getMemProfFuncName(Base: VI ->second.name(), CloneNo);
2340	if (isa<AllocInfo *>(Val: Call))
2341	return CallerName + " -> alloc";
2342	else {
2343	auto Callsite = dyn_cast_if_present<CallsiteInfo >(Val: Call);
2344	return CallerName + " -> " +
2345	getMemProfFuncName(Base: Callsite->Callee.name(),
2346	CloneNo: Callsite->Clones [CloneNo]);
2347	}
2348	}
2349
2350	std::vector<uint64_t>
2351	ModuleCallsiteContextGraph::getStackIdsWithContextNodesForCall(
2352	Instruction *Call) {
2353	CallStack<MDNode, MDNode::op_iterator> CallsiteContext(
2354	Call->getMetadata(KindID: LLVMContext::MD_callsite));
2355	return getStackIdsWithContextNodes<MDNode, MDNode::op_iterator>(
2356	CallsiteContext);
2357	}
2358
2359	std::vector<uint64_t>
2360	IndexCallsiteContextGraph::getStackIdsWithContextNodesForCall(IndexCall &Call) {
2361	assert(isa<CallsiteInfo *>(Call));
2362	CallStack<CallsiteInfo, SmallVector<unsigned>::const_iterator>
2363	CallsiteContext(dyn_cast_if_present<CallsiteInfo *>(Val&: Call));
2364	return getStackIdsWithContextNodes<CallsiteInfo,
2365	SmallVector<unsigned>::const_iterator>(
2366	CallsiteContext);
2367	}
2368
2369	template <typename DerivedCCG, typename FuncTy, typename CallTy>
2370	template <class NodeT, class IteratorT>
2371	std::vector<uint64_t>
2372	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::getStackIdsWithContextNodes(
2373	CallStack<NodeT, IteratorT> &CallsiteContext) {
2374	std::vector<uint64_t> StackIds;
2375	for (auto IdOrIndex : CallsiteContext) {
2376	auto StackId = getStackId(IdOrIndex);
2377	ContextNode *Node = getNodeForStackId(StackId);
2378	if (!Node)
2379	break;
2380	StackIds.push_back(StackId);
2381	}
2382	return StackIds;
2383	}
2384
2385	ModuleCallsiteContextGraph::ModuleCallsiteContextGraph(
2386	Module &M,
2387	llvm::function_ref<OptimizationRemarkEmitter &(Function *)> OREGetter)
2388	: Mod(M), OREGetter (OREGetter) {
2389	// Map for keeping track of the largest cold contexts up to the number given
2390	// by MemProfTopNImportant. Must be a std::map (not DenseMap) because keys
2391	// must be sorted.
2392	std::map<uint64_t, uint32_t> TotalSizeToContextIdTopNCold;
2393	for (auto &F : M) {
2394	std::vector<CallInfo> CallsWithMetadata;
2395	for (auto &BB : F) {
2396	for (auto &I : BB) {
2397	if (!isa<CallBase>(Val: I))
2398	continue;
2399	if (auto *MemProfMD = I.getMetadata(KindID: LLVMContext::MD_memprof)) {
2400	CallsWithMetadata.push_back(x: &I);
2401	auto *AllocNode = addAllocNode(Call: &I, F: &F);
2402	auto *CallsiteMD = I.getMetadata(KindID: LLVMContext::MD_callsite);
2403	assert(CallsiteMD);
2404	CallStack<MDNode, MDNode::op_iterator> CallsiteContext(CallsiteMD);
2405	// Add all of the MIBs and their stack nodes.
2406	for (auto &MDOp : MemProfMD->operands()) {
2407	auto MIBMD = cast<const* MDNode>(Val: MDOp);
2408	std::vector<ContextTotalSize> ContextSizeInfo;
2409	// Collect the context size information if it exists.
2410	if (MIBMD->getNumOperands() > `2`) {
2411	for (unsigned I = `2`; I < MIBMD->getNumOperands(); I++) {
2412	MDNode *ContextSizePair =
2413	dyn_cast<MDNode>(Val: MIBMD->getOperand(I));
2414	assert(ContextSizePair->getNumOperands() == `2`);
2415	uint64_t FullStackId = mdconst::dyn_extract<ConstantInt>(
2416	MD: ContextSizePair->getOperand(I: `0`))
2417	->getZExtValue();
2418	uint64_t TotalSize = mdconst::dyn_extract<ConstantInt>(
2419	MD: ContextSizePair->getOperand(I: `1`))
2420	->getZExtValue();
2421	ContextSizeInfo.push_back(x: {.FullStackId: FullStackId, .TotalSize: TotalSize});
2422	}
2423	}
2424	MDNode *StackNode = getMIBStackNode(MIB: MIBMD);
2425	assert(StackNode);
2426	CallStack<MDNode, MDNode::op_iterator> StackContext(StackNode);
2427	addStackNodesForMIB<MDNode, MDNode::op_iterator>(
2428	AllocNode, StackContext, CallsiteContext,
2429	AllocType: getMIBAllocType(MIB: MIBMD), ContextSizeInfo,
2430	TotalSizeToContextIdTopNCold);
2431	}
2432	// If exporting the graph to dot and an allocation id of interest was
2433	// specified, record all the context ids for this allocation node.
2434	if (ExportToDot && AllocNode->OrigStackOrAllocId == AllocIdForDot)
2435	DotAllocContextIds = AllocNode->getContextIds();
2436	assert(AllocNode->AllocTypes != (uint8_t)AllocationType::None);
2437	// Memprof and callsite metadata on memory allocations no longer
2438	// needed.
2439	I.setMetadata(KindID: LLVMContext::MD_memprof, Node: nullptr);
2440	I.setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
2441	}
2442	// For callsite metadata, add to list for this function for later use.
2443	else if (I.getMetadata(KindID: LLVMContext::MD_callsite)) {
2444	CallsWithMetadata.push_back(x: &I);
2445	}
2446	}
2447	}
2448	if (!CallsWithMetadata.empty())
2449	FuncToCallsWithMetadata [&F] = CallsWithMetadata;
2450	}
2451
2452	if (DumpCCG) {
2453	dbgs() << "CCG before updating call stack chains:\n";
2454	dbgs() << *this;
2455	}
2456
2457	if (ExportToDot)
2458	exportToDot(Label: "prestackupdate");
2459
2460	updateStackNodes();
2461
2462	if (ExportToDot)
2463	exportToDot(Label: "poststackupdate");
2464
2465	handleCallsitesWithMultipleTargets();
2466
2467	markBackedges();
2468
2469	// Strip off remaining callsite metadata, no longer needed.
2470	for (auto &FuncEntry : FuncToCallsWithMetadata)
2471	for (auto &Call : FuncEntry.second)
2472	Call.call()->setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
2473	}
2474
2475	// Finds the set of GUIDs for weak aliasees that are prevailing in different
2476	// modules than any of their aliases. We need to handle these specially.
2477	DenseSet<GlobalValue::GUID>
2478	IndexCallsiteContextGraph::findAliaseeGUIDsPrevailingInDifferentModule() {
2479	DenseSet<GlobalValue::GUID> AliaseeGUIDs;
2480	for (auto &I : Index) {
2481	auto VI = Index.getValueInfo(R: I);
2482	for (auto &S : VI.getSummaryList()) {
2483	// We only care about aliases to functions.
2484	auto *AS = dyn_cast<AliasSummary>(Val: S.get());
2485	if (!AS)
2486	continue;
2487	auto *AliaseeSummary = &AS->getAliasee();
2488	auto *AliaseeFS = dyn_cast<FunctionSummary>(Val: AliaseeSummary);
2489	if (!AliaseeFS)
2490	continue;
2491	// Skip this summary if it is not for the prevailing symbol for this GUID.
2492	// The linker doesn't resolve local linkage values so don't check whether
2493	// those are prevailing.
2494	if (!GlobalValue::isLocalLinkage(Linkage: S ->linkage()) &&
2495	!isPrevailing (VI.getGUID(), S.get()))
2496	continue;
2497	// Prevailing aliasee could be in a different module only if it is weak.
2498	if (!GlobalValue::isWeakForLinker(Linkage: AliaseeSummary->linkage()))
2499	continue;
2500	auto AliaseeGUID = AS->getAliaseeGUID();
2501	// If the aliasee copy in this module is not prevailing, record it.
2502	if (!isPrevailing (AliaseeGUID, AliaseeSummary))
2503	AliaseeGUIDs.insert(V: AliaseeGUID);
2504	}
2505	}
2506	AliaseesPrevailingInDiffModuleFromAlias += AliaseeGUIDs.size();
2507	return AliaseeGUIDs;
2508	}
2509
2510	IndexCallsiteContextGraph::IndexCallsiteContextGraph(
2511	ModuleSummaryIndex &Index,
2512	llvm::function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
2513	isPrevailing)
2514	: Index(Index), isPrevailing (isPrevailing) {
2515	// Since we use the aliasee summary info to create the necessary clones for
2516	// its aliases, conservatively skip recording the aliasee function's callsites
2517	// in the CCG for any that are prevailing in a different module than one of
2518	// its aliases. We could record the necessary information to do this in the
2519	// summary, but this case should not be common.
2520	DenseSet<GlobalValue::GUID> GUIDsToSkip =
2521	findAliaseeGUIDsPrevailingInDifferentModule();
2522	// Map for keeping track of the largest cold contexts up to the number given
2523	// by MemProfTopNImportant. Must be a std::map (not DenseMap) because keys
2524	// must be sorted.
2525	std::map<uint64_t, uint32_t> TotalSizeToContextIdTopNCold;
2526	// Sort by GUID for deterministic graph construction order.
2527	// TODO: This sort has a measurable cost on the thin link when memprof is
2528	// enabled. Investigate gating it behind an option that is only enabled for
2529	// tests that check internal state.
2530	for (const auto &I : Index.sortedGlobalValueSummariesRange()) {
2531	auto VI = Index.getValueInfo(R: I);
2532	if (GUIDsToSkip.contains(V: VI.getGUID()))
2533	continue;
2534	for (auto &S : VI.getSummaryList()) {
2535	// We should only add the prevailing nodes. Otherwise we may try to clone
2536	// in a weak copy that won't be linked (and may be different than the
2537	// prevailing version).
2538	// We only keep the memprof summary on the prevailing copy now when
2539	// building the combined index, as a space optimization, however don't
2540	// rely on this optimization. The linker doesn't resolve local linkage
2541	// values so don't check whether those are prevailing.
2542	if (!GlobalValue::isLocalLinkage(Linkage: S ->linkage()) &&
2543	!isPrevailing (VI.getGUID(), S.get()))
2544	continue;
2545	auto *FS = dyn_cast<FunctionSummary>(Val: S.get());
2546	if (!FS)
2547	continue;
2548	std::vector<CallInfo> CallsWithMetadata;
2549	if (!FS->allocs().empty()) {
2550	for (auto &AN : FS->mutableAllocs()) {
2551	// This can happen because of recursion elimination handling that
2552	// currently exists in ModuleSummaryAnalysis. Skip these for now.
2553	// We still added them to the summary because we need to be able to
2554	// correlate properly in applyImport in the backends.
2555	if (AN.MIBs.empty())
2556	continue;
2557	IndexCall AllocCall(&AN);
2558	CallsWithMetadata.push_back(x: AllocCall);
2559	auto *AllocNode = addAllocNode(Call: AllocCall, F: FS);
2560	// Pass an empty CallStack to the CallsiteContext (second)
2561	// parameter, since for ThinLTO we already collapsed out the inlined
2562	// stack ids on the allocation call during ModuleSummaryAnalysis.
2563	CallStack<MIBInfo, SmallVector<unsigned>::const_iterator>
2564	EmptyContext;
2565	unsigned I = `0`;
2566	assert(!metadataMayIncludeContextSizeInfo() \|\|
2567	AN.ContextSizeInfos.size() == AN.MIBs.size());
2568	// Now add all of the MIBs and their stack nodes.
2569	for (auto &MIB : AN.MIBs) {
2570	CallStack<MIBInfo, SmallVector<unsigned>::const_iterator>
2571	StackContext(&MIB);
2572	std::vector<ContextTotalSize> ContextSizeInfo;
2573	if (!AN.ContextSizeInfos.empty()) {
2574	for (auto [FullStackId, TotalSize] : AN.ContextSizeInfos [I])
2575	ContextSizeInfo.push_back(x: {.FullStackId: FullStackId, .TotalSize: TotalSize});
2576	}
2577	addStackNodesForMIB<MIBInfo, SmallVector<unsigned>::const_iterator>(
2578	AllocNode, StackContext, CallsiteContext&: EmptyContext, AllocType: MIB.AllocType,
2579	ContextSizeInfo, TotalSizeToContextIdTopNCold);
2580	I++;
2581	}
2582	// If exporting the graph to dot and an allocation id of interest was
2583	// specified, record all the context ids for this allocation node.
2584	if (ExportToDot && AllocNode->OrigStackOrAllocId == AllocIdForDot)
2585	DotAllocContextIds = AllocNode->getContextIds();
2586	assert(AllocNode->AllocTypes != (uint8_t)AllocationType::None);
2587	// Initialize version 0 on the summary alloc node to the current alloc
2588	// type, unless it has both types in which case make it default, so
2589	// that in the case where we aren't able to clone the original version
2590	// always ends up with the default allocation behavior.
2591	AN.Versions [`0`] = (uint8_t)allocTypeToUse(AllocTypes: AllocNode->AllocTypes);
2592	}
2593	}
2594	// For callsite metadata, add to list for this function for later use.
2595	if (!FS->callsites().empty())
2596	for (auto &SN : FS->mutableCallsites()) {
2597	IndexCall StackNodeCall(&SN);
2598	CallsWithMetadata.push_back(x: StackNodeCall);
2599	}
2600
2601	if (!CallsWithMetadata.empty())
2602	FuncToCallsWithMetadata [FS] = CallsWithMetadata;
2603
2604	if (!FS->allocs().empty() \|\| !FS->callsites().empty())
2605	FSToVIMap [FS] = VI;
2606	}
2607	}
2608
2609	if (DumpCCG) {
2610	dbgs() << "CCG before updating call stack chains:\n";
2611	dbgs() << *this;
2612	}
2613
2614	if (ExportToDot)
2615	exportToDot(Label: "prestackupdate");
2616
2617	updateStackNodes();
2618
2619	if (ExportToDot)
2620	exportToDot(Label: "poststackupdate");
2621
2622	handleCallsitesWithMultipleTargets();
2623
2624	markBackedges();
2625	}
2626
2627	template <typename DerivedCCG, typename FuncTy, typename CallTy>
2628	void CallsiteContextGraph<DerivedCCG, FuncTy,
2629	CallTy>::handleCallsitesWithMultipleTargets() {
2630	// Look for and workaround callsites that call multiple functions.
2631	// This can happen for indirect calls, which needs better handling, and in
2632	// more rare cases (e.g. macro expansion).
2633	// TODO: To fix this for indirect calls we will want to perform speculative
2634	// devirtualization using either the normal PGO info with ICP, or using the
2635	// information in the profiled MemProf contexts. We can do this prior to
2636	// this transformation for regular LTO, and for ThinLTO we can simulate that
2637	// effect in the summary and perform the actual speculative devirtualization
2638	// while cloning in the ThinLTO backend.
2639
2640	// Keep track of the new nodes synthesized for discovered tail calls missing
2641	// from the profiled contexts.
2642	MapVector<CallInfo, ContextNode *> TailCallToContextNodeMap;
2643
2644	std::vector<std::pair<CallInfo, ContextNode *>> NewCallToNode;
2645	for (auto &Entry : NonAllocationCallToContextNodeMap) {
2646	auto *Node = Entry.second;
2647	assert(Node->Clones.empty());
2648	// Check all node callees and see if in the same function.
2649	// We need to check all of the calls recorded in this Node, because in some
2650	// cases we may have had multiple calls with the same debug info calling
2651	// different callees. This can happen, for example, when an object is
2652	// constructed in the paramter list - the destructor call of the object has
2653	// the same debug info (line/col) as the call the object was passed to.
2654	// Here we will prune any that don't match all callee nodes.
2655	std::vector<CallInfo> AllCalls;
2656	AllCalls.reserve(Node->MatchingCalls.size() + `1`);
2657	AllCalls.push_back(Node->Call);
2658	llvm::append_range(AllCalls, Node->MatchingCalls);
2659
2660	// First see if we can partition the calls by callee function, creating new
2661	// nodes to host each set of calls calling the same callees. This is
2662	// necessary for support indirect calls with ThinLTO, for which we
2663	// synthesized CallsiteInfo records for each target. They will all have the
2664	// same callsite stack ids and would be sharing a context node at this
2665	// point. We need to perform separate cloning for each, which will be
2666	// applied along with speculative devirtualization in the ThinLTO backends
2667	// as needed. Note this does not currently support looking through tail
2668	// calls, it is unclear if we need that for indirect call targets.
2669	// First partition calls by callee func. Map indexed by func, value is
2670	// struct with list of matching calls, assigned node.
2671	if (partitionCallsByCallee(Node, AllCalls, NewCallToNode))
2672	continue;
2673
2674	auto It = AllCalls.begin();
2675	// Iterate through the calls until we find the first that matches.
2676	for (; It != AllCalls.end(); ++It) {
2677	auto ThisCall = *It;
2678	bool Match = true;
2679	for (auto EI = Node->CalleeEdges.begin(); EI != Node->CalleeEdges.end();
2680	++EI) {
2681	auto Edge = *EI;
2682	if (!Edge->Callee->hasCall())
2683	continue;
2684	assert(NodeToCallingFunc.count(Edge->Callee));
2685	// Check if the called function matches that of the callee node.
2686	if (!calleesMatch(Call: ThisCall.call(), EI, TailCallToContextNodeMap)) {
2687	Match = false;
2688	break;
2689	}
2690	}
2691	// Found a call that matches the callee nodes, we can quit now.
2692	if (Match) {
2693	// If the first match is not the primary call on the Node, update it
2694	// now. We will update the list of matching calls further below.
2695	if (Node->Call != ThisCall) {
2696	Node->setCall(ThisCall);
2697	// We need to update the NonAllocationCallToContextNodeMap, but don't
2698	// want to do this during iteration over that map, so save the calls
2699	// that need updated entries.
2700	NewCallToNode.push_back({ThisCall, Node});
2701	}
2702	break;
2703	}
2704	}
2705	// We will update this list below (or leave it cleared if there was no
2706	// match found above).
2707	Node->MatchingCalls.clear();
2708	// If we hit the end of the AllCalls vector, no call matching the callee
2709	// nodes was found, clear the call information in the node.
2710	if (It == AllCalls.end()) {
2711	RemovedEdgesWithMismatchedCallees ++;
2712	// Work around by setting Node to have a null call, so it gets
2713	// skipped during cloning. Otherwise assignFunctions will assert
2714	// because its data structures are not designed to handle this case.
2715	Node->setCall(CallInfo());
2716	continue;
2717	}
2718	// Now add back any matching calls that call the same function as the
2719	// matching primary call on Node.
2720	for (++It; It != AllCalls.end(); ++It) {
2721	auto ThisCall = *It;
2722	if (!sameCallee(Call1: Node->Call.call(), Call2: ThisCall.call()))
2723	continue;
2724	Node->MatchingCalls.push_back(ThisCall);
2725	}
2726	}
2727
2728	// Remove all mismatched nodes identified in the above loop from the node map
2729	// (checking whether they have a null call which is set above). For a
2730	// MapVector like NonAllocationCallToContextNodeMap it is much more efficient
2731	// to do the removal via remove_if than by individually erasing entries above.
2732	// Also remove any entries if we updated the node's primary call above.
2733	NonAllocationCallToContextNodeMap.remove_if([](const auto &it) {
2734	return !it.second->hasCall() \|\| it.second->Call != it.first;
2735	});
2736
2737	// Add entries for any new primary calls recorded above.
2738	for (auto &[Call, Node] : NewCallToNode)
2739	NonAllocationCallToContextNodeMap[Call] = Node;
2740
2741	// Add the new nodes after the above loop so that the iteration is not
2742	// invalidated.
2743	for (auto &[Call, Node] : TailCallToContextNodeMap)
2744	NonAllocationCallToContextNodeMap[Call] = Node;
2745	}
2746
2747	template <typename DerivedCCG, typename FuncTy, typename CallTy>
2748	bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::partitionCallsByCallee(
2749	ContextNode *Node, ArrayRef<CallInfo> AllCalls,
2750	std::vector<std::pair<CallInfo, ContextNode *>> &NewCallToNode) {
2751	// Struct to keep track of all the calls having the same callee function,
2752	// and the node we eventually assign to them. Eventually we will record the
2753	// context node assigned to this group of calls.
2754	struct CallsWithSameCallee {
2755	std::vector<CallInfo> Calls;
2756	ContextNode Node = nullptr*;
2757	};
2758
2759	// First partition calls by callee function. Build map from each function
2760	// to the list of matching calls.
2761	DenseMap<const FuncTy *, CallsWithSameCallee> CalleeFuncToCallInfo;
2762	for (auto ThisCall : AllCalls) {
2763	auto *F = getCalleeFunc(Call: ThisCall.call());
2764	if (F)
2765	CalleeFuncToCallInfo[F].Calls.push_back(ThisCall);
2766	}
2767
2768	// Next, walk through all callee edges. For each callee node, get its
2769	// containing function and see if it was recorded in the above map (meaning we
2770	// have at least one matching call). Build another map from each callee node
2771	// with a matching call to the structure instance created above containing all
2772	// the calls.
2773	DenseMap<ContextNode , CallsWithSameCallee > CalleeNodeToCallInfo;
2774	for (const auto &Edge : Node->CalleeEdges) {
2775	if (!Edge->Callee->hasCall())
2776	continue;
2777	const FuncTy *ProfiledCalleeFunc = NodeToCallingFunc[Edge->Callee];
2778	if (CalleeFuncToCallInfo.contains(ProfiledCalleeFunc))
2779	CalleeNodeToCallInfo[Edge->Callee] =
2780	&CalleeFuncToCallInfo[ProfiledCalleeFunc];
2781	}
2782
2783	// If there are entries in the second map, then there were no matching
2784	// calls/callees, nothing to do here. Return so we can go to the handling that
2785	// looks through tail calls.
2786	if (CalleeNodeToCallInfo.empty())
2787	return false;
2788
2789	// Walk through all callee edges again. Any and all callee edges that didn't
2790	// match any calls (callee not in the CalleeNodeToCallInfo map) are moved to a
2791	// new caller node (UnmatchedCalleesNode) which gets a null call so that it is
2792	// ignored during cloning. If it is in the map, then we use the node recorded
2793	// in that entry (creating it if needed), and move the callee edge to it.
2794	// The first callee will use the original node instead of creating a new one.
2795	// Note that any of the original calls on this node (in AllCalls) that didn't
2796	// have a callee function automatically get dropped from the node as part of
2797	// this process.
2798	ContextNode UnmatchedCalleesNode = nullptr*;
2799	// Track whether we already assigned original node to a callee.
2800	bool UsedOrigNode = false;
2801	assert(NodeToCallingFunc[Node]);
2802	// Iterate over a copy of Node's callee edges, since we may need to remove
2803	// edges in moveCalleeEdgeToNewCaller, and this simplifies the handling and
2804	// makes it less error-prone.
2805	auto CalleeEdges = Node->CalleeEdges;
2806	for (auto &Edge : CalleeEdges) {
2807	if (!Edge->Callee->hasCall())
2808	continue;
2809
2810	// Will be updated below to point to whatever (caller) node this callee edge
2811	// should be moved to.
2812	ContextNode CallerNodeToUse = nullptr*;
2813
2814	// Handle the case where there were no matching calls first. Move this
2815	// callee edge to the UnmatchedCalleesNode, creating it if needed.
2816	if (!CalleeNodeToCallInfo.contains(Edge->Callee)) {
2817	if (!UnmatchedCalleesNode)
2818	UnmatchedCalleesNode =
2819	createNewNode(/IsAllocation=/false, F: NodeToCallingFunc[Node]);
2820	CallerNodeToUse = UnmatchedCalleesNode;
2821	} else {
2822	// Look up the information recorded for this callee node, and use the
2823	// recorded caller node (creating it if needed).
2824	auto *Info = CalleeNodeToCallInfo[Edge->Callee];
2825	if (!Info->Node) {
2826	// If we haven't assigned any callees to the original node use it.
2827	if (!UsedOrigNode) {
2828	Info->Node = Node;
2829	// Clear the set of matching calls which will be updated below.
2830	Node->MatchingCalls.clear();
2831	UsedOrigNode = true;
2832	} else
2833	Info->Node =
2834	createNewNode(/IsAllocation=/false, F: NodeToCallingFunc[Node]);
2835	assert(!Info->Calls.empty());
2836	// The first call becomes the primary call for this caller node, and the
2837	// rest go in the matching calls list.
2838	Info->Node->setCall(Info->Calls.front());
2839	llvm::append_range(Info->Node->MatchingCalls,
2840	llvm::drop_begin(Info->Calls));
2841	// Save the primary call to node correspondence so that we can update
2842	// the NonAllocationCallToContextNodeMap, which is being iterated in the
2843	// caller of this function.
2844	NewCallToNode.push_back({Info->Node->Call, Info->Node});
2845	}
2846	CallerNodeToUse = Info->Node;
2847	}
2848
2849	// Don't need to move edge if we are using the original node;
2850	if (CallerNodeToUse == Node)
2851	continue;
2852
2853	moveCalleeEdgeToNewCaller(Edge, NewCaller: CallerNodeToUse);
2854	}
2855	// Now that we are done moving edges, clean up any caller edges that ended
2856	// up with no type or context ids. During moveCalleeEdgeToNewCaller all
2857	// caller edges from Node are replicated onto the new callers, and it
2858	// simplifies the handling to leave them until we have moved all
2859	// edges/context ids.
2860	for (auto &I : CalleeNodeToCallInfo)
2861	removeNoneTypeCallerEdges(Node: I.second->Node);
2862	if (UnmatchedCalleesNode)
2863	removeNoneTypeCallerEdges(Node: UnmatchedCalleesNode);
2864	removeNoneTypeCallerEdges(Node);
2865
2866	return true;
2867	}
2868
2869	uint64_t ModuleCallsiteContextGraph::getStackId(uint64_t IdOrIndex) const {
2870	// In the Module (IR) case this is already the Id.
2871	return IdOrIndex;
2872	}
2873
2874	uint64_t IndexCallsiteContextGraph::getStackId(uint64_t IdOrIndex) const {
2875	// In the Index case this is an index into the stack id list in the summary
2876	// index, convert it to an Id.
2877	return Index.getStackIdAtIndex(Index: IdOrIndex);
2878	}
2879
2880	template <typename DerivedCCG, typename FuncTy, typename CallTy>
2881	bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::calleesMatch(
2882	CallTy Call, EdgeIter &EI,
2883	MapVector<CallInfo, ContextNode *> &TailCallToContextNodeMap) {
2884	auto Edge = *EI;
2885	const FuncTy *ProfiledCalleeFunc = NodeToCallingFunc[Edge->Callee];
2886	const FuncTy *CallerFunc = NodeToCallingFunc[Edge->Caller];
2887	// Will be populated in order of callee to caller if we find a chain of tail
2888	// calls between the profiled caller and callee.
2889	std::vector<std::pair<CallTy, FuncTy *>> FoundCalleeChain;
2890	if (!calleeMatchesFunc(Call, Func: ProfiledCalleeFunc, CallerFunc,
2891	FoundCalleeChain))
2892	return false;
2893
2894	// The usual case where the profiled callee matches that of the IR/summary.
2895	if (FoundCalleeChain.empty())
2896	return true;
2897
2898	auto AddEdge = [Edge, &EI](ContextNode Caller, ContextNode Callee) {
2899	auto *CurEdge = Callee->findEdgeFromCaller(Caller);
2900	// If there is already an edge between these nodes, simply update it and
2901	// return.
2902	if (CurEdge) {
2903	CurEdge->ContextIds.insert_range(Edge->ContextIds);
2904	CurEdge->AllocTypes \|= Edge->AllocTypes;
2905	return;
2906	}
2907	// Otherwise, create a new edge and insert it into the caller and callee
2908	// lists.
2909	auto NewEdge = std::make_shared<ContextEdge>(
2910	Callee, Caller, Edge->AllocTypes, Edge->ContextIds);
2911	Callee->CallerEdges.push_back(NewEdge);
2912	if (Caller == Edge->Caller) {
2913	// If we are inserting the new edge into the current edge's caller, insert
2914	// the new edge before the current iterator position, and then increment
2915	// back to the current edge.
2916	EI = Caller->CalleeEdges.insert(EI, NewEdge);
2917	++EI;
2918	assert(*EI == Edge &&
2919	"Iterator position not restored after insert and increment");
2920	} else
2921	Caller->CalleeEdges.push_back(NewEdge);
2922	};
2923
2924	// Create new nodes for each found callee and connect in between the profiled
2925	// caller and callee.
2926	auto *CurCalleeNode = Edge->Callee;
2927	for (auto &[NewCall, Func] : FoundCalleeChain) {
2928	ContextNode NewNode = nullptr*;
2929	// First check if we have already synthesized a node for this tail call.
2930	if (TailCallToContextNodeMap.count(NewCall)) {
2931	NewNode = TailCallToContextNodeMap[NewCall];
2932	NewNode->AllocTypes \|= Edge->AllocTypes;
2933	} else {
2934	FuncToCallsWithMetadata[Func].push_back({NewCall});
2935	// Create Node and record node info.
2936	NewNode = createNewNode(/IsAllocation=/false, F: Func, C: NewCall);
2937	TailCallToContextNodeMap[NewCall] = NewNode;
2938	NewNode->AllocTypes = Edge->AllocTypes;
2939	}
2940
2941	// Hook up node to its callee node
2942	AddEdge(NewNode, CurCalleeNode);
2943
2944	CurCalleeNode = NewNode;
2945	}
2946
2947	// Hook up edge's original caller to new callee node.
2948	AddEdge(Edge->Caller, CurCalleeNode);
2949
2950	#ifndef NDEBUG
2951	// Save this because Edge's fields get cleared below when removed.
2952	auto *Caller = Edge->Caller;
2953	#endif
2954
2955	// Remove old edge
2956	removeEdgeFromGraph(Edge: Edge.get(), EI: &EI, /CalleeIter=/true);
2957
2958	// To simplify the increment of EI in the caller, subtract one from EI.
2959	// In the final AddEdge call we would have either added a new callee edge,
2960	// to Edge->Caller, or found an existing one. Either way we are guaranteed
2961	// that there is at least one callee edge.
2962	assert(!Caller->CalleeEdges.empty());
2963	--EI;
2964
2965	return true;
2966	}
2967
2968	bool ModuleCallsiteContextGraph::findProfiledCalleeThroughTailCalls(
2969	const Function ProfiledCallee, Value CurCallee, unsigned Depth,
2970	std::vector<std::pair<Instruction , Function >> &FoundCalleeChain,
2971	bool &FoundMultipleCalleeChains) {
2972	// Stop recursive search if we have already explored the maximum specified
2973	// depth.
2974	if (Depth > TailCallSearchDepth)
2975	return false;
2976
2977	auto SaveCallsiteInfo = [&](Instruction Callsite, Function F) {
2978	FoundCalleeChain.push_back(x: {Callsite, F});
2979	};
2980
2981	auto *CalleeFunc = dyn_cast<Function>(Val: CurCallee);
2982	if (!CalleeFunc) {
2983	auto *Alias = dyn_cast<GlobalAlias>(Val: CurCallee);
2984	assert(Alias);
2985	CalleeFunc = dyn_cast<Function>(Val: Alias->getAliasee());
2986	assert(CalleeFunc);
2987	}
2988
2989	// Look for tail calls in this function, and check if they either call the
2990	// profiled callee directly, or indirectly (via a recursive search).
2991	// Only succeed if there is a single unique tail call chain found between the
2992	// profiled caller and callee, otherwise we could perform incorrect cloning.
2993	bool FoundSingleCalleeChain = false;
2994	for (auto &BB : *CalleeFunc) {
2995	for (auto &I : BB) {
2996	auto *CB = dyn_cast<CallBase>(Val: &I);
2997	if (!CB \|\| !CB->isTailCall())
2998	continue;
2999	auto *CalledValue = CB->getCalledOperand();
3000	auto *CalledFunction = CB->getCalledFunction();
3001	if (CalledValue && !CalledFunction) {
3002	CalledValue = CalledValue->stripPointerCasts();
3003	// Stripping pointer casts can reveal a called function.
3004	CalledFunction = dyn_cast<Function>(Val: CalledValue);
3005	}
3006	// Check if this is an alias to a function. If so, get the
3007	// called aliasee for the checks below.
3008	if (auto *GA = dyn_cast<GlobalAlias>(Val: CalledValue)) {
3009	assert(!CalledFunction &&
3010	"Expected null called function in callsite for alias");
3011	CalledFunction = dyn_cast<Function>(Val: GA->getAliaseeObject());
3012	}
3013	if (!CalledFunction)
3014	continue;
3015	if (CalledFunction == ProfiledCallee) {
3016	if (FoundSingleCalleeChain) {
3017	FoundMultipleCalleeChains = true;
3018	return false;
3019	}
3020	FoundSingleCalleeChain = true;
3021	FoundProfiledCalleeCount ++;
3022	FoundProfiledCalleeDepth += Depth;
3023	if (Depth > FoundProfiledCalleeMaxDepth)
3024	FoundProfiledCalleeMaxDepth = Depth;
3025	SaveCallsiteInfo (&I, CalleeFunc);
3026	} else if (findProfiledCalleeThroughTailCalls(
3027	ProfiledCallee, CurCallee: CalledFunction, Depth: Depth + `1`,
3028	FoundCalleeChain, FoundMultipleCalleeChains)) {
3029	// findProfiledCalleeThroughTailCalls should not have returned
3030	// true if FoundMultipleCalleeChains.
3031	assert(!FoundMultipleCalleeChains);
3032	if (FoundSingleCalleeChain) {
3033	FoundMultipleCalleeChains = true;
3034	return false;
3035	}
3036	FoundSingleCalleeChain = true;
3037	SaveCallsiteInfo (&I, CalleeFunc);
3038	} else if (FoundMultipleCalleeChains)
3039	return false;
3040	}
3041	}
3042
3043	return FoundSingleCalleeChain;
3044	}
3045
3046	const Function ModuleCallsiteContextGraph::getCalleeFunc(Instruction Call) {
3047	auto *CB = dyn_cast<CallBase>(Val: Call);
3048	if (!CB->getCalledOperand() \|\| CB->isIndirectCall())
3049	return nullptr;
3050	auto *CalleeVal = CB->getCalledOperand()->stripPointerCasts();
3051	auto *Alias = dyn_cast<GlobalAlias>(Val: CalleeVal);
3052	if (Alias)
3053	return dyn_cast<Function>(Val: Alias->getAliasee());
3054	return dyn_cast<Function>(Val: CalleeVal);
3055	}
3056
3057	bool ModuleCallsiteContextGraph::calleeMatchesFunc(
3058	Instruction Call, const* Function Func, const* Function *CallerFunc,
3059	std::vector<std::pair<Instruction , Function >> &FoundCalleeChain) {
3060	auto *CB = dyn_cast<CallBase>(Val: Call);
3061	if (!CB->getCalledOperand() \|\| CB->isIndirectCall())
3062	return false;
3063	auto *CalleeVal = CB->getCalledOperand()->stripPointerCasts();
3064	auto *CalleeFunc = dyn_cast<Function>(Val: CalleeVal);
3065	if (CalleeFunc == Func)
3066	return true;
3067	auto *Alias = dyn_cast<GlobalAlias>(Val: CalleeVal);
3068	if (Alias && Alias->getAliasee() == Func)
3069	return true;
3070
3071	// Recursively search for the profiled callee through tail calls starting with
3072	// the actual Callee. The discovered tail call chain is saved in
3073	// FoundCalleeChain, and we will fixup the graph to include these callsites
3074	// after returning.
3075	// FIXME: We will currently redo the same recursive walk if we find the same
3076	// mismatched callee from another callsite. We can improve this with more
3077	// bookkeeping of the created chain of new nodes for each mismatch.
3078	unsigned Depth = `1`;
3079	bool FoundMultipleCalleeChains = false;
3080	if (!findProfiledCalleeThroughTailCalls(ProfiledCallee: Func, CurCallee: CalleeVal, Depth,
3081	FoundCalleeChain,
3082	FoundMultipleCalleeChains)) {
3083	LLVM_DEBUG(dbgs() << "Not found through unique tail call chain: "
3084	<< Func->getName() << " from " << CallerFunc->getName()
3085	<< " that actually called " << CalleeVal->getName()
3086	<< (FoundMultipleCalleeChains
3087	? " (found multiple possible chains)"
3088	: "")
3089	<< "\n");
3090	if (FoundMultipleCalleeChains)
3091	FoundProfiledCalleeNonUniquelyCount ++;
3092	return false;
3093	}
3094
3095	return true;
3096	}
3097
3098	bool ModuleCallsiteContextGraph::sameCallee(Instruction *Call1,
3099	Instruction *Call2) {
3100	auto *CB1 = cast<CallBase>(Val: Call1);
3101	if (!CB1->getCalledOperand() \|\| CB1->isIndirectCall())
3102	return false;
3103	auto *CalleeVal1 = CB1->getCalledOperand()->stripPointerCasts();
3104	auto *CalleeFunc1 = dyn_cast<Function>(Val: CalleeVal1);
3105	auto *CB2 = cast<CallBase>(Val: Call2);
3106	if (!CB2->getCalledOperand() \|\| CB2->isIndirectCall())
3107	return false;
3108	auto *CalleeVal2 = CB2->getCalledOperand()->stripPointerCasts();
3109	auto *CalleeFunc2 = dyn_cast<Function>(Val: CalleeVal2);
3110	return CalleeFunc1 == CalleeFunc2;
3111	}
3112
3113	bool IndexCallsiteContextGraph::findProfiledCalleeThroughTailCalls(
3114	ValueInfo ProfiledCallee, ValueInfo CurCallee, unsigned Depth,
3115	std::vector<std::pair<IndexCall, FunctionSummary *>> &FoundCalleeChain,
3116	bool &FoundMultipleCalleeChains) {
3117	// Stop recursive search if we have already explored the maximum specified
3118	// depth.
3119	if (Depth > TailCallSearchDepth)
3120	return false;
3121
3122	auto CreateAndSaveCallsiteInfo = [&](ValueInfo Callee, FunctionSummary *FS) {
3123	// Make a CallsiteInfo for each discovered callee, if one hasn't already
3124	// been synthesized.
3125	if (!FunctionCalleesToSynthesizedCallsiteInfos.count(Val: FS) \|\|
3126	!FunctionCalleesToSynthesizedCallsiteInfos [FS].count(x: Callee))
3127	// StackIds is empty (we don't have debug info available in the index for
3128	// these callsites)
3129	FunctionCalleesToSynthesizedCallsiteInfos [FS][Callee] =
3130	std::make_unique<CallsiteInfo>(args&: Callee, args: SmallVector<unsigned>());
3131	CallsiteInfo *NewCallsiteInfo =
3132	FunctionCalleesToSynthesizedCallsiteInfos [FS][Callee].get();
3133	FoundCalleeChain.push_back(x: {NewCallsiteInfo, FS});
3134	};
3135
3136	// Look for tail calls in this function, and check if they either call the
3137	// profiled callee directly, or indirectly (via a recursive search).
3138	// Only succeed if there is a single unique tail call chain found between the
3139	// profiled caller and callee, otherwise we could perform incorrect cloning.
3140	bool FoundSingleCalleeChain = false;
3141	for (auto &S : CurCallee.getSummaryList()) {
3142	if (!GlobalValue::isLocalLinkage(Linkage: S ->linkage()) &&
3143	!isPrevailing (CurCallee.getGUID(), S.get()))
3144	continue;
3145	auto *FS = dyn_cast<FunctionSummary>(Val: S ->getBaseObject());
3146	if (!FS)
3147	continue;
3148	auto FSVI = CurCallee;
3149	auto *AS = dyn_cast<AliasSummary>(Val: S.get());
3150	if (AS)
3151	FSVI = AS->getAliaseeVI();
3152	for (auto &CallEdge : FS->calls()) {
3153	if (!CallEdge.second.hasTailCall())
3154	continue;
3155	if (CallEdge.first == ProfiledCallee) {
3156	if (FoundSingleCalleeChain) {
3157	FoundMultipleCalleeChains = true;
3158	return false;
3159	}
3160	FoundSingleCalleeChain = true;
3161	FoundProfiledCalleeCount ++;
3162	FoundProfiledCalleeDepth += Depth;
3163	if (Depth > FoundProfiledCalleeMaxDepth)
3164	FoundProfiledCalleeMaxDepth = Depth;
3165	CreateAndSaveCallsiteInfo (CallEdge.first, FS);
3166	// Add FS to FSToVIMap in case it isn't already there.
3167	assert(!FSToVIMap.count(FS) \|\| FSToVIMap[FS] == FSVI);
3168	FSToVIMap [FS] = FSVI;
3169	} else if (findProfiledCalleeThroughTailCalls(
3170	ProfiledCallee, CurCallee: CallEdge.first, Depth: Depth + `1`,
3171	FoundCalleeChain, FoundMultipleCalleeChains)) {
3172	// findProfiledCalleeThroughTailCalls should not have returned
3173	// true if FoundMultipleCalleeChains.
3174	assert(!FoundMultipleCalleeChains);
3175	if (FoundSingleCalleeChain) {
3176	FoundMultipleCalleeChains = true;
3177	return false;
3178	}
3179	FoundSingleCalleeChain = true;
3180	CreateAndSaveCallsiteInfo (CallEdge.first, FS);
3181	// Add FS to FSToVIMap in case it isn't already there.
3182	assert(!FSToVIMap.count(FS) \|\| FSToVIMap[FS] == FSVI);
3183	FSToVIMap [FS] = FSVI;
3184	} else if (FoundMultipleCalleeChains)
3185	return false;
3186	}
3187	}
3188
3189	return FoundSingleCalleeChain;
3190	}
3191
3192	const FunctionSummary *
3193	IndexCallsiteContextGraph::getCalleeFunc(IndexCall &Call) {
3194	ValueInfo Callee = dyn_cast_if_present<CallsiteInfo *>(Val&: Call)->Callee;
3195	if (Callee.getSummaryList().empty())
3196	return nullptr;
3197	return dyn_cast<FunctionSummary>(Val: Callee.getSummaryList()[`0`]->getBaseObject());
3198	}
3199
3200	bool IndexCallsiteContextGraph::calleeMatchesFunc(
3201	IndexCall &Call, const FunctionSummary *Func,
3202	const FunctionSummary *CallerFunc,
3203	std::vector<std::pair<IndexCall, FunctionSummary *>> &FoundCalleeChain) {
3204	ValueInfo Callee = dyn_cast_if_present<CallsiteInfo *>(Val&: Call)->Callee;
3205	// If there is no summary list then this is a call to an externally defined
3206	// symbol.
3207	AliasSummary *Alias =
3208	Callee.getSummaryList().empty()
3209	? nullptr
3210	: dyn_cast<AliasSummary>(Val: Callee.getSummaryList()[`0`].get());
3211	assert(FSToVIMap.count(Func));
3212	auto FuncVI = FSToVIMap [Func];
3213	if (Callee == FuncVI \|\|
3214	// If callee is an alias, check the aliasee, since only function
3215	// summary base objects will contain the stack node summaries and thus
3216	// get a context node.
3217	(Alias && Alias->getAliaseeVI() == FuncVI))
3218	return true;
3219
3220	// Recursively search for the profiled callee through tail calls starting with
3221	// the actual Callee. The discovered tail call chain is saved in
3222	// FoundCalleeChain, and we will fixup the graph to include these callsites
3223	// after returning.
3224	// FIXME: We will currently redo the same recursive walk if we find the same
3225	// mismatched callee from another callsite. We can improve this with more
3226	// bookkeeping of the created chain of new nodes for each mismatch.
3227	unsigned Depth = `1`;
3228	bool FoundMultipleCalleeChains = false;
3229	if (!findProfiledCalleeThroughTailCalls(
3230	ProfiledCallee: FuncVI, CurCallee: Callee, Depth, FoundCalleeChain, FoundMultipleCalleeChains)) {
3231	LLVM_DEBUG(dbgs() << "Not found through unique tail call chain: " << FuncVI
3232	<< " from " << FSToVIMap[CallerFunc]
3233	<< " that actually called " << Callee
3234	<< (FoundMultipleCalleeChains
3235	? " (found multiple possible chains)"
3236	: "")
3237	<< "\n");
3238	if (FoundMultipleCalleeChains)
3239	FoundProfiledCalleeNonUniquelyCount ++;
3240	return false;
3241	}
3242
3243	return true;
3244	}
3245
3246	bool IndexCallsiteContextGraph::sameCallee(IndexCall &Call1, IndexCall &Call2) {
3247	ValueInfo Callee1 = dyn_cast_if_present<CallsiteInfo *>(Val&: Call1)->Callee;
3248	ValueInfo Callee2 = dyn_cast_if_present<CallsiteInfo *>(Val&: Call2)->Callee;
3249	return Callee1 == Callee2;
3250	}
3251
3252	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3253	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::dump()
3254	const {
3255	print(OS&: dbgs());
3256	dbgs() << "\n";
3257	}
3258
3259	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3260	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::print(
3261	raw_ostream &OS) const {
3262	OS << "Node " << this << "\n";
3263	OS << "\t";
3264	printCall(OS);
3265	if (Recursive)
3266	OS << " (recursive)";
3267	OS << "\n";
3268	if (!MatchingCalls.empty()) {
3269	OS << "\tMatchingCalls:\n";
3270	for (auto &MatchingCall : MatchingCalls) {
3271	OS << "\t";
3272	MatchingCall.print(OS);
3273	OS << "\n";
3274	}
3275	}
3276	OS << "\tNodeId: " << NodeId << "\n";
3277	OS << "\tAllocTypes: " << getAllocTypeString(AllocTypes) << "\n";
3278	OS << "\tContextIds:";
3279	// Make a copy of the computed context ids that we can sort for stability.
3280	auto ContextIds = getContextIds();
3281	std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
3282	std::sort(first: SortedIds.begin(), last: SortedIds.end());
3283	for (auto Id : SortedIds)
3284	OS << " " << Id;
3285	OS << "\n";
3286	OS << "\tCalleeEdges:\n";
3287	for (auto &Edge : CalleeEdges)
3288	OS << "\t\t" << *Edge << " (Callee NodeId: " << Edge->Callee->NodeId
3289	<< ")\n";
3290	OS << "\tCallerEdges:\n";
3291	for (auto &Edge : CallerEdges)
3292	OS << "\t\t" << *Edge << " (Caller NodeId: " << Edge->Caller->NodeId
3293	<< ")\n";
3294	if (!Clones.empty()) {
3295	OS << "\tClones: ";
3296	ListSeparator LS;
3297	for (auto *C : Clones)
3298	OS << LS << C << " NodeId: " << C->NodeId;
3299	OS << "\n";
3300	} else if (CloneOf) {
3301	OS << "\tClone of " << CloneOf << " NodeId: " << CloneOf->NodeId << "\n";
3302	}
3303	}
3304
3305	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3306	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge::dump()
3307	const {
3308	print(OS&: dbgs());
3309	dbgs() << "\n";
3310	}
3311
3312	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3313	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge::print(
3314	raw_ostream &OS) const {
3315	OS << "Edge from Callee " << Callee << " to Caller: " << Caller
3316	<< (IsBackedge ? " (BE)" : "")
3317	<< " AllocTypes: " << getAllocTypeString(AllocTypes);
3318	OS << " ContextIds:";
3319	std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
3320	std::sort(first: SortedIds.begin(), last: SortedIds.end());
3321	for (auto Id : SortedIds)
3322	OS << " " << Id;
3323	}
3324
3325	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3326	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::dump() const {
3327	print(OS&: dbgs());
3328	}
3329
3330	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3331	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::print(
3332	raw_ostream &OS) const {
3333	OS << "Callsite Context Graph:\n";
3334	using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
3335	for (const auto Node : nodes<GraphType>(this)) {
3336	if (Node->isRemoved())
3337	continue;
3338	Node->print(OS);
3339	OS << "\n";
3340	}
3341	}
3342
3343	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3344	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::printTotalSizes(
3345	raw_ostream &OS,
3346	function_ref<void(StringRef, StringRef, const Twine &)> EmitRemark) const {
3347	using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
3348	for (const auto Node : nodes<GraphType>(this)) {
3349	if (Node->isRemoved())
3350	continue;
3351	if (!Node->IsAllocation)
3352	continue;
3353	DenseSet<uint32_t> ContextIds = Node->getContextIds();
3354	auto AllocTypeFromCall = getAllocationCallType(Call: Node->Call);
3355	std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
3356	std::sort(first: SortedIds.begin(), last: SortedIds.end());
3357	for (auto Id : SortedIds) {
3358	auto TypeI = ContextIdToAllocationType.find(Val: Id);
3359	assert(TypeI != ContextIdToAllocationType.end());
3360	auto CSI = ContextIdToContextSizeInfos.find(Val: Id);
3361	if (CSI != ContextIdToContextSizeInfos.end()) {
3362	for (auto &Info : CSI ->second) {
3363	std::string Msg =
3364	"MemProf hinting: " + getAllocTypeString(AllocTypes: (uint8_t)TypeI ->second) +
3365	" full allocation context " + std::to_string(val: Info.FullStackId) +
3366	" with total size " + std::to_string(val: Info.TotalSize) + " is " +
3367	getAllocTypeString(Node->AllocTypes) + " after cloning";
3368	if (allocTypeToUse(Node->AllocTypes) != AllocTypeFromCall)
3369	Msg += " marked " + getAllocTypeString(AllocTypes: (uint8_t)AllocTypeFromCall) +
3370	" due to cold byte percent";
3371	// Print the internal context id to aid debugging and visualization.
3372	Msg += " (internal context id " + std::to_string(val: Id) + ")";
3373	if (MemProfReportHintedSizes)
3374	OS << Msg << "\n";
3375	if (EmitRemark)
3376	EmitRemark (DEBUG_TYPE, "MemProfReport", Msg);
3377	}
3378	} else {
3379	// This is only emitted if the context size info is not present.
3380	std::string Msg =
3381	"MemProf hinting: " + getAllocTypeString(AllocTypes: (uint8_t)TypeI ->second) +
3382	" context is " + getAllocTypeString(Node->AllocTypes) +
3383	" after cloning";
3384	if (allocTypeToUse(Node->AllocTypes) != AllocTypeFromCall)
3385	Msg += " marked " + getAllocTypeString(AllocTypes: (uint8_t)AllocTypeFromCall) +
3386	" due to cold byte percent";
3387	// Print the internal context id to aid debugging and visualization.
3388	Msg += " (internal context id " + std::to_string(val: Id) + ")";
3389	if (MemProfReportHintedSizes)
3390	OS << Msg << "\n";
3391	if (EmitRemark)
3392	EmitRemark (DEBUG_TYPE, "MemProfReport", Msg);
3393	}
3394	}
3395	}
3396	}
3397
3398	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3399	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::check() const {
3400	using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
3401	for (const auto Node : nodes<GraphType>(this)) {
3402	checkNode<DerivedCCG, FuncTy, CallTy>(Node, /CheckEdges=/false);
3403	for (auto &Edge : Node->CallerEdges)
3404	checkEdge<DerivedCCG, FuncTy, CallTy>(Edge);
3405	}
3406	}
3407
3408	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3409	struct GraphTraits<const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *> {
3410	using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
3411	using NodeRef = const ContextNode<DerivedCCG, FuncTy, CallTy> *;
3412
3413	using NodePtrTy = std::unique_ptr<ContextNode<DerivedCCG, FuncTy, CallTy>>;
3414	static NodeRef getNode(const NodePtrTy &P) { return P.get(); }
3415
3416	using nodes_iterator =
3417	mapped_iterator<typename std::vector<NodePtrTy>::const_iterator,
3418	decltype(&getNode)>;
3419
3420	static nodes_iterator nodes_begin(GraphType G) {
3421	return nodes_iterator(G->NodeOwner.begin(), &getNode);
3422	}
3423
3424	static nodes_iterator nodes_end(GraphType G) {
3425	return nodes_iterator(G->NodeOwner.end(), &getNode);
3426	}
3427
3428	static NodeRef getEntryNode(GraphType G) {
3429	return G->NodeOwner.begin()->get();
3430	}
3431
3432	using EdgePtrTy = std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>>;
3433	static const ContextNode<DerivedCCG, FuncTy, CallTy> *
3434	GetCallee(const EdgePtrTy &P) {
3435	return P->Callee;
3436	}
3437
3438	using ChildIteratorType =
3439	mapped_iterator<typename std::vector<std::shared_ptr<ContextEdge<
3440	DerivedCCG, FuncTy, CallTy>>>::const_iterator,
3441	decltype(&GetCallee)>;
3442
3443	static ChildIteratorType child_begin(NodeRef N) {
3444	return ChildIteratorType(N->CalleeEdges.begin(), &GetCallee);
3445	}
3446
3447	static ChildIteratorType child_end(NodeRef N) {
3448	return ChildIteratorType(N->CalleeEdges.end(), &GetCallee);
3449	}
3450	};
3451
3452	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3453	struct DOTGraphTraits<const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>
3454	: public DefaultDOTGraphTraits {
3455	DOTGraphTraits(bool IsSimple = false) : DefaultDOTGraphTraits(IsSimple) {
3456	// If the user requested the full graph to be exported, but provided an
3457	// allocation id, or if the user gave a context id and requested more than
3458	// just a specific context to be exported, note that highlighting is
3459	// enabled.
3460	DoHighlight =
3461	(AllocIdForDot.getNumOccurrences() && DotGraphScope == DotScope::All) \|\|
3462	(ContextIdForDot.getNumOccurrences() &&
3463	DotGraphScope != DotScope::Context);
3464	}
3465
3466	using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
3467	using GTraits = GraphTraits<GraphType>;
3468	using NodeRef = typename GTraits::NodeRef;
3469	using ChildIteratorType = typename GTraits::ChildIteratorType;
3470
3471	static std::string getNodeLabel(NodeRef Node, GraphType G) {
3472	std::string LabelString =
3473	(Twine ("OrigId: ") + (Node->IsAllocation ? "Alloc" : "") +
3474	Twine(Node->OrigStackOrAllocId) + " NodeId: " + Twine(Node->NodeId))
3475	.str();
3476	LabelString += "\n";
3477	if (Node->hasCall()) {
3478	auto Func = G->NodeToCallingFunc.find(Node);
3479	assert(Func != G->NodeToCallingFunc.end());
3480	LabelString +=
3481	G->getLabel(Func->second, Node->Call.call(), Node->Call.cloneNo());
3482	for (auto &MatchingCall : Node->MatchingCalls) {
3483	LabelString += "\n";
3484	LabelString += G->getLabel(Func->second, MatchingCall.call(),
3485	MatchingCall.cloneNo());
3486	}
3487	} else {
3488	LabelString += "null call";
3489	if (Node->Recursive)
3490	LabelString += " (recursive)";
3491	else
3492	LabelString += " (external)";
3493	}
3494	return LabelString;
3495	}
3496
3497	static std::string getNodeAttributes(NodeRef Node, GraphType G) {
3498	auto ContextIds = Node->getContextIds();
3499	// If highlighting enabled, see if this node contains any of the context ids
3500	// of interest. If so, it will use a different color and a larger fontsize
3501	// (which makes the node larger as well).
3502	bool Highlight = false;
3503	if (DoHighlight) {
3504	assert(ContextIdForDot.getNumOccurrences() \|\|
3505	AllocIdForDot.getNumOccurrences());
3506	if (ContextIdForDot.getNumOccurrences())
3507	Highlight = ContextIds.contains(ContextIdForDot);
3508	else
3509	Highlight = set_intersects(ContextIds, G->DotAllocContextIds);
3510	}
3511	std::string AttributeString = (Twine ("tooltip=\"") + getNodeId(Node) + " " +
3512	getContextIds(ContextIds) + "\"")
3513	.str();
3514	// Default fontsize is 14
3515	if (Highlight)
3516	AttributeString += ",fontsize=\"30\"";
3517	AttributeString +=
3518	(Twine (",fillcolor=\"") + getColor(AllocTypes: Node->AllocTypes, Highlight) + "\"")
3519	.str();
3520	if (Node->CloneOf) {
3521	AttributeString += ",color=\"blue\"";
3522	AttributeString += ",style=\"filled,bold,dashed\"";
3523	} else
3524	AttributeString += ",style=\"filled\"";
3525	return AttributeString;
3526	}
3527
3528	static std::string getEdgeAttributes(NodeRef, ChildIteratorType ChildIter,
3529	GraphType G) {
3530	auto &Edge = *(ChildIter.getCurrent());
3531	// If highlighting enabled, see if this edge contains any of the context ids
3532	// of interest. If so, it will use a different color and a heavier arrow
3533	// size and weight (the larger weight makes the highlighted path
3534	// straighter).
3535	bool Highlight = false;
3536	if (DoHighlight) {
3537	assert(ContextIdForDot.getNumOccurrences() \|\|
3538	AllocIdForDot.getNumOccurrences());
3539	if (ContextIdForDot.getNumOccurrences())
3540	Highlight = Edge->ContextIds.contains(ContextIdForDot);
3541	else
3542	Highlight = set_intersects(Edge->ContextIds, G->DotAllocContextIds);
3543	}
3544	auto Color = getColor(AllocTypes: Edge->AllocTypes, Highlight);
3545	std::string AttributeString =
3546	(Twine ("tooltip=\"") + getContextIds(ContextIds: Edge->ContextIds) + "\"" +
3547	// fillcolor is the arrow head and color is the line
3548	Twine (",fillcolor=\"") + Color + "\"" + Twine (",color=\"") + Color +
3549	"\"")
3550	.str();
3551	if (Edge->IsBackedge)
3552	AttributeString += ",style=\"dotted\"";
3553	// Default penwidth and weight are both 1.
3554	if (Highlight)
3555	AttributeString += ",penwidth=\"2.0\",weight=\"2\"";
3556	return AttributeString;
3557	}
3558
3559	// Since the NodeOwners list includes nodes that are no longer connected to
3560	// the graph, skip them here.
3561	static bool isNodeHidden(NodeRef Node, GraphType G) {
3562	if (Node->isRemoved())
3563	return true;
3564	// If a scope smaller than the full graph was requested, see if this node
3565	// contains any of the context ids of interest.
3566	if (DotGraphScope == DotScope::Alloc)
3567	return !set_intersects(Node->getContextIds(), G->DotAllocContextIds);
3568	if (DotGraphScope == DotScope::Context)
3569	return !Node->getContextIds().contains(ContextIdForDot);
3570	return false;
3571	}
3572
3573	private:
3574	static std::string getContextIds(const DenseSet<uint32_t> &ContextIds) {
3575	std::string IdString = "ContextIds:";
3576	if (ContextIds.size() < `100`) {
3577	std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
3578	std::sort(first: SortedIds.begin(), last: SortedIds.end());
3579	for (auto Id : SortedIds)
3580	IdString += (" " + Twine (Id)).str();
3581	} else {
3582	IdString += (" (" + Twine (ContextIds.size()) + " ids)").str();
3583	}
3584	return IdString;
3585	}
3586
3587	static std::string getColor(uint8_t AllocTypes, bool Highlight) {
3588	// If DoHighlight is not enabled, we want to use the highlight colors for
3589	// NotCold and Cold, and the non-highlight color for NotCold+Cold. This is
3590	// both compatible with the color scheme before highlighting was supported,
3591	// and for the NotCold+Cold color the non-highlight color is a bit more
3592	// readable.
3593	if (AllocTypes == (uint8_t)AllocationType::NotCold)
3594	// Color "brown1" actually looks like a lighter red.
3595	return !DoHighlight \|\| Highlight ? "brown1" : "lightpink";
3596	if (AllocTypes == (uint8_t)AllocationType::Cold)
3597	return !DoHighlight \|\| Highlight ? "cyan" : "lightskyblue";
3598	if (AllocTypes ==
3599	((uint8_t)AllocationType::NotCold \| (uint8_t)AllocationType::Cold))
3600	return Highlight ? "magenta" : "mediumorchid1";
3601	return "gray";
3602	}
3603
3604	static std::string getNodeId(NodeRef Node) {
3605	std::stringstream SStream;
3606	SStream << std::hex << "N0x" << (unsigned long long)Node;
3607	std::string Result = SStream.str();
3608	return Result;
3609	}
3610
3611	// True if we should highlight a specific context or allocation's contexts in
3612	// the emitted graph.
3613	static bool DoHighlight;
3614	};
3615
3616	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3617	bool DOTGraphTraits<
3618	const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>::DoHighlight =
3619	false;
3620
3621	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3622	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::exportToDot(
3623	std::string Label) const {
3624	WriteGraph(this, "", false, Label,
3625	DotFilePathPrefix + "ccg." + Label + ".dot");
3626	}
3627
3628	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3629	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
3630	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::moveEdgeToNewCalleeClone(
3631	const std::shared_ptr<ContextEdge> &Edge,
3632	DenseSet<uint32_t> ContextIdsToMove) {
3633	ContextNode *Node = Edge->Callee;
3634	assert(NodeToCallingFunc.count(Node));
3635	ContextNode *Clone =
3636	createNewNode(IsAllocation: Node->IsAllocation, F: NodeToCallingFunc[Node], C: Node->Call);
3637	Node->addClone(Clone);
3638	Clone->MatchingCalls = Node->MatchingCalls;
3639	moveEdgeToExistingCalleeClone(Edge, NewCallee: Clone, /NewClone=/true,
3640	ContextIdsToMove);
3641	return Clone;
3642	}
3643
3644	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3645	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
3646	moveEdgeToExistingCalleeClone(const std::shared_ptr<ContextEdge> &Edge,
3647	ContextNode NewCallee, bool* NewClone,
3648	DenseSet<uint32_t> ContextIdsToMove) {
3649	// NewCallee and Edge's current callee must be clones of the same original
3650	// node (Edge's current callee may be the original node too).
3651	assert(NewCallee->getOrigNode() == Edge->Callee->getOrigNode());
3652
3653	bool EdgeIsRecursive = Edge->Callee == Edge->Caller;
3654
3655	ContextNode *OldCallee = Edge->Callee;
3656
3657	// We might already have an edge to the new callee from earlier cloning for a
3658	// different allocation. If one exists we will reuse it.
3659	auto ExistingEdgeToNewCallee = NewCallee->findEdgeFromCaller(Edge->Caller);
3660
3661	// Callers will pass an empty ContextIdsToMove set when they want to move the
3662	// edge. Copy in Edge's ids for simplicity.
3663	if (ContextIdsToMove.empty())
3664	ContextIdsToMove = Edge->getContextIds();
3665
3666	// If we are moving all of Edge's ids, then just move the whole Edge.
3667	// Otherwise only move the specified subset, to a new edge if needed.
3668	if (Edge->getContextIds().size() == ContextIdsToMove.size()) {
3669	// First, update the alloc types on New Callee from Edge.
3670	// Do this before we potentially clear Edge's fields below!
3671	NewCallee->AllocTypes \|= Edge->AllocTypes;
3672	// Moving the whole Edge.
3673	if (ExistingEdgeToNewCallee) {
3674	// Since we already have an edge to NewCallee, simply move the ids
3675	// onto it, and remove the existing Edge.
3676	ExistingEdgeToNewCallee->getContextIds().insert_range(ContextIdsToMove);
3677	ExistingEdgeToNewCallee->AllocTypes \|= Edge->AllocTypes;
3678	assert(Edge->ContextIds == ContextIdsToMove);
3679	removeEdgeFromGraph(Edge: Edge.get());
3680	} else {
3681	// Otherwise just reconnect Edge to NewCallee.
3682	Edge->Callee = NewCallee;
3683	NewCallee->CallerEdges.push_back(Edge);
3684	// Remove it from callee where it was previously connected.
3685	OldCallee->eraseCallerEdge(Edge.get());
3686	// Don't need to update Edge's context ids since we are simply
3687	// reconnecting it.
3688	}
3689	} else {
3690	// Only moving a subset of Edge's ids.
3691	// Compute the alloc type of the subset of ids being moved.
3692	auto CallerEdgeAllocType = computeAllocType(ContextIds&: ContextIdsToMove);
3693	if (ExistingEdgeToNewCallee) {
3694	// Since we already have an edge to NewCallee, simply move the ids
3695	// onto it.
3696	ExistingEdgeToNewCallee->getContextIds().insert_range(ContextIdsToMove);
3697	ExistingEdgeToNewCallee->AllocTypes \|= CallerEdgeAllocType;
3698	} else {
3699	// Otherwise, create a new edge to NewCallee for the ids being moved.
3700	auto NewEdge = std::make_shared<ContextEdge>(
3701	NewCallee, Edge->Caller, CallerEdgeAllocType, ContextIdsToMove);
3702	Edge->Caller->CalleeEdges.push_back(NewEdge);
3703	NewCallee->CallerEdges.push_back(NewEdge);
3704	}
3705	// In either case, need to update the alloc types on NewCallee, and remove
3706	// those ids and update the alloc type on the original Edge.
3707	NewCallee->AllocTypes \|= CallerEdgeAllocType;
3708	set_subtract(Edge->ContextIds, ContextIdsToMove);
3709	Edge->AllocTypes = computeAllocType(ContextIds&: Edge->ContextIds);
3710	}
3711	// Now walk the old callee node's callee edges and move Edge's context ids
3712	// over to the corresponding edge into the clone (which is created here if
3713	// this is a newly created clone).
3714	for (auto &OldCalleeEdge : OldCallee->CalleeEdges) {
3715	ContextNode *CalleeToUse = OldCalleeEdge->Callee;
3716	// If this is a direct recursion edge, use NewCallee (the clone) as the
3717	// callee as well, so that any edge updated/created here is also direct
3718	// recursive.
3719	if (CalleeToUse == OldCallee) {
3720	// If this is a recursive edge, see if we already moved a recursive edge
3721	// (which would have to have been this one) - if we were only moving a
3722	// subset of context ids it would still be on OldCallee.
3723	if (EdgeIsRecursive) {
3724	assert(OldCalleeEdge == Edge);
3725	continue;
3726	}
3727	CalleeToUse = NewCallee;
3728	}
3729	// The context ids moving to the new callee are the subset of this edge's
3730	// context ids and the context ids on the caller edge being moved.
3731	DenseSet<uint32_t> EdgeContextIdsToMove =
3732	set_intersection(OldCalleeEdge->getContextIds(), ContextIdsToMove);
3733	set_subtract(OldCalleeEdge->getContextIds(), EdgeContextIdsToMove);
3734	OldCalleeEdge->AllocTypes =
3735	computeAllocType(ContextIds&: OldCalleeEdge->getContextIds());
3736	if (!NewClone) {
3737	// Update context ids / alloc type on corresponding edge to NewCallee.
3738	// There is a chance this may not exist if we are reusing an existing
3739	// clone, specifically during function assignment, where we would have
3740	// removed none type edges after creating the clone. If we can't find
3741	// a corresponding edge there, fall through to the cloning below.
3742	if (auto *NewCalleeEdge = NewCallee->findEdgeFromCallee(CalleeToUse)) {
3743	NewCalleeEdge->getContextIds().insert_range(EdgeContextIdsToMove);
3744	NewCalleeEdge->AllocTypes \|= computeAllocType(ContextIds&: EdgeContextIdsToMove);
3745	continue;
3746	}
3747	}
3748	auto NewEdge = std::make_shared<ContextEdge>(
3749	CalleeToUse, NewCallee, computeAllocType(ContextIds&: EdgeContextIdsToMove),
3750	EdgeContextIdsToMove);
3751	NewCallee->CalleeEdges.push_back(NewEdge);
3752	NewEdge->Callee->CallerEdges.push_back(NewEdge);
3753	}
3754	// Recompute the node alloc type now that its callee edges have been
3755	// updated (since we will compute from those edges).
3756	OldCallee->AllocTypes = OldCallee->computeAllocType();
3757	// OldCallee alloc type should be None iff its context id set is now empty.
3758	assert((OldCallee->AllocTypes == (uint8_t)AllocationType::None) ==
3759	OldCallee->emptyContextIds());
3760	if (VerifyCCG) {
3761	checkNode<DerivedCCG, FuncTy, CallTy>(OldCallee, /CheckEdges=/false);
3762	checkNode<DerivedCCG, FuncTy, CallTy>(NewCallee, /CheckEdges=/false);
3763	for (const auto &OldCalleeEdge : OldCallee->CalleeEdges)
3764	checkNode<DerivedCCG, FuncTy, CallTy>(OldCalleeEdge->Callee,
3765	/CheckEdges=/false);
3766	for (const auto &NewCalleeEdge : NewCallee->CalleeEdges)
3767	checkNode<DerivedCCG, FuncTy, CallTy>(NewCalleeEdge->Callee,
3768	/CheckEdges=/false);
3769	}
3770	}
3771
3772	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3773	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
3774	moveCalleeEdgeToNewCaller(const std::shared_ptr<ContextEdge> &Edge,
3775	ContextNode *NewCaller) {
3776	auto *OldCallee = Edge->Callee;
3777	auto *NewCallee = OldCallee;
3778	// If this edge was direct recursive, make any new/updated edge also direct
3779	// recursive to NewCaller.
3780	bool Recursive = Edge->Caller == Edge->Callee;
3781	if (Recursive)
3782	NewCallee = NewCaller;
3783
3784	ContextNode *OldCaller = Edge->Caller;
3785	OldCaller->eraseCalleeEdge(Edge.get());
3786
3787	// We might already have an edge to the new caller. If one exists we will
3788	// reuse it.
3789	auto ExistingEdgeToNewCaller = NewCaller->findEdgeFromCallee(NewCallee);
3790
3791	if (ExistingEdgeToNewCaller) {
3792	// Since we already have an edge to NewCaller, simply move the ids
3793	// onto it, and remove the existing Edge.
3794	ExistingEdgeToNewCaller->getContextIds().insert_range(
3795	Edge->getContextIds());
3796	ExistingEdgeToNewCaller->AllocTypes \|= Edge->AllocTypes;
3797	Edge->ContextIds.clear();
3798	Edge->AllocTypes = (uint8_t)AllocationType::None;
3799	OldCallee->eraseCallerEdge(Edge.get());
3800	} else {
3801	// Otherwise just reconnect Edge to NewCaller.
3802	Edge->Caller = NewCaller;
3803	NewCaller->CalleeEdges.push_back(Edge);
3804	if (Recursive) {
3805	assert(NewCallee == NewCaller);
3806	// In the case of (direct) recursive edges, we update the callee as well
3807	// so that it becomes recursive on the new caller.
3808	Edge->Callee = NewCallee;
3809	NewCallee->CallerEdges.push_back(Edge);
3810	OldCallee->eraseCallerEdge(Edge.get());
3811	}
3812	// Don't need to update Edge's context ids since we are simply
3813	// reconnecting it.
3814	}
3815	// In either case, need to update the alloc types on New Caller.
3816	NewCaller->AllocTypes \|= Edge->AllocTypes;
3817
3818	// Now walk the old caller node's caller edges and move Edge's context ids
3819	// over to the corresponding edge into the node (which is created here if
3820	// this is a newly created node). We can tell whether this is a newly created
3821	// node by seeing if it has any caller edges yet.
3822	#ifndef NDEBUG
3823	bool IsNewNode = NewCaller->CallerEdges.empty();
3824	#endif
3825	// If we just moved a direct recursive edge, presumably its context ids should
3826	// also flow out of OldCaller via some other non-recursive callee edge. We
3827	// don't want to remove the recursive context ids from other caller edges yet,
3828	// otherwise the context ids get into an inconsistent state on OldCaller.
3829	// We will update these context ids on the non-recursive caller edge when and
3830	// if they are updated on the non-recursive callee.
3831	if (!Recursive) {
3832	for (auto &OldCallerEdge : OldCaller->CallerEdges) {
3833	auto OldCallerCaller = OldCallerEdge->Caller;
3834	// The context ids moving to the new caller are the subset of this edge's
3835	// context ids and the context ids on the callee edge being moved.
3836	DenseSet<uint32_t> EdgeContextIdsToMove = set_intersection(
3837	OldCallerEdge->getContextIds(), Edge->getContextIds());
3838	if (OldCaller == OldCallerCaller) {
3839	OldCallerCaller = NewCaller;
3840	// Don't actually move this one. The caller will move it directly via a
3841	// call to this function with this as the Edge if it is appropriate to
3842	// move to a diff node that has a matching callee (itself).
3843	continue;
3844	}
3845	set_subtract(OldCallerEdge->getContextIds(), EdgeContextIdsToMove);
3846	OldCallerEdge->AllocTypes =
3847	computeAllocType(ContextIds&: OldCallerEdge->getContextIds());
3848	// In this function we expect that any pre-existing node already has edges
3849	// from the same callers as the old node. That should be true in the
3850	// current use case, where we will remove None-type edges after copying
3851	// over all caller edges from the callee.
3852	auto *ExistingCallerEdge = NewCaller->findEdgeFromCaller(OldCallerCaller);
3853	// Since we would have skipped caller edges when moving a direct recursive
3854	// edge, this may not hold true when recursive handling enabled.
3855	assert(IsNewNode \|\| ExistingCallerEdge \|\| AllowRecursiveCallsites);
3856	if (ExistingCallerEdge) {
3857	ExistingCallerEdge->getContextIds().insert_range(EdgeContextIdsToMove);
3858	ExistingCallerEdge->AllocTypes \|=
3859	computeAllocType(ContextIds&: EdgeContextIdsToMove);
3860	continue;
3861	}
3862	auto NewEdge = std::make_shared<ContextEdge>(
3863	NewCaller, OldCallerCaller, computeAllocType(ContextIds&: EdgeContextIdsToMove),
3864	EdgeContextIdsToMove);
3865	NewCaller->CallerEdges.push_back(NewEdge);
3866	NewEdge->Caller->CalleeEdges.push_back(NewEdge);
3867	}
3868	}
3869	// Recompute the node alloc type now that its caller edges have been
3870	// updated (since we will compute from those edges).
3871	OldCaller->AllocTypes = OldCaller->computeAllocType();
3872	// OldCaller alloc type should be None iff its context id set is now empty.
3873	assert((OldCaller->AllocTypes == (uint8_t)AllocationType::None) ==
3874	OldCaller->emptyContextIds());
3875	if (VerifyCCG) {
3876	checkNode<DerivedCCG, FuncTy, CallTy>(OldCaller, /CheckEdges=/false);
3877	checkNode<DerivedCCG, FuncTy, CallTy>(NewCaller, /CheckEdges=/false);
3878	for (const auto &OldCallerEdge : OldCaller->CallerEdges)
3879	checkNode<DerivedCCG, FuncTy, CallTy>(OldCallerEdge->Caller,
3880	/CheckEdges=/false);
3881	for (const auto &NewCallerEdge : NewCaller->CallerEdges)
3882	checkNode<DerivedCCG, FuncTy, CallTy>(NewCallerEdge->Caller,
3883	/CheckEdges=/false);
3884	}
3885	}
3886
3887	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3888	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
3889	recursivelyRemoveNoneTypeCalleeEdges(
3890	ContextNode Node, DenseSet<const* ContextNode *> &Visited) {
3891	auto Inserted = Visited.insert(Node);
3892	if (!Inserted.second)
3893	return;
3894
3895	removeNoneTypeCalleeEdges(Node);
3896
3897	for (auto *Clone : Node->Clones)
3898	recursivelyRemoveNoneTypeCalleeEdges(Node: Clone, Visited);
3899
3900	// The recursive call may remove some of this Node's caller edges.
3901	// Iterate over a copy and skip any that were removed.
3902	auto CallerEdges = Node->CallerEdges;
3903	for (auto &Edge : CallerEdges) {
3904	// Skip any that have been removed by an earlier recursive call.
3905	if (Edge->isRemoved()) {
3906	assert(!is_contained(Node->CallerEdges, Edge));
3907	continue;
3908	}
3909	recursivelyRemoveNoneTypeCalleeEdges(Node: Edge->Caller, Visited);
3910	}
3911	}
3912
3913	// This is the standard DFS based backedge discovery algorithm.
3914	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3915	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::markBackedges() {
3916	// If we are cloning recursive contexts, find and mark backedges from all root
3917	// callers, using the typical DFS based backedge analysis.
3918	if (!CloneRecursiveContexts)
3919	return;
3920	DenseSet<const ContextNode *> Visited;
3921	DenseSet<const ContextNode *> CurrentStack;
3922	for (auto &Entry : NonAllocationCallToContextNodeMap) {
3923	auto *Node = Entry.second;
3924	if (Node->isRemoved())
3925	continue;
3926	// It is a root if it doesn't have callers.
3927	if (!Node->CallerEdges.empty())
3928	continue;
3929	markBackedges(Node, Visited, CurrentStack);
3930	assert(CurrentStack.empty());
3931	}
3932	}
3933
3934	// Recursive helper for above markBackedges method.
3935	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3936	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::markBackedges(
3937	ContextNode Node, DenseSet<const* ContextNode *> &Visited,
3938	DenseSet<const ContextNode *> &CurrentStack) {
3939	auto I = Visited.insert(Node);
3940	// We should only call this for unvisited nodes.
3941	assert(I.second);
3942	(void)I;
3943	for (auto &CalleeEdge : Node->CalleeEdges) {
3944	auto *Callee = CalleeEdge->Callee;
3945	if (Visited.count(Callee)) {
3946	// Since this was already visited we need to check if it is currently on
3947	// the recursive stack in which case it is a backedge.
3948	if (CurrentStack.count(Callee))
3949	CalleeEdge->IsBackedge = true;
3950	continue;
3951	}
3952	CurrentStack.insert(Callee);
3953	markBackedges(Callee, Visited, CurrentStack);
3954	CurrentStack.erase(Callee);
3955	}
3956	}
3957
3958	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3959	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::identifyClones() {
3960	DenseSet<const ContextNode *> Visited;
3961	for (auto &Entry : AllocationCallToContextNodeMap) {
3962	Visited.clear();
3963	identifyClones(Entry.second, Visited, Entry.second->getContextIds());
3964	}
3965	Visited.clear();
3966	for (auto &Entry : AllocationCallToContextNodeMap)
3967	recursivelyRemoveNoneTypeCalleeEdges(Node: Entry.second, Visited);
3968	if (VerifyCCG)
3969	check();
3970	}
3971
3972	// helper function to check an AllocType is cold or notcold or both.
3973	bool checkColdOrNotCold(uint8_t AllocType) {
3974	return (AllocType == (uint8_t)AllocationType::Cold) \|\|
3975	(AllocType == (uint8_t)AllocationType::NotCold) \|\|
3976	(AllocType ==
3977	((uint8_t)AllocationType::Cold \| (uint8_t)AllocationType::NotCold));
3978	}
3979
3980	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3981	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::identifyClones(
3982	ContextNode Node, DenseSet<const* ContextNode *> &Visited,
3983	const DenseSet<uint32_t> &AllocContextIds) {
3984	if (VerifyNodes)
3985	checkNode<DerivedCCG, FuncTy, CallTy>(Node, /CheckEdges=/false);
3986	assert(!Node->CloneOf);
3987
3988	// If Node as a null call, then either it wasn't found in the module (regular
3989	// LTO) or summary index (ThinLTO), or there were other conditions blocking
3990	// cloning (e.g. recursion, calls multiple targets, etc).
3991	// Do this here so that we don't try to recursively clone callers below, which
3992	// isn't useful at least for this node.
3993	if (!Node->hasCall())
3994	return;
3995
3996	// No need to look at any callers if allocation type already unambiguous.
3997	if (hasSingleAllocType(Node->AllocTypes))
3998	return;
3999
4000	#ifndef NDEBUG
4001	auto Insert =
4002	#endif
4003	Visited.insert(Node);
4004	// We should not have visited this node yet.
4005	assert(Insert.second);
4006	// The recursive call to identifyClones may delete the current edge from the
4007	// CallerEdges vector. Make a copy and iterate on that, simpler than passing
4008	// in an iterator and having recursive call erase from it. Other edges may
4009	// also get removed during the recursion, which will have null Callee and
4010	// Caller pointers (and are deleted later), so we skip those below.
4011	{
4012	auto CallerEdges = Node->CallerEdges;
4013	for (auto &Edge : CallerEdges) {
4014	// Skip any that have been removed by an earlier recursive call.
4015	if (Edge->isRemoved()) {
4016	assert(!is_contained(Node->CallerEdges, Edge));
4017	continue;
4018	}
4019	// Defer backedges. See comments further below where these edges are
4020	// handled during the cloning of this Node.
4021	if (Edge->IsBackedge) {
4022	// We should only mark these if cloning recursive contexts, where we
4023	// need to do this deferral.
4024	assert(CloneRecursiveContexts);
4025	continue;
4026	}
4027	// Ignore any caller we previously visited via another edge.
4028	if (!Visited.count(Edge->Caller) && !Edge->Caller->CloneOf) {
4029	identifyClones(Edge->Caller, Visited, AllocContextIds);
4030	}
4031	}
4032	}
4033
4034	// Check if we reached an unambiguous call or have have only a single caller.
4035	if (hasSingleAllocType(Node->AllocTypes) \|\| Node->CallerEdges.size() <= `1`)
4036	return;
4037
4038	// We need to clone.
4039
4040	// Try to keep the original version as alloc type NotCold. This will make
4041	// cases with indirect calls or any other situation with an unknown call to
4042	// the original function get the default behavior. We do this by sorting the
4043	// CallerEdges of the Node we will clone by alloc type.
4044	//
4045	// Give NotCold edge the lowest sort priority so those edges are at the end of
4046	// the caller edges vector, and stay on the original version (since the below
4047	// code clones greedily until it finds all remaining edges have the same type
4048	// and leaves the remaining ones on the original Node).
4049	//
4050	// We shouldn't actually have any None type edges, so the sorting priority for
4051	// that is arbitrary, and we assert in that case below.
4052	const unsigned AllocTypeCloningPriority[] = {/None/ `3`, /NotCold/ `4`,
4053	/Cold/ `1`,
4054	/NotColdCold/ `2`};
4055	llvm::stable_sort(Node->CallerEdges,
4056	[&](const std::shared_ptr<ContextEdge> &A,
4057	const std::shared_ptr<ContextEdge> &B) {
4058	// Nodes with non-empty context ids should be sorted
4059	// before those with empty context ids.
4060	if (A->ContextIds.empty())
4061	// Either B ContextIds are non-empty (in which case we
4062	// should return false because B < A), or B ContextIds
4063	// are empty, in which case they are equal, and we
4064	// should maintain the original relative ordering.
4065	return false;
4066	if (B->ContextIds.empty())
4067	return true;
4068
4069	if (A->AllocTypes == B->AllocTypes)
4070	// Use the first context id for each edge as a
4071	// tie-breaker.
4072	return A->ContextIds.begin() < B->ContextIds.begin();
4073	return AllocTypeCloningPriority[A->AllocTypes] <
4074	AllocTypeCloningPriority[B->AllocTypes];
4075	});
4076
4077	assert(Node->AllocTypes != (uint8_t)AllocationType::None);
4078
4079	DenseSet<uint32_t> RecursiveContextIds;
4080	assert(AllowRecursiveContexts \|\| !CloneRecursiveContexts);
4081	// If we are allowing recursive callsites, but have also disabled recursive
4082	// contexts, look for context ids that show up in multiple caller edges.
4083	if (AllowRecursiveCallsites && !AllowRecursiveContexts) {
4084	DenseSet<uint32_t> AllCallerContextIds;
4085	for (auto &CE : Node->CallerEdges) {
4086	// Resize to the largest set of caller context ids, since we know the
4087	// final set will be at least that large.
4088	AllCallerContextIds.reserve(Size: CE->getContextIds().size());
4089	for (auto Id : CE->getContextIds())
4090	if (!AllCallerContextIds.insert(Id).second)
4091	RecursiveContextIds.insert(Id);
4092	}
4093	}
4094
4095	// Iterate until we find no more opportunities for disambiguating the alloc
4096	// types via cloning. In most cases this loop will terminate once the Node
4097	// has a single allocation type, in which case no more cloning is needed.
4098	// Iterate over a copy of Node's caller edges, since we may need to remove
4099	// edges in the moveEdgeTo methods, and this simplifies the handling and*
4100	// makes it less error-prone.
4101	auto CallerEdges = Node->CallerEdges;
4102	for (auto &CallerEdge : CallerEdges) {
4103	// Skip any that have been removed by an earlier recursive call.
4104	if (CallerEdge->isRemoved()) {
4105	assert(!is_contained(Node->CallerEdges, CallerEdge));
4106	continue;
4107	}
4108	assert(CallerEdge->Callee == Node);
4109
4110	// See if cloning the prior caller edge left this node with a single alloc
4111	// type or a single caller. In that case no more cloning of Node is needed.
4112	if (hasSingleAllocType(Node->AllocTypes) \|\| Node->CallerEdges.size() <= `1`)
4113	break;
4114
4115	// If the caller was not successfully matched to a call in the IR/summary,
4116	// there is no point in trying to clone for it as we can't update that call.
4117	if (!CallerEdge->Caller->hasCall())
4118	continue;
4119
4120	// Only need to process the ids along this edge pertaining to the given
4121	// allocation.
4122	auto CallerEdgeContextsForAlloc =
4123	set_intersection(CallerEdge->getContextIds(), AllocContextIds);
4124	if (!RecursiveContextIds.empty())
4125	CallerEdgeContextsForAlloc =
4126	set_difference(CallerEdgeContextsForAlloc, RecursiveContextIds);
4127	if (CallerEdgeContextsForAlloc.empty())
4128	continue;
4129
4130	auto CallerAllocTypeForAlloc = computeAllocType(ContextIds&: CallerEdgeContextsForAlloc);
4131
4132	// Compute the node callee edge alloc types corresponding to the context ids
4133	// for this caller edge.
4134	std::vector<uint8_t> CalleeEdgeAllocTypesForCallerEdge;
4135	CalleeEdgeAllocTypesForCallerEdge.reserve(n: Node->CalleeEdges.size());
4136	for (auto &CalleeEdge : Node->CalleeEdges)
4137	CalleeEdgeAllocTypesForCallerEdge.push_back(intersectAllocTypes(
4138	Node1Ids: CalleeEdge->getContextIds(), Node2Ids: CallerEdgeContextsForAlloc));
4139
4140	// Don't clone if doing so will not disambiguate any alloc types amongst
4141	// caller edges (including the callee edges that would be cloned).
4142	// Otherwise we will simply move all edges to the clone.
4143	//
4144	// First check if by cloning we will disambiguate the caller allocation
4145	// type from node's allocation type. Query allocTypeToUse so that we don't
4146	// bother cloning to distinguish NotCold+Cold from NotCold. Note that
4147	// neither of these should be None type.
4148	//
4149	// Then check if by cloning node at least one of the callee edges will be
4150	// disambiguated by splitting out different context ids.
4151	//
4152	// However, always do the cloning if this is a backedge, in which case we
4153	// have not yet cloned along this caller edge.
4154	assert(CallerEdge->AllocTypes != (uint8_t)AllocationType::None);
4155	assert(Node->AllocTypes != (uint8_t)AllocationType::None);
4156	if (!CallerEdge->IsBackedge &&
4157	allocTypeToUse(CallerAllocTypeForAlloc) ==
4158	allocTypeToUse(Node->AllocTypes) &&
4159	allocTypesMatch<DerivedCCG, FuncTy, CallTy>(
4160	CalleeEdgeAllocTypesForCallerEdge, Node->CalleeEdges)) {
4161	continue;
4162	}
4163
4164	if (CallerEdge->IsBackedge) {
4165	// We should only mark these if cloning recursive contexts, where we
4166	// need to do this deferral.
4167	assert(CloneRecursiveContexts);
4168	DeferredBackedges ++;
4169	}
4170
4171	// If this is a backedge, we now do recursive cloning starting from its
4172	// caller since we may have moved unambiguous caller contexts to a clone
4173	// of this Node in a previous iteration of the current loop, giving more
4174	// opportunity for cloning through the backedge. Because we sorted the
4175	// caller edges earlier so that cold caller edges are first, we would have
4176	// visited and cloned this node for any unamibiguously cold non-recursive
4177	// callers before any ambiguous backedge callers. Note that we don't do this
4178	// if the caller is already cloned or visited during cloning (e.g. via a
4179	// different context path from the allocation).
4180	// TODO: Can we do better in the case where the caller was already visited?
4181	if (CallerEdge->IsBackedge && !CallerEdge->Caller->CloneOf &&
4182	!Visited.count(CallerEdge->Caller)) {
4183	const auto OrigIdCount = CallerEdge->getContextIds().size();
4184	// Now do the recursive cloning of this backedge's caller, which was
4185	// deferred earlier.
4186	identifyClones(CallerEdge->Caller, Visited, CallerEdgeContextsForAlloc);
4187	removeNoneTypeCalleeEdges(Node: CallerEdge->Caller);
4188	// See if the recursive call to identifyClones moved the context ids to a
4189	// new edge from this node to a clone of caller, and switch to looking at
4190	// that new edge so that we clone Node for the new caller clone.
4191	bool UpdatedEdge = false;
4192	if (OrigIdCount > CallerEdge->getContextIds().size()) {
4193	for (auto E : Node->CallerEdges) {
4194	// Only interested in clones of the current edges caller.
4195	if (E->Caller->CloneOf != CallerEdge->Caller)
4196	continue;
4197	// See if this edge contains any of the context ids originally on the
4198	// current caller edge.
4199	auto CallerEdgeContextsForAllocNew =
4200	set_intersection(CallerEdgeContextsForAlloc, E->getContextIds());
4201	if (CallerEdgeContextsForAllocNew.empty())
4202	continue;
4203	// Make sure we don't pick a previously existing caller edge of this
4204	// Node, which would be processed on a different iteration of the
4205	// outer loop over the saved CallerEdges.
4206	if (llvm::is_contained(CallerEdges, E))
4207	continue;
4208	// The CallerAllocTypeForAlloc and CalleeEdgeAllocTypesForCallerEdge
4209	// are updated further below for all cases where we just invoked
4210	// identifyClones recursively.
4211	CallerEdgeContextsForAlloc.swap(CallerEdgeContextsForAllocNew);
4212	CallerEdge = E;
4213	UpdatedEdge = true;
4214	break;
4215	}
4216	}
4217	// If cloning removed this edge (and we didn't update it to a new edge
4218	// above), we're done with this edge. It's possible we moved all of the
4219	// context ids to an existing clone, in which case there's no need to do
4220	// further processing for them.
4221	if (CallerEdge->isRemoved())
4222	continue;
4223
4224	// Now we need to update the information used for the cloning decisions
4225	// further below, as we may have modified edges and their context ids.
4226
4227	// Note if we changed the CallerEdge above we would have already updated
4228	// the context ids.
4229	if (!UpdatedEdge) {
4230	CallerEdgeContextsForAlloc = set_intersection(
4231	CallerEdgeContextsForAlloc, CallerEdge->getContextIds());
4232	if (CallerEdgeContextsForAlloc.empty())
4233	continue;
4234	}
4235	// Update the other information that depends on the edges and on the now
4236	// updated CallerEdgeContextsForAlloc.
4237	CallerAllocTypeForAlloc = computeAllocType(ContextIds&: CallerEdgeContextsForAlloc);
4238	CalleeEdgeAllocTypesForCallerEdge.clear();
4239	for (auto &CalleeEdge : Node->CalleeEdges) {
4240	CalleeEdgeAllocTypesForCallerEdge.push_back(intersectAllocTypes(
4241	Node1Ids: CalleeEdge->getContextIds(), Node2Ids: CallerEdgeContextsForAlloc));
4242	}
4243	}
4244
4245	// First see if we can use an existing clone. Check each clone and its
4246	// callee edges for matching alloc types.
4247	ContextNode Clone = nullptr*;
4248	for (auto *CurClone : Node->Clones) {
4249	if (allocTypeToUse(CurClone->AllocTypes) !=
4250	allocTypeToUse(CallerAllocTypeForAlloc))
4251	continue;
4252
4253	bool BothSingleAlloc = hasSingleAllocType(CurClone->AllocTypes) &&
4254	hasSingleAllocType(CallerAllocTypeForAlloc);
4255	// The above check should mean that if both have single alloc types that
4256	// they should be equal.
4257	assert(!BothSingleAlloc \|\|
4258	CurClone->AllocTypes == CallerAllocTypeForAlloc);
4259
4260	// If either both have a single alloc type (which are the same), or if the
4261	// clone's callee edges have the same alloc types as those for the current
4262	// allocation on Node's callee edges (CalleeEdgeAllocTypesForCallerEdge),
4263	// then we can reuse this clone.
4264	if (BothSingleAlloc \|\| allocTypesMatchClone<DerivedCCG, FuncTy, CallTy>(
4265	CalleeEdgeAllocTypesForCallerEdge, CurClone)) {
4266	Clone = CurClone;
4267	break;
4268	}
4269	}
4270
4271	// The edge iterator is adjusted when we move the CallerEdge to the clone.
4272	if (Clone)
4273	moveEdgeToExistingCalleeClone(Edge: CallerEdge, NewCallee: Clone, /NewClone=/false,
4274	ContextIdsToMove: CallerEdgeContextsForAlloc);
4275	else
4276	Clone = moveEdgeToNewCalleeClone(Edge: CallerEdge, ContextIdsToMove: CallerEdgeContextsForAlloc);
4277
4278	// Sanity check that no alloc types on clone or its edges are None.
4279	assert(Clone->AllocTypes != (uint8_t)AllocationType::None);
4280	}
4281
4282	// We should still have some context ids on the original Node.
4283	assert(!Node->emptyContextIds());
4284
4285	// Sanity check that no alloc types on node or edges are None.
4286	assert(Node->AllocTypes != (uint8_t)AllocationType::None);
4287
4288	if (VerifyNodes)
4289	checkNode<DerivedCCG, FuncTy, CallTy>(Node, /CheckEdges=/false);
4290	}
4291
4292	void ModuleCallsiteContextGraph::updateAllocationCall(
4293	CallInfo &Call, AllocationType AllocType) {
4294	std::string AllocTypeString = getAllocTypeAttributeString(Type: AllocType);
4295	removeAnyExistingAmbiguousAttribute(CB: cast<CallBase>(Val: Call.call()));
4296	auto A = llvm::Attribute::get(Context&: Call.call()->getFunction()->getContext(),
4297	Kind: "memprof", Val: AllocTypeString);
4298	cast<CallBase>(Val: Call.call())->addFnAttr(Attr: A);
4299	OREGetter (Call.call()->getFunction())
4300	.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofAttribute", Call.call())
4301	<< ore::NV ("AllocationCall", Call.call()) << " in clone "
4302	<< ore::NV ("Caller", Call.call()->getFunction())
4303	<< " marked with memprof allocation attribute "
4304	<< ore::NV ("Attribute", AllocTypeString));
4305	}
4306
4307	void IndexCallsiteContextGraph::updateAllocationCall(CallInfo &Call,
4308	AllocationType AllocType) {
4309	auto AI = cast<AllocInfo >(Val: Call.call());
4310	assert(AI);
4311	assert(AI->Versions.size() > Call.cloneNo());
4312	AI->Versions [Call.cloneNo()] = (uint8_t)AllocType;
4313	}
4314
4315	AllocationType
4316	ModuleCallsiteContextGraph::getAllocationCallType(const CallInfo &Call) const {
4317	const auto *CB = cast<CallBase>(Val: Call.call());
4318	if (!CB->getAttributes().hasFnAttr(Kind: "memprof"))
4319	return AllocationType::None;
4320	return CB->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "cold"
4321	? AllocationType::Cold
4322	: AllocationType::NotCold;
4323	}
4324
4325	AllocationType
4326	IndexCallsiteContextGraph::getAllocationCallType(const CallInfo &Call) const {
4327	const auto AI = cast<AllocInfo >(Val: Call.call());
4328	assert(AI->Versions.size() > Call.cloneNo());
4329	return (AllocationType)AI->Versions [Call.cloneNo()];
4330	}
4331
4332	void ModuleCallsiteContextGraph::updateCall(CallInfo &CallerCall,
4333	FuncInfo CalleeFunc) {
4334	auto *CurF = getCalleeFunc(Call: CallerCall.call());
4335	auto NewCalleeCloneNo = CalleeFunc.cloneNo();
4336	if (isMemProfClone(F: *CurF)) {
4337	// If we already assigned this callsite to call a specific non-default
4338	// clone (i.e. not the original function which is clone 0), ensure that we
4339	// aren't trying to now update it to call a different clone, which is
4340	// indicative of a bug in the graph or function assignment.
4341	auto CurCalleeCloneNo = getMemProfCloneNum(F: *CurF);
4342	if (CurCalleeCloneNo != NewCalleeCloneNo) {
4343	LLVM_DEBUG(dbgs() << "Mismatch in call clone assignment: was "
4344	<< CurCalleeCloneNo << " now " << NewCalleeCloneNo
4345	<< "\n");
4346	MismatchedCloneAssignments ++;
4347	}
4348	}
4349	if (NewCalleeCloneNo > `0`)
4350	cast<CallBase>(Val: CallerCall.call())->setCalledFunction(CalleeFunc.func());
4351	OREGetter (CallerCall.call()->getFunction())
4352	.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofCall", CallerCall.call())
4353	<< ore::NV ("Call", CallerCall.call()) << " in clone "
4354	<< ore::NV ("Caller", CallerCall.call()->getFunction())
4355	<< " assigned to call function clone "
4356	<< ore::NV ("Callee", CalleeFunc.func()));
4357	}
4358
4359	void IndexCallsiteContextGraph::updateCall(CallInfo &CallerCall,
4360	FuncInfo CalleeFunc) {
4361	auto CI = cast<CallsiteInfo >(Val: CallerCall.call());
4362	assert(CI &&
4363	"Caller cannot be an allocation which should not have profiled calls");
4364	assert(CI->Clones.size() > CallerCall.cloneNo());
4365	auto NewCalleeCloneNo = CalleeFunc.cloneNo();
4366	auto &CurCalleeCloneNo = CI->Clones [CallerCall.cloneNo()];
4367	// If we already assigned this callsite to call a specific non-default
4368	// clone (i.e. not the original function which is clone 0), ensure that we
4369	// aren't trying to now update it to call a different clone, which is
4370	// indicative of a bug in the graph or function assignment.
4371	if (CurCalleeCloneNo != `0` && CurCalleeCloneNo != NewCalleeCloneNo) {
4372	LLVM_DEBUG(dbgs() << "Mismatch in call clone assignment: was "
4373	<< CurCalleeCloneNo << " now " << NewCalleeCloneNo
4374	<< "\n");
4375	MismatchedCloneAssignments ++;
4376	}
4377	CurCalleeCloneNo = NewCalleeCloneNo;
4378	}
4379
4380	// Update the debug information attached to NewFunc to use the clone Name. Note
4381	// this needs to be done for both any existing DISubprogram for the definition,
4382	// as well as any separate declaration DISubprogram.
4383	static void updateSubprogramLinkageName(Function *NewFunc, StringRef Name) {
4384	assert(Name == NewFunc->getName());
4385	auto *SP = NewFunc->getSubprogram();
4386	if (!SP)
4387	return;
4388	auto *MDName = MDString::get(Context&: NewFunc->getParent()->getContext(), Str: Name);
4389	SP->replaceLinkageName(LN: MDName);
4390	DISubprogram *Decl = SP->getDeclaration();
4391	if (!Decl)
4392	return;
4393	TempDISubprogram NewDecl = Decl->clone();
4394	NewDecl ->replaceLinkageName(LN: MDName);
4395	SP->replaceDeclaration(Decl: MDNode::replaceWithUniqued(N: std::move(NewDecl)));
4396	}
4397
4398	CallsiteContextGraph<ModuleCallsiteContextGraph, Function,
4399	Instruction *>::FuncInfo
4400	ModuleCallsiteContextGraph::cloneFunctionForCallsite(
4401	FuncInfo &Func, CallInfo &Call, DenseMap<CallInfo, CallInfo> &CallMap,
4402	std::vector<CallInfo> &CallsWithMetadataInFunc, unsigned CloneNo) {
4403	// Use existing LLVM facilities for cloning and obtaining Call in clone
4404	ValueToValueMapTy VMap;
4405	auto *NewFunc = CloneFunction(F: Func.func(), VMap);
4406	std::string Name = getMemProfFuncName(Base: Func.func()->getName(), CloneNo);
4407	assert(!Func.func()->getParent()->getFunction(Name));
4408	NewFunc->setName(Name);
4409	updateSubprogramLinkageName(NewFunc, Name);
4410	for (auto &Inst : CallsWithMetadataInFunc) {
4411	// This map always has the initial version in it.
4412	assert(Inst.cloneNo() == `0`);
4413	CallMap [Inst] = {cast<Instruction>(Val&: VMap [Inst.call()]), CloneNo};
4414	}
4415	OREGetter (Func.func())
4416	.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofClone", Func.func())
4417	<< "created clone " << ore::NV ("NewFunction", NewFunc));
4418	return {NewFunc, CloneNo};
4419	}
4420
4421	CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
4422	IndexCall>::FuncInfo
4423	IndexCallsiteContextGraph::cloneFunctionForCallsite(
4424	FuncInfo &Func, CallInfo &Call, DenseMap<CallInfo, CallInfo> &CallMap,
4425	std::vector<CallInfo> &CallsWithMetadataInFunc, unsigned CloneNo) {
4426	// Check how many clones we have of Call (and therefore function).
4427	// The next clone number is the current size of versions array.
4428	// Confirm this matches the CloneNo provided by the caller, which is based on
4429	// the number of function clones we have.
4430	assert(CloneNo == (isa<AllocInfo *>(Call.call())
4431	? cast<AllocInfo *>(Call.call())->Versions.size()
4432	: cast<CallsiteInfo *>(Call.call())->Clones.size()));
4433	// Walk all the instructions in this function. Create a new version for
4434	// each (by adding an entry to the Versions/Clones summary array), and copy
4435	// over the version being called for the function clone being cloned here.
4436	// Additionally, add an entry to the CallMap for the new function clone,
4437	// mapping the original call (clone 0, what is in CallsWithMetadataInFunc)
4438	// to the new call clone.
4439	for (auto &Inst : CallsWithMetadataInFunc) {
4440	// This map always has the initial version in it.
4441	assert(Inst.cloneNo() == `0`);
4442	if (auto AI = dyn_cast<AllocInfo >(Val: Inst.call())) {
4443	assert(AI->Versions.size() == CloneNo);
4444	// We assign the allocation type later (in updateAllocationCall), just add
4445	// an entry for it here.
4446	AI->Versions.push_back(Elt: `0`);
4447	} else {
4448	auto CI = cast<CallsiteInfo >(Val: Inst.call());
4449	assert(CI && CI->Clones.size() == CloneNo);
4450	// We assign the clone number later (in updateCall), just add an entry for
4451	// it here.
4452	CI->Clones.push_back(Elt: `0`);
4453	}
4454	CallMap [Inst] = {Inst.call(), CloneNo};
4455	}
4456	return {Func.func(), CloneNo};
4457	}
4458
4459	// We perform cloning for each allocation node separately. However, this
4460	// sometimes results in a situation where the same node calls multiple
4461	// clones of the same callee, created for different allocations. This
4462	// causes issues when assigning functions to these clones, as each node can
4463	// in reality only call a single callee clone.
4464	//
4465	// To address this, before assigning functions, merge callee clone nodes as
4466	// needed using a post order traversal from the allocations. We attempt to
4467	// use existing clones as the merge node when legal, and to share them
4468	// among callers with the same properties (callers calling the same set of
4469	// callee clone nodes for the same allocations).
4470	//
4471	// Without this fix, in some cases incorrect function assignment will lead
4472	// to calling the wrong allocation clone.
4473	template <typename DerivedCCG, typename FuncTy, typename CallTy>
4474	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::mergeClones() {
4475	if (!MergeClones)
4476	return;
4477
4478	// Generate a map from context id to the associated allocation node for use
4479	// when merging clones.
4480	DenseMap<uint32_t, ContextNode *> ContextIdToAllocationNode;
4481	for (auto &Entry : AllocationCallToContextNodeMap) {
4482	auto *Node = Entry.second;
4483	for (auto Id : Node->getContextIds())
4484	ContextIdToAllocationNode[Id] = Node->getOrigNode();
4485	for (auto *Clone : Node->Clones) {
4486	for (auto Id : Clone->getContextIds())
4487	ContextIdToAllocationNode[Id] = Clone->getOrigNode();
4488	}
4489	}
4490
4491	// Post order traversal starting from allocations to ensure each callsite
4492	// calls a single clone of its callee. Callee nodes that are clones of each
4493	// other are merged (via new merge nodes if needed) to achieve this.
4494	DenseSet<const ContextNode *> Visited;
4495	for (auto &Entry : AllocationCallToContextNodeMap) {
4496	auto *Node = Entry.second;
4497
4498	mergeClones(Node, Visited, ContextIdToAllocationNode);
4499
4500	// Make a copy so the recursive post order traversal that may create new
4501	// clones doesn't mess up iteration. Note that the recursive traversal
4502	// itself does not call mergeClones on any of these nodes, which are all
4503	// (clones of) allocations.
4504	auto Clones = Node->Clones;
4505	for (auto *Clone : Clones)
4506	mergeClones(Clone, Visited, ContextIdToAllocationNode);
4507	}
4508
4509	if (DumpCCG) {
4510	dbgs() << "CCG after merging:\n";
4511	dbgs() << *this;
4512	}
4513	if (ExportToDot)
4514	exportToDot(Label: "aftermerge");
4515
4516	if (VerifyCCG) {
4517	check();
4518	}
4519	}
4520
4521	// Recursive helper for above mergeClones method.
4522	template <typename DerivedCCG, typename FuncTy, typename CallTy>
4523	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::mergeClones(
4524	ContextNode Node, DenseSet<const* ContextNode *> &Visited,
4525	DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode) {
4526	auto Inserted = Visited.insert(Node);
4527	if (!Inserted.second)
4528	return;
4529
4530	// Iteratively perform merging on this node to handle new caller nodes created
4531	// during the recursive traversal. We could do something more elegant such as
4532	// maintain a worklist, but this is a simple approach that doesn't cause a
4533	// measureable compile time effect, as most nodes don't have many caller
4534	// edges to check.
4535	bool FoundUnvisited = true;
4536	unsigned Iters = `0`;
4537	while (FoundUnvisited) {
4538	Iters++;
4539	FoundUnvisited = false;
4540	// Make a copy since the recursive call may move a caller edge to a new
4541	// callee, messing up the iterator.
4542	auto CallerEdges = Node->CallerEdges;
4543	for (auto CallerEdge : CallerEdges) {
4544	// Skip any caller edge moved onto a different callee during recursion.
4545	if (CallerEdge->Callee != Node)
4546	continue;
4547	// If we found an unvisited caller, note that we should check the caller
4548	// edges again as mergeClones may add or change caller nodes.
4549	if (DoMergeIteration && !Visited.contains(CallerEdge->Caller))
4550	FoundUnvisited = true;
4551	mergeClones(CallerEdge->Caller, Visited, ContextIdToAllocationNode);
4552	}
4553	}
4554
4555	TotalMergeInvokes ++;
4556	TotalMergeIters += Iters;
4557	if (Iters > MaxMergeIters)
4558	MaxMergeIters = Iters;
4559
4560	// Merge for this node after we handle its callers.
4561	mergeNodeCalleeClones(Node, Visited, ContextIdToAllocationNode);
4562	}
4563
4564	template <typename DerivedCCG, typename FuncTy, typename CallTy>
4565	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::mergeNodeCalleeClones(
4566	ContextNode Node, DenseSet<const* ContextNode *> &Visited,
4567	DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode) {
4568	// Ignore Node if we moved all of its contexts to clones.
4569	if (Node->emptyContextIds())
4570	return;
4571
4572	// First identify groups of clones among Node's callee edges, by building
4573	// a map from each callee base node to the associated callee edges from Node.
4574	MapVector<ContextNode *, std::vector<std::shared_ptr<ContextEdge>>>
4575	OrigNodeToCloneEdges;
4576	for (const auto &E : Node->CalleeEdges) {
4577	auto *Callee = E->Callee;
4578	if (!Callee->CloneOf && Callee->Clones.empty())
4579	continue;
4580	ContextNode *Base = Callee->getOrigNode();
4581	OrigNodeToCloneEdges[Base].push_back(E);
4582	}
4583
4584	// Helper for callee edge sorting below. Return true if A's callee has fewer
4585	// caller edges than B, or if A is a clone and B is not, or if A's first
4586	// context id is smaller than B's.
4587	auto CalleeCallerEdgeLessThan = [](const std::shared_ptr<ContextEdge> &A,
4588	const std::shared_ptr<ContextEdge> &B) {
4589	if (A->Callee->CallerEdges.size() != B->Callee->CallerEdges.size())
4590	return A->Callee->CallerEdges.size() < B->Callee->CallerEdges.size();
4591	if (A->Callee->CloneOf && !B->Callee->CloneOf)
4592	return true;
4593	else if (!A->Callee->CloneOf && B->Callee->CloneOf)
4594	return false;
4595	// Use the first context id for each edge as a
4596	// tie-breaker.
4597	return A->ContextIds.begin() < B->ContextIds.begin();
4598	};
4599
4600	// Process each set of callee clones called by Node, performing the needed
4601	// merging.
4602	for (auto Entry : OrigNodeToCloneEdges) {
4603	// CalleeEdges is the set of edges from Node reaching callees that are
4604	// mutual clones of each other.
4605	auto &CalleeEdges = Entry.second;
4606	auto NumCalleeClones = CalleeEdges.size();
4607	// A single edge means there is no merging needed.
4608	if (NumCalleeClones == `1`)
4609	continue;
4610	// Sort the CalleeEdges calling this group of clones in ascending order of
4611	// their caller edge counts, putting the original non-clone node first in
4612	// cases of a tie. This simplifies finding an existing node to use as the
4613	// merge node.
4614	llvm::stable_sort(CalleeEdges, CalleeCallerEdgeLessThan);
4615
4616	/// Find other callers of the given set of callee edges that can
4617	/// share the same callee merge node. See the comments at this method
4618	/// definition for details.
4619	DenseSet<ContextNode *> OtherCallersToShareMerge;
4620	findOtherCallersToShareMerge(Node, CalleeEdges, ContextIdToAllocationNode,
4621	OtherCallersToShareMerge);
4622
4623	// Now do the actual merging. Identify existing or create a new MergeNode
4624	// during the first iteration. Move each callee over, along with edges from
4625	// other callers we've determined above can share the same merge node.
4626	ContextNode MergeNode = nullptr*;
4627	DenseMap<ContextNode , unsigned*> CallerToMoveCount;
4628	for (auto CalleeEdge : CalleeEdges) {
4629	auto *OrigCallee = CalleeEdge->Callee;
4630	// If we don't have a MergeNode yet (only happens on the first iteration,
4631	// as a new one will be created when we go to move the first callee edge
4632	// over as needed), see if we can use this callee.
4633	if (!MergeNode) {
4634	// If there are no other callers, simply use this callee.
4635	if (CalleeEdge->Callee->CallerEdges.size() == `1`) {
4636	MergeNode = OrigCallee;
4637	NonNewMergedNodes ++;
4638	continue;
4639	}
4640	// Otherwise, if we have identified other caller nodes that can share
4641	// the merge node with Node, see if all of OrigCallee's callers are
4642	// going to share the same merge node. In that case we can use callee
4643	// (since all of its callers would move to the new merge node).
4644	if (!OtherCallersToShareMerge.empty()) {
4645	bool MoveAllCallerEdges = true;
4646	for (auto CalleeCallerE : OrigCallee->CallerEdges) {
4647	if (CalleeCallerE == CalleeEdge)
4648	continue;
4649	if (!OtherCallersToShareMerge.contains(CalleeCallerE->Caller)) {
4650	MoveAllCallerEdges = false;
4651	break;
4652	}
4653	}
4654	// If we are going to move all callers over, we can use this callee as
4655	// the MergeNode.
4656	if (MoveAllCallerEdges) {
4657	MergeNode = OrigCallee;
4658	NonNewMergedNodes ++;
4659	continue;
4660	}
4661	}
4662	}
4663	// Move this callee edge, creating a new merge node if necessary.
4664	if (MergeNode) {
4665	assert(MergeNode != OrigCallee);
4666	moveEdgeToExistingCalleeClone(Edge: CalleeEdge, NewCallee: MergeNode,
4667	/NewClone/ false);
4668	} else {
4669	MergeNode = moveEdgeToNewCalleeClone(Edge: CalleeEdge);
4670	NewMergedNodes ++;
4671	}
4672	// Now move all identified edges from other callers over to the merge node
4673	// as well.
4674	if (!OtherCallersToShareMerge.empty()) {
4675	// Make and iterate over a copy of OrigCallee's caller edges because
4676	// some of these will be moved off of the OrigCallee and that would mess
4677	// up the iteration from OrigCallee.
4678	auto OrigCalleeCallerEdges = OrigCallee->CallerEdges;
4679	for (auto &CalleeCallerE : OrigCalleeCallerEdges) {
4680	if (CalleeCallerE == CalleeEdge)
4681	continue;
4682	if (!OtherCallersToShareMerge.contains(CalleeCallerE->Caller))
4683	continue;
4684	CallerToMoveCount[CalleeCallerE->Caller]++;
4685	moveEdgeToExistingCalleeClone(Edge: CalleeCallerE, NewCallee: MergeNode,
4686	/NewClone/ false);
4687	}
4688	}
4689	removeNoneTypeCalleeEdges(Node: OrigCallee);
4690	removeNoneTypeCalleeEdges(Node: MergeNode);
4691	}
4692	}
4693	}
4694
4695	// Look for other nodes that have edges to the same set of callee
4696	// clones as the current Node. Those can share the eventual merge node
4697	// (reducing cloning and binary size overhead) iff:
4698	// - they have edges to the same set of callee clones
4699	// - each callee edge reaches a subset of the same allocations as Node's
4700	// corresponding edge to the same callee clone.
4701	// The second requirement is to ensure that we don't undo any of the
4702	// necessary cloning to distinguish contexts with different allocation
4703	// behavior.
4704	// FIXME: This is somewhat conservative, as we really just need to ensure
4705	// that they don't reach the same allocations as contexts on edges from Node
4706	// going to any of the other* callee clones being merged. However, that*
4707	// requires more tracking and checking to get right.
4708	template <typename DerivedCCG, typename FuncTy, typename CallTy>
4709	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
4710	findOtherCallersToShareMerge(
4711	ContextNode *Node,
4712	std::vector<std::shared_ptr<ContextEdge>> &CalleeEdges,
4713	DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode,
4714	DenseSet<ContextNode *> &OtherCallersToShareMerge) {
4715	auto NumCalleeClones = CalleeEdges.size();
4716	// This map counts how many edges to the same callee clone exist for other
4717	// caller nodes of each callee clone.
4718	DenseMap<ContextNode , unsigned*> OtherCallersToSharedCalleeEdgeCount;
4719	// Counts the number of other caller nodes that have edges to all callee
4720	// clones that don't violate the allocation context checking.
4721	unsigned PossibleOtherCallerNodes = `0`;
4722
4723	// We only need to look at other Caller nodes if the first callee edge has
4724	// multiple callers (recall they are sorted in ascending order above).
4725	if (CalleeEdges[`0`]->Callee->CallerEdges.size() < `2`)
4726	return;
4727
4728	// For each callee edge:
4729	// - Collect the count of other caller nodes calling the same callees.
4730	// - Collect the alloc nodes reached by contexts on each callee edge.
4731	DenseMap<ContextEdge , DenseSet<ContextNode >> CalleeEdgeToAllocNodes;
4732	for (auto CalleeEdge : CalleeEdges) {
4733	assert(CalleeEdge->Callee->CallerEdges.size() > `1`);
4734	// For each other caller of the same callee, increment the count of
4735	// edges reaching the same callee clone.
4736	for (auto CalleeCallerEdges : CalleeEdge->Callee->CallerEdges) {
4737	if (CalleeCallerEdges->Caller == Node) {
4738	assert(CalleeCallerEdges == CalleeEdge);
4739	continue;
4740	}
4741	OtherCallersToSharedCalleeEdgeCount[CalleeCallerEdges->Caller]++;
4742	// If this caller edge now reaches all of the same callee clones,
4743	// increment the count of candidate other caller nodes.
4744	if (OtherCallersToSharedCalleeEdgeCount[CalleeCallerEdges->Caller] ==
4745	NumCalleeClones)
4746	PossibleOtherCallerNodes++;
4747	}
4748	// Collect the alloc nodes reached by contexts on each callee edge, for
4749	// later analysis.
4750	for (auto Id : CalleeEdge->getContextIds()) {
4751	auto *Alloc = ContextIdToAllocationNode.lookup(Id);
4752	if (!Alloc) {
4753	// FIXME: unclear why this happens occasionally, presumably
4754	// imperfect graph updates possibly with recursion.
4755	MissingAllocForContextId ++;
4756	continue;
4757	}
4758	CalleeEdgeToAllocNodes[CalleeEdge.get()].insert(Alloc);
4759	}
4760	}
4761
4762	// Now walk the callee edges again, and make sure that for each candidate
4763	// caller node all of its edges to the callees reach the same allocs (or
4764	// a subset) as those along the corresponding callee edge from Node.
4765	for (auto CalleeEdge : CalleeEdges) {
4766	assert(CalleeEdge->Callee->CallerEdges.size() > `1`);
4767	// Stop if we do not have any (more) candidate other caller nodes.
4768	if (!PossibleOtherCallerNodes)
4769	break;
4770	auto &CurCalleeAllocNodes = CalleeEdgeToAllocNodes[CalleeEdge.get()];
4771	// Check each other caller of this callee clone.
4772	for (auto &CalleeCallerE : CalleeEdge->Callee->CallerEdges) {
4773	// Not interested in the callee edge from Node itself.
4774	if (CalleeCallerE == CalleeEdge)
4775	continue;
4776	// Skip any callers that didn't have callee edges to all the same
4777	// callee clones.
4778	if (OtherCallersToSharedCalleeEdgeCount[CalleeCallerE->Caller] !=
4779	NumCalleeClones)
4780	continue;
4781	// Make sure that each context along edge from candidate caller node
4782	// reaches an allocation also reached by this callee edge from Node.
4783	for (auto Id : CalleeCallerE->getContextIds()) {
4784	auto *Alloc = ContextIdToAllocationNode.lookup(Id);
4785	if (!Alloc)
4786	continue;
4787	// If not, simply reset the map entry to 0 so caller is ignored, and
4788	// reduce the count of candidate other caller nodes.
4789	if (!CurCalleeAllocNodes.contains(Alloc)) {
4790	OtherCallersToSharedCalleeEdgeCount[CalleeCallerE->Caller] = `0`;
4791	PossibleOtherCallerNodes--;
4792	break;
4793	}
4794	}
4795	}
4796	}
4797
4798	if (!PossibleOtherCallerNodes)
4799	return;
4800
4801	// Build the set of other caller nodes that can use the same callee merge
4802	// node.
4803	for (auto &[OtherCaller, Count] : OtherCallersToSharedCalleeEdgeCount) {
4804	if (Count != NumCalleeClones)
4805	continue;
4806	OtherCallersToShareMerge.insert(OtherCaller);
4807	}
4808	}
4809
4810	// This method assigns cloned callsites to functions, cloning the functions as
4811	// needed. The assignment is greedy and proceeds roughly as follows:
4812	//
4813	// For each function Func:
4814	// For each call with graph Node having clones:
4815	// Initialize ClonesWorklist to Node and its clones
4816	// Initialize NodeCloneCount to 0
4817	// While ClonesWorklist is not empty:
4818	// Clone = pop front ClonesWorklist
4819	// NodeCloneCount++
4820	// If Func has been cloned less than NodeCloneCount times:
4821	// If NodeCloneCount is 1:
4822	// Assign Clone to original Func
4823	// Continue
4824	// Create a new function clone
4825	// If other callers not assigned to call a function clone yet:
4826	// Assign them to call new function clone
4827	// Continue
4828	// Assign any other caller calling the cloned version to new clone
4829	//
4830	// For each caller of Clone:
4831	// If caller is assigned to call a specific function clone:
4832	// If we cannot assign Clone to that function clone:
4833	// Create new callsite Clone NewClone
4834	// Add NewClone to ClonesWorklist
4835	// Continue
4836	// Assign Clone to existing caller's called function clone
4837	// Else:
4838	// If Clone not already assigned to a function clone:
4839	// Assign to first function clone without assignment
4840	// Assign caller to selected function clone
4841	// For each call with graph Node having clones:
4842	// If number func clones > number call's callsite Node clones:
4843	// Record func CallInfo clones without Node clone in UnassignedCallClones
4844	// For callsite Nodes in DFS order from allocations:
4845	// If IsAllocation:
4846	// Update allocation with alloc type
4847	// Else:
4848	// For Call, all MatchingCalls, and associated UnnassignedCallClones:
4849	// Update call to call recorded callee clone
4850	//
4851	template <typename DerivedCCG, typename FuncTy, typename CallTy>
4852	bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::assignFunctions() {
4853	bool Changed = false;
4854
4855	mergeClones();
4856
4857	// Keep track of the assignment of nodes (callsites) to function clones they
4858	// call.
4859	DenseMap<ContextNode *, FuncInfo> CallsiteToCalleeFuncCloneMap;
4860
4861	// Update caller node to call function version CalleeFunc, by recording the
4862	// assignment in CallsiteToCalleeFuncCloneMap.
4863	auto RecordCalleeFuncOfCallsite = [&](ContextNode *Caller,
4864	const FuncInfo &CalleeFunc) {
4865	assert(Caller->hasCall());
4866	CallsiteToCalleeFuncCloneMap[Caller] = CalleeFunc;
4867	};
4868
4869	// Information for a single clone of this Func.
4870	struct FuncCloneInfo {
4871	// The function clone.
4872	FuncInfo FuncClone;
4873	// Remappings of each call of interest (from original uncloned call to the
4874	// corresponding cloned call in this function clone).
4875	DenseMap<CallInfo, CallInfo> CallMap;
4876	};
4877
4878	// Map to keep track of information needed to update calls in function clones
4879	// when their corresponding callsite node was not itself cloned for that
4880	// function clone. Because of call context pruning (i.e. we only keep as much
4881	// caller information as needed to distinguish hot vs cold), we may not have
4882	// caller edges coming to each callsite node from all possible function
4883	// callers. A function clone may get created for other callsites in the
4884	// function for which there are caller edges that were not pruned. Any other
4885	// callsites in that function clone, which were not themselved cloned for
4886	// that function clone, should get updated the same way as the corresponding
4887	// callsite in the original function (which may call a clone of its callee).
4888	//
4889	// We build this map after completing function cloning for each function, so
4890	// that we can record the information from its call maps before they are
4891	// destructed. The map will be used as we update calls to update any still
4892	// unassigned call clones. Note that we may create new node clones as we clone
4893	// other functions, so later on we check which node clones were still not
4894	// created. To this end, the inner map is a map from function clone number to
4895	// the list of calls cloned for that function (can be more than one due to the
4896	// Node's MatchingCalls array).
4897	//
4898	// The alternative is creating new callsite clone nodes below as we clone the
4899	// function, but that is tricker to get right and likely more overhead.
4900	//
4901	// Inner map is a std::map so sorted by key (clone number), in order to get
4902	// ordered remarks in the full LTO case.
4903	DenseMap<const ContextNode , std::map<unsigned*, SmallVector<CallInfo, `0`>>>
4904	UnassignedCallClones;
4905
4906	// Walk all functions for which we saw calls with memprof metadata, and handle
4907	// cloning for each of its calls.
4908	for (auto &[Func, CallsWithMetadata] : FuncToCallsWithMetadata) {
4909	FuncInfo OrigFunc(Func);
4910	// Map from each clone number of OrigFunc to information about that function
4911	// clone (the function clone FuncInfo and call remappings). The index into
4912	// the vector is the clone number, as function clones are created and
4913	// numbered sequentially.
4914	std::vector<FuncCloneInfo> FuncCloneInfos;
4915	for (auto &Call : CallsWithMetadata) {
4916	ContextNode *Node = getNodeForInst(C: Call);
4917	// Skip call if we do not have a node for it (all uses of its stack ids
4918	// were either on inlined chains or pruned from the MIBs), or if we did
4919	// not create any clones for it.
4920	if (!Node \|\| Node->Clones.empty())
4921	continue;
4922	assert(Node->hasCall() &&
4923	"Not having a call should have prevented cloning");
4924
4925	// Track the assignment of function clones to clones of the current
4926	// callsite Node being handled.
4927	std::map<FuncInfo, ContextNode *> FuncCloneToCurNodeCloneMap;
4928
4929	// Assign callsite version CallsiteClone to function version FuncClone,
4930	// and also assign (possibly cloned) Call to CallsiteClone.
4931	auto AssignCallsiteCloneToFuncClone = [&](const FuncInfo &FuncClone,
4932	CallInfo &Call,
4933	ContextNode *CallsiteClone,
4934	bool IsAlloc) {
4935	// Record the clone of callsite node assigned to this function clone.
4936	FuncCloneToCurNodeCloneMap[FuncClone] = CallsiteClone;
4937
4938	assert(FuncCloneInfos.size() > FuncClone.cloneNo());
4939	DenseMap<CallInfo, CallInfo> &CallMap =
4940	FuncCloneInfos[FuncClone.cloneNo()].CallMap;
4941	CallInfo CallClone(Call);
4942	if (auto It = CallMap.find(Call); It != CallMap.end())
4943	CallClone = It->second;
4944	CallsiteClone->setCall(CallClone);
4945	// Need to do the same for all matching calls.
4946	for (auto &MatchingCall : Node->MatchingCalls) {
4947	CallInfo CallClone(MatchingCall);
4948	if (auto It = CallMap.find(MatchingCall); It != CallMap.end())
4949	CallClone = It->second;
4950	// Updates the call in the list.
4951	MatchingCall = CallClone;
4952	}
4953	};
4954
4955	// Invokes moveEdgeToNewCalleeClone which creates a new clone, and then
4956	// performs the necessary fixups (removing none type edges, and
4957	// importantly, propagating any function call assignment of the original
4958	// node to the new clone).
4959	auto MoveEdgeToNewCalleeCloneAndSetUp =
4960	[&](const std::shared_ptr<ContextEdge> &Edge) {
4961	ContextNode *OrigCallee = Edge->Callee;
4962	ContextNode *NewClone = moveEdgeToNewCalleeClone(Edge);
4963	removeNoneTypeCalleeEdges(Node: NewClone);
4964	assert(NewClone->AllocTypes != (uint8_t)AllocationType::None);
4965	// If the original Callee was already assigned to call a specific
4966	// function version, make sure its new clone is assigned to call
4967	// that same function clone.
4968	if (CallsiteToCalleeFuncCloneMap.count(OrigCallee))
4969	RecordCalleeFuncOfCallsite(
4970	NewClone, CallsiteToCalleeFuncCloneMap[OrigCallee]);
4971	return NewClone;
4972	};
4973
4974	// Keep track of the clones of callsite Node that need to be assigned to
4975	// function clones. This list may be expanded in the loop body below if we
4976	// find additional cloning is required.
4977	std::deque<ContextNode *> ClonesWorklist;
4978	// Ignore original Node if we moved all of its contexts to clones.
4979	if (!Node->emptyContextIds())
4980	ClonesWorklist.push_back(Node);
4981	llvm::append_range(ClonesWorklist, Node->Clones);
4982
4983	// Now walk through all of the clones of this callsite Node that we need,
4984	// and determine the assignment to a corresponding clone of the current
4985	// function (creating new function clones as needed).
4986	unsigned NodeCloneCount = `0`;
4987	while (!ClonesWorklist.empty()) {
4988	ContextNode *Clone = ClonesWorklist.front();
4989	ClonesWorklist.pop_front();
4990	NodeCloneCount++;
4991	if (VerifyNodes)
4992	checkNode<DerivedCCG, FuncTy, CallTy>(Clone);
4993
4994	// Need to create a new function clone if we have more callsite clones
4995	// than existing function clones, which would have been assigned to an
4996	// earlier clone in the list (we assign callsite clones to function
4997	// clones greedily).
4998	if (FuncCloneInfos.size() < NodeCloneCount) {
4999	// If this is the first callsite copy, assign to original function.
5000	if (NodeCloneCount == `1`) {
5001	// Since FuncCloneInfos is empty in this case, no clones have
5002	// been created for this function yet, and no callers should have
5003	// been assigned a function clone for this callee node yet.
5004	assert(llvm::none_of(
5005	Clone->CallerEdges, [&](const std::shared_ptr<ContextEdge> &E) {
5006	return CallsiteToCalleeFuncCloneMap.count(E->Caller);
5007	}));
5008	// Initialize with empty call map, assign Clone to original function
5009	// and its callers, and skip to the next clone.
5010	FuncCloneInfos.push_back(
5011	{OrigFunc, DenseMap<CallInfo, CallInfo>()});
5012	AssignCallsiteCloneToFuncClone(
5013	OrigFunc, Call, Clone,
5014	AllocationCallToContextNodeMap.count(Call));
5015	for (auto &CE : Clone->CallerEdges) {
5016	// Ignore any caller that does not have a recorded callsite Call.
5017	if (!CE->Caller->hasCall())
5018	continue;
5019	RecordCalleeFuncOfCallsite(CE->Caller, OrigFunc);
5020	}
5021	continue;
5022	}
5023
5024	// First locate which copy of OrigFunc to clone again. If a caller
5025	// of this callsite clone was already assigned to call a particular
5026	// function clone, we need to redirect all of those callers to the
5027	// new function clone, and update their other callees within this
5028	// function.
5029	FuncInfo PreviousAssignedFuncClone;
5030	auto EI = llvm::find_if(
5031	Clone->CallerEdges, [&](const std::shared_ptr<ContextEdge> &E) {
5032	return CallsiteToCalleeFuncCloneMap.count(E->Caller);
5033	});
5034	bool CallerAssignedToCloneOfFunc = false;
5035	if (EI != Clone->CallerEdges.end()) {
5036	const std::shared_ptr<ContextEdge> &Edge = *EI;
5037	PreviousAssignedFuncClone =
5038	CallsiteToCalleeFuncCloneMap[Edge->Caller];
5039	CallerAssignedToCloneOfFunc = true;
5040	}
5041
5042	// Clone function and save it along with the CallInfo map created
5043	// during cloning in the FuncCloneInfos.
5044	DenseMap<CallInfo, CallInfo> NewCallMap;
5045	unsigned CloneNo = FuncCloneInfos.size();
5046	assert(CloneNo > `0` && "Clone 0 is the original function, which "
5047	"should already exist in the map");
5048	FuncInfo NewFuncClone = cloneFunctionForCallsite(
5049	Func&: OrigFunc, Call, CallMap&: NewCallMap, CallsWithMetadataInFunc&: CallsWithMetadata, CloneNo);
5050	FuncCloneInfos.push_back({NewFuncClone, std::move(NewCallMap)});
5051	FunctionClonesAnalysis ++;
5052	Changed = true;
5053
5054	// If no caller callsites were already assigned to a clone of this
5055	// function, we can simply assign this clone to the new func clone
5056	// and update all callers to it, then skip to the next clone.
5057	if (!CallerAssignedToCloneOfFunc) {
5058	AssignCallsiteCloneToFuncClone(
5059	NewFuncClone, Call, Clone,
5060	AllocationCallToContextNodeMap.count(Call));
5061	for (auto &CE : Clone->CallerEdges) {
5062	// Ignore any caller that does not have a recorded callsite Call.
5063	if (!CE->Caller->hasCall())
5064	continue;
5065	RecordCalleeFuncOfCallsite(CE->Caller, NewFuncClone);
5066	}
5067	continue;
5068	}
5069
5070	// We may need to do additional node cloning in this case.
5071	// Reset the CallsiteToCalleeFuncCloneMap entry for any callers
5072	// that were previously assigned to call PreviousAssignedFuncClone,
5073	// to record that they now call NewFuncClone.
5074	// The none type edge removal may remove some of this Clone's caller
5075	// edges, if it is reached via another of its caller's callees.
5076	// Iterate over a copy and skip any that were removed.
5077	auto CallerEdges = Clone->CallerEdges;
5078	for (auto CE : CallerEdges) {
5079	// Skip any that have been removed on an earlier iteration.
5080	if (CE->isRemoved()) {
5081	assert(!is_contained(Clone->CallerEdges, CE));
5082	continue;
5083	}
5084	assert(CE);
5085	// Ignore any caller that does not have a recorded callsite Call.
5086	if (!CE->Caller->hasCall())
5087	continue;
5088
5089	if (!CallsiteToCalleeFuncCloneMap.count(CE->Caller) \|\|
5090	// We subsequently fall through to later handling that
5091	// will perform any additional cloning required for
5092	// callers that were calling other function clones.
5093	CallsiteToCalleeFuncCloneMap[CE->Caller] !=
5094	PreviousAssignedFuncClone)
5095	continue;
5096
5097	RecordCalleeFuncOfCallsite(CE->Caller, NewFuncClone);
5098
5099	// If we are cloning a function that was already assigned to some
5100	// callers, then essentially we are creating new callsite clones
5101	// of the other callsites in that function that are reached by those
5102	// callers. Clone the other callees of the current callsite's caller
5103	// that were already assigned to PreviousAssignedFuncClone
5104	// accordingly. This is important since we subsequently update the
5105	// calls from the nodes in the graph and their assignments to callee
5106	// functions recorded in CallsiteToCalleeFuncCloneMap.
5107	// The none type edge removal may remove some of this caller's
5108	// callee edges, if it is reached via another of its callees.
5109	// Iterate over a copy and skip any that were removed.
5110	auto CalleeEdges = CE->Caller->CalleeEdges;
5111	for (auto CalleeEdge : CalleeEdges) {
5112	// Skip any that have been removed on an earlier iteration when
5113	// cleaning up newly None type callee edges.
5114	if (CalleeEdge->isRemoved()) {
5115	assert(!is_contained(CE->Caller->CalleeEdges, CalleeEdge));
5116	continue;
5117	}
5118	assert(CalleeEdge);
5119	ContextNode *Callee = CalleeEdge->Callee;
5120	// Skip the current callsite, we are looking for other
5121	// callsites Caller calls, as well as any that does not have a
5122	// recorded callsite Call.
5123	if (Callee == Clone \|\| !Callee->hasCall())
5124	continue;
5125	// Skip direct recursive calls. We don't need/want to clone the
5126	// caller node again, and this loop will not behave as expected if
5127	// we tried.
5128	if (Callee == CalleeEdge->Caller)
5129	continue;
5130	ContextNode *NewClone =
5131	MoveEdgeToNewCalleeCloneAndSetUp(CalleeEdge);
5132	// Moving the edge may have resulted in some none type
5133	// callee edges on the original Callee.
5134	removeNoneTypeCalleeEdges(Node: Callee);
5135	// Update NewClone with the new Call clone of this callsite's Call
5136	// created for the new function clone created earlier.
5137	// Recall that we have already ensured when building the graph
5138	// that each caller can only call callsites within the same
5139	// function, so we are guaranteed that Callee Call is in the
5140	// current OrigFunc.
5141	// CallMap is set up as indexed by original Call at clone 0.
5142	CallInfo OrigCall(Callee->getOrigNode()->Call);
5143	OrigCall.setCloneNo(`0`);
5144	DenseMap<CallInfo, CallInfo> &CallMap =
5145	FuncCloneInfos[NewFuncClone.cloneNo()].CallMap;
5146	assert(CallMap.count(OrigCall));
5147	CallInfo NewCall(CallMap[OrigCall]);
5148	assert(NewCall);
5149	NewClone->setCall(NewCall);
5150	// Need to do the same for all matching calls.
5151	for (auto &MatchingCall : NewClone->MatchingCalls) {
5152	CallInfo OrigMatchingCall(MatchingCall);
5153	OrigMatchingCall.setCloneNo(`0`);
5154	assert(CallMap.count(OrigMatchingCall));
5155	CallInfo NewCall(CallMap[OrigMatchingCall]);
5156	assert(NewCall);
5157	// Updates the call in the list.
5158	MatchingCall = NewCall;
5159	}
5160	}
5161	}
5162	// Fall through to handling below to perform the recording of the
5163	// function for this callsite clone. This enables handling of cases
5164	// where the callers were assigned to different clones of a function.
5165	}
5166
5167	auto FindFirstAvailFuncClone = [&]() {
5168	// Find first function in FuncCloneInfos without an assigned
5169	// clone of this callsite Node. We should always have one
5170	// available at this point due to the earlier cloning when the
5171	// FuncCloneInfos size was smaller than the clone number.
5172	for (auto &CF : FuncCloneInfos) {
5173	if (!FuncCloneToCurNodeCloneMap.count(CF.FuncClone))
5174	return CF.FuncClone;
5175	}
5176	llvm_unreachable(
5177	"Expected an available func clone for this callsite clone");
5178	};
5179
5180	// See if we can use existing function clone. Walk through
5181	// all caller edges to see if any have already been assigned to
5182	// a clone of this callsite's function. If we can use it, do so. If not,
5183	// because that function clone is already assigned to a different clone
5184	// of this callsite, then we need to clone again.
5185	// Basically, this checking is needed to handle the case where different
5186	// caller functions/callsites may need versions of this function
5187	// containing different mixes of callsite clones across the different
5188	// callsites within the function. If that happens, we need to create
5189	// additional function clones to handle the various combinations.
5190	//
5191	// Keep track of any new clones of this callsite created by the
5192	// following loop, as well as any existing clone that we decided to
5193	// assign this clone to.
5194	std::map<FuncInfo, ContextNode *> FuncCloneToNewCallsiteCloneMap;
5195	FuncInfo FuncCloneAssignedToCurCallsiteClone;
5196	// Iterate over a copy of Clone's caller edges, since we may need to
5197	// remove edges in the moveEdgeTo methods, and this simplifies the*
5198	// handling and makes it less error-prone.
5199	auto CloneCallerEdges = Clone->CallerEdges;
5200	for (auto &Edge : CloneCallerEdges) {
5201	// Skip removed edges (due to direct recursive edges updated when
5202	// updating callee edges when moving an edge and subsequently
5203	// removed by call to removeNoneTypeCalleeEdges on the Clone).
5204	if (Edge->isRemoved())
5205	continue;
5206	// Ignore any caller that does not have a recorded callsite Call.
5207	if (!Edge->Caller->hasCall())
5208	continue;
5209	// If this caller already assigned to call a version of OrigFunc, need
5210	// to ensure we can assign this callsite clone to that function clone.
5211	if (CallsiteToCalleeFuncCloneMap.count(Edge->Caller)) {
5212	FuncInfo FuncCloneCalledByCaller =
5213	CallsiteToCalleeFuncCloneMap[Edge->Caller];
5214	// First we need to confirm that this function clone is available
5215	// for use by this callsite node clone.
5216	//
5217	// While FuncCloneToCurNodeCloneMap is built only for this Node and
5218	// its callsite clones, one of those callsite clones X could have
5219	// been assigned to the same function clone called by Edge's caller
5220	// - if Edge's caller calls another callsite within Node's original
5221	// function, and that callsite has another caller reaching clone X.
5222	// We need to clone Node again in this case.
5223	if ((FuncCloneToCurNodeCloneMap.count(FuncCloneCalledByCaller) &&
5224	FuncCloneToCurNodeCloneMap[FuncCloneCalledByCaller] !=
5225	Clone) \|\|
5226	// Detect when we have multiple callers of this callsite that
5227	// have already been assigned to specific, and different, clones
5228	// of OrigFunc (due to other unrelated callsites in Func they
5229	// reach via call contexts). Is this Clone of callsite Node
5230	// assigned to a different clone of OrigFunc? If so, clone Node
5231	// again.
5232	(FuncCloneAssignedToCurCallsiteClone &&
5233	FuncCloneAssignedToCurCallsiteClone !=
5234	FuncCloneCalledByCaller)) {
5235	// We need to use a different newly created callsite clone, in
5236	// order to assign it to another new function clone on a
5237	// subsequent iteration over the Clones array (adjusted below).
5238	// Note we specifically do not reset the
5239	// CallsiteToCalleeFuncCloneMap entry for this caller, so that
5240	// when this new clone is processed later we know which version of
5241	// the function to copy (so that other callsite clones we have
5242	// assigned to that function clone are properly cloned over). See
5243	// comments in the function cloning handling earlier.
5244
5245	// Check if we already have cloned this callsite again while
5246	// walking through caller edges, for a caller calling the same
5247	// function clone. If so, we can move this edge to that new clone
5248	// rather than creating yet another new clone.
5249	if (FuncCloneToNewCallsiteCloneMap.count(
5250	FuncCloneCalledByCaller)) {
5251	ContextNode *NewClone =
5252	FuncCloneToNewCallsiteCloneMap[FuncCloneCalledByCaller];
5253	moveEdgeToExistingCalleeClone(Edge, NewCallee: NewClone);
5254	// Cleanup any none type edges cloned over.
5255	removeNoneTypeCalleeEdges(Node: NewClone);
5256	} else {
5257	// Create a new callsite clone.
5258	ContextNode *NewClone = MoveEdgeToNewCalleeCloneAndSetUp(Edge);
5259	FuncCloneToNewCallsiteCloneMap[FuncCloneCalledByCaller] =
5260	NewClone;
5261	// Add to list of clones and process later.
5262	ClonesWorklist.push_back(NewClone);
5263	}
5264	// Moving the caller edge may have resulted in some none type
5265	// callee edges.
5266	removeNoneTypeCalleeEdges(Node: Clone);
5267	// We will handle the newly created callsite clone in a subsequent
5268	// iteration over this Node's Clones.
5269	continue;
5270	}
5271
5272	// Otherwise, we can use the function clone already assigned to this
5273	// caller.
5274	if (!FuncCloneAssignedToCurCallsiteClone) {
5275	FuncCloneAssignedToCurCallsiteClone = FuncCloneCalledByCaller;
5276	// Assign Clone to FuncCloneCalledByCaller
5277	AssignCallsiteCloneToFuncClone(
5278	FuncCloneCalledByCaller, Call, Clone,
5279	AllocationCallToContextNodeMap.count(Call));
5280	} else
5281	// Don't need to do anything - callsite is already calling this
5282	// function clone.
5283	assert(FuncCloneAssignedToCurCallsiteClone ==
5284	FuncCloneCalledByCaller);
5285
5286	} else {
5287	// We have not already assigned this caller to a version of
5288	// OrigFunc. Do the assignment now.
5289
5290	// First check if we have already assigned this callsite clone to a
5291	// clone of OrigFunc for another caller during this iteration over
5292	// its caller edges.
5293	if (!FuncCloneAssignedToCurCallsiteClone) {
5294	FuncCloneAssignedToCurCallsiteClone = FindFirstAvailFuncClone();
5295	assert(FuncCloneAssignedToCurCallsiteClone);
5296	// Assign Clone to FuncCloneAssignedToCurCallsiteClone
5297	AssignCallsiteCloneToFuncClone(
5298	FuncCloneAssignedToCurCallsiteClone, Call, Clone,
5299	AllocationCallToContextNodeMap.count(Call));
5300	} else
5301	assert(FuncCloneToCurNodeCloneMap
5302	[FuncCloneAssignedToCurCallsiteClone] == Clone);
5303	// Update callers to record function version called.
5304	RecordCalleeFuncOfCallsite(Edge->Caller,
5305	FuncCloneAssignedToCurCallsiteClone);
5306	}
5307	}
5308	// If we didn't assign a function clone to this callsite clone yet, e.g.
5309	// none of its callers has a non-null call, do the assignment here.
5310	// We want to ensure that every callsite clone is assigned to some
5311	// function clone, so that the call updates below work as expected.
5312	// In particular if this is the original callsite, we want to ensure it
5313	// is assigned to the original function, otherwise the original function
5314	// will appear available for assignment to other callsite clones,
5315	// leading to unintended effects. For one, the unknown and not updated
5316	// callers will call into cloned paths leading to the wrong hints,
5317	// because they still call the original function (clone 0). Also,
5318	// because all callsites start out as being clone 0 by default, we can't
5319	// easily distinguish between callsites explicitly assigned to clone 0
5320	// vs those never assigned, which can lead to multiple updates of the
5321	// calls when invoking updateCall below, with mismatched clone values.
5322	// TODO: Add a flag to the callsite nodes or some other mechanism to
5323	// better distinguish and identify callsite clones that are not getting
5324	// assigned to function clones as expected.
5325	if (!FuncCloneAssignedToCurCallsiteClone) {
5326	FuncCloneAssignedToCurCallsiteClone = FindFirstAvailFuncClone();
5327	assert(FuncCloneAssignedToCurCallsiteClone &&
5328	"No available func clone for this callsite clone");
5329	AssignCallsiteCloneToFuncClone(
5330	FuncCloneAssignedToCurCallsiteClone, Call, Clone,
5331	/IsAlloc=/AllocationCallToContextNodeMap.contains(Call));
5332	}
5333	}
5334	if (VerifyCCG) {
5335	checkNode<DerivedCCG, FuncTy, CallTy>(Node);
5336	for (const auto &PE : Node->CalleeEdges)
5337	checkNode<DerivedCCG, FuncTy, CallTy>(PE->Callee);
5338	for (const auto &CE : Node->CallerEdges)
5339	checkNode<DerivedCCG, FuncTy, CallTy>(CE->Caller);
5340	for (auto *Clone : Node->Clones) {
5341	checkNode<DerivedCCG, FuncTy, CallTy>(Clone);
5342	for (const auto &PE : Clone->CalleeEdges)
5343	checkNode<DerivedCCG, FuncTy, CallTy>(PE->Callee);
5344	for (const auto &CE : Clone->CallerEdges)
5345	checkNode<DerivedCCG, FuncTy, CallTy>(CE->Caller);
5346	}
5347	}
5348	}
5349
5350	if (FuncCloneInfos.size() < `2`)
5351	continue;
5352
5353	// In this case there is more than just the original function copy.
5354	// Record call clones of any callsite nodes in the function that did not
5355	// themselves get cloned for all of the function clones.
5356	for (auto &Call : CallsWithMetadata) {
5357	ContextNode *Node = getNodeForInst(C: Call);
5358	if (!Node \|\| !Node->hasCall() \|\| Node->emptyContextIds())
5359	continue;
5360	// If Node has enough clones already to cover all function clones, we can
5361	// skip it. Need to add one for the original copy.
5362	// Use >= in case there were clones that were skipped due to having empty
5363	// context ids
5364	if (Node->Clones.size() + `1` >= FuncCloneInfos.size())
5365	continue;
5366	// First collect all function clones we cloned this callsite node for.
5367	// They may not be sequential due to empty clones e.g.
5368	DenseSet<unsigned> NodeCallClones;
5369	for (auto *C : Node->Clones)
5370	NodeCallClones.insert(C->Call.cloneNo());
5371	unsigned I = `0`;
5372	// Now check all the function clones.
5373	for (auto &FC : FuncCloneInfos) {
5374	// Function clones should be sequential.
5375	assert(FC.FuncClone.cloneNo() == I);
5376	// Skip the first clone which got the original call.
5377	// Also skip any other clones created for this Node.
5378	if (++I == `1` \|\| NodeCallClones.contains(V: I)) {
5379	continue;
5380	}
5381	// Record the call clones created for this callsite in this function
5382	// clone.
5383	auto &CallVector = UnassignedCallClones[Node][I];
5384	DenseMap<CallInfo, CallInfo> &CallMap = FC.CallMap;
5385	if (auto It = CallMap.find(Call); It != CallMap.end()) {
5386	CallInfo CallClone = It->second;
5387	CallVector.push_back(CallClone);
5388	} else {
5389	// All but the original clone (skipped earlier) should have an entry
5390	// for all calls.
5391	assert(false && "Expected to find call in CallMap");
5392	}
5393	// Need to do the same for all matching calls.
5394	for (auto &MatchingCall : Node->MatchingCalls) {
5395	if (auto It = CallMap.find(MatchingCall); It != CallMap.end()) {
5396	CallInfo CallClone = It->second;
5397	CallVector.push_back(CallClone);
5398	} else {
5399	// All but the original clone (skipped earlier) should have an entry
5400	// for all calls.
5401	assert(false && "Expected to find call in CallMap");
5402	}
5403	}
5404	}
5405	}
5406	}
5407
5408	uint8_t BothTypes =
5409	(uint8_t)AllocationType::Cold \| (uint8_t)AllocationType::NotCold;
5410
5411	auto UpdateCalls = [&](ContextNode *Node,
5412	DenseSet<const ContextNode *> &Visited,
5413	auto &&UpdateCalls) {
5414	auto Inserted = Visited.insert(Node);
5415	if (!Inserted.second)
5416	return;
5417
5418	for (auto *Clone : Node->Clones)
5419	UpdateCalls(Clone, Visited, UpdateCalls);
5420
5421	for (auto &Edge : Node->CallerEdges)
5422	UpdateCalls(Edge->Caller, Visited, UpdateCalls);
5423
5424	// Skip if either no call to update, or if we ended up with no context ids
5425	// (we moved all edges onto other clones).
5426	if (!Node->hasCall() \|\| Node->emptyContextIds())
5427	return;
5428
5429	if (Node->IsAllocation) {
5430	auto AT = allocTypeToUse(Node->AllocTypes);
5431	// If the allocation type is ambiguous, and more aggressive hinting
5432	// has been enabled via the MinClonedColdBytePercent flag, see if this
5433	// allocation should be hinted cold anyway because its fraction cold bytes
5434	// allocated is at least the given threshold.
5435	if (Node->AllocTypes == BothTypes && MinClonedColdBytePercent < `100` &&
5436	!ContextIdToContextSizeInfos.empty()) {
5437	uint64_t TotalCold = `0`;
5438	uint64_t Total = `0`;
5439	for (auto Id : Node->getContextIds()) {
5440	auto TypeI = ContextIdToAllocationType.find(Id);
5441	assert(TypeI != ContextIdToAllocationType.end());
5442	auto CSI = ContextIdToContextSizeInfos.find(Id);
5443	if (CSI != ContextIdToContextSizeInfos.end()) {
5444	for (auto &Info : CSI->second) {
5445	Total += Info.TotalSize;
5446	if (TypeI->second == AllocationType::Cold)
5447	TotalCold += Info.TotalSize;
5448	}
5449	}
5450	}
5451	if (TotalCold * `100` >= Total * MinClonedColdBytePercent)
5452	AT = AllocationType::Cold;
5453	}
5454	updateAllocationCall(Call&: Node->Call, AllocType: AT);
5455	assert(Node->MatchingCalls.empty());
5456	return;
5457	}
5458
5459	if (!CallsiteToCalleeFuncCloneMap.count(Node))
5460	return;
5461
5462	auto CalleeFunc = CallsiteToCalleeFuncCloneMap[Node];
5463	updateCall(CallerCall&: Node->Call, CalleeFunc);
5464	// Update all the matching calls as well.
5465	for (auto &Call : Node->MatchingCalls)
5466	updateCall(CallerCall&: Call, CalleeFunc);
5467
5468	// Now update all calls recorded earlier that are still in function clones
5469	// which don't have a clone of this callsite node.
5470	if (!UnassignedCallClones.contains(Node))
5471	return;
5472	DenseSet<unsigned> NodeCallClones;
5473	for (auto *C : Node->Clones)
5474	NodeCallClones.insert(C->Call.cloneNo());
5475	// Note that we already confirmed Node is in this map a few lines above.
5476	auto &ClonedCalls = UnassignedCallClones[Node];
5477	for (auto &[CloneNo, CallVector] : ClonedCalls) {
5478	// Should start at 1 as we never create an entry for original node.
5479	assert(CloneNo > `0`);
5480	// If we subsequently created a clone, skip this one.
5481	if (NodeCallClones.contains(V: CloneNo))
5482	continue;
5483	// Use the original Node's CalleeFunc.
5484	for (auto &Call : CallVector)
5485	updateCall(CallerCall&: Call, CalleeFunc);
5486	}
5487	};
5488
5489	// Performs DFS traversal starting from allocation nodes to update calls to
5490	// reflect cloning decisions recorded earlier. For regular LTO this will
5491	// update the actual calls in the IR to call the appropriate function clone
5492	// (and add attributes to allocation calls), whereas for ThinLTO the decisions
5493	// are recorded in the summary entries.
5494	DenseSet<const ContextNode *> Visited;
5495	for (auto &Entry : AllocationCallToContextNodeMap)
5496	UpdateCalls(Entry.second, Visited, UpdateCalls);
5497
5498	return Changed;
5499	}
5500
5501	// Compute a SHA1 hash of the callsite and alloc version information of clone I
5502	// in the summary, to use in detection of duplicate clones.
5503	uint64_t ComputeHash(const FunctionSummary FS, unsigned* I) {
5504	SHA1 Hasher;
5505	// Update hash with any callsites that call non-default (non-zero) callee
5506	// versions.
5507	for (auto &SN : FS->callsites()) {
5508	// In theory all callsites and allocs in this function should have the same
5509	// number of clone entries, but handle any discrepancies gracefully below
5510	// for NDEBUG builds.
5511	assert(
5512	SN.Clones.size() > I &&
5513	"Callsite summary has fewer entries than other summaries in function");
5514	if (SN.Clones.size() <= I \|\| !SN.Clones [I])
5515	continue;
5516	uint8_t Data[sizeof(SN.Clones [I])];
5517	support::endian::write32le(P: Data, V: SN.Clones [I]);
5518	Hasher.update(Data);
5519	}
5520	// Update hash with any allocs that have non-default (non-None) hints.
5521	for (auto &AN : FS->allocs()) {
5522	// In theory all callsites and allocs in this function should have the same
5523	// number of clone entries, but handle any discrepancies gracefully below
5524	// for NDEBUG builds.
5525	assert(AN.Versions.size() > I &&
5526	"Alloc summary has fewer entries than other summaries in function");
5527	if (AN.Versions.size() <= I \|\|
5528	(AllocationType)AN.Versions [I] == AllocationType::None)
5529	continue;
5530	Hasher.update(Data: ArrayRef<uint8_t>(&AN.Versions [I], `1`));
5531	}
5532	return support::endian::read64le(P: Hasher.result().data());
5533	}
5534
5535	static SmallVector<std::unique_ptr<ValueToValueMapTy>, `4`> createFunctionClones(
5536	Function &F, unsigned NumClones, Module &M, OptimizationRemarkEmitter &ORE,
5537	std::map<const Function , SmallPtrSet<const* GlobalAlias *, `1`>>
5538	&FuncToAliasMap,
5539	FunctionSummary *FS) {
5540	auto TakeDeclNameAndReplace = [](GlobalValue DeclGV, GlobalValue NewGV) {
5541	// We might have created this when adjusting callsite in another
5542	// function. It should be a declaration.
5543	assert(DeclGV->isDeclaration());
5544	NewGV->takeName(V: DeclGV);
5545	DeclGV->replaceAllUsesWith(V: NewGV);
5546	DeclGV->eraseFromParent();
5547	};
5548
5549	// Handle aliases to this function, and create analogous alias clones to the
5550	// provided clone of this function.
5551	auto CloneFuncAliases = [&](Function NewF, unsigned* I) {
5552	if (!FuncToAliasMap.count(x: &F))
5553	return;
5554	for (auto *A : FuncToAliasMap [&F]) {
5555	std::string AliasName = getMemProfFuncName(Base: A->getName(), CloneNo: I);
5556	auto *PrevA = M.getNamedAlias(Name: AliasName);
5557	auto *NewA = GlobalAlias::create(Ty: A->getValueType(),
5558	AddressSpace: A->getType()->getPointerAddressSpace(),
5559	Linkage: A->getLinkage(), Name: AliasName, Aliasee: NewF);
5560	NewA->copyAttributesFrom(Src: A);
5561	if (PrevA)
5562	TakeDeclNameAndReplace (PrevA, NewA);
5563	}
5564	};
5565
5566	// The first "clone" is the original copy, we should only call this if we
5567	// needed to create new clones.
5568	assert(NumClones > `1`);
5569	SmallVector<std::unique_ptr<ValueToValueMapTy>, `4`> VMaps;
5570	VMaps.reserve(N: NumClones - `1`);
5571	FunctionsClonedThinBackend ++;
5572
5573	// Map of hash of callsite/alloc versions to the instantiated function clone
5574	// (possibly the original) implementing those calls. Used to avoid
5575	// instantiating duplicate function clones.
5576	// FIXME: Ideally the thin link would not generate such duplicate clones to
5577	// start with, but right now it happens due to phase ordering in the function
5578	// assignment and possible new clones that produces. We simply make each
5579	// duplicate an alias to the matching instantiated clone recorded in the map
5580	// (except for available_externally which are made declarations as they would
5581	// be aliases in the prevailing module, and available_externally aliases are
5582	// not well supported right now).
5583	DenseMap<uint64_t, Function *> HashToFunc;
5584
5585	// Save the hash of the original function version.
5586	HashToFunc [ComputeHash(FS, I: `0`)] = &F;
5587
5588	for (unsigned I = `1`; I < NumClones; I++) {
5589	VMaps.emplace_back(Args: std::make_unique<ValueToValueMapTy>());
5590	std::string Name = getMemProfFuncName(Base: F.getName(), CloneNo: I);
5591	auto Hash = ComputeHash(FS, I);
5592	// If this clone would duplicate a previously seen clone, don't generate the
5593	// duplicate clone body, just make an alias to satisfy any (potentially
5594	// cross-module) references.
5595	if (HashToFunc.contains(Val: Hash)) {
5596	FunctionCloneDuplicatesThinBackend ++;
5597	auto *Func = HashToFunc [Hash];
5598	if (Func->hasAvailableExternallyLinkage()) {
5599	// Skip these as EliminateAvailableExternallyPass does not handle
5600	// available_externally aliases correctly and we end up with an
5601	// available_externally alias to a declaration. Just create a
5602	// declaration for now as we know we will have a definition in another
5603	// module.
5604	auto Decl = M.getOrInsertFunction(Name, T: Func->getFunctionType());
5605	ORE.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofClone", &F)
5606	<< "created clone decl " << ore::NV ("Decl", Decl.getCallee()));
5607	continue;
5608	}
5609	auto *PrevF = M.getFunction(Name);
5610	auto *Alias = GlobalAlias::create(Name, Aliasee: Func);
5611	if (PrevF)
5612	TakeDeclNameAndReplace (PrevF, Alias);
5613	ORE.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofClone", &F)
5614	<< "created clone alias " << ore::NV ("Alias", Alias));
5615
5616	// Now handle aliases to this function, and clone those as well.
5617	CloneFuncAliases (Func, I);
5618	continue;
5619	}
5620	auto NewF = CloneFunction(F: &F, VMap&: VMaps.back());
5621	HashToFunc [Hash] = NewF;
5622	FunctionClonesThinBackend ++;
5623	// Strip memprof and callsite metadata from clone as they are no longer
5624	// needed.
5625	for (auto &BB : *NewF) {
5626	for (auto &Inst : BB) {
5627	Inst.setMetadata(KindID: LLVMContext::MD_memprof, Node: nullptr);
5628	Inst.setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
5629	}
5630	}
5631	auto *PrevF = M.getFunction(Name);
5632	if (PrevF)
5633	TakeDeclNameAndReplace (PrevF, NewF);
5634	else
5635	NewF->setName(Name);
5636	updateSubprogramLinkageName(NewFunc: NewF, Name);
5637	ORE.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofClone", &F)
5638	<< "created clone " << ore::NV ("NewFunction", NewF));
5639
5640	// Now handle aliases to this function, and clone those as well.
5641	CloneFuncAliases (NewF, I);
5642	}
5643	return VMaps;
5644	}
5645
5646	// Locate the summary for F. This is complicated by the fact that it might
5647	// have been internalized or promoted.
5648	static ValueInfo findValueInfoForFunc(const Function &F, const Module &M,
5649	const ModuleSummaryIndex *ImportSummary,
5650	const Function CallingFunc = nullptr*) {
5651	// FIXME: Ideally we would retain the original GUID in some fashion on the
5652	// function (e.g. as metadata), but for now do our best to locate the
5653	// summary without that information.
5654	ValueInfo TheFnVI = ImportSummary->getValueInfo(GUID: F.getGUID());
5655	if (!TheFnVI)
5656	// See if theFn was internalized, by checking index directly with
5657	// original name (this avoids the name adjustment done by getGUID() for
5658	// internal symbols).
5659	TheFnVI = ImportSummary->getValueInfo(
5660	GUID: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: F.getName()));
5661	if (TheFnVI)
5662	return TheFnVI;
5663	// Now query with the original name before any promotion was performed.
5664	StringRef OrigName =
5665	ModuleSummaryIndex::getOriginalNameBeforePromote(Name: F.getName());
5666	// When this pass is enabled, we always add thinlto_src_file provenance
5667	// metadata to imported function definitions, which allows us to recreate the
5668	// original internal symbol's GUID.
5669	auto SrcFileMD = F.getMetadata(Kind: "thinlto_src_file");
5670	// If this is a call to an imported/promoted local for which we didn't import
5671	// the definition, the metadata will not exist on the declaration. However,
5672	// since we are doing this early, before any inlining in the LTO backend, we
5673	// can simply look at the metadata on the calling function which must have
5674	// been from the same module if F was an internal symbol originally.
5675	if (!SrcFileMD && F.isDeclaration()) {
5676	// We would only call this for a declaration for a direct callsite, in which
5677	// case the caller would have provided the calling function pointer.
5678	assert(CallingFunc);
5679	SrcFileMD = CallingFunc->getMetadata(Kind: "thinlto_src_file");
5680	// If this is a promoted local (OrigName != F.getName()), since this is a
5681	// declaration, it must be imported from a different module and therefore we
5682	// should always find the metadata on its calling function. Any call to a
5683	// promoted local that came from this module should still be a definition.
5684	assert(SrcFileMD \|\| OrigName == F.getName());
5685	}
5686	StringRef SrcFile = M.getSourceFileName();
5687	if (SrcFileMD)
5688	SrcFile = dyn_cast<MDString>(Val: SrcFileMD->getOperand(I: `0`))->getString();
5689	std::string OrigId = GlobalValue::getGlobalIdentifier(
5690	Name: OrigName, Linkage: GlobalValue::InternalLinkage, FileName: SrcFile);
5691	TheFnVI = ImportSummary->getValueInfo(
5692	GUID: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigId));
5693	// Internal func in original module may have gotten a numbered suffix if we
5694	// imported an external function with the same name. This happens
5695	// automatically during IR linking for naming conflicts. It would have to
5696	// still be internal in that case (otherwise it would have been renamed on
5697	// promotion in which case we wouldn't have a naming conflict).
5698	if (!TheFnVI && OrigName == F.getName() && F.hasLocalLinkage() &&
5699	F.getName().contains(C: `'.'`)) {
5700	OrigName = F.getName().rsplit(Separator: `'.'`).first;
5701	OrigId = GlobalValue::getGlobalIdentifier(
5702	Name: OrigName, Linkage: GlobalValue::InternalLinkage, FileName: SrcFile);
5703	TheFnVI = ImportSummary->getValueInfo(
5704	GUID: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigId));
5705	}
5706	// The only way we may not have a VI is if this is a declaration created for
5707	// an imported reference. For distributed ThinLTO we may not have a VI for
5708	// such declarations in the distributed summary.
5709	assert(TheFnVI \|\| F.isDeclaration());
5710	return TheFnVI;
5711	}
5712
5713	bool MemProfContextDisambiguation::initializeIndirectCallPromotionInfo(
5714	Module &M) {
5715	ICallAnalysis = std::make_unique<ICallPromotionAnalysis>();
5716	Symtab = std::make_unique<InstrProfSymtab>();
5717	// Don't add canonical names, to avoid multiple functions to the symtab
5718	// when they both have the same root name with "." suffixes stripped.
5719	// If we pick the wrong one then this could lead to incorrect ICP and calling
5720	// a memprof clone that we don't actually create (resulting in linker unsats).
5721	// What this means is that the GUID of the function (or its PGOFuncName
5722	// metadata) must* match that in the VP metadata to allow promotion.*
5723	// In practice this should not be a limitation, since local functions should
5724	// have PGOFuncName metadata and global function names shouldn't need any
5725	// special handling (they should not get the ".llvm." suffix that the*
5726	// canonicalization handling is attempting to strip).
5727	if (Error E = Symtab ->create(M, /InLTO=/true, /AddCanonical=/false)) {
5728	std::string SymtabFailure = toString(E: std::move(E));
5729	M.getContext().emitError(ErrorStr: "Failed to create symtab: " + SymtabFailure);
5730	return false;
5731	}
5732	return true;
5733	}
5734
5735	#ifndef NDEBUG
5736	// Sanity check that the MIB stack ids match between the summary and
5737	// instruction metadata.
5738	static void checkAllocContextIds(
5739	const AllocInfo &AllocNode, const MDNode *MemProfMD,
5740	const CallStack<MDNode, MDNode::op_iterator> &CallsiteContext,
5741	const ModuleSummaryIndex *ImportSummary) {
5742	auto MIBIter = AllocNode.MIBs.begin();
5743	for (auto &MDOp : MemProfMD->operands()) {
5744	assert(MIBIter != AllocNode.MIBs.end());
5745	auto StackIdIndexIter = MIBIter->StackIdIndices.begin();
5746	auto MIBMD = cast<const* MDNode>(MDOp);
5747	MDNode *StackMDNode = getMIBStackNode(MIBMD);
5748	assert(StackMDNode);
5749	CallStack<MDNode, MDNode::op_iterator> StackContext(StackMDNode);
5750	auto ContextIterBegin =
5751	StackContext.beginAfterSharedPrefix(CallsiteContext);
5752	// Skip the checking on the first iteration.
5753	uint64_t LastStackContextId =
5754	(ContextIterBegin != StackContext.end() && *ContextIterBegin == `0`) ? `1`
5755	: `0`;
5756	for (auto ContextIter = ContextIterBegin; ContextIter != StackContext.end();
5757	++ContextIter) {
5758	// If this is a direct recursion, simply skip the duplicate
5759	// entries, to be consistent with how the summary ids were
5760	// generated during ModuleSummaryAnalysis.
5761	if (LastStackContextId == *ContextIter)
5762	continue;
5763	LastStackContextId = *ContextIter;
5764	assert(StackIdIndexIter != MIBIter->StackIdIndices.end());
5765	assert(ImportSummary->getStackIdAtIndex(*StackIdIndexIter) ==
5766	*ContextIter);
5767	StackIdIndexIter++;
5768	}
5769	MIBIter++;
5770	}
5771	}
5772	#endif
5773
5774	bool MemProfContextDisambiguation::applyImport(Module &M) {
5775	assert(ImportSummary);
5776	bool Changed = false;
5777
5778	// We also need to clone any aliases that reference cloned functions, because
5779	// the modified callsites may invoke via the alias. Keep track of the aliases
5780	// for each function.
5781	std::map<const Function , SmallPtrSet<const* GlobalAlias *, `1`>>
5782	FuncToAliasMap;
5783	for (auto &A : M.aliases()) {
5784	auto *Aliasee = A.getAliaseeObject();
5785	if (auto *F = dyn_cast<Function>(Val: Aliasee))
5786	FuncToAliasMap [F].insert(Ptr: &A);
5787	}
5788
5789	if (!initializeIndirectCallPromotionInfo(M))
5790	return false;
5791
5792	for (auto &F : M) {
5793	if (F.isDeclaration() \|\| isMemProfClone(F))
5794	continue;
5795
5796	OptimizationRemarkEmitter ORE(&F);
5797
5798	SmallVector<std::unique_ptr<ValueToValueMapTy>, `4`> VMaps;
5799	bool ClonesCreated = false;
5800	unsigned NumClonesCreated = `0`;
5801	auto CloneFuncIfNeeded = [&](unsigned NumClones, FunctionSummary *FS) {
5802	// We should at least have version 0 which is the original copy.
5803	assert(NumClones > `0`);
5804	// If only one copy needed use original.
5805	if (NumClones == `1`)
5806	return;
5807	// If we already performed cloning of this function, confirm that the
5808	// requested number of clones matches (the thin link should ensure the
5809	// number of clones for each constituent callsite is consistent within
5810	// each function), before returning.
5811	if (ClonesCreated) {
5812	assert(NumClonesCreated == NumClones);
5813	return;
5814	}
5815	VMaps = createFunctionClones(F, NumClones, M, ORE, FuncToAliasMap, FS);
5816	// The first "clone" is the original copy, which doesn't have a VMap.
5817	assert(VMaps.size() == NumClones - `1`);
5818	Changed = true;
5819	ClonesCreated = true;
5820	NumClonesCreated = NumClones;
5821	};
5822
5823	auto CloneCallsite = [&](const CallsiteInfo &StackNode, CallBase *CB,
5824	Function CalledFunction, FunctionSummary FS) {
5825	// Perform cloning if not yet done.
5826	CloneFuncIfNeeded (/NumClones=/StackNode.Clones.size(), FS);
5827
5828	assert(!isMemProfClone(*CalledFunction));
5829
5830	// Because we update the cloned calls by calling setCalledOperand (see
5831	// comment below), out of an abundance of caution make sure the called
5832	// function was actually the called operand (or its aliasee). We also
5833	// strip pointer casts when looking for calls (to match behavior during
5834	// summary generation), however, with opaque pointers in theory this
5835	// should not be an issue. Note we still clone the current function
5836	// (containing this call) above, as that could be needed for its callers.
5837	auto *GA = dyn_cast_or_null<GlobalAlias>(Val: CB->getCalledOperand());
5838	if (CalledFunction != CB->getCalledOperand() &&
5839	(!GA \|\| CalledFunction != GA->getAliaseeObject())) {
5840	SkippedCallsCloning ++;
5841	return;
5842	}
5843	// Update the calls per the summary info.
5844	// Save orig name since it gets updated in the first iteration
5845	// below.
5846	auto CalleeOrigName = CalledFunction->getName();
5847	for (unsigned J = `0`; J < StackNode.Clones.size(); J++) {
5848	// If the VMap is empty, this clone was a duplicate of another and was
5849	// created as an alias or a declaration.
5850	if (J > `0` && VMaps [J - `1`]->empty())
5851	continue;
5852	// Do nothing if this version calls the original version of its
5853	// callee.
5854	if (!StackNode.Clones [J])
5855	continue;
5856	auto NewF = M.getOrInsertFunction(
5857	Name: getMemProfFuncName(Base: CalleeOrigName, CloneNo: StackNode.Clones [J]),
5858	T: CalledFunction->getFunctionType());
5859	CallBase *CBClone;
5860	// Copy 0 is the original function.
5861	if (!J)
5862	CBClone = CB;
5863	else
5864	CBClone = cast<CallBase>(Val&: (*VMaps [J - `1`])[CB]);
5865	// Set the called operand directly instead of calling setCalledFunction,
5866	// as the latter mutates the function type on the call. In rare cases
5867	// we may have a slightly different type on a callee function
5868	// declaration due to it being imported from a different module with
5869	// incomplete types. We really just want to change the name of the
5870	// function to the clone, and not make any type changes.
5871	CBClone->setCalledOperand(NewF.getCallee());
5872	ORE.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofCall", CBClone)
5873	<< ore::NV ("Call", CBClone) << " in clone "
5874	<< ore::NV ("Caller", CBClone->getFunction())
5875	<< " assigned to call function clone "
5876	<< ore::NV ("Callee", NewF.getCallee()));
5877	}
5878	};
5879
5880	// Locate the summary for F.
5881	ValueInfo TheFnVI = findValueInfoForFunc(F, M, ImportSummary);
5882	// If not found, this could be an imported local (see comment in
5883	// findValueInfoForFunc). Skip for now as it will be cloned in its original
5884	// module (where it would have been promoted to global scope so should
5885	// satisfy any reference in this module).
5886	if (!TheFnVI)
5887	continue;
5888
5889	auto *GVSummary =
5890	ImportSummary->findSummaryInModule(VI: TheFnVI, ModuleId: M.getModuleIdentifier());
5891	if (!GVSummary) {
5892	// Must have been imported, use the summary which matches the definition。
5893	// (might be multiple if this was a linkonce_odr).
5894	auto SrcModuleMD = F.getMetadata(Kind: "thinlto_src_module");
5895	assert(SrcModuleMD &&
5896	"enable-import-metadata is needed to emit thinlto_src_module");
5897	StringRef SrcModule =
5898	dyn_cast<MDString>(Val: SrcModuleMD->getOperand(I: `0`))->getString();
5899	for (auto &GVS : TheFnVI.getSummaryList()) {
5900	if (GVS ->modulePath() == SrcModule) {
5901	GVSummary = GVS.get();
5902	break;
5903	}
5904	}
5905	// TODO: Put back the assert once we have metadata on imported copies of
5906	// aliases linking them back to the original alias GUID, which would allow
5907	// us to locate the alias summary here.
5908	// assert(GVSummary && GVSummary->modulePath() == SrcModule);
5909	}
5910
5911	// GVSummary can be null if this is a function imported as a copy of an
5912	// alias, and we don't have the aliasee's summary in our distributed index.
5913	// TODO: Once we can locate the original GUID for imported aliases (e.g. via
5914	// TBD additional metadata), we should find the alias summary instead, and
5915	// we can remove this check and fall back to the original check below.
5916	if (!GVSummary)
5917	continue;
5918
5919	// If this was an imported alias skip it as we won't have the function
5920	// summary, and it should be cloned in the original module.
5921	if (isa<AliasSummary>(Val: GVSummary))
5922	continue;
5923
5924	auto *FS = cast<FunctionSummary>(Val: GVSummary->getBaseObject());
5925
5926	if (FS->allocs().empty() && FS->callsites().empty())
5927	continue;
5928
5929	auto SI = FS->callsites().begin();
5930	auto AI = FS->allocs().begin();
5931
5932	// To handle callsite infos synthesized for tail calls which have missing
5933	// frames in the profiled context, map callee VI to the synthesized callsite
5934	// info.
5935	DenseMap<ValueInfo, CallsiteInfo> MapTailCallCalleeVIToCallsite;
5936	// Iterate the callsites for this function in reverse, since we place all
5937	// those synthesized for tail calls at the end.
5938	for (auto CallsiteIt = FS->callsites().rbegin();
5939	CallsiteIt != FS->callsites().rend(); CallsiteIt ++) {
5940	auto &Callsite = *CallsiteIt;
5941	// Stop as soon as we see a non-synthesized callsite info (see comment
5942	// above loop). All the entries added for discovered tail calls have empty
5943	// stack ids.
5944	if (!Callsite.StackIdIndices.empty())
5945	break;
5946	MapTailCallCalleeVIToCallsite.insert(KV: {Callsite.Callee, Callsite});
5947	}
5948
5949	// Keeps track of needed ICP for the function.
5950	SmallVector<ICallAnalysisData> ICallAnalysisInfo;
5951
5952	// Assume for now that the instructions are in the exact same order
5953	// as when the summary was created, but confirm this is correct by
5954	// matching the stack ids.
5955	for (auto &BB : F) {
5956	for (auto &I : BB) {
5957	auto *CB = dyn_cast<CallBase>(Val: &I);
5958	// Same handling as when creating module summary.
5959	if (!mayHaveMemprofSummary(CB))
5960	continue;
5961
5962	auto *CalledValue = CB->getCalledOperand();
5963	auto *CalledFunction = CB->getCalledFunction();
5964	if (CalledValue && !CalledFunction) {
5965	CalledValue = CalledValue->stripPointerCasts();
5966	// Stripping pointer casts can reveal a called function.
5967	CalledFunction = dyn_cast<Function>(Val: CalledValue);
5968	}
5969	// Check if this is an alias to a function. If so, get the
5970	// called aliasee for the checks below.
5971	if (auto *GA = dyn_cast<GlobalAlias>(Val: CalledValue)) {
5972	assert(!CalledFunction &&
5973	"Expected null called function in callsite for alias");
5974	CalledFunction = dyn_cast<Function>(Val: GA->getAliaseeObject());
5975	}
5976
5977	CallStack<MDNode, MDNode::op_iterator> CallsiteContext(
5978	I.getMetadata(KindID: LLVMContext::MD_callsite));
5979	auto *MemProfMD = I.getMetadata(KindID: LLVMContext::MD_memprof);
5980
5981	// Include allocs that were already assigned a memprof function
5982	// attribute in the statistics. Only do this for those that do not have
5983	// memprof metadata, since we add an "ambiguous" memprof attribute by
5984	// default.
5985	if (CB->getAttributes().hasFnAttr(Kind: "memprof") && !MemProfMD) {
5986	CB->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "cold"
5987	? AllocTypeColdThinBackend ++
5988	: AllocTypeNotColdThinBackend ++;
5989	OrigAllocsThinBackend ++;
5990	AllocVersionsThinBackend ++;
5991	if (!MaxAllocVersionsThinBackend)
5992	MaxAllocVersionsThinBackend = `1`;
5993	continue;
5994	}
5995
5996	if (MemProfMD) {
5997	// Consult the next alloc node.
5998	assert(AI != FS->allocs().end());
5999	auto &AllocNode = *(AI++);
6000
6001	#ifndef NDEBUG
6002	checkAllocContextIds(AllocNode, MemProfMD, CallsiteContext,
6003	ImportSummary);
6004	#endif
6005
6006	// Perform cloning if not yet done.
6007	CloneFuncIfNeeded (/NumClones=/AllocNode.Versions.size(), FS);
6008
6009	OrigAllocsThinBackend ++;
6010	AllocVersionsThinBackend += AllocNode.Versions.size();
6011	if (MaxAllocVersionsThinBackend < AllocNode.Versions.size())
6012	MaxAllocVersionsThinBackend = AllocNode.Versions.size();
6013
6014	// If there is only one version that means we didn't end up
6015	// considering this function for cloning, and in that case the alloc
6016	// will still be none type or should have gotten the default NotCold.
6017	// Skip that after calling clone helper since that does some sanity
6018	// checks that confirm we haven't decided yet that we need cloning.
6019	// We might have a single version that is cold due to the
6020	// MinClonedColdBytePercent heuristic, make sure we don't skip in that
6021	// case.
6022	if (AllocNode.Versions.size() == `1` &&
6023	(AllocationType)AllocNode.Versions [`0`] != AllocationType::Cold) {
6024	assert((AllocationType)AllocNode.Versions[`0`] ==
6025	AllocationType::NotCold \|\|
6026	(AllocationType)AllocNode.Versions[`0`] ==
6027	AllocationType::None);
6028	UnclonableAllocsThinBackend ++;
6029	continue;
6030	}
6031
6032	// All versions should have a singular allocation type.
6033	assert(llvm::none_of(AllocNode.Versions, [](uint8_t Type) {
6034	return Type == ((uint8_t)AllocationType::NotCold \|
6035	(uint8_t)AllocationType::Cold);
6036	}));
6037
6038	// Update the allocation types per the summary info.
6039	for (unsigned J = `0`; J < AllocNode.Versions.size(); J++) {
6040	// If the VMap is empty, this clone was a duplicate of another and
6041	// was created as an alias or a declaration.
6042	if (J > `0` && VMaps [J - `1`]->empty())
6043	continue;
6044	// Ignore any that didn't get an assigned allocation type.
6045	if (AllocNode.Versions [J] == (uint8_t)AllocationType::None)
6046	continue;
6047	AllocationType AllocTy = (AllocationType)AllocNode.Versions [J];
6048	AllocTy == AllocationType::Cold ? AllocTypeColdThinBackend ++
6049	: AllocTypeNotColdThinBackend ++;
6050	std::string AllocTypeString = getAllocTypeAttributeString(Type: AllocTy);
6051	auto A = llvm::Attribute::get(Context&: F.getContext(), Kind: "memprof",
6052	Val: AllocTypeString);
6053	CallBase *CBClone;
6054	// Copy 0 is the original function.
6055	if (!J)
6056	CBClone = CB;
6057	else
6058	// Since VMaps are only created for new clones, we index with
6059	// clone J-1 (J==0 is the original clone and does not have a VMaps
6060	// entry).
6061	CBClone = cast<CallBase>(Val&: (*VMaps [J - `1`])[CB]);
6062	removeAnyExistingAmbiguousAttribute(CB: CBClone);
6063	CBClone->addFnAttr(Attr: A);
6064	ORE.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofAttribute", CBClone)
6065	<< ore::NV ("AllocationCall", CBClone) << " in clone "
6066	<< ore::NV ("Caller", CBClone->getFunction())
6067	<< " marked with memprof allocation attribute "
6068	<< ore::NV ("Attribute", AllocTypeString));
6069	}
6070	} else if (!CallsiteContext.empty()) {
6071	if (!CalledFunction) {
6072	#ifndef NDEBUG
6073	// We should have skipped inline assembly calls.
6074	auto *CI = dyn_cast<CallInst>(CB);
6075	assert(!CI \|\| !CI->isInlineAsm());
6076	#endif
6077	// We should have skipped direct calls via a Constant.
6078	assert(CalledValue && !isa<Constant>(CalledValue));
6079
6080	// This is an indirect call, see if we have profile information and
6081	// whether any clones were recorded for the profiled targets (that
6082	// we synthesized CallsiteInfo summary records for when building the
6083	// index).
6084	auto NumClones =
6085	recordICPInfo(CB, AllCallsites: FS->callsites(), SI, ICallAnalysisInfo);
6086
6087	// Perform cloning if not yet done. This is done here in case
6088	// we don't need to do ICP, but might need to clone this
6089	// function as it is the target of other cloned calls.
6090	if (NumClones)
6091	CloneFuncIfNeeded (NumClones, FS);
6092	}
6093
6094	else {
6095	// Consult the next callsite node.
6096	assert(SI != FS->callsites().end());
6097	auto &StackNode = *(SI++);
6098
6099	#ifndef NDEBUG
6100	// Sanity check that the stack ids match between the summary and
6101	// instruction metadata.
6102	auto StackIdIndexIter = StackNode.StackIdIndices.begin();
6103	for (auto StackId : CallsiteContext) {
6104	assert(StackIdIndexIter != StackNode.StackIdIndices.end());
6105	assert(ImportSummary->getStackIdAtIndex(*StackIdIndexIter) ==
6106	StackId);
6107	StackIdIndexIter++;
6108	}
6109	#endif
6110
6111	CloneCallsite (StackNode, CB, CalledFunction, FS);
6112	}
6113	} else if (CB->isTailCall() && CalledFunction) {
6114	// Locate the synthesized callsite info for the callee VI, if any was
6115	// created, and use that for cloning.
6116	ValueInfo CalleeVI =
6117	findValueInfoForFunc(F: *CalledFunction, M, ImportSummary, CallingFunc: &F);
6118	if (CalleeVI && MapTailCallCalleeVIToCallsite.count(Val: CalleeVI)) {
6119	auto Callsite = MapTailCallCalleeVIToCallsite.find(Val: CalleeVI);
6120	assert(Callsite != MapTailCallCalleeVIToCallsite.end());
6121	CloneCallsite (Callsite ->second, CB, CalledFunction, FS);
6122	}
6123	}
6124	}
6125	}
6126
6127	// Now do any promotion required for cloning.
6128	performICP(M, AllCallsites: FS->callsites(), VMaps, ICallAnalysisInfo, ORE);
6129	}
6130
6131	// We skip some of the functions and instructions above, so remove all the
6132	// metadata in a single sweep here.
6133	for (auto &F : M) {
6134	// We can skip memprof clones because createFunctionClones already strips
6135	// the metadata from the newly created clones.
6136	if (F.isDeclaration() \|\| isMemProfClone(F))
6137	continue;
6138	for (auto &BB : F) {
6139	for (auto &I : BB) {
6140	if (!isa<CallBase>(Val: I))
6141	continue;
6142	I.setMetadata(KindID: LLVMContext::MD_memprof, Node: nullptr);
6143	I.setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
6144	}
6145	}
6146	}
6147
6148	return Changed;
6149	}
6150
6151	unsigned MemProfContextDisambiguation::recordICPInfo(
6152	CallBase *CB, ArrayRef<CallsiteInfo> AllCallsites,
6153	ArrayRef<CallsiteInfo>::iterator &SI,
6154	SmallVector<ICallAnalysisData> &ICallAnalysisInfo) {
6155	// First see if we have profile information for this indirect call.
6156	uint32_t NumCandidates;
6157	uint64_t TotalCount;
6158	auto CandidateProfileData =
6159	ICallAnalysis ->getPromotionCandidatesForInstruction(
6160	I: CB, TotalCount, NumCandidates, MaxNumValueData: MaxSummaryIndirectEdges);
6161	if (CandidateProfileData.empty())
6162	return `0`;
6163
6164	// Iterate through all of the candidate profiled targets along with the
6165	// CallsiteInfo summary records synthesized for them when building the index,
6166	// and see if any are cloned and/or refer to clones.
6167	bool ICPNeeded = false;
6168	unsigned NumClones = `0`;
6169	size_t CallsiteInfoStartIndex = std::distance(first: AllCallsites.begin(), last: SI);
6170	for (const auto &Candidate : CandidateProfileData) {
6171	#ifndef NDEBUG
6172	auto CalleeValueInfo =
6173	#endif
6174	ImportSummary->getValueInfo(GUID: Candidate.Value);
6175	// We might not have a ValueInfo if this is a distributed
6176	// ThinLTO backend and decided not to import that function.
6177	assert(!CalleeValueInfo \|\| SI->Callee == CalleeValueInfo);
6178	assert(SI != AllCallsites.end());
6179	auto &StackNode = *(SI++);
6180	// See if any of the clones of the indirect callsite for this
6181	// profiled target should call a cloned version of the profiled
6182	// target. We only need to do the ICP here if so.
6183	ICPNeeded \|= llvm::any_of(Range: StackNode.Clones,
6184	P: [](unsigned CloneNo) { return CloneNo != `0`; });
6185	// Every callsite in the same function should have been cloned the same
6186	// number of times.
6187	assert(!NumClones \|\| NumClones == StackNode.Clones.size());
6188	NumClones = StackNode.Clones.size();
6189	}
6190	if (!ICPNeeded)
6191	return NumClones;
6192	// Save information for ICP, which is performed later to avoid messing up the
6193	// current function traversal.
6194	ICallAnalysisInfo.push_back(Elt: {.CB: CB, .CandidateProfileData: CandidateProfileData.vec(), .NumCandidates: NumCandidates,
6195	.TotalCount: TotalCount, .CallsiteInfoStartIndex: CallsiteInfoStartIndex});
6196	return NumClones;
6197	}
6198
6199	void MemProfContextDisambiguation::performICP(
6200	Module &M, ArrayRef<CallsiteInfo> AllCallsites,
6201	ArrayRef<std::unique_ptr<ValueToValueMapTy>> VMaps,
6202	ArrayRef<ICallAnalysisData> ICallAnalysisInfo,
6203	OptimizationRemarkEmitter &ORE) {
6204	// Now do any promotion required for cloning. Specifically, for each
6205	// recorded ICP candidate (which was only recorded because one clone of that
6206	// candidate should call a cloned target), we perform ICP (speculative
6207	// devirtualization) for each clone of the callsite, and update its callee
6208	// to the appropriate clone. Note that the ICP compares against the original
6209	// version of the target, which is what is in the vtable.
6210	for (auto &Info : ICallAnalysisInfo) {
6211	auto *CB = Info.CB;
6212	auto CallsiteIndex = Info.CallsiteInfoStartIndex;
6213	auto TotalCount = Info.TotalCount;
6214	unsigned NumClones = `0`;
6215	SmallVector<InstrProfValueData, `8`> RemainingCandidates;
6216
6217	for (auto &Candidate : Info.CandidateProfileData) {
6218	auto &StackNode = AllCallsites [CallsiteIndex++];
6219
6220	// All calls in the same function must have the same number of clones.
6221	assert(!NumClones \|\| NumClones == StackNode.Clones.size());
6222	NumClones = StackNode.Clones.size();
6223
6224	// See if the target is in the module. If it wasn't imported, it is
6225	// possible that this profile could have been collected on a different
6226	// target (or version of the code), and we need to be conservative
6227	// (similar to what is done in the ICP pass).
6228	Function *TargetFunction = Symtab ->getFunction(FuncMD5Hash: Candidate.Value);
6229	if (TargetFunction == nullptr \|\|
6230	// Any ThinLTO global dead symbol removal should have already
6231	// occurred, so it should be safe to promote when the target is a
6232	// declaration.
6233	// TODO: Remove internal option once more fully tested.
6234	(MemProfRequireDefinitionForPromotion &&
6235	TargetFunction->isDeclaration())) {
6236	ORE.emit(RemarkBuilder: [&]() {
6237	return OptimizationRemarkMissed (DEBUG_TYPE, "UnableToFindTarget", CB)
6238	<< "Memprof cannot promote indirect call: target with md5sum "
6239	<< ore::NV ("target md5sum", Candidate.Value) << " not found";
6240	});
6241	// FIXME: See if we can use the new declaration importing support to
6242	// at least get the declarations imported for this case. Hot indirect
6243	// targets should have been imported normally, however.
6244	RemainingCandidates.push_back(Elt: Candidate);
6245	continue;
6246	}
6247
6248	// Check if legal to promote
6249	const char Reason = nullptr*;
6250	if (!isLegalToPromote(CB: *CB, Callee: TargetFunction, FailureReason: &Reason)) {
6251	ORE.emit(RemarkBuilder: [&]() {
6252	return OptimizationRemarkMissed (DEBUG_TYPE, "UnableToPromote", CB)
6253	<< "Memprof cannot promote indirect call to "
6254	<< ore::NV ("TargetFunction", TargetFunction)
6255	<< " with count of " << ore::NV ("TotalCount", TotalCount)
6256	<< ": " << Reason;
6257	});
6258	RemainingCandidates.push_back(Elt: Candidate);
6259	continue;
6260	}
6261
6262	assert(!isMemProfClone(*TargetFunction));
6263
6264	// Handle each call clone, applying ICP so that each clone directly
6265	// calls the specified callee clone, guarded by the appropriate ICP
6266	// check.
6267	CallBase *CBClone = CB;
6268	for (unsigned J = `0`; J < NumClones; J++) {
6269	// If the VMap is empty, this clone was a duplicate of another and was
6270	// created as an alias or a declaration.
6271	if (J > `0` && VMaps [J - `1`]->empty())
6272	continue;
6273	// Copy 0 is the original function.
6274	if (J > `0`)
6275	CBClone = cast<CallBase>(Val&: (*VMaps [J - `1`])[CB]);
6276	// We do the promotion using the original name, so that the comparison
6277	// is against the name in the vtable. Then just below, change the new
6278	// direct call to call the cloned function.
6279	auto &DirectCall =
6280	pgo::promoteIndirectCall(CB&: *CBClone, F: TargetFunction, Count: Candidate.Count,
6281	TotalCount, AttachProfToDirectCall: isSamplePGO, ORE: &ORE);
6282	auto *TargetToUse = TargetFunction;
6283	// Call original if this version calls the original version of its
6284	// callee.
6285	if (StackNode.Clones [J]) {
6286	TargetToUse =
6287	cast<Function>(Val: M.getOrInsertFunction(
6288	Name: getMemProfFuncName(Base: TargetFunction->getName(),
6289	CloneNo: StackNode.Clones [J]),
6290	T: TargetFunction->getFunctionType())
6291	.getCallee());
6292	}
6293	DirectCall.setCalledFunction(TargetToUse);
6294	// During matching we generate synthetic VP metadata for indirect calls
6295	// not already having any, from the memprof profile's callee GUIDs. If
6296	// we subsequently promote and inline those callees, we currently lose
6297	// the ability to generate this synthetic VP metadata. Optionally apply
6298	// a noinline attribute to promoted direct calls, where the threshold is
6299	// set to capture synthetic VP metadata targets which get a count of 1.
6300	if (MemProfICPNoInlineThreshold &&
6301	Candidate.Count < MemProfICPNoInlineThreshold)
6302	DirectCall.setIsNoInline();
6303	ORE.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofCall", CBClone)
6304	<< ore::NV ("Call", CBClone) << " in clone "
6305	<< ore::NV ("Caller", CBClone->getFunction())
6306	<< " promoted and assigned to call function clone "
6307	<< ore::NV ("Callee", TargetToUse));
6308	}
6309
6310	// Update TotalCount (all clones should get same count above)
6311	TotalCount -= Candidate.Count;
6312	}
6313	// Adjust the MD.prof metadata for all clones, now that we have the new
6314	// TotalCount and the remaining candidates.
6315	CallBase *CBClone = CB;
6316	for (unsigned J = `0`; J < NumClones; J++) {
6317	// If the VMap is empty, this clone was a duplicate of another and was
6318	// created as an alias or a declaration.
6319	if (J > `0` && VMaps [J - `1`]->empty())
6320	continue;
6321	// Copy 0 is the original function.
6322	if (J > `0`)
6323	CBClone = cast<CallBase>(Val&: (*VMaps [J - `1`])[CB]);
6324	// First delete the old one.
6325	CBClone->setMetadata(KindID: LLVMContext::MD_prof, Node: nullptr);
6326	// If all promoted, we don't need the MD.prof metadata.
6327	// Otherwise we need update with the un-promoted records back.
6328	if (TotalCount != `0`)
6329	annotateValueSite(M, Inst&: *CBClone, VDs: RemainingCandidates, Sum: TotalCount,
6330	ValueKind: IPVK_IndirectCallTarget, MaxMDCount: Info.NumCandidates);
6331	}
6332	}
6333	}
6334
6335	template <typename DerivedCCG, typename FuncTy, typename CallTy>
6336	bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::process(
6337	function_ref<void(StringRef, StringRef, const Twine &)> EmitRemark,
6338	bool AllowExtraAnalysis) {
6339	if (DumpCCG) {
6340	dbgs() << "CCG before cloning:\n";
6341	dbgs() << *this;
6342	}
6343	if (ExportToDot)
6344	exportToDot(Label: "postbuild");
6345
6346	if (VerifyCCG) {
6347	check();
6348	}
6349
6350	identifyClones();
6351
6352	if (VerifyCCG) {
6353	check();
6354	}
6355
6356	if (DumpCCG) {
6357	dbgs() << "CCG after cloning:\n";
6358	dbgs() << *this;
6359	}
6360	if (ExportToDot)
6361	exportToDot(Label: "cloned");
6362
6363	bool Changed = assignFunctions();
6364
6365	if (DumpCCG) {
6366	dbgs() << "CCG after assigning function clones:\n";
6367	dbgs() << *this;
6368	}
6369	if (ExportToDot)
6370	exportToDot(Label: "clonefuncassign");
6371
6372	if (MemProfReportHintedSizes \|\| AllowExtraAnalysis)
6373	printTotalSizes(OS&: errs(), EmitRemark);
6374
6375	return Changed;
6376	}
6377
6378	bool MemProfContextDisambiguation::processModule(
6379	Module &M,
6380	llvm::function_ref<OptimizationRemarkEmitter &(Function *)> OREGetter) {
6381
6382	// If we have an import summary, then the cloning decisions were made during
6383	// the thin link on the index. Apply them and return.
6384	if (ImportSummary)
6385	return applyImport(M);
6386
6387	// TODO: If/when other types of memprof cloning are enabled beyond just for
6388	// hot and cold, we will need to change this to individually control the
6389	// AllocationType passed to addStackNodesForMIB during CCG construction.
6390	// Note that we specifically check this after applying imports above, so that
6391	// the option isn't needed to be passed to distributed ThinLTO backend
6392	// clang processes, which won't necessarily have visibility into the linker
6393	// dependences. Instead the information is communicated from the LTO link to
6394	// the backends via the combined summary index.
6395	if (!SupportsHotColdNew)
6396	return false;
6397
6398	ModuleCallsiteContextGraph CCG(M, OREGetter);
6399	// TODO: Set up remarks for regular LTO. We need to decide what function to
6400	// use in the callback.
6401	return CCG.process();
6402	}
6403
6404	MemProfContextDisambiguation::MemProfContextDisambiguation(
6405	const ModuleSummaryIndex Summary, bool* isSamplePGO)
6406	: ImportSummary(Summary), isSamplePGO(isSamplePGO) {
6407	// Check the dot graph printing options once here, to make sure we have valid
6408	// and expected combinations.
6409	if (DotGraphScope == DotScope::Alloc && !AllocIdForDot.getNumOccurrences())
6410	llvm::report_fatal_error(
6411	reason: "-memprof-dot-scope=alloc requires -memprof-dot-alloc-id");
6412	if (DotGraphScope == DotScope::Context &&
6413	!ContextIdForDot.getNumOccurrences())
6414	llvm::report_fatal_error(
6415	reason: "-memprof-dot-scope=context requires -memprof-dot-context-id");
6416	if (DotGraphScope == DotScope::All && AllocIdForDot.getNumOccurrences() &&
6417	ContextIdForDot.getNumOccurrences())
6418	llvm::report_fatal_error(
6419	reason: "-memprof-dot-scope=all can't have both -memprof-dot-alloc-id and "
6420	"-memprof-dot-context-id");
6421	if (ImportSummary) {
6422	// The MemProfImportSummary should only be used for testing ThinLTO
6423	// distributed backend handling via opt, in which case we don't have a
6424	// summary from the pass pipeline.
6425	assert(MemProfImportSummary.empty());
6426	return;
6427	}
6428	if (MemProfImportSummary.empty())
6429	return;
6430
6431	auto ReadSummaryFile =
6432	errorOrToExpected(EO: MemoryBuffer::getFile(Filename: MemProfImportSummary));
6433	if (!ReadSummaryFile) {
6434	logAllUnhandledErrors(E: ReadSummaryFile.takeError(), OS&: errs(),
6435	ErrorBanner: "Error loading file '" + MemProfImportSummary +
6436	"': ");
6437	return;
6438	}
6439	auto ImportSummaryForTestingOrErr = getModuleSummaryIndex(Buffer: **ReadSummaryFile);
6440	if (!ImportSummaryForTestingOrErr) {
6441	logAllUnhandledErrors(E: ImportSummaryForTestingOrErr.takeError(), OS&: errs(),
6442	ErrorBanner: "Error parsing file '" + MemProfImportSummary +
6443	"': ");
6444	return;
6445	}
6446	ImportSummaryForTesting = std::move(*ImportSummaryForTestingOrErr);
6447	ImportSummary = ImportSummaryForTesting.get();
6448	}
6449
6450	PreservedAnalyses MemProfContextDisambiguation::run(Module &M,
6451	ModuleAnalysisManager &AM) {
6452	auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
6453	auto OREGetter = [&](Function *F) -> OptimizationRemarkEmitter & {
6454	return FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: *F);
6455	};
6456	if (!processModule(M, OREGetter))
6457	return PreservedAnalyses::all();
6458	return PreservedAnalyses::none();
6459	}
6460
6461	void MemProfContextDisambiguation::run(
6462	ModuleSummaryIndex &Index,
6463	llvm::function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
6464	isPrevailing,
6465	LLVMContext &Ctx,
6466	function_ref<void(StringRef, StringRef, const Twine &)> EmitRemark) {
6467	// TODO: If/when other types of memprof cloning are enabled beyond just for
6468	// hot and cold, we will need to change this to individually control the
6469	// AllocationType passed to addStackNodesForMIB during CCG construction.
6470	// The index was set from the option, so these should be in sync.
6471	assert(Index.withSupportsHotColdNew() == SupportsHotColdNew);
6472	if (!SupportsHotColdNew)
6473	return;
6474
6475	bool AllowExtraAnalysis =
6476	OptimizationRemarkEmitter::allowExtraAnalysis(Ctx, DEBUG_TYPE);
6477
6478	IndexCallsiteContextGraph CCG(Index, isPrevailing);
6479	CCG.process(EmitRemark, AllowExtraAnalysis);
6480	}
6481
6482	// Strips MemProf attributes and metadata. Can be invoked by the pass pipeline
6483	// when we don't have an index that has recorded that we are linking with
6484	// allocation libraries containing the necessary APIs for downstream
6485	// transformations.
6486	PreservedAnalyses MemProfRemoveInfo::run(Module &M, ModuleAnalysisManager &AM) {
6487	// The profile matcher applies hotness attributes directly for allocations,
6488	// and those will cause us to generate calls to the hot/cold interfaces
6489	// unconditionally. If supports-hot-cold-new was not enabled in the LTO
6490	// link then assume we don't want these calls (e.g. not linking with
6491	// the appropriate library, or otherwise trying to disable this behavior).
6492	bool Changed = false;
6493	for (auto &F : M) {
6494	for (auto &BB : F) {
6495	for (auto &I : BB) {
6496	auto *CI = dyn_cast<CallBase>(Val: &I);
6497	if (!CI)
6498	continue;
6499	if (CI->hasFnAttr(Kind: "memprof")) {
6500	CI->removeFnAttr(Kind: "memprof");
6501	Changed = true;
6502	}
6503	if (!CI->hasMetadata(KindID: LLVMContext::MD_callsite)) {
6504	assert(!CI->hasMetadata(LLVMContext::MD_memprof));
6505	continue;
6506	}
6507	// Strip off all memprof metadata as it is no longer needed.
6508	// Importantly, this avoids the addition of new memprof attributes
6509	// after inlining propagation.
6510	CI->setMetadata(KindID: LLVMContext::MD_memprof, Node: nullptr);
6511	CI->setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
6512	Changed = true;
6513	}
6514	}
6515	}
6516	if (!Changed)
6517	return PreservedAnalyses::all();
6518	return PreservedAnalyses::none();
6519	}
6520

Browse the source code of llvm_projects/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp