MemProfContextDisambiguation.cpp source code [llvm_projects/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp]

1	//==-- MemProfContextDisambiguation.cpp - Disambiguate contexts -------------=//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file implements support for context disambiguation of allocation
10	// calls for profile guided heap optimization. Specifically, it uses Memprof
11	// profiles which indicate context specific allocation behavior (currently
12	// distinguishing cold vs hot memory allocations). Cloning is performed to
13	// expose the cold allocation call contexts, and the allocation calls are
14	// subsequently annotated with an attribute for later transformation.
15	//
16	// The transformations can be performed either directly on IR (regular LTO), or
17	// on a ThinLTO index (and later applied to the IR during the ThinLTO backend).
18	// Both types of LTO operate on a the same base graph representation, which
19	// uses CRTP to support either IR or Index formats.
20	//
21	//===----------------------------------------------------------------------===//
22
23	#include "llvm/Transforms/IPO/MemProfContextDisambiguation.h"
24	#include "llvm/ADT/DenseMap.h"
25	#include "llvm/ADT/DenseSet.h"
26	#include "llvm/ADT/MapVector.h"
27	#include "llvm/ADT/SetOperations.h"
28	#include "llvm/ADT/SmallPtrSet.h"
29	#include "llvm/ADT/SmallSet.h"
30	#include "llvm/ADT/SmallVector.h"
31	#include "llvm/ADT/Statistic.h"
32	#include "llvm/ADT/StringExtras.h"
33	#include "llvm/Analysis/MemoryProfileInfo.h"
34	#include "llvm/Analysis/ModuleSummaryAnalysis.h"
35	#include "llvm/Analysis/OptimizationRemarkEmitter.h"
36	#include "llvm/Bitcode/BitcodeReader.h"
37	#include "llvm/IR/Instructions.h"
38	#include "llvm/IR/Module.h"
39	#include "llvm/IR/ModuleSummaryIndex.h"
40	#include "llvm/Pass.h"
41	#include "llvm/Support/CommandLine.h"
42	#include "llvm/Support/GraphWriter.h"
43	#include "llvm/Support/SHA1.h"
44	#include "llvm/Support/raw_ostream.h"
45	#include "llvm/Transforms/IPO.h"
46	#include "llvm/Transforms/Utils/CallPromotionUtils.h"
47	#include "llvm/Transforms/Utils/Cloning.h"
48	#include "llvm/Transforms/Utils/Instrumentation.h"
49	#include <deque>
50	#include <sstream>
51	#include <vector>
52	using namespace llvm;
53	using namespace llvm::memprof;
54
55	#define DEBUG_TYPE "memprof-context-disambiguation"
56
57	STATISTIC(FunctionClonesAnalysis,
58	"Number of function clones created during whole program analysis");
59	STATISTIC(FunctionClonesThinBackend,
60	"Number of function clones created during ThinLTO backend");
61	STATISTIC(FunctionsClonedThinBackend,
62	"Number of functions that had clones created during ThinLTO backend");
63	STATISTIC(
64	FunctionCloneDuplicatesThinBackend,
65	"Number of function clone duplicates detected during ThinLTO backend");
66	STATISTIC(AllocTypeNotCold, "Number of not cold static allocations (possibly "
67	"cloned) during whole program analysis");
68	STATISTIC(AllocTypeCold, "Number of cold static allocations (possibly cloned) "
69	"during whole program analysis");
70	STATISTIC(AllocTypeNotColdThinBackend,
71	"Number of not cold static allocations (possibly cloned) during "
72	"ThinLTO backend");
73	STATISTIC(AllocTypeColdThinBackend, "Number of cold static allocations "
74	"(possibly cloned) during ThinLTO backend");
75	STATISTIC(OrigAllocsThinBackend,
76	"Number of original (not cloned) allocations with memprof profiles "
77	"during ThinLTO backend");
78	STATISTIC(
79	AllocVersionsThinBackend,
80	"Number of allocation versions (including clones) during ThinLTO backend");
81	STATISTIC(MaxAllocVersionsThinBackend,
82	"Maximum number of allocation versions created for an original "
83	"allocation during ThinLTO backend");
84	STATISTIC(UnclonableAllocsThinBackend,
85	"Number of unclonable ambigous allocations during ThinLTO backend");
86	STATISTIC(RemovedEdgesWithMismatchedCallees,
87	"Number of edges removed due to mismatched callees (profiled vs IR)");
88	STATISTIC(FoundProfiledCalleeCount,
89	"Number of profiled callees found via tail calls");
90	STATISTIC(FoundProfiledCalleeDepth,
91	"Aggregate depth of profiled callees found via tail calls");
92	STATISTIC(FoundProfiledCalleeMaxDepth,
93	"Maximum depth of profiled callees found via tail calls");
94	STATISTIC(FoundProfiledCalleeNonUniquelyCount,
95	"Number of profiled callees found via multiple tail call chains");
96	STATISTIC(DeferredBackedges, "Number of backedges with deferred cloning");
97	STATISTIC(NewMergedNodes, "Number of new nodes created during merging");
98	STATISTIC(NonNewMergedNodes, "Number of non new nodes used during merging");
99	STATISTIC(MissingAllocForContextId,
100	"Number of missing alloc nodes for context ids");
101	STATISTIC(SkippedCallsCloning,
102	"Number of calls skipped during cloning due to unexpected operand");
103	STATISTIC(MismatchedCloneAssignments,
104	"Number of callsites assigned to call multiple non-matching clones");
105	STATISTIC(TotalMergeInvokes, "Number of merge invocations for nodes");
106	STATISTIC(TotalMergeIters, "Number of merge iterations for nodes");
107	STATISTIC(MaxMergeIters, "Max merge iterations for nodes");
108	STATISTIC(NumImportantContextIds, "Number of important context ids");
109	STATISTIC(NumFixupEdgeIdsInserted, "Number of fixup edge ids inserted");
110	STATISTIC(NumFixupEdgesAdded, "Number of fixup edges added");
111	STATISTIC(NumFixedContexts, "Number of contexts with fixed edges");
112	STATISTIC(AliaseesPrevailingInDiffModuleFromAlias,
113	"Number of aliasees prevailing in a different module than its alias");
114
115	static cl::opt<std::string> DotFilePathPrefix(
116	"memprof-dot-file-path-prefix", cl::init(Val: ""), cl::Hidden,
117	cl::value_desc("filename"),
118	cl::desc("Specify the path prefix of the MemProf dot files."));
119
120	static cl::opt<bool> ExportToDot("memprof-export-to-dot", cl::init(Val: false),
121	cl::Hidden,
122	cl::desc("Export graph to dot files."));
123
124	// TODO: Remove this option once new handling is validated more widely.
125	static cl::opt<bool> DoMergeIteration(
126	"memprof-merge-iteration", cl::init(Val: true), cl::Hidden,
127	cl::desc("Iteratively apply merging on a node to catch new callers"));
128
129	// How much of the graph to export to dot.
130	enum DotScope {
131	All, // The full CCG graph.
132	Alloc, // Only contexts for the specified allocation.
133	Context, // Only the specified context.
134	};
135
136	static cl::opt<DotScope> DotGraphScope(
137	"memprof-dot-scope", cl::desc("Scope of graph to export to dot"),
138	cl::Hidden, cl::init(Val: DotScope::All),
139	cl::values(
140	clEnumValN(DotScope::All, "all", "Export full callsite graph"),
141	clEnumValN(DotScope::Alloc, "alloc",
142	"Export only nodes with contexts feeding given "
143	"-memprof-dot-alloc-id"),
144	clEnumValN(DotScope::Context, "context",
145	"Export only nodes with given -memprof-dot-context-id")));
146
147	static cl::opt<unsigned>
148	AllocIdForDot("memprof-dot-alloc-id", cl::init(Val: `0`), cl::Hidden,
149	cl::desc("Id of alloc to export if -memprof-dot-scope=alloc "
150	"or to highlight if -memprof-dot-scope=all"));
151
152	static cl::opt<unsigned> ContextIdForDot(
153	"memprof-dot-context-id", cl::init(Val: `0`), cl::Hidden,
154	cl::desc("Id of context to export if -memprof-dot-scope=context or to "
155	"highlight otherwise"));
156
157	static cl::opt<bool>
158	DumpCCG("memprof-dump-ccg", cl::init(Val: false), cl::Hidden,
159	cl::desc("Dump CallingContextGraph to stdout after each stage."));
160
161	static cl::opt<bool>
162	VerifyCCG("memprof-verify-ccg", cl::init(Val: false), cl::Hidden,
163	cl::desc("Perform verification checks on CallingContextGraph."));
164
165	static cl::opt<bool>
166	VerifyNodes("memprof-verify-nodes", cl::init(Val: false), cl::Hidden,
167	cl::desc("Perform frequent verification checks on nodes."));
168
169	static cl::opt<std::string> MemProfImportSummary(
170	"memprof-import-summary",
171	cl::desc("Import summary to use for testing the ThinLTO backend via opt"),
172	cl::Hidden);
173
174	static cl::opt<unsigned>
175	TailCallSearchDepth("memprof-tail-call-search-depth", cl::init(Val: `5`),
176	cl::Hidden,
177	cl::desc("Max depth to recursively search for missing "
178	"frames through tail calls."));
179
180	// Optionally enable cloning of callsites involved with recursive cycles
181	static cl::opt<bool> AllowRecursiveCallsites(
182	"memprof-allow-recursive-callsites", cl::init(Val: true), cl::Hidden,
183	cl::desc("Allow cloning of callsites involved in recursive cycles"));
184
185	static cl::opt<bool> CloneRecursiveContexts(
186	"memprof-clone-recursive-contexts", cl::init(Val: true), cl::Hidden,
187	cl::desc("Allow cloning of contexts through recursive cycles"));
188
189	// Generally this is needed for correct assignment of allocation clones to
190	// function clones, however, allow it to be disabled for debugging while the
191	// functionality is new and being tested more widely.
192	static cl::opt<bool>
193	MergeClones("memprof-merge-clones", cl::init(Val: true), cl::Hidden,
194	cl::desc("Merge clones before assigning functions"));
195
196	// When disabled, try to detect and prevent cloning of recursive contexts.
197	// This is only necessary until we support cloning through recursive cycles.
198	// Leave on by default for now, as disabling requires a little bit of compile
199	// time overhead and doesn't affect correctness, it will just inflate the cold
200	// hinted bytes reporting a bit when -memprof-report-hinted-sizes is enabled.
201	static cl::opt<bool> AllowRecursiveContexts(
202	"memprof-allow-recursive-contexts", cl::init(Val: true), cl::Hidden,
203	cl::desc("Allow cloning of contexts having recursive cycles"));
204
205	// Set the minimum absolute count threshold for allowing inlining of indirect
206	// calls promoted during cloning.
207	static cl::opt<unsigned> MemProfICPNoInlineThreshold(
208	"memprof-icp-noinline-threshold", cl::init(Val: `0`), cl::Hidden,
209	cl::desc("Minimum absolute count for promoted target to be inlinable"));
210
211	namespace llvm {
212	cl::opt<bool> EnableMemProfContextDisambiguation(
213	"enable-memprof-context-disambiguation", cl::Hidden,
214	cl::desc("Enable MemProf context disambiguation"));
215
216	// Indicate we are linking with an allocator that supports hot/cold operator
217	// new interfaces.
218	cl::opt<bool> SupportsHotColdNew(
219	"supports-hot-cold-new", cl::init(Val: false), cl::Hidden,
220	cl::desc("Linking with hot/cold operator new interfaces"));
221
222	static cl::opt<bool> MemProfRequireDefinitionForPromotion(
223	"memprof-require-definition-for-promotion", cl::init(Val: false), cl::Hidden,
224	cl::desc(
225	"Require target function definition when promoting indirect calls"));
226
227	extern cl::opt<bool> MemProfReportHintedSizes;
228	extern cl::opt<unsigned> MinClonedColdBytePercent;
229
230	cl::opt<unsigned> MemProfTopNImportant(
231	"memprof-top-n-important", cl::init(Val: `10`), cl::Hidden,
232	cl::desc("Number of largest cold contexts to consider important"));
233
234	cl::opt<bool> MemProfFixupImportant(
235	"memprof-fixup-important", cl::init(Val: true), cl::Hidden,
236	cl::desc("Enables edge fixup for important contexts"));
237
238	extern cl::opt<unsigned> MaxSummaryIndirectEdges;
239
240	} // namespace llvm
241
242	namespace {
243
244	/// CRTP base for graphs built from either IR or ThinLTO summary index.
245	///
246	/// The graph represents the call contexts in all memprof metadata on allocation
247	/// calls, with nodes for the allocations themselves, as well as for the calls
248	/// in each context. The graph is initially built from the allocation memprof
249	/// metadata (or summary) MIBs. It is then updated to match calls with callsite
250	/// metadata onto the nodes, updating it to reflect any inlining performed on
251	/// those calls.
252	///
253	/// Each MIB (representing an allocation's call context with allocation
254	/// behavior) is assigned a unique context id during the graph build. The edges
255	/// and nodes in the graph are decorated with the context ids they carry. This
256	/// is used to correctly update the graph when cloning is performed so that we
257	/// can uniquify the context for a single (possibly cloned) allocation.
258	template <typename DerivedCCG, typename FuncTy, typename CallTy>
259	class CallsiteContextGraph {
260	public:
261	CallsiteContextGraph() = default;
262	CallsiteContextGraph(const CallsiteContextGraph &) = default;
263	CallsiteContextGraph(CallsiteContextGraph &&) = default;
264
265	/// Main entry point to perform analysis and transformations on graph.
266	bool process(function_ref<void(StringRef, StringRef, const Twine &)>
267	EmitRemark = nullptr,
268	bool AllowExtraAnalysis = false);
269
270	/// Perform cloning on the graph necessary to uniquely identify the allocation
271	/// behavior of an allocation based on its context.
272	void identifyClones();
273
274	/// Assign callsite clones to functions, cloning functions as needed to
275	/// accommodate the combinations of their callsite clones reached by callers.
276	/// For regular LTO this clones functions and callsites in the IR, but for
277	/// ThinLTO the cloning decisions are noted in the summaries and later applied
278	/// in applyImport.
279	bool assignFunctions();
280
281	void dump() const;
282	void print(raw_ostream &OS) const;
283	void printTotalSizes(raw_ostream &OS,
284	function_ref<void(StringRef, StringRef, const Twine &)>
285	EmitRemark = nullptr) const;
286
287	friend raw_ostream &operator<<(raw_ostream &OS,
288	const CallsiteContextGraph &CCG) {
289	CCG.print(OS);
290	return OS;
291	}
292
293	friend struct GraphTraits<
294	const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>;
295	friend struct DOTGraphTraits<
296	const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>;
297
298	void exportToDot(std::string Label) const;
299
300	/// Represents a function clone via FuncTy pointer and clone number pair.
301	struct FuncInfo final
302	: public std::pair<FuncTy , unsigned* /Clone number/> {
303	using Base = std::pair<FuncTy , unsigned*>;
304	FuncInfo(const Base &B) : Base(B) {}
305	FuncInfo(FuncTy F = nullptr, unsigned* CloneNo = `0`) : Base(F, CloneNo) {}
306	explicit operator bool() const { return this->first != nullptr; }
307	FuncTy func() const* { return this->first; }
308	unsigned cloneNo() const { return this->second; }
309	};
310
311	/// Represents a callsite clone via CallTy and clone number pair.
312	struct CallInfo final : public std::pair<CallTy, unsigned /Clone number/> {
313	using Base = std::pair<CallTy, unsigned>;
314	CallInfo(const Base &B) : Base(B) {}
315	CallInfo(CallTy Call = nullptr, unsigned CloneNo = `0`)
316	: Base(Call, CloneNo) {}
317	explicit operator bool() const { return (bool)this->first; }
318	CallTy call() const { return this->first; }
319	unsigned cloneNo() const { return this->second; }
320	void setCloneNo(unsigned N) { this->second = N; }
321	void print(raw_ostream &OS) const {
322	if (!operator bool()) {
323	assert(!cloneNo());
324	OS << "null Call";
325	return;
326	}
327	call()->print(OS);
328	OS << "\t(clone " << cloneNo() << ")";
329	}
330	void dump() const {
331	print(OS&: dbgs());
332	dbgs() << "\n";
333	}
334	friend raw_ostream &operator<<(raw_ostream &OS, const CallInfo &Call) {
335	Call.print(OS);
336	return OS;
337	}
338	};
339
340	struct ContextEdge;
341
342	/// Node in the Callsite Context Graph
343	struct ContextNode {
344	// Assigned to nodes as they are created, useful for debugging.
345	unsigned NodeId = `0`;
346
347	// Keep this for now since in the IR case where we have an Instruction it*
348	// is not as immediately discoverable. Used for printing richer information
349	// when dumping graph.
350	bool IsAllocation;
351
352	// Keeps track of when the Call was reset to null because there was
353	// recursion.
354	bool Recursive = false;
355
356	// This will be formed by ORing together the AllocationType enum values
357	// for contexts including this node.
358	uint8_t AllocTypes = `0`;
359
360	// The corresponding allocation or interior call. This is the primary call
361	// for which we have created this node.
362	CallInfo Call;
363
364	// List of other calls that can be treated the same as the primary call
365	// through cloning. I.e. located in the same function and have the same
366	// (possibly pruned) stack ids. They will be updated the same way as the
367	// primary call when assigning to function clones.
368	SmallVector<CallInfo, `0`> MatchingCalls;
369
370	// For alloc nodes this is a unique id assigned when constructed, and for
371	// callsite stack nodes it is the original stack id when the node is
372	// constructed from the memprof MIB metadata on the alloc nodes. Note that
373	// this is only used when matching callsite metadata onto the stack nodes
374	// created when processing the allocation memprof MIBs, and for labeling
375	// nodes in the dot graph. Therefore we don't bother to assign a value for
376	// clones.
377	uint64_t OrigStackOrAllocId = `0`;
378
379	// Edges to all callees in the profiled call stacks.
380	// TODO: Should this be a map (from Callee node) for more efficient lookup?
381	std::vector<std::shared_ptr<ContextEdge>> CalleeEdges;
382
383	// Edges to all callers in the profiled call stacks.
384	// TODO: Should this be a map (from Caller node) for more efficient lookup?
385	std::vector<std::shared_ptr<ContextEdge>> CallerEdges;
386
387	// Returns true if we need to look at the callee edges for determining the
388	// node context ids and allocation type.
389	bool useCallerEdgesForContextInfo() const {
390	// Typically if the callee edges are empty either the caller edges are
391	// also empty, or this is an allocation (leaf node). However, if we are
392	// allowing recursive callsites and contexts this will be violated for
393	// incompletely cloned recursive cycles.
394	assert(!CalleeEdges.empty() \|\| CallerEdges.empty() \|\| IsAllocation \|\|
395	(AllowRecursiveCallsites && AllowRecursiveContexts));
396	// When cloning for a recursive context, during cloning we might be in the
397	// midst of cloning for a recurrence and have moved context ids off of a
398	// caller edge onto the clone but not yet off of the incoming caller
399	// (back) edge. If we don't look at those we miss the fact that this node
400	// still has context ids of interest.
401	return IsAllocation \|\| CloneRecursiveContexts;
402	}
403
404	// Compute the context ids for this node from the union of its edge context
405	// ids.
406	DenseSet<uint32_t> getContextIds() const {
407	unsigned Count = `0`;
408	// Compute the number of ids for reserve below. In general we only need to
409	// look at one set of edges, typically the callee edges, since other than
410	// allocations and in some cases during recursion cloning, all the context
411	// ids on the callers should also flow out via callee edges.
412	for (auto &Edge : CalleeEdges.empty() ? CallerEdges : CalleeEdges)
413	Count += Edge->getContextIds().size();
414	DenseSet<uint32_t> ContextIds;
415	ContextIds.reserve(Size: Count);
416	auto Edges = llvm::concat<const std::shared_ptr<ContextEdge>>(
417	CalleeEdges, useCallerEdgesForContextInfo()
418	? CallerEdges
419	: std::vector<std::shared_ptr<ContextEdge>>());
420	for (const auto &Edge : Edges)
421	ContextIds.insert_range(Edge->getContextIds());
422	return ContextIds;
423	}
424
425	// Compute the allocation type for this node from the OR of its edge
426	// allocation types.
427	uint8_t computeAllocType() const {
428	uint8_t BothTypes =
429	(uint8_t)AllocationType::Cold \| (uint8_t)AllocationType::NotCold;
430	uint8_t AllocType = (uint8_t)AllocationType::None;
431	auto Edges = llvm::concat<const std::shared_ptr<ContextEdge>>(
432	CalleeEdges, useCallerEdgesForContextInfo()
433	? CallerEdges
434	: std::vector<std::shared_ptr<ContextEdge>>());
435	for (const auto &Edge : Edges) {
436	AllocType \|= Edge->AllocTypes;
437	// Bail early if alloc type reached both, no further refinement.
438	if (AllocType == BothTypes)
439	return AllocType;
440	}
441	return AllocType;
442	}
443
444	// The context ids set for this node is empty if its edge context ids are
445	// also all empty.
446	bool emptyContextIds() const {
447	auto Edges = llvm::concat<const std::shared_ptr<ContextEdge>>(
448	CalleeEdges, useCallerEdgesForContextInfo()
449	? CallerEdges
450	: std::vector<std::shared_ptr<ContextEdge>>());
451	for (const auto &Edge : Edges) {
452	if (!Edge->getContextIds().empty())
453	return false;
454	}
455	return true;
456	}
457
458	// List of clones of this ContextNode, initially empty.
459	std::vector<ContextNode *> Clones;
460
461	// If a clone, points to the original uncloned node.
462	ContextNode CloneOf = nullptr*;
463
464	ContextNode(bool IsAllocation) : IsAllocation(IsAllocation), Call() {}
465
466	ContextNode(bool IsAllocation, CallInfo C)
467	: IsAllocation(IsAllocation), Call(C) {}
468
469	void addClone(ContextNode *Clone) {
470	if (CloneOf) {
471	CloneOf->Clones.push_back(Clone);
472	Clone->CloneOf = CloneOf;
473	} else {
474	Clones.push_back(Clone);
475	assert(!Clone->CloneOf);
476	Clone->CloneOf = this;
477	}
478	}
479
480	ContextNode *getOrigNode() {
481	if (!CloneOf)
482	return this;
483	return CloneOf;
484	}
485
486	void addOrUpdateCallerEdge(ContextNode *Caller, AllocationType AllocType,
487	unsigned int ContextId);
488
489	ContextEdge findEdgeFromCallee(const* ContextNode *Callee);
490	ContextEdge findEdgeFromCaller(const* ContextNode *Caller);
491	void eraseCalleeEdge(const ContextEdge *Edge);
492	void eraseCallerEdge(const ContextEdge *Edge);
493
494	void setCall(CallInfo C) { Call = std::move(C); }
495
496	bool hasCall() const { return (bool)Call.call(); }
497
498	void printCall(raw_ostream &OS) const { Call.print(OS); }
499
500	// True if this node was effectively removed from the graph, in which case
501	// it should have an allocation type of None and empty context ids.
502	bool isRemoved() const {
503	// Typically if the callee edges are empty either the caller edges are
504	// also empty, or this is an allocation (leaf node). However, if we are
505	// allowing recursive callsites and contexts this will be violated for
506	// incompletely cloned recursive cycles.
507	assert((AllowRecursiveCallsites && AllowRecursiveContexts) \|\|
508	(AllocTypes == (uint8_t)AllocationType::None) ==
509	emptyContextIds());
510	return AllocTypes == (uint8_t)AllocationType::None;
511	}
512
513	void dump() const;
514	void print(raw_ostream &OS) const;
515
516	friend raw_ostream &operator<<(raw_ostream &OS, const ContextNode &Node) {
517	Node.print(OS);
518	return OS;
519	}
520	};
521
522	/// Edge in the Callsite Context Graph from a ContextNode N to a caller or
523	/// callee.
524	struct ContextEdge {
525	ContextNode *Callee;
526	ContextNode *Caller;
527
528	// This will be formed by ORing together the AllocationType enum values
529	// for contexts including this edge.
530	uint8_t AllocTypes = `0`;
531
532	// Set just before initiating cloning when cloning of recursive contexts is
533	// enabled. Used to defer cloning of backedges until we have done cloning of
534	// the callee node for non-backedge caller edges. This exposes cloning
535	// opportunities through the backedge of the cycle.
536	// TODO: Note that this is not updated during cloning, and it is unclear
537	// whether that would be needed.
538	bool IsBackedge = false;
539
540	// The set of IDs for contexts including this edge.
541	DenseSet<uint32_t> ContextIds;
542
543	ContextEdge(ContextNode Callee, ContextNode Caller, uint8_t AllocType,
544	DenseSet<uint32_t> ContextIds)
545	: Callee(Callee), Caller(Caller), AllocTypes(AllocType),
546	ContextIds (std::move(ContextIds)) {}
547
548	DenseSet<uint32_t> &getContextIds() { return ContextIds; }
549
550	// Helper to clear the fields of this edge when we are removing it from the
551	// graph.
552	inline void clear() {
553	ContextIds.clear();
554	AllocTypes = (uint8_t)AllocationType::None;
555	Caller = nullptr;
556	Callee = nullptr;
557	}
558
559	// Check if edge was removed from the graph. This is useful while iterating
560	// over a copy of edge lists when performing operations that mutate the
561	// graph in ways that might remove one of the edges.
562	inline bool isRemoved() const {
563	if (Callee \|\| Caller)
564	return false;
565	// Any edges that have been removed from the graph but are still in a
566	// shared_ptr somewhere should have all fields null'ed out by clear()
567	// above.
568	assert(AllocTypes == (uint8_t)AllocationType::None);
569	assert(ContextIds.empty());
570	return true;
571	}
572
573	void dump() const;
574	void print(raw_ostream &OS) const;
575
576	friend raw_ostream &operator<<(raw_ostream &OS, const ContextEdge &Edge) {
577	Edge.print(OS);
578	return OS;
579	}
580	};
581
582	/// Helpers to remove edges that have allocation type None (due to not
583	/// carrying any context ids) after transformations.
584	void removeNoneTypeCalleeEdges(ContextNode *Node);
585	void removeNoneTypeCallerEdges(ContextNode *Node);
586	void
587	recursivelyRemoveNoneTypeCalleeEdges(ContextNode *Node,
588	DenseSet<const ContextNode *> &Visited);
589
590	protected:
591	/// Get a list of nodes corresponding to the stack ids in the given callsite
592	/// context.
593	template <class NodeT, class IteratorT>
594	std::vector<uint64_t>
595	getStackIdsWithContextNodes(CallStack<NodeT, IteratorT> &CallsiteContext);
596
597	/// Adds nodes for the given allocation and any stack ids on its memprof MIB
598	/// metadata (or summary).
599	ContextNode addAllocNode(CallInfo Call, const* FuncTy *F);
600
601	/// Adds nodes for the given MIB stack ids.
602	template <class NodeT, class IteratorT>
603	void addStackNodesForMIB(
604	ContextNode *AllocNode, CallStack<NodeT, IteratorT> &StackContext,
605	CallStack<NodeT, IteratorT> &CallsiteContext, AllocationType AllocType,
606	ArrayRef<ContextTotalSize> ContextSizeInfo,
607	std::map<uint64_t, uint32_t> &TotalSizeToContextIdTopNCold);
608
609	/// Matches all callsite metadata (or summary) to the nodes created for
610	/// allocation memprof MIB metadata, synthesizing new nodes to reflect any
611	/// inlining performed on those callsite instructions.
612	void updateStackNodes();
613
614	/// Optionally fixup edges for the N largest cold contexts to better enable
615	/// cloning. This is particularly helpful if the context includes recursion
616	/// as well as inlining, resulting in a single stack node for multiple stack
617	/// ids in the context. With recursion it is particularly difficult to get the
618	/// edge updates correct as in the general case we have lost the original
619	/// stack id ordering for the context. Do more expensive fixup for the largest
620	/// contexts, controlled by MemProfTopNImportant and MemProfFixupImportant.
621	void fixupImportantContexts();
622
623	/// Update graph to conservatively handle any callsite stack nodes that target
624	/// multiple different callee target functions.
625	void handleCallsitesWithMultipleTargets();
626
627	/// Mark backedges via the standard DFS based backedge algorithm.
628	void markBackedges();
629
630	/// Merge clones generated during cloning for different allocations but that
631	/// are called by the same caller node, to ensure proper function assignment.
632	void mergeClones();
633
634	// Try to partition calls on the given node (already placed into the AllCalls
635	// array) by callee function, creating new copies of Node as needed to hold
636	// calls with different callees, and moving the callee edges appropriately.
637	// Returns true if partitioning was successful.
638	bool partitionCallsByCallee(
639	ContextNode *Node, ArrayRef<CallInfo> AllCalls,
640	std::vector<std::pair<CallInfo, ContextNode *>> &NewCallToNode);
641
642	/// Save lists of calls with MemProf metadata in each function, for faster
643	/// iteration.
644	MapVector<FuncTy *, std::vector<CallInfo>> FuncToCallsWithMetadata;
645
646	/// Map from callsite node to the enclosing caller function.
647	std::map<const ContextNode , const* FuncTy *> NodeToCallingFunc;
648
649	// When exporting to dot, and an allocation id is specified, contains the
650	// context ids on that allocation.
651	DenseSet<uint32_t> DotAllocContextIds;
652
653	private:
654	using EdgeIter = typename std::vector<std::shared_ptr<ContextEdge>>::iterator;
655
656	// Structure to keep track of information for each call as we are matching
657	// non-allocation callsites onto context nodes created from the allocation
658	// call metadata / summary contexts.
659	struct CallContextInfo {
660	// The callsite we're trying to match.
661	CallTy Call;
662	// The callsites stack ids that have a context node in the graph.
663	std::vector<uint64_t> StackIds;
664	// The function containing this callsite.
665	const FuncTy *Func;
666	// Initially empty, if needed this will be updated to contain the context
667	// ids for use in a new context node created for this callsite.
668	DenseSet<uint32_t> ContextIds;
669	};
670
671	/// Helper to remove edge from graph, updating edge iterator if it is provided
672	/// (in which case CalleeIter indicates which edge list is being iterated).
673	/// This will also perform the necessary clearing of the ContextEdge members
674	/// to enable later checking if the edge has been removed (since we may have
675	/// other copies of the shared_ptr in existence, and in fact rely on this to
676	/// enable removal while iterating over a copy of a node's edge list).
677	void removeEdgeFromGraph(ContextEdge Edge, EdgeIter EI = nullptr,
678	bool CalleeIter = true);
679
680	/// Assigns the given Node to calls at or inlined into the location with
681	/// the Node's stack id, after post order traversing and processing its
682	/// caller nodes. Uses the call information recorded in the given
683	/// StackIdToMatchingCalls map, and creates new nodes for inlined sequences
684	/// as needed. Called by updateStackNodes which sets up the given
685	/// StackIdToMatchingCalls map.
686	void assignStackNodesPostOrder(
687	ContextNode Node, DenseSet<const* ContextNode *> &Visited,
688	DenseMap<uint64_t, std::vector<CallContextInfo>> &StackIdToMatchingCalls,
689	DenseMap<CallInfo, CallInfo> &CallToMatchingCall,
690	const DenseSet<uint32_t> &ImportantContextIds);
691
692	/// Duplicates the given set of context ids, updating the provided
693	/// map from each original id with the newly generated context ids,
694	/// and returning the new duplicated id set.
695	DenseSet<uint32_t> duplicateContextIds(
696	const DenseSet<uint32_t> &StackSequenceContextIds,
697	DenseMap<uint32_t, DenseSet<uint32_t>> &OldToNewContextIds);
698
699	/// Propagates all duplicated context ids across the graph.
700	void propagateDuplicateContextIds(
701	const DenseMap<uint32_t, DenseSet<uint32_t>> &OldToNewContextIds);
702
703	/// Connect the NewNode to OrigNode's callees if TowardsCallee is true,
704	/// else to its callers. Also updates OrigNode's edges to remove any context
705	/// ids moved to the newly created edge.
706	void connectNewNode(ContextNode NewNode, ContextNode OrigNode,
707	bool TowardsCallee, const DenseSet<uint32_t> &ContextIds);
708
709	/// Get the stack id corresponding to the given Id or Index (for IR this will
710	/// return itself, for a summary index this will return the id recorded in the
711	/// index for that stack id index value).
712	uint64_t getStackId(uint64_t IdOrIndex) const {
713	return static_cast<const DerivedCCG >(this*)->getStackId(IdOrIndex);
714	}
715
716	/// Returns true if the given call targets the callee of the given edge, or if
717	/// we were able to identify the call chain through intermediate tail calls.
718	/// In the latter case new context nodes are added to the graph for the
719	/// identified tail calls, and their synthesized nodes are added to
720	/// TailCallToContextNodeMap. The EdgeIter is updated in the latter case for
721	/// the updated edges and to prepare it for an increment in the caller.
722	bool
723	calleesMatch(CallTy Call, EdgeIter &EI,
724	MapVector<CallInfo, ContextNode *> &TailCallToContextNodeMap);
725
726	// Return the callee function of the given call, or nullptr if it can't be
727	// determined
728	const FuncTy *getCalleeFunc(CallTy Call) {
729	return static_cast<DerivedCCG >(this*)->getCalleeFunc(Call);
730	}
731
732	/// Returns true if the given call targets the given function, or if we were
733	/// able to identify the call chain through intermediate tail calls (in which
734	/// case FoundCalleeChain will be populated).
735	bool calleeMatchesFunc(
736	CallTy Call, const FuncTy Func, const* FuncTy *CallerFunc,
737	std::vector<std::pair<CallTy, FuncTy *>> &FoundCalleeChain) {
738	return static_cast<DerivedCCG >(this*)->calleeMatchesFunc(
739	Call, Func, CallerFunc, FoundCalleeChain);
740	}
741
742	/// Returns true if both call instructions have the same callee.
743	bool sameCallee(CallTy Call1, CallTy Call2) {
744	return static_cast<DerivedCCG >(this*)->sameCallee(Call1, Call2);
745	}
746
747	/// Get a list of nodes corresponding to the stack ids in the given
748	/// callsite's context.
749	std::vector<uint64_t> getStackIdsWithContextNodesForCall(CallTy Call) {
750	return static_cast<DerivedCCG >(this*)->getStackIdsWithContextNodesForCall(
751	Call);
752	}
753
754	/// Get the last stack id in the context for callsite.
755	uint64_t getLastStackId(CallTy Call) {
756	return static_cast<DerivedCCG >(this*)->getLastStackId(Call);
757	}
758
759	/// Update the allocation call to record type of allocated memory.
760	void updateAllocationCall(CallInfo &Call, AllocationType AllocType) {
761	AllocType == AllocationType::Cold ? AllocTypeCold ++ : AllocTypeNotCold ++;
762	static_cast<DerivedCCG >(this*)->updateAllocationCall(Call, AllocType);
763	}
764
765	/// Get the AllocationType assigned to the given allocation instruction clone.
766	AllocationType getAllocationCallType(const CallInfo &Call) const {
767	return static_cast<const DerivedCCG >(this*)->getAllocationCallType(Call);
768	}
769
770	/// Update non-allocation call to invoke (possibly cloned) function
771	/// CalleeFunc.
772	void updateCall(CallInfo &CallerCall, FuncInfo CalleeFunc) {
773	static_cast<DerivedCCG >(this*)->updateCall(CallerCall, CalleeFunc);
774	}
775
776	/// Clone the given function for the given callsite, recording mapping of all
777	/// of the functions tracked calls to their new versions in the CallMap.
778	/// Assigns new clones to clone number CloneNo.
779	FuncInfo cloneFunctionForCallsite(
780	FuncInfo &Func, CallInfo &Call, DenseMap<CallInfo, CallInfo> &CallMap,
781	std::vector<CallInfo> &CallsWithMetadataInFunc, unsigned CloneNo) {
782	return static_cast<DerivedCCG >(this*)->cloneFunctionForCallsite(
783	Func, Call, CallMap, CallsWithMetadataInFunc, CloneNo);
784	}
785
786	/// Gets a label to use in the dot graph for the given call clone in the given
787	/// function.
788	std::string getLabel(const FuncTy Func, const* CallTy Call,
789	unsigned CloneNo) const {
790	return static_cast<const DerivedCCG >(this*)->getLabel(Func, Call, CloneNo);
791	}
792
793	// Create and return a new ContextNode.
794	ContextNode createNewNode(bool* IsAllocation, const FuncTy F = nullptr*,
795	CallInfo C = CallInfo()) {
796	NodeOwner.push_back(std::make_unique<ContextNode>(IsAllocation, C));
797	auto *NewNode = NodeOwner.back().get();
798	if (F)
799	NodeToCallingFunc[NewNode] = F;
800	NewNode->NodeId = NodeOwner.size();
801	return NewNode;
802	}
803
804	/// Helpers to find the node corresponding to the given call or stackid.
805	ContextNode getNodeForInst(const* CallInfo &C);
806	ContextNode getNodeForAlloc(const* CallInfo &C);
807	ContextNode *getNodeForStackId(uint64_t StackId);
808
809	/// Computes the alloc type corresponding to the given context ids, by
810	/// unioning their recorded alloc types.
811	uint8_t computeAllocType(DenseSet<uint32_t> &ContextIds) const;
812
813	/// Returns the allocation type of the intersection of the contexts of two
814	/// nodes (based on their provided context id sets), optimized for the case
815	/// when Node1Ids is smaller than Node2Ids.
816	uint8_t intersectAllocTypesImpl(const DenseSet<uint32_t> &Node1Ids,
817	const DenseSet<uint32_t> &Node2Ids) const;
818
819	/// Returns the allocation type of the intersection of the contexts of two
820	/// nodes (based on their provided context id sets).
821	uint8_t intersectAllocTypes(const DenseSet<uint32_t> &Node1Ids,
822	const DenseSet<uint32_t> &Node2Ids) const;
823
824	/// Create a clone of Edge's callee and move Edge to that new callee node,
825	/// performing the necessary context id and allocation type updates.
826	/// If ContextIdsToMove is non-empty, only that subset of Edge's ids are
827	/// moved to an edge to the new callee.
828	ContextNode *
829	moveEdgeToNewCalleeClone(const std::shared_ptr<ContextEdge> &Edge,
830	DenseSet<uint32_t> ContextIdsToMove = {});
831
832	/// Change the callee of Edge to existing callee clone NewCallee, performing
833	/// the necessary context id and allocation type updates.
834	/// If ContextIdsToMove is non-empty, only that subset of Edge's ids are
835	/// moved to an edge to the new callee.
836	void moveEdgeToExistingCalleeClone(const std::shared_ptr<ContextEdge> &Edge,
837	ContextNode *NewCallee,
838	bool NewClone = false,
839	DenseSet<uint32_t> ContextIdsToMove = {});
840
841	/// Change the caller of the edge at the given callee edge iterator to be
842	/// NewCaller, performing the necessary context id and allocation type
843	/// updates. This is similar to the above moveEdgeToExistingCalleeClone, but
844	/// a simplified version of it as we always move the given edge and all of its
845	/// context ids.
846	void moveCalleeEdgeToNewCaller(const std::shared_ptr<ContextEdge> &Edge,
847	ContextNode *NewCaller);
848
849	/// Recursive helper for marking backedges via DFS.
850	void markBackedges(ContextNode Node, DenseSet<const* ContextNode *> &Visited,
851	DenseSet<const ContextNode *> &CurrentStack);
852
853	/// Recursive helper for merging clones.
854	void
855	mergeClones(ContextNode Node, DenseSet<const* ContextNode *> &Visited,
856	DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode);
857	/// Main worker for merging callee clones for a given node.
858	void mergeNodeCalleeClones(
859	ContextNode Node, DenseSet<const* ContextNode *> &Visited,
860	DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode);
861	/// Helper to find other callers of the given set of callee edges that can
862	/// share the same callee merge node.
863	void findOtherCallersToShareMerge(
864	ContextNode *Node, std::vector<std::shared_ptr<ContextEdge>> &CalleeEdges,
865	DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode,
866	DenseSet<ContextNode *> &OtherCallersToShareMerge);
867
868	/// Recursively perform cloning on the graph for the given Node and its
869	/// callers, in order to uniquely identify the allocation behavior of an
870	/// allocation given its context. The context ids of the allocation being
871	/// processed are given in AllocContextIds.
872	void identifyClones(ContextNode Node, DenseSet<const* ContextNode *> &Visited,
873	const DenseSet<uint32_t> &AllocContextIds);
874
875	/// Map from each context ID to the AllocationType assigned to that context.
876	DenseMap<uint32_t, AllocationType> ContextIdToAllocationType;
877
878	/// Map from each contextID to the profiled full contexts and their total
879	/// sizes (there may be more than one due to context trimming),
880	/// optionally populated when requested (via MemProfReportHintedSizes or
881	/// MinClonedColdBytePercent).
882	DenseMap<uint32_t, std::vector<ContextTotalSize>> ContextIdToContextSizeInfos;
883
884	/// Identifies the context node created for a stack id when adding the MIB
885	/// contexts to the graph. This is used to locate the context nodes when
886	/// trying to assign the corresponding callsites with those stack ids to these
887	/// nodes.
888	DenseMap<uint64_t, ContextNode *> StackEntryIdToContextNodeMap;
889
890	/// Saves information for the contexts identified as important (the largest
891	/// cold contexts up to MemProfTopNImportant).
892	struct ImportantContextInfo {
893	// The original list of leaf first stack ids corresponding to this context.
894	std::vector<uint64_t> StackIds;
895	// Max length of stack ids corresponding to a single stack ContextNode for
896	// this context (i.e. the max length of a key in StackIdsToNode below).
897	unsigned MaxLength = `0`;
898	// Mapping of slices of the stack ids to the corresponding ContextNode
899	// (there can be multiple stack ids due to inlining). Populated when
900	// updating stack nodes while matching them to the IR or summary.
901	std::map<std::vector<uint64_t>, ContextNode *> StackIdsToNode;
902	};
903
904	// Map of important full context ids to information about each.
905	DenseMap<uint32_t, ImportantContextInfo> ImportantContextIdInfo;
906
907	// For each important context id found in Node (if any), records the list of
908	// stack ids that corresponded to the given callsite Node. There can be more
909	// than one in the case of inlining.
910	void recordStackNode(std::vector<uint64_t> &StackIds, ContextNode *Node,
911	// We pass in the Node's context ids to avoid the
912	// overhead of computing them as the caller already has
913	// them in some cases.
914	const DenseSet<uint32_t> &NodeContextIds,
915	const DenseSet<uint32_t> &ImportantContextIds) {
916	if (!MemProfTopNImportant) {
917	assert(ImportantContextIds.empty());
918	return;
919	}
920	DenseSet<uint32_t> Ids =
921	set_intersection(S1: NodeContextIds, S2: ImportantContextIds);
922	if (Ids.empty())
923	return;
924	auto Size = StackIds.size();
925	for (auto Id : Ids) {
926	auto &Entry = ImportantContextIdInfo[Id];
927	Entry.StackIdsToNode[StackIds] = Node;
928	// Keep track of the max to simplify later analysis.
929	if (Size > Entry.MaxLength)
930	Entry.MaxLength = Size;
931	}
932	}
933
934	/// Maps to track the calls to their corresponding nodes in the graph.
935	MapVector<CallInfo, ContextNode *> AllocationCallToContextNodeMap;
936	MapVector<CallInfo, ContextNode *> NonAllocationCallToContextNodeMap;
937
938	/// Owner of all ContextNode unique_ptrs.
939	std::vector<std::unique_ptr<ContextNode>> NodeOwner;
940
941	/// Perform sanity checks on graph when requested.
942	void check() const;
943
944	/// Keeps track of the last unique context id assigned.
945	unsigned int LastContextId = `0`;
946	};
947
948	template <typename DerivedCCG, typename FuncTy, typename CallTy>
949	using ContextNode =
950	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode;
951	template <typename DerivedCCG, typename FuncTy, typename CallTy>
952	using ContextEdge =
953	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge;
954	template <typename DerivedCCG, typename FuncTy, typename CallTy>
955	using FuncInfo =
956	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::FuncInfo;
957	template <typename DerivedCCG, typename FuncTy, typename CallTy>
958	using CallInfo =
959	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::CallInfo;
960
961	/// CRTP derived class for graphs built from IR (regular LTO).
962	class ModuleCallsiteContextGraph
963	: public CallsiteContextGraph<ModuleCallsiteContextGraph, Function,
964	Instruction *> {
965	public:
966	ModuleCallsiteContextGraph(
967	Module &M,
968	llvm::function_ref<OptimizationRemarkEmitter &(Function *)> OREGetter);
969
970	private:
971	friend CallsiteContextGraph<ModuleCallsiteContextGraph, Function,
972	Instruction *>;
973
974	uint64_t getStackId(uint64_t IdOrIndex) const;
975	const Function getCalleeFunc(Instruction Call);
976	bool calleeMatchesFunc(
977	Instruction Call, const* Function Func, const* Function *CallerFunc,
978	std::vector<std::pair<Instruction , Function >> &FoundCalleeChain);
979	bool sameCallee(Instruction Call1, Instruction Call2);
980	bool findProfiledCalleeThroughTailCalls(
981	const Function ProfiledCallee, Value CurCallee, unsigned Depth,
982	std::vector<std::pair<Instruction , Function >> &FoundCalleeChain,
983	bool &FoundMultipleCalleeChains);
984	uint64_t getLastStackId(Instruction *Call);
985	std::vector<uint64_t> getStackIdsWithContextNodesForCall(Instruction *Call);
986	void updateAllocationCall(CallInfo &Call, AllocationType AllocType);
987	AllocationType getAllocationCallType(const CallInfo &Call) const;
988	void updateCall(CallInfo &CallerCall, FuncInfo CalleeFunc);
989	CallsiteContextGraph<ModuleCallsiteContextGraph, Function,
990	Instruction *>::FuncInfo
991	cloneFunctionForCallsite(FuncInfo &Func, CallInfo &Call,
992	DenseMap<CallInfo, CallInfo> &CallMap,
993	std::vector<CallInfo> &CallsWithMetadataInFunc,
994	unsigned CloneNo);
995	std::string getLabel(const Function Func, const* Instruction *Call,
996	unsigned CloneNo) const;
997
998	const Module &Mod;
999	llvm::function_ref<OptimizationRemarkEmitter &(Function *)> OREGetter;
1000	};
1001
1002	/// Represents a call in the summary index graph, which can either be an
1003	/// allocation or an interior callsite node in an allocation's context.
1004	/// Holds a pointer to the corresponding data structure in the index.
1005	struct IndexCall : public PointerUnion<CallsiteInfo , AllocInfo > {
1006	IndexCall() : PointerUnion () {}
1007	IndexCall(std::nullptr_t) : IndexCall () {}
1008	IndexCall(CallsiteInfo *StackNode) : PointerUnion (StackNode) {}
1009	IndexCall(AllocInfo *AllocNode) : PointerUnion (AllocNode) {}
1010	IndexCall(PointerUnion PT) : PointerUnion (PT) {}
1011
1012	IndexCall *operator->() { return this; }
1013
1014	void print(raw_ostream &OS) const {
1015	PointerUnion<CallsiteInfo , AllocInfo > Base = *this;
1016	if (auto AI = llvm::dyn_cast_if_present<AllocInfo >(Val&: Base)) {
1017	OS << *AI;
1018	} else {
1019	auto CI = llvm::dyn_cast_if_present<CallsiteInfo >(Val&: Base);
1020	assert(CI);
1021	OS << *CI;
1022	}
1023	}
1024	};
1025	} // namespace
1026
1027	namespace llvm {
1028	template <> struct simplify_type<IndexCall> {
1029	using SimpleType = PointerUnion<CallsiteInfo , AllocInfo >;
1030	static SimpleType getSimplifiedValue(IndexCall &Val) { return Val; }
1031	};
1032	template <> struct simplify_type<const IndexCall> {
1033	using SimpleType = const PointerUnion<CallsiteInfo , AllocInfo >;
1034	static SimpleType getSimplifiedValue(const IndexCall &Val) { return Val; }
1035	};
1036	} // namespace llvm
1037
1038	namespace {
1039	/// CRTP derived class for graphs built from summary index (ThinLTO).
1040	class IndexCallsiteContextGraph
1041	: public CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
1042	IndexCall> {
1043	public:
1044	IndexCallsiteContextGraph(
1045	ModuleSummaryIndex &Index,
1046	llvm::function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1047	isPrevailing);
1048
1049	~IndexCallsiteContextGraph() {
1050	// Now that we are done with the graph it is safe to add the new
1051	// CallsiteInfo structs to the function summary vectors. The graph nodes
1052	// point into locations within these vectors, so we don't want to add them
1053	// any earlier.
1054	for (auto &I : FunctionCalleesToSynthesizedCallsiteInfos) {
1055	auto *FS = I.first;
1056	for (auto &Callsite : I.second)
1057	FS->addCallsite(Callsite: std::move(*Callsite.second));
1058	}
1059	}
1060
1061	private:
1062	friend CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
1063	IndexCall>;
1064
1065	uint64_t getStackId(uint64_t IdOrIndex) const;
1066	const FunctionSummary *getCalleeFunc(IndexCall &Call);
1067	bool calleeMatchesFunc(
1068	IndexCall &Call, const FunctionSummary *Func,
1069	const FunctionSummary *CallerFunc,
1070	std::vector<std::pair<IndexCall, FunctionSummary *>> &FoundCalleeChain);
1071	bool sameCallee(IndexCall &Call1, IndexCall &Call2);
1072	bool findProfiledCalleeThroughTailCalls(
1073	ValueInfo ProfiledCallee, ValueInfo CurCallee, unsigned Depth,
1074	std::vector<std::pair<IndexCall, FunctionSummary *>> &FoundCalleeChain,
1075	bool &FoundMultipleCalleeChains);
1076	uint64_t getLastStackId(IndexCall &Call);
1077	std::vector<uint64_t> getStackIdsWithContextNodesForCall(IndexCall &Call);
1078	void updateAllocationCall(CallInfo &Call, AllocationType AllocType);
1079	AllocationType getAllocationCallType(const CallInfo &Call) const;
1080	void updateCall(CallInfo &CallerCall, FuncInfo CalleeFunc);
1081	CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
1082	IndexCall>::FuncInfo
1083	cloneFunctionForCallsite(FuncInfo &Func, CallInfo &Call,
1084	DenseMap<CallInfo, CallInfo> &CallMap,
1085	std::vector<CallInfo> &CallsWithMetadataInFunc,
1086	unsigned CloneNo);
1087	std::string getLabel(const FunctionSummary Func, const* IndexCall &Call,
1088	unsigned CloneNo) const;
1089	DenseSet<GlobalValue::GUID> findAliaseeGUIDsPrevailingInDifferentModule();
1090
1091	// Saves mapping from function summaries containing memprof records back to
1092	// its VI, for use in checking and debugging.
1093	std::map<const FunctionSummary *, ValueInfo> FSToVIMap;
1094
1095	const ModuleSummaryIndex &Index;
1096	llvm::function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1097	isPrevailing;
1098
1099	// Saves/owns the callsite info structures synthesized for missing tail call
1100	// frames that we discover while building the graph.
1101	// It maps from the summary of the function making the tail call, to a map
1102	// of callee ValueInfo to corresponding synthesized callsite info.
1103	DenseMap<FunctionSummary *,
1104	std::map<ValueInfo, std::unique_ptr<CallsiteInfo>>>
1105	FunctionCalleesToSynthesizedCallsiteInfos;
1106	};
1107	} // namespace
1108
1109	template <>
1110	struct llvm::DenseMapInfo<CallsiteContextGraph<
1111	ModuleCallsiteContextGraph, Function, Instruction *>::CallInfo>
1112	: public DenseMapInfo<std::pair<Instruction , unsigned*>> {};
1113	template <>
1114	struct llvm::DenseMapInfo<CallsiteContextGraph<
1115	IndexCallsiteContextGraph, FunctionSummary, IndexCall>::CallInfo>
1116	: public DenseMapInfo<std::pair<IndexCall, unsigned>> {};
1117	template <>
1118	struct llvm::DenseMapInfo<IndexCall>
1119	: public DenseMapInfo<PointerUnion<CallsiteInfo , AllocInfo >> {};
1120
1121	namespace {
1122
1123	// Map the uint8_t alloc types (which may contain NotCold\|Cold) to the alloc
1124	// type we should actually use on the corresponding allocation.
1125	// If we can't clone a node that has NotCold+Cold alloc type, we will fall
1126	// back to using NotCold. So don't bother cloning to distinguish NotCold+Cold
1127	// from NotCold.
1128	AllocationType allocTypeToUse(uint8_t AllocTypes) {
1129	assert(AllocTypes != (uint8_t)AllocationType::None);
1130	if (AllocTypes ==
1131	((uint8_t)AllocationType::NotCold \| (uint8_t)AllocationType::Cold))
1132	return AllocationType::NotCold;
1133	else
1134	return (AllocationType)AllocTypes;
1135	}
1136
1137	// Helper to check if the alloc types for all edges recorded in the
1138	// InAllocTypes vector match the alloc types for all edges in the Edges
1139	// vector.
1140	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1141	bool allocTypesMatch(
1142	const std::vector<uint8_t> &InAllocTypes,
1143	const std::vector<std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>>>
1144	&Edges) {
1145	// This should be called only when the InAllocTypes vector was computed for
1146	// this set of Edges. Make sure the sizes are the same.
1147	assert(InAllocTypes.size() == Edges.size());
1148	return std::equal(
1149	InAllocTypes.begin(), InAllocTypes.end(), Edges.begin(), Edges.end(),
1150	[](const uint8_t &l,
1151	const std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>> &r) {
1152	// Can share if one of the edges is None type - don't
1153	// care about the type along that edge as it doesn't
1154	// exist for those context ids.
1155	if (l == (uint8_t)AllocationType::None \|\|
1156	r->AllocTypes == (uint8_t)AllocationType::None)
1157	return true;
1158	return allocTypeToUse(AllocTypes: l) == allocTypeToUse(r->AllocTypes);
1159	});
1160	}
1161
1162	// Helper to check if the alloc types for all edges recorded in the
1163	// InAllocTypes vector match the alloc types for callee edges in the given
1164	// clone. Because the InAllocTypes were computed from the original node's callee
1165	// edges, and other cloning could have happened after this clone was created, we
1166	// need to find the matching clone callee edge, which may or may not exist.
1167	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1168	bool allocTypesMatchClone(
1169	const std::vector<uint8_t> &InAllocTypes,
1170	const ContextNode<DerivedCCG, FuncTy, CallTy> *Clone) {
1171	const ContextNode<DerivedCCG, FuncTy, CallTy> *Node = Clone->CloneOf;
1172	assert(Node);
1173	// InAllocTypes should have been computed for the original node's callee
1174	// edges.
1175	assert(InAllocTypes.size() == Node->CalleeEdges.size());
1176	// First create a map of the clone callee edge callees to the edge alloc type.
1177	DenseMap<const ContextNode<DerivedCCG, FuncTy, CallTy> *, uint8_t>
1178	EdgeCalleeMap;
1179	for (const auto &E : Clone->CalleeEdges) {
1180	assert(!EdgeCalleeMap.contains(E->Callee));
1181	EdgeCalleeMap[E->Callee] = E->AllocTypes;
1182	}
1183	// Next, walk the original node's callees, and look for the corresponding
1184	// clone edge to that callee.
1185	for (unsigned I = `0`; I < Node->CalleeEdges.size(); I++) {
1186	auto Iter = EdgeCalleeMap.find(Node->CalleeEdges[I]->Callee);
1187	// Not found is ok, we will simply add an edge if we use this clone.
1188	if (Iter == EdgeCalleeMap.end())
1189	continue;
1190	// Can share if one of the edges is None type - don't
1191	// care about the type along that edge as it doesn't
1192	// exist for those context ids.
1193	if (InAllocTypes [I] == (uint8_t)AllocationType::None \|\|
1194	Iter->second == (uint8_t)AllocationType::None)
1195	continue;
1196	if (allocTypeToUse(Iter->second) != allocTypeToUse(AllocTypes: InAllocTypes [I]))
1197	return false;
1198	}
1199	return true;
1200	}
1201
1202	} // end anonymous namespace
1203
1204	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1205	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
1206	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::getNodeForInst(
1207	const CallInfo &C) {
1208	ContextNode *Node = getNodeForAlloc(C);
1209	if (Node)
1210	return Node;
1211
1212	return NonAllocationCallToContextNodeMap.lookup(C);
1213	}
1214
1215	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1216	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
1217	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::getNodeForAlloc(
1218	const CallInfo &C) {
1219	return AllocationCallToContextNodeMap.lookup(C);
1220	}
1221
1222	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1223	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
1224	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::getNodeForStackId(
1225	uint64_t StackId) {
1226	auto StackEntryNode = StackEntryIdToContextNodeMap.find(StackId);
1227	if (StackEntryNode != StackEntryIdToContextNodeMap.end())
1228	return StackEntryNode->second;
1229	return nullptr;
1230	}
1231
1232	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1233	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1234	addOrUpdateCallerEdge(ContextNode *Caller, AllocationType AllocType,
1235	unsigned int ContextId) {
1236	for (auto &Edge : CallerEdges) {
1237	if (Edge->Caller == Caller) {
1238	Edge->AllocTypes \|= (uint8_t)AllocType;
1239	Edge->getContextIds().insert(ContextId);
1240	return;
1241	}
1242	}
1243	std::shared_ptr<ContextEdge> Edge = std::make_shared<ContextEdge>(
1244	this, Caller, (uint8_t)AllocType, DenseSet<uint32_t>({ContextId}));
1245	CallerEdges.push_back(Edge);
1246	Caller->CalleeEdges.push_back(Edge);
1247	}
1248
1249	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1250	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::removeEdgeFromGraph(
1251	ContextEdge Edge, EdgeIter EI, bool CalleeIter) {
1252	assert(!EI \|\| (*EI)->get() == Edge);
1253	assert(!Edge->isRemoved());
1254	// Save the Caller and Callee pointers so we can erase Edge from their edge
1255	// lists after clearing Edge below. We do the clearing first in case it is
1256	// destructed after removing from the edge lists (if those were the last
1257	// shared_ptr references to Edge).
1258	auto *Callee = Edge->Callee;
1259	auto *Caller = Edge->Caller;
1260
1261	// Make sure the edge fields are cleared out so we can properly detect
1262	// removed edges if Edge is not destructed because there is still a shared_ptr
1263	// reference.
1264	Edge->clear();
1265
1266	#ifndef NDEBUG
1267	auto CalleeCallerCount = Callee->CallerEdges.size();
1268	auto CallerCalleeCount = Caller->CalleeEdges.size();
1269	#endif
1270	if (!EI) {
1271	Callee->eraseCallerEdge(Edge);
1272	Caller->eraseCalleeEdge(Edge);
1273	} else if (CalleeIter) {
1274	Callee->eraseCallerEdge(Edge);
1275	EI = Caller->CalleeEdges.erase(EI);
1276	} else {
1277	Caller->eraseCalleeEdge(Edge);
1278	EI = Callee->CallerEdges.erase(EI);
1279	}
1280	assert(Callee->CallerEdges.size() < CalleeCallerCount);
1281	assert(Caller->CalleeEdges.size() < CallerCalleeCount);
1282	}
1283
1284	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1285	void CallsiteContextGraph<
1286	DerivedCCG, FuncTy, CallTy>::removeNoneTypeCalleeEdges(ContextNode *Node) {
1287	for (auto EI = Node->CalleeEdges.begin(); EI != Node->CalleeEdges.end();) {
1288	auto Edge = *EI;
1289	if (Edge->AllocTypes == (uint8_t)AllocationType::None) {
1290	assert(Edge->ContextIds.empty());
1291	removeEdgeFromGraph(Edge: Edge.get(), EI: &EI, /CalleeIter=/true);
1292	} else
1293	++EI;
1294	}
1295	}
1296
1297	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1298	void CallsiteContextGraph<
1299	DerivedCCG, FuncTy, CallTy>::removeNoneTypeCallerEdges(ContextNode *Node) {
1300	for (auto EI = Node->CallerEdges.begin(); EI != Node->CallerEdges.end();) {
1301	auto Edge = *EI;
1302	if (Edge->AllocTypes == (uint8_t)AllocationType::None) {
1303	assert(Edge->ContextIds.empty());
1304	Edge->Caller->eraseCalleeEdge(Edge.get());
1305	EI = Node->CallerEdges.erase(EI);
1306	} else
1307	++EI;
1308	}
1309	}
1310
1311	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1312	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge *
1313	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1314	findEdgeFromCallee(const ContextNode *Callee) {
1315	for (const auto &Edge : CalleeEdges)
1316	if (Edge->Callee == Callee)
1317	return Edge.get();
1318	return nullptr;
1319	}
1320
1321	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1322	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge *
1323	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1324	findEdgeFromCaller(const ContextNode *Caller) {
1325	for (const auto &Edge : CallerEdges)
1326	if (Edge->Caller == Caller)
1327	return Edge.get();
1328	return nullptr;
1329	}
1330
1331	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1332	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1333	eraseCalleeEdge(const ContextEdge *Edge) {
1334	auto EI = llvm::find_if(
1335	CalleeEdges, [Edge](const std::shared_ptr<ContextEdge> &CalleeEdge) {
1336	return CalleeEdge.get() == Edge;
1337	});
1338	assert(EI != CalleeEdges.end());
1339	CalleeEdges.erase(EI);
1340	}
1341
1342	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1343	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1344	eraseCallerEdge(const ContextEdge *Edge) {
1345	auto EI = llvm::find_if(
1346	CallerEdges, [Edge](const std::shared_ptr<ContextEdge> &CallerEdge) {
1347	return CallerEdge.get() == Edge;
1348	});
1349	assert(EI != CallerEdges.end());
1350	CallerEdges.erase(EI);
1351	}
1352
1353	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1354	uint8_t CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::computeAllocType(
1355	DenseSet<uint32_t> &ContextIds) const {
1356	uint8_t BothTypes =
1357	(uint8_t)AllocationType::Cold \| (uint8_t)AllocationType::NotCold;
1358	uint8_t AllocType = (uint8_t)AllocationType::None;
1359	for (auto Id : ContextIds) {
1360	AllocType \|= (uint8_t)ContextIdToAllocationType.at(Val: Id);
1361	// Bail early if alloc type reached both, no further refinement.
1362	if (AllocType == BothTypes)
1363	return AllocType;
1364	}
1365	return AllocType;
1366	}
1367
1368	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1369	uint8_t
1370	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::intersectAllocTypesImpl(
1371	const DenseSet<uint32_t> &Node1Ids,
1372	const DenseSet<uint32_t> &Node2Ids) const {
1373	uint8_t BothTypes =
1374	(uint8_t)AllocationType::Cold \| (uint8_t)AllocationType::NotCold;
1375	uint8_t AllocType = (uint8_t)AllocationType::None;
1376	for (auto Id : Node1Ids) {
1377	if (!Node2Ids.count(V: Id))
1378	continue;
1379	AllocType \|= (uint8_t)ContextIdToAllocationType.at(Val: Id);
1380	// Bail early if alloc type reached both, no further refinement.
1381	if (AllocType == BothTypes)
1382	return AllocType;
1383	}
1384	return AllocType;
1385	}
1386
1387	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1388	uint8_t CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::intersectAllocTypes(
1389	const DenseSet<uint32_t> &Node1Ids,
1390	const DenseSet<uint32_t> &Node2Ids) const {
1391	if (Node1Ids.size() < Node2Ids.size())
1392	return intersectAllocTypesImpl(Node1Ids, Node2Ids);
1393	else
1394	return intersectAllocTypesImpl(Node1Ids: Node2Ids, Node2Ids: Node1Ids);
1395	}
1396
1397	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1398	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
1399	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::addAllocNode(
1400	CallInfo Call, const FuncTy *F) {
1401	assert(!getNodeForAlloc(Call));
1402	ContextNode AllocNode = createNewNode(/IsAllocation=/*true, F, C: Call);
1403	AllocationCallToContextNodeMap[Call] = AllocNode;
1404	// Use LastContextId as a uniq id for MIB allocation nodes.
1405	AllocNode->OrigStackOrAllocId = LastContextId;
1406	// Alloc type should be updated as we add in the MIBs. We should assert
1407	// afterwards that it is not still None.
1408	AllocNode->AllocTypes = (uint8_t)AllocationType::None;
1409
1410	return AllocNode;
1411	}
1412
1413	static std::string getAllocTypeString(uint8_t AllocTypes) {
1414	if (!AllocTypes)
1415	return "None";
1416	std::string Str;
1417	if (AllocTypes & (uint8_t)AllocationType::NotCold)
1418	Str += "NotCold";
1419	if (AllocTypes & (uint8_t)AllocationType::Cold)
1420	Str += "Cold";
1421	return Str;
1422	}
1423
1424	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1425	template <class NodeT, class IteratorT>
1426	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::addStackNodesForMIB(
1427	ContextNode *AllocNode, CallStack<NodeT, IteratorT> &StackContext,
1428	CallStack<NodeT, IteratorT> &CallsiteContext, AllocationType AllocType,
1429	ArrayRef<ContextTotalSize> ContextSizeInfo,
1430	std::map<uint64_t, uint32_t> &TotalSizeToContextIdTopNCold) {
1431	// Treating the hot alloc type as NotCold before the disambiguation for "hot"
1432	// is done.
1433	if (AllocType == AllocationType::Hot)
1434	AllocType = AllocationType::NotCold;
1435
1436	ContextIdToAllocationType [++LastContextId] = AllocType;
1437
1438	bool IsImportant = false;
1439	if (!ContextSizeInfo.empty()) {
1440	auto &Entry = ContextIdToContextSizeInfos [LastContextId];
1441	// If this is a cold allocation, and we are collecting non-zero largest
1442	// contexts, see if this is a candidate.
1443	if (AllocType == AllocationType::Cold && MemProfTopNImportant > `0`) {
1444	uint64_t TotalCold = `0`;
1445	for (auto &CSI : ContextSizeInfo)
1446	TotalCold += CSI.TotalSize;
1447	// Record this context if either we haven't found the first top-n largest
1448	// yet, or if it is larger than the smallest already recorded.
1449	if (TotalSizeToContextIdTopNCold.size() < MemProfTopNImportant \|\|
1450	// Since TotalSizeToContextIdTopNCold is a std::map, it is implicitly
1451	// sorted in ascending size of its key which is the size.
1452	TotalCold > TotalSizeToContextIdTopNCold.begin()->first) {
1453	if (TotalSizeToContextIdTopNCold.size() == MemProfTopNImportant) {
1454	// Remove old one and its associated entries.
1455	auto IdToRemove = TotalSizeToContextIdTopNCold.begin()->second;
1456	TotalSizeToContextIdTopNCold.erase(
1457	position: TotalSizeToContextIdTopNCold.begin());
1458	assert(ImportantContextIdInfo.count(IdToRemove));
1459	ImportantContextIdInfo.erase(IdToRemove);
1460	}
1461	TotalSizeToContextIdTopNCold [TotalCold] = LastContextId;
1462	IsImportant = true;
1463	}
1464	}
1465	Entry.insert(position: Entry.begin(), first: ContextSizeInfo.begin(), last: ContextSizeInfo.end());
1466	}
1467
1468	// Update alloc type and context ids for this MIB.
1469	AllocNode->AllocTypes \|= (uint8_t)AllocType;
1470
1471	// Now add or update nodes for each stack id in alloc's context.
1472	// Later when processing the stack ids on non-alloc callsites we will adjust
1473	// for any inlining in the context.
1474	ContextNode *PrevNode = AllocNode;
1475	// Look for recursion (direct recursion should have been collapsed by
1476	// module summary analysis, here we should just be detecting mutual
1477	// recursion). Mark these nodes so we don't try to clone.
1478	SmallSet<uint64_t, `8`> StackIdSet;
1479	// Skip any on the allocation call (inlining).
1480	for (auto ContextIter = StackContext.beginAfterSharedPrefix(CallsiteContext);
1481	ContextIter != StackContext.end(); ++ContextIter) {
1482	auto StackId = getStackId(IdOrIndex: *ContextIter);
1483	if (IsImportant)
1484	ImportantContextIdInfo[LastContextId].StackIds.push_back(StackId);
1485	ContextNode *StackNode = getNodeForStackId(StackId);
1486	if (!StackNode) {
1487	StackNode = createNewNode(/IsAllocation=/false);
1488	StackEntryIdToContextNodeMap[StackId] = StackNode;
1489	StackNode->OrigStackOrAllocId = StackId;
1490	}
1491	// Marking a node recursive will prevent its cloning completely, even for
1492	// non-recursive contexts flowing through it.
1493	if (!AllowRecursiveCallsites) {
1494	auto Ins = StackIdSet.insert(StackId);
1495	if (!Ins.second)
1496	StackNode->Recursive = true;
1497	}
1498	StackNode->AllocTypes \|= (uint8_t)AllocType;
1499	PrevNode->addOrUpdateCallerEdge(StackNode, AllocType, LastContextId);
1500	PrevNode = StackNode;
1501	}
1502	}
1503
1504	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1505	DenseSet<uint32_t>
1506	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::duplicateContextIds(
1507	const DenseSet<uint32_t> &StackSequenceContextIds,
1508	DenseMap<uint32_t, DenseSet<uint32_t>> &OldToNewContextIds) {
1509	DenseSet<uint32_t> NewContextIds;
1510	for (auto OldId : StackSequenceContextIds) {
1511	NewContextIds.insert(V: ++LastContextId);
1512	OldToNewContextIds [OldId].insert(V: LastContextId);
1513	assert(ContextIdToAllocationType.count(OldId));
1514	// The new context has the same allocation type and size info as original.
1515	ContextIdToAllocationType [LastContextId] = ContextIdToAllocationType [OldId];
1516	auto CSI = ContextIdToContextSizeInfos.find(Val: OldId);
1517	if (CSI != ContextIdToContextSizeInfos.end())
1518	ContextIdToContextSizeInfos [LastContextId] = CSI ->second;
1519	if (DotAllocContextIds.contains(V: OldId))
1520	DotAllocContextIds.insert(V: LastContextId);
1521	}
1522	return NewContextIds;
1523	}
1524
1525	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1526	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
1527	propagateDuplicateContextIds(
1528	const DenseMap<uint32_t, DenseSet<uint32_t>> &OldToNewContextIds) {
1529	// Build a set of duplicated context ids corresponding to the input id set.
1530	auto GetNewIds = [&OldToNewContextIds](const DenseSet<uint32_t> &ContextIds) {
1531	DenseSet<uint32_t> NewIds;
1532	for (auto Id : ContextIds)
1533	if (auto NewId = OldToNewContextIds.find(Val: Id);
1534	NewId != OldToNewContextIds.end())
1535	NewIds.insert_range(R: NewId ->second);
1536	return NewIds;
1537	};
1538
1539	// Recursively update context ids sets along caller edges.
1540	auto UpdateCallers = [&](ContextNode *Node,
1541	DenseSet<const ContextEdge *> &Visited,
1542	auto &&UpdateCallers) -> void {
1543	for (const auto &Edge : Node->CallerEdges) {
1544	auto Inserted = Visited.insert(Edge.get());
1545	if (!Inserted.second)
1546	continue;
1547	ContextNode *NextNode = Edge->Caller;
1548	DenseSet<uint32_t> NewIdsToAdd = GetNewIds(Edge->getContextIds());
1549	// Only need to recursively iterate to NextNode via this caller edge if
1550	// it resulted in any added ids to NextNode.
1551	if (!NewIdsToAdd.empty()) {
1552	Edge->getContextIds().insert_range(NewIdsToAdd);
1553	UpdateCallers(NextNode, Visited, UpdateCallers);
1554	}
1555	}
1556	};
1557
1558	DenseSet<const ContextEdge *> Visited;
1559	for (auto &Entry : AllocationCallToContextNodeMap) {
1560	auto *Node = Entry.second;
1561	UpdateCallers(Node, Visited, UpdateCallers);
1562	}
1563	}
1564
1565	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1566	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::connectNewNode(
1567	ContextNode NewNode, ContextNode OrigNode, bool TowardsCallee,
1568	const DenseSet<uint32_t> &ContextIds) {
1569	auto &OrigEdges =
1570	TowardsCallee ? OrigNode->CalleeEdges : OrigNode->CallerEdges;
1571	// Increment iterator in loop so that we can remove edges as needed.
1572	for (auto EI = OrigEdges.begin(); EI != OrigEdges.end();) {
1573	auto Edge = *EI;
1574	DenseSet<uint32_t> NewEdgeContextIds;
1575	// Remove any matching context ids from Edge, return set that were found and
1576	// removed, these are the new edge's context ids.
1577	set_subtract(Edge->getContextIds(), ContextIds, NewEdgeContextIds);
1578	// If no matching context ids for this edge, skip it.
1579	if (NewEdgeContextIds.empty()) {
1580	++EI;
1581	continue;
1582	}
1583	if (TowardsCallee) {
1584	uint8_t NewAllocType = computeAllocType(ContextIds&: NewEdgeContextIds);
1585	auto NewEdge = std::make_shared<ContextEdge>(
1586	Edge->Callee, NewNode, NewAllocType, std::move(NewEdgeContextIds));
1587	NewNode->CalleeEdges.push_back(NewEdge);
1588	NewEdge->Callee->CallerEdges.push_back(NewEdge);
1589	} else {
1590	uint8_t NewAllocType = computeAllocType(ContextIds&: NewEdgeContextIds);
1591	auto NewEdge = std::make_shared<ContextEdge>(
1592	NewNode, Edge->Caller, NewAllocType, std::move(NewEdgeContextIds));
1593	NewNode->CallerEdges.push_back(NewEdge);
1594	NewEdge->Caller->CalleeEdges.push_back(NewEdge);
1595	}
1596	// Remove old edge if context ids empty.
1597	if (Edge->getContextIds().empty()) {
1598	removeEdgeFromGraph(Edge: Edge.get(), EI: &EI, CalleeIter: TowardsCallee);
1599	continue;
1600	}
1601	++EI;
1602	}
1603	}
1604
1605	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1606	static void checkEdge(
1607	const std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>> &Edge) {
1608	// Confirm that alloc type is not None and that we have at least one context
1609	// id.
1610	assert(Edge->AllocTypes != (uint8_t)AllocationType::None);
1611	assert(!Edge->ContextIds.empty());
1612	}
1613
1614	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1615	static void checkNode(const ContextNode<DerivedCCG, FuncTy, CallTy> *Node,
1616	bool CheckEdges = true) {
1617	if (Node->isRemoved())
1618	return;
1619	#ifndef NDEBUG
1620	// Compute node's context ids once for use in asserts.
1621	auto NodeContextIds = Node->getContextIds();
1622	#endif
1623	// Node's context ids should be the union of both its callee and caller edge
1624	// context ids.
1625	if (Node->CallerEdges.size()) {
1626	DenseSet<uint32_t> CallerEdgeContextIds(
1627	Node->CallerEdges.front()->ContextIds);
1628	for (const auto &Edge : llvm::drop_begin(Node->CallerEdges)) {
1629	if (CheckEdges)
1630	checkEdge<DerivedCCG, FuncTy, CallTy>(Edge);
1631	set_union(CallerEdgeContextIds, Edge->ContextIds);
1632	}
1633	// Node can have more context ids than callers if some contexts terminate at
1634	// node and some are longer. If we are allowing recursive callsites and
1635	// contexts this will be violated for incompletely cloned recursive cycles,
1636	// so skip the checking in that case.
1637	assert((AllowRecursiveCallsites && AllowRecursiveContexts) \|\|
1638	NodeContextIds == CallerEdgeContextIds \|\|
1639	set_is_subset(CallerEdgeContextIds, NodeContextIds));
1640	}
1641	if (Node->CalleeEdges.size()) {
1642	DenseSet<uint32_t> CalleeEdgeContextIds(
1643	Node->CalleeEdges.front()->ContextIds);
1644	for (const auto &Edge : llvm::drop_begin(Node->CalleeEdges)) {
1645	if (CheckEdges)
1646	checkEdge<DerivedCCG, FuncTy, CallTy>(Edge);
1647	set_union(CalleeEdgeContextIds, Edge->getContextIds());
1648	}
1649	// If we are allowing recursive callsites and contexts this will be violated
1650	// for incompletely cloned recursive cycles, so skip the checking in that
1651	// case.
1652	assert((AllowRecursiveCallsites && AllowRecursiveContexts) \|\|
1653	NodeContextIds == CalleeEdgeContextIds);
1654	}
1655	// FIXME: Since this checking is only invoked under an option, we should
1656	// change the error checking from using assert to something that will trigger
1657	// an error on a release build.
1658	#ifndef NDEBUG
1659	// Make sure we don't end up with duplicate edges between the same caller and
1660	// callee.
1661	DenseSet<ContextNode<DerivedCCG, FuncTy, CallTy> *> NodeSet;
1662	for (const auto &E : Node->CalleeEdges)
1663	NodeSet.insert(E->Callee);
1664	assert(NodeSet.size() == Node->CalleeEdges.size());
1665	#endif
1666	}
1667
1668	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1669	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
1670	assignStackNodesPostOrder(ContextNode *Node,
1671	DenseSet<const ContextNode *> &Visited,
1672	DenseMap<uint64_t, std::vector<CallContextInfo>>
1673	&StackIdToMatchingCalls,
1674	DenseMap<CallInfo, CallInfo> &CallToMatchingCall,
1675	const DenseSet<uint32_t> &ImportantContextIds) {
1676	auto Inserted = Visited.insert(Node);
1677	if (!Inserted.second)
1678	return;
1679	// Post order traversal. Iterate over a copy since we may add nodes and
1680	// therefore new callers during the recursive call, invalidating any
1681	// iterator over the original edge vector. We don't need to process these
1682	// new nodes as they were already processed on creation.
1683	auto CallerEdges = Node->CallerEdges;
1684	for (auto &Edge : CallerEdges) {
1685	// Skip any that have been removed during the recursion.
1686	if (Edge->isRemoved()) {
1687	assert(!is_contained(Node->CallerEdges, Edge));
1688	continue;
1689	}
1690	assignStackNodesPostOrder(Node: Edge->Caller, Visited, StackIdToMatchingCalls,
1691	CallToMatchingCall, ImportantContextIds);
1692	}
1693
1694	// If this node's stack id is in the map, update the graph to contain new
1695	// nodes representing any inlining at interior callsites. Note we move the
1696	// associated context ids over to the new nodes.
1697
1698	// Ignore this node if it is for an allocation or we didn't record any
1699	// stack id lists ending at it.
1700	if (Node->IsAllocation \|\|
1701	!StackIdToMatchingCalls.count(Node->OrigStackOrAllocId))
1702	return;
1703
1704	auto &Calls = StackIdToMatchingCalls[Node->OrigStackOrAllocId];
1705	// Handle the simple case first. A single call with a single stack id.
1706	// In this case there is no need to create any new context nodes, simply
1707	// assign the context node for stack id to this Call.
1708	if (Calls.size() == `1`) {
1709	auto &[Call, Ids, Func, SavedContextIds] = Calls[`0`];
1710	if (Ids.size() == `1`) {
1711	assert(SavedContextIds.empty());
1712	// It should be this Node
1713	assert(Node == getNodeForStackId(Ids[`0`]));
1714	if (Node->Recursive)
1715	return;
1716	Node->setCall(Call);
1717	NonAllocationCallToContextNodeMap[Call] = Node;
1718	NodeToCallingFunc[Node] = Func;
1719	recordStackNode(StackIds&: Ids, Node, NodeContextIds: Node->getContextIds(), ImportantContextIds);
1720	return;
1721	}
1722	}
1723
1724	#ifndef NDEBUG
1725	// Find the node for the last stack id, which should be the same
1726	// across all calls recorded for this id, and is this node's id.
1727	uint64_t LastId = Node->OrigStackOrAllocId;
1728	ContextNode *LastNode = getNodeForStackId(LastId);
1729	// We should only have kept stack ids that had nodes.
1730	assert(LastNode);
1731	assert(LastNode == Node);
1732	#else
1733	ContextNode *LastNode = Node;
1734	#endif
1735
1736	// Compute the last node's context ids once, as it is shared by all calls in
1737	// this entry.
1738	DenseSet<uint32_t> LastNodeContextIds = LastNode->getContextIds();
1739
1740	[[maybe_unused]] bool PrevIterCreatedNode = false;
1741	bool CreatedNode = false;
1742	for (unsigned I = `0`; I < Calls.size();
1743	I++, PrevIterCreatedNode = CreatedNode) {
1744	CreatedNode = false;
1745	auto &[Call, Ids, Func, SavedContextIds] = Calls[I];
1746	// Skip any for which we didn't assign any ids, these don't get a node in
1747	// the graph.
1748	if (SavedContextIds.empty()) {
1749	// If this call has a matching call (located in the same function and
1750	// having the same stack ids), simply add it to the context node created
1751	// for its matching call earlier. These can be treated the same through
1752	// cloning and get updated at the same time.
1753	if (!CallToMatchingCall.contains(Call))
1754	continue;
1755	auto MatchingCall = CallToMatchingCall[Call];
1756	if (!NonAllocationCallToContextNodeMap.contains(MatchingCall)) {
1757	// This should only happen if we had a prior iteration, and it didn't
1758	// create a node because of the below recomputation of context ids
1759	// finding none remaining and continuing early.
1760	assert(I > `0` && !PrevIterCreatedNode);
1761	continue;
1762	}
1763	NonAllocationCallToContextNodeMap[MatchingCall]->MatchingCalls.push_back(
1764	Call);
1765	continue;
1766	}
1767
1768	assert(LastId == Ids.back());
1769
1770	// Recompute the context ids for this stack id sequence (the
1771	// intersection of the context ids of the corresponding nodes).
1772	// Start with the ids we saved in the map for this call, which could be
1773	// duplicated context ids. We have to recompute as we might have overlap
1774	// overlap between the saved context ids for different last nodes, and
1775	// removed them already during the post order traversal.
1776	set_intersect(SavedContextIds, LastNodeContextIds);
1777	ContextNode *PrevNode = LastNode;
1778	bool Skip = false;
1779	// Iterate backwards through the stack Ids, starting after the last Id
1780	// in the list, which was handled once outside for all Calls.
1781	for (auto IdIter = Ids.rbegin() + `1`; IdIter != Ids.rend(); IdIter++) {
1782	auto Id = *IdIter;
1783	ContextNode *CurNode = getNodeForStackId(StackId: Id);
1784	// We should only have kept stack ids that had nodes and weren't
1785	// recursive.
1786	assert(CurNode);
1787	assert(!CurNode->Recursive);
1788
1789	auto *Edge = CurNode->findEdgeFromCaller(PrevNode);
1790	if (!Edge) {
1791	Skip = true;
1792	break;
1793	}
1794	PrevNode = CurNode;
1795
1796	// Update the context ids, which is the intersection of the ids along
1797	// all edges in the sequence.
1798	set_intersect(SavedContextIds, Edge->getContextIds());
1799
1800	// If we now have no context ids for clone, skip this call.
1801	if (SavedContextIds.empty()) {
1802	Skip = true;
1803	break;
1804	}
1805	}
1806	if (Skip)
1807	continue;
1808
1809	// Create new context node.
1810	ContextNode NewNode = createNewNode(/IsAllocation=/*false, F: Func, C: Call);
1811	NonAllocationCallToContextNodeMap[Call] = NewNode;
1812	CreatedNode = true;
1813	NewNode->AllocTypes = computeAllocType(ContextIds&: SavedContextIds);
1814
1815	ContextNode *FirstNode = getNodeForStackId(StackId: Ids[`0`]);
1816	assert(FirstNode);
1817
1818	// Connect to callees of innermost stack frame in inlined call chain.
1819	// This updates context ids for FirstNode's callee's to reflect those
1820	// moved to NewNode.
1821	connectNewNode(NewNode, OrigNode: FirstNode, /TowardsCallee=/true, ContextIds: SavedContextIds);
1822
1823	// Connect to callers of outermost stack frame in inlined call chain.
1824	// This updates context ids for FirstNode's caller's to reflect those
1825	// moved to NewNode.
1826	connectNewNode(NewNode, OrigNode: LastNode, /TowardsCallee=/false, ContextIds: SavedContextIds);
1827
1828	// Now we need to remove context ids from edges/nodes between First and
1829	// Last Node.
1830	PrevNode = nullptr;
1831	for (auto Id : Ids) {
1832	ContextNode *CurNode = getNodeForStackId(StackId: Id);
1833	// We should only have kept stack ids that had nodes.
1834	assert(CurNode);
1835
1836	// Remove the context ids moved to NewNode from CurNode, and the
1837	// edge from the prior node.
1838	if (PrevNode) {
1839	auto *PrevEdge = CurNode->findEdgeFromCallee(PrevNode);
1840	// If the sequence contained recursion, we might have already removed
1841	// some edges during the connectNewNode calls above.
1842	if (!PrevEdge) {
1843	PrevNode = CurNode;
1844	continue;
1845	}
1846	set_subtract(PrevEdge->getContextIds(), SavedContextIds);
1847	if (PrevEdge->getContextIds().empty())
1848	removeEdgeFromGraph(Edge: PrevEdge);
1849	}
1850	// Since we update the edges from leaf to tail, only look at the callee
1851	// edges. This isn't an alloc node, so if there are no callee edges, the
1852	// alloc type is None.
1853	CurNode->AllocTypes = CurNode->CalleeEdges.empty()
1854	? (uint8_t)AllocationType::None
1855	: CurNode->computeAllocType();
1856	PrevNode = CurNode;
1857	}
1858
1859	recordStackNode(StackIds&: Ids, Node: NewNode, NodeContextIds: SavedContextIds, ImportantContextIds);
1860
1861	if (VerifyNodes) {
1862	checkNode<DerivedCCG, FuncTy, CallTy>(NewNode, /CheckEdges=/true);
1863	for (auto Id : Ids) {
1864	ContextNode *CurNode = getNodeForStackId(StackId: Id);
1865	// We should only have kept stack ids that had nodes.
1866	assert(CurNode);
1867	checkNode<DerivedCCG, FuncTy, CallTy>(CurNode, /CheckEdges=/true);
1868	}
1869	}
1870	}
1871	}
1872
1873	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1874	void CallsiteContextGraph<DerivedCCG, FuncTy,
1875	CallTy>::fixupImportantContexts() {
1876	if (ImportantContextIdInfo.empty())
1877	return;
1878
1879	// Update statistics as we are done building this map at this point.
1880	NumImportantContextIds = ImportantContextIdInfo.size();
1881
1882	if (!MemProfFixupImportant)
1883	return;
1884
1885	if (ExportToDot)
1886	exportToDot(Label: "beforestackfixup");
1887
1888	// For each context we identified as important, walk through the saved context
1889	// stack ids in order from leaf upwards, and make sure all edges are correct.
1890	// These can be difficult to get right when updating the graph while mapping
1891	// nodes onto summary or IR, especially when there is recursion. In
1892	// particular, when we have created new nodes to reflect inlining, it is
1893	// sometimes impossible to know exactly how to update the edges in the face of
1894	// recursion, as we have lost the original ordering of the stack ids in the
1895	// contexts.
1896	// TODO: Consider only doing this if we detect the context has recursive
1897	// cycles.
1898	//
1899	// I.e. assume we have a context with stack ids like: {A B A C A D E}
1900	// and let's say A was inlined into B, C, and D. The original graph will have
1901	// multiple recursive cycles through A. When we match the original context
1902	// nodes onto the IR or summary, we will merge {A B} into one context node,
1903	// {A C} onto another, and {A D} onto another. Looking at the stack sequence
1904	// above, we should end up with a non-cyclic set of edges like:
1905	// {AB} <- {AC} <- {AD} <- E. However, because we normally have lost the
1906	// original ordering, we won't get the edges correct initially (it's
1907	// impossible without the original ordering). Here we do the fixup (add and
1908	// removing edges where necessary) for this context. In the
1909	// ImportantContextInfo struct in this case we should have a MaxLength = 2,
1910	// and map entries for {A B}, {A C}, {A D}, and {E}.
1911	for (auto &[CurContextId, Info] : ImportantContextIdInfo) {
1912	if (Info.StackIdsToNode.empty())
1913	continue;
1914	bool Changed = false;
1915	ContextNode PrevNode = nullptr*;
1916	ContextNode CurNode = nullptr*;
1917	DenseSet<const ContextEdge *> VisitedEdges;
1918	ArrayRef<uint64_t> AllStackIds(Info.StackIds);
1919	// Try to identify what callsite ContextNode maps to which slice of the
1920	// context's ordered stack ids.
1921	for (unsigned I = `0`; I < AllStackIds.size(); I++, PrevNode = CurNode) {
1922	// We will do this greedily, trying up to MaxLength stack ids in a row, to
1923	// see if we recorded a context node for that sequence.
1924	auto Len = Info.MaxLength;
1925	auto LenToEnd = AllStackIds.size() - I;
1926	if (Len > LenToEnd)
1927	Len = LenToEnd;
1928	CurNode = nullptr;
1929	// Try to find a recorded context node starting with the longest length
1930	// recorded, and on down until we check for just a single stack node.
1931	for (; Len > `0`; Len--) {
1932	// Get the slice of the original stack id sequence to check.
1933	auto CheckStackIds = AllStackIds.slice(I, Len);
1934	auto EntryIt = Info.StackIdsToNode.find(CheckStackIds);
1935	if (EntryIt == Info.StackIdsToNode.end())
1936	continue;
1937	CurNode = EntryIt->second;
1938	// Skip forward so we don't try to look for the ones we just matched.
1939	// We increment by Len - 1, because the outer for loop will increment I.
1940	I += Len - `1`;
1941	break;
1942	}
1943	// Give up if we couldn't find a node. Since we need to clone from the
1944	// leaf allocation upwards, no sense in doing anymore fixup further up
1945	// the context if we couldn't match part of the original stack context
1946	// onto a callsite node.
1947	if (!CurNode)
1948	break;
1949	// No edges to fix up until we have a pair of nodes that should be
1950	// adjacent in the graph.
1951	if (!PrevNode)
1952	continue;
1953	// See if we already have a call edge from CurNode to PrevNode.
1954	auto *CurEdge = PrevNode->findEdgeFromCaller(CurNode);
1955	if (CurEdge) {
1956	// We already have an edge. Make sure it contains this context id.
1957	if (CurEdge->getContextIds().insert(CurContextId).second) {
1958	NumFixupEdgeIdsInserted ++;
1959	Changed = true;
1960	}
1961	} else {
1962	// No edge exists - add one.
1963	NumFixupEdgesAdded ++;
1964	DenseSet<uint32_t> ContextIds({CurContextId});
1965	auto AllocType = computeAllocType(ContextIds);
1966	auto NewEdge = std::make_shared<ContextEdge>(
1967	PrevNode, CurNode, AllocType, std::move(ContextIds));
1968	PrevNode->CallerEdges.push_back(NewEdge);
1969	CurNode->CalleeEdges.push_back(NewEdge);
1970	// Save the new edge for the below handling.
1971	CurEdge = NewEdge.get();
1972	Changed = true;
1973	}
1974	VisitedEdges.insert(CurEdge);
1975	// Now remove this context id from any other caller edges calling
1976	// PrevNode.
1977	for (auto &Edge : PrevNode->CallerEdges) {
1978	// Skip the edge updating/created above and edges we have already
1979	// visited (due to recursion).
1980	if (Edge.get() != CurEdge && !VisitedEdges.contains(Edge.get()))
1981	Edge->getContextIds().erase(CurContextId);
1982	}
1983	}
1984	if (Changed)
1985	NumFixedContexts ++;
1986	}
1987	}
1988
1989	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1990	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::updateStackNodes() {
1991	// Map of stack id to all calls with that as the last (outermost caller)
1992	// callsite id that has a context node (some might not due to pruning
1993	// performed during matching of the allocation profile contexts).
1994	// The CallContextInfo contains the Call and a list of its stack ids with
1995	// ContextNodes, the function containing Call, and the set of context ids
1996	// the analysis will eventually identify for use in any new node created
1997	// for that callsite.
1998	DenseMap<uint64_t, std::vector<CallContextInfo>> StackIdToMatchingCalls;
1999	for (auto &[Func, CallsWithMetadata] : FuncToCallsWithMetadata) {
2000	for (auto &Call : CallsWithMetadata) {
2001	// Ignore allocations, already handled.
2002	if (AllocationCallToContextNodeMap.count(Call))
2003	continue;
2004	auto StackIdsWithContextNodes =
2005	getStackIdsWithContextNodesForCall(Call: Call.call());
2006	// If there were no nodes created for MIBs on allocs (maybe this was in
2007	// the unambiguous part of the MIB stack that was pruned), ignore.
2008	if (StackIdsWithContextNodes.empty())
2009	continue;
2010	// Otherwise, record this Call along with the list of ids for the last
2011	// (outermost caller) stack id with a node.
2012	StackIdToMatchingCalls[StackIdsWithContextNodes.back()].push_back(
2013	{Call.call(), StackIdsWithContextNodes, Func, {}});
2014	}
2015	}
2016
2017	// First make a pass through all stack ids that correspond to a call,
2018	// as identified in the above loop. Compute the context ids corresponding to
2019	// each of these calls when they correspond to multiple stack ids due to
2020	// due to inlining. Perform any duplication of context ids required when
2021	// there is more than one call with the same stack ids. Their (possibly newly
2022	// duplicated) context ids are saved in the StackIdToMatchingCalls map.
2023	DenseMap<uint32_t, DenseSet<uint32_t>> OldToNewContextIds;
2024	// Save a map from each call to any that are found to match it. I.e. located
2025	// in the same function and have the same (possibly pruned) stack ids. We use
2026	// this to avoid creating extra graph nodes as they can be treated the same.
2027	DenseMap<CallInfo, CallInfo> CallToMatchingCall;
2028	for (auto &It : StackIdToMatchingCalls) {
2029	auto &Calls = It.getSecond();
2030	// Skip single calls with a single stack id. These don't need a new node.
2031	if (Calls.size() == `1`) {
2032	auto &Ids = Calls[`0`].StackIds;
2033	if (Ids.size() == `1`)
2034	continue;
2035	}
2036	// In order to do the best and maximal matching of inlined calls to context
2037	// node sequences we will sort the vectors of stack ids in descending order
2038	// of length, and within each length, lexicographically by stack id. The
2039	// latter is so that we can specially handle calls that have identical stack
2040	// id sequences (either due to cloning or artificially because of the MIB
2041	// context pruning). Those with the same Ids are then sorted by function to
2042	// facilitate efficiently mapping them to the same context node.
2043	// Because the functions are pointers, to ensure a stable sort first assign
2044	// each function pointer to its first index in the Calls array, and then use
2045	// that to sort by.
2046	DenseMap<const FuncTy , unsigned*> FuncToIndex;
2047	for (const auto &[Idx, CallCtxInfo] : enumerate(Calls))
2048	FuncToIndex.insert({CallCtxInfo.Func, Idx});
2049	llvm::stable_sort(
2050	Calls,
2051	[&FuncToIndex](const CallContextInfo &A, const CallContextInfo &B) {
2052	return A.StackIds.size() > B.StackIds.size() \|\|
2053	(A.StackIds.size() == B.StackIds.size() &&
2054	(A.StackIds < B.StackIds \|\|
2055	(A.StackIds == B.StackIds &&
2056	FuncToIndex[A.Func] < FuncToIndex[B.Func])));
2057	});
2058
2059	// Find the node for the last stack id, which should be the same
2060	// across all calls recorded for this id, and is the id for this
2061	// entry in the StackIdToMatchingCalls map.
2062	uint64_t LastId = It.getFirst();
2063	ContextNode *LastNode = getNodeForStackId(StackId: LastId);
2064	// We should only have kept stack ids that had nodes.
2065	assert(LastNode);
2066
2067	if (LastNode->Recursive)
2068	continue;
2069
2070	// Initialize the context ids with the last node's. We will subsequently
2071	// refine the context ids by computing the intersection along all edges.
2072	DenseSet<uint32_t> LastNodeContextIds = LastNode->getContextIds();
2073	assert(!LastNodeContextIds.empty());
2074
2075	#ifndef NDEBUG
2076	// Save the set of functions seen for a particular set of the same stack
2077	// ids. This is used to ensure that they have been correctly sorted to be
2078	// adjacent in the Calls list, since we rely on that to efficiently place
2079	// all such matching calls onto the same context node.
2080	DenseSet<const FuncTy *> MatchingIdsFuncSet;
2081	#endif
2082
2083	for (unsigned I = `0`; I < Calls.size(); I++) {
2084	auto &[Call, Ids, Func, SavedContextIds] = Calls[I];
2085	assert(SavedContextIds.empty());
2086	assert(LastId == Ids.back());
2087
2088	#ifndef NDEBUG
2089	// If this call has a different set of ids than the last one, clear the
2090	// set used to ensure they are sorted properly.
2091	if (I > `0` && Ids != Calls[I - `1`].StackIds)
2092	MatchingIdsFuncSet.clear();
2093	#endif
2094
2095	// First compute the context ids for this stack id sequence (the
2096	// intersection of the context ids of the corresponding nodes).
2097	// Start with the remaining saved ids for the last node.
2098	assert(!LastNodeContextIds.empty());
2099	DenseSet<uint32_t> StackSequenceContextIds = LastNodeContextIds;
2100
2101	ContextNode *PrevNode = LastNode;
2102	ContextNode *CurNode = LastNode;
2103	bool Skip = false;
2104
2105	// Iterate backwards through the stack Ids, starting after the last Id
2106	// in the list, which was handled once outside for all Calls.
2107	for (auto IdIter = Ids.rbegin() + `1`; IdIter != Ids.rend(); IdIter++) {
2108	auto Id = *IdIter;
2109	CurNode = getNodeForStackId(StackId: Id);
2110	// We should only have kept stack ids that had nodes.
2111	assert(CurNode);
2112
2113	if (CurNode->Recursive) {
2114	Skip = true;
2115	break;
2116	}
2117
2118	auto *Edge = CurNode->findEdgeFromCaller(PrevNode);
2119	// If there is no edge then the nodes belong to different MIB contexts,
2120	// and we should skip this inlined context sequence. For example, this
2121	// particular inlined context may include stack ids A->B, and we may
2122	// indeed have nodes for both A and B, but it is possible that they were
2123	// never profiled in sequence in a single MIB for any allocation (i.e.
2124	// we might have profiled an allocation that involves the callsite A,
2125	// but through a different one of its callee callsites, and we might
2126	// have profiled an allocation that involves callsite B, but reached
2127	// from a different caller callsite).
2128	if (!Edge) {
2129	Skip = true;
2130	break;
2131	}
2132	PrevNode = CurNode;
2133
2134	// Update the context ids, which is the intersection of the ids along
2135	// all edges in the sequence.
2136	set_intersect(StackSequenceContextIds, Edge->getContextIds());
2137
2138	// If we now have no context ids for clone, skip this call.
2139	if (StackSequenceContextIds.empty()) {
2140	Skip = true;
2141	break;
2142	}
2143	}
2144	if (Skip)
2145	continue;
2146
2147	// If some of this call's stack ids did not have corresponding nodes (due
2148	// to pruning), don't include any context ids for contexts that extend
2149	// beyond these nodes. Otherwise we would be matching part of unrelated /
2150	// not fully matching stack contexts. To do this, subtract any context ids
2151	// found in caller nodes of the last node found above.
2152	if (Ids.back() != getLastStackId(Call)) {
2153	for (const auto &PE : LastNode->CallerEdges) {
2154	set_subtract(StackSequenceContextIds, PE->getContextIds());
2155	if (StackSequenceContextIds.empty())
2156	break;
2157	}
2158	// If we now have no context ids for clone, skip this call.
2159	if (StackSequenceContextIds.empty())
2160	continue;
2161	}
2162
2163	#ifndef NDEBUG
2164	// If the prior call had the same stack ids this set would not be empty.
2165	// Check if we already have a call that "matches" because it is located
2166	// in the same function. If the Calls list was sorted properly we should
2167	// not encounter this situation as all such entries should be adjacent
2168	// and processed in bulk further below.
2169	assert(!MatchingIdsFuncSet.contains(Func));
2170
2171	MatchingIdsFuncSet.insert(Func);
2172	#endif
2173
2174	// Check if the next set of stack ids is the same (since the Calls vector
2175	// of tuples is sorted by the stack ids we can just look at the next one).
2176	// If so, save them in the CallToMatchingCall map so that they get
2177	// assigned to the same context node, and skip them.
2178	bool DuplicateContextIds = false;
2179	for (unsigned J = I + `1`; J < Calls.size(); J++) {
2180	auto &CallCtxInfo = Calls[J];
2181	auto &NextIds = CallCtxInfo.StackIds;
2182	if (NextIds != Ids)
2183	break;
2184	auto *NextFunc = CallCtxInfo.Func;
2185	if (NextFunc != Func) {
2186	// We have another Call with the same ids but that cannot share this
2187	// node, must duplicate ids for it.
2188	DuplicateContextIds = true;
2189	break;
2190	}
2191	auto &NextCall = CallCtxInfo.Call;
2192	CallToMatchingCall[NextCall] = Call;
2193	// Update I so that it gets incremented correctly to skip this call.
2194	I = J;
2195	}
2196
2197	// If we don't have duplicate context ids, then we can assign all the
2198	// context ids computed for the original node sequence to this call.
2199	// If there are duplicate calls with the same stack ids then we synthesize
2200	// new context ids that are duplicates of the originals. These are
2201	// assigned to SavedContextIds, which is a reference into the map entry
2202	// for this call, allowing us to access these ids later on.
2203	OldToNewContextIds.reserve(NumEntries: OldToNewContextIds.size() +
2204	StackSequenceContextIds.size());
2205	SavedContextIds =
2206	DuplicateContextIds
2207	? duplicateContextIds(StackSequenceContextIds, OldToNewContextIds)
2208	: StackSequenceContextIds;
2209	assert(!SavedContextIds.empty());
2210
2211	if (!DuplicateContextIds) {
2212	// Update saved last node's context ids to remove those that are
2213	// assigned to other calls, so that it is ready for the next call at
2214	// this stack id.
2215	set_subtract(S1&: LastNodeContextIds, S2: StackSequenceContextIds);
2216	if (LastNodeContextIds.empty())
2217	break;
2218	}
2219	}
2220	}
2221
2222	// Propagate the duplicate context ids over the graph.
2223	propagateDuplicateContextIds(OldToNewContextIds);
2224
2225	if (VerifyCCG)
2226	check();
2227
2228	// Now perform a post-order traversal over the graph, starting with the
2229	// allocation nodes, essentially processing nodes from callers to callees.
2230	// For any that contains an id in the map, update the graph to contain new
2231	// nodes representing any inlining at interior callsites. Note we move the
2232	// associated context ids over to the new nodes.
2233	DenseSet<const ContextNode *> Visited;
2234	DenseSet<uint32_t> ImportantContextIds(llvm::from_range,
2235	ImportantContextIdInfo.keys());
2236	for (auto &Entry : AllocationCallToContextNodeMap)
2237	assignStackNodesPostOrder(Node: Entry.second, Visited, StackIdToMatchingCalls,
2238	CallToMatchingCall, ImportantContextIds);
2239
2240	fixupImportantContexts();
2241
2242	if (VerifyCCG)
2243	check();
2244	}
2245
2246	uint64_t ModuleCallsiteContextGraph::getLastStackId(Instruction *Call) {
2247	CallStack<MDNode, MDNode::op_iterator> CallsiteContext(
2248	Call->getMetadata(KindID: LLVMContext::MD_callsite));
2249	return CallsiteContext.back();
2250	}
2251
2252	uint64_t IndexCallsiteContextGraph::getLastStackId(IndexCall &Call) {
2253	assert(isa<CallsiteInfo *>(Call));
2254	CallStack<CallsiteInfo, SmallVector<unsigned>::const_iterator>
2255	CallsiteContext(dyn_cast_if_present<CallsiteInfo *>(Val&: Call));
2256	// Need to convert index into stack id.
2257	return Index.getStackIdAtIndex(Index: CallsiteContext.back());
2258	}
2259
2260	static const std::string MemProfCloneSuffix = ".memprof.";
2261
2262	static std::string getMemProfFuncName(Twine Base, unsigned CloneNo) {
2263	// We use CloneNo == 0 to refer to the original version, which doesn't get
2264	// renamed with a suffix.
2265	if (!CloneNo)
2266	return Base.str();
2267	return (Base + MemProfCloneSuffix + Twine (CloneNo)).str();
2268	}
2269
2270	static bool isMemProfClone(const Function &F) {
2271	return F.getName().contains(Other: MemProfCloneSuffix);
2272	}
2273
2274	// Return the clone number of the given function by extracting it from the
2275	// memprof suffix. Assumes the caller has already confirmed it is a memprof
2276	// clone.
2277	static unsigned getMemProfCloneNum(const Function &F) {
2278	assert(isMemProfClone(F));
2279	auto Pos = F.getName().find_last_of(C: `'.'`);
2280	assert(Pos > `0`);
2281	unsigned CloneNo;
2282	bool Err = F.getName().drop_front(N: Pos + `1`).getAsInteger(Radix: `10`, Result&: CloneNo);
2283	assert(!Err);
2284	(void)Err;
2285	return CloneNo;
2286	}
2287
2288	std::string ModuleCallsiteContextGraph::getLabel(const Function *Func,
2289	const Instruction *Call,
2290	unsigned CloneNo) const {
2291	return (Twine (Call->getFunction()->getName()) + " -> " +
2292	cast<CallBase>(Val: Call)->getCalledFunction()->getName())
2293	.str();
2294	}
2295
2296	std::string IndexCallsiteContextGraph::getLabel(const FunctionSummary *Func,
2297	const IndexCall &Call,
2298	unsigned CloneNo) const {
2299	auto VI = FSToVIMap.find(x: Func);
2300	assert(VI != FSToVIMap.end());
2301	std::string CallerName = getMemProfFuncName(Base: VI ->second.name(), CloneNo);
2302	if (isa<AllocInfo *>(Val: Call))
2303	return CallerName + " -> alloc";
2304	else {
2305	auto Callsite = dyn_cast_if_present<CallsiteInfo >(Val: Call);
2306	return CallerName + " -> " +
2307	getMemProfFuncName(Base: Callsite->Callee.name(),
2308	CloneNo: Callsite->Clones [CloneNo]);
2309	}
2310	}
2311
2312	std::vector<uint64_t>
2313	ModuleCallsiteContextGraph::getStackIdsWithContextNodesForCall(
2314	Instruction *Call) {
2315	CallStack<MDNode, MDNode::op_iterator> CallsiteContext(
2316	Call->getMetadata(KindID: LLVMContext::MD_callsite));
2317	return getStackIdsWithContextNodes<MDNode, MDNode::op_iterator>(
2318	CallsiteContext);
2319	}
2320
2321	std::vector<uint64_t>
2322	IndexCallsiteContextGraph::getStackIdsWithContextNodesForCall(IndexCall &Call) {
2323	assert(isa<CallsiteInfo *>(Call));
2324	CallStack<CallsiteInfo, SmallVector<unsigned>::const_iterator>
2325	CallsiteContext(dyn_cast_if_present<CallsiteInfo *>(Val&: Call));
2326	return getStackIdsWithContextNodes<CallsiteInfo,
2327	SmallVector<unsigned>::const_iterator>(
2328	CallsiteContext);
2329	}
2330
2331	template <typename DerivedCCG, typename FuncTy, typename CallTy>
2332	template <class NodeT, class IteratorT>
2333	std::vector<uint64_t>
2334	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::getStackIdsWithContextNodes(
2335	CallStack<NodeT, IteratorT> &CallsiteContext) {
2336	std::vector<uint64_t> StackIds;
2337	for (auto IdOrIndex : CallsiteContext) {
2338	auto StackId = getStackId(IdOrIndex);
2339	ContextNode *Node = getNodeForStackId(StackId);
2340	if (!Node)
2341	break;
2342	StackIds.push_back(StackId);
2343	}
2344	return StackIds;
2345	}
2346
2347	ModuleCallsiteContextGraph::ModuleCallsiteContextGraph(
2348	Module &M,
2349	llvm::function_ref<OptimizationRemarkEmitter &(Function *)> OREGetter)
2350	: Mod(M), OREGetter (OREGetter) {
2351	// Map for keeping track of the largest cold contexts up to the number given
2352	// by MemProfTopNImportant. Must be a std::map (not DenseMap) because keys
2353	// must be sorted.
2354	std::map<uint64_t, uint32_t> TotalSizeToContextIdTopNCold;
2355	for (auto &F : M) {
2356	std::vector<CallInfo> CallsWithMetadata;
2357	for (auto &BB : F) {
2358	for (auto &I : BB) {
2359	if (!isa<CallBase>(Val: I))
2360	continue;
2361	if (auto *MemProfMD = I.getMetadata(KindID: LLVMContext::MD_memprof)) {
2362	CallsWithMetadata.push_back(x: &I);
2363	auto *AllocNode = addAllocNode(Call: &I, F: &F);
2364	auto *CallsiteMD = I.getMetadata(KindID: LLVMContext::MD_callsite);
2365	assert(CallsiteMD);
2366	CallStack<MDNode, MDNode::op_iterator> CallsiteContext(CallsiteMD);
2367	// Add all of the MIBs and their stack nodes.
2368	for (auto &MDOp : MemProfMD->operands()) {
2369	auto MIBMD = cast<const* MDNode>(Val: MDOp);
2370	std::vector<ContextTotalSize> ContextSizeInfo;
2371	// Collect the context size information if it exists.
2372	if (MIBMD->getNumOperands() > `2`) {
2373	for (unsigned I = `2`; I < MIBMD->getNumOperands(); I++) {
2374	MDNode *ContextSizePair =
2375	dyn_cast<MDNode>(Val: MIBMD->getOperand(I));
2376	assert(ContextSizePair->getNumOperands() == `2`);
2377	uint64_t FullStackId = mdconst::dyn_extract<ConstantInt>(
2378	MD: ContextSizePair->getOperand(I: `0`))
2379	->getZExtValue();
2380	uint64_t TotalSize = mdconst::dyn_extract<ConstantInt>(
2381	MD: ContextSizePair->getOperand(I: `1`))
2382	->getZExtValue();
2383	ContextSizeInfo.push_back(x: {.FullStackId: FullStackId, .TotalSize: TotalSize});
2384	}
2385	}
2386	MDNode *StackNode = getMIBStackNode(MIB: MIBMD);
2387	assert(StackNode);
2388	CallStack<MDNode, MDNode::op_iterator> StackContext(StackNode);
2389	addStackNodesForMIB<MDNode, MDNode::op_iterator>(
2390	AllocNode, StackContext, CallsiteContext,
2391	AllocType: getMIBAllocType(MIB: MIBMD), ContextSizeInfo,
2392	TotalSizeToContextIdTopNCold);
2393	}
2394	// If exporting the graph to dot and an allocation id of interest was
2395	// specified, record all the context ids for this allocation node.
2396	if (ExportToDot && AllocNode->OrigStackOrAllocId == AllocIdForDot)
2397	DotAllocContextIds = AllocNode->getContextIds();
2398	assert(AllocNode->AllocTypes != (uint8_t)AllocationType::None);
2399	// Memprof and callsite metadata on memory allocations no longer
2400	// needed.
2401	I.setMetadata(KindID: LLVMContext::MD_memprof, Node: nullptr);
2402	I.setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
2403	}
2404	// For callsite metadata, add to list for this function for later use.
2405	else if (I.getMetadata(KindID: LLVMContext::MD_callsite)) {
2406	CallsWithMetadata.push_back(x: &I);
2407	}
2408	}
2409	}
2410	if (!CallsWithMetadata.empty())
2411	FuncToCallsWithMetadata [&F] = CallsWithMetadata;
2412	}
2413
2414	if (DumpCCG) {
2415	dbgs() << "CCG before updating call stack chains:\n";
2416	dbgs() << *this;
2417	}
2418
2419	if (ExportToDot)
2420	exportToDot(Label: "prestackupdate");
2421
2422	updateStackNodes();
2423
2424	if (ExportToDot)
2425	exportToDot(Label: "poststackupdate");
2426
2427	handleCallsitesWithMultipleTargets();
2428
2429	markBackedges();
2430
2431	// Strip off remaining callsite metadata, no longer needed.
2432	for (auto &FuncEntry : FuncToCallsWithMetadata)
2433	for (auto &Call : FuncEntry.second)
2434	Call.call()->setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
2435	}
2436
2437	// Finds the set of GUIDs for weak aliasees that are prevailing in different
2438	// modules than any of their aliases. We need to handle these specially.
2439	DenseSet<GlobalValue::GUID>
2440	IndexCallsiteContextGraph::findAliaseeGUIDsPrevailingInDifferentModule() {
2441	DenseSet<GlobalValue::GUID> AliaseeGUIDs;
2442	for (auto &I : Index) {
2443	auto VI = Index.getValueInfo(R: I);
2444	for (auto &S : VI.getSummaryList()) {
2445	// We only care about aliases to functions.
2446	auto *AS = dyn_cast<AliasSummary>(Val: S.get());
2447	if (!AS)
2448	continue;
2449	auto *AliaseeSummary = &AS->getAliasee();
2450	auto *AliaseeFS = dyn_cast<FunctionSummary>(Val: AliaseeSummary);
2451	if (!AliaseeFS)
2452	continue;
2453	// Skip this summary if it is not for the prevailing symbol for this GUID.
2454	// The linker doesn't resolve local linkage values so don't check whether
2455	// those are prevailing.
2456	if (!GlobalValue::isLocalLinkage(Linkage: S ->linkage()) &&
2457	!isPrevailing (VI.getGUID(), S.get()))
2458	continue;
2459	// Prevailing aliasee could be in a different module only if it is weak.
2460	if (!GlobalValue::isWeakForLinker(Linkage: AliaseeSummary->linkage()))
2461	continue;
2462	auto AliaseeGUID = AS->getAliaseeGUID();
2463	// If the aliasee copy in this module is not prevailing, record it.
2464	if (!isPrevailing (AliaseeGUID, AliaseeSummary))
2465	AliaseeGUIDs.insert(V: AliaseeGUID);
2466	}
2467	}
2468	AliaseesPrevailingInDiffModuleFromAlias += AliaseeGUIDs.size();
2469	return AliaseeGUIDs;
2470	}
2471
2472	IndexCallsiteContextGraph::IndexCallsiteContextGraph(
2473	ModuleSummaryIndex &Index,
2474	llvm::function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
2475	isPrevailing)
2476	: Index(Index), isPrevailing (isPrevailing) {
2477	// Since we use the aliasee summary info to create the necessary clones for
2478	// its aliases, conservatively skip recording the aliasee function's callsites
2479	// in the CCG for any that are prevailing in a different module than one of
2480	// its aliases. We could record the necessary information to do this in the
2481	// summary, but this case should not be common.
2482	DenseSet<GlobalValue::GUID> GUIDsToSkip =
2483	findAliaseeGUIDsPrevailingInDifferentModule();
2484	// Map for keeping track of the largest cold contexts up to the number given
2485	// by MemProfTopNImportant. Must be a std::map (not DenseMap) because keys
2486	// must be sorted.
2487	std::map<uint64_t, uint32_t> TotalSizeToContextIdTopNCold;
2488	// Sort by GUID for deterministic graph construction order.
2489	// TODO: This sort has a measurable cost on the thin link when memprof is
2490	// enabled. Investigate gating it behind an option that is only enabled for
2491	// tests that check internal state.
2492	for (const auto &I : Index.sortedGlobalValueSummariesRange()) {
2493	auto VI = Index.getValueInfo(R: I);
2494	if (GUIDsToSkip.contains(V: VI.getGUID()))
2495	continue;
2496	for (auto &S : VI.getSummaryList()) {
2497	// We should only add the prevailing nodes. Otherwise we may try to clone
2498	// in a weak copy that won't be linked (and may be different than the
2499	// prevailing version).
2500	// We only keep the memprof summary on the prevailing copy now when
2501	// building the combined index, as a space optimization, however don't
2502	// rely on this optimization. The linker doesn't resolve local linkage
2503	// values so don't check whether those are prevailing.
2504	if (!GlobalValue::isLocalLinkage(Linkage: S ->linkage()) &&
2505	!isPrevailing (VI.getGUID(), S.get()))
2506	continue;
2507	auto *FS = dyn_cast<FunctionSummary>(Val: S.get());
2508	if (!FS)
2509	continue;
2510	std::vector<CallInfo> CallsWithMetadata;
2511	if (!FS->allocs().empty()) {
2512	for (auto &AN : FS->mutableAllocs()) {
2513	// This can happen because of recursion elimination handling that
2514	// currently exists in ModuleSummaryAnalysis. Skip these for now.
2515	// We still added them to the summary because we need to be able to
2516	// correlate properly in applyImport in the backends.
2517	if (AN.MIBs.empty())
2518	continue;
2519	IndexCall AllocCall(&AN);
2520	CallsWithMetadata.push_back(x: AllocCall);
2521	auto *AllocNode = addAllocNode(Call: AllocCall, F: FS);
2522	// Pass an empty CallStack to the CallsiteContext (second)
2523	// parameter, since for ThinLTO we already collapsed out the inlined
2524	// stack ids on the allocation call during ModuleSummaryAnalysis.
2525	CallStack<MIBInfo, SmallVector<unsigned>::const_iterator>
2526	EmptyContext;
2527	unsigned I = `0`;
2528	assert(!metadataMayIncludeContextSizeInfo() \|\|
2529	AN.ContextSizeInfos.size() == AN.MIBs.size());
2530	// Now add all of the MIBs and their stack nodes.
2531	for (auto &MIB : AN.MIBs) {
2532	CallStack<MIBInfo, SmallVector<unsigned>::const_iterator>
2533	StackContext(&MIB);
2534	std::vector<ContextTotalSize> ContextSizeInfo;
2535	if (!AN.ContextSizeInfos.empty()) {
2536	for (auto [FullStackId, TotalSize] : AN.ContextSizeInfos [I])
2537	ContextSizeInfo.push_back(x: {.FullStackId: FullStackId, .TotalSize: TotalSize});
2538	}
2539	addStackNodesForMIB<MIBInfo, SmallVector<unsigned>::const_iterator>(
2540	AllocNode, StackContext, CallsiteContext&: EmptyContext, AllocType: MIB.AllocType,
2541	ContextSizeInfo, TotalSizeToContextIdTopNCold);
2542	I++;
2543	}
2544	// If exporting the graph to dot and an allocation id of interest was
2545	// specified, record all the context ids for this allocation node.
2546	if (ExportToDot && AllocNode->OrigStackOrAllocId == AllocIdForDot)
2547	DotAllocContextIds = AllocNode->getContextIds();
2548	assert(AllocNode->AllocTypes != (uint8_t)AllocationType::None);
2549	// Initialize version 0 on the summary alloc node to the current alloc
2550	// type, unless it has both types in which case make it default, so
2551	// that in the case where we aren't able to clone the original version
2552	// always ends up with the default allocation behavior.
2553	AN.Versions [`0`] = (uint8_t)allocTypeToUse(AllocTypes: AllocNode->AllocTypes);
2554	}
2555	}
2556	// For callsite metadata, add to list for this function for later use.
2557	if (!FS->callsites().empty())
2558	for (auto &SN : FS->mutableCallsites()) {
2559	IndexCall StackNodeCall(&SN);
2560	CallsWithMetadata.push_back(x: StackNodeCall);
2561	}
2562
2563	if (!CallsWithMetadata.empty())
2564	FuncToCallsWithMetadata [FS] = CallsWithMetadata;
2565
2566	if (!FS->allocs().empty() \|\| !FS->callsites().empty())
2567	FSToVIMap [FS] = VI;
2568	}
2569	}
2570
2571	if (DumpCCG) {
2572	dbgs() << "CCG before updating call stack chains:\n";
2573	dbgs() << *this;
2574	}
2575
2576	if (ExportToDot)
2577	exportToDot(Label: "prestackupdate");
2578
2579	updateStackNodes();
2580
2581	if (ExportToDot)
2582	exportToDot(Label: "poststackupdate");
2583
2584	handleCallsitesWithMultipleTargets();
2585
2586	markBackedges();
2587	}
2588
2589	template <typename DerivedCCG, typename FuncTy, typename CallTy>
2590	void CallsiteContextGraph<DerivedCCG, FuncTy,
2591	CallTy>::handleCallsitesWithMultipleTargets() {
2592	// Look for and workaround callsites that call multiple functions.
2593	// This can happen for indirect calls, which needs better handling, and in
2594	// more rare cases (e.g. macro expansion).
2595	// TODO: To fix this for indirect calls we will want to perform speculative
2596	// devirtualization using either the normal PGO info with ICP, or using the
2597	// information in the profiled MemProf contexts. We can do this prior to
2598	// this transformation for regular LTO, and for ThinLTO we can simulate that
2599	// effect in the summary and perform the actual speculative devirtualization
2600	// while cloning in the ThinLTO backend.
2601
2602	// Keep track of the new nodes synthesized for discovered tail calls missing
2603	// from the profiled contexts.
2604	MapVector<CallInfo, ContextNode *> TailCallToContextNodeMap;
2605
2606	std::vector<std::pair<CallInfo, ContextNode *>> NewCallToNode;
2607	for (auto &Entry : NonAllocationCallToContextNodeMap) {
2608	auto *Node = Entry.second;
2609	assert(Node->Clones.empty());
2610	// Check all node callees and see if in the same function.
2611	// We need to check all of the calls recorded in this Node, because in some
2612	// cases we may have had multiple calls with the same debug info calling
2613	// different callees. This can happen, for example, when an object is
2614	// constructed in the paramter list - the destructor call of the object has
2615	// the same debug info (line/col) as the call the object was passed to.
2616	// Here we will prune any that don't match all callee nodes.
2617	std::vector<CallInfo> AllCalls;
2618	AllCalls.reserve(Node->MatchingCalls.size() + `1`);
2619	AllCalls.push_back(Node->Call);
2620	llvm::append_range(AllCalls, Node->MatchingCalls);
2621
2622	// First see if we can partition the calls by callee function, creating new
2623	// nodes to host each set of calls calling the same callees. This is
2624	// necessary for support indirect calls with ThinLTO, for which we
2625	// synthesized CallsiteInfo records for each target. They will all have the
2626	// same callsite stack ids and would be sharing a context node at this
2627	// point. We need to perform separate cloning for each, which will be
2628	// applied along with speculative devirtualization in the ThinLTO backends
2629	// as needed. Note this does not currently support looking through tail
2630	// calls, it is unclear if we need that for indirect call targets.
2631	// First partition calls by callee func. Map indexed by func, value is
2632	// struct with list of matching calls, assigned node.
2633	if (partitionCallsByCallee(Node, AllCalls, NewCallToNode))
2634	continue;
2635
2636	auto It = AllCalls.begin();
2637	// Iterate through the calls until we find the first that matches.
2638	for (; It != AllCalls.end(); ++It) {
2639	auto ThisCall = *It;
2640	bool Match = true;
2641	for (auto EI = Node->CalleeEdges.begin(); EI != Node->CalleeEdges.end();
2642	++EI) {
2643	auto Edge = *EI;
2644	if (!Edge->Callee->hasCall())
2645	continue;
2646	assert(NodeToCallingFunc.count(Edge->Callee));
2647	// Check if the called function matches that of the callee node.
2648	if (!calleesMatch(Call: ThisCall.call(), EI, TailCallToContextNodeMap)) {
2649	Match = false;
2650	break;
2651	}
2652	}
2653	// Found a call that matches the callee nodes, we can quit now.
2654	if (Match) {
2655	// If the first match is not the primary call on the Node, update it
2656	// now. We will update the list of matching calls further below.
2657	if (Node->Call != ThisCall) {
2658	Node->setCall(ThisCall);
2659	// We need to update the NonAllocationCallToContextNodeMap, but don't
2660	// want to do this during iteration over that map, so save the calls
2661	// that need updated entries.
2662	NewCallToNode.push_back({ThisCall, Node});
2663	}
2664	break;
2665	}
2666	}
2667	// We will update this list below (or leave it cleared if there was no
2668	// match found above).
2669	Node->MatchingCalls.clear();
2670	// If we hit the end of the AllCalls vector, no call matching the callee
2671	// nodes was found, clear the call information in the node.
2672	if (It == AllCalls.end()) {
2673	RemovedEdgesWithMismatchedCallees ++;
2674	// Work around by setting Node to have a null call, so it gets
2675	// skipped during cloning. Otherwise assignFunctions will assert
2676	// because its data structures are not designed to handle this case.
2677	Node->setCall(CallInfo());
2678	continue;
2679	}
2680	// Now add back any matching calls that call the same function as the
2681	// matching primary call on Node.
2682	for (++It; It != AllCalls.end(); ++It) {
2683	auto ThisCall = *It;
2684	if (!sameCallee(Call1: Node->Call.call(), Call2: ThisCall.call()))
2685	continue;
2686	Node->MatchingCalls.push_back(ThisCall);
2687	}
2688	}
2689
2690	// Remove all mismatched nodes identified in the above loop from the node map
2691	// (checking whether they have a null call which is set above). For a
2692	// MapVector like NonAllocationCallToContextNodeMap it is much more efficient
2693	// to do the removal via remove_if than by individually erasing entries above.
2694	// Also remove any entries if we updated the node's primary call above.
2695	NonAllocationCallToContextNodeMap.remove_if([](const auto &it) {
2696	return !it.second->hasCall() \|\| it.second->Call != it.first;
2697	});
2698
2699	// Add entries for any new primary calls recorded above.
2700	for (auto &[Call, Node] : NewCallToNode)
2701	NonAllocationCallToContextNodeMap[Call] = Node;
2702
2703	// Add the new nodes after the above loop so that the iteration is not
2704	// invalidated.
2705	for (auto &[Call, Node] : TailCallToContextNodeMap)
2706	NonAllocationCallToContextNodeMap[Call] = Node;
2707	}
2708
2709	template <typename DerivedCCG, typename FuncTy, typename CallTy>
2710	bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::partitionCallsByCallee(
2711	ContextNode *Node, ArrayRef<CallInfo> AllCalls,
2712	std::vector<std::pair<CallInfo, ContextNode *>> &NewCallToNode) {
2713	// Struct to keep track of all the calls having the same callee function,
2714	// and the node we eventually assign to them. Eventually we will record the
2715	// context node assigned to this group of calls.
2716	struct CallsWithSameCallee {
2717	std::vector<CallInfo> Calls;
2718	ContextNode Node = nullptr*;
2719	};
2720
2721	// First partition calls by callee function. Build map from each function
2722	// to the list of matching calls.
2723	DenseMap<const FuncTy *, CallsWithSameCallee> CalleeFuncToCallInfo;
2724	for (auto ThisCall : AllCalls) {
2725	auto *F = getCalleeFunc(Call: ThisCall.call());
2726	if (F)
2727	CalleeFuncToCallInfo[F].Calls.push_back(ThisCall);
2728	}
2729
2730	// Next, walk through all callee edges. For each callee node, get its
2731	// containing function and see if it was recorded in the above map (meaning we
2732	// have at least one matching call). Build another map from each callee node
2733	// with a matching call to the structure instance created above containing all
2734	// the calls.
2735	DenseMap<ContextNode , CallsWithSameCallee > CalleeNodeToCallInfo;
2736	for (const auto &Edge : Node->CalleeEdges) {
2737	if (!Edge->Callee->hasCall())
2738	continue;
2739	const FuncTy *ProfiledCalleeFunc = NodeToCallingFunc[Edge->Callee];
2740	if (CalleeFuncToCallInfo.contains(ProfiledCalleeFunc))
2741	CalleeNodeToCallInfo[Edge->Callee] =
2742	&CalleeFuncToCallInfo[ProfiledCalleeFunc];
2743	}
2744
2745	// If there are entries in the second map, then there were no matching
2746	// calls/callees, nothing to do here. Return so we can go to the handling that
2747	// looks through tail calls.
2748	if (CalleeNodeToCallInfo.empty())
2749	return false;
2750
2751	// Walk through all callee edges again. Any and all callee edges that didn't
2752	// match any calls (callee not in the CalleeNodeToCallInfo map) are moved to a
2753	// new caller node (UnmatchedCalleesNode) which gets a null call so that it is
2754	// ignored during cloning. If it is in the map, then we use the node recorded
2755	// in that entry (creating it if needed), and move the callee edge to it.
2756	// The first callee will use the original node instead of creating a new one.
2757	// Note that any of the original calls on this node (in AllCalls) that didn't
2758	// have a callee function automatically get dropped from the node as part of
2759	// this process.
2760	ContextNode UnmatchedCalleesNode = nullptr*;
2761	// Track whether we already assigned original node to a callee.
2762	bool UsedOrigNode = false;
2763	assert(NodeToCallingFunc[Node]);
2764	// Iterate over a copy of Node's callee edges, since we may need to remove
2765	// edges in moveCalleeEdgeToNewCaller, and this simplifies the handling and
2766	// makes it less error-prone.
2767	auto CalleeEdges = Node->CalleeEdges;
2768	for (auto &Edge : CalleeEdges) {
2769	if (!Edge->Callee->hasCall())
2770	continue;
2771
2772	// Will be updated below to point to whatever (caller) node this callee edge
2773	// should be moved to.
2774	ContextNode CallerNodeToUse = nullptr*;
2775
2776	// Handle the case where there were no matching calls first. Move this
2777	// callee edge to the UnmatchedCalleesNode, creating it if needed.
2778	if (!CalleeNodeToCallInfo.contains(Edge->Callee)) {
2779	if (!UnmatchedCalleesNode)
2780	UnmatchedCalleesNode =
2781	createNewNode(/IsAllocation=/false, F: NodeToCallingFunc[Node]);
2782	CallerNodeToUse = UnmatchedCalleesNode;
2783	} else {
2784	// Look up the information recorded for this callee node, and use the
2785	// recorded caller node (creating it if needed).
2786	auto *Info = CalleeNodeToCallInfo[Edge->Callee];
2787	if (!Info->Node) {
2788	// If we haven't assigned any callees to the original node use it.
2789	if (!UsedOrigNode) {
2790	Info->Node = Node;
2791	// Clear the set of matching calls which will be updated below.
2792	Node->MatchingCalls.clear();
2793	UsedOrigNode = true;
2794	} else
2795	Info->Node =
2796	createNewNode(/IsAllocation=/false, F: NodeToCallingFunc[Node]);
2797	assert(!Info->Calls.empty());
2798	// The first call becomes the primary call for this caller node, and the
2799	// rest go in the matching calls list.
2800	Info->Node->setCall(Info->Calls.front());
2801	llvm::append_range(Info->Node->MatchingCalls,
2802	llvm::drop_begin(Info->Calls));
2803	// Save the primary call to node correspondence so that we can update
2804	// the NonAllocationCallToContextNodeMap, which is being iterated in the
2805	// caller of this function.
2806	NewCallToNode.push_back({Info->Node->Call, Info->Node});
2807	}
2808	CallerNodeToUse = Info->Node;
2809	}
2810
2811	// Don't need to move edge if we are using the original node;
2812	if (CallerNodeToUse == Node)
2813	continue;
2814
2815	moveCalleeEdgeToNewCaller(Edge, NewCaller: CallerNodeToUse);
2816	}
2817	// Now that we are done moving edges, clean up any caller edges that ended
2818	// up with no type or context ids. During moveCalleeEdgeToNewCaller all
2819	// caller edges from Node are replicated onto the new callers, and it
2820	// simplifies the handling to leave them until we have moved all
2821	// edges/context ids.
2822	for (auto &I : CalleeNodeToCallInfo)
2823	removeNoneTypeCallerEdges(Node: I.second->Node);
2824	if (UnmatchedCalleesNode)
2825	removeNoneTypeCallerEdges(Node: UnmatchedCalleesNode);
2826	removeNoneTypeCallerEdges(Node);
2827
2828	return true;
2829	}
2830
2831	uint64_t ModuleCallsiteContextGraph::getStackId(uint64_t IdOrIndex) const {
2832	// In the Module (IR) case this is already the Id.
2833	return IdOrIndex;
2834	}
2835
2836	uint64_t IndexCallsiteContextGraph::getStackId(uint64_t IdOrIndex) const {
2837	// In the Index case this is an index into the stack id list in the summary
2838	// index, convert it to an Id.
2839	return Index.getStackIdAtIndex(Index: IdOrIndex);
2840	}
2841
2842	template <typename DerivedCCG, typename FuncTy, typename CallTy>
2843	bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::calleesMatch(
2844	CallTy Call, EdgeIter &EI,
2845	MapVector<CallInfo, ContextNode *> &TailCallToContextNodeMap) {
2846	auto Edge = *EI;
2847	const FuncTy *ProfiledCalleeFunc = NodeToCallingFunc[Edge->Callee];
2848	const FuncTy *CallerFunc = NodeToCallingFunc[Edge->Caller];
2849	// Will be populated in order of callee to caller if we find a chain of tail
2850	// calls between the profiled caller and callee.
2851	std::vector<std::pair<CallTy, FuncTy *>> FoundCalleeChain;
2852	if (!calleeMatchesFunc(Call, Func: ProfiledCalleeFunc, CallerFunc,
2853	FoundCalleeChain))
2854	return false;
2855
2856	// The usual case where the profiled callee matches that of the IR/summary.
2857	if (FoundCalleeChain.empty())
2858	return true;
2859
2860	auto AddEdge = [Edge, &EI](ContextNode Caller, ContextNode Callee) {
2861	auto *CurEdge = Callee->findEdgeFromCaller(Caller);
2862	// If there is already an edge between these nodes, simply update it and
2863	// return.
2864	if (CurEdge) {
2865	CurEdge->ContextIds.insert_range(Edge->ContextIds);
2866	CurEdge->AllocTypes \|= Edge->AllocTypes;
2867	return;
2868	}
2869	// Otherwise, create a new edge and insert it into the caller and callee
2870	// lists.
2871	auto NewEdge = std::make_shared<ContextEdge>(
2872	Callee, Caller, Edge->AllocTypes, Edge->ContextIds);
2873	Callee->CallerEdges.push_back(NewEdge);
2874	if (Caller == Edge->Caller) {
2875	// If we are inserting the new edge into the current edge's caller, insert
2876	// the new edge before the current iterator position, and then increment
2877	// back to the current edge.
2878	EI = Caller->CalleeEdges.insert(EI, NewEdge);
2879	++EI;
2880	assert(*EI == Edge &&
2881	"Iterator position not restored after insert and increment");
2882	} else
2883	Caller->CalleeEdges.push_back(NewEdge);
2884	};
2885
2886	// Create new nodes for each found callee and connect in between the profiled
2887	// caller and callee.
2888	auto *CurCalleeNode = Edge->Callee;
2889	for (auto &[NewCall, Func] : FoundCalleeChain) {
2890	ContextNode NewNode = nullptr*;
2891	// First check if we have already synthesized a node for this tail call.
2892	if (TailCallToContextNodeMap.count(NewCall)) {
2893	NewNode = TailCallToContextNodeMap[NewCall];
2894	NewNode->AllocTypes \|= Edge->AllocTypes;
2895	} else {
2896	FuncToCallsWithMetadata[Func].push_back({NewCall});
2897	// Create Node and record node info.
2898	NewNode = createNewNode(/IsAllocation=/false, F: Func, C: NewCall);
2899	TailCallToContextNodeMap[NewCall] = NewNode;
2900	NewNode->AllocTypes = Edge->AllocTypes;
2901	}
2902
2903	// Hook up node to its callee node
2904	AddEdge(NewNode, CurCalleeNode);
2905
2906	CurCalleeNode = NewNode;
2907	}
2908
2909	// Hook up edge's original caller to new callee node.
2910	AddEdge(Edge->Caller, CurCalleeNode);
2911
2912	#ifndef NDEBUG
2913	// Save this because Edge's fields get cleared below when removed.
2914	auto *Caller = Edge->Caller;
2915	#endif
2916
2917	// Remove old edge
2918	removeEdgeFromGraph(Edge: Edge.get(), EI: &EI, /CalleeIter=/true);
2919
2920	// To simplify the increment of EI in the caller, subtract one from EI.
2921	// In the final AddEdge call we would have either added a new callee edge,
2922	// to Edge->Caller, or found an existing one. Either way we are guaranteed
2923	// that there is at least one callee edge.
2924	assert(!Caller->CalleeEdges.empty());
2925	--EI;
2926
2927	return true;
2928	}
2929
2930	bool ModuleCallsiteContextGraph::findProfiledCalleeThroughTailCalls(
2931	const Function ProfiledCallee, Value CurCallee, unsigned Depth,
2932	std::vector<std::pair<Instruction , Function >> &FoundCalleeChain,
2933	bool &FoundMultipleCalleeChains) {
2934	// Stop recursive search if we have already explored the maximum specified
2935	// depth.
2936	if (Depth > TailCallSearchDepth)
2937	return false;
2938
2939	auto SaveCallsiteInfo = [&](Instruction Callsite, Function F) {
2940	FoundCalleeChain.push_back(x: {Callsite, F});
2941	};
2942
2943	auto *CalleeFunc = dyn_cast<Function>(Val: CurCallee);
2944	if (!CalleeFunc) {
2945	auto *Alias = dyn_cast<GlobalAlias>(Val: CurCallee);
2946	assert(Alias);
2947	CalleeFunc = dyn_cast<Function>(Val: Alias->getAliasee());
2948	assert(CalleeFunc);
2949	}
2950
2951	// Look for tail calls in this function, and check if they either call the
2952	// profiled callee directly, or indirectly (via a recursive search).
2953	// Only succeed if there is a single unique tail call chain found between the
2954	// profiled caller and callee, otherwise we could perform incorrect cloning.
2955	bool FoundSingleCalleeChain = false;
2956	for (auto &BB : *CalleeFunc) {
2957	for (auto &I : BB) {
2958	auto *CB = dyn_cast<CallBase>(Val: &I);
2959	if (!CB \|\| !CB->isTailCall())
2960	continue;
2961	auto *CalledValue = CB->getCalledOperand();
2962	auto *CalledFunction = CB->getCalledFunction();
2963	if (CalledValue && !CalledFunction) {
2964	CalledValue = CalledValue->stripPointerCasts();
2965	// Stripping pointer casts can reveal a called function.
2966	CalledFunction = dyn_cast<Function>(Val: CalledValue);
2967	}
2968	// Check if this is an alias to a function. If so, get the
2969	// called aliasee for the checks below.
2970	if (auto *GA = dyn_cast<GlobalAlias>(Val: CalledValue)) {
2971	assert(!CalledFunction &&
2972	"Expected null called function in callsite for alias");
2973	CalledFunction = dyn_cast<Function>(Val: GA->getAliaseeObject());
2974	}
2975	if (!CalledFunction)
2976	continue;
2977	if (CalledFunction == ProfiledCallee) {
2978	if (FoundSingleCalleeChain) {
2979	FoundMultipleCalleeChains = true;
2980	return false;
2981	}
2982	FoundSingleCalleeChain = true;
2983	FoundProfiledCalleeCount ++;
2984	FoundProfiledCalleeDepth += Depth;
2985	if (Depth > FoundProfiledCalleeMaxDepth)
2986	FoundProfiledCalleeMaxDepth = Depth;
2987	SaveCallsiteInfo (&I, CalleeFunc);
2988	} else if (findProfiledCalleeThroughTailCalls(
2989	ProfiledCallee, CurCallee: CalledFunction, Depth: Depth + `1`,
2990	FoundCalleeChain, FoundMultipleCalleeChains)) {
2991	// findProfiledCalleeThroughTailCalls should not have returned
2992	// true if FoundMultipleCalleeChains.
2993	assert(!FoundMultipleCalleeChains);
2994	if (FoundSingleCalleeChain) {
2995	FoundMultipleCalleeChains = true;
2996	return false;
2997	}
2998	FoundSingleCalleeChain = true;
2999	SaveCallsiteInfo (&I, CalleeFunc);
3000	} else if (FoundMultipleCalleeChains)
3001	return false;
3002	}
3003	}
3004
3005	return FoundSingleCalleeChain;
3006	}
3007
3008	const Function ModuleCallsiteContextGraph::getCalleeFunc(Instruction Call) {
3009	auto *CB = dyn_cast<CallBase>(Val: Call);
3010	if (!CB->getCalledOperand() \|\| CB->isIndirectCall())
3011	return nullptr;
3012	auto *CalleeVal = CB->getCalledOperand()->stripPointerCasts();
3013	auto *Alias = dyn_cast<GlobalAlias>(Val: CalleeVal);
3014	if (Alias)
3015	return dyn_cast<Function>(Val: Alias->getAliasee());
3016	return dyn_cast<Function>(Val: CalleeVal);
3017	}
3018
3019	bool ModuleCallsiteContextGraph::calleeMatchesFunc(
3020	Instruction Call, const* Function Func, const* Function *CallerFunc,
3021	std::vector<std::pair<Instruction , Function >> &FoundCalleeChain) {
3022	auto *CB = dyn_cast<CallBase>(Val: Call);
3023	if (!CB->getCalledOperand() \|\| CB->isIndirectCall())
3024	return false;
3025	auto *CalleeVal = CB->getCalledOperand()->stripPointerCasts();
3026	auto *CalleeFunc = dyn_cast<Function>(Val: CalleeVal);
3027	if (CalleeFunc == Func)
3028	return true;
3029	auto *Alias = dyn_cast<GlobalAlias>(Val: CalleeVal);
3030	if (Alias && Alias->getAliasee() == Func)
3031	return true;
3032
3033	// Recursively search for the profiled callee through tail calls starting with
3034	// the actual Callee. The discovered tail call chain is saved in
3035	// FoundCalleeChain, and we will fixup the graph to include these callsites
3036	// after returning.
3037	// FIXME: We will currently redo the same recursive walk if we find the same
3038	// mismatched callee from another callsite. We can improve this with more
3039	// bookkeeping of the created chain of new nodes for each mismatch.
3040	unsigned Depth = `1`;
3041	bool FoundMultipleCalleeChains = false;
3042	if (!findProfiledCalleeThroughTailCalls(ProfiledCallee: Func, CurCallee: CalleeVal, Depth,
3043	FoundCalleeChain,
3044	FoundMultipleCalleeChains)) {
3045	LLVM_DEBUG(dbgs() << "Not found through unique tail call chain: "
3046	<< Func->getName() << " from " << CallerFunc->getName()
3047	<< " that actually called " << CalleeVal->getName()
3048	<< (FoundMultipleCalleeChains
3049	? " (found multiple possible chains)"
3050	: "")
3051	<< "\n");
3052	if (FoundMultipleCalleeChains)
3053	FoundProfiledCalleeNonUniquelyCount ++;
3054	return false;
3055	}
3056
3057	return true;
3058	}
3059
3060	bool ModuleCallsiteContextGraph::sameCallee(Instruction *Call1,
3061	Instruction *Call2) {
3062	auto *CB1 = cast<CallBase>(Val: Call1);
3063	if (!CB1->getCalledOperand() \|\| CB1->isIndirectCall())
3064	return false;
3065	auto *CalleeVal1 = CB1->getCalledOperand()->stripPointerCasts();
3066	auto *CalleeFunc1 = dyn_cast<Function>(Val: CalleeVal1);
3067	auto *CB2 = cast<CallBase>(Val: Call2);
3068	if (!CB2->getCalledOperand() \|\| CB2->isIndirectCall())
3069	return false;
3070	auto *CalleeVal2 = CB2->getCalledOperand()->stripPointerCasts();
3071	auto *CalleeFunc2 = dyn_cast<Function>(Val: CalleeVal2);
3072	return CalleeFunc1 == CalleeFunc2;
3073	}
3074
3075	bool IndexCallsiteContextGraph::findProfiledCalleeThroughTailCalls(
3076	ValueInfo ProfiledCallee, ValueInfo CurCallee, unsigned Depth,
3077	std::vector<std::pair<IndexCall, FunctionSummary *>> &FoundCalleeChain,
3078	bool &FoundMultipleCalleeChains) {
3079	// Stop recursive search if we have already explored the maximum specified
3080	// depth.
3081	if (Depth > TailCallSearchDepth)
3082	return false;
3083
3084	auto CreateAndSaveCallsiteInfo = [&](ValueInfo Callee, FunctionSummary *FS) {
3085	// Make a CallsiteInfo for each discovered callee, if one hasn't already
3086	// been synthesized.
3087	if (!FunctionCalleesToSynthesizedCallsiteInfos.count(Val: FS) \|\|
3088	!FunctionCalleesToSynthesizedCallsiteInfos [FS].count(x: Callee))
3089	// StackIds is empty (we don't have debug info available in the index for
3090	// these callsites)
3091	FunctionCalleesToSynthesizedCallsiteInfos [FS][Callee] =
3092	std::make_unique<CallsiteInfo>(args&: Callee, args: SmallVector<unsigned>());
3093	CallsiteInfo *NewCallsiteInfo =
3094	FunctionCalleesToSynthesizedCallsiteInfos [FS][Callee].get();
3095	FoundCalleeChain.push_back(x: {NewCallsiteInfo, FS});
3096	};
3097
3098	// Look for tail calls in this function, and check if they either call the
3099	// profiled callee directly, or indirectly (via a recursive search).
3100	// Only succeed if there is a single unique tail call chain found between the
3101	// profiled caller and callee, otherwise we could perform incorrect cloning.
3102	bool FoundSingleCalleeChain = false;
3103	for (auto &S : CurCallee.getSummaryList()) {
3104	if (!GlobalValue::isLocalLinkage(Linkage: S ->linkage()) &&
3105	!isPrevailing (CurCallee.getGUID(), S.get()))
3106	continue;
3107	auto *FS = dyn_cast<FunctionSummary>(Val: S ->getBaseObject());
3108	if (!FS)
3109	continue;
3110	auto FSVI = CurCallee;
3111	auto *AS = dyn_cast<AliasSummary>(Val: S.get());
3112	if (AS)
3113	FSVI = AS->getAliaseeVI();
3114	for (auto &CallEdge : FS->calls()) {
3115	if (!CallEdge.second.hasTailCall())
3116	continue;
3117	if (CallEdge.first == ProfiledCallee) {
3118	if (FoundSingleCalleeChain) {
3119	FoundMultipleCalleeChains = true;
3120	return false;
3121	}
3122	FoundSingleCalleeChain = true;
3123	FoundProfiledCalleeCount ++;
3124	FoundProfiledCalleeDepth += Depth;
3125	if (Depth > FoundProfiledCalleeMaxDepth)
3126	FoundProfiledCalleeMaxDepth = Depth;
3127	CreateAndSaveCallsiteInfo (CallEdge.first, FS);
3128	// Add FS to FSToVIMap in case it isn't already there.
3129	assert(!FSToVIMap.count(FS) \|\| FSToVIMap[FS] == FSVI);
3130	FSToVIMap [FS] = FSVI;
3131	} else if (findProfiledCalleeThroughTailCalls(
3132	ProfiledCallee, CurCallee: CallEdge.first, Depth: Depth + `1`,
3133	FoundCalleeChain, FoundMultipleCalleeChains)) {
3134	// findProfiledCalleeThroughTailCalls should not have returned
3135	// true if FoundMultipleCalleeChains.
3136	assert(!FoundMultipleCalleeChains);
3137	if (FoundSingleCalleeChain) {
3138	FoundMultipleCalleeChains = true;
3139	return false;
3140	}
3141	FoundSingleCalleeChain = true;
3142	CreateAndSaveCallsiteInfo (CallEdge.first, FS);
3143	// Add FS to FSToVIMap in case it isn't already there.
3144	assert(!FSToVIMap.count(FS) \|\| FSToVIMap[FS] == FSVI);
3145	FSToVIMap [FS] = FSVI;
3146	} else if (FoundMultipleCalleeChains)
3147	return false;
3148	}
3149	}
3150
3151	return FoundSingleCalleeChain;
3152	}
3153
3154	const FunctionSummary *
3155	IndexCallsiteContextGraph::getCalleeFunc(IndexCall &Call) {
3156	ValueInfo Callee = dyn_cast_if_present<CallsiteInfo *>(Val&: Call)->Callee;
3157	if (Callee.getSummaryList().empty())
3158	return nullptr;
3159	return dyn_cast<FunctionSummary>(Val: Callee.getSummaryList()[`0`]->getBaseObject());
3160	}
3161
3162	bool IndexCallsiteContextGraph::calleeMatchesFunc(
3163	IndexCall &Call, const FunctionSummary *Func,
3164	const FunctionSummary *CallerFunc,
3165	std::vector<std::pair<IndexCall, FunctionSummary *>> &FoundCalleeChain) {
3166	ValueInfo Callee = dyn_cast_if_present<CallsiteInfo *>(Val&: Call)->Callee;
3167	// If there is no summary list then this is a call to an externally defined
3168	// symbol.
3169	AliasSummary *Alias =
3170	Callee.getSummaryList().empty()
3171	? nullptr
3172	: dyn_cast<AliasSummary>(Val: Callee.getSummaryList()[`0`].get());
3173	assert(FSToVIMap.count(Func));
3174	auto FuncVI = FSToVIMap [Func];
3175	if (Callee == FuncVI \|\|
3176	// If callee is an alias, check the aliasee, since only function
3177	// summary base objects will contain the stack node summaries and thus
3178	// get a context node.
3179	(Alias && Alias->getAliaseeVI() == FuncVI))
3180	return true;
3181
3182	// Recursively search for the profiled callee through tail calls starting with
3183	// the actual Callee. The discovered tail call chain is saved in
3184	// FoundCalleeChain, and we will fixup the graph to include these callsites
3185	// after returning.
3186	// FIXME: We will currently redo the same recursive walk if we find the same
3187	// mismatched callee from another callsite. We can improve this with more
3188	// bookkeeping of the created chain of new nodes for each mismatch.
3189	unsigned Depth = `1`;
3190	bool FoundMultipleCalleeChains = false;
3191	if (!findProfiledCalleeThroughTailCalls(
3192	ProfiledCallee: FuncVI, CurCallee: Callee, Depth, FoundCalleeChain, FoundMultipleCalleeChains)) {
3193	LLVM_DEBUG(dbgs() << "Not found through unique tail call chain: " << FuncVI
3194	<< " from " << FSToVIMap[CallerFunc]
3195	<< " that actually called " << Callee
3196	<< (FoundMultipleCalleeChains
3197	? " (found multiple possible chains)"
3198	: "")
3199	<< "\n");
3200	if (FoundMultipleCalleeChains)
3201	FoundProfiledCalleeNonUniquelyCount ++;
3202	return false;
3203	}
3204
3205	return true;
3206	}
3207
3208	bool IndexCallsiteContextGraph::sameCallee(IndexCall &Call1, IndexCall &Call2) {
3209	ValueInfo Callee1 = dyn_cast_if_present<CallsiteInfo *>(Val&: Call1)->Callee;
3210	ValueInfo Callee2 = dyn_cast_if_present<CallsiteInfo *>(Val&: Call2)->Callee;
3211	return Callee1 == Callee2;
3212	}
3213
3214	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3215	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::dump()
3216	const {
3217	print(OS&: dbgs());
3218	dbgs() << "\n";
3219	}
3220
3221	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3222	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::print(
3223	raw_ostream &OS) const {
3224	OS << "Node " << this << "\n";
3225	OS << "\t";
3226	printCall(OS);
3227	if (Recursive)
3228	OS << " (recursive)";
3229	OS << "\n";
3230	if (!MatchingCalls.empty()) {
3231	OS << "\tMatchingCalls:\n";
3232	for (auto &MatchingCall : MatchingCalls) {
3233	OS << "\t";
3234	MatchingCall.print(OS);
3235	OS << "\n";
3236	}
3237	}
3238	OS << "\tNodeId: " << NodeId << "\n";
3239	OS << "\tAllocTypes: " << getAllocTypeString(AllocTypes) << "\n";
3240	OS << "\tContextIds:";
3241	// Make a copy of the computed context ids that we can sort for stability.
3242	auto ContextIds = getContextIds();
3243	std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
3244	std::sort(first: SortedIds.begin(), last: SortedIds.end());
3245	for (auto Id : SortedIds)
3246	OS << " " << Id;
3247	OS << "\n";
3248	OS << "\tCalleeEdges:\n";
3249	for (auto &Edge : CalleeEdges)
3250	OS << "\t\t" << *Edge << " (Callee NodeId: " << Edge->Callee->NodeId
3251	<< ")\n";
3252	OS << "\tCallerEdges:\n";
3253	for (auto &Edge : CallerEdges)
3254	OS << "\t\t" << *Edge << " (Caller NodeId: " << Edge->Caller->NodeId
3255	<< ")\n";
3256	if (!Clones.empty()) {
3257	OS << "\tClones: ";
3258	ListSeparator LS;
3259	for (auto *C : Clones)
3260	OS << LS << C << " NodeId: " << C->NodeId;
3261	OS << "\n";
3262	} else if (CloneOf) {
3263	OS << "\tClone of " << CloneOf << " NodeId: " << CloneOf->NodeId << "\n";
3264	}
3265	}
3266
3267	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3268	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge::dump()
3269	const {
3270	print(OS&: dbgs());
3271	dbgs() << "\n";
3272	}
3273
3274	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3275	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge::print(
3276	raw_ostream &OS) const {
3277	OS << "Edge from Callee " << Callee << " to Caller: " << Caller
3278	<< (IsBackedge ? " (BE)" : "")
3279	<< " AllocTypes: " << getAllocTypeString(AllocTypes);
3280	OS << " ContextIds:";
3281	std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
3282	std::sort(first: SortedIds.begin(), last: SortedIds.end());
3283	for (auto Id : SortedIds)
3284	OS << " " << Id;
3285	}
3286
3287	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3288	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::dump() const {
3289	print(OS&: dbgs());
3290	}
3291
3292	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3293	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::print(
3294	raw_ostream &OS) const {
3295	OS << "Callsite Context Graph:\n";
3296	using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
3297	for (const auto Node : nodes<GraphType>(this)) {
3298	if (Node->isRemoved())
3299	continue;
3300	Node->print(OS);
3301	OS << "\n";
3302	}
3303	}
3304
3305	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3306	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::printTotalSizes(
3307	raw_ostream &OS,
3308	function_ref<void(StringRef, StringRef, const Twine &)> EmitRemark) const {
3309	using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
3310	for (const auto Node : nodes<GraphType>(this)) {
3311	if (Node->isRemoved())
3312	continue;
3313	if (!Node->IsAllocation)
3314	continue;
3315	DenseSet<uint32_t> ContextIds = Node->getContextIds();
3316	auto AllocTypeFromCall = getAllocationCallType(Call: Node->Call);
3317	std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
3318	std::sort(first: SortedIds.begin(), last: SortedIds.end());
3319	for (auto Id : SortedIds) {
3320	auto TypeI = ContextIdToAllocationType.find(Val: Id);
3321	assert(TypeI != ContextIdToAllocationType.end());
3322	auto CSI = ContextIdToContextSizeInfos.find(Val: Id);
3323	if (CSI != ContextIdToContextSizeInfos.end()) {
3324	for (auto &Info : CSI ->second) {
3325	std::string Msg =
3326	"MemProf hinting: " + getAllocTypeString(AllocTypes: (uint8_t)TypeI ->second) +
3327	" full allocation context " + std::to_string(val: Info.FullStackId) +
3328	" with total size " + std::to_string(val: Info.TotalSize) + " is " +
3329	getAllocTypeString(Node->AllocTypes) + " after cloning";
3330	if (allocTypeToUse(Node->AllocTypes) != AllocTypeFromCall)
3331	Msg += " marked " + getAllocTypeString(AllocTypes: (uint8_t)AllocTypeFromCall) +
3332	" due to cold byte percent";
3333	// Print the internal context id to aid debugging and visualization.
3334	Msg += " (internal context id " + std::to_string(val: Id) + ")";
3335	if (MemProfReportHintedSizes)
3336	OS << Msg << "\n";
3337	if (EmitRemark)
3338	EmitRemark (DEBUG_TYPE, "MemProfReport", Msg);
3339	}
3340	} else {
3341	// This is only emitted if the context size info is not present.
3342	std::string Msg =
3343	"MemProf hinting: " + getAllocTypeString(AllocTypes: (uint8_t)TypeI ->second) +
3344	" context is " + getAllocTypeString(Node->AllocTypes) +
3345	" after cloning";
3346	if (allocTypeToUse(Node->AllocTypes) != AllocTypeFromCall)
3347	Msg += " marked " + getAllocTypeString(AllocTypes: (uint8_t)AllocTypeFromCall) +
3348	" due to cold byte percent";
3349	// Print the internal context id to aid debugging and visualization.
3350	Msg += " (internal context id " + std::to_string(val: Id) + ")";
3351	if (MemProfReportHintedSizes)
3352	OS << Msg << "\n";
3353	if (EmitRemark)
3354	EmitRemark (DEBUG_TYPE, "MemProfReport", Msg);
3355	}
3356	}
3357	}
3358	}
3359
3360	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3361	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::check() const {
3362	using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
3363	for (const auto Node : nodes<GraphType>(this)) {
3364	checkNode<DerivedCCG, FuncTy, CallTy>(Node, /CheckEdges=/false);
3365	for (auto &Edge : Node->CallerEdges)
3366	checkEdge<DerivedCCG, FuncTy, CallTy>(Edge);
3367	}
3368	}
3369
3370	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3371	struct GraphTraits<const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *> {
3372	using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
3373	using NodeRef = const ContextNode<DerivedCCG, FuncTy, CallTy> *;
3374
3375	using NodePtrTy = std::unique_ptr<ContextNode<DerivedCCG, FuncTy, CallTy>>;
3376	static NodeRef getNode(const NodePtrTy &P) { return P.get(); }
3377
3378	using nodes_iterator =
3379	mapped_iterator<typename std::vector<NodePtrTy>::const_iterator,
3380	decltype(&getNode)>;
3381
3382	static nodes_iterator nodes_begin(GraphType G) {
3383	return nodes_iterator(G->NodeOwner.begin(), &getNode);
3384	}
3385
3386	static nodes_iterator nodes_end(GraphType G) {
3387	return nodes_iterator(G->NodeOwner.end(), &getNode);
3388	}
3389
3390	static NodeRef getEntryNode(GraphType G) {
3391	return G->NodeOwner.begin()->get();
3392	}
3393
3394	using EdgePtrTy = std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>>;
3395	static const ContextNode<DerivedCCG, FuncTy, CallTy> *
3396	GetCallee(const EdgePtrTy &P) {
3397	return P->Callee;
3398	}
3399
3400	using ChildIteratorType =
3401	mapped_iterator<typename std::vector<std::shared_ptr<ContextEdge<
3402	DerivedCCG, FuncTy, CallTy>>>::const_iterator,
3403	decltype(&GetCallee)>;
3404
3405	static ChildIteratorType child_begin(NodeRef N) {
3406	return ChildIteratorType(N->CalleeEdges.begin(), &GetCallee);
3407	}
3408
3409	static ChildIteratorType child_end(NodeRef N) {
3410	return ChildIteratorType(N->CalleeEdges.end(), &GetCallee);
3411	}
3412	};
3413
3414	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3415	struct DOTGraphTraits<const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>
3416	: public DefaultDOTGraphTraits {
3417	DOTGraphTraits(bool IsSimple = false) : DefaultDOTGraphTraits(IsSimple) {
3418	// If the user requested the full graph to be exported, but provided an
3419	// allocation id, or if the user gave a context id and requested more than
3420	// just a specific context to be exported, note that highlighting is
3421	// enabled.
3422	DoHighlight =
3423	(AllocIdForDot.getNumOccurrences() && DotGraphScope == DotScope::All) \|\|
3424	(ContextIdForDot.getNumOccurrences() &&
3425	DotGraphScope != DotScope::Context);
3426	}
3427
3428	using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
3429	using GTraits = GraphTraits<GraphType>;
3430	using NodeRef = typename GTraits::NodeRef;
3431	using ChildIteratorType = typename GTraits::ChildIteratorType;
3432
3433	static std::string getNodeLabel(NodeRef Node, GraphType G) {
3434	std::string LabelString =
3435	(Twine ("OrigId: ") + (Node->IsAllocation ? "Alloc" : "") +
3436	Twine(Node->OrigStackOrAllocId) + " NodeId: " + Twine(Node->NodeId))
3437	.str();
3438	LabelString += "\n";
3439	if (Node->hasCall()) {
3440	auto Func = G->NodeToCallingFunc.find(Node);
3441	assert(Func != G->NodeToCallingFunc.end());
3442	LabelString +=
3443	G->getLabel(Func->second, Node->Call.call(), Node->Call.cloneNo());
3444	for (auto &MatchingCall : Node->MatchingCalls) {
3445	LabelString += "\n";
3446	LabelString += G->getLabel(Func->second, MatchingCall.call(),
3447	MatchingCall.cloneNo());
3448	}
3449	} else {
3450	LabelString += "null call";
3451	if (Node->Recursive)
3452	LabelString += " (recursive)";
3453	else
3454	LabelString += " (external)";
3455	}
3456	return LabelString;
3457	}
3458
3459	static std::string getNodeAttributes(NodeRef Node, GraphType G) {
3460	auto ContextIds = Node->getContextIds();
3461	// If highlighting enabled, see if this node contains any of the context ids
3462	// of interest. If so, it will use a different color and a larger fontsize
3463	// (which makes the node larger as well).
3464	bool Highlight = false;
3465	if (DoHighlight) {
3466	assert(ContextIdForDot.getNumOccurrences() \|\|
3467	AllocIdForDot.getNumOccurrences());
3468	if (ContextIdForDot.getNumOccurrences())
3469	Highlight = ContextIds.contains(ContextIdForDot);
3470	else
3471	Highlight = set_intersects(ContextIds, G->DotAllocContextIds);
3472	}
3473	std::string AttributeString = (Twine ("tooltip=\"") + getNodeId(Node) + " " +
3474	getContextIds(ContextIds) + "\"")
3475	.str();
3476	// Default fontsize is 14
3477	if (Highlight)
3478	AttributeString += ",fontsize=\"30\"";
3479	AttributeString +=
3480	(Twine (",fillcolor=\"") + getColor(AllocTypes: Node->AllocTypes, Highlight) + "\"")
3481	.str();
3482	if (Node->CloneOf) {
3483	AttributeString += ",color=\"blue\"";
3484	AttributeString += ",style=\"filled,bold,dashed\"";
3485	} else
3486	AttributeString += ",style=\"filled\"";
3487	return AttributeString;
3488	}
3489
3490	static std::string getEdgeAttributes(NodeRef, ChildIteratorType ChildIter,
3491	GraphType G) {
3492	auto &Edge = *(ChildIter.getCurrent());
3493	// If highlighting enabled, see if this edge contains any of the context ids
3494	// of interest. If so, it will use a different color and a heavier arrow
3495	// size and weight (the larger weight makes the highlighted path
3496	// straighter).
3497	bool Highlight = false;
3498	if (DoHighlight) {
3499	assert(ContextIdForDot.getNumOccurrences() \|\|
3500	AllocIdForDot.getNumOccurrences());
3501	if (ContextIdForDot.getNumOccurrences())
3502	Highlight = Edge->ContextIds.contains(ContextIdForDot);
3503	else
3504	Highlight = set_intersects(Edge->ContextIds, G->DotAllocContextIds);
3505	}
3506	auto Color = getColor(AllocTypes: Edge->AllocTypes, Highlight);
3507	std::string AttributeString =
3508	(Twine ("tooltip=\"") + getContextIds(ContextIds: Edge->ContextIds) + "\"" +
3509	// fillcolor is the arrow head and color is the line
3510	Twine (",fillcolor=\"") + Color + "\"" + Twine (",color=\"") + Color +
3511	"\"")
3512	.str();
3513	if (Edge->IsBackedge)
3514	AttributeString += ",style=\"dotted\"";
3515	// Default penwidth and weight are both 1.
3516	if (Highlight)
3517	AttributeString += ",penwidth=\"2.0\",weight=\"2\"";
3518	return AttributeString;
3519	}
3520
3521	// Since the NodeOwners list includes nodes that are no longer connected to
3522	// the graph, skip them here.
3523	static bool isNodeHidden(NodeRef Node, GraphType G) {
3524	if (Node->isRemoved())
3525	return true;
3526	// If a scope smaller than the full graph was requested, see if this node
3527	// contains any of the context ids of interest.
3528	if (DotGraphScope == DotScope::Alloc)
3529	return !set_intersects(Node->getContextIds(), G->DotAllocContextIds);
3530	if (DotGraphScope == DotScope::Context)
3531	return !Node->getContextIds().contains(ContextIdForDot);
3532	return false;
3533	}
3534
3535	private:
3536	static std::string getContextIds(const DenseSet<uint32_t> &ContextIds) {
3537	std::string IdString = "ContextIds:";
3538	if (ContextIds.size() < `100`) {
3539	std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
3540	std::sort(first: SortedIds.begin(), last: SortedIds.end());
3541	for (auto Id : SortedIds)
3542	IdString += (" " + Twine (Id)).str();
3543	} else {
3544	IdString += (" (" + Twine (ContextIds.size()) + " ids)").str();
3545	}
3546	return IdString;
3547	}
3548
3549	static std::string getColor(uint8_t AllocTypes, bool Highlight) {
3550	// If DoHighlight is not enabled, we want to use the highlight colors for
3551	// NotCold and Cold, and the non-highlight color for NotCold+Cold. This is
3552	// both compatible with the color scheme before highlighting was supported,
3553	// and for the NotCold+Cold color the non-highlight color is a bit more
3554	// readable.
3555	if (AllocTypes == (uint8_t)AllocationType::NotCold)
3556	// Color "brown1" actually looks like a lighter red.
3557	return !DoHighlight \|\| Highlight ? "brown1" : "lightpink";
3558	if (AllocTypes == (uint8_t)AllocationType::Cold)
3559	return !DoHighlight \|\| Highlight ? "cyan" : "lightskyblue";
3560	if (AllocTypes ==
3561	((uint8_t)AllocationType::NotCold \| (uint8_t)AllocationType::Cold))
3562	return Highlight ? "magenta" : "mediumorchid1";
3563	return "gray";
3564	}
3565
3566	static std::string getNodeId(NodeRef Node) {
3567	std::stringstream SStream;
3568	SStream << std::hex << "N0x" << (unsigned long long)Node;
3569	std::string Result = SStream.str();
3570	return Result;
3571	}
3572
3573	// True if we should highlight a specific context or allocation's contexts in
3574	// the emitted graph.
3575	static bool DoHighlight;
3576	};
3577
3578	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3579	bool DOTGraphTraits<
3580	const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>::DoHighlight =
3581	false;
3582
3583	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3584	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::exportToDot(
3585	std::string Label) const {
3586	WriteGraph(this, "", false, Label,
3587	DotFilePathPrefix + "ccg." + Label + ".dot");
3588	}
3589
3590	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3591	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
3592	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::moveEdgeToNewCalleeClone(
3593	const std::shared_ptr<ContextEdge> &Edge,
3594	DenseSet<uint32_t> ContextIdsToMove) {
3595	ContextNode *Node = Edge->Callee;
3596	assert(NodeToCallingFunc.count(Node));
3597	ContextNode *Clone =
3598	createNewNode(IsAllocation: Node->IsAllocation, F: NodeToCallingFunc[Node], C: Node->Call);
3599	Node->addClone(Clone);
3600	Clone->MatchingCalls = Node->MatchingCalls;
3601	moveEdgeToExistingCalleeClone(Edge, NewCallee: Clone, /NewClone=/true,
3602	ContextIdsToMove);
3603	return Clone;
3604	}
3605
3606	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3607	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
3608	moveEdgeToExistingCalleeClone(const std::shared_ptr<ContextEdge> &Edge,
3609	ContextNode NewCallee, bool* NewClone,
3610	DenseSet<uint32_t> ContextIdsToMove) {
3611	// NewCallee and Edge's current callee must be clones of the same original
3612	// node (Edge's current callee may be the original node too).
3613	assert(NewCallee->getOrigNode() == Edge->Callee->getOrigNode());
3614
3615	bool EdgeIsRecursive = Edge->Callee == Edge->Caller;
3616
3617	ContextNode *OldCallee = Edge->Callee;
3618
3619	// We might already have an edge to the new callee from earlier cloning for a
3620	// different allocation. If one exists we will reuse it.
3621	auto ExistingEdgeToNewCallee = NewCallee->findEdgeFromCaller(Edge->Caller);
3622
3623	// Callers will pass an empty ContextIdsToMove set when they want to move the
3624	// edge. Copy in Edge's ids for simplicity.
3625	if (ContextIdsToMove.empty())
3626	ContextIdsToMove = Edge->getContextIds();
3627
3628	// If we are moving all of Edge's ids, then just move the whole Edge.
3629	// Otherwise only move the specified subset, to a new edge if needed.
3630	if (Edge->getContextIds().size() == ContextIdsToMove.size()) {
3631	// First, update the alloc types on New Callee from Edge.
3632	// Do this before we potentially clear Edge's fields below!
3633	NewCallee->AllocTypes \|= Edge->AllocTypes;
3634	// Moving the whole Edge.
3635	if (ExistingEdgeToNewCallee) {
3636	// Since we already have an edge to NewCallee, simply move the ids
3637	// onto it, and remove the existing Edge.
3638	ExistingEdgeToNewCallee->getContextIds().insert_range(ContextIdsToMove);
3639	ExistingEdgeToNewCallee->AllocTypes \|= Edge->AllocTypes;
3640	assert(Edge->ContextIds == ContextIdsToMove);
3641	removeEdgeFromGraph(Edge: Edge.get());
3642	} else {
3643	// Otherwise just reconnect Edge to NewCallee.
3644	Edge->Callee = NewCallee;
3645	NewCallee->CallerEdges.push_back(Edge);
3646	// Remove it from callee where it was previously connected.
3647	OldCallee->eraseCallerEdge(Edge.get());
3648	// Don't need to update Edge's context ids since we are simply
3649	// reconnecting it.
3650	}
3651	} else {
3652	// Only moving a subset of Edge's ids.
3653	// Compute the alloc type of the subset of ids being moved.
3654	auto CallerEdgeAllocType = computeAllocType(ContextIds&: ContextIdsToMove);
3655	if (ExistingEdgeToNewCallee) {
3656	// Since we already have an edge to NewCallee, simply move the ids
3657	// onto it.
3658	ExistingEdgeToNewCallee->getContextIds().insert_range(ContextIdsToMove);
3659	ExistingEdgeToNewCallee->AllocTypes \|= CallerEdgeAllocType;
3660	} else {
3661	// Otherwise, create a new edge to NewCallee for the ids being moved.
3662	auto NewEdge = std::make_shared<ContextEdge>(
3663	NewCallee, Edge->Caller, CallerEdgeAllocType, ContextIdsToMove);
3664	Edge->Caller->CalleeEdges.push_back(NewEdge);
3665	NewCallee->CallerEdges.push_back(NewEdge);
3666	}
3667	// In either case, need to update the alloc types on NewCallee, and remove
3668	// those ids and update the alloc type on the original Edge.
3669	NewCallee->AllocTypes \|= CallerEdgeAllocType;
3670	set_subtract(Edge->ContextIds, ContextIdsToMove);
3671	Edge->AllocTypes = computeAllocType(ContextIds&: Edge->ContextIds);
3672	}
3673	// Now walk the old callee node's callee edges and move Edge's context ids
3674	// over to the corresponding edge into the clone (which is created here if
3675	// this is a newly created clone).
3676	for (auto &OldCalleeEdge : OldCallee->CalleeEdges) {
3677	ContextNode *CalleeToUse = OldCalleeEdge->Callee;
3678	// If this is a direct recursion edge, use NewCallee (the clone) as the
3679	// callee as well, so that any edge updated/created here is also direct
3680	// recursive.
3681	if (CalleeToUse == OldCallee) {
3682	// If this is a recursive edge, see if we already moved a recursive edge
3683	// (which would have to have been this one) - if we were only moving a
3684	// subset of context ids it would still be on OldCallee.
3685	if (EdgeIsRecursive) {
3686	assert(OldCalleeEdge == Edge);
3687	continue;
3688	}
3689	CalleeToUse = NewCallee;
3690	}
3691	// The context ids moving to the new callee are the subset of this edge's
3692	// context ids and the context ids on the caller edge being moved.
3693	DenseSet<uint32_t> EdgeContextIdsToMove =
3694	set_intersection(OldCalleeEdge->getContextIds(), ContextIdsToMove);
3695	set_subtract(OldCalleeEdge->getContextIds(), EdgeContextIdsToMove);
3696	OldCalleeEdge->AllocTypes =
3697	computeAllocType(ContextIds&: OldCalleeEdge->getContextIds());
3698	if (!NewClone) {
3699	// Update context ids / alloc type on corresponding edge to NewCallee.
3700	// There is a chance this may not exist if we are reusing an existing
3701	// clone, specifically during function assignment, where we would have
3702	// removed none type edges after creating the clone. If we can't find
3703	// a corresponding edge there, fall through to the cloning below.
3704	if (auto *NewCalleeEdge = NewCallee->findEdgeFromCallee(CalleeToUse)) {
3705	NewCalleeEdge->getContextIds().insert_range(EdgeContextIdsToMove);
3706	NewCalleeEdge->AllocTypes \|= computeAllocType(ContextIds&: EdgeContextIdsToMove);
3707	continue;
3708	}
3709	}
3710	auto NewEdge = std::make_shared<ContextEdge>(
3711	CalleeToUse, NewCallee, computeAllocType(ContextIds&: EdgeContextIdsToMove),
3712	EdgeContextIdsToMove);
3713	NewCallee->CalleeEdges.push_back(NewEdge);
3714	NewEdge->Callee->CallerEdges.push_back(NewEdge);
3715	}
3716	// Recompute the node alloc type now that its callee edges have been
3717	// updated (since we will compute from those edges).
3718	OldCallee->AllocTypes = OldCallee->computeAllocType();
3719	// OldCallee alloc type should be None iff its context id set is now empty.
3720	assert((OldCallee->AllocTypes == (uint8_t)AllocationType::None) ==
3721	OldCallee->emptyContextIds());
3722	if (VerifyCCG) {
3723	checkNode<DerivedCCG, FuncTy, CallTy>(OldCallee, /CheckEdges=/false);
3724	checkNode<DerivedCCG, FuncTy, CallTy>(NewCallee, /CheckEdges=/false);
3725	for (const auto &OldCalleeEdge : OldCallee->CalleeEdges)
3726	checkNode<DerivedCCG, FuncTy, CallTy>(OldCalleeEdge->Callee,
3727	/CheckEdges=/false);
3728	for (const auto &NewCalleeEdge : NewCallee->CalleeEdges)
3729	checkNode<DerivedCCG, FuncTy, CallTy>(NewCalleeEdge->Callee,
3730	/CheckEdges=/false);
3731	}
3732	}
3733
3734	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3735	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
3736	moveCalleeEdgeToNewCaller(const std::shared_ptr<ContextEdge> &Edge,
3737	ContextNode *NewCaller) {
3738	auto *OldCallee = Edge->Callee;
3739	auto *NewCallee = OldCallee;
3740	// If this edge was direct recursive, make any new/updated edge also direct
3741	// recursive to NewCaller.
3742	bool Recursive = Edge->Caller == Edge->Callee;
3743	if (Recursive)
3744	NewCallee = NewCaller;
3745
3746	ContextNode *OldCaller = Edge->Caller;
3747	OldCaller->eraseCalleeEdge(Edge.get());
3748
3749	// We might already have an edge to the new caller. If one exists we will
3750	// reuse it.
3751	auto ExistingEdgeToNewCaller = NewCaller->findEdgeFromCallee(NewCallee);
3752
3753	if (ExistingEdgeToNewCaller) {
3754	// Since we already have an edge to NewCaller, simply move the ids
3755	// onto it, and remove the existing Edge.
3756	ExistingEdgeToNewCaller->getContextIds().insert_range(
3757	Edge->getContextIds());
3758	ExistingEdgeToNewCaller->AllocTypes \|= Edge->AllocTypes;
3759	Edge->ContextIds.clear();
3760	Edge->AllocTypes = (uint8_t)AllocationType::None;
3761	OldCallee->eraseCallerEdge(Edge.get());
3762	} else {
3763	// Otherwise just reconnect Edge to NewCaller.
3764	Edge->Caller = NewCaller;
3765	NewCaller->CalleeEdges.push_back(Edge);
3766	if (Recursive) {
3767	assert(NewCallee == NewCaller);
3768	// In the case of (direct) recursive edges, we update the callee as well
3769	// so that it becomes recursive on the new caller.
3770	Edge->Callee = NewCallee;
3771	NewCallee->CallerEdges.push_back(Edge);
3772	OldCallee->eraseCallerEdge(Edge.get());
3773	}
3774	// Don't need to update Edge's context ids since we are simply
3775	// reconnecting it.
3776	}
3777	// In either case, need to update the alloc types on New Caller.
3778	NewCaller->AllocTypes \|= Edge->AllocTypes;
3779
3780	// Now walk the old caller node's caller edges and move Edge's context ids
3781	// over to the corresponding edge into the node (which is created here if
3782	// this is a newly created node). We can tell whether this is a newly created
3783	// node by seeing if it has any caller edges yet.
3784	#ifndef NDEBUG
3785	bool IsNewNode = NewCaller->CallerEdges.empty();
3786	#endif
3787	// If we just moved a direct recursive edge, presumably its context ids should
3788	// also flow out of OldCaller via some other non-recursive callee edge. We
3789	// don't want to remove the recursive context ids from other caller edges yet,
3790	// otherwise the context ids get into an inconsistent state on OldCaller.
3791	// We will update these context ids on the non-recursive caller edge when and
3792	// if they are updated on the non-recursive callee.
3793	if (!Recursive) {
3794	for (auto &OldCallerEdge : OldCaller->CallerEdges) {
3795	auto OldCallerCaller = OldCallerEdge->Caller;
3796	// The context ids moving to the new caller are the subset of this edge's
3797	// context ids and the context ids on the callee edge being moved.
3798	DenseSet<uint32_t> EdgeContextIdsToMove = set_intersection(
3799	OldCallerEdge->getContextIds(), Edge->getContextIds());
3800	if (OldCaller == OldCallerCaller) {
3801	OldCallerCaller = NewCaller;
3802	// Don't actually move this one. The caller will move it directly via a
3803	// call to this function with this as the Edge if it is appropriate to
3804	// move to a diff node that has a matching callee (itself).
3805	continue;
3806	}
3807	set_subtract(OldCallerEdge->getContextIds(), EdgeContextIdsToMove);
3808	OldCallerEdge->AllocTypes =
3809	computeAllocType(ContextIds&: OldCallerEdge->getContextIds());
3810	// In this function we expect that any pre-existing node already has edges
3811	// from the same callers as the old node. That should be true in the
3812	// current use case, where we will remove None-type edges after copying
3813	// over all caller edges from the callee.
3814	auto *ExistingCallerEdge = NewCaller->findEdgeFromCaller(OldCallerCaller);
3815	// Since we would have skipped caller edges when moving a direct recursive
3816	// edge, this may not hold true when recursive handling enabled.
3817	assert(IsNewNode \|\| ExistingCallerEdge \|\| AllowRecursiveCallsites);
3818	if (ExistingCallerEdge) {
3819	ExistingCallerEdge->getContextIds().insert_range(EdgeContextIdsToMove);
3820	ExistingCallerEdge->AllocTypes \|=
3821	computeAllocType(ContextIds&: EdgeContextIdsToMove);
3822	continue;
3823	}
3824	auto NewEdge = std::make_shared<ContextEdge>(
3825	NewCaller, OldCallerCaller, computeAllocType(ContextIds&: EdgeContextIdsToMove),
3826	EdgeContextIdsToMove);
3827	NewCaller->CallerEdges.push_back(NewEdge);
3828	NewEdge->Caller->CalleeEdges.push_back(NewEdge);
3829	}
3830	}
3831	// Recompute the node alloc type now that its caller edges have been
3832	// updated (since we will compute from those edges).
3833	OldCaller->AllocTypes = OldCaller->computeAllocType();
3834	// OldCaller alloc type should be None iff its context id set is now empty.
3835	assert((OldCaller->AllocTypes == (uint8_t)AllocationType::None) ==
3836	OldCaller->emptyContextIds());
3837	if (VerifyCCG) {
3838	checkNode<DerivedCCG, FuncTy, CallTy>(OldCaller, /CheckEdges=/false);
3839	checkNode<DerivedCCG, FuncTy, CallTy>(NewCaller, /CheckEdges=/false);
3840	for (const auto &OldCallerEdge : OldCaller->CallerEdges)
3841	checkNode<DerivedCCG, FuncTy, CallTy>(OldCallerEdge->Caller,
3842	/CheckEdges=/false);
3843	for (const auto &NewCallerEdge : NewCaller->CallerEdges)
3844	checkNode<DerivedCCG, FuncTy, CallTy>(NewCallerEdge->Caller,
3845	/CheckEdges=/false);
3846	}
3847	}
3848
3849	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3850	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
3851	recursivelyRemoveNoneTypeCalleeEdges(
3852	ContextNode Node, DenseSet<const* ContextNode *> &Visited) {
3853	auto Inserted = Visited.insert(Node);
3854	if (!Inserted.second)
3855	return;
3856
3857	removeNoneTypeCalleeEdges(Node);
3858
3859	for (auto *Clone : Node->Clones)
3860	recursivelyRemoveNoneTypeCalleeEdges(Node: Clone, Visited);
3861
3862	// The recursive call may remove some of this Node's caller edges.
3863	// Iterate over a copy and skip any that were removed.
3864	auto CallerEdges = Node->CallerEdges;
3865	for (auto &Edge : CallerEdges) {
3866	// Skip any that have been removed by an earlier recursive call.
3867	if (Edge->isRemoved()) {
3868	assert(!is_contained(Node->CallerEdges, Edge));
3869	continue;
3870	}
3871	recursivelyRemoveNoneTypeCalleeEdges(Node: Edge->Caller, Visited);
3872	}
3873	}
3874
3875	// This is the standard DFS based backedge discovery algorithm.
3876	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3877	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::markBackedges() {
3878	// If we are cloning recursive contexts, find and mark backedges from all root
3879	// callers, using the typical DFS based backedge analysis.
3880	if (!CloneRecursiveContexts)
3881	return;
3882	DenseSet<const ContextNode *> Visited;
3883	DenseSet<const ContextNode *> CurrentStack;
3884	for (auto &Entry : NonAllocationCallToContextNodeMap) {
3885	auto *Node = Entry.second;
3886	if (Node->isRemoved())
3887	continue;
3888	// It is a root if it doesn't have callers.
3889	if (!Node->CallerEdges.empty())
3890	continue;
3891	markBackedges(Node, Visited, CurrentStack);
3892	assert(CurrentStack.empty());
3893	}
3894	}
3895
3896	// Recursive helper for above markBackedges method.
3897	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3898	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::markBackedges(
3899	ContextNode Node, DenseSet<const* ContextNode *> &Visited,
3900	DenseSet<const ContextNode *> &CurrentStack) {
3901	auto I = Visited.insert(Node);
3902	// We should only call this for unvisited nodes.
3903	assert(I.second);
3904	(void)I;
3905	for (auto &CalleeEdge : Node->CalleeEdges) {
3906	auto *Callee = CalleeEdge->Callee;
3907	if (Visited.count(Callee)) {
3908	// Since this was already visited we need to check if it is currently on
3909	// the recursive stack in which case it is a backedge.
3910	if (CurrentStack.count(Callee))
3911	CalleeEdge->IsBackedge = true;
3912	continue;
3913	}
3914	CurrentStack.insert(Callee);
3915	markBackedges(Callee, Visited, CurrentStack);
3916	CurrentStack.erase(Callee);
3917	}
3918	}
3919
3920	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3921	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::identifyClones() {
3922	DenseSet<const ContextNode *> Visited;
3923	for (auto &Entry : AllocationCallToContextNodeMap) {
3924	Visited.clear();
3925	identifyClones(Entry.second, Visited, Entry.second->getContextIds());
3926	}
3927	Visited.clear();
3928	for (auto &Entry : AllocationCallToContextNodeMap)
3929	recursivelyRemoveNoneTypeCalleeEdges(Node: Entry.second, Visited);
3930	if (VerifyCCG)
3931	check();
3932	}
3933
3934	// helper function to check an AllocType is cold or notcold or both.
3935	bool checkColdOrNotCold(uint8_t AllocType) {
3936	return (AllocType == (uint8_t)AllocationType::Cold) \|\|
3937	(AllocType == (uint8_t)AllocationType::NotCold) \|\|
3938	(AllocType ==
3939	((uint8_t)AllocationType::Cold \| (uint8_t)AllocationType::NotCold));
3940	}
3941
3942	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3943	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::identifyClones(
3944	ContextNode Node, DenseSet<const* ContextNode *> &Visited,
3945	const DenseSet<uint32_t> &AllocContextIds) {
3946	if (VerifyNodes)
3947	checkNode<DerivedCCG, FuncTy, CallTy>(Node, /CheckEdges=/false);
3948	assert(!Node->CloneOf);
3949
3950	// If Node as a null call, then either it wasn't found in the module (regular
3951	// LTO) or summary index (ThinLTO), or there were other conditions blocking
3952	// cloning (e.g. recursion, calls multiple targets, etc).
3953	// Do this here so that we don't try to recursively clone callers below, which
3954	// isn't useful at least for this node.
3955	if (!Node->hasCall())
3956	return;
3957
3958	// No need to look at any callers if allocation type already unambiguous.
3959	if (hasSingleAllocType(Node->AllocTypes))
3960	return;
3961
3962	#ifndef NDEBUG
3963	auto Insert =
3964	#endif
3965	Visited.insert(Node);
3966	// We should not have visited this node yet.
3967	assert(Insert.second);
3968	// The recursive call to identifyClones may delete the current edge from the
3969	// CallerEdges vector. Make a copy and iterate on that, simpler than passing
3970	// in an iterator and having recursive call erase from it. Other edges may
3971	// also get removed during the recursion, which will have null Callee and
3972	// Caller pointers (and are deleted later), so we skip those below.
3973	{
3974	auto CallerEdges = Node->CallerEdges;
3975	for (auto &Edge : CallerEdges) {
3976	// Skip any that have been removed by an earlier recursive call.
3977	if (Edge->isRemoved()) {
3978	assert(!is_contained(Node->CallerEdges, Edge));
3979	continue;
3980	}
3981	// Defer backedges. See comments further below where these edges are
3982	// handled during the cloning of this Node.
3983	if (Edge->IsBackedge) {
3984	// We should only mark these if cloning recursive contexts, where we
3985	// need to do this deferral.
3986	assert(CloneRecursiveContexts);
3987	continue;
3988	}
3989	// Ignore any caller we previously visited via another edge.
3990	if (!Visited.count(Edge->Caller) && !Edge->Caller->CloneOf) {
3991	identifyClones(Edge->Caller, Visited, AllocContextIds);
3992	}
3993	}
3994	}
3995
3996	// Check if we reached an unambiguous call or have have only a single caller.
3997	if (hasSingleAllocType(Node->AllocTypes) \|\| Node->CallerEdges.size() <= `1`)
3998	return;
3999
4000	// We need to clone.
4001
4002	// Try to keep the original version as alloc type NotCold. This will make
4003	// cases with indirect calls or any other situation with an unknown call to
4004	// the original function get the default behavior. We do this by sorting the
4005	// CallerEdges of the Node we will clone by alloc type.
4006	//
4007	// Give NotCold edge the lowest sort priority so those edges are at the end of
4008	// the caller edges vector, and stay on the original version (since the below
4009	// code clones greedily until it finds all remaining edges have the same type
4010	// and leaves the remaining ones on the original Node).
4011	//
4012	// We shouldn't actually have any None type edges, so the sorting priority for
4013	// that is arbitrary, and we assert in that case below.
4014	const unsigned AllocTypeCloningPriority[] = {/None/ `3`, /NotCold/ `4`,
4015	/Cold/ `1`,
4016	/NotColdCold/ `2`};
4017	llvm::stable_sort(Node->CallerEdges,
4018	[&](const std::shared_ptr<ContextEdge> &A,
4019	const std::shared_ptr<ContextEdge> &B) {
4020	// Nodes with non-empty context ids should be sorted
4021	// before those with empty context ids.
4022	if (A->ContextIds.empty())
4023	// Either B ContextIds are non-empty (in which case we
4024	// should return false because B < A), or B ContextIds
4025	// are empty, in which case they are equal, and we
4026	// should maintain the original relative ordering.
4027	return false;
4028	if (B->ContextIds.empty())
4029	return true;
4030
4031	if (A->AllocTypes == B->AllocTypes)
4032	// Use the caller node id as a deterministic
4033	// tie-breaker.
4034	return A->Caller->NodeId < B->Caller->NodeId;
4035	return AllocTypeCloningPriority[A->AllocTypes] <
4036	AllocTypeCloningPriority[B->AllocTypes];
4037	});
4038
4039	assert(Node->AllocTypes != (uint8_t)AllocationType::None);
4040
4041	DenseSet<uint32_t> RecursiveContextIds;
4042	assert(AllowRecursiveContexts \|\| !CloneRecursiveContexts);
4043	// If we are allowing recursive callsites, but have also disabled recursive
4044	// contexts, look for context ids that show up in multiple caller edges.
4045	if (AllowRecursiveCallsites && !AllowRecursiveContexts) {
4046	DenseSet<uint32_t> AllCallerContextIds;
4047	for (auto &CE : Node->CallerEdges) {
4048	// Resize to the largest set of caller context ids, since we know the
4049	// final set will be at least that large.
4050	AllCallerContextIds.reserve(Size: CE->getContextIds().size());
4051	for (auto Id : CE->getContextIds())
4052	if (!AllCallerContextIds.insert(Id).second)
4053	RecursiveContextIds.insert(Id);
4054	}
4055	}
4056
4057	// Iterate until we find no more opportunities for disambiguating the alloc
4058	// types via cloning. In most cases this loop will terminate once the Node
4059	// has a single allocation type, in which case no more cloning is needed.
4060	// Iterate over a copy of Node's caller edges, since we may need to remove
4061	// edges in the moveEdgeTo methods, and this simplifies the handling and*
4062	// makes it less error-prone.
4063	auto CallerEdges = Node->CallerEdges;
4064	for (auto &CallerEdge : CallerEdges) {
4065	// Skip any that have been removed by an earlier recursive call.
4066	if (CallerEdge->isRemoved()) {
4067	assert(!is_contained(Node->CallerEdges, CallerEdge));
4068	continue;
4069	}
4070	assert(CallerEdge->Callee == Node);
4071
4072	// See if cloning the prior caller edge left this node with a single alloc
4073	// type or a single caller. In that case no more cloning of Node is needed.
4074	if (hasSingleAllocType(Node->AllocTypes) \|\| Node->CallerEdges.size() <= `1`)
4075	break;
4076
4077	// If the caller was not successfully matched to a call in the IR/summary,
4078	// there is no point in trying to clone for it as we can't update that call.
4079	if (!CallerEdge->Caller->hasCall())
4080	continue;
4081
4082	// Only need to process the ids along this edge pertaining to the given
4083	// allocation.
4084	auto CallerEdgeContextsForAlloc =
4085	set_intersection(CallerEdge->getContextIds(), AllocContextIds);
4086	if (!RecursiveContextIds.empty())
4087	CallerEdgeContextsForAlloc =
4088	set_difference(CallerEdgeContextsForAlloc, RecursiveContextIds);
4089	if (CallerEdgeContextsForAlloc.empty())
4090	continue;
4091
4092	auto CallerAllocTypeForAlloc = computeAllocType(ContextIds&: CallerEdgeContextsForAlloc);
4093
4094	// Compute the node callee edge alloc types corresponding to the context ids
4095	// for this caller edge.
4096	std::vector<uint8_t> CalleeEdgeAllocTypesForCallerEdge;
4097	CalleeEdgeAllocTypesForCallerEdge.reserve(n: Node->CalleeEdges.size());
4098	for (auto &CalleeEdge : Node->CalleeEdges)
4099	CalleeEdgeAllocTypesForCallerEdge.push_back(intersectAllocTypes(
4100	Node1Ids: CalleeEdge->getContextIds(), Node2Ids: CallerEdgeContextsForAlloc));
4101
4102	// Don't clone if doing so will not disambiguate any alloc types amongst
4103	// caller edges (including the callee edges that would be cloned).
4104	// Otherwise we will simply move all edges to the clone.
4105	//
4106	// First check if by cloning we will disambiguate the caller allocation
4107	// type from node's allocation type. Query allocTypeToUse so that we don't
4108	// bother cloning to distinguish NotCold+Cold from NotCold. Note that
4109	// neither of these should be None type.
4110	//
4111	// Then check if by cloning node at least one of the callee edges will be
4112	// disambiguated by splitting out different context ids.
4113	//
4114	// However, always do the cloning if this is a backedge, in which case we
4115	// have not yet cloned along this caller edge.
4116	assert(CallerEdge->AllocTypes != (uint8_t)AllocationType::None);
4117	assert(Node->AllocTypes != (uint8_t)AllocationType::None);
4118	if (!CallerEdge->IsBackedge &&
4119	allocTypeToUse(CallerAllocTypeForAlloc) ==
4120	allocTypeToUse(Node->AllocTypes) &&
4121	allocTypesMatch<DerivedCCG, FuncTy, CallTy>(
4122	CalleeEdgeAllocTypesForCallerEdge, Node->CalleeEdges)) {
4123	continue;
4124	}
4125
4126	if (CallerEdge->IsBackedge) {
4127	// We should only mark these if cloning recursive contexts, where we
4128	// need to do this deferral.
4129	assert(CloneRecursiveContexts);
4130	DeferredBackedges ++;
4131	}
4132
4133	// If this is a backedge, we now do recursive cloning starting from its
4134	// caller since we may have moved unambiguous caller contexts to a clone
4135	// of this Node in a previous iteration of the current loop, giving more
4136	// opportunity for cloning through the backedge. Because we sorted the
4137	// caller edges earlier so that cold caller edges are first, we would have
4138	// visited and cloned this node for any unamibiguously cold non-recursive
4139	// callers before any ambiguous backedge callers. Note that we don't do this
4140	// if the caller is already cloned or visited during cloning (e.g. via a
4141	// different context path from the allocation).
4142	// TODO: Can we do better in the case where the caller was already visited?
4143	if (CallerEdge->IsBackedge && !CallerEdge->Caller->CloneOf &&
4144	!Visited.count(CallerEdge->Caller)) {
4145	const auto OrigIdCount = CallerEdge->getContextIds().size();
4146	// Now do the recursive cloning of this backedge's caller, which was
4147	// deferred earlier.
4148	identifyClones(CallerEdge->Caller, Visited, CallerEdgeContextsForAlloc);
4149	removeNoneTypeCalleeEdges(Node: CallerEdge->Caller);
4150	// See if the recursive call to identifyClones moved the context ids to a
4151	// new edge from this node to a clone of caller, and switch to looking at
4152	// that new edge so that we clone Node for the new caller clone.
4153	bool UpdatedEdge = false;
4154	if (OrigIdCount > CallerEdge->getContextIds().size()) {
4155	for (auto E : Node->CallerEdges) {
4156	// Only interested in clones of the current edges caller.
4157	if (E->Caller->CloneOf != CallerEdge->Caller)
4158	continue;
4159	// See if this edge contains any of the context ids originally on the
4160	// current caller edge.
4161	auto CallerEdgeContextsForAllocNew =
4162	set_intersection(CallerEdgeContextsForAlloc, E->getContextIds());
4163	if (CallerEdgeContextsForAllocNew.empty())
4164	continue;
4165	// Make sure we don't pick a previously existing caller edge of this
4166	// Node, which would be processed on a different iteration of the
4167	// outer loop over the saved CallerEdges.
4168	if (llvm::is_contained(CallerEdges, E))
4169	continue;
4170	// The CallerAllocTypeForAlloc and CalleeEdgeAllocTypesForCallerEdge
4171	// are updated further below for all cases where we just invoked
4172	// identifyClones recursively.
4173	CallerEdgeContextsForAlloc.swap(CallerEdgeContextsForAllocNew);
4174	CallerEdge = E;
4175	UpdatedEdge = true;
4176	break;
4177	}
4178	}
4179	// If cloning removed this edge (and we didn't update it to a new edge
4180	// above), we're done with this edge. It's possible we moved all of the
4181	// context ids to an existing clone, in which case there's no need to do
4182	// further processing for them.
4183	if (CallerEdge->isRemoved())
4184	continue;
4185
4186	// Now we need to update the information used for the cloning decisions
4187	// further below, as we may have modified edges and their context ids.
4188
4189	// Note if we changed the CallerEdge above we would have already updated
4190	// the context ids.
4191	if (!UpdatedEdge) {
4192	CallerEdgeContextsForAlloc = set_intersection(
4193	CallerEdgeContextsForAlloc, CallerEdge->getContextIds());
4194	if (CallerEdgeContextsForAlloc.empty())
4195	continue;
4196	}
4197	// Update the other information that depends on the edges and on the now
4198	// updated CallerEdgeContextsForAlloc.
4199	CallerAllocTypeForAlloc = computeAllocType(ContextIds&: CallerEdgeContextsForAlloc);
4200	CalleeEdgeAllocTypesForCallerEdge.clear();
4201	for (auto &CalleeEdge : Node->CalleeEdges) {
4202	CalleeEdgeAllocTypesForCallerEdge.push_back(intersectAllocTypes(
4203	Node1Ids: CalleeEdge->getContextIds(), Node2Ids: CallerEdgeContextsForAlloc));
4204	}
4205	}
4206
4207	// First see if we can use an existing clone. Check each clone and its
4208	// callee edges for matching alloc types.
4209	ContextNode Clone = nullptr*;
4210	for (auto *CurClone : Node->Clones) {
4211	if (allocTypeToUse(CurClone->AllocTypes) !=
4212	allocTypeToUse(CallerAllocTypeForAlloc))
4213	continue;
4214
4215	bool BothSingleAlloc = hasSingleAllocType(CurClone->AllocTypes) &&
4216	hasSingleAllocType(CallerAllocTypeForAlloc);
4217	// The above check should mean that if both have single alloc types that
4218	// they should be equal.
4219	assert(!BothSingleAlloc \|\|
4220	CurClone->AllocTypes == CallerAllocTypeForAlloc);
4221
4222	// If either both have a single alloc type (which are the same), or if the
4223	// clone's callee edges have the same alloc types as those for the current
4224	// allocation on Node's callee edges (CalleeEdgeAllocTypesForCallerEdge),
4225	// then we can reuse this clone.
4226	if (BothSingleAlloc \|\| allocTypesMatchClone<DerivedCCG, FuncTy, CallTy>(
4227	CalleeEdgeAllocTypesForCallerEdge, CurClone)) {
4228	Clone = CurClone;
4229	break;
4230	}
4231	}
4232
4233	// The edge iterator is adjusted when we move the CallerEdge to the clone.
4234	if (Clone)
4235	moveEdgeToExistingCalleeClone(Edge: CallerEdge, NewCallee: Clone, /NewClone=/false,
4236	ContextIdsToMove: CallerEdgeContextsForAlloc);
4237	else
4238	Clone = moveEdgeToNewCalleeClone(Edge: CallerEdge, ContextIdsToMove: CallerEdgeContextsForAlloc);
4239
4240	// Sanity check that no alloc types on clone or its edges are None.
4241	assert(Clone->AllocTypes != (uint8_t)AllocationType::None);
4242	}
4243
4244	// We should still have some context ids on the original Node.
4245	assert(!Node->emptyContextIds());
4246
4247	// Sanity check that no alloc types on node or edges are None.
4248	assert(Node->AllocTypes != (uint8_t)AllocationType::None);
4249
4250	if (VerifyNodes)
4251	checkNode<DerivedCCG, FuncTy, CallTy>(Node, /CheckEdges=/false);
4252	}
4253
4254	void ModuleCallsiteContextGraph::updateAllocationCall(
4255	CallInfo &Call, AllocationType AllocType) {
4256	std::string AllocTypeString = getAllocTypeAttributeString(Type: AllocType);
4257	removeAnyExistingAmbiguousAttribute(CB: cast<CallBase>(Val: Call.call()));
4258	auto A = llvm::Attribute::get(Context&: Call.call()->getFunction()->getContext(),
4259	Kind: "memprof", Val: AllocTypeString);
4260	cast<CallBase>(Val: Call.call())->addFnAttr(Attr: A);
4261	OREGetter (Call.call()->getFunction())
4262	.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofAttribute", Call.call())
4263	<< ore::NV ("AllocationCall", Call.call()) << " in clone "
4264	<< ore::NV ("Caller", Call.call()->getFunction())
4265	<< " marked with memprof allocation attribute "
4266	<< ore::NV ("Attribute", AllocTypeString));
4267	}
4268
4269	void IndexCallsiteContextGraph::updateAllocationCall(CallInfo &Call,
4270	AllocationType AllocType) {
4271	auto AI = cast<AllocInfo >(Val: Call.call());
4272	assert(AI);
4273	assert(AI->Versions.size() > Call.cloneNo());
4274	AI->Versions [Call.cloneNo()] = (uint8_t)AllocType;
4275	}
4276
4277	AllocationType
4278	ModuleCallsiteContextGraph::getAllocationCallType(const CallInfo &Call) const {
4279	const auto *CB = cast<CallBase>(Val: Call.call());
4280	if (!CB->getAttributes().hasFnAttr(Kind: "memprof"))
4281	return AllocationType::None;
4282	return CB->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "cold"
4283	? AllocationType::Cold
4284	: AllocationType::NotCold;
4285	}
4286
4287	AllocationType
4288	IndexCallsiteContextGraph::getAllocationCallType(const CallInfo &Call) const {
4289	const auto AI = cast<AllocInfo >(Val: Call.call());
4290	assert(AI->Versions.size() > Call.cloneNo());
4291	return (AllocationType)AI->Versions [Call.cloneNo()];
4292	}
4293
4294	void ModuleCallsiteContextGraph::updateCall(CallInfo &CallerCall,
4295	FuncInfo CalleeFunc) {
4296	auto *CurF = getCalleeFunc(Call: CallerCall.call());
4297	auto NewCalleeCloneNo = CalleeFunc.cloneNo();
4298	if (isMemProfClone(F: *CurF)) {
4299	// If we already assigned this callsite to call a specific non-default
4300	// clone (i.e. not the original function which is clone 0), ensure that we
4301	// aren't trying to now update it to call a different clone, which is
4302	// indicative of a bug in the graph or function assignment.
4303	auto CurCalleeCloneNo = getMemProfCloneNum(F: *CurF);
4304	if (CurCalleeCloneNo != NewCalleeCloneNo) {
4305	LLVM_DEBUG(dbgs() << "Mismatch in call clone assignment: was "
4306	<< CurCalleeCloneNo << " now " << NewCalleeCloneNo
4307	<< "\n");
4308	MismatchedCloneAssignments ++;
4309	}
4310	}
4311	if (NewCalleeCloneNo > `0`)
4312	cast<CallBase>(Val: CallerCall.call())->setCalledFunction(CalleeFunc.func());
4313	OREGetter (CallerCall.call()->getFunction())
4314	.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofCall", CallerCall.call())
4315	<< ore::NV ("Call", CallerCall.call()) << " in clone "
4316	<< ore::NV ("Caller", CallerCall.call()->getFunction())
4317	<< " assigned to call function clone "
4318	<< ore::NV ("Callee", CalleeFunc.func()));
4319	}
4320
4321	void IndexCallsiteContextGraph::updateCall(CallInfo &CallerCall,
4322	FuncInfo CalleeFunc) {
4323	auto CI = cast<CallsiteInfo >(Val: CallerCall.call());
4324	assert(CI &&
4325	"Caller cannot be an allocation which should not have profiled calls");
4326	assert(CI->Clones.size() > CallerCall.cloneNo());
4327	auto NewCalleeCloneNo = CalleeFunc.cloneNo();
4328	auto &CurCalleeCloneNo = CI->Clones [CallerCall.cloneNo()];
4329	// If we already assigned this callsite to call a specific non-default
4330	// clone (i.e. not the original function which is clone 0), ensure that we
4331	// aren't trying to now update it to call a different clone, which is
4332	// indicative of a bug in the graph or function assignment.
4333	if (CurCalleeCloneNo != `0` && CurCalleeCloneNo != NewCalleeCloneNo) {
4334	LLVM_DEBUG(dbgs() << "Mismatch in call clone assignment: was "
4335	<< CurCalleeCloneNo << " now " << NewCalleeCloneNo
4336	<< "\n");
4337	MismatchedCloneAssignments ++;
4338	}
4339	CurCalleeCloneNo = NewCalleeCloneNo;
4340	}
4341
4342	// Update the debug information attached to NewFunc to use the clone Name. Note
4343	// this needs to be done for both any existing DISubprogram for the definition,
4344	// as well as any separate declaration DISubprogram.
4345	static void updateSubprogramLinkageName(Function *NewFunc, StringRef Name) {
4346	assert(Name == NewFunc->getName());
4347	auto *SP = NewFunc->getSubprogram();
4348	if (!SP)
4349	return;
4350	auto *MDName = MDString::get(Context&: NewFunc->getParent()->getContext(), Str: Name);
4351	SP->replaceLinkageName(LN: MDName);
4352	DISubprogram *Decl = SP->getDeclaration();
4353	if (!Decl)
4354	return;
4355	TempDISubprogram NewDecl = Decl->clone();
4356	NewDecl ->replaceLinkageName(LN: MDName);
4357	SP->replaceDeclaration(Decl: MDNode::replaceWithUniqued(N: std::move(NewDecl)));
4358	}
4359
4360	CallsiteContextGraph<ModuleCallsiteContextGraph, Function,
4361	Instruction *>::FuncInfo
4362	ModuleCallsiteContextGraph::cloneFunctionForCallsite(
4363	FuncInfo &Func, CallInfo &Call, DenseMap<CallInfo, CallInfo> &CallMap,
4364	std::vector<CallInfo> &CallsWithMetadataInFunc, unsigned CloneNo) {
4365	// Use existing LLVM facilities for cloning and obtaining Call in clone
4366	ValueToValueMapTy VMap;
4367	auto *NewFunc = CloneFunction(F: Func.func(), VMap);
4368	std::string Name = getMemProfFuncName(Base: Func.func()->getName(), CloneNo);
4369	assert(!Func.func()->getParent()->getFunction(Name));
4370	NewFunc->setName(Name);
4371	updateSubprogramLinkageName(NewFunc, Name);
4372	for (auto &Inst : CallsWithMetadataInFunc) {
4373	// This map always has the initial version in it.
4374	assert(Inst.cloneNo() == `0`);
4375	CallMap [Inst] = {cast<Instruction>(Val&: VMap [Inst.call()]), CloneNo};
4376	}
4377	OREGetter (Func.func())
4378	.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofClone", Func.func())
4379	<< "created clone " << ore::NV ("NewFunction", NewFunc));
4380	return {NewFunc, CloneNo};
4381	}
4382
4383	CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
4384	IndexCall>::FuncInfo
4385	IndexCallsiteContextGraph::cloneFunctionForCallsite(
4386	FuncInfo &Func, CallInfo &Call, DenseMap<CallInfo, CallInfo> &CallMap,
4387	std::vector<CallInfo> &CallsWithMetadataInFunc, unsigned CloneNo) {
4388	// Check how many clones we have of Call (and therefore function).
4389	// The next clone number is the current size of versions array.
4390	// Confirm this matches the CloneNo provided by the caller, which is based on
4391	// the number of function clones we have.
4392	assert(CloneNo == (isa<AllocInfo *>(Call.call())
4393	? cast<AllocInfo *>(Call.call())->Versions.size()
4394	: cast<CallsiteInfo *>(Call.call())->Clones.size()));
4395	// Walk all the instructions in this function. Create a new version for
4396	// each (by adding an entry to the Versions/Clones summary array), and copy
4397	// over the version being called for the function clone being cloned here.
4398	// Additionally, add an entry to the CallMap for the new function clone,
4399	// mapping the original call (clone 0, what is in CallsWithMetadataInFunc)
4400	// to the new call clone.
4401	for (auto &Inst : CallsWithMetadataInFunc) {
4402	// This map always has the initial version in it.
4403	assert(Inst.cloneNo() == `0`);
4404	if (auto AI = dyn_cast<AllocInfo >(Val: Inst.call())) {
4405	assert(AI->Versions.size() == CloneNo);
4406	// We assign the allocation type later (in updateAllocationCall), just add
4407	// an entry for it here.
4408	AI->Versions.push_back(Elt: `0`);
4409	} else {
4410	auto CI = cast<CallsiteInfo >(Val: Inst.call());
4411	assert(CI && CI->Clones.size() == CloneNo);
4412	// We assign the clone number later (in updateCall), just add an entry for
4413	// it here.
4414	CI->Clones.push_back(Elt: `0`);
4415	}
4416	CallMap [Inst] = {Inst.call(), CloneNo};
4417	}
4418	return {Func.func(), CloneNo};
4419	}
4420
4421	// We perform cloning for each allocation node separately. However, this
4422	// sometimes results in a situation where the same node calls multiple
4423	// clones of the same callee, created for different allocations. This
4424	// causes issues when assigning functions to these clones, as each node can
4425	// in reality only call a single callee clone.
4426	//
4427	// To address this, before assigning functions, merge callee clone nodes as
4428	// needed using a post order traversal from the allocations. We attempt to
4429	// use existing clones as the merge node when legal, and to share them
4430	// among callers with the same properties (callers calling the same set of
4431	// callee clone nodes for the same allocations).
4432	//
4433	// Without this fix, in some cases incorrect function assignment will lead
4434	// to calling the wrong allocation clone.
4435	template <typename DerivedCCG, typename FuncTy, typename CallTy>
4436	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::mergeClones() {
4437	if (!MergeClones)
4438	return;
4439
4440	// Generate a map from context id to the associated allocation node for use
4441	// when merging clones.
4442	DenseMap<uint32_t, ContextNode *> ContextIdToAllocationNode;
4443	for (auto &Entry : AllocationCallToContextNodeMap) {
4444	auto *Node = Entry.second;
4445	for (auto Id : Node->getContextIds())
4446	ContextIdToAllocationNode[Id] = Node->getOrigNode();
4447	for (auto *Clone : Node->Clones) {
4448	for (auto Id : Clone->getContextIds())
4449	ContextIdToAllocationNode[Id] = Clone->getOrigNode();
4450	}
4451	}
4452
4453	// Post order traversal starting from allocations to ensure each callsite
4454	// calls a single clone of its callee. Callee nodes that are clones of each
4455	// other are merged (via new merge nodes if needed) to achieve this.
4456	DenseSet<const ContextNode *> Visited;
4457	for (auto &Entry : AllocationCallToContextNodeMap) {
4458	auto *Node = Entry.second;
4459
4460	mergeClones(Node, Visited, ContextIdToAllocationNode);
4461
4462	// Make a copy so the recursive post order traversal that may create new
4463	// clones doesn't mess up iteration. Note that the recursive traversal
4464	// itself does not call mergeClones on any of these nodes, which are all
4465	// (clones of) allocations.
4466	auto Clones = Node->Clones;
4467	for (auto *Clone : Clones)
4468	mergeClones(Clone, Visited, ContextIdToAllocationNode);
4469	}
4470
4471	if (DumpCCG) {
4472	dbgs() << "CCG after merging:\n";
4473	dbgs() << *this;
4474	}
4475	if (ExportToDot)
4476	exportToDot(Label: "aftermerge");
4477
4478	if (VerifyCCG) {
4479	check();
4480	}
4481	}
4482
4483	// Recursive helper for above mergeClones method.
4484	template <typename DerivedCCG, typename FuncTy, typename CallTy>
4485	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::mergeClones(
4486	ContextNode Node, DenseSet<const* ContextNode *> &Visited,
4487	DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode) {
4488	auto Inserted = Visited.insert(Node);
4489	if (!Inserted.second)
4490	return;
4491
4492	// Iteratively perform merging on this node to handle new caller nodes created
4493	// during the recursive traversal. We could do something more elegant such as
4494	// maintain a worklist, but this is a simple approach that doesn't cause a
4495	// measureable compile time effect, as most nodes don't have many caller
4496	// edges to check.
4497	bool FoundUnvisited = true;
4498	unsigned Iters = `0`;
4499	while (FoundUnvisited) {
4500	Iters++;
4501	FoundUnvisited = false;
4502	// Make a copy since the recursive call may move a caller edge to a new
4503	// callee, messing up the iterator.
4504	auto CallerEdges = Node->CallerEdges;
4505	for (auto CallerEdge : CallerEdges) {
4506	// Skip any caller edge moved onto a different callee during recursion.
4507	if (CallerEdge->Callee != Node)
4508	continue;
4509	// If we found an unvisited caller, note that we should check the caller
4510	// edges again as mergeClones may add or change caller nodes.
4511	if (DoMergeIteration && !Visited.contains(CallerEdge->Caller))
4512	FoundUnvisited = true;
4513	mergeClones(CallerEdge->Caller, Visited, ContextIdToAllocationNode);
4514	}
4515	}
4516
4517	TotalMergeInvokes ++;
4518	TotalMergeIters += Iters;
4519	if (Iters > MaxMergeIters)
4520	MaxMergeIters = Iters;
4521
4522	// Merge for this node after we handle its callers.
4523	mergeNodeCalleeClones(Node, Visited, ContextIdToAllocationNode);
4524	}
4525
4526	template <typename DerivedCCG, typename FuncTy, typename CallTy>
4527	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::mergeNodeCalleeClones(
4528	ContextNode Node, DenseSet<const* ContextNode *> &Visited,
4529	DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode) {
4530	// Ignore Node if we moved all of its contexts to clones.
4531	if (Node->emptyContextIds())
4532	return;
4533
4534	// First identify groups of clones among Node's callee edges, by building
4535	// a map from each callee base node to the associated callee edges from Node.
4536	MapVector<ContextNode *, std::vector<std::shared_ptr<ContextEdge>>>
4537	OrigNodeToCloneEdges;
4538	for (const auto &E : Node->CalleeEdges) {
4539	auto *Callee = E->Callee;
4540	if (!Callee->CloneOf && Callee->Clones.empty())
4541	continue;
4542	ContextNode *Base = Callee->getOrigNode();
4543	OrigNodeToCloneEdges[Base].push_back(E);
4544	}
4545
4546	// Helper for callee edge sorting below. Return true if A's callee has fewer
4547	// caller edges than B, or if A is a clone and B is not, or if A's callee
4548	// node id is smaller than B's.
4549	auto CalleeCallerEdgeLessThan = [](const std::shared_ptr<ContextEdge> &A,
4550	const std::shared_ptr<ContextEdge> &B) {
4551	if (A->Callee->CallerEdges.size() != B->Callee->CallerEdges.size())
4552	return A->Callee->CallerEdges.size() < B->Callee->CallerEdges.size();
4553	if (A->Callee->CloneOf && !B->Callee->CloneOf)
4554	return true;
4555	else if (!A->Callee->CloneOf && B->Callee->CloneOf)
4556	return false;
4557	// Use the callee node id as a deterministic tie-breaker.
4558	return A->Callee->NodeId < B->Callee->NodeId;
4559	};
4560
4561	// Process each set of callee clones called by Node, performing the needed
4562	// merging.
4563	for (auto Entry : OrigNodeToCloneEdges) {
4564	// CalleeEdges is the set of edges from Node reaching callees that are
4565	// mutual clones of each other.
4566	auto &CalleeEdges = Entry.second;
4567	auto NumCalleeClones = CalleeEdges.size();
4568	// A single edge means there is no merging needed.
4569	if (NumCalleeClones == `1`)
4570	continue;
4571	// Sort the CalleeEdges calling this group of clones in ascending order of
4572	// their caller edge counts, putting the original non-clone node first in
4573	// cases of a tie. This simplifies finding an existing node to use as the
4574	// merge node.
4575	llvm::stable_sort(CalleeEdges, CalleeCallerEdgeLessThan);
4576
4577	/// Find other callers of the given set of callee edges that can
4578	/// share the same callee merge node. See the comments at this method
4579	/// definition for details.
4580	DenseSet<ContextNode *> OtherCallersToShareMerge;
4581	findOtherCallersToShareMerge(Node, CalleeEdges, ContextIdToAllocationNode,
4582	OtherCallersToShareMerge);
4583
4584	// Now do the actual merging. Identify existing or create a new MergeNode
4585	// during the first iteration. Move each callee over, along with edges from
4586	// other callers we've determined above can share the same merge node.
4587	ContextNode MergeNode = nullptr*;
4588	DenseMap<ContextNode , unsigned*> CallerToMoveCount;
4589	for (auto CalleeEdge : CalleeEdges) {
4590	auto *OrigCallee = CalleeEdge->Callee;
4591	// If we don't have a MergeNode yet (only happens on the first iteration,
4592	// as a new one will be created when we go to move the first callee edge
4593	// over as needed), see if we can use this callee.
4594	if (!MergeNode) {
4595	// If there are no other callers, simply use this callee.
4596	if (CalleeEdge->Callee->CallerEdges.size() == `1`) {
4597	MergeNode = OrigCallee;
4598	NonNewMergedNodes ++;
4599	continue;
4600	}
4601	// Otherwise, if we have identified other caller nodes that can share
4602	// the merge node with Node, see if all of OrigCallee's callers are
4603	// going to share the same merge node. In that case we can use callee
4604	// (since all of its callers would move to the new merge node).
4605	if (!OtherCallersToShareMerge.empty()) {
4606	bool MoveAllCallerEdges = true;
4607	for (auto CalleeCallerE : OrigCallee->CallerEdges) {
4608	if (CalleeCallerE == CalleeEdge)
4609	continue;
4610	if (!OtherCallersToShareMerge.contains(CalleeCallerE->Caller)) {
4611	MoveAllCallerEdges = false;
4612	break;
4613	}
4614	}
4615	// If we are going to move all callers over, we can use this callee as
4616	// the MergeNode.
4617	if (MoveAllCallerEdges) {
4618	MergeNode = OrigCallee;
4619	NonNewMergedNodes ++;
4620	continue;
4621	}
4622	}
4623	}
4624	// Move this callee edge, creating a new merge node if necessary.
4625	if (MergeNode) {
4626	assert(MergeNode != OrigCallee);
4627	moveEdgeToExistingCalleeClone(Edge: CalleeEdge, NewCallee: MergeNode,
4628	/NewClone/ false);
4629	} else {
4630	MergeNode = moveEdgeToNewCalleeClone(Edge: CalleeEdge);
4631	NewMergedNodes ++;
4632	}
4633	// Now move all identified edges from other callers over to the merge node
4634	// as well.
4635	if (!OtherCallersToShareMerge.empty()) {
4636	// Make and iterate over a copy of OrigCallee's caller edges because
4637	// some of these will be moved off of the OrigCallee and that would mess
4638	// up the iteration from OrigCallee.
4639	auto OrigCalleeCallerEdges = OrigCallee->CallerEdges;
4640	for (auto &CalleeCallerE : OrigCalleeCallerEdges) {
4641	if (CalleeCallerE == CalleeEdge)
4642	continue;
4643	if (!OtherCallersToShareMerge.contains(CalleeCallerE->Caller))
4644	continue;
4645	CallerToMoveCount[CalleeCallerE->Caller]++;
4646	moveEdgeToExistingCalleeClone(Edge: CalleeCallerE, NewCallee: MergeNode,
4647	/NewClone/ false);
4648	}
4649	}
4650	removeNoneTypeCalleeEdges(Node: OrigCallee);
4651	removeNoneTypeCalleeEdges(Node: MergeNode);
4652	}
4653	}
4654	}
4655
4656	// Look for other nodes that have edges to the same set of callee
4657	// clones as the current Node. Those can share the eventual merge node
4658	// (reducing cloning and binary size overhead) iff:
4659	// - they have edges to the same set of callee clones
4660	// - each callee edge reaches a subset of the same allocations as Node's
4661	// corresponding edge to the same callee clone.
4662	// The second requirement is to ensure that we don't undo any of the
4663	// necessary cloning to distinguish contexts with different allocation
4664	// behavior.
4665	// FIXME: This is somewhat conservative, as we really just need to ensure
4666	// that they don't reach the same allocations as contexts on edges from Node
4667	// going to any of the other* callee clones being merged. However, that*
4668	// requires more tracking and checking to get right.
4669	template <typename DerivedCCG, typename FuncTy, typename CallTy>
4670	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
4671	findOtherCallersToShareMerge(
4672	ContextNode *Node,
4673	std::vector<std::shared_ptr<ContextEdge>> &CalleeEdges,
4674	DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode,
4675	DenseSet<ContextNode *> &OtherCallersToShareMerge) {
4676	auto NumCalleeClones = CalleeEdges.size();
4677	// This map counts how many edges to the same callee clone exist for other
4678	// caller nodes of each callee clone.
4679	DenseMap<ContextNode , unsigned*> OtherCallersToSharedCalleeEdgeCount;
4680	// Counts the number of other caller nodes that have edges to all callee
4681	// clones that don't violate the allocation context checking.
4682	unsigned PossibleOtherCallerNodes = `0`;
4683
4684	// We only need to look at other Caller nodes if the first callee edge has
4685	// multiple callers (recall they are sorted in ascending order above).
4686	if (CalleeEdges[`0`]->Callee->CallerEdges.size() < `2`)
4687	return;
4688
4689	// For each callee edge:
4690	// - Collect the count of other caller nodes calling the same callees.
4691	// - Collect the alloc nodes reached by contexts on each callee edge.
4692	DenseMap<ContextEdge , DenseSet<ContextNode >> CalleeEdgeToAllocNodes;
4693	for (auto CalleeEdge : CalleeEdges) {
4694	assert(CalleeEdge->Callee->CallerEdges.size() > `1`);
4695	// For each other caller of the same callee, increment the count of
4696	// edges reaching the same callee clone.
4697	for (auto CalleeCallerEdges : CalleeEdge->Callee->CallerEdges) {
4698	if (CalleeCallerEdges->Caller == Node) {
4699	assert(CalleeCallerEdges == CalleeEdge);
4700	continue;
4701	}
4702	OtherCallersToSharedCalleeEdgeCount[CalleeCallerEdges->Caller]++;
4703	// If this caller edge now reaches all of the same callee clones,
4704	// increment the count of candidate other caller nodes.
4705	if (OtherCallersToSharedCalleeEdgeCount[CalleeCallerEdges->Caller] ==
4706	NumCalleeClones)
4707	PossibleOtherCallerNodes++;
4708	}
4709	// Collect the alloc nodes reached by contexts on each callee edge, for
4710	// later analysis.
4711	for (auto Id : CalleeEdge->getContextIds()) {
4712	auto *Alloc = ContextIdToAllocationNode.lookup(Id);
4713	if (!Alloc) {
4714	// FIXME: unclear why this happens occasionally, presumably
4715	// imperfect graph updates possibly with recursion.
4716	MissingAllocForContextId ++;
4717	continue;
4718	}
4719	CalleeEdgeToAllocNodes[CalleeEdge.get()].insert(Alloc);
4720	}
4721	}
4722
4723	// Now walk the callee edges again, and make sure that for each candidate
4724	// caller node all of its edges to the callees reach the same allocs (or
4725	// a subset) as those along the corresponding callee edge from Node.
4726	for (auto CalleeEdge : CalleeEdges) {
4727	assert(CalleeEdge->Callee->CallerEdges.size() > `1`);
4728	// Stop if we do not have any (more) candidate other caller nodes.
4729	if (!PossibleOtherCallerNodes)
4730	break;
4731	auto &CurCalleeAllocNodes = CalleeEdgeToAllocNodes[CalleeEdge.get()];
4732	// Check each other caller of this callee clone.
4733	for (auto &CalleeCallerE : CalleeEdge->Callee->CallerEdges) {
4734	// Not interested in the callee edge from Node itself.
4735	if (CalleeCallerE == CalleeEdge)
4736	continue;
4737	// Skip any callers that didn't have callee edges to all the same
4738	// callee clones.
4739	if (OtherCallersToSharedCalleeEdgeCount[CalleeCallerE->Caller] !=
4740	NumCalleeClones)
4741	continue;
4742	// Make sure that each context along edge from candidate caller node
4743	// reaches an allocation also reached by this callee edge from Node.
4744	for (auto Id : CalleeCallerE->getContextIds()) {
4745	auto *Alloc = ContextIdToAllocationNode.lookup(Id);
4746	if (!Alloc)
4747	continue;
4748	// If not, simply reset the map entry to 0 so caller is ignored, and
4749	// reduce the count of candidate other caller nodes.
4750	if (!CurCalleeAllocNodes.contains(Alloc)) {
4751	OtherCallersToSharedCalleeEdgeCount[CalleeCallerE->Caller] = `0`;
4752	PossibleOtherCallerNodes--;
4753	break;
4754	}
4755	}
4756	}
4757	}
4758
4759	if (!PossibleOtherCallerNodes)
4760	return;
4761
4762	// Build the set of other caller nodes that can use the same callee merge
4763	// node.
4764	for (auto &[OtherCaller, Count] : OtherCallersToSharedCalleeEdgeCount) {
4765	if (Count != NumCalleeClones)
4766	continue;
4767	OtherCallersToShareMerge.insert(OtherCaller);
4768	}
4769	}
4770
4771	// This method assigns cloned callsites to functions, cloning the functions as
4772	// needed. The assignment is greedy and proceeds roughly as follows:
4773	//
4774	// For each function Func:
4775	// For each call with graph Node having clones:
4776	// Initialize ClonesWorklist to Node and its clones
4777	// Initialize NodeCloneCount to 0
4778	// While ClonesWorklist is not empty:
4779	// Clone = pop front ClonesWorklist
4780	// NodeCloneCount++
4781	// If Func has been cloned less than NodeCloneCount times:
4782	// If NodeCloneCount is 1:
4783	// Assign Clone to original Func
4784	// Continue
4785	// Create a new function clone
4786	// If other callers not assigned to call a function clone yet:
4787	// Assign them to call new function clone
4788	// Continue
4789	// Assign any other caller calling the cloned version to new clone
4790	//
4791	// For each caller of Clone:
4792	// If caller is assigned to call a specific function clone:
4793	// If we cannot assign Clone to that function clone:
4794	// Create new callsite Clone NewClone
4795	// Add NewClone to ClonesWorklist
4796	// Continue
4797	// Assign Clone to existing caller's called function clone
4798	// Else:
4799	// If Clone not already assigned to a function clone:
4800	// Assign to first function clone without assignment
4801	// Assign caller to selected function clone
4802	// For each call with graph Node having clones:
4803	// If number func clones > number call's callsite Node clones:
4804	// Record func CallInfo clones without Node clone in UnassignedCallClones
4805	// For callsite Nodes in DFS order from allocations:
4806	// If IsAllocation:
4807	// Update allocation with alloc type
4808	// Else:
4809	// For Call, all MatchingCalls, and associated UnnassignedCallClones:
4810	// Update call to call recorded callee clone
4811	//
4812	template <typename DerivedCCG, typename FuncTy, typename CallTy>
4813	bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::assignFunctions() {
4814	bool Changed = false;
4815
4816	mergeClones();
4817
4818	// Keep track of the assignment of nodes (callsites) to function clones they
4819	// call.
4820	DenseMap<ContextNode *, FuncInfo> CallsiteToCalleeFuncCloneMap;
4821
4822	// Update caller node to call function version CalleeFunc, by recording the
4823	// assignment in CallsiteToCalleeFuncCloneMap.
4824	auto RecordCalleeFuncOfCallsite = [&](ContextNode *Caller,
4825	const FuncInfo &CalleeFunc) {
4826	assert(Caller->hasCall());
4827	CallsiteToCalleeFuncCloneMap[Caller] = CalleeFunc;
4828	};
4829
4830	// Information for a single clone of this Func.
4831	struct FuncCloneInfo {
4832	// The function clone.
4833	FuncInfo FuncClone;
4834	// Remappings of each call of interest (from original uncloned call to the
4835	// corresponding cloned call in this function clone).
4836	DenseMap<CallInfo, CallInfo> CallMap;
4837	};
4838
4839	// Map to keep track of information needed to update calls in function clones
4840	// when their corresponding callsite node was not itself cloned for that
4841	// function clone. Because of call context pruning (i.e. we only keep as much
4842	// caller information as needed to distinguish hot vs cold), we may not have
4843	// caller edges coming to each callsite node from all possible function
4844	// callers. A function clone may get created for other callsites in the
4845	// function for which there are caller edges that were not pruned. Any other
4846	// callsites in that function clone, which were not themselved cloned for
4847	// that function clone, should get updated the same way as the corresponding
4848	// callsite in the original function (which may call a clone of its callee).
4849	//
4850	// We build this map after completing function cloning for each function, so
4851	// that we can record the information from its call maps before they are
4852	// destructed. The map will be used as we update calls to update any still
4853	// unassigned call clones. Note that we may create new node clones as we clone
4854	// other functions, so later on we check which node clones were still not
4855	// created. To this end, the inner map is a map from function clone number to
4856	// the list of calls cloned for that function (can be more than one due to the
4857	// Node's MatchingCalls array).
4858	//
4859	// The alternative is creating new callsite clone nodes below as we clone the
4860	// function, but that is tricker to get right and likely more overhead.
4861	//
4862	// Inner map is a std::map so sorted by key (clone number), in order to get
4863	// ordered remarks in the full LTO case.
4864	DenseMap<const ContextNode , std::map<unsigned*, SmallVector<CallInfo, `0`>>>
4865	UnassignedCallClones;
4866
4867	// Walk all functions for which we saw calls with memprof metadata, and handle
4868	// cloning for each of its calls.
4869	for (auto &[Func, CallsWithMetadata] : FuncToCallsWithMetadata) {
4870	FuncInfo OrigFunc(Func);
4871	// Map from each clone number of OrigFunc to information about that function
4872	// clone (the function clone FuncInfo and call remappings). The index into
4873	// the vector is the clone number, as function clones are created and
4874	// numbered sequentially.
4875	std::vector<FuncCloneInfo> FuncCloneInfos;
4876	for (auto &Call : CallsWithMetadata) {
4877	ContextNode *Node = getNodeForInst(C: Call);
4878	// Skip call if we do not have a node for it (all uses of its stack ids
4879	// were either on inlined chains or pruned from the MIBs), or if we did
4880	// not create any clones for it.
4881	if (!Node \|\| Node->Clones.empty())
4882	continue;
4883	assert(Node->hasCall() &&
4884	"Not having a call should have prevented cloning");
4885
4886	// Track the assignment of function clones to clones of the current
4887	// callsite Node being handled.
4888	std::map<FuncInfo, ContextNode *> FuncCloneToCurNodeCloneMap;
4889
4890	// Assign callsite version CallsiteClone to function version FuncClone,
4891	// and also assign (possibly cloned) Call to CallsiteClone.
4892	auto AssignCallsiteCloneToFuncClone = [&](const FuncInfo &FuncClone,
4893	CallInfo &Call,
4894	ContextNode *CallsiteClone,
4895	bool IsAlloc) {
4896	// Record the clone of callsite node assigned to this function clone.
4897	FuncCloneToCurNodeCloneMap[FuncClone] = CallsiteClone;
4898
4899	assert(FuncCloneInfos.size() > FuncClone.cloneNo());
4900	DenseMap<CallInfo, CallInfo> &CallMap =
4901	FuncCloneInfos[FuncClone.cloneNo()].CallMap;
4902	CallInfo CallClone(Call);
4903	if (auto It = CallMap.find(Call); It != CallMap.end())
4904	CallClone = It->second;
4905	CallsiteClone->setCall(CallClone);
4906	// Need to do the same for all matching calls.
4907	for (auto &MatchingCall : Node->MatchingCalls) {
4908	CallInfo CallClone(MatchingCall);
4909	if (auto It = CallMap.find(MatchingCall); It != CallMap.end())
4910	CallClone = It->second;
4911	// Updates the call in the list.
4912	MatchingCall = CallClone;
4913	}
4914	};
4915
4916	// Invokes moveEdgeToNewCalleeClone which creates a new clone, and then
4917	// performs the necessary fixups (removing none type edges, and
4918	// importantly, propagating any function call assignment of the original
4919	// node to the new clone).
4920	auto MoveEdgeToNewCalleeCloneAndSetUp =
4921	[&](const std::shared_ptr<ContextEdge> &Edge) {
4922	ContextNode *OrigCallee = Edge->Callee;
4923	ContextNode *NewClone = moveEdgeToNewCalleeClone(Edge);
4924	removeNoneTypeCalleeEdges(Node: NewClone);
4925	assert(NewClone->AllocTypes != (uint8_t)AllocationType::None);
4926	// If the original Callee was already assigned to call a specific
4927	// function version, make sure its new clone is assigned to call
4928	// that same function clone.
4929	if (CallsiteToCalleeFuncCloneMap.count(OrigCallee))
4930	RecordCalleeFuncOfCallsite(
4931	NewClone, CallsiteToCalleeFuncCloneMap[OrigCallee]);
4932	return NewClone;
4933	};
4934
4935	// Keep track of the clones of callsite Node that need to be assigned to
4936	// function clones. This list may be expanded in the loop body below if we
4937	// find additional cloning is required.
4938	std::deque<ContextNode *> ClonesWorklist;
4939	// Ignore original Node if we moved all of its contexts to clones.
4940	if (!Node->emptyContextIds())
4941	ClonesWorklist.push_back(Node);
4942	llvm::append_range(ClonesWorklist, Node->Clones);
4943
4944	// Now walk through all of the clones of this callsite Node that we need,
4945	// and determine the assignment to a corresponding clone of the current
4946	// function (creating new function clones as needed).
4947	unsigned NodeCloneCount = `0`;
4948	while (!ClonesWorklist.empty()) {
4949	ContextNode *Clone = ClonesWorklist.front();
4950	ClonesWorklist.pop_front();
4951	NodeCloneCount++;
4952	if (VerifyNodes)
4953	checkNode<DerivedCCG, FuncTy, CallTy>(Clone);
4954
4955	// Need to create a new function clone if we have more callsite clones
4956	// than existing function clones, which would have been assigned to an
4957	// earlier clone in the list (we assign callsite clones to function
4958	// clones greedily).
4959	if (FuncCloneInfos.size() < NodeCloneCount) {
4960	// If this is the first callsite copy, assign to original function.
4961	if (NodeCloneCount == `1`) {
4962	// Since FuncCloneInfos is empty in this case, no clones have
4963	// been created for this function yet, and no callers should have
4964	// been assigned a function clone for this callee node yet.
4965	assert(llvm::none_of(
4966	Clone->CallerEdges, [&](const std::shared_ptr<ContextEdge> &E) {
4967	return CallsiteToCalleeFuncCloneMap.count(E->Caller);
4968	}));
4969	// Initialize with empty call map, assign Clone to original function
4970	// and its callers, and skip to the next clone.
4971	FuncCloneInfos.push_back(
4972	{OrigFunc, DenseMap<CallInfo, CallInfo>()});
4973	AssignCallsiteCloneToFuncClone(
4974	OrigFunc, Call, Clone,
4975	AllocationCallToContextNodeMap.count(Call));
4976	for (auto &CE : Clone->CallerEdges) {
4977	// Ignore any caller that does not have a recorded callsite Call.
4978	if (!CE->Caller->hasCall())
4979	continue;
4980	RecordCalleeFuncOfCallsite(CE->Caller, OrigFunc);
4981	}
4982	continue;
4983	}
4984
4985	// First locate which copy of OrigFunc to clone again. If a caller
4986	// of this callsite clone was already assigned to call a particular
4987	// function clone, we need to redirect all of those callers to the
4988	// new function clone, and update their other callees within this
4989	// function.
4990	FuncInfo PreviousAssignedFuncClone;
4991	auto EI = llvm::find_if(
4992	Clone->CallerEdges, [&](const std::shared_ptr<ContextEdge> &E) {
4993	return CallsiteToCalleeFuncCloneMap.count(E->Caller);
4994	});
4995	bool CallerAssignedToCloneOfFunc = false;
4996	if (EI != Clone->CallerEdges.end()) {
4997	const std::shared_ptr<ContextEdge> &Edge = *EI;
4998	PreviousAssignedFuncClone =
4999	CallsiteToCalleeFuncCloneMap[Edge->Caller];
5000	CallerAssignedToCloneOfFunc = true;
5001	}
5002
5003	// Clone function and save it along with the CallInfo map created
5004	// during cloning in the FuncCloneInfos.
5005	DenseMap<CallInfo, CallInfo> NewCallMap;
5006	unsigned CloneNo = FuncCloneInfos.size();
5007	assert(CloneNo > `0` && "Clone 0 is the original function, which "
5008	"should already exist in the map");
5009	FuncInfo NewFuncClone = cloneFunctionForCallsite(
5010	Func&: OrigFunc, Call, CallMap&: NewCallMap, CallsWithMetadataInFunc&: CallsWithMetadata, CloneNo);
5011	FuncCloneInfos.push_back({NewFuncClone, std::move(NewCallMap)});
5012	FunctionClonesAnalysis ++;
5013	Changed = true;
5014
5015	// If no caller callsites were already assigned to a clone of this
5016	// function, we can simply assign this clone to the new func clone
5017	// and update all callers to it, then skip to the next clone.
5018	if (!CallerAssignedToCloneOfFunc) {
5019	AssignCallsiteCloneToFuncClone(
5020	NewFuncClone, Call, Clone,
5021	AllocationCallToContextNodeMap.count(Call));
5022	for (auto &CE : Clone->CallerEdges) {
5023	// Ignore any caller that does not have a recorded callsite Call.
5024	if (!CE->Caller->hasCall())
5025	continue;
5026	RecordCalleeFuncOfCallsite(CE->Caller, NewFuncClone);
5027	}
5028	continue;
5029	}
5030
5031	// We may need to do additional node cloning in this case.
5032	// Reset the CallsiteToCalleeFuncCloneMap entry for any callers
5033	// that were previously assigned to call PreviousAssignedFuncClone,
5034	// to record that they now call NewFuncClone.
5035	// The none type edge removal may remove some of this Clone's caller
5036	// edges, if it is reached via another of its caller's callees.
5037	// Iterate over a copy and skip any that were removed.
5038	auto CallerEdges = Clone->CallerEdges;
5039	for (auto CE : CallerEdges) {
5040	// Skip any that have been removed on an earlier iteration.
5041	if (CE->isRemoved()) {
5042	assert(!is_contained(Clone->CallerEdges, CE));
5043	continue;
5044	}
5045	assert(CE);
5046	// Ignore any caller that does not have a recorded callsite Call.
5047	if (!CE->Caller->hasCall())
5048	continue;
5049
5050	if (!CallsiteToCalleeFuncCloneMap.count(CE->Caller) \|\|
5051	// We subsequently fall through to later handling that
5052	// will perform any additional cloning required for
5053	// callers that were calling other function clones.
5054	CallsiteToCalleeFuncCloneMap[CE->Caller] !=
5055	PreviousAssignedFuncClone)
5056	continue;
5057
5058	RecordCalleeFuncOfCallsite(CE->Caller, NewFuncClone);
5059
5060	// If we are cloning a function that was already assigned to some
5061	// callers, then essentially we are creating new callsite clones
5062	// of the other callsites in that function that are reached by those
5063	// callers. Clone the other callees of the current callsite's caller
5064	// that were already assigned to PreviousAssignedFuncClone
5065	// accordingly. This is important since we subsequently update the
5066	// calls from the nodes in the graph and their assignments to callee
5067	// functions recorded in CallsiteToCalleeFuncCloneMap.
5068	// The none type edge removal may remove some of this caller's
5069	// callee edges, if it is reached via another of its callees.
5070	// Iterate over a copy and skip any that were removed.
5071	auto CalleeEdges = CE->Caller->CalleeEdges;
5072	for (auto CalleeEdge : CalleeEdges) {
5073	// Skip any that have been removed on an earlier iteration when
5074	// cleaning up newly None type callee edges.
5075	if (CalleeEdge->isRemoved()) {
5076	assert(!is_contained(CE->Caller->CalleeEdges, CalleeEdge));
5077	continue;
5078	}
5079	assert(CalleeEdge);
5080	ContextNode *Callee = CalleeEdge->Callee;
5081	// Skip the current callsite, we are looking for other
5082	// callsites Caller calls, as well as any that does not have a
5083	// recorded callsite Call.
5084	if (Callee == Clone \|\| !Callee->hasCall())
5085	continue;
5086	// Skip direct recursive calls. We don't need/want to clone the
5087	// caller node again, and this loop will not behave as expected if
5088	// we tried.
5089	if (Callee == CalleeEdge->Caller)
5090	continue;
5091	ContextNode *NewClone =
5092	MoveEdgeToNewCalleeCloneAndSetUp(CalleeEdge);
5093	// Moving the edge may have resulted in some none type
5094	// callee edges on the original Callee.
5095	removeNoneTypeCalleeEdges(Node: Callee);
5096	// Update NewClone with the new Call clone of this callsite's Call
5097	// created for the new function clone created earlier.
5098	// Recall that we have already ensured when building the graph
5099	// that each caller can only call callsites within the same
5100	// function, so we are guaranteed that Callee Call is in the
5101	// current OrigFunc.
5102	// CallMap is set up as indexed by original Call at clone 0.
5103	CallInfo OrigCall(Callee->getOrigNode()->Call);
5104	OrigCall.setCloneNo(`0`);
5105	DenseMap<CallInfo, CallInfo> &CallMap =
5106	FuncCloneInfos[NewFuncClone.cloneNo()].CallMap;
5107	assert(CallMap.count(OrigCall));
5108	CallInfo NewCall(CallMap[OrigCall]);
5109	assert(NewCall);
5110	NewClone->setCall(NewCall);
5111	// Need to do the same for all matching calls.
5112	for (auto &MatchingCall : NewClone->MatchingCalls) {
5113	CallInfo OrigMatchingCall(MatchingCall);
5114	OrigMatchingCall.setCloneNo(`0`);
5115	assert(CallMap.count(OrigMatchingCall));
5116	CallInfo NewCall(CallMap[OrigMatchingCall]);
5117	assert(NewCall);
5118	// Updates the call in the list.
5119	MatchingCall = NewCall;
5120	}
5121	}
5122	}
5123	// Fall through to handling below to perform the recording of the
5124	// function for this callsite clone. This enables handling of cases
5125	// where the callers were assigned to different clones of a function.
5126	}
5127
5128	auto FindFirstAvailFuncClone = [&]() {
5129	// Find first function in FuncCloneInfos without an assigned
5130	// clone of this callsite Node. We should always have one
5131	// available at this point due to the earlier cloning when the
5132	// FuncCloneInfos size was smaller than the clone number.
5133	for (auto &CF : FuncCloneInfos) {
5134	if (!FuncCloneToCurNodeCloneMap.count(CF.FuncClone))
5135	return CF.FuncClone;
5136	}
5137	llvm_unreachable(
5138	"Expected an available func clone for this callsite clone");
5139	};
5140
5141	// See if we can use existing function clone. Walk through
5142	// all caller edges to see if any have already been assigned to
5143	// a clone of this callsite's function. If we can use it, do so. If not,
5144	// because that function clone is already assigned to a different clone
5145	// of this callsite, then we need to clone again.
5146	// Basically, this checking is needed to handle the case where different
5147	// caller functions/callsites may need versions of this function
5148	// containing different mixes of callsite clones across the different
5149	// callsites within the function. If that happens, we need to create
5150	// additional function clones to handle the various combinations.
5151	//
5152	// Keep track of any new clones of this callsite created by the
5153	// following loop, as well as any existing clone that we decided to
5154	// assign this clone to.
5155	std::map<FuncInfo, ContextNode *> FuncCloneToNewCallsiteCloneMap;
5156	FuncInfo FuncCloneAssignedToCurCallsiteClone;
5157	// Iterate over a copy of Clone's caller edges, since we may need to
5158	// remove edges in the moveEdgeTo methods, and this simplifies the*
5159	// handling and makes it less error-prone.
5160	auto CloneCallerEdges = Clone->CallerEdges;
5161	for (auto &Edge : CloneCallerEdges) {
5162	// Skip removed edges (due to direct recursive edges updated when
5163	// updating callee edges when moving an edge and subsequently
5164	// removed by call to removeNoneTypeCalleeEdges on the Clone).
5165	if (Edge->isRemoved())
5166	continue;
5167	// Ignore any caller that does not have a recorded callsite Call.
5168	if (!Edge->Caller->hasCall())
5169	continue;
5170	// If this caller already assigned to call a version of OrigFunc, need
5171	// to ensure we can assign this callsite clone to that function clone.
5172	if (CallsiteToCalleeFuncCloneMap.count(Edge->Caller)) {
5173	FuncInfo FuncCloneCalledByCaller =
5174	CallsiteToCalleeFuncCloneMap[Edge->Caller];
5175	// First we need to confirm that this function clone is available
5176	// for use by this callsite node clone.
5177	//
5178	// While FuncCloneToCurNodeCloneMap is built only for this Node and
5179	// its callsite clones, one of those callsite clones X could have
5180	// been assigned to the same function clone called by Edge's caller
5181	// - if Edge's caller calls another callsite within Node's original
5182	// function, and that callsite has another caller reaching clone X.
5183	// We need to clone Node again in this case.
5184	if ((FuncCloneToCurNodeCloneMap.count(FuncCloneCalledByCaller) &&
5185	FuncCloneToCurNodeCloneMap[FuncCloneCalledByCaller] !=
5186	Clone) \|\|
5187	// Detect when we have multiple callers of this callsite that
5188	// have already been assigned to specific, and different, clones
5189	// of OrigFunc (due to other unrelated callsites in Func they
5190	// reach via call contexts). Is this Clone of callsite Node
5191	// assigned to a different clone of OrigFunc? If so, clone Node
5192	// again.
5193	(FuncCloneAssignedToCurCallsiteClone &&
5194	FuncCloneAssignedToCurCallsiteClone !=
5195	FuncCloneCalledByCaller)) {
5196	// We need to use a different newly created callsite clone, in
5197	// order to assign it to another new function clone on a
5198	// subsequent iteration over the Clones array (adjusted below).
5199	// Note we specifically do not reset the
5200	// CallsiteToCalleeFuncCloneMap entry for this caller, so that
5201	// when this new clone is processed later we know which version of
5202	// the function to copy (so that other callsite clones we have
5203	// assigned to that function clone are properly cloned over). See
5204	// comments in the function cloning handling earlier.
5205
5206	// Check if we already have cloned this callsite again while
5207	// walking through caller edges, for a caller calling the same
5208	// function clone. If so, we can move this edge to that new clone
5209	// rather than creating yet another new clone.
5210	if (FuncCloneToNewCallsiteCloneMap.count(
5211	FuncCloneCalledByCaller)) {
5212	ContextNode *NewClone =
5213	FuncCloneToNewCallsiteCloneMap[FuncCloneCalledByCaller];
5214	moveEdgeToExistingCalleeClone(Edge, NewCallee: NewClone);
5215	// Cleanup any none type edges cloned over.
5216	removeNoneTypeCalleeEdges(Node: NewClone);
5217	} else {
5218	// Create a new callsite clone.
5219	ContextNode *NewClone = MoveEdgeToNewCalleeCloneAndSetUp(Edge);
5220	FuncCloneToNewCallsiteCloneMap[FuncCloneCalledByCaller] =
5221	NewClone;
5222	// Add to list of clones and process later.
5223	ClonesWorklist.push_back(NewClone);
5224	}
5225	// Moving the caller edge may have resulted in some none type
5226	// callee edges.
5227	removeNoneTypeCalleeEdges(Node: Clone);
5228	// We will handle the newly created callsite clone in a subsequent
5229	// iteration over this Node's Clones.
5230	continue;
5231	}
5232
5233	// Otherwise, we can use the function clone already assigned to this
5234	// caller.
5235	if (!FuncCloneAssignedToCurCallsiteClone) {
5236	FuncCloneAssignedToCurCallsiteClone = FuncCloneCalledByCaller;
5237	// Assign Clone to FuncCloneCalledByCaller
5238	AssignCallsiteCloneToFuncClone(
5239	FuncCloneCalledByCaller, Call, Clone,
5240	AllocationCallToContextNodeMap.count(Call));
5241	} else
5242	// Don't need to do anything - callsite is already calling this
5243	// function clone.
5244	assert(FuncCloneAssignedToCurCallsiteClone ==
5245	FuncCloneCalledByCaller);
5246
5247	} else {
5248	// We have not already assigned this caller to a version of
5249	// OrigFunc. Do the assignment now.
5250
5251	// First check if we have already assigned this callsite clone to a
5252	// clone of OrigFunc for another caller during this iteration over
5253	// its caller edges.
5254	if (!FuncCloneAssignedToCurCallsiteClone) {
5255	FuncCloneAssignedToCurCallsiteClone = FindFirstAvailFuncClone();
5256	assert(FuncCloneAssignedToCurCallsiteClone);
5257	// Assign Clone to FuncCloneAssignedToCurCallsiteClone
5258	AssignCallsiteCloneToFuncClone(
5259	FuncCloneAssignedToCurCallsiteClone, Call, Clone,
5260	AllocationCallToContextNodeMap.count(Call));
5261	} else
5262	assert(FuncCloneToCurNodeCloneMap
5263	[FuncCloneAssignedToCurCallsiteClone] == Clone);
5264	// Update callers to record function version called.
5265	RecordCalleeFuncOfCallsite(Edge->Caller,
5266	FuncCloneAssignedToCurCallsiteClone);
5267	}
5268	}
5269	// If we didn't assign a function clone to this callsite clone yet, e.g.
5270	// none of its callers has a non-null call, do the assignment here.
5271	// We want to ensure that every callsite clone is assigned to some
5272	// function clone, so that the call updates below work as expected.
5273	// In particular if this is the original callsite, we want to ensure it
5274	// is assigned to the original function, otherwise the original function
5275	// will appear available for assignment to other callsite clones,
5276	// leading to unintended effects. For one, the unknown and not updated
5277	// callers will call into cloned paths leading to the wrong hints,
5278	// because they still call the original function (clone 0). Also,
5279	// because all callsites start out as being clone 0 by default, we can't
5280	// easily distinguish between callsites explicitly assigned to clone 0
5281	// vs those never assigned, which can lead to multiple updates of the
5282	// calls when invoking updateCall below, with mismatched clone values.
5283	// TODO: Add a flag to the callsite nodes or some other mechanism to
5284	// better distinguish and identify callsite clones that are not getting
5285	// assigned to function clones as expected.
5286	if (!FuncCloneAssignedToCurCallsiteClone) {
5287	FuncCloneAssignedToCurCallsiteClone = FindFirstAvailFuncClone();
5288	assert(FuncCloneAssignedToCurCallsiteClone &&
5289	"No available func clone for this callsite clone");
5290	AssignCallsiteCloneToFuncClone(
5291	FuncCloneAssignedToCurCallsiteClone, Call, Clone,
5292	/IsAlloc=/AllocationCallToContextNodeMap.contains(Call));
5293	}
5294	}
5295	if (VerifyCCG) {
5296	checkNode<DerivedCCG, FuncTy, CallTy>(Node);
5297	for (const auto &PE : Node->CalleeEdges)
5298	checkNode<DerivedCCG, FuncTy, CallTy>(PE->Callee);
5299	for (const auto &CE : Node->CallerEdges)
5300	checkNode<DerivedCCG, FuncTy, CallTy>(CE->Caller);
5301	for (auto *Clone : Node->Clones) {
5302	checkNode<DerivedCCG, FuncTy, CallTy>(Clone);
5303	for (const auto &PE : Clone->CalleeEdges)
5304	checkNode<DerivedCCG, FuncTy, CallTy>(PE->Callee);
5305	for (const auto &CE : Clone->CallerEdges)
5306	checkNode<DerivedCCG, FuncTy, CallTy>(CE->Caller);
5307	}
5308	}
5309	}
5310
5311	if (FuncCloneInfos.size() < `2`)
5312	continue;
5313
5314	// In this case there is more than just the original function copy.
5315	// Record call clones of any callsite nodes in the function that did not
5316	// themselves get cloned for all of the function clones.
5317	for (auto &Call : CallsWithMetadata) {
5318	ContextNode *Node = getNodeForInst(C: Call);
5319	if (!Node \|\| !Node->hasCall() \|\| Node->emptyContextIds())
5320	continue;
5321	// If Node has enough clones already to cover all function clones, we can
5322	// skip it. Need to add one for the original copy.
5323	// Use >= in case there were clones that were skipped due to having empty
5324	// context ids
5325	if (Node->Clones.size() + `1` >= FuncCloneInfos.size())
5326	continue;
5327	// First collect all function clones we cloned this callsite node for.
5328	// They may not be sequential due to empty clones e.g.
5329	DenseSet<unsigned> NodeCallClones;
5330	for (auto *C : Node->Clones)
5331	NodeCallClones.insert(C->Call.cloneNo());
5332	unsigned I = `0`;
5333	// Now check all the function clones.
5334	for (auto &FC : FuncCloneInfos) {
5335	// Function clones should be sequential.
5336	assert(FC.FuncClone.cloneNo() == I);
5337	// Skip the first clone which got the original call.
5338	// Also skip any other clones created for this Node.
5339	if (++I == `1` \|\| NodeCallClones.contains(V: I)) {
5340	continue;
5341	}
5342	// Record the call clones created for this callsite in this function
5343	// clone.
5344	auto &CallVector = UnassignedCallClones[Node][I];
5345	DenseMap<CallInfo, CallInfo> &CallMap = FC.CallMap;
5346	if (auto It = CallMap.find(Call); It != CallMap.end()) {
5347	CallInfo CallClone = It->second;
5348	CallVector.push_back(CallClone);
5349	} else {
5350	// All but the original clone (skipped earlier) should have an entry
5351	// for all calls.
5352	assert(false && "Expected to find call in CallMap");
5353	}
5354	// Need to do the same for all matching calls.
5355	for (auto &MatchingCall : Node->MatchingCalls) {
5356	if (auto It = CallMap.find(MatchingCall); It != CallMap.end()) {
5357	CallInfo CallClone = It->second;
5358	CallVector.push_back(CallClone);
5359	} else {
5360	// All but the original clone (skipped earlier) should have an entry
5361	// for all calls.
5362	assert(false && "Expected to find call in CallMap");
5363	}
5364	}
5365	}
5366	}
5367	}
5368
5369	uint8_t BothTypes =
5370	(uint8_t)AllocationType::Cold \| (uint8_t)AllocationType::NotCold;
5371
5372	auto UpdateCalls = [&](ContextNode *Node,
5373	DenseSet<const ContextNode *> &Visited,
5374	auto &&UpdateCalls) {
5375	auto Inserted = Visited.insert(Node);
5376	if (!Inserted.second)
5377	return;
5378
5379	for (auto *Clone : Node->Clones)
5380	UpdateCalls(Clone, Visited, UpdateCalls);
5381
5382	for (auto &Edge : Node->CallerEdges)
5383	UpdateCalls(Edge->Caller, Visited, UpdateCalls);
5384
5385	// Skip if either no call to update, or if we ended up with no context ids
5386	// (we moved all edges onto other clones).
5387	if (!Node->hasCall() \|\| Node->emptyContextIds())
5388	return;
5389
5390	if (Node->IsAllocation) {
5391	auto AT = allocTypeToUse(Node->AllocTypes);
5392	// If the allocation type is ambiguous, and more aggressive hinting
5393	// has been enabled via the MinClonedColdBytePercent flag, see if this
5394	// allocation should be hinted cold anyway because its fraction cold bytes
5395	// allocated is at least the given threshold.
5396	if (Node->AllocTypes == BothTypes && MinClonedColdBytePercent < `100` &&
5397	!ContextIdToContextSizeInfos.empty()) {
5398	uint64_t TotalCold = `0`;
5399	uint64_t Total = `0`;
5400	for (auto Id : Node->getContextIds()) {
5401	auto TypeI = ContextIdToAllocationType.find(Id);
5402	assert(TypeI != ContextIdToAllocationType.end());
5403	auto CSI = ContextIdToContextSizeInfos.find(Id);
5404	if (CSI != ContextIdToContextSizeInfos.end()) {
5405	for (auto &Info : CSI->second) {
5406	Total += Info.TotalSize;
5407	if (TypeI->second == AllocationType::Cold)
5408	TotalCold += Info.TotalSize;
5409	}
5410	}
5411	}
5412	if (TotalCold * `100` >= Total * MinClonedColdBytePercent)
5413	AT = AllocationType::Cold;
5414	}
5415	updateAllocationCall(Call&: Node->Call, AllocType: AT);
5416	assert(Node->MatchingCalls.empty());
5417	return;
5418	}
5419
5420	if (!CallsiteToCalleeFuncCloneMap.count(Node))
5421	return;
5422
5423	auto CalleeFunc = CallsiteToCalleeFuncCloneMap[Node];
5424	updateCall(CallerCall&: Node->Call, CalleeFunc);
5425	// Update all the matching calls as well.
5426	for (auto &Call : Node->MatchingCalls)
5427	updateCall(CallerCall&: Call, CalleeFunc);
5428
5429	// Now update all calls recorded earlier that are still in function clones
5430	// which don't have a clone of this callsite node.
5431	if (!UnassignedCallClones.contains(Node))
5432	return;
5433	DenseSet<unsigned> NodeCallClones;
5434	for (auto *C : Node->Clones)
5435	NodeCallClones.insert(C->Call.cloneNo());
5436	// Note that we already confirmed Node is in this map a few lines above.
5437	auto &ClonedCalls = UnassignedCallClones[Node];
5438	for (auto &[CloneNo, CallVector] : ClonedCalls) {
5439	// Should start at 1 as we never create an entry for original node.
5440	assert(CloneNo > `0`);
5441	// If we subsequently created a clone, skip this one.
5442	if (NodeCallClones.contains(V: CloneNo))
5443	continue;
5444	// Use the original Node's CalleeFunc.
5445	for (auto &Call : CallVector)
5446	updateCall(CallerCall&: Call, CalleeFunc);
5447	}
5448	};
5449
5450	// Performs DFS traversal starting from allocation nodes to update calls to
5451	// reflect cloning decisions recorded earlier. For regular LTO this will
5452	// update the actual calls in the IR to call the appropriate function clone
5453	// (and add attributes to allocation calls), whereas for ThinLTO the decisions
5454	// are recorded in the summary entries.
5455	DenseSet<const ContextNode *> Visited;
5456	for (auto &Entry : AllocationCallToContextNodeMap)
5457	UpdateCalls(Entry.second, Visited, UpdateCalls);
5458
5459	return Changed;
5460	}
5461
5462	// Compute a SHA1 hash of the callsite and alloc version information of clone I
5463	// in the summary, to use in detection of duplicate clones.
5464	uint64_t ComputeHash(const FunctionSummary FS, unsigned* I) {
5465	SHA1 Hasher;
5466	// Update hash with any callsites that call non-default (non-zero) callee
5467	// versions.
5468	for (auto &SN : FS->callsites()) {
5469	// In theory all callsites and allocs in this function should have the same
5470	// number of clone entries, but handle any discrepancies gracefully below
5471	// for NDEBUG builds.
5472	assert(
5473	SN.Clones.size() > I &&
5474	"Callsite summary has fewer entries than other summaries in function");
5475	if (SN.Clones.size() <= I \|\| !SN.Clones [I])
5476	continue;
5477	uint8_t Data[sizeof(SN.Clones [I])];
5478	support::endian::write32le(P: Data, V: SN.Clones [I]);
5479	Hasher.update(Data);
5480	}
5481	// Update hash with any allocs that have non-default (non-None) hints.
5482	for (auto &AN : FS->allocs()) {
5483	// In theory all callsites and allocs in this function should have the same
5484	// number of clone entries, but handle any discrepancies gracefully below
5485	// for NDEBUG builds.
5486	assert(AN.Versions.size() > I &&
5487	"Alloc summary has fewer entries than other summaries in function");
5488	if (AN.Versions.size() <= I \|\|
5489	(AllocationType)AN.Versions [I] == AllocationType::None)
5490	continue;
5491	Hasher.update(Data: ArrayRef<uint8_t>(&AN.Versions [I], `1`));
5492	}
5493	return support::endian::read64le(P: Hasher.result().data());
5494	}
5495
5496	static SmallVector<std::unique_ptr<ValueToValueMapTy>, `4`> createFunctionClones(
5497	Function &F, unsigned NumClones, Module &M, OptimizationRemarkEmitter &ORE,
5498	std::map<const Function , SmallPtrSet<const* GlobalAlias *, `1`>>
5499	&FuncToAliasMap,
5500	FunctionSummary *FS) {
5501	auto TakeDeclNameAndReplace = [](GlobalValue DeclGV, GlobalValue NewGV) {
5502	// We might have created this when adjusting callsite in another
5503	// function. It should be a declaration.
5504	assert(DeclGV->isDeclaration());
5505	NewGV->takeName(V: DeclGV);
5506	DeclGV->replaceAllUsesWith(V: NewGV);
5507	DeclGV->eraseFromParent();
5508	};
5509
5510	// Handle aliases to this function, and create analogous alias clones to the
5511	// provided clone of this function.
5512	auto CloneFuncAliases = [&](Function NewF, unsigned* I) {
5513	if (!FuncToAliasMap.count(x: &F))
5514	return;
5515	for (auto *A : FuncToAliasMap [&F]) {
5516	std::string AliasName = getMemProfFuncName(Base: A->getName(), CloneNo: I);
5517	auto *PrevA = M.getNamedAlias(Name: AliasName);
5518	auto *NewA = GlobalAlias::create(Ty: A->getValueType(),
5519	AddressSpace: A->getType()->getPointerAddressSpace(),
5520	Linkage: A->getLinkage(), Name: AliasName, Aliasee: NewF);
5521	NewA->copyAttributesFrom(Src: A);
5522	if (PrevA)
5523	TakeDeclNameAndReplace (PrevA, NewA);
5524	}
5525	};
5526
5527	// The first "clone" is the original copy, we should only call this if we
5528	// needed to create new clones.
5529	assert(NumClones > `1`);
5530	SmallVector<std::unique_ptr<ValueToValueMapTy>, `4`> VMaps;
5531	VMaps.reserve(N: NumClones - `1`);
5532	FunctionsClonedThinBackend ++;
5533
5534	// Map of hash of callsite/alloc versions to the instantiated function clone
5535	// (possibly the original) implementing those calls. Used to avoid
5536	// instantiating duplicate function clones.
5537	// FIXME: Ideally the thin link would not generate such duplicate clones to
5538	// start with, but right now it happens due to phase ordering in the function
5539	// assignment and possible new clones that produces. We simply make each
5540	// duplicate an alias to the matching instantiated clone recorded in the map
5541	// (except for available_externally which are made declarations as they would
5542	// be aliases in the prevailing module, and available_externally aliases are
5543	// not well supported right now).
5544	DenseMap<uint64_t, Function *> HashToFunc;
5545
5546	// Save the hash of the original function version.
5547	HashToFunc [ComputeHash(FS, I: `0`)] = &F;
5548
5549	for (unsigned I = `1`; I < NumClones; I++) {
5550	VMaps.emplace_back(Args: std::make_unique<ValueToValueMapTy>());
5551	std::string Name = getMemProfFuncName(Base: F.getName(), CloneNo: I);
5552	auto Hash = ComputeHash(FS, I);
5553	// If this clone would duplicate a previously seen clone, don't generate the
5554	// duplicate clone body, just make an alias to satisfy any (potentially
5555	// cross-module) references.
5556	if (HashToFunc.contains(Val: Hash)) {
5557	FunctionCloneDuplicatesThinBackend ++;
5558	auto *Func = HashToFunc [Hash];
5559	if (Func->hasAvailableExternallyLinkage()) {
5560	// Skip these as EliminateAvailableExternallyPass does not handle
5561	// available_externally aliases correctly and we end up with an
5562	// available_externally alias to a declaration. Just create a
5563	// declaration for now as we know we will have a definition in another
5564	// module.
5565	auto Decl = M.getOrInsertFunction(Name, T: Func->getFunctionType());
5566	ORE.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofClone", &F)
5567	<< "created clone decl " << ore::NV ("Decl", Decl.getCallee()));
5568	continue;
5569	}
5570	auto *PrevF = M.getFunction(Name);
5571	auto *Alias = GlobalAlias::create(Name, Aliasee: Func);
5572	if (PrevF)
5573	TakeDeclNameAndReplace (PrevF, Alias);
5574	ORE.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofClone", &F)
5575	<< "created clone alias " << ore::NV ("Alias", Alias));
5576
5577	// Now handle aliases to this function, and clone those as well.
5578	CloneFuncAliases (Func, I);
5579	continue;
5580	}
5581	auto NewF = CloneFunction(F: &F, VMap&: VMaps.back());
5582	HashToFunc [Hash] = NewF;
5583	FunctionClonesThinBackend ++;
5584	// Strip memprof and callsite metadata from clone as they are no longer
5585	// needed.
5586	for (auto &BB : *NewF) {
5587	for (auto &Inst : BB) {
5588	Inst.setMetadata(KindID: LLVMContext::MD_memprof, Node: nullptr);
5589	Inst.setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
5590	}
5591	}
5592	auto *PrevF = M.getFunction(Name);
5593	if (PrevF)
5594	TakeDeclNameAndReplace (PrevF, NewF);
5595	else
5596	NewF->setName(Name);
5597	updateSubprogramLinkageName(NewFunc: NewF, Name);
5598	ORE.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofClone", &F)
5599	<< "created clone " << ore::NV ("NewFunction", NewF));
5600
5601	// Now handle aliases to this function, and clone those as well.
5602	CloneFuncAliases (NewF, I);
5603	}
5604	return VMaps;
5605	}
5606
5607	// Locate the summary for F. This is complicated by the fact that it might
5608	// have been internalized or promoted.
5609	static ValueInfo findValueInfoForFunc(const Function &F, const Module &M,
5610	const ModuleSummaryIndex *ImportSummary,
5611	const Function CallingFunc = nullptr*) {
5612	// FIXME: Ideally we would retain the original GUID in some fashion on the
5613	// function (e.g. as metadata), but for now do our best to locate the
5614	// summary without that information.
5615	ValueInfo TheFnVI = ImportSummary->getValueInfo(GUID: F.getGUID());
5616	if (!TheFnVI)
5617	// See if theFn was internalized, by checking index directly with
5618	// original name (this avoids the name adjustment done by getGUID() for
5619	// internal symbols).
5620	TheFnVI = ImportSummary->getValueInfo(
5621	GUID: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: F.getName()));
5622	if (TheFnVI)
5623	return TheFnVI;
5624	// Now query with the original name before any promotion was performed.
5625	StringRef OrigName =
5626	ModuleSummaryIndex::getOriginalNameBeforePromote(Name: F.getName());
5627	// When this pass is enabled, we always add thinlto_src_file provenance
5628	// metadata to imported function definitions, which allows us to recreate the
5629	// original internal symbol's GUID.
5630	auto SrcFileMD = F.getMetadata(Kind: "thinlto_src_file");
5631	// If this is a call to an imported/promoted local for which we didn't import
5632	// the definition, the metadata will not exist on the declaration. However,
5633	// since we are doing this early, before any inlining in the LTO backend, we
5634	// can simply look at the metadata on the calling function which must have
5635	// been from the same module if F was an internal symbol originally.
5636	if (!SrcFileMD && F.isDeclaration()) {
5637	// We would only call this for a declaration for a direct callsite, in which
5638	// case the caller would have provided the calling function pointer.
5639	assert(CallingFunc);
5640	SrcFileMD = CallingFunc->getMetadata(Kind: "thinlto_src_file");
5641	// If this is a promoted local (OrigName != F.getName()), since this is a
5642	// declaration, it must be imported from a different module and therefore we
5643	// should always find the metadata on its calling function. Any call to a
5644	// promoted local that came from this module should still be a definition.
5645	assert(SrcFileMD \|\| OrigName == F.getName());
5646	}
5647	StringRef SrcFile = M.getSourceFileName();
5648	if (SrcFileMD)
5649	SrcFile = dyn_cast<MDString>(Val: SrcFileMD->getOperand(I: `0`))->getString();
5650	std::string OrigId = GlobalValue::getGlobalIdentifier(
5651	Name: OrigName, Linkage: GlobalValue::InternalLinkage, FileName: SrcFile);
5652	TheFnVI = ImportSummary->getValueInfo(
5653	GUID: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigId));
5654	// Internal func in original module may have gotten a numbered suffix if we
5655	// imported an external function with the same name. This happens
5656	// automatically during IR linking for naming conflicts. It would have to
5657	// still be internal in that case (otherwise it would have been renamed on
5658	// promotion in which case we wouldn't have a naming conflict).
5659	if (!TheFnVI && OrigName == F.getName() && F.hasLocalLinkage() &&
5660	F.getName().contains(C: `'.'`)) {
5661	OrigName = F.getName().rsplit(Separator: `'.'`).first;
5662	OrigId = GlobalValue::getGlobalIdentifier(
5663	Name: OrigName, Linkage: GlobalValue::InternalLinkage, FileName: SrcFile);
5664	TheFnVI = ImportSummary->getValueInfo(
5665	GUID: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigId));
5666	}
5667	// The only way we may not have a VI is if this is a declaration created for
5668	// an imported reference. For distributed ThinLTO we may not have a VI for
5669	// such declarations in the distributed summary.
5670	assert(TheFnVI \|\| F.isDeclaration());
5671	return TheFnVI;
5672	}
5673
5674	bool MemProfContextDisambiguation::initializeIndirectCallPromotionInfo(
5675	Module &M) {
5676	ICallAnalysis = std::make_unique<ICallPromotionAnalysis>();
5677	Symtab = std::make_unique<InstrProfSymtab>();
5678	// Don't add canonical names, to avoid multiple functions to the symtab
5679	// when they both have the same root name with "." suffixes stripped.
5680	// If we pick the wrong one then this could lead to incorrect ICP and calling
5681	// a memprof clone that we don't actually create (resulting in linker unsats).
5682	// What this means is that the GUID of the function must* match that in the*
5683	// VP metadata to allow promotion. In practice this should not be a
5684	// limitation, since local functions have a GUID from before they were
5685	// promoted and global function names shouldn't need any special handling
5686	// (they should not get the ".llvm." suffix that the canonicalization*
5687	// handling is attempting to strip).
5688	if (Error E = Symtab ->create(M, /InLTO=/true, /AddCanonical=/false)) {
5689	std::string SymtabFailure = toString(E: std::move(E));
5690	M.getContext().emitError(ErrorStr: "Failed to create symtab: " + SymtabFailure);
5691	return false;
5692	}
5693	return true;
5694	}
5695
5696	#ifndef NDEBUG
5697	// Sanity check that the MIB stack ids match between the summary and
5698	// instruction metadata.
5699	static void checkAllocContextIds(
5700	const AllocInfo &AllocNode, const MDNode *MemProfMD,
5701	const CallStack<MDNode, MDNode::op_iterator> &CallsiteContext,
5702	const ModuleSummaryIndex *ImportSummary) {
5703	auto MIBIter = AllocNode.MIBs.begin();
5704	for (auto &MDOp : MemProfMD->operands()) {
5705	assert(MIBIter != AllocNode.MIBs.end());
5706	// The combined index may have been written without the MIB contexts (see
5707	// -combined-index-memprof-context, which defaults to false in NDEBUG
5708	// builds), e.g. if the thin link was performed by a release build while
5709	// this backend has assertions enabled. Since the MIB context in the summary
5710	// can never be empty after trimming the allocation's callsite context, an
5711	// empty list means the context was not serialized, and there is nothing to
5712	// check.
5713	if (MIBIter->StackIdIndices.empty()) {
5714	MIBIter++;
5715	continue;
5716	}
5717	auto StackIdIndexIter = MIBIter->StackIdIndices.begin();
5718	auto MIBMD = cast<const* MDNode>(MDOp);
5719	MDNode *StackMDNode = getMIBStackNode(MIBMD);
5720	assert(StackMDNode);
5721	CallStack<MDNode, MDNode::op_iterator> StackContext(StackMDNode);
5722	auto ContextIterBegin =
5723	StackContext.beginAfterSharedPrefix(CallsiteContext);
5724	// Skip the checking on the first iteration.
5725	uint64_t LastStackContextId =
5726	(ContextIterBegin != StackContext.end() && *ContextIterBegin == `0`) ? `1`
5727	: `0`;
5728	for (auto ContextIter = ContextIterBegin; ContextIter != StackContext.end();
5729	++ContextIter) {
5730	// If this is a direct recursion, simply skip the duplicate
5731	// entries, to be consistent with how the summary ids were
5732	// generated during ModuleSummaryAnalysis.
5733	if (LastStackContextId == *ContextIter)
5734	continue;
5735	LastStackContextId = *ContextIter;
5736	assert(StackIdIndexIter != MIBIter->StackIdIndices.end());
5737	assert(ImportSummary->getStackIdAtIndex(*StackIdIndexIter) ==
5738	*ContextIter);
5739	StackIdIndexIter++;
5740	}
5741	MIBIter++;
5742	}
5743	}
5744	#endif
5745
5746	bool MemProfContextDisambiguation::applyImport(Module &M) {
5747	assert(ImportSummary);
5748	bool Changed = false;
5749
5750	// We also need to clone any aliases that reference cloned functions, because
5751	// the modified callsites may invoke via the alias. Keep track of the aliases
5752	// for each function.
5753	std::map<const Function , SmallPtrSet<const* GlobalAlias *, `1`>>
5754	FuncToAliasMap;
5755	for (auto &A : M.aliases()) {
5756	auto *Aliasee = A.getAliaseeObject();
5757	if (auto *F = dyn_cast<Function>(Val: Aliasee))
5758	FuncToAliasMap [F].insert(Ptr: &A);
5759	}
5760
5761	if (!initializeIndirectCallPromotionInfo(M))
5762	return false;
5763
5764	for (auto &F : M) {
5765	if (F.isDeclaration() \|\| isMemProfClone(F))
5766	continue;
5767
5768	OptimizationRemarkEmitter ORE(&F);
5769
5770	SmallVector<std::unique_ptr<ValueToValueMapTy>, `4`> VMaps;
5771	bool ClonesCreated = false;
5772	unsigned NumClonesCreated = `0`;
5773	auto CloneFuncIfNeeded = [&](unsigned NumClones, FunctionSummary *FS) {
5774	// We should at least have version 0 which is the original copy.
5775	assert(NumClones > `0`);
5776	// If only one copy needed use original.
5777	if (NumClones == `1`)
5778	return;
5779	// If we already performed cloning of this function, confirm that the
5780	// requested number of clones matches (the thin link should ensure the
5781	// number of clones for each constituent callsite is consistent within
5782	// each function), before returning.
5783	if (ClonesCreated) {
5784	assert(NumClonesCreated == NumClones);
5785	return;
5786	}
5787	VMaps = createFunctionClones(F, NumClones, M, ORE, FuncToAliasMap, FS);
5788	// The first "clone" is the original copy, which doesn't have a VMap.
5789	assert(VMaps.size() == NumClones - `1`);
5790	Changed = true;
5791	ClonesCreated = true;
5792	NumClonesCreated = NumClones;
5793	};
5794
5795	auto CloneCallsite = [&](const CallsiteInfo &StackNode, CallBase *CB,
5796	Function CalledFunction, FunctionSummary FS) {
5797	// Perform cloning if not yet done.
5798	CloneFuncIfNeeded (/NumClones=/StackNode.Clones.size(), FS);
5799
5800	assert(!isMemProfClone(*CalledFunction));
5801
5802	// Because we update the cloned calls by calling setCalledOperand (see
5803	// comment below), out of an abundance of caution make sure the called
5804	// function was actually the called operand (or its aliasee). We also
5805	// strip pointer casts when looking for calls (to match behavior during
5806	// summary generation), however, with opaque pointers in theory this
5807	// should not be an issue. Note we still clone the current function
5808	// (containing this call) above, as that could be needed for its callers.
5809	auto *GA = dyn_cast_or_null<GlobalAlias>(Val: CB->getCalledOperand());
5810	if (CalledFunction != CB->getCalledOperand() &&
5811	(!GA \|\| CalledFunction != GA->getAliaseeObject())) {
5812	SkippedCallsCloning ++;
5813	return;
5814	}
5815	// Update the calls per the summary info.
5816	// Save orig name since it gets updated in the first iteration
5817	// below.
5818	auto CalleeOrigName = CalledFunction->getName();
5819	for (unsigned J = `0`; J < StackNode.Clones.size(); J++) {
5820	// If the VMap is empty, this clone was a duplicate of another and was
5821	// created as an alias or a declaration.
5822	if (J > `0` && VMaps [J - `1`]->empty())
5823	continue;
5824	// Do nothing if this version calls the original version of its
5825	// callee.
5826	if (!StackNode.Clones [J])
5827	continue;
5828	auto NewF = M.getOrInsertFunction(
5829	Name: getMemProfFuncName(Base: CalleeOrigName, CloneNo: StackNode.Clones [J]),
5830	T: CalledFunction->getFunctionType());
5831	CallBase *CBClone;
5832	// Copy 0 is the original function.
5833	if (!J)
5834	CBClone = CB;
5835	else
5836	CBClone = cast<CallBase>(Val&: (*VMaps [J - `1`])[CB]);
5837	// Set the called operand directly instead of calling setCalledFunction,
5838	// as the latter mutates the function type on the call. In rare cases
5839	// we may have a slightly different type on a callee function
5840	// declaration due to it being imported from a different module with
5841	// incomplete types. We really just want to change the name of the
5842	// function to the clone, and not make any type changes.
5843	CBClone->setCalledOperand(NewF.getCallee());
5844	ORE.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofCall", CBClone)
5845	<< ore::NV ("Call", CBClone) << " in clone "
5846	<< ore::NV ("Caller", CBClone->getFunction())
5847	<< " assigned to call function clone "
5848	<< ore::NV ("Callee", NewF.getCallee()));
5849	}
5850	};
5851
5852	// Locate the summary for F.
5853	ValueInfo TheFnVI = findValueInfoForFunc(F, M, ImportSummary);
5854	// If not found, this could be an imported local (see comment in
5855	// findValueInfoForFunc). Skip for now as it will be cloned in its original
5856	// module (where it would have been promoted to global scope so should
5857	// satisfy any reference in this module).
5858	if (!TheFnVI)
5859	continue;
5860
5861	auto *GVSummary =
5862	ImportSummary->findSummaryInModule(VI: TheFnVI, ModuleId: M.getModuleIdentifier());
5863	if (!GVSummary) {
5864	// Must have been imported, use the summary which matches the definition。
5865	// (might be multiple if this was a linkonce_odr).
5866	auto SrcModuleMD = F.getMetadata(Kind: "thinlto_src_module");
5867	assert(SrcModuleMD &&
5868	"enable-import-metadata is needed to emit thinlto_src_module");
5869	StringRef SrcModule =
5870	dyn_cast<MDString>(Val: SrcModuleMD->getOperand(I: `0`))->getString();
5871	for (auto &GVS : TheFnVI.getSummaryList()) {
5872	if (GVS ->modulePath() == SrcModule) {
5873	GVSummary = GVS.get();
5874	break;
5875	}
5876	}
5877	// TODO: Put back the assert once we have metadata on imported copies of
5878	// aliases linking them back to the original alias GUID, which would allow
5879	// us to locate the alias summary here.
5880	// assert(GVSummary && GVSummary->modulePath() == SrcModule);
5881	}
5882
5883	// GVSummary can be null if this is a function imported as a copy of an
5884	// alias, and we don't have the aliasee's summary in our distributed index.
5885	// TODO: Once we can locate the original GUID for imported aliases (e.g. via
5886	// TBD additional metadata), we should find the alias summary instead, and
5887	// we can remove this check and fall back to the original check below.
5888	if (!GVSummary)
5889	continue;
5890
5891	// If this was an imported alias skip it as we won't have the function
5892	// summary, and it should be cloned in the original module.
5893	if (isa<AliasSummary>(Val: GVSummary))
5894	continue;
5895
5896	auto *FS = cast<FunctionSummary>(Val: GVSummary->getBaseObject());
5897
5898	if (FS->allocs().empty() && FS->callsites().empty())
5899	continue;
5900
5901	auto SI = FS->callsites().begin();
5902	auto AI = FS->allocs().begin();
5903
5904	// To handle callsite infos synthesized for tail calls which have missing
5905	// frames in the profiled context, map callee VI to the synthesized callsite
5906	// info.
5907	DenseMap<ValueInfo, CallsiteInfo> MapTailCallCalleeVIToCallsite;
5908	// Iterate the callsites for this function in reverse, since we place all
5909	// those synthesized for tail calls at the end.
5910	for (auto CallsiteIt = FS->callsites().rbegin();
5911	CallsiteIt != FS->callsites().rend(); CallsiteIt ++) {
5912	auto &Callsite = *CallsiteIt;
5913	// Stop as soon as we see a non-synthesized callsite info (see comment
5914	// above loop). All the entries added for discovered tail calls have empty
5915	// stack ids.
5916	if (!Callsite.StackIdIndices.empty())
5917	break;
5918	MapTailCallCalleeVIToCallsite.insert(KV: {Callsite.Callee, Callsite});
5919	}
5920
5921	// Keeps track of needed ICP for the function.
5922	SmallVector<ICallAnalysisData> ICallAnalysisInfo;
5923
5924	// Assume for now that the instructions are in the exact same order
5925	// as when the summary was created, but confirm this is correct by
5926	// matching the stack ids.
5927	for (auto &BB : F) {
5928	for (auto &I : BB) {
5929	auto *CB = dyn_cast<CallBase>(Val: &I);
5930	// Same handling as when creating module summary.
5931	if (!mayHaveMemprofSummary(CB))
5932	continue;
5933
5934	auto *CalledValue = CB->getCalledOperand();
5935	auto *CalledFunction = CB->getCalledFunction();
5936	if (CalledValue && !CalledFunction) {
5937	CalledValue = CalledValue->stripPointerCasts();
5938	// Stripping pointer casts can reveal a called function.
5939	CalledFunction = dyn_cast<Function>(Val: CalledValue);
5940	}
5941	// Check if this is an alias to a function. If so, get the
5942	// called aliasee for the checks below.
5943	if (auto *GA = dyn_cast<GlobalAlias>(Val: CalledValue)) {
5944	assert(!CalledFunction &&
5945	"Expected null called function in callsite for alias");
5946	CalledFunction = dyn_cast<Function>(Val: GA->getAliaseeObject());
5947	}
5948
5949	CallStack<MDNode, MDNode::op_iterator> CallsiteContext(
5950	I.getMetadata(KindID: LLVMContext::MD_callsite));
5951	auto *MemProfMD = I.getMetadata(KindID: LLVMContext::MD_memprof);
5952
5953	// Include allocs that were already assigned a memprof function
5954	// attribute in the statistics. Only do this for those that do not have
5955	// memprof metadata, since we add an "ambiguous" memprof attribute by
5956	// default.
5957	if (CB->getAttributes().hasFnAttr(Kind: "memprof") && !MemProfMD) {
5958	CB->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "cold"
5959	? AllocTypeColdThinBackend ++
5960	: AllocTypeNotColdThinBackend ++;
5961	OrigAllocsThinBackend ++;
5962	AllocVersionsThinBackend ++;
5963	if (!MaxAllocVersionsThinBackend)
5964	MaxAllocVersionsThinBackend = `1`;
5965	continue;
5966	}
5967
5968	if (MemProfMD) {
5969	// Consult the next alloc node.
5970	assert(AI != FS->allocs().end());
5971	auto &AllocNode = *(AI++);
5972
5973	#ifndef NDEBUG
5974	checkAllocContextIds(AllocNode, MemProfMD, CallsiteContext,
5975	ImportSummary);
5976	#endif
5977
5978	// Perform cloning if not yet done.
5979	CloneFuncIfNeeded (/NumClones=/AllocNode.Versions.size(), FS);
5980
5981	OrigAllocsThinBackend ++;
5982	AllocVersionsThinBackend += AllocNode.Versions.size();
5983	if (MaxAllocVersionsThinBackend < AllocNode.Versions.size())
5984	MaxAllocVersionsThinBackend = AllocNode.Versions.size();
5985
5986	// If there is only one version that means we didn't end up
5987	// considering this function for cloning, and in that case the alloc
5988	// will still be none type or should have gotten the default NotCold.
5989	// Skip that after calling clone helper since that does some sanity
5990	// checks that confirm we haven't decided yet that we need cloning.
5991	// We might have a single version that is cold due to the
5992	// MinClonedColdBytePercent heuristic, make sure we don't skip in that
5993	// case.
5994	if (AllocNode.Versions.size() == `1` &&
5995	(AllocationType)AllocNode.Versions [`0`] != AllocationType::Cold) {
5996	assert((AllocationType)AllocNode.Versions[`0`] ==
5997	AllocationType::NotCold \|\|
5998	(AllocationType)AllocNode.Versions[`0`] ==
5999	AllocationType::None);
6000	UnclonableAllocsThinBackend ++;
6001	continue;
6002	}
6003
6004	// All versions should have a singular allocation type.
6005	assert(llvm::none_of(AllocNode.Versions, [](uint8_t Type) {
6006	return Type == ((uint8_t)AllocationType::NotCold \|
6007	(uint8_t)AllocationType::Cold);
6008	}));
6009
6010	// Update the allocation types per the summary info.
6011	for (unsigned J = `0`; J < AllocNode.Versions.size(); J++) {
6012	// If the VMap is empty, this clone was a duplicate of another and
6013	// was created as an alias or a declaration.
6014	if (J > `0` && VMaps [J - `1`]->empty())
6015	continue;
6016	// Ignore any that didn't get an assigned allocation type.
6017	if (AllocNode.Versions [J] == (uint8_t)AllocationType::None)
6018	continue;
6019	AllocationType AllocTy = (AllocationType)AllocNode.Versions [J];
6020	AllocTy == AllocationType::Cold ? AllocTypeColdThinBackend ++
6021	: AllocTypeNotColdThinBackend ++;
6022	std::string AllocTypeString = getAllocTypeAttributeString(Type: AllocTy);
6023	auto A = llvm::Attribute::get(Context&: F.getContext(), Kind: "memprof",
6024	Val: AllocTypeString);
6025	CallBase *CBClone;
6026	// Copy 0 is the original function.
6027	if (!J)
6028	CBClone = CB;
6029	else
6030	// Since VMaps are only created for new clones, we index with
6031	// clone J-1 (J==0 is the original clone and does not have a VMaps
6032	// entry).
6033	CBClone = cast<CallBase>(Val&: (*VMaps [J - `1`])[CB]);
6034	removeAnyExistingAmbiguousAttribute(CB: CBClone);
6035	CBClone->addFnAttr(Attr: A);
6036	ORE.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofAttribute", CBClone)
6037	<< ore::NV ("AllocationCall", CBClone) << " in clone "
6038	<< ore::NV ("Caller", CBClone->getFunction())
6039	<< " marked with memprof allocation attribute "
6040	<< ore::NV ("Attribute", AllocTypeString));
6041	}
6042	} else if (!CallsiteContext.empty()) {
6043	if (!CalledFunction) {
6044	#ifndef NDEBUG
6045	// We should have skipped inline assembly calls.
6046	auto *CI = dyn_cast<CallInst>(CB);
6047	assert(!CI \|\| !CI->isInlineAsm());
6048	#endif
6049	// We should have skipped direct calls via a Constant.
6050	assert(CalledValue && !isa<Constant>(CalledValue));
6051
6052	// This is an indirect call, see if we have profile information and
6053	// whether any clones were recorded for the profiled targets (that
6054	// we synthesized CallsiteInfo summary records for when building the
6055	// index).
6056	auto NumClones =
6057	recordICPInfo(CB, AllCallsites: FS->callsites(), SI, ICallAnalysisInfo);
6058
6059	// Perform cloning if not yet done. This is done here in case
6060	// we don't need to do ICP, but might need to clone this
6061	// function as it is the target of other cloned calls.
6062	if (NumClones)
6063	CloneFuncIfNeeded (NumClones, FS);
6064	}
6065
6066	else {
6067	// Consult the next callsite node.
6068	assert(SI != FS->callsites().end());
6069	auto &StackNode = *(SI++);
6070
6071	#ifndef NDEBUG
6072	// Sanity check that the stack ids match between the summary and
6073	// instruction metadata.
6074	auto StackIdIndexIter = StackNode.StackIdIndices.begin();
6075	for (auto StackId : CallsiteContext) {
6076	assert(StackIdIndexIter != StackNode.StackIdIndices.end());
6077	assert(ImportSummary->getStackIdAtIndex(*StackIdIndexIter) ==
6078	StackId);
6079	StackIdIndexIter++;
6080	}
6081	#endif
6082
6083	CloneCallsite (StackNode, CB, CalledFunction, FS);
6084	}
6085	} else if (CB->isTailCall() && CalledFunction) {
6086	// Locate the synthesized callsite info for the callee VI, if any was
6087	// created, and use that for cloning.
6088	ValueInfo CalleeVI =
6089	findValueInfoForFunc(F: *CalledFunction, M, ImportSummary, CallingFunc: &F);
6090	if (CalleeVI && MapTailCallCalleeVIToCallsite.count(Val: CalleeVI)) {
6091	auto Callsite = MapTailCallCalleeVIToCallsite.find(Val: CalleeVI);
6092	assert(Callsite != MapTailCallCalleeVIToCallsite.end());
6093	CloneCallsite (Callsite ->second, CB, CalledFunction, FS);
6094	}
6095	}
6096	}
6097	}
6098
6099	// Now do any promotion required for cloning.
6100	performICP(M, AllCallsites: FS->callsites(), VMaps, ICallAnalysisInfo, ORE);
6101	}
6102
6103	// We skip some of the functions and instructions above, so remove all the
6104	// metadata in a single sweep here.
6105	for (auto &F : M) {
6106	// We can skip memprof clones because createFunctionClones already strips
6107	// the metadata from the newly created clones.
6108	if (F.isDeclaration() \|\| isMemProfClone(F))
6109	continue;
6110	for (auto &BB : F) {
6111	for (auto &I : BB) {
6112	if (!isa<CallBase>(Val: I))
6113	continue;
6114	I.setMetadata(KindID: LLVMContext::MD_memprof, Node: nullptr);
6115	I.setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
6116	}
6117	}
6118	}
6119
6120	return Changed;
6121	}
6122
6123	unsigned MemProfContextDisambiguation::recordICPInfo(
6124	CallBase *CB, ArrayRef<CallsiteInfo> AllCallsites,
6125	ArrayRef<CallsiteInfo>::iterator &SI,
6126	SmallVector<ICallAnalysisData> &ICallAnalysisInfo) {
6127	// First see if we have profile information for this indirect call.
6128	uint32_t NumCandidates;
6129	uint64_t TotalCount;
6130	auto CandidateProfileData =
6131	ICallAnalysis ->getPromotionCandidatesForInstruction(
6132	I: CB, TotalCount, NumCandidates, MaxNumValueData: MaxSummaryIndirectEdges);
6133	if (CandidateProfileData.empty())
6134	return `0`;
6135
6136	// Iterate through all of the candidate profiled targets along with the
6137	// CallsiteInfo summary records synthesized for them when building the index,
6138	// and see if any are cloned and/or refer to clones.
6139	bool ICPNeeded = false;
6140	unsigned NumClones = `0`;
6141	size_t CallsiteInfoStartIndex = std::distance(first: AllCallsites.begin(), last: SI);
6142	for (const auto &Candidate : CandidateProfileData) {
6143	#ifndef NDEBUG
6144	auto CalleeValueInfo =
6145	#endif
6146	ImportSummary->getValueInfo(GUID: Candidate.Value);
6147	// We might not have a ValueInfo if this is a distributed
6148	// ThinLTO backend and decided not to import that function. The thin link
6149	// may also have updated the callee from the original name GUID recorded in
6150	// the VP metadata to the GUID of the local target function (see
6151	// updateValueInfoForIndirectCalls).
6152	assert(!CalleeValueInfo \|\| SI->Callee == CalleeValueInfo \|\|
6153	(SI->Callee &&
6154	SI->Callee.getGUID() ==
6155	ImportSummary->getGUIDFromOriginalID(Candidate.Value)));
6156	assert(SI != AllCallsites.end());
6157	auto &StackNode = *(SI++);
6158	// See if any of the clones of the indirect callsite for this
6159	// profiled target should call a cloned version of the profiled
6160	// target. We only need to do the ICP here if so.
6161	ICPNeeded \|= llvm::any_of(Range: StackNode.Clones,
6162	P: [](unsigned CloneNo) { return CloneNo != `0`; });
6163	// Every callsite in the same function should have been cloned the same
6164	// number of times.
6165	assert(!NumClones \|\| NumClones == StackNode.Clones.size());
6166	NumClones = StackNode.Clones.size();
6167	}
6168	if (!ICPNeeded)
6169	return NumClones;
6170	// Save information for ICP, which is performed later to avoid messing up the
6171	// current function traversal.
6172	ICallAnalysisInfo.push_back(Elt: {.CB: CB, .CandidateProfileData: CandidateProfileData.vec(), .NumCandidates: NumCandidates,
6173	.TotalCount: TotalCount, .CallsiteInfoStartIndex: CallsiteInfoStartIndex});
6174	return NumClones;
6175	}
6176
6177	void MemProfContextDisambiguation::performICP(
6178	Module &M, ArrayRef<CallsiteInfo> AllCallsites,
6179	ArrayRef<std::unique_ptr<ValueToValueMapTy>> VMaps,
6180	ArrayRef<ICallAnalysisData> ICallAnalysisInfo,
6181	OptimizationRemarkEmitter &ORE) {
6182	// Now do any promotion required for cloning. Specifically, for each
6183	// recorded ICP candidate (which was only recorded because one clone of that
6184	// candidate should call a cloned target), we perform ICP (speculative
6185	// devirtualization) for each clone of the callsite, and update its callee
6186	// to the appropriate clone. Note that the ICP compares against the original
6187	// version of the target, which is what is in the vtable.
6188	for (auto &Info : ICallAnalysisInfo) {
6189	auto *CB = Info.CB;
6190	auto CallsiteIndex = Info.CallsiteInfoStartIndex;
6191	auto TotalCount = Info.TotalCount;
6192	unsigned NumClones = `0`;
6193	SmallVector<InstrProfValueData, `8`> RemainingCandidates;
6194
6195	for (auto &Candidate : Info.CandidateProfileData) {
6196	auto &StackNode = AllCallsites [CallsiteIndex++];
6197
6198	// All calls in the same function must have the same number of clones.
6199	assert(!NumClones \|\| NumClones == StackNode.Clones.size());
6200	NumClones = StackNode.Clones.size();
6201
6202	// See if the target is in the module. If it wasn't imported, it is
6203	// possible that this profile could have been collected on a different
6204	// target (or version of the code), and we need to be conservative
6205	// (similar to what is done in the ICP pass).
6206	Function *TargetFunction = Symtab ->getFunction(FuncMD5Hash: Candidate.Value);
6207	// If not found, try the GUID of the callee recorded in the summary. For
6208	// local targets the VP metadata may contain the GUID of the original
6209	// name without the source file prefix (e.g. if it was synthesized from
6210	// the MemProf profile's callee GUIDs or from a SamplePGO profile), which
6211	// the thin link translates to the actual GUID of the local function
6212	// (see updateValueInfoForIndirectCalls). The latter is recorded in the
6213	// function's !guid metadata, which is preserved across promotion and
6214	// importing, and is added to the Symtab.
6215	if (!TargetFunction && StackNode.Callee &&
6216	StackNode.Callee.getGUID() != Candidate.Value)
6217	TargetFunction = Symtab ->getFunction(FuncMD5Hash: StackNode.Callee.getGUID());
6218	if (TargetFunction == nullptr \|\|
6219	// Any ThinLTO global dead symbol removal should have already
6220	// occurred, so it should be safe to promote when the target is a
6221	// declaration.
6222	// TODO: Remove internal option once more fully tested.
6223	(MemProfRequireDefinitionForPromotion &&
6224	TargetFunction->isDeclaration())) {
6225	ORE.emit(RemarkBuilder: [&]() {
6226	return OptimizationRemarkMissed (DEBUG_TYPE, "UnableToFindTarget", CB)
6227	<< "Memprof cannot promote indirect call: target with md5sum "
6228	<< ore::NV ("target md5sum", Candidate.Value) << " not found";
6229	});
6230	// FIXME: See if we can use the new declaration importing support to
6231	// at least get the declarations imported for this case. Hot indirect
6232	// targets should have been imported normally, however.
6233	RemainingCandidates.push_back(Elt: Candidate);
6234	continue;
6235	}
6236
6237	// Check if legal to promote
6238	const char Reason = nullptr*;
6239	if (!isLegalToPromote(CB: *CB, Callee: TargetFunction, FailureReason: &Reason)) {
6240	ORE.emit(RemarkBuilder: [&]() {
6241	return OptimizationRemarkMissed (DEBUG_TYPE, "UnableToPromote", CB)
6242	<< "Memprof cannot promote indirect call to "
6243	<< ore::NV ("TargetFunction", TargetFunction)
6244	<< " with count of " << ore::NV ("TotalCount", TotalCount)
6245	<< ": " << Reason;
6246	});
6247	RemainingCandidates.push_back(Elt: Candidate);
6248	continue;
6249	}
6250
6251	assert(!isMemProfClone(*TargetFunction));
6252
6253	// Handle each call clone, applying ICP so that each clone directly
6254	// calls the specified callee clone, guarded by the appropriate ICP
6255	// check.
6256	CallBase *CBClone = CB;
6257	for (unsigned J = `0`; J < NumClones; J++) {
6258	// If the VMap is empty, this clone was a duplicate of another and was
6259	// created as an alias or a declaration.
6260	if (J > `0` && VMaps [J - `1`]->empty())
6261	continue;
6262	// Copy 0 is the original function.
6263	if (J > `0`)
6264	CBClone = cast<CallBase>(Val&: (*VMaps [J - `1`])[CB]);
6265	// We do the promotion using the original name, so that the comparison
6266	// is against the name in the vtable. Then just below, change the new
6267	// direct call to call the cloned function.
6268	auto &DirectCall =
6269	pgo::promoteIndirectCall(CB&: *CBClone, F: TargetFunction, Count: Candidate.Count,
6270	TotalCount, AttachProfToDirectCall: isSamplePGO, ORE: &ORE);
6271	auto *TargetToUse = TargetFunction;
6272	// Call original if this version calls the original version of its
6273	// callee.
6274	if (StackNode.Clones [J]) {
6275	TargetToUse =
6276	cast<Function>(Val: M.getOrInsertFunction(
6277	Name: getMemProfFuncName(Base: TargetFunction->getName(),
6278	CloneNo: StackNode.Clones [J]),
6279	T: TargetFunction->getFunctionType())
6280	.getCallee());
6281	}
6282	DirectCall.setCalledFunction(TargetToUse);
6283	// During matching we generate synthetic VP metadata for indirect calls
6284	// not already having any, from the memprof profile's callee GUIDs. If
6285	// we subsequently promote and inline those callees, we currently lose
6286	// the ability to generate this synthetic VP metadata. Optionally apply
6287	// a noinline attribute to promoted direct calls, where the threshold is
6288	// set to capture synthetic VP metadata targets which get a count of 1.
6289	if (MemProfICPNoInlineThreshold &&
6290	Candidate.Count < MemProfICPNoInlineThreshold)
6291	DirectCall.setIsNoInline();
6292	ORE.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofCall", CBClone)
6293	<< ore::NV ("Call", CBClone) << " in clone "
6294	<< ore::NV ("Caller", CBClone->getFunction())
6295	<< " promoted and assigned to call function clone "
6296	<< ore::NV ("Callee", TargetToUse));
6297	}
6298
6299	// Update TotalCount (all clones should get same count above)
6300	TotalCount -= Candidate.Count;
6301	}
6302	// Adjust the MD.prof metadata for all clones, now that we have the new
6303	// TotalCount and the remaining candidates.
6304	CallBase *CBClone = CB;
6305	for (unsigned J = `0`; J < NumClones; J++) {
6306	// If the VMap is empty, this clone was a duplicate of another and was
6307	// created as an alias or a declaration.
6308	if (J > `0` && VMaps [J - `1`]->empty())
6309	continue;
6310	// Copy 0 is the original function.
6311	if (J > `0`)
6312	CBClone = cast<CallBase>(Val&: (*VMaps [J - `1`])[CB]);
6313	// First delete the old one.
6314	CBClone->setMetadata(KindID: LLVMContext::MD_prof, Node: nullptr);
6315	// If all promoted, we don't need the MD.prof metadata.
6316	// Otherwise we need update with the un-promoted records back.
6317	if (TotalCount != `0`)
6318	annotateValueSite(M, Inst&: *CBClone, VDs: RemainingCandidates, Sum: TotalCount,
6319	ValueKind: IPVK_IndirectCallTarget, MaxMDCount: Info.NumCandidates);
6320	}
6321	}
6322	}
6323
6324	template <typename DerivedCCG, typename FuncTy, typename CallTy>
6325	bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::process(
6326	function_ref<void(StringRef, StringRef, const Twine &)> EmitRemark,
6327	bool AllowExtraAnalysis) {
6328	if (DumpCCG) {
6329	dbgs() << "CCG before cloning:\n";
6330	dbgs() << *this;
6331	}
6332	if (ExportToDot)
6333	exportToDot(Label: "postbuild");
6334
6335	if (VerifyCCG) {
6336	check();
6337	}
6338
6339	identifyClones();
6340
6341	if (VerifyCCG) {
6342	check();
6343	}
6344
6345	if (DumpCCG) {
6346	dbgs() << "CCG after cloning:\n";
6347	dbgs() << *this;
6348	}
6349	if (ExportToDot)
6350	exportToDot(Label: "cloned");
6351
6352	bool Changed = assignFunctions();
6353
6354	if (DumpCCG) {
6355	dbgs() << "CCG after assigning function clones:\n";
6356	dbgs() << *this;
6357	}
6358	if (ExportToDot)
6359	exportToDot(Label: "clonefuncassign");
6360
6361	if (MemProfReportHintedSizes \|\| AllowExtraAnalysis)
6362	printTotalSizes(OS&: errs(), EmitRemark);
6363
6364	return Changed;
6365	}
6366
6367	bool MemProfContextDisambiguation::processModule(
6368	Module &M,
6369	llvm::function_ref<OptimizationRemarkEmitter &(Function *)> OREGetter) {
6370
6371	// If we have an import summary, then the cloning decisions were made during
6372	// the thin link on the index. Apply them and return.
6373	if (ImportSummary)
6374	return applyImport(M);
6375
6376	// TODO: If/when other types of memprof cloning are enabled beyond just for
6377	// hot and cold, we will need to change this to individually control the
6378	// AllocationType passed to addStackNodesForMIB during CCG construction.
6379	// Note that we specifically check this after applying imports above, so that
6380	// the option isn't needed to be passed to distributed ThinLTO backend
6381	// clang processes, which won't necessarily have visibility into the linker
6382	// dependences. Instead the information is communicated from the LTO link to
6383	// the backends via the combined summary index.
6384	if (!SupportsHotColdNew)
6385	return false;
6386
6387	ModuleCallsiteContextGraph CCG(M, OREGetter);
6388	// TODO: Set up remarks for regular LTO. We need to decide what function to
6389	// use in the callback.
6390	return CCG.process();
6391	}
6392
6393	MemProfContextDisambiguation::MemProfContextDisambiguation(
6394	const ModuleSummaryIndex Summary, bool* isSamplePGO)
6395	: ImportSummary(Summary), isSamplePGO(isSamplePGO) {
6396	// Check the dot graph printing options once here, to make sure we have valid
6397	// and expected combinations.
6398	if (DotGraphScope == DotScope::Alloc && !AllocIdForDot.getNumOccurrences())
6399	llvm::report_fatal_error(
6400	reason: "-memprof-dot-scope=alloc requires -memprof-dot-alloc-id");
6401	if (DotGraphScope == DotScope::Context &&
6402	!ContextIdForDot.getNumOccurrences())
6403	llvm::report_fatal_error(
6404	reason: "-memprof-dot-scope=context requires -memprof-dot-context-id");
6405	if (DotGraphScope == DotScope::All && AllocIdForDot.getNumOccurrences() &&
6406	ContextIdForDot.getNumOccurrences())
6407	llvm::report_fatal_error(
6408	reason: "-memprof-dot-scope=all can't have both -memprof-dot-alloc-id and "
6409	"-memprof-dot-context-id");
6410	if (ImportSummary) {
6411	// The MemProfImportSummary should only be used for testing ThinLTO
6412	// distributed backend handling via opt, in which case we don't have a
6413	// summary from the pass pipeline.
6414	assert(MemProfImportSummary.empty());
6415	return;
6416	}
6417	if (MemProfImportSummary.empty())
6418	return;
6419
6420	auto ReadSummaryFile =
6421	errorOrToExpected(EO: MemoryBuffer::getFile(Filename: MemProfImportSummary));
6422	if (!ReadSummaryFile) {
6423	logAllUnhandledErrors(E: ReadSummaryFile.takeError(), OS&: errs(),
6424	ErrorBanner: "Error loading file '" + MemProfImportSummary +
6425	"': ");
6426	return;
6427	}
6428	auto ImportSummaryForTestingOrErr = getModuleSummaryIndex(Buffer: **ReadSummaryFile);
6429	if (!ImportSummaryForTestingOrErr) {
6430	logAllUnhandledErrors(E: ImportSummaryForTestingOrErr.takeError(), OS&: errs(),
6431	ErrorBanner: "Error parsing file '" + MemProfImportSummary +
6432	"': ");
6433	return;
6434	}
6435	ImportSummaryForTesting = std::move(*ImportSummaryForTestingOrErr);
6436	ImportSummary = ImportSummaryForTesting.get();
6437	}
6438
6439	PreservedAnalyses MemProfContextDisambiguation::run(Module &M,
6440	ModuleAnalysisManager &AM) {
6441	auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
6442	auto OREGetter = [&](Function *F) -> OptimizationRemarkEmitter & {
6443	return FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: *F);
6444	};
6445	if (!processModule(M, OREGetter))
6446	return PreservedAnalyses::all();
6447	return PreservedAnalyses::none();
6448	}
6449
6450	void MemProfContextDisambiguation::run(
6451	ModuleSummaryIndex &Index,
6452	llvm::function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
6453	isPrevailing,
6454	LLVMContext &Ctx,
6455	function_ref<void(StringRef, StringRef, const Twine &)> EmitRemark) {
6456	// TODO: If/when other types of memprof cloning are enabled beyond just for
6457	// hot and cold, we will need to change this to individually control the
6458	// AllocationType passed to addStackNodesForMIB during CCG construction.
6459	// The index was set from the option, so these should be in sync.
6460	assert(Index.withSupportsHotColdNew() == SupportsHotColdNew);
6461	if (!SupportsHotColdNew)
6462	return;
6463
6464	bool AllowExtraAnalysis =
6465	OptimizationRemarkEmitter::allowExtraAnalysis(Ctx, DEBUG_TYPE);
6466
6467	IndexCallsiteContextGraph CCG(Index, isPrevailing);
6468	CCG.process(EmitRemark, AllowExtraAnalysis);
6469	}
6470
6471	// Strips MemProf attributes and metadata. Can be invoked by the pass pipeline
6472	// when we don't have an index that has recorded that we are linking with
6473	// allocation libraries containing the necessary APIs for downstream
6474	// transformations.
6475	PreservedAnalyses MemProfRemoveInfo::run(Module &M, ModuleAnalysisManager &AM) {
6476	// The profile matcher applies hotness attributes directly for allocations,
6477	// and those will cause us to generate calls to the hot/cold interfaces
6478	// unconditionally. If supports-hot-cold-new was not enabled in the LTO
6479	// link then assume we don't want these calls (e.g. not linking with
6480	// the appropriate library, or otherwise trying to disable this behavior).
6481	bool Changed = false;
6482	for (auto &F : M) {
6483	for (auto &BB : F) {
6484	for (auto &I : BB) {
6485	auto *CI = dyn_cast<CallBase>(Val: &I);
6486	if (!CI)
6487	continue;
6488	if (CI->hasFnAttr(Kind: "memprof")) {
6489	CI->removeFnAttr(Kind: "memprof");
6490	Changed = true;
6491	}
6492	if (!CI->hasMetadata(KindID: LLVMContext::MD_callsite)) {
6493	assert(!CI->hasMetadata(LLVMContext::MD_memprof));
6494	continue;
6495	}
6496	// Strip off all memprof metadata as it is no longer needed.
6497	// Importantly, this avoids the addition of new memprof attributes
6498	// after inlining propagation.
6499	CI->setMetadata(KindID: LLVMContext::MD_memprof, Node: nullptr);
6500	CI->setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
6501	Changed = true;
6502	}
6503	}
6504	}
6505	if (!Changed)
6506	return PreservedAnalyses::all();
6507	return PreservedAnalyses::none();
6508	}
6509

Browse the source code of llvm_projects/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp