MemProfContextDisambiguation.cpp source code [llvm_projects/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp]

1	//==-- MemProfContextDisambiguation.cpp - Disambiguate contexts -------------=//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file implements support for context disambiguation of allocation
10	// calls for profile guided heap optimization. Specifically, it uses Memprof
11	// profiles which indicate context specific allocation behavior (currently
12	// distinguishing cold vs hot memory allocations). Cloning is performed to
13	// expose the cold allocation call contexts, and the allocation calls are
14	// subsequently annotated with an attribute for later transformation.
15	//
16	// The transformations can be performed either directly on IR (regular LTO), or
17	// on a ThinLTO index (and later applied to the IR during the ThinLTO backend).
18	// Both types of LTO operate on a the same base graph representation, which
19	// uses CRTP to support either IR or Index formats.
20	//
21	//===----------------------------------------------------------------------===//
22
23	#include "llvm/Transforms/IPO/MemProfContextDisambiguation.h"
24	#include "llvm/ADT/DenseMap.h"
25	#include "llvm/ADT/DenseSet.h"
26	#include "llvm/ADT/MapVector.h"
27	#include "llvm/ADT/SetOperations.h"
28	#include "llvm/ADT/SmallPtrSet.h"
29	#include "llvm/ADT/SmallSet.h"
30	#include "llvm/ADT/SmallVector.h"
31	#include "llvm/ADT/Statistic.h"
32	#include "llvm/ADT/StringExtras.h"
33	#include "llvm/Analysis/MemoryProfileInfo.h"
34	#include "llvm/Analysis/ModuleSummaryAnalysis.h"
35	#include "llvm/Analysis/OptimizationRemarkEmitter.h"
36	#include "llvm/Bitcode/BitcodeReader.h"
37	#include "llvm/IR/Instructions.h"
38	#include "llvm/IR/Module.h"
39	#include "llvm/IR/ModuleSummaryIndex.h"
40	#include "llvm/Pass.h"
41	#include "llvm/Support/CommandLine.h"
42	#include "llvm/Support/GraphWriter.h"
43	#include "llvm/Support/SHA1.h"
44	#include "llvm/Support/raw_ostream.h"
45	#include "llvm/Transforms/IPO.h"
46	#include "llvm/Transforms/Utils/CallPromotionUtils.h"
47	#include "llvm/Transforms/Utils/Cloning.h"
48	#include "llvm/Transforms/Utils/Instrumentation.h"
49	#include <deque>
50	#include <sstream>
51	#include <vector>
52	using namespace llvm;
53	using namespace llvm::memprof;
54
55	#define DEBUG_TYPE "memprof-context-disambiguation"
56
57	STATISTIC(FunctionClonesAnalysis,
58	"Number of function clones created during whole program analysis");
59	STATISTIC(FunctionClonesThinBackend,
60	"Number of function clones created during ThinLTO backend");
61	STATISTIC(FunctionsClonedThinBackend,
62	"Number of functions that had clones created during ThinLTO backend");
63	STATISTIC(
64	FunctionCloneDuplicatesThinBackend,
65	"Number of function clone duplicates detected during ThinLTO backend");
66	STATISTIC(AllocTypeNotCold, "Number of not cold static allocations (possibly "
67	"cloned) during whole program analysis");
68	STATISTIC(AllocTypeCold, "Number of cold static allocations (possibly cloned) "
69	"during whole program analysis");
70	STATISTIC(AllocTypeNotColdThinBackend,
71	"Number of not cold static allocations (possibly cloned) during "
72	"ThinLTO backend");
73	STATISTIC(AllocTypeColdThinBackend, "Number of cold static allocations "
74	"(possibly cloned) during ThinLTO backend");
75	STATISTIC(OrigAllocsThinBackend,
76	"Number of original (not cloned) allocations with memprof profiles "
77	"during ThinLTO backend");
78	STATISTIC(
79	AllocVersionsThinBackend,
80	"Number of allocation versions (including clones) during ThinLTO backend");
81	STATISTIC(MaxAllocVersionsThinBackend,
82	"Maximum number of allocation versions created for an original "
83	"allocation during ThinLTO backend");
84	STATISTIC(UnclonableAllocsThinBackend,
85	"Number of unclonable ambigous allocations during ThinLTO backend");
86	STATISTIC(RemovedEdgesWithMismatchedCallees,
87	"Number of edges removed due to mismatched callees (profiled vs IR)");
88	STATISTIC(FoundProfiledCalleeCount,
89	"Number of profiled callees found via tail calls");
90	STATISTIC(FoundProfiledCalleeDepth,
91	"Aggregate depth of profiled callees found via tail calls");
92	STATISTIC(FoundProfiledCalleeMaxDepth,
93	"Maximum depth of profiled callees found via tail calls");
94	STATISTIC(FoundProfiledCalleeNonUniquelyCount,
95	"Number of profiled callees found via multiple tail call chains");
96	STATISTIC(DeferredBackedges, "Number of backedges with deferred cloning");
97	STATISTIC(NewMergedNodes, "Number of new nodes created during merging");
98	STATISTIC(NonNewMergedNodes, "Number of non new nodes used during merging");
99	STATISTIC(MissingAllocForContextId,
100	"Number of missing alloc nodes for context ids");
101	STATISTIC(SkippedCallsCloning,
102	"Number of calls skipped during cloning due to unexpected operand");
103	STATISTIC(MismatchedCloneAssignments,
104	"Number of callsites assigned to call multiple non-matching clones");
105	STATISTIC(TotalMergeInvokes, "Number of merge invocations for nodes");
106	STATISTIC(TotalMergeIters, "Number of merge iterations for nodes");
107	STATISTIC(MaxMergeIters, "Max merge iterations for nodes");
108	STATISTIC(NumImportantContextIds, "Number of important context ids");
109	STATISTIC(NumFixupEdgeIdsInserted, "Number of fixup edge ids inserted");
110	STATISTIC(NumFixupEdgesAdded, "Number of fixup edges added");
111	STATISTIC(NumFixedContexts, "Number of contexts with fixed edges");
112	STATISTIC(AliaseesPrevailingInDiffModuleFromAlias,
113	"Number of aliasees prevailing in a different module than its alias");
114
115	static cl::opt<std::string> DotFilePathPrefix(
116	"memprof-dot-file-path-prefix", cl::init(Val: ""), cl::Hidden,
117	cl::value_desc("filename"),
118	cl::desc("Specify the path prefix of the MemProf dot files."));
119
120	static cl::opt<bool> ExportToDot("memprof-export-to-dot", cl::init(Val: false),
121	cl::Hidden,
122	cl::desc("Export graph to dot files."));
123
124	// TODO: Remove this option once new handling is validated more widely.
125	static cl::opt<bool> DoMergeIteration(
126	"memprof-merge-iteration", cl::init(Val: true), cl::Hidden,
127	cl::desc("Iteratively apply merging on a node to catch new callers"));
128
129	// How much of the graph to export to dot.
130	enum DotScope {
131	All, // The full CCG graph.
132	Alloc, // Only contexts for the specified allocation.
133	Context, // Only the specified context.
134	};
135
136	static cl::opt<DotScope> DotGraphScope(
137	"memprof-dot-scope", cl::desc("Scope of graph to export to dot"),
138	cl::Hidden, cl::init(Val: DotScope::All),
139	cl::values(
140	clEnumValN(DotScope::All, "all", "Export full callsite graph"),
141	clEnumValN(DotScope::Alloc, "alloc",
142	"Export only nodes with contexts feeding given "
143	"-memprof-dot-alloc-id"),
144	clEnumValN(DotScope::Context, "context",
145	"Export only nodes with given -memprof-dot-context-id")));
146
147	static cl::opt<unsigned>
148	AllocIdForDot("memprof-dot-alloc-id", cl::init(Val: `0`), cl::Hidden,
149	cl::desc("Id of alloc to export if -memprof-dot-scope=alloc "
150	"or to highlight if -memprof-dot-scope=all"));
151
152	static cl::opt<unsigned> ContextIdForDot(
153	"memprof-dot-context-id", cl::init(Val: `0`), cl::Hidden,
154	cl::desc("Id of context to export if -memprof-dot-scope=context or to "
155	"highlight otherwise"));
156
157	static cl::opt<bool>
158	DumpCCG("memprof-dump-ccg", cl::init(Val: false), cl::Hidden,
159	cl::desc("Dump CallingContextGraph to stdout after each stage."));
160
161	static cl::opt<bool>
162	VerifyCCG("memprof-verify-ccg", cl::init(Val: false), cl::Hidden,
163	cl::desc("Perform verification checks on CallingContextGraph."));
164
165	static cl::opt<bool>
166	VerifyNodes("memprof-verify-nodes", cl::init(Val: false), cl::Hidden,
167	cl::desc("Perform frequent verification checks on nodes."));
168
169	static cl::opt<std::string> MemProfImportSummary(
170	"memprof-import-summary",
171	cl::desc("Import summary to use for testing the ThinLTO backend via opt"),
172	cl::Hidden);
173
174	static cl::opt<unsigned>
175	TailCallSearchDepth("memprof-tail-call-search-depth", cl::init(Val: `5`),
176	cl::Hidden,
177	cl::desc("Max depth to recursively search for missing "
178	"frames through tail calls."));
179
180	// Optionally enable cloning of callsites involved with recursive cycles
181	static cl::opt<bool> AllowRecursiveCallsites(
182	"memprof-allow-recursive-callsites", cl::init(Val: true), cl::Hidden,
183	cl::desc("Allow cloning of callsites involved in recursive cycles"));
184
185	static cl::opt<bool> CloneRecursiveContexts(
186	"memprof-clone-recursive-contexts", cl::init(Val: true), cl::Hidden,
187	cl::desc("Allow cloning of contexts through recursive cycles"));
188
189	// Generally this is needed for correct assignment of allocation clones to
190	// function clones, however, allow it to be disabled for debugging while the
191	// functionality is new and being tested more widely.
192	static cl::opt<bool>
193	MergeClones("memprof-merge-clones", cl::init(Val: true), cl::Hidden,
194	cl::desc("Merge clones before assigning functions"));
195
196	// When disabled, try to detect and prevent cloning of recursive contexts.
197	// This is only necessary until we support cloning through recursive cycles.
198	// Leave on by default for now, as disabling requires a little bit of compile
199	// time overhead and doesn't affect correctness, it will just inflate the cold
200	// hinted bytes reporting a bit when -memprof-report-hinted-sizes is enabled.
201	static cl::opt<bool> AllowRecursiveContexts(
202	"memprof-allow-recursive-contexts", cl::init(Val: true), cl::Hidden,
203	cl::desc("Allow cloning of contexts having recursive cycles"));
204
205	// Set the minimum absolute count threshold for allowing inlining of indirect
206	// calls promoted during cloning.
207	static cl::opt<unsigned> MemProfICPNoInlineThreshold(
208	"memprof-icp-noinline-threshold", cl::init(Val: `0`), cl::Hidden,
209	cl::desc("Minimum absolute count for promoted target to be inlinable"));
210
211	namespace llvm {
212	cl::opt<bool> EnableMemProfContextDisambiguation(
213	"enable-memprof-context-disambiguation", cl::Hidden,
214	cl::desc("Enable MemProf context disambiguation"));
215
216	// Indicate we are linking with an allocator that supports hot/cold operator
217	// new interfaces.
218	cl::opt<bool> SupportsHotColdNew(
219	"supports-hot-cold-new", cl::init(Val: false), cl::Hidden,
220	cl::desc("Linking with hot/cold operator new interfaces"));
221
222	static cl::opt<bool> MemProfRequireDefinitionForPromotion(
223	"memprof-require-definition-for-promotion", cl::init(Val: false), cl::Hidden,
224	cl::desc(
225	"Require target function definition when promoting indirect calls"));
226
227	extern cl::opt<bool> MemProfReportHintedSizes;
228	extern cl::opt<unsigned> MinClonedColdBytePercent;
229
230	cl::opt<unsigned> MemProfTopNImportant(
231	"memprof-top-n-important", cl::init(Val: `10`), cl::Hidden,
232	cl::desc("Number of largest cold contexts to consider important"));
233
234	cl::opt<bool> MemProfFixupImportant(
235	"memprof-fixup-important", cl::init(Val: true), cl::Hidden,
236	cl::desc("Enables edge fixup for important contexts"));
237
238	extern cl::opt<unsigned> MaxSummaryIndirectEdges;
239
240	} // namespace llvm
241
242	namespace {
243
244	/// CRTP base for graphs built from either IR or ThinLTO summary index.
245	///
246	/// The graph represents the call contexts in all memprof metadata on allocation
247	/// calls, with nodes for the allocations themselves, as well as for the calls
248	/// in each context. The graph is initially built from the allocation memprof
249	/// metadata (or summary) MIBs. It is then updated to match calls with callsite
250	/// metadata onto the nodes, updating it to reflect any inlining performed on
251	/// those calls.
252	///
253	/// Each MIB (representing an allocation's call context with allocation
254	/// behavior) is assigned a unique context id during the graph build. The edges
255	/// and nodes in the graph are decorated with the context ids they carry. This
256	/// is used to correctly update the graph when cloning is performed so that we
257	/// can uniquify the context for a single (possibly cloned) allocation.
258	template <typename DerivedCCG, typename FuncTy, typename CallTy>
259	class CallsiteContextGraph {
260	public:
261	CallsiteContextGraph() = default;
262	CallsiteContextGraph(const CallsiteContextGraph &) = default;
263	CallsiteContextGraph(CallsiteContextGraph &&) = default;
264
265	/// Main entry point to perform analysis and transformations on graph.
266	bool process(function_ref<void(StringRef, StringRef, const Twine &)>
267	EmitRemark = nullptr,
268	bool AllowExtraAnalysis = false);
269
270	/// Perform cloning on the graph necessary to uniquely identify the allocation
271	/// behavior of an allocation based on its context.
272	void identifyClones();
273
274	/// Assign callsite clones to functions, cloning functions as needed to
275	/// accommodate the combinations of their callsite clones reached by callers.
276	/// For regular LTO this clones functions and callsites in the IR, but for
277	/// ThinLTO the cloning decisions are noted in the summaries and later applied
278	/// in applyImport.
279	bool assignFunctions();
280
281	void dump() const;
282	void print(raw_ostream &OS) const;
283	void printTotalSizes(raw_ostream &OS,
284	function_ref<void(StringRef, StringRef, const Twine &)>
285	EmitRemark = nullptr) const;
286
287	friend raw_ostream &operator<<(raw_ostream &OS,
288	const CallsiteContextGraph &CCG) {
289	CCG.print(OS);
290	return OS;
291	}
292
293	friend struct GraphTraits<
294	const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>;
295	friend struct DOTGraphTraits<
296	const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>;
297
298	void exportToDot(std::string Label) const;
299
300	/// Represents a function clone via FuncTy pointer and clone number pair.
301	struct FuncInfo final
302	: public std::pair<FuncTy , unsigned* /Clone number/> {
303	using Base = std::pair<FuncTy , unsigned*>;
304	FuncInfo(const Base &B) : Base(B) {}
305	FuncInfo(FuncTy F = nullptr, unsigned* CloneNo = `0`) : Base(F, CloneNo) {}
306	explicit operator bool() const { return this->first != nullptr; }
307	FuncTy func() const* { return this->first; }
308	unsigned cloneNo() const { return this->second; }
309	};
310
311	/// Represents a callsite clone via CallTy and clone number pair.
312	struct CallInfo final : public std::pair<CallTy, unsigned /Clone number/> {
313	using Base = std::pair<CallTy, unsigned>;
314	CallInfo(const Base &B) : Base(B) {}
315	CallInfo(CallTy Call = nullptr, unsigned CloneNo = `0`)
316	: Base(Call, CloneNo) {}
317	explicit operator bool() const { return (bool)this->first; }
318	CallTy call() const { return this->first; }
319	unsigned cloneNo() const { return this->second; }
320	void setCloneNo(unsigned N) { this->second = N; }
321	void print(raw_ostream &OS) const {
322	if (!operator bool()) {
323	assert(!cloneNo());
324	OS << "null Call";
325	return;
326	}
327	call()->print(OS);
328	OS << "\t(clone " << cloneNo() << ")";
329	}
330	void dump() const {
331	print(OS&: dbgs());
332	dbgs() << "\n";
333	}
334	friend raw_ostream &operator<<(raw_ostream &OS, const CallInfo &Call) {
335	Call.print(OS);
336	return OS;
337	}
338	};
339
340	struct ContextEdge;
341
342	/// Node in the Callsite Context Graph
343	struct ContextNode {
344	// Assigned to nodes as they are created, useful for debugging.
345	unsigned NodeId = `0`;
346
347	// Keep this for now since in the IR case where we have an Instruction it*
348	// is not as immediately discoverable. Used for printing richer information
349	// when dumping graph.
350	bool IsAllocation;
351
352	// Keeps track of when the Call was reset to null because there was
353	// recursion.
354	bool Recursive = false;
355
356	// This will be formed by ORing together the AllocationType enum values
357	// for contexts including this node.
358	uint8_t AllocTypes = `0`;
359
360	// The corresponding allocation or interior call. This is the primary call
361	// for which we have created this node.
362	CallInfo Call;
363
364	// List of other calls that can be treated the same as the primary call
365	// through cloning. I.e. located in the same function and have the same
366	// (possibly pruned) stack ids. They will be updated the same way as the
367	// primary call when assigning to function clones.
368	SmallVector<CallInfo, `0`> MatchingCalls;
369
370	// For alloc nodes this is a unique id assigned when constructed, and for
371	// callsite stack nodes it is the original stack id when the node is
372	// constructed from the memprof MIB metadata on the alloc nodes. Note that
373	// this is only used when matching callsite metadata onto the stack nodes
374	// created when processing the allocation memprof MIBs, and for labeling
375	// nodes in the dot graph. Therefore we don't bother to assign a value for
376	// clones.
377	uint64_t OrigStackOrAllocId = `0`;
378
379	// Edges to all callees in the profiled call stacks.
380	// TODO: Should this be a map (from Callee node) for more efficient lookup?
381	std::vector<std::shared_ptr<ContextEdge>> CalleeEdges;
382
383	// Edges to all callers in the profiled call stacks.
384	// TODO: Should this be a map (from Caller node) for more efficient lookup?
385	std::vector<std::shared_ptr<ContextEdge>> CallerEdges;
386
387	// Returns true if we need to look at the callee edges for determining the
388	// node context ids and allocation type.
389	bool useCallerEdgesForContextInfo() const {
390	// Typically if the callee edges are empty either the caller edges are
391	// also empty, or this is an allocation (leaf node). However, if we are
392	// allowing recursive callsites and contexts this will be violated for
393	// incompletely cloned recursive cycles.
394	assert(!CalleeEdges.empty() \|\| CallerEdges.empty() \|\| IsAllocation \|\|
395	(AllowRecursiveCallsites && AllowRecursiveContexts));
396	// When cloning for a recursive context, during cloning we might be in the
397	// midst of cloning for a recurrence and have moved context ids off of a
398	// caller edge onto the clone but not yet off of the incoming caller
399	// (back) edge. If we don't look at those we miss the fact that this node
400	// still has context ids of interest.
401	return IsAllocation \|\| CloneRecursiveContexts;
402	}
403
404	// Compute the context ids for this node from the union of its edge context
405	// ids.
406	DenseSet<uint32_t> getContextIds() const {
407	unsigned Count = `0`;
408	// Compute the number of ids for reserve below. In general we only need to
409	// look at one set of edges, typically the callee edges, since other than
410	// allocations and in some cases during recursion cloning, all the context
411	// ids on the callers should also flow out via callee edges.
412	for (auto &Edge : CalleeEdges.empty() ? CallerEdges : CalleeEdges)
413	Count += Edge->getContextIds().size();
414	DenseSet<uint32_t> ContextIds;
415	ContextIds.reserve(Size: Count);
416	auto Edges = llvm::concat<const std::shared_ptr<ContextEdge>>(
417	CalleeEdges, useCallerEdgesForContextInfo()
418	? CallerEdges
419	: std::vector<std::shared_ptr<ContextEdge>>());
420	for (const auto &Edge : Edges)
421	ContextIds.insert_range(Edge->getContextIds());
422	return ContextIds;
423	}
424
425	// Compute the allocation type for this node from the OR of its edge
426	// allocation types.
427	uint8_t computeAllocType() const {
428	uint8_t BothTypes =
429	(uint8_t)AllocationType::Cold \| (uint8_t)AllocationType::NotCold;
430	uint8_t AllocType = (uint8_t)AllocationType::None;
431	auto Edges = llvm::concat<const std::shared_ptr<ContextEdge>>(
432	CalleeEdges, useCallerEdgesForContextInfo()
433	? CallerEdges
434	: std::vector<std::shared_ptr<ContextEdge>>());
435	for (const auto &Edge : Edges) {
436	AllocType \|= Edge->AllocTypes;
437	// Bail early if alloc type reached both, no further refinement.
438	if (AllocType == BothTypes)
439	return AllocType;
440	}
441	return AllocType;
442	}
443
444	// The context ids set for this node is empty if its edge context ids are
445	// also all empty.
446	bool emptyContextIds() const {
447	auto Edges = llvm::concat<const std::shared_ptr<ContextEdge>>(
448	CalleeEdges, useCallerEdgesForContextInfo()
449	? CallerEdges
450	: std::vector<std::shared_ptr<ContextEdge>>());
451	for (const auto &Edge : Edges) {
452	if (!Edge->getContextIds().empty())
453	return false;
454	}
455	return true;
456	}
457
458	// List of clones of this ContextNode, initially empty.
459	std::vector<ContextNode *> Clones;
460
461	// If a clone, points to the original uncloned node.
462	ContextNode CloneOf = nullptr*;
463
464	ContextNode(bool IsAllocation) : IsAllocation(IsAllocation), Call() {}
465
466	ContextNode(bool IsAllocation, CallInfo C)
467	: IsAllocation(IsAllocation), Call(C) {}
468
469	void addClone(ContextNode *Clone) {
470	if (CloneOf) {
471	CloneOf->Clones.push_back(Clone);
472	Clone->CloneOf = CloneOf;
473	} else {
474	Clones.push_back(Clone);
475	assert(!Clone->CloneOf);
476	Clone->CloneOf = this;
477	}
478	}
479
480	ContextNode *getOrigNode() {
481	if (!CloneOf)
482	return this;
483	return CloneOf;
484	}
485
486	void addOrUpdateCallerEdge(ContextNode *Caller, AllocationType AllocType,
487	unsigned int ContextId);
488
489	ContextEdge findEdgeFromCallee(const* ContextNode *Callee);
490	ContextEdge findEdgeFromCaller(const* ContextNode *Caller);
491	void eraseCalleeEdge(const ContextEdge *Edge);
492	void eraseCallerEdge(const ContextEdge *Edge);
493
494	void setCall(CallInfo C) { Call = std::move(C); }
495
496	bool hasCall() const { return (bool)Call.call(); }
497
498	void printCall(raw_ostream &OS) const { Call.print(OS); }
499
500	// True if this node was effectively removed from the graph, in which case
501	// it should have an allocation type of None and empty context ids.
502	bool isRemoved() const {
503	// Typically if the callee edges are empty either the caller edges are
504	// also empty, or this is an allocation (leaf node). However, if we are
505	// allowing recursive callsites and contexts this will be violated for
506	// incompletely cloned recursive cycles.
507	assert((AllowRecursiveCallsites && AllowRecursiveContexts) \|\|
508	(AllocTypes == (uint8_t)AllocationType::None) ==
509	emptyContextIds());
510	return AllocTypes == (uint8_t)AllocationType::None;
511	}
512
513	void dump() const;
514	void print(raw_ostream &OS) const;
515
516	friend raw_ostream &operator<<(raw_ostream &OS, const ContextNode &Node) {
517	Node.print(OS);
518	return OS;
519	}
520	};
521
522	/// Edge in the Callsite Context Graph from a ContextNode N to a caller or
523	/// callee.
524	struct ContextEdge {
525	ContextNode *Callee;
526	ContextNode *Caller;
527
528	// This will be formed by ORing together the AllocationType enum values
529	// for contexts including this edge.
530	uint8_t AllocTypes = `0`;
531
532	// Set just before initiating cloning when cloning of recursive contexts is
533	// enabled. Used to defer cloning of backedges until we have done cloning of
534	// the callee node for non-backedge caller edges. This exposes cloning
535	// opportunities through the backedge of the cycle.
536	// TODO: Note that this is not updated during cloning, and it is unclear
537	// whether that would be needed.
538	bool IsBackedge = false;
539
540	// The set of IDs for contexts including this edge.
541	DenseSet<uint32_t> ContextIds;
542
543	ContextEdge(ContextNode Callee, ContextNode Caller, uint8_t AllocType,
544	DenseSet<uint32_t> ContextIds)
545	: Callee(Callee), Caller(Caller), AllocTypes(AllocType),
546	ContextIds (std::move(ContextIds)) {}
547
548	DenseSet<uint32_t> &getContextIds() { return ContextIds; }
549
550	// Helper to clear the fields of this edge when we are removing it from the
551	// graph.
552	inline void clear() {
553	ContextIds.clear();
554	AllocTypes = (uint8_t)AllocationType::None;
555	Caller = nullptr;
556	Callee = nullptr;
557	}
558
559	// Check if edge was removed from the graph. This is useful while iterating
560	// over a copy of edge lists when performing operations that mutate the
561	// graph in ways that might remove one of the edges.
562	inline bool isRemoved() const {
563	if (Callee \|\| Caller)
564	return false;
565	// Any edges that have been removed from the graph but are still in a
566	// shared_ptr somewhere should have all fields null'ed out by clear()
567	// above.
568	assert(AllocTypes == (uint8_t)AllocationType::None);
569	assert(ContextIds.empty());
570	return true;
571	}
572
573	void dump() const;
574	void print(raw_ostream &OS) const;
575
576	friend raw_ostream &operator<<(raw_ostream &OS, const ContextEdge &Edge) {
577	Edge.print(OS);
578	return OS;
579	}
580	};
581
582	/// Helpers to remove edges that have allocation type None (due to not
583	/// carrying any context ids) after transformations.
584	void removeNoneTypeCalleeEdges(ContextNode *Node);
585	void removeNoneTypeCallerEdges(ContextNode *Node);
586	void
587	recursivelyRemoveNoneTypeCalleeEdges(ContextNode *Node,
588	DenseSet<const ContextNode *> &Visited);
589
590	protected:
591	/// Get a list of nodes corresponding to the stack ids in the given callsite
592	/// context.
593	template <class NodeT, class IteratorT>
594	std::vector<uint64_t>
595	getStackIdsWithContextNodes(CallStack<NodeT, IteratorT> &CallsiteContext);
596
597	/// Adds nodes for the given allocation and any stack ids on its memprof MIB
598	/// metadata (or summary).
599	ContextNode addAllocNode(CallInfo Call, const* FuncTy *F);
600
601	/// Adds nodes for the given MIB stack ids.
602	template <class NodeT, class IteratorT>
603	void addStackNodesForMIB(
604	ContextNode *AllocNode, CallStack<NodeT, IteratorT> &StackContext,
605	CallStack<NodeT, IteratorT> &CallsiteContext, AllocationType AllocType,
606	ArrayRef<ContextTotalSize> ContextSizeInfo,
607	std::map<uint64_t, uint32_t> &TotalSizeToContextIdTopNCold);
608
609	/// Matches all callsite metadata (or summary) to the nodes created for
610	/// allocation memprof MIB metadata, synthesizing new nodes to reflect any
611	/// inlining performed on those callsite instructions.
612	void updateStackNodes();
613
614	/// Optionally fixup edges for the N largest cold contexts to better enable
615	/// cloning. This is particularly helpful if the context includes recursion
616	/// as well as inlining, resulting in a single stack node for multiple stack
617	/// ids in the context. With recursion it is particularly difficult to get the
618	/// edge updates correct as in the general case we have lost the original
619	/// stack id ordering for the context. Do more expensive fixup for the largest
620	/// contexts, controlled by MemProfTopNImportant and MemProfFixupImportant.
621	void fixupImportantContexts();
622
623	/// Update graph to conservatively handle any callsite stack nodes that target
624	/// multiple different callee target functions.
625	void handleCallsitesWithMultipleTargets();
626
627	/// Mark backedges via the standard DFS based backedge algorithm.
628	void markBackedges();
629
630	/// Merge clones generated during cloning for different allocations but that
631	/// are called by the same caller node, to ensure proper function assignment.
632	void mergeClones();
633
634	// Try to partition calls on the given node (already placed into the AllCalls
635	// array) by callee function, creating new copies of Node as needed to hold
636	// calls with different callees, and moving the callee edges appropriately.
637	// Returns true if partitioning was successful.
638	bool partitionCallsByCallee(
639	ContextNode *Node, ArrayRef<CallInfo> AllCalls,
640	std::vector<std::pair<CallInfo, ContextNode *>> &NewCallToNode);
641
642	/// Save lists of calls with MemProf metadata in each function, for faster
643	/// iteration.
644	MapVector<FuncTy *, std::vector<CallInfo>> FuncToCallsWithMetadata;
645
646	/// Map from callsite node to the enclosing caller function.
647	std::map<const ContextNode , const* FuncTy *> NodeToCallingFunc;
648
649	// When exporting to dot, and an allocation id is specified, contains the
650	// context ids on that allocation.
651	DenseSet<uint32_t> DotAllocContextIds;
652
653	private:
654	using EdgeIter = typename std::vector<std::shared_ptr<ContextEdge>>::iterator;
655
656	// Structure to keep track of information for each call as we are matching
657	// non-allocation callsites onto context nodes created from the allocation
658	// call metadata / summary contexts.
659	struct CallContextInfo {
660	// The callsite we're trying to match.
661	CallTy Call;
662	// The callsites stack ids that have a context node in the graph.
663	std::vector<uint64_t> StackIds;
664	// The function containing this callsite.
665	const FuncTy *Func;
666	// Initially empty, if needed this will be updated to contain the context
667	// ids for use in a new context node created for this callsite.
668	DenseSet<uint32_t> ContextIds;
669	};
670
671	/// Helper to remove edge from graph, updating edge iterator if it is provided
672	/// (in which case CalleeIter indicates which edge list is being iterated).
673	/// This will also perform the necessary clearing of the ContextEdge members
674	/// to enable later checking if the edge has been removed (since we may have
675	/// other copies of the shared_ptr in existence, and in fact rely on this to
676	/// enable removal while iterating over a copy of a node's edge list).
677	void removeEdgeFromGraph(ContextEdge Edge, EdgeIter EI = nullptr,
678	bool CalleeIter = true);
679
680	/// Assigns the given Node to calls at or inlined into the location with
681	/// the Node's stack id, after post order traversing and processing its
682	/// caller nodes. Uses the call information recorded in the given
683	/// StackIdToMatchingCalls map, and creates new nodes for inlined sequences
684	/// as needed. Called by updateStackNodes which sets up the given
685	/// StackIdToMatchingCalls map.
686	void assignStackNodesPostOrder(
687	ContextNode Node, DenseSet<const* ContextNode *> &Visited,
688	DenseMap<uint64_t, std::vector<CallContextInfo>> &StackIdToMatchingCalls,
689	DenseMap<CallInfo, CallInfo> &CallToMatchingCall,
690	const DenseSet<uint32_t> &ImportantContextIds);
691
692	/// Duplicates the given set of context ids, updating the provided
693	/// map from each original id with the newly generated context ids,
694	/// and returning the new duplicated id set.
695	DenseSet<uint32_t> duplicateContextIds(
696	const DenseSet<uint32_t> &StackSequenceContextIds,
697	DenseMap<uint32_t, DenseSet<uint32_t>> &OldToNewContextIds);
698
699	/// Propagates all duplicated context ids across the graph.
700	void propagateDuplicateContextIds(
701	const DenseMap<uint32_t, DenseSet<uint32_t>> &OldToNewContextIds);
702
703	/// Connect the NewNode to OrigNode's callees if TowardsCallee is true,
704	/// else to its callers. Also updates OrigNode's edges to remove any context
705	/// ids moved to the newly created edge.
706	void connectNewNode(ContextNode NewNode, ContextNode OrigNode,
707	bool TowardsCallee,
708	DenseSet<uint32_t> RemainingContextIds);
709
710	/// Get the stack id corresponding to the given Id or Index (for IR this will
711	/// return itself, for a summary index this will return the id recorded in the
712	/// index for that stack id index value).
713	uint64_t getStackId(uint64_t IdOrIndex) const {
714	return static_cast<const DerivedCCG >(this*)->getStackId(IdOrIndex);
715	}
716
717	/// Returns true if the given call targets the callee of the given edge, or if
718	/// we were able to identify the call chain through intermediate tail calls.
719	/// In the latter case new context nodes are added to the graph for the
720	/// identified tail calls, and their synthesized nodes are added to
721	/// TailCallToContextNodeMap. The EdgeIter is updated in the latter case for
722	/// the updated edges and to prepare it for an increment in the caller.
723	bool
724	calleesMatch(CallTy Call, EdgeIter &EI,
725	MapVector<CallInfo, ContextNode *> &TailCallToContextNodeMap);
726
727	// Return the callee function of the given call, or nullptr if it can't be
728	// determined
729	const FuncTy *getCalleeFunc(CallTy Call) {
730	return static_cast<DerivedCCG >(this*)->getCalleeFunc(Call);
731	}
732
733	/// Returns true if the given call targets the given function, or if we were
734	/// able to identify the call chain through intermediate tail calls (in which
735	/// case FoundCalleeChain will be populated).
736	bool calleeMatchesFunc(
737	CallTy Call, const FuncTy Func, const* FuncTy *CallerFunc,
738	std::vector<std::pair<CallTy, FuncTy *>> &FoundCalleeChain) {
739	return static_cast<DerivedCCG >(this*)->calleeMatchesFunc(
740	Call, Func, CallerFunc, FoundCalleeChain);
741	}
742
743	/// Returns true if both call instructions have the same callee.
744	bool sameCallee(CallTy Call1, CallTy Call2) {
745	return static_cast<DerivedCCG >(this*)->sameCallee(Call1, Call2);
746	}
747
748	/// Get a list of nodes corresponding to the stack ids in the given
749	/// callsite's context.
750	std::vector<uint64_t> getStackIdsWithContextNodesForCall(CallTy Call) {
751	return static_cast<DerivedCCG >(this*)->getStackIdsWithContextNodesForCall(
752	Call);
753	}
754
755	/// Get the last stack id in the context for callsite.
756	uint64_t getLastStackId(CallTy Call) {
757	return static_cast<DerivedCCG >(this*)->getLastStackId(Call);
758	}
759
760	/// Update the allocation call to record type of allocated memory.
761	void updateAllocationCall(CallInfo &Call, AllocationType AllocType) {
762	AllocType == AllocationType::Cold ? AllocTypeCold ++ : AllocTypeNotCold ++;
763	static_cast<DerivedCCG >(this*)->updateAllocationCall(Call, AllocType);
764	}
765
766	/// Get the AllocationType assigned to the given allocation instruction clone.
767	AllocationType getAllocationCallType(const CallInfo &Call) const {
768	return static_cast<const DerivedCCG >(this*)->getAllocationCallType(Call);
769	}
770
771	/// Update non-allocation call to invoke (possibly cloned) function
772	/// CalleeFunc.
773	void updateCall(CallInfo &CallerCall, FuncInfo CalleeFunc) {
774	static_cast<DerivedCCG >(this*)->updateCall(CallerCall, CalleeFunc);
775	}
776
777	/// Clone the given function for the given callsite, recording mapping of all
778	/// of the functions tracked calls to their new versions in the CallMap.
779	/// Assigns new clones to clone number CloneNo.
780	FuncInfo cloneFunctionForCallsite(
781	FuncInfo &Func, CallInfo &Call, DenseMap<CallInfo, CallInfo> &CallMap,
782	std::vector<CallInfo> &CallsWithMetadataInFunc, unsigned CloneNo) {
783	return static_cast<DerivedCCG >(this*)->cloneFunctionForCallsite(
784	Func, Call, CallMap, CallsWithMetadataInFunc, CloneNo);
785	}
786
787	/// Gets a label to use in the dot graph for the given call clone in the given
788	/// function.
789	std::string getLabel(const FuncTy Func, const* CallTy Call,
790	unsigned CloneNo) const {
791	return static_cast<const DerivedCCG >(this*)->getLabel(Func, Call, CloneNo);
792	}
793
794	// Create and return a new ContextNode.
795	ContextNode createNewNode(bool* IsAllocation, const FuncTy F = nullptr*,
796	CallInfo C = CallInfo()) {
797	NodeOwner.push_back(std::make_unique<ContextNode>(IsAllocation, C));
798	auto *NewNode = NodeOwner.back().get();
799	if (F)
800	NodeToCallingFunc[NewNode] = F;
801	NewNode->NodeId = NodeOwner.size();
802	return NewNode;
803	}
804
805	/// Helpers to find the node corresponding to the given call or stackid.
806	ContextNode getNodeForInst(const* CallInfo &C);
807	ContextNode getNodeForAlloc(const* CallInfo &C);
808	ContextNode *getNodeForStackId(uint64_t StackId);
809
810	/// Computes the alloc type corresponding to the given context ids, by
811	/// unioning their recorded alloc types.
812	uint8_t computeAllocType(DenseSet<uint32_t> &ContextIds) const;
813
814	/// Returns the allocation type of the intersection of the contexts of two
815	/// nodes (based on their provided context id sets), optimized for the case
816	/// when Node1Ids is smaller than Node2Ids.
817	uint8_t intersectAllocTypesImpl(const DenseSet<uint32_t> &Node1Ids,
818	const DenseSet<uint32_t> &Node2Ids) const;
819
820	/// Returns the allocation type of the intersection of the contexts of two
821	/// nodes (based on their provided context id sets).
822	uint8_t intersectAllocTypes(const DenseSet<uint32_t> &Node1Ids,
823	const DenseSet<uint32_t> &Node2Ids) const;
824
825	/// Create a clone of Edge's callee and move Edge to that new callee node,
826	/// performing the necessary context id and allocation type updates.
827	/// If ContextIdsToMove is non-empty, only that subset of Edge's ids are
828	/// moved to an edge to the new callee.
829	ContextNode *
830	moveEdgeToNewCalleeClone(const std::shared_ptr<ContextEdge> &Edge,
831	DenseSet<uint32_t> ContextIdsToMove = {});
832
833	/// Change the callee of Edge to existing callee clone NewCallee, performing
834	/// the necessary context id and allocation type updates.
835	/// If ContextIdsToMove is non-empty, only that subset of Edge's ids are
836	/// moved to an edge to the new callee.
837	void moveEdgeToExistingCalleeClone(const std::shared_ptr<ContextEdge> &Edge,
838	ContextNode *NewCallee,
839	bool NewClone = false,
840	DenseSet<uint32_t> ContextIdsToMove = {});
841
842	/// Change the caller of the edge at the given callee edge iterator to be
843	/// NewCaller, performing the necessary context id and allocation type
844	/// updates. This is similar to the above moveEdgeToExistingCalleeClone, but
845	/// a simplified version of it as we always move the given edge and all of its
846	/// context ids.
847	void moveCalleeEdgeToNewCaller(const std::shared_ptr<ContextEdge> &Edge,
848	ContextNode *NewCaller);
849
850	/// Recursive helper for marking backedges via DFS.
851	void markBackedges(ContextNode Node, DenseSet<const* ContextNode *> &Visited,
852	DenseSet<const ContextNode *> &CurrentStack);
853
854	/// Recursive helper for merging clones.
855	void
856	mergeClones(ContextNode Node, DenseSet<const* ContextNode *> &Visited,
857	DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode);
858	/// Main worker for merging callee clones for a given node.
859	void mergeNodeCalleeClones(
860	ContextNode Node, DenseSet<const* ContextNode *> &Visited,
861	DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode);
862	/// Helper to find other callers of the given set of callee edges that can
863	/// share the same callee merge node.
864	void findOtherCallersToShareMerge(
865	ContextNode *Node, std::vector<std::shared_ptr<ContextEdge>> &CalleeEdges,
866	DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode,
867	DenseSet<ContextNode *> &OtherCallersToShareMerge);
868
869	/// Recursively perform cloning on the graph for the given Node and its
870	/// callers, in order to uniquely identify the allocation behavior of an
871	/// allocation given its context. The context ids of the allocation being
872	/// processed are given in AllocContextIds.
873	void identifyClones(ContextNode Node, DenseSet<const* ContextNode *> &Visited,
874	const DenseSet<uint32_t> &AllocContextIds);
875
876	/// Map from each context ID to the AllocationType assigned to that context.
877	DenseMap<uint32_t, AllocationType> ContextIdToAllocationType;
878
879	/// Map from each contextID to the profiled full contexts and their total
880	/// sizes (there may be more than one due to context trimming),
881	/// optionally populated when requested (via MemProfReportHintedSizes or
882	/// MinClonedColdBytePercent).
883	DenseMap<uint32_t, std::vector<ContextTotalSize>> ContextIdToContextSizeInfos;
884
885	/// Identifies the context node created for a stack id when adding the MIB
886	/// contexts to the graph. This is used to locate the context nodes when
887	/// trying to assign the corresponding callsites with those stack ids to these
888	/// nodes.
889	DenseMap<uint64_t, ContextNode *> StackEntryIdToContextNodeMap;
890
891	/// Saves information for the contexts identified as important (the largest
892	/// cold contexts up to MemProfTopNImportant).
893	struct ImportantContextInfo {
894	// The original list of leaf first stack ids corresponding to this context.
895	std::vector<uint64_t> StackIds;
896	// Max length of stack ids corresponding to a single stack ContextNode for
897	// this context (i.e. the max length of a key in StackIdsToNode below).
898	unsigned MaxLength = `0`;
899	// Mapping of slices of the stack ids to the corresponding ContextNode
900	// (there can be multiple stack ids due to inlining). Populated when
901	// updating stack nodes while matching them to the IR or summary.
902	std::map<std::vector<uint64_t>, ContextNode *> StackIdsToNode;
903	};
904
905	// Map of important full context ids to information about each.
906	DenseMap<uint32_t, ImportantContextInfo> ImportantContextIdInfo;
907
908	// For each important context id found in Node (if any), records the list of
909	// stack ids that corresponded to the given callsite Node. There can be more
910	// than one in the case of inlining.
911	void recordStackNode(std::vector<uint64_t> &StackIds, ContextNode *Node,
912	// We pass in the Node's context ids to avoid the
913	// overhead of computing them as the caller already has
914	// them in some cases.
915	const DenseSet<uint32_t> &NodeContextIds,
916	const DenseSet<uint32_t> &ImportantContextIds) {
917	if (!MemProfTopNImportant) {
918	assert(ImportantContextIds.empty());
919	return;
920	}
921	DenseSet<uint32_t> Ids =
922	set_intersection(S1: NodeContextIds, S2: ImportantContextIds);
923	if (Ids.empty())
924	return;
925	auto Size = StackIds.size();
926	for (auto Id : Ids) {
927	auto &Entry = ImportantContextIdInfo[Id];
928	Entry.StackIdsToNode[StackIds] = Node;
929	// Keep track of the max to simplify later analysis.
930	if (Size > Entry.MaxLength)
931	Entry.MaxLength = Size;
932	}
933	}
934
935	/// Maps to track the calls to their corresponding nodes in the graph.
936	MapVector<CallInfo, ContextNode *> AllocationCallToContextNodeMap;
937	MapVector<CallInfo, ContextNode *> NonAllocationCallToContextNodeMap;
938
939	/// Owner of all ContextNode unique_ptrs.
940	std::vector<std::unique_ptr<ContextNode>> NodeOwner;
941
942	/// Perform sanity checks on graph when requested.
943	void check() const;
944
945	/// Keeps track of the last unique context id assigned.
946	unsigned int LastContextId = `0`;
947	};
948
949	template <typename DerivedCCG, typename FuncTy, typename CallTy>
950	using ContextNode =
951	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode;
952	template <typename DerivedCCG, typename FuncTy, typename CallTy>
953	using ContextEdge =
954	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge;
955	template <typename DerivedCCG, typename FuncTy, typename CallTy>
956	using FuncInfo =
957	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::FuncInfo;
958	template <typename DerivedCCG, typename FuncTy, typename CallTy>
959	using CallInfo =
960	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::CallInfo;
961
962	/// CRTP derived class for graphs built from IR (regular LTO).
963	class ModuleCallsiteContextGraph
964	: public CallsiteContextGraph<ModuleCallsiteContextGraph, Function,
965	Instruction *> {
966	public:
967	ModuleCallsiteContextGraph(
968	Module &M,
969	llvm::function_ref<OptimizationRemarkEmitter &(Function *)> OREGetter);
970
971	private:
972	friend CallsiteContextGraph<ModuleCallsiteContextGraph, Function,
973	Instruction *>;
974
975	uint64_t getStackId(uint64_t IdOrIndex) const;
976	const Function getCalleeFunc(Instruction Call);
977	bool calleeMatchesFunc(
978	Instruction Call, const* Function Func, const* Function *CallerFunc,
979	std::vector<std::pair<Instruction , Function >> &FoundCalleeChain);
980	bool sameCallee(Instruction Call1, Instruction Call2);
981	bool findProfiledCalleeThroughTailCalls(
982	const Function ProfiledCallee, Value CurCallee, unsigned Depth,
983	std::vector<std::pair<Instruction , Function >> &FoundCalleeChain,
984	bool &FoundMultipleCalleeChains);
985	uint64_t getLastStackId(Instruction *Call);
986	std::vector<uint64_t> getStackIdsWithContextNodesForCall(Instruction *Call);
987	void updateAllocationCall(CallInfo &Call, AllocationType AllocType);
988	AllocationType getAllocationCallType(const CallInfo &Call) const;
989	void updateCall(CallInfo &CallerCall, FuncInfo CalleeFunc);
990	CallsiteContextGraph<ModuleCallsiteContextGraph, Function,
991	Instruction *>::FuncInfo
992	cloneFunctionForCallsite(FuncInfo &Func, CallInfo &Call,
993	DenseMap<CallInfo, CallInfo> &CallMap,
994	std::vector<CallInfo> &CallsWithMetadataInFunc,
995	unsigned CloneNo);
996	std::string getLabel(const Function Func, const* Instruction *Call,
997	unsigned CloneNo) const;
998
999	const Module &Mod;
1000	llvm::function_ref<OptimizationRemarkEmitter &(Function *)> OREGetter;
1001	};
1002
1003	/// Represents a call in the summary index graph, which can either be an
1004	/// allocation or an interior callsite node in an allocation's context.
1005	/// Holds a pointer to the corresponding data structure in the index.
1006	struct IndexCall : public PointerUnion<CallsiteInfo , AllocInfo > {
1007	IndexCall() : PointerUnion () {}
1008	IndexCall(std::nullptr_t) : IndexCall () {}
1009	IndexCall(CallsiteInfo *StackNode) : PointerUnion (StackNode) {}
1010	IndexCall(AllocInfo *AllocNode) : PointerUnion (AllocNode) {}
1011	IndexCall(PointerUnion PT) : PointerUnion (PT) {}
1012
1013	IndexCall *operator->() { return this; }
1014
1015	void print(raw_ostream &OS) const {
1016	PointerUnion<CallsiteInfo , AllocInfo > Base = *this;
1017	if (auto AI = llvm::dyn_cast_if_present<AllocInfo >(Val&: Base)) {
1018	OS << *AI;
1019	} else {
1020	auto CI = llvm::dyn_cast_if_present<CallsiteInfo >(Val&: Base);
1021	assert(CI);
1022	OS << *CI;
1023	}
1024	}
1025	};
1026	} // namespace
1027
1028	namespace llvm {
1029	template <> struct simplify_type<IndexCall> {
1030	using SimpleType = PointerUnion<CallsiteInfo , AllocInfo >;
1031	static SimpleType getSimplifiedValue(IndexCall &Val) { return Val; }
1032	};
1033	template <> struct simplify_type<const IndexCall> {
1034	using SimpleType = const PointerUnion<CallsiteInfo , AllocInfo >;
1035	static SimpleType getSimplifiedValue(const IndexCall &Val) { return Val; }
1036	};
1037	} // namespace llvm
1038
1039	namespace {
1040	/// CRTP derived class for graphs built from summary index (ThinLTO).
1041	class IndexCallsiteContextGraph
1042	: public CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
1043	IndexCall> {
1044	public:
1045	IndexCallsiteContextGraph(
1046	ModuleSummaryIndex &Index,
1047	llvm::function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1048	isPrevailing);
1049
1050	~IndexCallsiteContextGraph() {
1051	// Now that we are done with the graph it is safe to add the new
1052	// CallsiteInfo structs to the function summary vectors. The graph nodes
1053	// point into locations within these vectors, so we don't want to add them
1054	// any earlier.
1055	for (auto &I : FunctionCalleesToSynthesizedCallsiteInfos) {
1056	auto *FS = I.first;
1057	for (auto &Callsite : I.second)
1058	FS->addCallsite(Callsite: std::move(*Callsite.second));
1059	}
1060	}
1061
1062	private:
1063	friend CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
1064	IndexCall>;
1065
1066	uint64_t getStackId(uint64_t IdOrIndex) const;
1067	const FunctionSummary *getCalleeFunc(IndexCall &Call);
1068	bool calleeMatchesFunc(
1069	IndexCall &Call, const FunctionSummary *Func,
1070	const FunctionSummary *CallerFunc,
1071	std::vector<std::pair<IndexCall, FunctionSummary *>> &FoundCalleeChain);
1072	bool sameCallee(IndexCall &Call1, IndexCall &Call2);
1073	bool findProfiledCalleeThroughTailCalls(
1074	ValueInfo ProfiledCallee, ValueInfo CurCallee, unsigned Depth,
1075	std::vector<std::pair<IndexCall, FunctionSummary *>> &FoundCalleeChain,
1076	bool &FoundMultipleCalleeChains);
1077	uint64_t getLastStackId(IndexCall &Call);
1078	std::vector<uint64_t> getStackIdsWithContextNodesForCall(IndexCall &Call);
1079	void updateAllocationCall(CallInfo &Call, AllocationType AllocType);
1080	AllocationType getAllocationCallType(const CallInfo &Call) const;
1081	void updateCall(CallInfo &CallerCall, FuncInfo CalleeFunc);
1082	CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
1083	IndexCall>::FuncInfo
1084	cloneFunctionForCallsite(FuncInfo &Func, CallInfo &Call,
1085	DenseMap<CallInfo, CallInfo> &CallMap,
1086	std::vector<CallInfo> &CallsWithMetadataInFunc,
1087	unsigned CloneNo);
1088	std::string getLabel(const FunctionSummary Func, const* IndexCall &Call,
1089	unsigned CloneNo) const;
1090	DenseSet<GlobalValue::GUID> findAliaseeGUIDsPrevailingInDifferentModule();
1091
1092	// Saves mapping from function summaries containing memprof records back to
1093	// its VI, for use in checking and debugging.
1094	std::map<const FunctionSummary *, ValueInfo> FSToVIMap;
1095
1096	const ModuleSummaryIndex &Index;
1097	llvm::function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1098	isPrevailing;
1099
1100	// Saves/owns the callsite info structures synthesized for missing tail call
1101	// frames that we discover while building the graph.
1102	// It maps from the summary of the function making the tail call, to a map
1103	// of callee ValueInfo to corresponding synthesized callsite info.
1104	DenseMap<FunctionSummary *,
1105	std::map<ValueInfo, std::unique_ptr<CallsiteInfo>>>
1106	FunctionCalleesToSynthesizedCallsiteInfos;
1107	};
1108	} // namespace
1109
1110	template <>
1111	struct llvm::DenseMapInfo<CallsiteContextGraph<
1112	ModuleCallsiteContextGraph, Function, Instruction *>::CallInfo>
1113	: public DenseMapInfo<std::pair<Instruction , unsigned*>> {};
1114	template <>
1115	struct llvm::DenseMapInfo<CallsiteContextGraph<
1116	IndexCallsiteContextGraph, FunctionSummary, IndexCall>::CallInfo>
1117	: public DenseMapInfo<std::pair<IndexCall, unsigned>> {};
1118	template <>
1119	struct llvm::DenseMapInfo<IndexCall>
1120	: public DenseMapInfo<PointerUnion<CallsiteInfo , AllocInfo >> {};
1121
1122	namespace {
1123
1124	// Map the uint8_t alloc types (which may contain NotCold\|Cold) to the alloc
1125	// type we should actually use on the corresponding allocation.
1126	// If we can't clone a node that has NotCold+Cold alloc type, we will fall
1127	// back to using NotCold. So don't bother cloning to distinguish NotCold+Cold
1128	// from NotCold.
1129	AllocationType allocTypeToUse(uint8_t AllocTypes) {
1130	assert(AllocTypes != (uint8_t)AllocationType::None);
1131	if (AllocTypes ==
1132	((uint8_t)AllocationType::NotCold \| (uint8_t)AllocationType::Cold))
1133	return AllocationType::NotCold;
1134	else
1135	return (AllocationType)AllocTypes;
1136	}
1137
1138	// Helper to check if the alloc types for all edges recorded in the
1139	// InAllocTypes vector match the alloc types for all edges in the Edges
1140	// vector.
1141	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1142	bool allocTypesMatch(
1143	const std::vector<uint8_t> &InAllocTypes,
1144	const std::vector<std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>>>
1145	&Edges) {
1146	// This should be called only when the InAllocTypes vector was computed for
1147	// this set of Edges. Make sure the sizes are the same.
1148	assert(InAllocTypes.size() == Edges.size());
1149	return std::equal(
1150	InAllocTypes.begin(), InAllocTypes.end(), Edges.begin(), Edges.end(),
1151	[](const uint8_t &l,
1152	const std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>> &r) {
1153	// Can share if one of the edges is None type - don't
1154	// care about the type along that edge as it doesn't
1155	// exist for those context ids.
1156	if (l == (uint8_t)AllocationType::None \|\|
1157	r->AllocTypes == (uint8_t)AllocationType::None)
1158	return true;
1159	return allocTypeToUse(AllocTypes: l) == allocTypeToUse(r->AllocTypes);
1160	});
1161	}
1162
1163	// Helper to check if the alloc types for all edges recorded in the
1164	// InAllocTypes vector match the alloc types for callee edges in the given
1165	// clone. Because the InAllocTypes were computed from the original node's callee
1166	// edges, and other cloning could have happened after this clone was created, we
1167	// need to find the matching clone callee edge, which may or may not exist.
1168	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1169	bool allocTypesMatchClone(
1170	const std::vector<uint8_t> &InAllocTypes,
1171	const ContextNode<DerivedCCG, FuncTy, CallTy> *Clone) {
1172	const ContextNode<DerivedCCG, FuncTy, CallTy> *Node = Clone->CloneOf;
1173	assert(Node);
1174	// InAllocTypes should have been computed for the original node's callee
1175	// edges.
1176	assert(InAllocTypes.size() == Node->CalleeEdges.size());
1177	// First create a map of the clone callee edge callees to the edge alloc type.
1178	DenseMap<const ContextNode<DerivedCCG, FuncTy, CallTy> *, uint8_t>
1179	EdgeCalleeMap;
1180	for (const auto &E : Clone->CalleeEdges) {
1181	assert(!EdgeCalleeMap.contains(E->Callee));
1182	EdgeCalleeMap[E->Callee] = E->AllocTypes;
1183	}
1184	// Next, walk the original node's callees, and look for the corresponding
1185	// clone edge to that callee.
1186	for (unsigned I = `0`; I < Node->CalleeEdges.size(); I++) {
1187	auto Iter = EdgeCalleeMap.find(Node->CalleeEdges[I]->Callee);
1188	// Not found is ok, we will simply add an edge if we use this clone.
1189	if (Iter == EdgeCalleeMap.end())
1190	continue;
1191	// Can share if one of the edges is None type - don't
1192	// care about the type along that edge as it doesn't
1193	// exist for those context ids.
1194	if (InAllocTypes [I] == (uint8_t)AllocationType::None \|\|
1195	Iter->second == (uint8_t)AllocationType::None)
1196	continue;
1197	if (allocTypeToUse(Iter->second) != allocTypeToUse(AllocTypes: InAllocTypes [I]))
1198	return false;
1199	}
1200	return true;
1201	}
1202
1203	} // end anonymous namespace
1204
1205	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1206	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
1207	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::getNodeForInst(
1208	const CallInfo &C) {
1209	ContextNode *Node = getNodeForAlloc(C);
1210	if (Node)
1211	return Node;
1212
1213	return NonAllocationCallToContextNodeMap.lookup(C);
1214	}
1215
1216	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1217	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
1218	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::getNodeForAlloc(
1219	const CallInfo &C) {
1220	return AllocationCallToContextNodeMap.lookup(C);
1221	}
1222
1223	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1224	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
1225	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::getNodeForStackId(
1226	uint64_t StackId) {
1227	auto StackEntryNode = StackEntryIdToContextNodeMap.find(StackId);
1228	if (StackEntryNode != StackEntryIdToContextNodeMap.end())
1229	return StackEntryNode->second;
1230	return nullptr;
1231	}
1232
1233	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1234	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1235	addOrUpdateCallerEdge(ContextNode *Caller, AllocationType AllocType,
1236	unsigned int ContextId) {
1237	for (auto &Edge : CallerEdges) {
1238	if (Edge->Caller == Caller) {
1239	Edge->AllocTypes \|= (uint8_t)AllocType;
1240	Edge->getContextIds().insert(ContextId);
1241	return;
1242	}
1243	}
1244	std::shared_ptr<ContextEdge> Edge = std::make_shared<ContextEdge>(
1245	this, Caller, (uint8_t)AllocType, DenseSet<uint32_t>({ContextId}));
1246	CallerEdges.push_back(Edge);
1247	Caller->CalleeEdges.push_back(Edge);
1248	}
1249
1250	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1251	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::removeEdgeFromGraph(
1252	ContextEdge Edge, EdgeIter EI, bool CalleeIter) {
1253	assert(!EI \|\| (*EI)->get() == Edge);
1254	assert(!Edge->isRemoved());
1255	// Save the Caller and Callee pointers so we can erase Edge from their edge
1256	// lists after clearing Edge below. We do the clearing first in case it is
1257	// destructed after removing from the edge lists (if those were the last
1258	// shared_ptr references to Edge).
1259	auto *Callee = Edge->Callee;
1260	auto *Caller = Edge->Caller;
1261
1262	// Make sure the edge fields are cleared out so we can properly detect
1263	// removed edges if Edge is not destructed because there is still a shared_ptr
1264	// reference.
1265	Edge->clear();
1266
1267	#ifndef NDEBUG
1268	auto CalleeCallerCount = Callee->CallerEdges.size();
1269	auto CallerCalleeCount = Caller->CalleeEdges.size();
1270	#endif
1271	if (!EI) {
1272	Callee->eraseCallerEdge(Edge);
1273	Caller->eraseCalleeEdge(Edge);
1274	} else if (CalleeIter) {
1275	Callee->eraseCallerEdge(Edge);
1276	EI = Caller->CalleeEdges.erase(EI);
1277	} else {
1278	Caller->eraseCalleeEdge(Edge);
1279	EI = Callee->CallerEdges.erase(EI);
1280	}
1281	assert(Callee->CallerEdges.size() < CalleeCallerCount);
1282	assert(Caller->CalleeEdges.size() < CallerCalleeCount);
1283	}
1284
1285	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1286	void CallsiteContextGraph<
1287	DerivedCCG, FuncTy, CallTy>::removeNoneTypeCalleeEdges(ContextNode *Node) {
1288	for (auto EI = Node->CalleeEdges.begin(); EI != Node->CalleeEdges.end();) {
1289	auto Edge = *EI;
1290	if (Edge->AllocTypes == (uint8_t)AllocationType::None) {
1291	assert(Edge->ContextIds.empty());
1292	removeEdgeFromGraph(Edge: Edge.get(), EI: &EI, /CalleeIter=/true);
1293	} else
1294	++EI;
1295	}
1296	}
1297
1298	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1299	void CallsiteContextGraph<
1300	DerivedCCG, FuncTy, CallTy>::removeNoneTypeCallerEdges(ContextNode *Node) {
1301	for (auto EI = Node->CallerEdges.begin(); EI != Node->CallerEdges.end();) {
1302	auto Edge = *EI;
1303	if (Edge->AllocTypes == (uint8_t)AllocationType::None) {
1304	assert(Edge->ContextIds.empty());
1305	Edge->Caller->eraseCalleeEdge(Edge.get());
1306	EI = Node->CallerEdges.erase(EI);
1307	} else
1308	++EI;
1309	}
1310	}
1311
1312	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1313	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge *
1314	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1315	findEdgeFromCallee(const ContextNode *Callee) {
1316	for (const auto &Edge : CalleeEdges)
1317	if (Edge->Callee == Callee)
1318	return Edge.get();
1319	return nullptr;
1320	}
1321
1322	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1323	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge *
1324	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1325	findEdgeFromCaller(const ContextNode *Caller) {
1326	for (const auto &Edge : CallerEdges)
1327	if (Edge->Caller == Caller)
1328	return Edge.get();
1329	return nullptr;
1330	}
1331
1332	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1333	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1334	eraseCalleeEdge(const ContextEdge *Edge) {
1335	auto EI = llvm::find_if(
1336	CalleeEdges, [Edge](const std::shared_ptr<ContextEdge> &CalleeEdge) {
1337	return CalleeEdge.get() == Edge;
1338	});
1339	assert(EI != CalleeEdges.end());
1340	CalleeEdges.erase(EI);
1341	}
1342
1343	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1344	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1345	eraseCallerEdge(const ContextEdge *Edge) {
1346	auto EI = llvm::find_if(
1347	CallerEdges, [Edge](const std::shared_ptr<ContextEdge> &CallerEdge) {
1348	return CallerEdge.get() == Edge;
1349	});
1350	assert(EI != CallerEdges.end());
1351	CallerEdges.erase(EI);
1352	}
1353
1354	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1355	uint8_t CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::computeAllocType(
1356	DenseSet<uint32_t> &ContextIds) const {
1357	uint8_t BothTypes =
1358	(uint8_t)AllocationType::Cold \| (uint8_t)AllocationType::NotCold;
1359	uint8_t AllocType = (uint8_t)AllocationType::None;
1360	for (auto Id : ContextIds) {
1361	AllocType \|= (uint8_t)ContextIdToAllocationType.at(Val: Id);
1362	// Bail early if alloc type reached both, no further refinement.
1363	if (AllocType == BothTypes)
1364	return AllocType;
1365	}
1366	return AllocType;
1367	}
1368
1369	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1370	uint8_t
1371	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::intersectAllocTypesImpl(
1372	const DenseSet<uint32_t> &Node1Ids,
1373	const DenseSet<uint32_t> &Node2Ids) const {
1374	uint8_t BothTypes =
1375	(uint8_t)AllocationType::Cold \| (uint8_t)AllocationType::NotCold;
1376	uint8_t AllocType = (uint8_t)AllocationType::None;
1377	for (auto Id : Node1Ids) {
1378	if (!Node2Ids.count(V: Id))
1379	continue;
1380	AllocType \|= (uint8_t)ContextIdToAllocationType.at(Val: Id);
1381	// Bail early if alloc type reached both, no further refinement.
1382	if (AllocType == BothTypes)
1383	return AllocType;
1384	}
1385	return AllocType;
1386	}
1387
1388	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1389	uint8_t CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::intersectAllocTypes(
1390	const DenseSet<uint32_t> &Node1Ids,
1391	const DenseSet<uint32_t> &Node2Ids) const {
1392	if (Node1Ids.size() < Node2Ids.size())
1393	return intersectAllocTypesImpl(Node1Ids, Node2Ids);
1394	else
1395	return intersectAllocTypesImpl(Node1Ids: Node2Ids, Node2Ids: Node1Ids);
1396	}
1397
1398	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1399	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
1400	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::addAllocNode(
1401	CallInfo Call, const FuncTy *F) {
1402	assert(!getNodeForAlloc(Call));
1403	ContextNode AllocNode = createNewNode(/IsAllocation=/*true, F, C: Call);
1404	AllocationCallToContextNodeMap[Call] = AllocNode;
1405	// Use LastContextId as a uniq id for MIB allocation nodes.
1406	AllocNode->OrigStackOrAllocId = LastContextId;
1407	// Alloc type should be updated as we add in the MIBs. We should assert
1408	// afterwards that it is not still None.
1409	AllocNode->AllocTypes = (uint8_t)AllocationType::None;
1410
1411	return AllocNode;
1412	}
1413
1414	static std::string getAllocTypeString(uint8_t AllocTypes) {
1415	if (!AllocTypes)
1416	return "None";
1417	std::string Str;
1418	if (AllocTypes & (uint8_t)AllocationType::NotCold)
1419	Str += "NotCold";
1420	if (AllocTypes & (uint8_t)AllocationType::Cold)
1421	Str += "Cold";
1422	return Str;
1423	}
1424
1425	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1426	template <class NodeT, class IteratorT>
1427	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::addStackNodesForMIB(
1428	ContextNode *AllocNode, CallStack<NodeT, IteratorT> &StackContext,
1429	CallStack<NodeT, IteratorT> &CallsiteContext, AllocationType AllocType,
1430	ArrayRef<ContextTotalSize> ContextSizeInfo,
1431	std::map<uint64_t, uint32_t> &TotalSizeToContextIdTopNCold) {
1432	// Treating the hot alloc type as NotCold before the disambiguation for "hot"
1433	// is done.
1434	if (AllocType == AllocationType::Hot)
1435	AllocType = AllocationType::NotCold;
1436
1437	ContextIdToAllocationType [++LastContextId] = AllocType;
1438
1439	bool IsImportant = false;
1440	if (!ContextSizeInfo.empty()) {
1441	auto &Entry = ContextIdToContextSizeInfos [LastContextId];
1442	// If this is a cold allocation, and we are collecting non-zero largest
1443	// contexts, see if this is a candidate.
1444	if (AllocType == AllocationType::Cold && MemProfTopNImportant > `0`) {
1445	uint64_t TotalCold = `0`;
1446	for (auto &CSI : ContextSizeInfo)
1447	TotalCold += CSI.TotalSize;
1448	// Record this context if either we haven't found the first top-n largest
1449	// yet, or if it is larger than the smallest already recorded.
1450	if (TotalSizeToContextIdTopNCold.size() < MemProfTopNImportant \|\|
1451	// Since TotalSizeToContextIdTopNCold is a std::map, it is implicitly
1452	// sorted in ascending size of its key which is the size.
1453	TotalCold > TotalSizeToContextIdTopNCold.begin()->first) {
1454	if (TotalSizeToContextIdTopNCold.size() == MemProfTopNImportant) {
1455	// Remove old one and its associated entries.
1456	auto IdToRemove = TotalSizeToContextIdTopNCold.begin()->second;
1457	TotalSizeToContextIdTopNCold.erase(
1458	position: TotalSizeToContextIdTopNCold.begin());
1459	assert(ImportantContextIdInfo.count(IdToRemove));
1460	ImportantContextIdInfo.erase(IdToRemove);
1461	}
1462	TotalSizeToContextIdTopNCold [TotalCold] = LastContextId;
1463	IsImportant = true;
1464	}
1465	}
1466	Entry.insert(position: Entry.begin(), first: ContextSizeInfo.begin(), last: ContextSizeInfo.end());
1467	}
1468
1469	// Update alloc type and context ids for this MIB.
1470	AllocNode->AllocTypes \|= (uint8_t)AllocType;
1471
1472	// Now add or update nodes for each stack id in alloc's context.
1473	// Later when processing the stack ids on non-alloc callsites we will adjust
1474	// for any inlining in the context.
1475	ContextNode *PrevNode = AllocNode;
1476	// Look for recursion (direct recursion should have been collapsed by
1477	// module summary analysis, here we should just be detecting mutual
1478	// recursion). Mark these nodes so we don't try to clone.
1479	SmallSet<uint64_t, `8`> StackIdSet;
1480	// Skip any on the allocation call (inlining).
1481	for (auto ContextIter = StackContext.beginAfterSharedPrefix(CallsiteContext);
1482	ContextIter != StackContext.end(); ++ContextIter) {
1483	auto StackId = getStackId(IdOrIndex: *ContextIter);
1484	if (IsImportant)
1485	ImportantContextIdInfo[LastContextId].StackIds.push_back(StackId);
1486	ContextNode *StackNode = getNodeForStackId(StackId);
1487	if (!StackNode) {
1488	StackNode = createNewNode(/IsAllocation=/false);
1489	StackEntryIdToContextNodeMap[StackId] = StackNode;
1490	StackNode->OrigStackOrAllocId = StackId;
1491	}
1492	// Marking a node recursive will prevent its cloning completely, even for
1493	// non-recursive contexts flowing through it.
1494	if (!AllowRecursiveCallsites) {
1495	auto Ins = StackIdSet.insert(StackId);
1496	if (!Ins.second)
1497	StackNode->Recursive = true;
1498	}
1499	StackNode->AllocTypes \|= (uint8_t)AllocType;
1500	PrevNode->addOrUpdateCallerEdge(StackNode, AllocType, LastContextId);
1501	PrevNode = StackNode;
1502	}
1503	}
1504
1505	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1506	DenseSet<uint32_t>
1507	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::duplicateContextIds(
1508	const DenseSet<uint32_t> &StackSequenceContextIds,
1509	DenseMap<uint32_t, DenseSet<uint32_t>> &OldToNewContextIds) {
1510	DenseSet<uint32_t> NewContextIds;
1511	for (auto OldId : StackSequenceContextIds) {
1512	NewContextIds.insert(V: ++LastContextId);
1513	OldToNewContextIds [OldId].insert(V: LastContextId);
1514	assert(ContextIdToAllocationType.count(OldId));
1515	// The new context has the same allocation type and size info as original.
1516	ContextIdToAllocationType [LastContextId] = ContextIdToAllocationType [OldId];
1517	auto CSI = ContextIdToContextSizeInfos.find(Val: OldId);
1518	if (CSI != ContextIdToContextSizeInfos.end())
1519	ContextIdToContextSizeInfos [LastContextId] = CSI ->second;
1520	if (DotAllocContextIds.contains(V: OldId))
1521	DotAllocContextIds.insert(V: LastContextId);
1522	}
1523	return NewContextIds;
1524	}
1525
1526	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1527	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
1528	propagateDuplicateContextIds(
1529	const DenseMap<uint32_t, DenseSet<uint32_t>> &OldToNewContextIds) {
1530	// Build a set of duplicated context ids corresponding to the input id set.
1531	auto GetNewIds = [&OldToNewContextIds](const DenseSet<uint32_t> &ContextIds) {
1532	DenseSet<uint32_t> NewIds;
1533	for (auto Id : ContextIds)
1534	if (auto NewId = OldToNewContextIds.find(Val: Id);
1535	NewId != OldToNewContextIds.end())
1536	NewIds.insert_range(R: NewId ->second);
1537	return NewIds;
1538	};
1539
1540	// Recursively update context ids sets along caller edges.
1541	auto UpdateCallers = [&](ContextNode *Node,
1542	DenseSet<const ContextEdge *> &Visited,
1543	auto &&UpdateCallers) -> void {
1544	for (const auto &Edge : Node->CallerEdges) {
1545	auto Inserted = Visited.insert(Edge.get());
1546	if (!Inserted.second)
1547	continue;
1548	ContextNode *NextNode = Edge->Caller;
1549	DenseSet<uint32_t> NewIdsToAdd = GetNewIds(Edge->getContextIds());
1550	// Only need to recursively iterate to NextNode via this caller edge if
1551	// it resulted in any added ids to NextNode.
1552	if (!NewIdsToAdd.empty()) {
1553	Edge->getContextIds().insert_range(NewIdsToAdd);
1554	UpdateCallers(NextNode, Visited, UpdateCallers);
1555	}
1556	}
1557	};
1558
1559	DenseSet<const ContextEdge *> Visited;
1560	for (auto &Entry : AllocationCallToContextNodeMap) {
1561	auto *Node = Entry.second;
1562	UpdateCallers(Node, Visited, UpdateCallers);
1563	}
1564	}
1565
1566	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1567	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::connectNewNode(
1568	ContextNode NewNode, ContextNode OrigNode, bool TowardsCallee,
1569	// This must be passed by value to make a copy since it will be adjusted
1570	// as ids are moved.
1571	DenseSet<uint32_t> RemainingContextIds) {
1572	auto &OrigEdges =
1573	TowardsCallee ? OrigNode->CalleeEdges : OrigNode->CallerEdges;
1574	DenseSet<uint32_t> RecursiveContextIds;
1575	DenseSet<uint32_t> AllCallerContextIds;
1576	if (AllowRecursiveCallsites) {
1577	// Identify which context ids are recursive which is needed to properly
1578	// update the RemainingContextIds set. The relevant recursive context ids
1579	// are those that are in multiple edges.
1580	for (auto &CE : OrigEdges) {
1581	AllCallerContextIds.reserve(Size: CE->getContextIds().size());
1582	for (auto Id : CE->getContextIds())
1583	if (!AllCallerContextIds.insert(Id).second)
1584	RecursiveContextIds.insert(Id);
1585	}
1586	}
1587	// Increment iterator in loop so that we can remove edges as needed.
1588	for (auto EI = OrigEdges.begin(); EI != OrigEdges.end();) {
1589	auto Edge = *EI;
1590	DenseSet<uint32_t> NewEdgeContextIds;
1591	// Remove any matching context ids from Edge, return set that were found and
1592	// removed, these are the new edge's context ids.
1593	set_subtract(Edge->getContextIds(), RemainingContextIds, NewEdgeContextIds);
1594	// If no matching context ids for this edge, skip it.
1595	if (NewEdgeContextIds.empty()) {
1596	++EI;
1597	continue;
1598	}
1599	// Update the remaining context ids set for the later edges. This is a
1600	// compile time optimization.
1601	if (RecursiveContextIds.empty()) {
1602	set_subtract(S1&: RemainingContextIds, S2: NewEdgeContextIds);
1603	} else {
1604	// Keep the recursive ids in the remaining set as we expect to see those
1605	// on another edge. We can remove the non-recursive remaining ids that
1606	// were seen on this edge, however. We already have the set of remaining
1607	// ids that were on this edge (in NewEdgeContextIds). Figure out which are
1608	// non-recursive and only remove those. Note that despite the higher
1609	// overhead of updating the remaining context ids set when recursion
1610	// handling is enabled, it was found to be at worst performance neutral
1611	// and in one case a clear win.
1612	DenseSet<uint32_t> NonRecursiveRemainingCurEdgeIds =
1613	set_difference(S1: NewEdgeContextIds, S2: RecursiveContextIds);
1614	set_subtract(S1&: RemainingContextIds, S2: NonRecursiveRemainingCurEdgeIds);
1615	}
1616	if (TowardsCallee) {
1617	uint8_t NewAllocType = computeAllocType(ContextIds&: NewEdgeContextIds);
1618	auto NewEdge = std::make_shared<ContextEdge>(
1619	Edge->Callee, NewNode, NewAllocType, std::move(NewEdgeContextIds));
1620	NewNode->CalleeEdges.push_back(NewEdge);
1621	NewEdge->Callee->CallerEdges.push_back(NewEdge);
1622	} else {
1623	uint8_t NewAllocType = computeAllocType(ContextIds&: NewEdgeContextIds);
1624	auto NewEdge = std::make_shared<ContextEdge>(
1625	NewNode, Edge->Caller, NewAllocType, std::move(NewEdgeContextIds));
1626	NewNode->CallerEdges.push_back(NewEdge);
1627	NewEdge->Caller->CalleeEdges.push_back(NewEdge);
1628	}
1629	// Remove old edge if context ids empty.
1630	if (Edge->getContextIds().empty()) {
1631	removeEdgeFromGraph(Edge: Edge.get(), EI: &EI, CalleeIter: TowardsCallee);
1632	continue;
1633	}
1634	++EI;
1635	}
1636	}
1637
1638	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1639	static void checkEdge(
1640	const std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>> &Edge) {
1641	// Confirm that alloc type is not None and that we have at least one context
1642	// id.
1643	assert(Edge->AllocTypes != (uint8_t)AllocationType::None);
1644	assert(!Edge->ContextIds.empty());
1645	}
1646
1647	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1648	static void checkNode(const ContextNode<DerivedCCG, FuncTy, CallTy> *Node,
1649	bool CheckEdges = true) {
1650	if (Node->isRemoved())
1651	return;
1652	#ifndef NDEBUG
1653	// Compute node's context ids once for use in asserts.
1654	auto NodeContextIds = Node->getContextIds();
1655	#endif
1656	// Node's context ids should be the union of both its callee and caller edge
1657	// context ids.
1658	if (Node->CallerEdges.size()) {
1659	DenseSet<uint32_t> CallerEdgeContextIds(
1660	Node->CallerEdges.front()->ContextIds);
1661	for (const auto &Edge : llvm::drop_begin(Node->CallerEdges)) {
1662	if (CheckEdges)
1663	checkEdge<DerivedCCG, FuncTy, CallTy>(Edge);
1664	set_union(CallerEdgeContextIds, Edge->ContextIds);
1665	}
1666	// Node can have more context ids than callers if some contexts terminate at
1667	// node and some are longer. If we are allowing recursive callsites and
1668	// contexts this will be violated for incompletely cloned recursive cycles,
1669	// so skip the checking in that case.
1670	assert((AllowRecursiveCallsites && AllowRecursiveContexts) \|\|
1671	NodeContextIds == CallerEdgeContextIds \|\|
1672	set_is_subset(CallerEdgeContextIds, NodeContextIds));
1673	}
1674	if (Node->CalleeEdges.size()) {
1675	DenseSet<uint32_t> CalleeEdgeContextIds(
1676	Node->CalleeEdges.front()->ContextIds);
1677	for (const auto &Edge : llvm::drop_begin(Node->CalleeEdges)) {
1678	if (CheckEdges)
1679	checkEdge<DerivedCCG, FuncTy, CallTy>(Edge);
1680	set_union(CalleeEdgeContextIds, Edge->getContextIds());
1681	}
1682	// If we are allowing recursive callsites and contexts this will be violated
1683	// for incompletely cloned recursive cycles, so skip the checking in that
1684	// case.
1685	assert((AllowRecursiveCallsites && AllowRecursiveContexts) \|\|
1686	NodeContextIds == CalleeEdgeContextIds);
1687	}
1688	// FIXME: Since this checking is only invoked under an option, we should
1689	// change the error checking from using assert to something that will trigger
1690	// an error on a release build.
1691	#ifndef NDEBUG
1692	// Make sure we don't end up with duplicate edges between the same caller and
1693	// callee.
1694	DenseSet<ContextNode<DerivedCCG, FuncTy, CallTy> *> NodeSet;
1695	for (const auto &E : Node->CalleeEdges)
1696	NodeSet.insert(E->Callee);
1697	assert(NodeSet.size() == Node->CalleeEdges.size());
1698	#endif
1699	}
1700
1701	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1702	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
1703	assignStackNodesPostOrder(ContextNode *Node,
1704	DenseSet<const ContextNode *> &Visited,
1705	DenseMap<uint64_t, std::vector<CallContextInfo>>
1706	&StackIdToMatchingCalls,
1707	DenseMap<CallInfo, CallInfo> &CallToMatchingCall,
1708	const DenseSet<uint32_t> &ImportantContextIds) {
1709	auto Inserted = Visited.insert(Node);
1710	if (!Inserted.second)
1711	return;
1712	// Post order traversal. Iterate over a copy since we may add nodes and
1713	// therefore new callers during the recursive call, invalidating any
1714	// iterator over the original edge vector. We don't need to process these
1715	// new nodes as they were already processed on creation.
1716	auto CallerEdges = Node->CallerEdges;
1717	for (auto &Edge : CallerEdges) {
1718	// Skip any that have been removed during the recursion.
1719	if (Edge->isRemoved()) {
1720	assert(!is_contained(Node->CallerEdges, Edge));
1721	continue;
1722	}
1723	assignStackNodesPostOrder(Node: Edge->Caller, Visited, StackIdToMatchingCalls,
1724	CallToMatchingCall, ImportantContextIds);
1725	}
1726
1727	// If this node's stack id is in the map, update the graph to contain new
1728	// nodes representing any inlining at interior callsites. Note we move the
1729	// associated context ids over to the new nodes.
1730
1731	// Ignore this node if it is for an allocation or we didn't record any
1732	// stack id lists ending at it.
1733	if (Node->IsAllocation \|\|
1734	!StackIdToMatchingCalls.count(Node->OrigStackOrAllocId))
1735	return;
1736
1737	auto &Calls = StackIdToMatchingCalls[Node->OrigStackOrAllocId];
1738	// Handle the simple case first. A single call with a single stack id.
1739	// In this case there is no need to create any new context nodes, simply
1740	// assign the context node for stack id to this Call.
1741	if (Calls.size() == `1`) {
1742	auto &[Call, Ids, Func, SavedContextIds] = Calls[`0`];
1743	if (Ids.size() == `1`) {
1744	assert(SavedContextIds.empty());
1745	// It should be this Node
1746	assert(Node == getNodeForStackId(Ids[`0`]));
1747	if (Node->Recursive)
1748	return;
1749	Node->setCall(Call);
1750	NonAllocationCallToContextNodeMap[Call] = Node;
1751	NodeToCallingFunc[Node] = Func;
1752	recordStackNode(StackIds&: Ids, Node, NodeContextIds: Node->getContextIds(), ImportantContextIds);
1753	return;
1754	}
1755	}
1756
1757	#ifndef NDEBUG
1758	// Find the node for the last stack id, which should be the same
1759	// across all calls recorded for this id, and is this node's id.
1760	uint64_t LastId = Node->OrigStackOrAllocId;
1761	ContextNode *LastNode = getNodeForStackId(LastId);
1762	// We should only have kept stack ids that had nodes.
1763	assert(LastNode);
1764	assert(LastNode == Node);
1765	#else
1766	ContextNode *LastNode = Node;
1767	#endif
1768
1769	// Compute the last node's context ids once, as it is shared by all calls in
1770	// this entry.
1771	DenseSet<uint32_t> LastNodeContextIds = LastNode->getContextIds();
1772
1773	[[maybe_unused]] bool PrevIterCreatedNode = false;
1774	bool CreatedNode = false;
1775	for (unsigned I = `0`; I < Calls.size();
1776	I++, PrevIterCreatedNode = CreatedNode) {
1777	CreatedNode = false;
1778	auto &[Call, Ids, Func, SavedContextIds] = Calls[I];
1779	// Skip any for which we didn't assign any ids, these don't get a node in
1780	// the graph.
1781	if (SavedContextIds.empty()) {
1782	// If this call has a matching call (located in the same function and
1783	// having the same stack ids), simply add it to the context node created
1784	// for its matching call earlier. These can be treated the same through
1785	// cloning and get updated at the same time.
1786	if (!CallToMatchingCall.contains(Call))
1787	continue;
1788	auto MatchingCall = CallToMatchingCall[Call];
1789	if (!NonAllocationCallToContextNodeMap.contains(MatchingCall)) {
1790	// This should only happen if we had a prior iteration, and it didn't
1791	// create a node because of the below recomputation of context ids
1792	// finding none remaining and continuing early.
1793	assert(I > `0` && !PrevIterCreatedNode);
1794	continue;
1795	}
1796	NonAllocationCallToContextNodeMap[MatchingCall]->MatchingCalls.push_back(
1797	Call);
1798	continue;
1799	}
1800
1801	assert(LastId == Ids.back());
1802
1803	// Recompute the context ids for this stack id sequence (the
1804	// intersection of the context ids of the corresponding nodes).
1805	// Start with the ids we saved in the map for this call, which could be
1806	// duplicated context ids. We have to recompute as we might have overlap
1807	// overlap between the saved context ids for different last nodes, and
1808	// removed them already during the post order traversal.
1809	set_intersect(SavedContextIds, LastNodeContextIds);
1810	ContextNode *PrevNode = LastNode;
1811	bool Skip = false;
1812	// Iterate backwards through the stack Ids, starting after the last Id
1813	// in the list, which was handled once outside for all Calls.
1814	for (auto IdIter = Ids.rbegin() + `1`; IdIter != Ids.rend(); IdIter++) {
1815	auto Id = *IdIter;
1816	ContextNode *CurNode = getNodeForStackId(StackId: Id);
1817	// We should only have kept stack ids that had nodes and weren't
1818	// recursive.
1819	assert(CurNode);
1820	assert(!CurNode->Recursive);
1821
1822	auto *Edge = CurNode->findEdgeFromCaller(PrevNode);
1823	if (!Edge) {
1824	Skip = true;
1825	break;
1826	}
1827	PrevNode = CurNode;
1828
1829	// Update the context ids, which is the intersection of the ids along
1830	// all edges in the sequence.
1831	set_intersect(SavedContextIds, Edge->getContextIds());
1832
1833	// If we now have no context ids for clone, skip this call.
1834	if (SavedContextIds.empty()) {
1835	Skip = true;
1836	break;
1837	}
1838	}
1839	if (Skip)
1840	continue;
1841
1842	// Create new context node.
1843	ContextNode NewNode = createNewNode(/IsAllocation=/*false, F: Func, C: Call);
1844	NonAllocationCallToContextNodeMap[Call] = NewNode;
1845	CreatedNode = true;
1846	NewNode->AllocTypes = computeAllocType(ContextIds&: SavedContextIds);
1847
1848	ContextNode *FirstNode = getNodeForStackId(StackId: Ids[`0`]);
1849	assert(FirstNode);
1850
1851	// Connect to callees of innermost stack frame in inlined call chain.
1852	// This updates context ids for FirstNode's callee's to reflect those
1853	// moved to NewNode.
1854	connectNewNode(NewNode, OrigNode: FirstNode, /TowardsCallee=/true, RemainingContextIds: SavedContextIds);
1855
1856	// Connect to callers of outermost stack frame in inlined call chain.
1857	// This updates context ids for FirstNode's caller's to reflect those
1858	// moved to NewNode.
1859	connectNewNode(NewNode, OrigNode: LastNode, /TowardsCallee=/false, RemainingContextIds: SavedContextIds);
1860
1861	// Now we need to remove context ids from edges/nodes between First and
1862	// Last Node.
1863	PrevNode = nullptr;
1864	for (auto Id : Ids) {
1865	ContextNode *CurNode = getNodeForStackId(StackId: Id);
1866	// We should only have kept stack ids that had nodes.
1867	assert(CurNode);
1868
1869	// Remove the context ids moved to NewNode from CurNode, and the
1870	// edge from the prior node.
1871	if (PrevNode) {
1872	auto *PrevEdge = CurNode->findEdgeFromCallee(PrevNode);
1873	// If the sequence contained recursion, we might have already removed
1874	// some edges during the connectNewNode calls above.
1875	if (!PrevEdge) {
1876	PrevNode = CurNode;
1877	continue;
1878	}
1879	set_subtract(PrevEdge->getContextIds(), SavedContextIds);
1880	if (PrevEdge->getContextIds().empty())
1881	removeEdgeFromGraph(Edge: PrevEdge);
1882	}
1883	// Since we update the edges from leaf to tail, only look at the callee
1884	// edges. This isn't an alloc node, so if there are no callee edges, the
1885	// alloc type is None.
1886	CurNode->AllocTypes = CurNode->CalleeEdges.empty()
1887	? (uint8_t)AllocationType::None
1888	: CurNode->computeAllocType();
1889	PrevNode = CurNode;
1890	}
1891
1892	recordStackNode(StackIds&: Ids, Node: NewNode, NodeContextIds: SavedContextIds, ImportantContextIds);
1893
1894	if (VerifyNodes) {
1895	checkNode<DerivedCCG, FuncTy, CallTy>(NewNode, /CheckEdges=/true);
1896	for (auto Id : Ids) {
1897	ContextNode *CurNode = getNodeForStackId(StackId: Id);
1898	// We should only have kept stack ids that had nodes.
1899	assert(CurNode);
1900	checkNode<DerivedCCG, FuncTy, CallTy>(CurNode, /CheckEdges=/true);
1901	}
1902	}
1903	}
1904	}
1905
1906	template <typename DerivedCCG, typename FuncTy, typename CallTy>
1907	void CallsiteContextGraph<DerivedCCG, FuncTy,
1908	CallTy>::fixupImportantContexts() {
1909	if (ImportantContextIdInfo.empty())
1910	return;
1911
1912	// Update statistics as we are done building this map at this point.
1913	NumImportantContextIds = ImportantContextIdInfo.size();
1914
1915	if (!MemProfFixupImportant)
1916	return;
1917
1918	if (ExportToDot)
1919	exportToDot(Label: "beforestackfixup");
1920
1921	// For each context we identified as important, walk through the saved context
1922	// stack ids in order from leaf upwards, and make sure all edges are correct.
1923	// These can be difficult to get right when updating the graph while mapping
1924	// nodes onto summary or IR, especially when there is recursion. In
1925	// particular, when we have created new nodes to reflect inlining, it is
1926	// sometimes impossible to know exactly how to update the edges in the face of
1927	// recursion, as we have lost the original ordering of the stack ids in the
1928	// contexts.
1929	// TODO: Consider only doing this if we detect the context has recursive
1930	// cycles.
1931	//
1932	// I.e. assume we have a context with stack ids like: {A B A C A D E}
1933	// and let's say A was inlined into B, C, and D. The original graph will have
1934	// multiple recursive cycles through A. When we match the original context
1935	// nodes onto the IR or summary, we will merge {A B} into one context node,
1936	// {A C} onto another, and {A D} onto another. Looking at the stack sequence
1937	// above, we should end up with a non-cyclic set of edges like:
1938	// {AB} <- {AC} <- {AD} <- E. However, because we normally have lost the
1939	// original ordering, we won't get the edges correct initially (it's
1940	// impossible without the original ordering). Here we do the fixup (add and
1941	// removing edges where necessary) for this context. In the
1942	// ImportantContextInfo struct in this case we should have a MaxLength = 2,
1943	// and map entries for {A B}, {A C}, {A D}, and {E}.
1944	for (auto &[CurContextId, Info] : ImportantContextIdInfo) {
1945	if (Info.StackIdsToNode.empty())
1946	continue;
1947	bool Changed = false;
1948	ContextNode PrevNode = nullptr*;
1949	ContextNode CurNode = nullptr*;
1950	DenseSet<const ContextEdge *> VisitedEdges;
1951	ArrayRef<uint64_t> AllStackIds(Info.StackIds);
1952	// Try to identify what callsite ContextNode maps to which slice of the
1953	// context's ordered stack ids.
1954	for (unsigned I = `0`; I < AllStackIds.size(); I++, PrevNode = CurNode) {
1955	// We will do this greedily, trying up to MaxLength stack ids in a row, to
1956	// see if we recorded a context node for that sequence.
1957	auto Len = Info.MaxLength;
1958	auto LenToEnd = AllStackIds.size() - I;
1959	if (Len > LenToEnd)
1960	Len = LenToEnd;
1961	CurNode = nullptr;
1962	// Try to find a recorded context node starting with the longest length
1963	// recorded, and on down until we check for just a single stack node.
1964	for (; Len > `0`; Len--) {
1965	// Get the slice of the original stack id sequence to check.
1966	auto CheckStackIds = AllStackIds.slice(I, Len);
1967	auto EntryIt = Info.StackIdsToNode.find(CheckStackIds);
1968	if (EntryIt == Info.StackIdsToNode.end())
1969	continue;
1970	CurNode = EntryIt->second;
1971	// Skip forward so we don't try to look for the ones we just matched.
1972	// We increment by Len - 1, because the outer for loop will increment I.
1973	I += Len - `1`;
1974	break;
1975	}
1976	// Give up if we couldn't find a node. Since we need to clone from the
1977	// leaf allocation upwards, no sense in doing anymore fixup further up
1978	// the context if we couldn't match part of the original stack context
1979	// onto a callsite node.
1980	if (!CurNode)
1981	break;
1982	// No edges to fix up until we have a pair of nodes that should be
1983	// adjacent in the graph.
1984	if (!PrevNode)
1985	continue;
1986	// See if we already have a call edge from CurNode to PrevNode.
1987	auto *CurEdge = PrevNode->findEdgeFromCaller(CurNode);
1988	if (CurEdge) {
1989	// We already have an edge. Make sure it contains this context id.
1990	if (CurEdge->getContextIds().insert(CurContextId).second) {
1991	NumFixupEdgeIdsInserted ++;
1992	Changed = true;
1993	}
1994	} else {
1995	// No edge exists - add one.
1996	NumFixupEdgesAdded ++;
1997	DenseSet<uint32_t> ContextIds({CurContextId});
1998	auto AllocType = computeAllocType(ContextIds);
1999	auto NewEdge = std::make_shared<ContextEdge>(
2000	PrevNode, CurNode, AllocType, std::move(ContextIds));
2001	PrevNode->CallerEdges.push_back(NewEdge);
2002	CurNode->CalleeEdges.push_back(NewEdge);
2003	// Save the new edge for the below handling.
2004	CurEdge = NewEdge.get();
2005	Changed = true;
2006	}
2007	VisitedEdges.insert(CurEdge);
2008	// Now remove this context id from any other caller edges calling
2009	// PrevNode.
2010	for (auto &Edge : PrevNode->CallerEdges) {
2011	// Skip the edge updating/created above and edges we have already
2012	// visited (due to recursion).
2013	if (Edge.get() != CurEdge && !VisitedEdges.contains(Edge.get()))
2014	Edge->getContextIds().erase(CurContextId);
2015	}
2016	}
2017	if (Changed)
2018	NumFixedContexts ++;
2019	}
2020	}
2021
2022	template <typename DerivedCCG, typename FuncTy, typename CallTy>
2023	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::updateStackNodes() {
2024	// Map of stack id to all calls with that as the last (outermost caller)
2025	// callsite id that has a context node (some might not due to pruning
2026	// performed during matching of the allocation profile contexts).
2027	// The CallContextInfo contains the Call and a list of its stack ids with
2028	// ContextNodes, the function containing Call, and the set of context ids
2029	// the analysis will eventually identify for use in any new node created
2030	// for that callsite.
2031	DenseMap<uint64_t, std::vector<CallContextInfo>> StackIdToMatchingCalls;
2032	for (auto &[Func, CallsWithMetadata] : FuncToCallsWithMetadata) {
2033	for (auto &Call : CallsWithMetadata) {
2034	// Ignore allocations, already handled.
2035	if (AllocationCallToContextNodeMap.count(Call))
2036	continue;
2037	auto StackIdsWithContextNodes =
2038	getStackIdsWithContextNodesForCall(Call: Call.call());
2039	// If there were no nodes created for MIBs on allocs (maybe this was in
2040	// the unambiguous part of the MIB stack that was pruned), ignore.
2041	if (StackIdsWithContextNodes.empty())
2042	continue;
2043	// Otherwise, record this Call along with the list of ids for the last
2044	// (outermost caller) stack id with a node.
2045	StackIdToMatchingCalls[StackIdsWithContextNodes.back()].push_back(
2046	{Call.call(), StackIdsWithContextNodes, Func, {}});
2047	}
2048	}
2049
2050	// First make a pass through all stack ids that correspond to a call,
2051	// as identified in the above loop. Compute the context ids corresponding to
2052	// each of these calls when they correspond to multiple stack ids due to
2053	// due to inlining. Perform any duplication of context ids required when
2054	// there is more than one call with the same stack ids. Their (possibly newly
2055	// duplicated) context ids are saved in the StackIdToMatchingCalls map.
2056	DenseMap<uint32_t, DenseSet<uint32_t>> OldToNewContextIds;
2057	// Save a map from each call to any that are found to match it. I.e. located
2058	// in the same function and have the same (possibly pruned) stack ids. We use
2059	// this to avoid creating extra graph nodes as they can be treated the same.
2060	DenseMap<CallInfo, CallInfo> CallToMatchingCall;
2061	for (auto &It : StackIdToMatchingCalls) {
2062	auto &Calls = It.getSecond();
2063	// Skip single calls with a single stack id. These don't need a new node.
2064	if (Calls.size() == `1`) {
2065	auto &Ids = Calls[`0`].StackIds;
2066	if (Ids.size() == `1`)
2067	continue;
2068	}
2069	// In order to do the best and maximal matching of inlined calls to context
2070	// node sequences we will sort the vectors of stack ids in descending order
2071	// of length, and within each length, lexicographically by stack id. The
2072	// latter is so that we can specially handle calls that have identical stack
2073	// id sequences (either due to cloning or artificially because of the MIB
2074	// context pruning). Those with the same Ids are then sorted by function to
2075	// facilitate efficiently mapping them to the same context node.
2076	// Because the functions are pointers, to ensure a stable sort first assign
2077	// each function pointer to its first index in the Calls array, and then use
2078	// that to sort by.
2079	DenseMap<const FuncTy , unsigned*> FuncToIndex;
2080	for (const auto &[Idx, CallCtxInfo] : enumerate(Calls))
2081	FuncToIndex.insert({CallCtxInfo.Func, Idx});
2082	llvm::stable_sort(
2083	Calls,
2084	[&FuncToIndex](const CallContextInfo &A, const CallContextInfo &B) {
2085	return A.StackIds.size() > B.StackIds.size() \|\|
2086	(A.StackIds.size() == B.StackIds.size() &&
2087	(A.StackIds < B.StackIds \|\|
2088	(A.StackIds == B.StackIds &&
2089	FuncToIndex[A.Func] < FuncToIndex[B.Func])));
2090	});
2091
2092	// Find the node for the last stack id, which should be the same
2093	// across all calls recorded for this id, and is the id for this
2094	// entry in the StackIdToMatchingCalls map.
2095	uint64_t LastId = It.getFirst();
2096	ContextNode *LastNode = getNodeForStackId(StackId: LastId);
2097	// We should only have kept stack ids that had nodes.
2098	assert(LastNode);
2099
2100	if (LastNode->Recursive)
2101	continue;
2102
2103	// Initialize the context ids with the last node's. We will subsequently
2104	// refine the context ids by computing the intersection along all edges.
2105	DenseSet<uint32_t> LastNodeContextIds = LastNode->getContextIds();
2106	assert(!LastNodeContextIds.empty());
2107
2108	#ifndef NDEBUG
2109	// Save the set of functions seen for a particular set of the same stack
2110	// ids. This is used to ensure that they have been correctly sorted to be
2111	// adjacent in the Calls list, since we rely on that to efficiently place
2112	// all such matching calls onto the same context node.
2113	DenseSet<const FuncTy *> MatchingIdsFuncSet;
2114	#endif
2115
2116	for (unsigned I = `0`; I < Calls.size(); I++) {
2117	auto &[Call, Ids, Func, SavedContextIds] = Calls[I];
2118	assert(SavedContextIds.empty());
2119	assert(LastId == Ids.back());
2120
2121	#ifndef NDEBUG
2122	// If this call has a different set of ids than the last one, clear the
2123	// set used to ensure they are sorted properly.
2124	if (I > `0` && Ids != Calls[I - `1`].StackIds)
2125	MatchingIdsFuncSet.clear();
2126	#endif
2127
2128	// First compute the context ids for this stack id sequence (the
2129	// intersection of the context ids of the corresponding nodes).
2130	// Start with the remaining saved ids for the last node.
2131	assert(!LastNodeContextIds.empty());
2132	DenseSet<uint32_t> StackSequenceContextIds = LastNodeContextIds;
2133
2134	ContextNode *PrevNode = LastNode;
2135	ContextNode *CurNode = LastNode;
2136	bool Skip = false;
2137
2138	// Iterate backwards through the stack Ids, starting after the last Id
2139	// in the list, which was handled once outside for all Calls.
2140	for (auto IdIter = Ids.rbegin() + `1`; IdIter != Ids.rend(); IdIter++) {
2141	auto Id = *IdIter;
2142	CurNode = getNodeForStackId(StackId: Id);
2143	// We should only have kept stack ids that had nodes.
2144	assert(CurNode);
2145
2146	if (CurNode->Recursive) {
2147	Skip = true;
2148	break;
2149	}
2150
2151	auto *Edge = CurNode->findEdgeFromCaller(PrevNode);
2152	// If there is no edge then the nodes belong to different MIB contexts,
2153	// and we should skip this inlined context sequence. For example, this
2154	// particular inlined context may include stack ids A->B, and we may
2155	// indeed have nodes for both A and B, but it is possible that they were
2156	// never profiled in sequence in a single MIB for any allocation (i.e.
2157	// we might have profiled an allocation that involves the callsite A,
2158	// but through a different one of its callee callsites, and we might
2159	// have profiled an allocation that involves callsite B, but reached
2160	// from a different caller callsite).
2161	if (!Edge) {
2162	Skip = true;
2163	break;
2164	}
2165	PrevNode = CurNode;
2166
2167	// Update the context ids, which is the intersection of the ids along
2168	// all edges in the sequence.
2169	set_intersect(StackSequenceContextIds, Edge->getContextIds());
2170
2171	// If we now have no context ids for clone, skip this call.
2172	if (StackSequenceContextIds.empty()) {
2173	Skip = true;
2174	break;
2175	}
2176	}
2177	if (Skip)
2178	continue;
2179
2180	// If some of this call's stack ids did not have corresponding nodes (due
2181	// to pruning), don't include any context ids for contexts that extend
2182	// beyond these nodes. Otherwise we would be matching part of unrelated /
2183	// not fully matching stack contexts. To do this, subtract any context ids
2184	// found in caller nodes of the last node found above.
2185	if (Ids.back() != getLastStackId(Call)) {
2186	for (const auto &PE : LastNode->CallerEdges) {
2187	set_subtract(StackSequenceContextIds, PE->getContextIds());
2188	if (StackSequenceContextIds.empty())
2189	break;
2190	}
2191	// If we now have no context ids for clone, skip this call.
2192	if (StackSequenceContextIds.empty())
2193	continue;
2194	}
2195
2196	#ifndef NDEBUG
2197	// If the prior call had the same stack ids this set would not be empty.
2198	// Check if we already have a call that "matches" because it is located
2199	// in the same function. If the Calls list was sorted properly we should
2200	// not encounter this situation as all such entries should be adjacent
2201	// and processed in bulk further below.
2202	assert(!MatchingIdsFuncSet.contains(Func));
2203
2204	MatchingIdsFuncSet.insert(Func);
2205	#endif
2206
2207	// Check if the next set of stack ids is the same (since the Calls vector
2208	// of tuples is sorted by the stack ids we can just look at the next one).
2209	// If so, save them in the CallToMatchingCall map so that they get
2210	// assigned to the same context node, and skip them.
2211	bool DuplicateContextIds = false;
2212	for (unsigned J = I + `1`; J < Calls.size(); J++) {
2213	auto &CallCtxInfo = Calls[J];
2214	auto &NextIds = CallCtxInfo.StackIds;
2215	if (NextIds != Ids)
2216	break;
2217	auto *NextFunc = CallCtxInfo.Func;
2218	if (NextFunc != Func) {
2219	// We have another Call with the same ids but that cannot share this
2220	// node, must duplicate ids for it.
2221	DuplicateContextIds = true;
2222	break;
2223	}
2224	auto &NextCall = CallCtxInfo.Call;
2225	CallToMatchingCall[NextCall] = Call;
2226	// Update I so that it gets incremented correctly to skip this call.
2227	I = J;
2228	}
2229
2230	// If we don't have duplicate context ids, then we can assign all the
2231	// context ids computed for the original node sequence to this call.
2232	// If there are duplicate calls with the same stack ids then we synthesize
2233	// new context ids that are duplicates of the originals. These are
2234	// assigned to SavedContextIds, which is a reference into the map entry
2235	// for this call, allowing us to access these ids later on.
2236	OldToNewContextIds.reserve(NumEntries: OldToNewContextIds.size() +
2237	StackSequenceContextIds.size());
2238	SavedContextIds =
2239	DuplicateContextIds
2240	? duplicateContextIds(StackSequenceContextIds, OldToNewContextIds)
2241	: StackSequenceContextIds;
2242	assert(!SavedContextIds.empty());
2243
2244	if (!DuplicateContextIds) {
2245	// Update saved last node's context ids to remove those that are
2246	// assigned to other calls, so that it is ready for the next call at
2247	// this stack id.
2248	set_subtract(S1&: LastNodeContextIds, S2: StackSequenceContextIds);
2249	if (LastNodeContextIds.empty())
2250	break;
2251	}
2252	}
2253	}
2254
2255	// Propagate the duplicate context ids over the graph.
2256	propagateDuplicateContextIds(OldToNewContextIds);
2257
2258	if (VerifyCCG)
2259	check();
2260
2261	// Now perform a post-order traversal over the graph, starting with the
2262	// allocation nodes, essentially processing nodes from callers to callees.
2263	// For any that contains an id in the map, update the graph to contain new
2264	// nodes representing any inlining at interior callsites. Note we move the
2265	// associated context ids over to the new nodes.
2266	DenseSet<const ContextNode *> Visited;
2267	DenseSet<uint32_t> ImportantContextIds(llvm::from_range,
2268	ImportantContextIdInfo.keys());
2269	for (auto &Entry : AllocationCallToContextNodeMap)
2270	assignStackNodesPostOrder(Node: Entry.second, Visited, StackIdToMatchingCalls,
2271	CallToMatchingCall, ImportantContextIds);
2272
2273	fixupImportantContexts();
2274
2275	if (VerifyCCG)
2276	check();
2277	}
2278
2279	uint64_t ModuleCallsiteContextGraph::getLastStackId(Instruction *Call) {
2280	CallStack<MDNode, MDNode::op_iterator> CallsiteContext(
2281	Call->getMetadata(KindID: LLVMContext::MD_callsite));
2282	return CallsiteContext.back();
2283	}
2284
2285	uint64_t IndexCallsiteContextGraph::getLastStackId(IndexCall &Call) {
2286	assert(isa<CallsiteInfo *>(Call));
2287	CallStack<CallsiteInfo, SmallVector<unsigned>::const_iterator>
2288	CallsiteContext(dyn_cast_if_present<CallsiteInfo *>(Val&: Call));
2289	// Need to convert index into stack id.
2290	return Index.getStackIdAtIndex(Index: CallsiteContext.back());
2291	}
2292
2293	static const std::string MemProfCloneSuffix = ".memprof.";
2294
2295	static std::string getMemProfFuncName(Twine Base, unsigned CloneNo) {
2296	// We use CloneNo == 0 to refer to the original version, which doesn't get
2297	// renamed with a suffix.
2298	if (!CloneNo)
2299	return Base.str();
2300	return (Base + MemProfCloneSuffix + Twine (CloneNo)).str();
2301	}
2302
2303	static bool isMemProfClone(const Function &F) {
2304	return F.getName().contains(Other: MemProfCloneSuffix);
2305	}
2306
2307	// Return the clone number of the given function by extracting it from the
2308	// memprof suffix. Assumes the caller has already confirmed it is a memprof
2309	// clone.
2310	static unsigned getMemProfCloneNum(const Function &F) {
2311	assert(isMemProfClone(F));
2312	auto Pos = F.getName().find_last_of(C: `'.'`);
2313	assert(Pos > `0`);
2314	unsigned CloneNo;
2315	bool Err = F.getName().drop_front(N: Pos + `1`).getAsInteger(Radix: `10`, Result&: CloneNo);
2316	assert(!Err);
2317	(void)Err;
2318	return CloneNo;
2319	}
2320
2321	std::string ModuleCallsiteContextGraph::getLabel(const Function *Func,
2322	const Instruction *Call,
2323	unsigned CloneNo) const {
2324	return (Twine (Call->getFunction()->getName()) + " -> " +
2325	cast<CallBase>(Val: Call)->getCalledFunction()->getName())
2326	.str();
2327	}
2328
2329	std::string IndexCallsiteContextGraph::getLabel(const FunctionSummary *Func,
2330	const IndexCall &Call,
2331	unsigned CloneNo) const {
2332	auto VI = FSToVIMap.find(x: Func);
2333	assert(VI != FSToVIMap.end());
2334	std::string CallerName = getMemProfFuncName(Base: VI ->second.name(), CloneNo);
2335	if (isa<AllocInfo *>(Val: Call))
2336	return CallerName + " -> alloc";
2337	else {
2338	auto Callsite = dyn_cast_if_present<CallsiteInfo >(Val: Call);
2339	return CallerName + " -> " +
2340	getMemProfFuncName(Base: Callsite->Callee.name(),
2341	CloneNo: Callsite->Clones [CloneNo]);
2342	}
2343	}
2344
2345	std::vector<uint64_t>
2346	ModuleCallsiteContextGraph::getStackIdsWithContextNodesForCall(
2347	Instruction *Call) {
2348	CallStack<MDNode, MDNode::op_iterator> CallsiteContext(
2349	Call->getMetadata(KindID: LLVMContext::MD_callsite));
2350	return getStackIdsWithContextNodes<MDNode, MDNode::op_iterator>(
2351	CallsiteContext);
2352	}
2353
2354	std::vector<uint64_t>
2355	IndexCallsiteContextGraph::getStackIdsWithContextNodesForCall(IndexCall &Call) {
2356	assert(isa<CallsiteInfo *>(Call));
2357	CallStack<CallsiteInfo, SmallVector<unsigned>::const_iterator>
2358	CallsiteContext(dyn_cast_if_present<CallsiteInfo *>(Val&: Call));
2359	return getStackIdsWithContextNodes<CallsiteInfo,
2360	SmallVector<unsigned>::const_iterator>(
2361	CallsiteContext);
2362	}
2363
2364	template <typename DerivedCCG, typename FuncTy, typename CallTy>
2365	template <class NodeT, class IteratorT>
2366	std::vector<uint64_t>
2367	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::getStackIdsWithContextNodes(
2368	CallStack<NodeT, IteratorT> &CallsiteContext) {
2369	std::vector<uint64_t> StackIds;
2370	for (auto IdOrIndex : CallsiteContext) {
2371	auto StackId = getStackId(IdOrIndex);
2372	ContextNode *Node = getNodeForStackId(StackId);
2373	if (!Node)
2374	break;
2375	StackIds.push_back(StackId);
2376	}
2377	return StackIds;
2378	}
2379
2380	ModuleCallsiteContextGraph::ModuleCallsiteContextGraph(
2381	Module &M,
2382	llvm::function_ref<OptimizationRemarkEmitter &(Function *)> OREGetter)
2383	: Mod(M), OREGetter (OREGetter) {
2384	// Map for keeping track of the largest cold contexts up to the number given
2385	// by MemProfTopNImportant. Must be a std::map (not DenseMap) because keys
2386	// must be sorted.
2387	std::map<uint64_t, uint32_t> TotalSizeToContextIdTopNCold;
2388	for (auto &F : M) {
2389	std::vector<CallInfo> CallsWithMetadata;
2390	for (auto &BB : F) {
2391	for (auto &I : BB) {
2392	if (!isa<CallBase>(Val: I))
2393	continue;
2394	if (auto *MemProfMD = I.getMetadata(KindID: LLVMContext::MD_memprof)) {
2395	CallsWithMetadata.push_back(x: &I);
2396	auto *AllocNode = addAllocNode(Call: &I, F: &F);
2397	auto *CallsiteMD = I.getMetadata(KindID: LLVMContext::MD_callsite);
2398	assert(CallsiteMD);
2399	CallStack<MDNode, MDNode::op_iterator> CallsiteContext(CallsiteMD);
2400	// Add all of the MIBs and their stack nodes.
2401	for (auto &MDOp : MemProfMD->operands()) {
2402	auto MIBMD = cast<const* MDNode>(Val: MDOp);
2403	std::vector<ContextTotalSize> ContextSizeInfo;
2404	// Collect the context size information if it exists.
2405	if (MIBMD->getNumOperands() > `2`) {
2406	for (unsigned I = `2`; I < MIBMD->getNumOperands(); I++) {
2407	MDNode *ContextSizePair =
2408	dyn_cast<MDNode>(Val: MIBMD->getOperand(I));
2409	assert(ContextSizePair->getNumOperands() == `2`);
2410	uint64_t FullStackId = mdconst::dyn_extract<ConstantInt>(
2411	MD: ContextSizePair->getOperand(I: `0`))
2412	->getZExtValue();
2413	uint64_t TotalSize = mdconst::dyn_extract<ConstantInt>(
2414	MD: ContextSizePair->getOperand(I: `1`))
2415	->getZExtValue();
2416	ContextSizeInfo.push_back(x: {.FullStackId: FullStackId, .TotalSize: TotalSize});
2417	}
2418	}
2419	MDNode *StackNode = getMIBStackNode(MIB: MIBMD);
2420	assert(StackNode);
2421	CallStack<MDNode, MDNode::op_iterator> StackContext(StackNode);
2422	addStackNodesForMIB<MDNode, MDNode::op_iterator>(
2423	AllocNode, StackContext, CallsiteContext,
2424	AllocType: getMIBAllocType(MIB: MIBMD), ContextSizeInfo,
2425	TotalSizeToContextIdTopNCold);
2426	}
2427	// If exporting the graph to dot and an allocation id of interest was
2428	// specified, record all the context ids for this allocation node.
2429	if (ExportToDot && AllocNode->OrigStackOrAllocId == AllocIdForDot)
2430	DotAllocContextIds = AllocNode->getContextIds();
2431	assert(AllocNode->AllocTypes != (uint8_t)AllocationType::None);
2432	// Memprof and callsite metadata on memory allocations no longer
2433	// needed.
2434	I.setMetadata(KindID: LLVMContext::MD_memprof, Node: nullptr);
2435	I.setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
2436	}
2437	// For callsite metadata, add to list for this function for later use.
2438	else if (I.getMetadata(KindID: LLVMContext::MD_callsite)) {
2439	CallsWithMetadata.push_back(x: &I);
2440	}
2441	}
2442	}
2443	if (!CallsWithMetadata.empty())
2444	FuncToCallsWithMetadata [&F] = CallsWithMetadata;
2445	}
2446
2447	if (DumpCCG) {
2448	dbgs() << "CCG before updating call stack chains:\n";
2449	dbgs() << *this;
2450	}
2451
2452	if (ExportToDot)
2453	exportToDot(Label: "prestackupdate");
2454
2455	updateStackNodes();
2456
2457	if (ExportToDot)
2458	exportToDot(Label: "poststackupdate");
2459
2460	handleCallsitesWithMultipleTargets();
2461
2462	markBackedges();
2463
2464	// Strip off remaining callsite metadata, no longer needed.
2465	for (auto &FuncEntry : FuncToCallsWithMetadata)
2466	for (auto &Call : FuncEntry.second)
2467	Call.call()->setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
2468	}
2469
2470	// Finds the set of GUIDs for weak aliasees that are prevailing in different
2471	// modules than any of their aliases. We need to handle these specially.
2472	DenseSet<GlobalValue::GUID>
2473	IndexCallsiteContextGraph::findAliaseeGUIDsPrevailingInDifferentModule() {
2474	DenseSet<GlobalValue::GUID> AliaseeGUIDs;
2475	for (auto &I : Index) {
2476	auto VI = Index.getValueInfo(R: I);
2477	for (auto &S : VI.getSummaryList()) {
2478	// We only care about aliases to functions.
2479	auto *AS = dyn_cast<AliasSummary>(Val: S.get());
2480	if (!AS)
2481	continue;
2482	auto *AliaseeSummary = &AS->getAliasee();
2483	auto *AliaseeFS = dyn_cast<FunctionSummary>(Val: AliaseeSummary);
2484	if (!AliaseeFS)
2485	continue;
2486	// Skip this summary if it is not for the prevailing symbol for this GUID.
2487	// The linker doesn't resolve local linkage values so don't check whether
2488	// those are prevailing.
2489	if (!GlobalValue::isLocalLinkage(Linkage: S ->linkage()) &&
2490	!isPrevailing (VI.getGUID(), S.get()))
2491	continue;
2492	// Prevailing aliasee could be in a different module only if it is weak.
2493	if (!GlobalValue::isWeakForLinker(Linkage: AliaseeSummary->linkage()))
2494	continue;
2495	auto AliaseeGUID = AS->getAliaseeGUID();
2496	// If the aliasee copy in this module is not prevailing, record it.
2497	if (!isPrevailing (AliaseeGUID, AliaseeSummary))
2498	AliaseeGUIDs.insert(V: AliaseeGUID);
2499	}
2500	}
2501	AliaseesPrevailingInDiffModuleFromAlias += AliaseeGUIDs.size();
2502	return AliaseeGUIDs;
2503	}
2504
2505	IndexCallsiteContextGraph::IndexCallsiteContextGraph(
2506	ModuleSummaryIndex &Index,
2507	llvm::function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
2508	isPrevailing)
2509	: Index(Index), isPrevailing (isPrevailing) {
2510	// Since we use the aliasee summary info to create the necessary clones for
2511	// its aliases, conservatively skip recording the aliasee function's callsites
2512	// in the CCG for any that are prevailing in a different module than one of
2513	// its aliases. We could record the necessary information to do this in the
2514	// summary, but this case should not be common.
2515	DenseSet<GlobalValue::GUID> GUIDsToSkip =
2516	findAliaseeGUIDsPrevailingInDifferentModule();
2517	// Map for keeping track of the largest cold contexts up to the number given
2518	// by MemProfTopNImportant. Must be a std::map (not DenseMap) because keys
2519	// must be sorted.
2520	std::map<uint64_t, uint32_t> TotalSizeToContextIdTopNCold;
2521	// Sort by GUID for deterministic graph construction order.
2522	// TODO: This sort has a measurable cost on the thin link when memprof is
2523	// enabled. Investigate gating it behind an option that is only enabled for
2524	// tests that check internal state.
2525	for (const auto &I : Index.sortedGlobalValueSummariesRange()) {
2526	auto VI = Index.getValueInfo(R: I);
2527	if (GUIDsToSkip.contains(V: VI.getGUID()))
2528	continue;
2529	for (auto &S : VI.getSummaryList()) {
2530	// We should only add the prevailing nodes. Otherwise we may try to clone
2531	// in a weak copy that won't be linked (and may be different than the
2532	// prevailing version).
2533	// We only keep the memprof summary on the prevailing copy now when
2534	// building the combined index, as a space optimization, however don't
2535	// rely on this optimization. The linker doesn't resolve local linkage
2536	// values so don't check whether those are prevailing.
2537	if (!GlobalValue::isLocalLinkage(Linkage: S ->linkage()) &&
2538	!isPrevailing (VI.getGUID(), S.get()))
2539	continue;
2540	auto *FS = dyn_cast<FunctionSummary>(Val: S.get());
2541	if (!FS)
2542	continue;
2543	std::vector<CallInfo> CallsWithMetadata;
2544	if (!FS->allocs().empty()) {
2545	for (auto &AN : FS->mutableAllocs()) {
2546	// This can happen because of recursion elimination handling that
2547	// currently exists in ModuleSummaryAnalysis. Skip these for now.
2548	// We still added them to the summary because we need to be able to
2549	// correlate properly in applyImport in the backends.
2550	if (AN.MIBs.empty())
2551	continue;
2552	IndexCall AllocCall(&AN);
2553	CallsWithMetadata.push_back(x: AllocCall);
2554	auto *AllocNode = addAllocNode(Call: AllocCall, F: FS);
2555	// Pass an empty CallStack to the CallsiteContext (second)
2556	// parameter, since for ThinLTO we already collapsed out the inlined
2557	// stack ids on the allocation call during ModuleSummaryAnalysis.
2558	CallStack<MIBInfo, SmallVector<unsigned>::const_iterator>
2559	EmptyContext;
2560	unsigned I = `0`;
2561	assert(!metadataMayIncludeContextSizeInfo() \|\|
2562	AN.ContextSizeInfos.size() == AN.MIBs.size());
2563	// Now add all of the MIBs and their stack nodes.
2564	for (auto &MIB : AN.MIBs) {
2565	CallStack<MIBInfo, SmallVector<unsigned>::const_iterator>
2566	StackContext(&MIB);
2567	std::vector<ContextTotalSize> ContextSizeInfo;
2568	if (!AN.ContextSizeInfos.empty()) {
2569	for (auto [FullStackId, TotalSize] : AN.ContextSizeInfos [I])
2570	ContextSizeInfo.push_back(x: {.FullStackId: FullStackId, .TotalSize: TotalSize});
2571	}
2572	addStackNodesForMIB<MIBInfo, SmallVector<unsigned>::const_iterator>(
2573	AllocNode, StackContext, CallsiteContext&: EmptyContext, AllocType: MIB.AllocType,
2574	ContextSizeInfo, TotalSizeToContextIdTopNCold);
2575	I++;
2576	}
2577	// If exporting the graph to dot and an allocation id of interest was
2578	// specified, record all the context ids for this allocation node.
2579	if (ExportToDot && AllocNode->OrigStackOrAllocId == AllocIdForDot)
2580	DotAllocContextIds = AllocNode->getContextIds();
2581	assert(AllocNode->AllocTypes != (uint8_t)AllocationType::None);
2582	// Initialize version 0 on the summary alloc node to the current alloc
2583	// type, unless it has both types in which case make it default, so
2584	// that in the case where we aren't able to clone the original version
2585	// always ends up with the default allocation behavior.
2586	AN.Versions [`0`] = (uint8_t)allocTypeToUse(AllocTypes: AllocNode->AllocTypes);
2587	}
2588	}
2589	// For callsite metadata, add to list for this function for later use.
2590	if (!FS->callsites().empty())
2591	for (auto &SN : FS->mutableCallsites()) {
2592	IndexCall StackNodeCall(&SN);
2593	CallsWithMetadata.push_back(x: StackNodeCall);
2594	}
2595
2596	if (!CallsWithMetadata.empty())
2597	FuncToCallsWithMetadata [FS] = CallsWithMetadata;
2598
2599	if (!FS->allocs().empty() \|\| !FS->callsites().empty())
2600	FSToVIMap [FS] = VI;
2601	}
2602	}
2603
2604	if (DumpCCG) {
2605	dbgs() << "CCG before updating call stack chains:\n";
2606	dbgs() << *this;
2607	}
2608
2609	if (ExportToDot)
2610	exportToDot(Label: "prestackupdate");
2611
2612	updateStackNodes();
2613
2614	if (ExportToDot)
2615	exportToDot(Label: "poststackupdate");
2616
2617	handleCallsitesWithMultipleTargets();
2618
2619	markBackedges();
2620	}
2621
2622	template <typename DerivedCCG, typename FuncTy, typename CallTy>
2623	void CallsiteContextGraph<DerivedCCG, FuncTy,
2624	CallTy>::handleCallsitesWithMultipleTargets() {
2625	// Look for and workaround callsites that call multiple functions.
2626	// This can happen for indirect calls, which needs better handling, and in
2627	// more rare cases (e.g. macro expansion).
2628	// TODO: To fix this for indirect calls we will want to perform speculative
2629	// devirtualization using either the normal PGO info with ICP, or using the
2630	// information in the profiled MemProf contexts. We can do this prior to
2631	// this transformation for regular LTO, and for ThinLTO we can simulate that
2632	// effect in the summary and perform the actual speculative devirtualization
2633	// while cloning in the ThinLTO backend.
2634
2635	// Keep track of the new nodes synthesized for discovered tail calls missing
2636	// from the profiled contexts.
2637	MapVector<CallInfo, ContextNode *> TailCallToContextNodeMap;
2638
2639	std::vector<std::pair<CallInfo, ContextNode *>> NewCallToNode;
2640	for (auto &Entry : NonAllocationCallToContextNodeMap) {
2641	auto *Node = Entry.second;
2642	assert(Node->Clones.empty());
2643	// Check all node callees and see if in the same function.
2644	// We need to check all of the calls recorded in this Node, because in some
2645	// cases we may have had multiple calls with the same debug info calling
2646	// different callees. This can happen, for example, when an object is
2647	// constructed in the paramter list - the destructor call of the object has
2648	// the same debug info (line/col) as the call the object was passed to.
2649	// Here we will prune any that don't match all callee nodes.
2650	std::vector<CallInfo> AllCalls;
2651	AllCalls.reserve(Node->MatchingCalls.size() + `1`);
2652	AllCalls.push_back(Node->Call);
2653	llvm::append_range(AllCalls, Node->MatchingCalls);
2654
2655	// First see if we can partition the calls by callee function, creating new
2656	// nodes to host each set of calls calling the same callees. This is
2657	// necessary for support indirect calls with ThinLTO, for which we
2658	// synthesized CallsiteInfo records for each target. They will all have the
2659	// same callsite stack ids and would be sharing a context node at this
2660	// point. We need to perform separate cloning for each, which will be
2661	// applied along with speculative devirtualization in the ThinLTO backends
2662	// as needed. Note this does not currently support looking through tail
2663	// calls, it is unclear if we need that for indirect call targets.
2664	// First partition calls by callee func. Map indexed by func, value is
2665	// struct with list of matching calls, assigned node.
2666	if (partitionCallsByCallee(Node, AllCalls, NewCallToNode))
2667	continue;
2668
2669	auto It = AllCalls.begin();
2670	// Iterate through the calls until we find the first that matches.
2671	for (; It != AllCalls.end(); ++It) {
2672	auto ThisCall = *It;
2673	bool Match = true;
2674	for (auto EI = Node->CalleeEdges.begin(); EI != Node->CalleeEdges.end();
2675	++EI) {
2676	auto Edge = *EI;
2677	if (!Edge->Callee->hasCall())
2678	continue;
2679	assert(NodeToCallingFunc.count(Edge->Callee));
2680	// Check if the called function matches that of the callee node.
2681	if (!calleesMatch(Call: ThisCall.call(), EI, TailCallToContextNodeMap)) {
2682	Match = false;
2683	break;
2684	}
2685	}
2686	// Found a call that matches the callee nodes, we can quit now.
2687	if (Match) {
2688	// If the first match is not the primary call on the Node, update it
2689	// now. We will update the list of matching calls further below.
2690	if (Node->Call != ThisCall) {
2691	Node->setCall(ThisCall);
2692	// We need to update the NonAllocationCallToContextNodeMap, but don't
2693	// want to do this during iteration over that map, so save the calls
2694	// that need updated entries.
2695	NewCallToNode.push_back({ThisCall, Node});
2696	}
2697	break;
2698	}
2699	}
2700	// We will update this list below (or leave it cleared if there was no
2701	// match found above).
2702	Node->MatchingCalls.clear();
2703	// If we hit the end of the AllCalls vector, no call matching the callee
2704	// nodes was found, clear the call information in the node.
2705	if (It == AllCalls.end()) {
2706	RemovedEdgesWithMismatchedCallees ++;
2707	// Work around by setting Node to have a null call, so it gets
2708	// skipped during cloning. Otherwise assignFunctions will assert
2709	// because its data structures are not designed to handle this case.
2710	Node->setCall(CallInfo());
2711	continue;
2712	}
2713	// Now add back any matching calls that call the same function as the
2714	// matching primary call on Node.
2715	for (++It; It != AllCalls.end(); ++It) {
2716	auto ThisCall = *It;
2717	if (!sameCallee(Call1: Node->Call.call(), Call2: ThisCall.call()))
2718	continue;
2719	Node->MatchingCalls.push_back(ThisCall);
2720	}
2721	}
2722
2723	// Remove all mismatched nodes identified in the above loop from the node map
2724	// (checking whether they have a null call which is set above). For a
2725	// MapVector like NonAllocationCallToContextNodeMap it is much more efficient
2726	// to do the removal via remove_if than by individually erasing entries above.
2727	// Also remove any entries if we updated the node's primary call above.
2728	NonAllocationCallToContextNodeMap.remove_if([](const auto &it) {
2729	return !it.second->hasCall() \|\| it.second->Call != it.first;
2730	});
2731
2732	// Add entries for any new primary calls recorded above.
2733	for (auto &[Call, Node] : NewCallToNode)
2734	NonAllocationCallToContextNodeMap[Call] = Node;
2735
2736	// Add the new nodes after the above loop so that the iteration is not
2737	// invalidated.
2738	for (auto &[Call, Node] : TailCallToContextNodeMap)
2739	NonAllocationCallToContextNodeMap[Call] = Node;
2740	}
2741
2742	template <typename DerivedCCG, typename FuncTy, typename CallTy>
2743	bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::partitionCallsByCallee(
2744	ContextNode *Node, ArrayRef<CallInfo> AllCalls,
2745	std::vector<std::pair<CallInfo, ContextNode *>> &NewCallToNode) {
2746	// Struct to keep track of all the calls having the same callee function,
2747	// and the node we eventually assign to them. Eventually we will record the
2748	// context node assigned to this group of calls.
2749	struct CallsWithSameCallee {
2750	std::vector<CallInfo> Calls;
2751	ContextNode Node = nullptr*;
2752	};
2753
2754	// First partition calls by callee function. Build map from each function
2755	// to the list of matching calls.
2756	DenseMap<const FuncTy *, CallsWithSameCallee> CalleeFuncToCallInfo;
2757	for (auto ThisCall : AllCalls) {
2758	auto *F = getCalleeFunc(Call: ThisCall.call());
2759	if (F)
2760	CalleeFuncToCallInfo[F].Calls.push_back(ThisCall);
2761	}
2762
2763	// Next, walk through all callee edges. For each callee node, get its
2764	// containing function and see if it was recorded in the above map (meaning we
2765	// have at least one matching call). Build another map from each callee node
2766	// with a matching call to the structure instance created above containing all
2767	// the calls.
2768	DenseMap<ContextNode , CallsWithSameCallee > CalleeNodeToCallInfo;
2769	for (const auto &Edge : Node->CalleeEdges) {
2770	if (!Edge->Callee->hasCall())
2771	continue;
2772	const FuncTy *ProfiledCalleeFunc = NodeToCallingFunc[Edge->Callee];
2773	if (CalleeFuncToCallInfo.contains(ProfiledCalleeFunc))
2774	CalleeNodeToCallInfo[Edge->Callee] =
2775	&CalleeFuncToCallInfo[ProfiledCalleeFunc];
2776	}
2777
2778	// If there are entries in the second map, then there were no matching
2779	// calls/callees, nothing to do here. Return so we can go to the handling that
2780	// looks through tail calls.
2781	if (CalleeNodeToCallInfo.empty())
2782	return false;
2783
2784	// Walk through all callee edges again. Any and all callee edges that didn't
2785	// match any calls (callee not in the CalleeNodeToCallInfo map) are moved to a
2786	// new caller node (UnmatchedCalleesNode) which gets a null call so that it is
2787	// ignored during cloning. If it is in the map, then we use the node recorded
2788	// in that entry (creating it if needed), and move the callee edge to it.
2789	// The first callee will use the original node instead of creating a new one.
2790	// Note that any of the original calls on this node (in AllCalls) that didn't
2791	// have a callee function automatically get dropped from the node as part of
2792	// this process.
2793	ContextNode UnmatchedCalleesNode = nullptr*;
2794	// Track whether we already assigned original node to a callee.
2795	bool UsedOrigNode = false;
2796	assert(NodeToCallingFunc[Node]);
2797	// Iterate over a copy of Node's callee edges, since we may need to remove
2798	// edges in moveCalleeEdgeToNewCaller, and this simplifies the handling and
2799	// makes it less error-prone.
2800	auto CalleeEdges = Node->CalleeEdges;
2801	for (auto &Edge : CalleeEdges) {
2802	if (!Edge->Callee->hasCall())
2803	continue;
2804
2805	// Will be updated below to point to whatever (caller) node this callee edge
2806	// should be moved to.
2807	ContextNode CallerNodeToUse = nullptr*;
2808
2809	// Handle the case where there were no matching calls first. Move this
2810	// callee edge to the UnmatchedCalleesNode, creating it if needed.
2811	if (!CalleeNodeToCallInfo.contains(Edge->Callee)) {
2812	if (!UnmatchedCalleesNode)
2813	UnmatchedCalleesNode =
2814	createNewNode(/IsAllocation=/false, F: NodeToCallingFunc[Node]);
2815	CallerNodeToUse = UnmatchedCalleesNode;
2816	} else {
2817	// Look up the information recorded for this callee node, and use the
2818	// recorded caller node (creating it if needed).
2819	auto *Info = CalleeNodeToCallInfo[Edge->Callee];
2820	if (!Info->Node) {
2821	// If we haven't assigned any callees to the original node use it.
2822	if (!UsedOrigNode) {
2823	Info->Node = Node;
2824	// Clear the set of matching calls which will be updated below.
2825	Node->MatchingCalls.clear();
2826	UsedOrigNode = true;
2827	} else
2828	Info->Node =
2829	createNewNode(/IsAllocation=/false, F: NodeToCallingFunc[Node]);
2830	assert(!Info->Calls.empty());
2831	// The first call becomes the primary call for this caller node, and the
2832	// rest go in the matching calls list.
2833	Info->Node->setCall(Info->Calls.front());
2834	llvm::append_range(Info->Node->MatchingCalls,
2835	llvm::drop_begin(Info->Calls));
2836	// Save the primary call to node correspondence so that we can update
2837	// the NonAllocationCallToContextNodeMap, which is being iterated in the
2838	// caller of this function.
2839	NewCallToNode.push_back({Info->Node->Call, Info->Node});
2840	}
2841	CallerNodeToUse = Info->Node;
2842	}
2843
2844	// Don't need to move edge if we are using the original node;
2845	if (CallerNodeToUse == Node)
2846	continue;
2847
2848	moveCalleeEdgeToNewCaller(Edge, NewCaller: CallerNodeToUse);
2849	}
2850	// Now that we are done moving edges, clean up any caller edges that ended
2851	// up with no type or context ids. During moveCalleeEdgeToNewCaller all
2852	// caller edges from Node are replicated onto the new callers, and it
2853	// simplifies the handling to leave them until we have moved all
2854	// edges/context ids.
2855	for (auto &I : CalleeNodeToCallInfo)
2856	removeNoneTypeCallerEdges(Node: I.second->Node);
2857	if (UnmatchedCalleesNode)
2858	removeNoneTypeCallerEdges(Node: UnmatchedCalleesNode);
2859	removeNoneTypeCallerEdges(Node);
2860
2861	return true;
2862	}
2863
2864	uint64_t ModuleCallsiteContextGraph::getStackId(uint64_t IdOrIndex) const {
2865	// In the Module (IR) case this is already the Id.
2866	return IdOrIndex;
2867	}
2868
2869	uint64_t IndexCallsiteContextGraph::getStackId(uint64_t IdOrIndex) const {
2870	// In the Index case this is an index into the stack id list in the summary
2871	// index, convert it to an Id.
2872	return Index.getStackIdAtIndex(Index: IdOrIndex);
2873	}
2874
2875	template <typename DerivedCCG, typename FuncTy, typename CallTy>
2876	bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::calleesMatch(
2877	CallTy Call, EdgeIter &EI,
2878	MapVector<CallInfo, ContextNode *> &TailCallToContextNodeMap) {
2879	auto Edge = *EI;
2880	const FuncTy *ProfiledCalleeFunc = NodeToCallingFunc[Edge->Callee];
2881	const FuncTy *CallerFunc = NodeToCallingFunc[Edge->Caller];
2882	// Will be populated in order of callee to caller if we find a chain of tail
2883	// calls between the profiled caller and callee.
2884	std::vector<std::pair<CallTy, FuncTy *>> FoundCalleeChain;
2885	if (!calleeMatchesFunc(Call, Func: ProfiledCalleeFunc, CallerFunc,
2886	FoundCalleeChain))
2887	return false;
2888
2889	// The usual case where the profiled callee matches that of the IR/summary.
2890	if (FoundCalleeChain.empty())
2891	return true;
2892
2893	auto AddEdge = [Edge, &EI](ContextNode Caller, ContextNode Callee) {
2894	auto *CurEdge = Callee->findEdgeFromCaller(Caller);
2895	// If there is already an edge between these nodes, simply update it and
2896	// return.
2897	if (CurEdge) {
2898	CurEdge->ContextIds.insert_range(Edge->ContextIds);
2899	CurEdge->AllocTypes \|= Edge->AllocTypes;
2900	return;
2901	}
2902	// Otherwise, create a new edge and insert it into the caller and callee
2903	// lists.
2904	auto NewEdge = std::make_shared<ContextEdge>(
2905	Callee, Caller, Edge->AllocTypes, Edge->ContextIds);
2906	Callee->CallerEdges.push_back(NewEdge);
2907	if (Caller == Edge->Caller) {
2908	// If we are inserting the new edge into the current edge's caller, insert
2909	// the new edge before the current iterator position, and then increment
2910	// back to the current edge.
2911	EI = Caller->CalleeEdges.insert(EI, NewEdge);
2912	++EI;
2913	assert(*EI == Edge &&
2914	"Iterator position not restored after insert and increment");
2915	} else
2916	Caller->CalleeEdges.push_back(NewEdge);
2917	};
2918
2919	// Create new nodes for each found callee and connect in between the profiled
2920	// caller and callee.
2921	auto *CurCalleeNode = Edge->Callee;
2922	for (auto &[NewCall, Func] : FoundCalleeChain) {
2923	ContextNode NewNode = nullptr*;
2924	// First check if we have already synthesized a node for this tail call.
2925	if (TailCallToContextNodeMap.count(NewCall)) {
2926	NewNode = TailCallToContextNodeMap[NewCall];
2927	NewNode->AllocTypes \|= Edge->AllocTypes;
2928	} else {
2929	FuncToCallsWithMetadata[Func].push_back({NewCall});
2930	// Create Node and record node info.
2931	NewNode = createNewNode(/IsAllocation=/false, F: Func, C: NewCall);
2932	TailCallToContextNodeMap[NewCall] = NewNode;
2933	NewNode->AllocTypes = Edge->AllocTypes;
2934	}
2935
2936	// Hook up node to its callee node
2937	AddEdge(NewNode, CurCalleeNode);
2938
2939	CurCalleeNode = NewNode;
2940	}
2941
2942	// Hook up edge's original caller to new callee node.
2943	AddEdge(Edge->Caller, CurCalleeNode);
2944
2945	#ifndef NDEBUG
2946	// Save this because Edge's fields get cleared below when removed.
2947	auto *Caller = Edge->Caller;
2948	#endif
2949
2950	// Remove old edge
2951	removeEdgeFromGraph(Edge: Edge.get(), EI: &EI, /CalleeIter=/true);
2952
2953	// To simplify the increment of EI in the caller, subtract one from EI.
2954	// In the final AddEdge call we would have either added a new callee edge,
2955	// to Edge->Caller, or found an existing one. Either way we are guaranteed
2956	// that there is at least one callee edge.
2957	assert(!Caller->CalleeEdges.empty());
2958	--EI;
2959
2960	return true;
2961	}
2962
2963	bool ModuleCallsiteContextGraph::findProfiledCalleeThroughTailCalls(
2964	const Function ProfiledCallee, Value CurCallee, unsigned Depth,
2965	std::vector<std::pair<Instruction , Function >> &FoundCalleeChain,
2966	bool &FoundMultipleCalleeChains) {
2967	// Stop recursive search if we have already explored the maximum specified
2968	// depth.
2969	if (Depth > TailCallSearchDepth)
2970	return false;
2971
2972	auto SaveCallsiteInfo = [&](Instruction Callsite, Function F) {
2973	FoundCalleeChain.push_back(x: {Callsite, F});
2974	};
2975
2976	auto *CalleeFunc = dyn_cast<Function>(Val: CurCallee);
2977	if (!CalleeFunc) {
2978	auto *Alias = dyn_cast<GlobalAlias>(Val: CurCallee);
2979	assert(Alias);
2980	CalleeFunc = dyn_cast<Function>(Val: Alias->getAliasee());
2981	assert(CalleeFunc);
2982	}
2983
2984	// Look for tail calls in this function, and check if they either call the
2985	// profiled callee directly, or indirectly (via a recursive search).
2986	// Only succeed if there is a single unique tail call chain found between the
2987	// profiled caller and callee, otherwise we could perform incorrect cloning.
2988	bool FoundSingleCalleeChain = false;
2989	for (auto &BB : *CalleeFunc) {
2990	for (auto &I : BB) {
2991	auto *CB = dyn_cast<CallBase>(Val: &I);
2992	if (!CB \|\| !CB->isTailCall())
2993	continue;
2994	auto *CalledValue = CB->getCalledOperand();
2995	auto *CalledFunction = CB->getCalledFunction();
2996	if (CalledValue && !CalledFunction) {
2997	CalledValue = CalledValue->stripPointerCasts();
2998	// Stripping pointer casts can reveal a called function.
2999	CalledFunction = dyn_cast<Function>(Val: CalledValue);
3000	}
3001	// Check if this is an alias to a function. If so, get the
3002	// called aliasee for the checks below.
3003	if (auto *GA = dyn_cast<GlobalAlias>(Val: CalledValue)) {
3004	assert(!CalledFunction &&
3005	"Expected null called function in callsite for alias");
3006	CalledFunction = dyn_cast<Function>(Val: GA->getAliaseeObject());
3007	}
3008	if (!CalledFunction)
3009	continue;
3010	if (CalledFunction == ProfiledCallee) {
3011	if (FoundSingleCalleeChain) {
3012	FoundMultipleCalleeChains = true;
3013	return false;
3014	}
3015	FoundSingleCalleeChain = true;
3016	FoundProfiledCalleeCount ++;
3017	FoundProfiledCalleeDepth += Depth;
3018	if (Depth > FoundProfiledCalleeMaxDepth)
3019	FoundProfiledCalleeMaxDepth = Depth;
3020	SaveCallsiteInfo (&I, CalleeFunc);
3021	} else if (findProfiledCalleeThroughTailCalls(
3022	ProfiledCallee, CurCallee: CalledFunction, Depth: Depth + `1`,
3023	FoundCalleeChain, FoundMultipleCalleeChains)) {
3024	// findProfiledCalleeThroughTailCalls should not have returned
3025	// true if FoundMultipleCalleeChains.
3026	assert(!FoundMultipleCalleeChains);
3027	if (FoundSingleCalleeChain) {
3028	FoundMultipleCalleeChains = true;
3029	return false;
3030	}
3031	FoundSingleCalleeChain = true;
3032	SaveCallsiteInfo (&I, CalleeFunc);
3033	} else if (FoundMultipleCalleeChains)
3034	return false;
3035	}
3036	}
3037
3038	return FoundSingleCalleeChain;
3039	}
3040
3041	const Function ModuleCallsiteContextGraph::getCalleeFunc(Instruction Call) {
3042	auto *CB = dyn_cast<CallBase>(Val: Call);
3043	if (!CB->getCalledOperand() \|\| CB->isIndirectCall())
3044	return nullptr;
3045	auto *CalleeVal = CB->getCalledOperand()->stripPointerCasts();
3046	auto *Alias = dyn_cast<GlobalAlias>(Val: CalleeVal);
3047	if (Alias)
3048	return dyn_cast<Function>(Val: Alias->getAliasee());
3049	return dyn_cast<Function>(Val: CalleeVal);
3050	}
3051
3052	bool ModuleCallsiteContextGraph::calleeMatchesFunc(
3053	Instruction Call, const* Function Func, const* Function *CallerFunc,
3054	std::vector<std::pair<Instruction , Function >> &FoundCalleeChain) {
3055	auto *CB = dyn_cast<CallBase>(Val: Call);
3056	if (!CB->getCalledOperand() \|\| CB->isIndirectCall())
3057	return false;
3058	auto *CalleeVal = CB->getCalledOperand()->stripPointerCasts();
3059	auto *CalleeFunc = dyn_cast<Function>(Val: CalleeVal);
3060	if (CalleeFunc == Func)
3061	return true;
3062	auto *Alias = dyn_cast<GlobalAlias>(Val: CalleeVal);
3063	if (Alias && Alias->getAliasee() == Func)
3064	return true;
3065
3066	// Recursively search for the profiled callee through tail calls starting with
3067	// the actual Callee. The discovered tail call chain is saved in
3068	// FoundCalleeChain, and we will fixup the graph to include these callsites
3069	// after returning.
3070	// FIXME: We will currently redo the same recursive walk if we find the same
3071	// mismatched callee from another callsite. We can improve this with more
3072	// bookkeeping of the created chain of new nodes for each mismatch.
3073	unsigned Depth = `1`;
3074	bool FoundMultipleCalleeChains = false;
3075	if (!findProfiledCalleeThroughTailCalls(ProfiledCallee: Func, CurCallee: CalleeVal, Depth,
3076	FoundCalleeChain,
3077	FoundMultipleCalleeChains)) {
3078	LLVM_DEBUG(dbgs() << "Not found through unique tail call chain: "
3079	<< Func->getName() << " from " << CallerFunc->getName()
3080	<< " that actually called " << CalleeVal->getName()
3081	<< (FoundMultipleCalleeChains
3082	? " (found multiple possible chains)"
3083	: "")
3084	<< "\n");
3085	if (FoundMultipleCalleeChains)
3086	FoundProfiledCalleeNonUniquelyCount ++;
3087	return false;
3088	}
3089
3090	return true;
3091	}
3092
3093	bool ModuleCallsiteContextGraph::sameCallee(Instruction *Call1,
3094	Instruction *Call2) {
3095	auto *CB1 = cast<CallBase>(Val: Call1);
3096	if (!CB1->getCalledOperand() \|\| CB1->isIndirectCall())
3097	return false;
3098	auto *CalleeVal1 = CB1->getCalledOperand()->stripPointerCasts();
3099	auto *CalleeFunc1 = dyn_cast<Function>(Val: CalleeVal1);
3100	auto *CB2 = cast<CallBase>(Val: Call2);
3101	if (!CB2->getCalledOperand() \|\| CB2->isIndirectCall())
3102	return false;
3103	auto *CalleeVal2 = CB2->getCalledOperand()->stripPointerCasts();
3104	auto *CalleeFunc2 = dyn_cast<Function>(Val: CalleeVal2);
3105	return CalleeFunc1 == CalleeFunc2;
3106	}
3107
3108	bool IndexCallsiteContextGraph::findProfiledCalleeThroughTailCalls(
3109	ValueInfo ProfiledCallee, ValueInfo CurCallee, unsigned Depth,
3110	std::vector<std::pair<IndexCall, FunctionSummary *>> &FoundCalleeChain,
3111	bool &FoundMultipleCalleeChains) {
3112	// Stop recursive search if we have already explored the maximum specified
3113	// depth.
3114	if (Depth > TailCallSearchDepth)
3115	return false;
3116
3117	auto CreateAndSaveCallsiteInfo = [&](ValueInfo Callee, FunctionSummary *FS) {
3118	// Make a CallsiteInfo for each discovered callee, if one hasn't already
3119	// been synthesized.
3120	if (!FunctionCalleesToSynthesizedCallsiteInfos.count(Val: FS) \|\|
3121	!FunctionCalleesToSynthesizedCallsiteInfos [FS].count(x: Callee))
3122	// StackIds is empty (we don't have debug info available in the index for
3123	// these callsites)
3124	FunctionCalleesToSynthesizedCallsiteInfos [FS][Callee] =
3125	std::make_unique<CallsiteInfo>(args&: Callee, args: SmallVector<unsigned>());
3126	CallsiteInfo *NewCallsiteInfo =
3127	FunctionCalleesToSynthesizedCallsiteInfos [FS][Callee].get();
3128	FoundCalleeChain.push_back(x: {NewCallsiteInfo, FS});
3129	};
3130
3131	// Look for tail calls in this function, and check if they either call the
3132	// profiled callee directly, or indirectly (via a recursive search).
3133	// Only succeed if there is a single unique tail call chain found between the
3134	// profiled caller and callee, otherwise we could perform incorrect cloning.
3135	bool FoundSingleCalleeChain = false;
3136	for (auto &S : CurCallee.getSummaryList()) {
3137	if (!GlobalValue::isLocalLinkage(Linkage: S ->linkage()) &&
3138	!isPrevailing (CurCallee.getGUID(), S.get()))
3139	continue;
3140	auto *FS = dyn_cast<FunctionSummary>(Val: S ->getBaseObject());
3141	if (!FS)
3142	continue;
3143	auto FSVI = CurCallee;
3144	auto *AS = dyn_cast<AliasSummary>(Val: S.get());
3145	if (AS)
3146	FSVI = AS->getAliaseeVI();
3147	for (auto &CallEdge : FS->calls()) {
3148	if (!CallEdge.second.hasTailCall())
3149	continue;
3150	if (CallEdge.first == ProfiledCallee) {
3151	if (FoundSingleCalleeChain) {
3152	FoundMultipleCalleeChains = true;
3153	return false;
3154	}
3155	FoundSingleCalleeChain = true;
3156	FoundProfiledCalleeCount ++;
3157	FoundProfiledCalleeDepth += Depth;
3158	if (Depth > FoundProfiledCalleeMaxDepth)
3159	FoundProfiledCalleeMaxDepth = Depth;
3160	CreateAndSaveCallsiteInfo (CallEdge.first, FS);
3161	// Add FS to FSToVIMap in case it isn't already there.
3162	assert(!FSToVIMap.count(FS) \|\| FSToVIMap[FS] == FSVI);
3163	FSToVIMap [FS] = FSVI;
3164	} else if (findProfiledCalleeThroughTailCalls(
3165	ProfiledCallee, CurCallee: CallEdge.first, Depth: Depth + `1`,
3166	FoundCalleeChain, FoundMultipleCalleeChains)) {
3167	// findProfiledCalleeThroughTailCalls should not have returned
3168	// true if FoundMultipleCalleeChains.
3169	assert(!FoundMultipleCalleeChains);
3170	if (FoundSingleCalleeChain) {
3171	FoundMultipleCalleeChains = true;
3172	return false;
3173	}
3174	FoundSingleCalleeChain = true;
3175	CreateAndSaveCallsiteInfo (CallEdge.first, FS);
3176	// Add FS to FSToVIMap in case it isn't already there.
3177	assert(!FSToVIMap.count(FS) \|\| FSToVIMap[FS] == FSVI);
3178	FSToVIMap [FS] = FSVI;
3179	} else if (FoundMultipleCalleeChains)
3180	return false;
3181	}
3182	}
3183
3184	return FoundSingleCalleeChain;
3185	}
3186
3187	const FunctionSummary *
3188	IndexCallsiteContextGraph::getCalleeFunc(IndexCall &Call) {
3189	ValueInfo Callee = dyn_cast_if_present<CallsiteInfo *>(Val&: Call)->Callee;
3190	if (Callee.getSummaryList().empty())
3191	return nullptr;
3192	return dyn_cast<FunctionSummary>(Val: Callee.getSummaryList()[`0`]->getBaseObject());
3193	}
3194
3195	bool IndexCallsiteContextGraph::calleeMatchesFunc(
3196	IndexCall &Call, const FunctionSummary *Func,
3197	const FunctionSummary *CallerFunc,
3198	std::vector<std::pair<IndexCall, FunctionSummary *>> &FoundCalleeChain) {
3199	ValueInfo Callee = dyn_cast_if_present<CallsiteInfo *>(Val&: Call)->Callee;
3200	// If there is no summary list then this is a call to an externally defined
3201	// symbol.
3202	AliasSummary *Alias =
3203	Callee.getSummaryList().empty()
3204	? nullptr
3205	: dyn_cast<AliasSummary>(Val: Callee.getSummaryList()[`0`].get());
3206	assert(FSToVIMap.count(Func));
3207	auto FuncVI = FSToVIMap [Func];
3208	if (Callee == FuncVI \|\|
3209	// If callee is an alias, check the aliasee, since only function
3210	// summary base objects will contain the stack node summaries and thus
3211	// get a context node.
3212	(Alias && Alias->getAliaseeVI() == FuncVI))
3213	return true;
3214
3215	// Recursively search for the profiled callee through tail calls starting with
3216	// the actual Callee. The discovered tail call chain is saved in
3217	// FoundCalleeChain, and we will fixup the graph to include these callsites
3218	// after returning.
3219	// FIXME: We will currently redo the same recursive walk if we find the same
3220	// mismatched callee from another callsite. We can improve this with more
3221	// bookkeeping of the created chain of new nodes for each mismatch.
3222	unsigned Depth = `1`;
3223	bool FoundMultipleCalleeChains = false;
3224	if (!findProfiledCalleeThroughTailCalls(
3225	ProfiledCallee: FuncVI, CurCallee: Callee, Depth, FoundCalleeChain, FoundMultipleCalleeChains)) {
3226	LLVM_DEBUG(dbgs() << "Not found through unique tail call chain: " << FuncVI
3227	<< " from " << FSToVIMap[CallerFunc]
3228	<< " that actually called " << Callee
3229	<< (FoundMultipleCalleeChains
3230	? " (found multiple possible chains)"
3231	: "")
3232	<< "\n");
3233	if (FoundMultipleCalleeChains)
3234	FoundProfiledCalleeNonUniquelyCount ++;
3235	return false;
3236	}
3237
3238	return true;
3239	}
3240
3241	bool IndexCallsiteContextGraph::sameCallee(IndexCall &Call1, IndexCall &Call2) {
3242	ValueInfo Callee1 = dyn_cast_if_present<CallsiteInfo *>(Val&: Call1)->Callee;
3243	ValueInfo Callee2 = dyn_cast_if_present<CallsiteInfo *>(Val&: Call2)->Callee;
3244	return Callee1 == Callee2;
3245	}
3246
3247	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3248	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::dump()
3249	const {
3250	print(OS&: dbgs());
3251	dbgs() << "\n";
3252	}
3253
3254	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3255	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::print(
3256	raw_ostream &OS) const {
3257	OS << "Node " << this << "\n";
3258	OS << "\t";
3259	printCall(OS);
3260	if (Recursive)
3261	OS << " (recursive)";
3262	OS << "\n";
3263	if (!MatchingCalls.empty()) {
3264	OS << "\tMatchingCalls:\n";
3265	for (auto &MatchingCall : MatchingCalls) {
3266	OS << "\t";
3267	MatchingCall.print(OS);
3268	OS << "\n";
3269	}
3270	}
3271	OS << "\tNodeId: " << NodeId << "\n";
3272	OS << "\tAllocTypes: " << getAllocTypeString(AllocTypes) << "\n";
3273	OS << "\tContextIds:";
3274	// Make a copy of the computed context ids that we can sort for stability.
3275	auto ContextIds = getContextIds();
3276	std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
3277	std::sort(first: SortedIds.begin(), last: SortedIds.end());
3278	for (auto Id : SortedIds)
3279	OS << " " << Id;
3280	OS << "\n";
3281	OS << "\tCalleeEdges:\n";
3282	for (auto &Edge : CalleeEdges)
3283	OS << "\t\t" << *Edge << " (Callee NodeId: " << Edge->Callee->NodeId
3284	<< ")\n";
3285	OS << "\tCallerEdges:\n";
3286	for (auto &Edge : CallerEdges)
3287	OS << "\t\t" << *Edge << " (Caller NodeId: " << Edge->Caller->NodeId
3288	<< ")\n";
3289	if (!Clones.empty()) {
3290	OS << "\tClones: ";
3291	ListSeparator LS;
3292	for (auto *C : Clones)
3293	OS << LS << C << " NodeId: " << C->NodeId;
3294	OS << "\n";
3295	} else if (CloneOf) {
3296	OS << "\tClone of " << CloneOf << " NodeId: " << CloneOf->NodeId << "\n";
3297	}
3298	}
3299
3300	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3301	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge::dump()
3302	const {
3303	print(OS&: dbgs());
3304	dbgs() << "\n";
3305	}
3306
3307	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3308	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge::print(
3309	raw_ostream &OS) const {
3310	OS << "Edge from Callee " << Callee << " to Caller: " << Caller
3311	<< (IsBackedge ? " (BE)" : "")
3312	<< " AllocTypes: " << getAllocTypeString(AllocTypes);
3313	OS << " ContextIds:";
3314	std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
3315	std::sort(first: SortedIds.begin(), last: SortedIds.end());
3316	for (auto Id : SortedIds)
3317	OS << " " << Id;
3318	}
3319
3320	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3321	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::dump() const {
3322	print(OS&: dbgs());
3323	}
3324
3325	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3326	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::print(
3327	raw_ostream &OS) const {
3328	OS << "Callsite Context Graph:\n";
3329	using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
3330	for (const auto Node : nodes<GraphType>(this)) {
3331	if (Node->isRemoved())
3332	continue;
3333	Node->print(OS);
3334	OS << "\n";
3335	}
3336	}
3337
3338	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3339	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::printTotalSizes(
3340	raw_ostream &OS,
3341	function_ref<void(StringRef, StringRef, const Twine &)> EmitRemark) const {
3342	using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
3343	for (const auto Node : nodes<GraphType>(this)) {
3344	if (Node->isRemoved())
3345	continue;
3346	if (!Node->IsAllocation)
3347	continue;
3348	DenseSet<uint32_t> ContextIds = Node->getContextIds();
3349	auto AllocTypeFromCall = getAllocationCallType(Call: Node->Call);
3350	std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
3351	std::sort(first: SortedIds.begin(), last: SortedIds.end());
3352	for (auto Id : SortedIds) {
3353	auto TypeI = ContextIdToAllocationType.find(Val: Id);
3354	assert(TypeI != ContextIdToAllocationType.end());
3355	auto CSI = ContextIdToContextSizeInfos.find(Val: Id);
3356	if (CSI != ContextIdToContextSizeInfos.end()) {
3357	for (auto &Info : CSI ->second) {
3358	std::string Msg =
3359	"MemProf hinting: " + getAllocTypeString(AllocTypes: (uint8_t)TypeI ->second) +
3360	" full allocation context " + std::to_string(val: Info.FullStackId) +
3361	" with total size " + std::to_string(val: Info.TotalSize) + " is " +
3362	getAllocTypeString(Node->AllocTypes) + " after cloning";
3363	if (allocTypeToUse(Node->AllocTypes) != AllocTypeFromCall)
3364	Msg += " marked " + getAllocTypeString(AllocTypes: (uint8_t)AllocTypeFromCall) +
3365	" due to cold byte percent";
3366	// Print the internal context id to aid debugging and visualization.
3367	Msg += " (internal context id " + std::to_string(val: Id) + ")";
3368	if (MemProfReportHintedSizes)
3369	OS << Msg << "\n";
3370	if (EmitRemark)
3371	EmitRemark (DEBUG_TYPE, "MemProfReport", Msg);
3372	}
3373	} else {
3374	// This is only emitted if the context size info is not present.
3375	std::string Msg =
3376	"MemProf hinting: " + getAllocTypeString(AllocTypes: (uint8_t)TypeI ->second) +
3377	" context is " + getAllocTypeString(Node->AllocTypes) +
3378	" after cloning";
3379	if (allocTypeToUse(Node->AllocTypes) != AllocTypeFromCall)
3380	Msg += " marked " + getAllocTypeString(AllocTypes: (uint8_t)AllocTypeFromCall) +
3381	" due to cold byte percent";
3382	// Print the internal context id to aid debugging and visualization.
3383	Msg += " (internal context id " + std::to_string(val: Id) + ")";
3384	if (MemProfReportHintedSizes)
3385	OS << Msg << "\n";
3386	if (EmitRemark)
3387	EmitRemark (DEBUG_TYPE, "MemProfReport", Msg);
3388	}
3389	}
3390	}
3391	}
3392
3393	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3394	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::check() const {
3395	using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
3396	for (const auto Node : nodes<GraphType>(this)) {
3397	checkNode<DerivedCCG, FuncTy, CallTy>(Node, /CheckEdges=/false);
3398	for (auto &Edge : Node->CallerEdges)
3399	checkEdge<DerivedCCG, FuncTy, CallTy>(Edge);
3400	}
3401	}
3402
3403	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3404	struct GraphTraits<const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *> {
3405	using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
3406	using NodeRef = const ContextNode<DerivedCCG, FuncTy, CallTy> *;
3407
3408	using NodePtrTy = std::unique_ptr<ContextNode<DerivedCCG, FuncTy, CallTy>>;
3409	static NodeRef getNode(const NodePtrTy &P) { return P.get(); }
3410
3411	using nodes_iterator =
3412	mapped_iterator<typename std::vector<NodePtrTy>::const_iterator,
3413	decltype(&getNode)>;
3414
3415	static nodes_iterator nodes_begin(GraphType G) {
3416	return nodes_iterator(G->NodeOwner.begin(), &getNode);
3417	}
3418
3419	static nodes_iterator nodes_end(GraphType G) {
3420	return nodes_iterator(G->NodeOwner.end(), &getNode);
3421	}
3422
3423	static NodeRef getEntryNode(GraphType G) {
3424	return G->NodeOwner.begin()->get();
3425	}
3426
3427	using EdgePtrTy = std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>>;
3428	static const ContextNode<DerivedCCG, FuncTy, CallTy> *
3429	GetCallee(const EdgePtrTy &P) {
3430	return P->Callee;
3431	}
3432
3433	using ChildIteratorType =
3434	mapped_iterator<typename std::vector<std::shared_ptr<ContextEdge<
3435	DerivedCCG, FuncTy, CallTy>>>::const_iterator,
3436	decltype(&GetCallee)>;
3437
3438	static ChildIteratorType child_begin(NodeRef N) {
3439	return ChildIteratorType(N->CalleeEdges.begin(), &GetCallee);
3440	}
3441
3442	static ChildIteratorType child_end(NodeRef N) {
3443	return ChildIteratorType(N->CalleeEdges.end(), &GetCallee);
3444	}
3445	};
3446
3447	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3448	struct DOTGraphTraits<const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>
3449	: public DefaultDOTGraphTraits {
3450	DOTGraphTraits(bool IsSimple = false) : DefaultDOTGraphTraits(IsSimple) {
3451	// If the user requested the full graph to be exported, but provided an
3452	// allocation id, or if the user gave a context id and requested more than
3453	// just a specific context to be exported, note that highlighting is
3454	// enabled.
3455	DoHighlight =
3456	(AllocIdForDot.getNumOccurrences() && DotGraphScope == DotScope::All) \|\|
3457	(ContextIdForDot.getNumOccurrences() &&
3458	DotGraphScope != DotScope::Context);
3459	}
3460
3461	using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
3462	using GTraits = GraphTraits<GraphType>;
3463	using NodeRef = typename GTraits::NodeRef;
3464	using ChildIteratorType = typename GTraits::ChildIteratorType;
3465
3466	static std::string getNodeLabel(NodeRef Node, GraphType G) {
3467	std::string LabelString =
3468	(Twine ("OrigId: ") + (Node->IsAllocation ? "Alloc" : "") +
3469	Twine(Node->OrigStackOrAllocId) + " NodeId: " + Twine(Node->NodeId))
3470	.str();
3471	LabelString += "\n";
3472	if (Node->hasCall()) {
3473	auto Func = G->NodeToCallingFunc.find(Node);
3474	assert(Func != G->NodeToCallingFunc.end());
3475	LabelString +=
3476	G->getLabel(Func->second, Node->Call.call(), Node->Call.cloneNo());
3477	for (auto &MatchingCall : Node->MatchingCalls) {
3478	LabelString += "\n";
3479	LabelString += G->getLabel(Func->second, MatchingCall.call(),
3480	MatchingCall.cloneNo());
3481	}
3482	} else {
3483	LabelString += "null call";
3484	if (Node->Recursive)
3485	LabelString += " (recursive)";
3486	else
3487	LabelString += " (external)";
3488	}
3489	return LabelString;
3490	}
3491
3492	static std::string getNodeAttributes(NodeRef Node, GraphType G) {
3493	auto ContextIds = Node->getContextIds();
3494	// If highlighting enabled, see if this node contains any of the context ids
3495	// of interest. If so, it will use a different color and a larger fontsize
3496	// (which makes the node larger as well).
3497	bool Highlight = false;
3498	if (DoHighlight) {
3499	assert(ContextIdForDot.getNumOccurrences() \|\|
3500	AllocIdForDot.getNumOccurrences());
3501	if (ContextIdForDot.getNumOccurrences())
3502	Highlight = ContextIds.contains(ContextIdForDot);
3503	else
3504	Highlight = set_intersects(ContextIds, G->DotAllocContextIds);
3505	}
3506	std::string AttributeString = (Twine ("tooltip=\"") + getNodeId(Node) + " " +
3507	getContextIds(ContextIds) + "\"")
3508	.str();
3509	// Default fontsize is 14
3510	if (Highlight)
3511	AttributeString += ",fontsize=\"30\"";
3512	AttributeString +=
3513	(Twine (",fillcolor=\"") + getColor(AllocTypes: Node->AllocTypes, Highlight) + "\"")
3514	.str();
3515	if (Node->CloneOf) {
3516	AttributeString += ",color=\"blue\"";
3517	AttributeString += ",style=\"filled,bold,dashed\"";
3518	} else
3519	AttributeString += ",style=\"filled\"";
3520	return AttributeString;
3521	}
3522
3523	static std::string getEdgeAttributes(NodeRef, ChildIteratorType ChildIter,
3524	GraphType G) {
3525	auto &Edge = *(ChildIter.getCurrent());
3526	// If highlighting enabled, see if this edge contains any of the context ids
3527	// of interest. If so, it will use a different color and a heavier arrow
3528	// size and weight (the larger weight makes the highlighted path
3529	// straighter).
3530	bool Highlight = false;
3531	if (DoHighlight) {
3532	assert(ContextIdForDot.getNumOccurrences() \|\|
3533	AllocIdForDot.getNumOccurrences());
3534	if (ContextIdForDot.getNumOccurrences())
3535	Highlight = Edge->ContextIds.contains(ContextIdForDot);
3536	else
3537	Highlight = set_intersects(Edge->ContextIds, G->DotAllocContextIds);
3538	}
3539	auto Color = getColor(AllocTypes: Edge->AllocTypes, Highlight);
3540	std::string AttributeString =
3541	(Twine ("tooltip=\"") + getContextIds(ContextIds: Edge->ContextIds) + "\"" +
3542	// fillcolor is the arrow head and color is the line
3543	Twine (",fillcolor=\"") + Color + "\"" + Twine (",color=\"") + Color +
3544	"\"")
3545	.str();
3546	if (Edge->IsBackedge)
3547	AttributeString += ",style=\"dotted\"";
3548	// Default penwidth and weight are both 1.
3549	if (Highlight)
3550	AttributeString += ",penwidth=\"2.0\",weight=\"2\"";
3551	return AttributeString;
3552	}
3553
3554	// Since the NodeOwners list includes nodes that are no longer connected to
3555	// the graph, skip them here.
3556	static bool isNodeHidden(NodeRef Node, GraphType G) {
3557	if (Node->isRemoved())
3558	return true;
3559	// If a scope smaller than the full graph was requested, see if this node
3560	// contains any of the context ids of interest.
3561	if (DotGraphScope == DotScope::Alloc)
3562	return !set_intersects(Node->getContextIds(), G->DotAllocContextIds);
3563	if (DotGraphScope == DotScope::Context)
3564	return !Node->getContextIds().contains(ContextIdForDot);
3565	return false;
3566	}
3567
3568	private:
3569	static std::string getContextIds(const DenseSet<uint32_t> &ContextIds) {
3570	std::string IdString = "ContextIds:";
3571	if (ContextIds.size() < `100`) {
3572	std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
3573	std::sort(first: SortedIds.begin(), last: SortedIds.end());
3574	for (auto Id : SortedIds)
3575	IdString += (" " + Twine (Id)).str();
3576	} else {
3577	IdString += (" (" + Twine (ContextIds.size()) + " ids)").str();
3578	}
3579	return IdString;
3580	}
3581
3582	static std::string getColor(uint8_t AllocTypes, bool Highlight) {
3583	// If DoHighlight is not enabled, we want to use the highlight colors for
3584	// NotCold and Cold, and the non-highlight color for NotCold+Cold. This is
3585	// both compatible with the color scheme before highlighting was supported,
3586	// and for the NotCold+Cold color the non-highlight color is a bit more
3587	// readable.
3588	if (AllocTypes == (uint8_t)AllocationType::NotCold)
3589	// Color "brown1" actually looks like a lighter red.
3590	return !DoHighlight \|\| Highlight ? "brown1" : "lightpink";
3591	if (AllocTypes == (uint8_t)AllocationType::Cold)
3592	return !DoHighlight \|\| Highlight ? "cyan" : "lightskyblue";
3593	if (AllocTypes ==
3594	((uint8_t)AllocationType::NotCold \| (uint8_t)AllocationType::Cold))
3595	return Highlight ? "magenta" : "mediumorchid1";
3596	return "gray";
3597	}
3598
3599	static std::string getNodeId(NodeRef Node) {
3600	std::stringstream SStream;
3601	SStream << std::hex << "N0x" << (unsigned long long)Node;
3602	std::string Result = SStream.str();
3603	return Result;
3604	}
3605
3606	// True if we should highlight a specific context or allocation's contexts in
3607	// the emitted graph.
3608	static bool DoHighlight;
3609	};
3610
3611	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3612	bool DOTGraphTraits<
3613	const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>::DoHighlight =
3614	false;
3615
3616	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3617	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::exportToDot(
3618	std::string Label) const {
3619	WriteGraph(this, "", false, Label,
3620	DotFilePathPrefix + "ccg." + Label + ".dot");
3621	}
3622
3623	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3624	typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
3625	CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::moveEdgeToNewCalleeClone(
3626	const std::shared_ptr<ContextEdge> &Edge,
3627	DenseSet<uint32_t> ContextIdsToMove) {
3628	ContextNode *Node = Edge->Callee;
3629	assert(NodeToCallingFunc.count(Node));
3630	ContextNode *Clone =
3631	createNewNode(IsAllocation: Node->IsAllocation, F: NodeToCallingFunc[Node], C: Node->Call);
3632	Node->addClone(Clone);
3633	Clone->MatchingCalls = Node->MatchingCalls;
3634	moveEdgeToExistingCalleeClone(Edge, NewCallee: Clone, /NewClone=/true,
3635	ContextIdsToMove);
3636	return Clone;
3637	}
3638
3639	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3640	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
3641	moveEdgeToExistingCalleeClone(const std::shared_ptr<ContextEdge> &Edge,
3642	ContextNode NewCallee, bool* NewClone,
3643	DenseSet<uint32_t> ContextIdsToMove) {
3644	// NewCallee and Edge's current callee must be clones of the same original
3645	// node (Edge's current callee may be the original node too).
3646	assert(NewCallee->getOrigNode() == Edge->Callee->getOrigNode());
3647
3648	bool EdgeIsRecursive = Edge->Callee == Edge->Caller;
3649
3650	ContextNode *OldCallee = Edge->Callee;
3651
3652	// We might already have an edge to the new callee from earlier cloning for a
3653	// different allocation. If one exists we will reuse it.
3654	auto ExistingEdgeToNewCallee = NewCallee->findEdgeFromCaller(Edge->Caller);
3655
3656	// Callers will pass an empty ContextIdsToMove set when they want to move the
3657	// edge. Copy in Edge's ids for simplicity.
3658	if (ContextIdsToMove.empty())
3659	ContextIdsToMove = Edge->getContextIds();
3660
3661	// If we are moving all of Edge's ids, then just move the whole Edge.
3662	// Otherwise only move the specified subset, to a new edge if needed.
3663	if (Edge->getContextIds().size() == ContextIdsToMove.size()) {
3664	// First, update the alloc types on New Callee from Edge.
3665	// Do this before we potentially clear Edge's fields below!
3666	NewCallee->AllocTypes \|= Edge->AllocTypes;
3667	// Moving the whole Edge.
3668	if (ExistingEdgeToNewCallee) {
3669	// Since we already have an edge to NewCallee, simply move the ids
3670	// onto it, and remove the existing Edge.
3671	ExistingEdgeToNewCallee->getContextIds().insert_range(ContextIdsToMove);
3672	ExistingEdgeToNewCallee->AllocTypes \|= Edge->AllocTypes;
3673	assert(Edge->ContextIds == ContextIdsToMove);
3674	removeEdgeFromGraph(Edge: Edge.get());
3675	} else {
3676	// Otherwise just reconnect Edge to NewCallee.
3677	Edge->Callee = NewCallee;
3678	NewCallee->CallerEdges.push_back(Edge);
3679	// Remove it from callee where it was previously connected.
3680	OldCallee->eraseCallerEdge(Edge.get());
3681	// Don't need to update Edge's context ids since we are simply
3682	// reconnecting it.
3683	}
3684	} else {
3685	// Only moving a subset of Edge's ids.
3686	// Compute the alloc type of the subset of ids being moved.
3687	auto CallerEdgeAllocType = computeAllocType(ContextIds&: ContextIdsToMove);
3688	if (ExistingEdgeToNewCallee) {
3689	// Since we already have an edge to NewCallee, simply move the ids
3690	// onto it.
3691	ExistingEdgeToNewCallee->getContextIds().insert_range(ContextIdsToMove);
3692	ExistingEdgeToNewCallee->AllocTypes \|= CallerEdgeAllocType;
3693	} else {
3694	// Otherwise, create a new edge to NewCallee for the ids being moved.
3695	auto NewEdge = std::make_shared<ContextEdge>(
3696	NewCallee, Edge->Caller, CallerEdgeAllocType, ContextIdsToMove);
3697	Edge->Caller->CalleeEdges.push_back(NewEdge);
3698	NewCallee->CallerEdges.push_back(NewEdge);
3699	}
3700	// In either case, need to update the alloc types on NewCallee, and remove
3701	// those ids and update the alloc type on the original Edge.
3702	NewCallee->AllocTypes \|= CallerEdgeAllocType;
3703	set_subtract(Edge->ContextIds, ContextIdsToMove);
3704	Edge->AllocTypes = computeAllocType(ContextIds&: Edge->ContextIds);
3705	}
3706	// Now walk the old callee node's callee edges and move Edge's context ids
3707	// over to the corresponding edge into the clone (which is created here if
3708	// this is a newly created clone).
3709	for (auto &OldCalleeEdge : OldCallee->CalleeEdges) {
3710	ContextNode *CalleeToUse = OldCalleeEdge->Callee;
3711	// If this is a direct recursion edge, use NewCallee (the clone) as the
3712	// callee as well, so that any edge updated/created here is also direct
3713	// recursive.
3714	if (CalleeToUse == OldCallee) {
3715	// If this is a recursive edge, see if we already moved a recursive edge
3716	// (which would have to have been this one) - if we were only moving a
3717	// subset of context ids it would still be on OldCallee.
3718	if (EdgeIsRecursive) {
3719	assert(OldCalleeEdge == Edge);
3720	continue;
3721	}
3722	CalleeToUse = NewCallee;
3723	}
3724	// The context ids moving to the new callee are the subset of this edge's
3725	// context ids and the context ids on the caller edge being moved.
3726	DenseSet<uint32_t> EdgeContextIdsToMove =
3727	set_intersection(OldCalleeEdge->getContextIds(), ContextIdsToMove);
3728	set_subtract(OldCalleeEdge->getContextIds(), EdgeContextIdsToMove);
3729	OldCalleeEdge->AllocTypes =
3730	computeAllocType(ContextIds&: OldCalleeEdge->getContextIds());
3731	if (!NewClone) {
3732	// Update context ids / alloc type on corresponding edge to NewCallee.
3733	// There is a chance this may not exist if we are reusing an existing
3734	// clone, specifically during function assignment, where we would have
3735	// removed none type edges after creating the clone. If we can't find
3736	// a corresponding edge there, fall through to the cloning below.
3737	if (auto *NewCalleeEdge = NewCallee->findEdgeFromCallee(CalleeToUse)) {
3738	NewCalleeEdge->getContextIds().insert_range(EdgeContextIdsToMove);
3739	NewCalleeEdge->AllocTypes \|= computeAllocType(ContextIds&: EdgeContextIdsToMove);
3740	continue;
3741	}
3742	}
3743	auto NewEdge = std::make_shared<ContextEdge>(
3744	CalleeToUse, NewCallee, computeAllocType(ContextIds&: EdgeContextIdsToMove),
3745	EdgeContextIdsToMove);
3746	NewCallee->CalleeEdges.push_back(NewEdge);
3747	NewEdge->Callee->CallerEdges.push_back(NewEdge);
3748	}
3749	// Recompute the node alloc type now that its callee edges have been
3750	// updated (since we will compute from those edges).
3751	OldCallee->AllocTypes = OldCallee->computeAllocType();
3752	// OldCallee alloc type should be None iff its context id set is now empty.
3753	assert((OldCallee->AllocTypes == (uint8_t)AllocationType::None) ==
3754	OldCallee->emptyContextIds());
3755	if (VerifyCCG) {
3756	checkNode<DerivedCCG, FuncTy, CallTy>(OldCallee, /CheckEdges=/false);
3757	checkNode<DerivedCCG, FuncTy, CallTy>(NewCallee, /CheckEdges=/false);
3758	for (const auto &OldCalleeEdge : OldCallee->CalleeEdges)
3759	checkNode<DerivedCCG, FuncTy, CallTy>(OldCalleeEdge->Callee,
3760	/CheckEdges=/false);
3761	for (const auto &NewCalleeEdge : NewCallee->CalleeEdges)
3762	checkNode<DerivedCCG, FuncTy, CallTy>(NewCalleeEdge->Callee,
3763	/CheckEdges=/false);
3764	}
3765	}
3766
3767	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3768	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
3769	moveCalleeEdgeToNewCaller(const std::shared_ptr<ContextEdge> &Edge,
3770	ContextNode *NewCaller) {
3771	auto *OldCallee = Edge->Callee;
3772	auto *NewCallee = OldCallee;
3773	// If this edge was direct recursive, make any new/updated edge also direct
3774	// recursive to NewCaller.
3775	bool Recursive = Edge->Caller == Edge->Callee;
3776	if (Recursive)
3777	NewCallee = NewCaller;
3778
3779	ContextNode *OldCaller = Edge->Caller;
3780	OldCaller->eraseCalleeEdge(Edge.get());
3781
3782	// We might already have an edge to the new caller. If one exists we will
3783	// reuse it.
3784	auto ExistingEdgeToNewCaller = NewCaller->findEdgeFromCallee(NewCallee);
3785
3786	if (ExistingEdgeToNewCaller) {
3787	// Since we already have an edge to NewCaller, simply move the ids
3788	// onto it, and remove the existing Edge.
3789	ExistingEdgeToNewCaller->getContextIds().insert_range(
3790	Edge->getContextIds());
3791	ExistingEdgeToNewCaller->AllocTypes \|= Edge->AllocTypes;
3792	Edge->ContextIds.clear();
3793	Edge->AllocTypes = (uint8_t)AllocationType::None;
3794	OldCallee->eraseCallerEdge(Edge.get());
3795	} else {
3796	// Otherwise just reconnect Edge to NewCaller.
3797	Edge->Caller = NewCaller;
3798	NewCaller->CalleeEdges.push_back(Edge);
3799	if (Recursive) {
3800	assert(NewCallee == NewCaller);
3801	// In the case of (direct) recursive edges, we update the callee as well
3802	// so that it becomes recursive on the new caller.
3803	Edge->Callee = NewCallee;
3804	NewCallee->CallerEdges.push_back(Edge);
3805	OldCallee->eraseCallerEdge(Edge.get());
3806	}
3807	// Don't need to update Edge's context ids since we are simply
3808	// reconnecting it.
3809	}
3810	// In either case, need to update the alloc types on New Caller.
3811	NewCaller->AllocTypes \|= Edge->AllocTypes;
3812
3813	// Now walk the old caller node's caller edges and move Edge's context ids
3814	// over to the corresponding edge into the node (which is created here if
3815	// this is a newly created node). We can tell whether this is a newly created
3816	// node by seeing if it has any caller edges yet.
3817	#ifndef NDEBUG
3818	bool IsNewNode = NewCaller->CallerEdges.empty();
3819	#endif
3820	// If we just moved a direct recursive edge, presumably its context ids should
3821	// also flow out of OldCaller via some other non-recursive callee edge. We
3822	// don't want to remove the recursive context ids from other caller edges yet,
3823	// otherwise the context ids get into an inconsistent state on OldCaller.
3824	// We will update these context ids on the non-recursive caller edge when and
3825	// if they are updated on the non-recursive callee.
3826	if (!Recursive) {
3827	for (auto &OldCallerEdge : OldCaller->CallerEdges) {
3828	auto OldCallerCaller = OldCallerEdge->Caller;
3829	// The context ids moving to the new caller are the subset of this edge's
3830	// context ids and the context ids on the callee edge being moved.
3831	DenseSet<uint32_t> EdgeContextIdsToMove = set_intersection(
3832	OldCallerEdge->getContextIds(), Edge->getContextIds());
3833	if (OldCaller == OldCallerCaller) {
3834	OldCallerCaller = NewCaller;
3835	// Don't actually move this one. The caller will move it directly via a
3836	// call to this function with this as the Edge if it is appropriate to
3837	// move to a diff node that has a matching callee (itself).
3838	continue;
3839	}
3840	set_subtract(OldCallerEdge->getContextIds(), EdgeContextIdsToMove);
3841	OldCallerEdge->AllocTypes =
3842	computeAllocType(ContextIds&: OldCallerEdge->getContextIds());
3843	// In this function we expect that any pre-existing node already has edges
3844	// from the same callers as the old node. That should be true in the
3845	// current use case, where we will remove None-type edges after copying
3846	// over all caller edges from the callee.
3847	auto *ExistingCallerEdge = NewCaller->findEdgeFromCaller(OldCallerCaller);
3848	// Since we would have skipped caller edges when moving a direct recursive
3849	// edge, this may not hold true when recursive handling enabled.
3850	assert(IsNewNode \|\| ExistingCallerEdge \|\| AllowRecursiveCallsites);
3851	if (ExistingCallerEdge) {
3852	ExistingCallerEdge->getContextIds().insert_range(EdgeContextIdsToMove);
3853	ExistingCallerEdge->AllocTypes \|=
3854	computeAllocType(ContextIds&: EdgeContextIdsToMove);
3855	continue;
3856	}
3857	auto NewEdge = std::make_shared<ContextEdge>(
3858	NewCaller, OldCallerCaller, computeAllocType(ContextIds&: EdgeContextIdsToMove),
3859	EdgeContextIdsToMove);
3860	NewCaller->CallerEdges.push_back(NewEdge);
3861	NewEdge->Caller->CalleeEdges.push_back(NewEdge);
3862	}
3863	}
3864	// Recompute the node alloc type now that its caller edges have been
3865	// updated (since we will compute from those edges).
3866	OldCaller->AllocTypes = OldCaller->computeAllocType();
3867	// OldCaller alloc type should be None iff its context id set is now empty.
3868	assert((OldCaller->AllocTypes == (uint8_t)AllocationType::None) ==
3869	OldCaller->emptyContextIds());
3870	if (VerifyCCG) {
3871	checkNode<DerivedCCG, FuncTy, CallTy>(OldCaller, /CheckEdges=/false);
3872	checkNode<DerivedCCG, FuncTy, CallTy>(NewCaller, /CheckEdges=/false);
3873	for (const auto &OldCallerEdge : OldCaller->CallerEdges)
3874	checkNode<DerivedCCG, FuncTy, CallTy>(OldCallerEdge->Caller,
3875	/CheckEdges=/false);
3876	for (const auto &NewCallerEdge : NewCaller->CallerEdges)
3877	checkNode<DerivedCCG, FuncTy, CallTy>(NewCallerEdge->Caller,
3878	/CheckEdges=/false);
3879	}
3880	}
3881
3882	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3883	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
3884	recursivelyRemoveNoneTypeCalleeEdges(
3885	ContextNode Node, DenseSet<const* ContextNode *> &Visited) {
3886	auto Inserted = Visited.insert(Node);
3887	if (!Inserted.second)
3888	return;
3889
3890	removeNoneTypeCalleeEdges(Node);
3891
3892	for (auto *Clone : Node->Clones)
3893	recursivelyRemoveNoneTypeCalleeEdges(Node: Clone, Visited);
3894
3895	// The recursive call may remove some of this Node's caller edges.
3896	// Iterate over a copy and skip any that were removed.
3897	auto CallerEdges = Node->CallerEdges;
3898	for (auto &Edge : CallerEdges) {
3899	// Skip any that have been removed by an earlier recursive call.
3900	if (Edge->isRemoved()) {
3901	assert(!is_contained(Node->CallerEdges, Edge));
3902	continue;
3903	}
3904	recursivelyRemoveNoneTypeCalleeEdges(Node: Edge->Caller, Visited);
3905	}
3906	}
3907
3908	// This is the standard DFS based backedge discovery algorithm.
3909	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3910	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::markBackedges() {
3911	// If we are cloning recursive contexts, find and mark backedges from all root
3912	// callers, using the typical DFS based backedge analysis.
3913	if (!CloneRecursiveContexts)
3914	return;
3915	DenseSet<const ContextNode *> Visited;
3916	DenseSet<const ContextNode *> CurrentStack;
3917	for (auto &Entry : NonAllocationCallToContextNodeMap) {
3918	auto *Node = Entry.second;
3919	if (Node->isRemoved())
3920	continue;
3921	// It is a root if it doesn't have callers.
3922	if (!Node->CallerEdges.empty())
3923	continue;
3924	markBackedges(Node, Visited, CurrentStack);
3925	assert(CurrentStack.empty());
3926	}
3927	}
3928
3929	// Recursive helper for above markBackedges method.
3930	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3931	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::markBackedges(
3932	ContextNode Node, DenseSet<const* ContextNode *> &Visited,
3933	DenseSet<const ContextNode *> &CurrentStack) {
3934	auto I = Visited.insert(Node);
3935	// We should only call this for unvisited nodes.
3936	assert(I.second);
3937	(void)I;
3938	for (auto &CalleeEdge : Node->CalleeEdges) {
3939	auto *Callee = CalleeEdge->Callee;
3940	if (Visited.count(Callee)) {
3941	// Since this was already visited we need to check if it is currently on
3942	// the recursive stack in which case it is a backedge.
3943	if (CurrentStack.count(Callee))
3944	CalleeEdge->IsBackedge = true;
3945	continue;
3946	}
3947	CurrentStack.insert(Callee);
3948	markBackedges(Callee, Visited, CurrentStack);
3949	CurrentStack.erase(Callee);
3950	}
3951	}
3952
3953	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3954	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::identifyClones() {
3955	DenseSet<const ContextNode *> Visited;
3956	for (auto &Entry : AllocationCallToContextNodeMap) {
3957	Visited.clear();
3958	identifyClones(Entry.second, Visited, Entry.second->getContextIds());
3959	}
3960	Visited.clear();
3961	for (auto &Entry : AllocationCallToContextNodeMap)
3962	recursivelyRemoveNoneTypeCalleeEdges(Node: Entry.second, Visited);
3963	if (VerifyCCG)
3964	check();
3965	}
3966
3967	// helper function to check an AllocType is cold or notcold or both.
3968	bool checkColdOrNotCold(uint8_t AllocType) {
3969	return (AllocType == (uint8_t)AllocationType::Cold) \|\|
3970	(AllocType == (uint8_t)AllocationType::NotCold) \|\|
3971	(AllocType ==
3972	((uint8_t)AllocationType::Cold \| (uint8_t)AllocationType::NotCold));
3973	}
3974
3975	template <typename DerivedCCG, typename FuncTy, typename CallTy>
3976	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::identifyClones(
3977	ContextNode Node, DenseSet<const* ContextNode *> &Visited,
3978	const DenseSet<uint32_t> &AllocContextIds) {
3979	if (VerifyNodes)
3980	checkNode<DerivedCCG, FuncTy, CallTy>(Node, /CheckEdges=/false);
3981	assert(!Node->CloneOf);
3982
3983	// If Node as a null call, then either it wasn't found in the module (regular
3984	// LTO) or summary index (ThinLTO), or there were other conditions blocking
3985	// cloning (e.g. recursion, calls multiple targets, etc).
3986	// Do this here so that we don't try to recursively clone callers below, which
3987	// isn't useful at least for this node.
3988	if (!Node->hasCall())
3989	return;
3990
3991	// No need to look at any callers if allocation type already unambiguous.
3992	if (hasSingleAllocType(Node->AllocTypes))
3993	return;
3994
3995	#ifndef NDEBUG
3996	auto Insert =
3997	#endif
3998	Visited.insert(Node);
3999	// We should not have visited this node yet.
4000	assert(Insert.second);
4001	// The recursive call to identifyClones may delete the current edge from the
4002	// CallerEdges vector. Make a copy and iterate on that, simpler than passing
4003	// in an iterator and having recursive call erase from it. Other edges may
4004	// also get removed during the recursion, which will have null Callee and
4005	// Caller pointers (and are deleted later), so we skip those below.
4006	{
4007	auto CallerEdges = Node->CallerEdges;
4008	for (auto &Edge : CallerEdges) {
4009	// Skip any that have been removed by an earlier recursive call.
4010	if (Edge->isRemoved()) {
4011	assert(!is_contained(Node->CallerEdges, Edge));
4012	continue;
4013	}
4014	// Defer backedges. See comments further below where these edges are
4015	// handled during the cloning of this Node.
4016	if (Edge->IsBackedge) {
4017	// We should only mark these if cloning recursive contexts, where we
4018	// need to do this deferral.
4019	assert(CloneRecursiveContexts);
4020	continue;
4021	}
4022	// Ignore any caller we previously visited via another edge.
4023	if (!Visited.count(Edge->Caller) && !Edge->Caller->CloneOf) {
4024	identifyClones(Edge->Caller, Visited, AllocContextIds);
4025	}
4026	}
4027	}
4028
4029	// Check if we reached an unambiguous call or have have only a single caller.
4030	if (hasSingleAllocType(Node->AllocTypes) \|\| Node->CallerEdges.size() <= `1`)
4031	return;
4032
4033	// We need to clone.
4034
4035	// Try to keep the original version as alloc type NotCold. This will make
4036	// cases with indirect calls or any other situation with an unknown call to
4037	// the original function get the default behavior. We do this by sorting the
4038	// CallerEdges of the Node we will clone by alloc type.
4039	//
4040	// Give NotCold edge the lowest sort priority so those edges are at the end of
4041	// the caller edges vector, and stay on the original version (since the below
4042	// code clones greedily until it finds all remaining edges have the same type
4043	// and leaves the remaining ones on the original Node).
4044	//
4045	// We shouldn't actually have any None type edges, so the sorting priority for
4046	// that is arbitrary, and we assert in that case below.
4047	const unsigned AllocTypeCloningPriority[] = {/None/ `3`, /NotCold/ `4`,
4048	/Cold/ `1`,
4049	/NotColdCold/ `2`};
4050	llvm::stable_sort(Node->CallerEdges,
4051	[&](const std::shared_ptr<ContextEdge> &A,
4052	const std::shared_ptr<ContextEdge> &B) {
4053	// Nodes with non-empty context ids should be sorted
4054	// before those with empty context ids.
4055	if (A->ContextIds.empty())
4056	// Either B ContextIds are non-empty (in which case we
4057	// should return false because B < A), or B ContextIds
4058	// are empty, in which case they are equal, and we
4059	// should maintain the original relative ordering.
4060	return false;
4061	if (B->ContextIds.empty())
4062	return true;
4063
4064	if (A->AllocTypes == B->AllocTypes)
4065	// Use the caller node id as a deterministic
4066	// tie-breaker.
4067	return A->Caller->NodeId < B->Caller->NodeId;
4068	return AllocTypeCloningPriority[A->AllocTypes] <
4069	AllocTypeCloningPriority[B->AllocTypes];
4070	});
4071
4072	assert(Node->AllocTypes != (uint8_t)AllocationType::None);
4073
4074	DenseSet<uint32_t> RecursiveContextIds;
4075	assert(AllowRecursiveContexts \|\| !CloneRecursiveContexts);
4076	// If we are allowing recursive callsites, but have also disabled recursive
4077	// contexts, look for context ids that show up in multiple caller edges.
4078	if (AllowRecursiveCallsites && !AllowRecursiveContexts) {
4079	DenseSet<uint32_t> AllCallerContextIds;
4080	for (auto &CE : Node->CallerEdges) {
4081	// Resize to the largest set of caller context ids, since we know the
4082	// final set will be at least that large.
4083	AllCallerContextIds.reserve(Size: CE->getContextIds().size());
4084	for (auto Id : CE->getContextIds())
4085	if (!AllCallerContextIds.insert(Id).second)
4086	RecursiveContextIds.insert(Id);
4087	}
4088	}
4089
4090	// Iterate until we find no more opportunities for disambiguating the alloc
4091	// types via cloning. In most cases this loop will terminate once the Node
4092	// has a single allocation type, in which case no more cloning is needed.
4093	// Iterate over a copy of Node's caller edges, since we may need to remove
4094	// edges in the moveEdgeTo methods, and this simplifies the handling and*
4095	// makes it less error-prone.
4096	auto CallerEdges = Node->CallerEdges;
4097	for (auto &CallerEdge : CallerEdges) {
4098	// Skip any that have been removed by an earlier recursive call.
4099	if (CallerEdge->isRemoved()) {
4100	assert(!is_contained(Node->CallerEdges, CallerEdge));
4101	continue;
4102	}
4103	assert(CallerEdge->Callee == Node);
4104
4105	// See if cloning the prior caller edge left this node with a single alloc
4106	// type or a single caller. In that case no more cloning of Node is needed.
4107	if (hasSingleAllocType(Node->AllocTypes) \|\| Node->CallerEdges.size() <= `1`)
4108	break;
4109
4110	// If the caller was not successfully matched to a call in the IR/summary,
4111	// there is no point in trying to clone for it as we can't update that call.
4112	if (!CallerEdge->Caller->hasCall())
4113	continue;
4114
4115	// Only need to process the ids along this edge pertaining to the given
4116	// allocation.
4117	auto CallerEdgeContextsForAlloc =
4118	set_intersection(CallerEdge->getContextIds(), AllocContextIds);
4119	if (!RecursiveContextIds.empty())
4120	CallerEdgeContextsForAlloc =
4121	set_difference(CallerEdgeContextsForAlloc, RecursiveContextIds);
4122	if (CallerEdgeContextsForAlloc.empty())
4123	continue;
4124
4125	auto CallerAllocTypeForAlloc = computeAllocType(ContextIds&: CallerEdgeContextsForAlloc);
4126
4127	// Compute the node callee edge alloc types corresponding to the context ids
4128	// for this caller edge.
4129	std::vector<uint8_t> CalleeEdgeAllocTypesForCallerEdge;
4130	CalleeEdgeAllocTypesForCallerEdge.reserve(n: Node->CalleeEdges.size());
4131	for (auto &CalleeEdge : Node->CalleeEdges)
4132	CalleeEdgeAllocTypesForCallerEdge.push_back(intersectAllocTypes(
4133	Node1Ids: CalleeEdge->getContextIds(), Node2Ids: CallerEdgeContextsForAlloc));
4134
4135	// Don't clone if doing so will not disambiguate any alloc types amongst
4136	// caller edges (including the callee edges that would be cloned).
4137	// Otherwise we will simply move all edges to the clone.
4138	//
4139	// First check if by cloning we will disambiguate the caller allocation
4140	// type from node's allocation type. Query allocTypeToUse so that we don't
4141	// bother cloning to distinguish NotCold+Cold from NotCold. Note that
4142	// neither of these should be None type.
4143	//
4144	// Then check if by cloning node at least one of the callee edges will be
4145	// disambiguated by splitting out different context ids.
4146	//
4147	// However, always do the cloning if this is a backedge, in which case we
4148	// have not yet cloned along this caller edge.
4149	assert(CallerEdge->AllocTypes != (uint8_t)AllocationType::None);
4150	assert(Node->AllocTypes != (uint8_t)AllocationType::None);
4151	if (!CallerEdge->IsBackedge &&
4152	allocTypeToUse(CallerAllocTypeForAlloc) ==
4153	allocTypeToUse(Node->AllocTypes) &&
4154	allocTypesMatch<DerivedCCG, FuncTy, CallTy>(
4155	CalleeEdgeAllocTypesForCallerEdge, Node->CalleeEdges)) {
4156	continue;
4157	}
4158
4159	if (CallerEdge->IsBackedge) {
4160	// We should only mark these if cloning recursive contexts, where we
4161	// need to do this deferral.
4162	assert(CloneRecursiveContexts);
4163	DeferredBackedges ++;
4164	}
4165
4166	// If this is a backedge, we now do recursive cloning starting from its
4167	// caller since we may have moved unambiguous caller contexts to a clone
4168	// of this Node in a previous iteration of the current loop, giving more
4169	// opportunity for cloning through the backedge. Because we sorted the
4170	// caller edges earlier so that cold caller edges are first, we would have
4171	// visited and cloned this node for any unamibiguously cold non-recursive
4172	// callers before any ambiguous backedge callers. Note that we don't do this
4173	// if the caller is already cloned or visited during cloning (e.g. via a
4174	// different context path from the allocation).
4175	// TODO: Can we do better in the case where the caller was already visited?
4176	if (CallerEdge->IsBackedge && !CallerEdge->Caller->CloneOf &&
4177	!Visited.count(CallerEdge->Caller)) {
4178	const auto OrigIdCount = CallerEdge->getContextIds().size();
4179	// Now do the recursive cloning of this backedge's caller, which was
4180	// deferred earlier.
4181	identifyClones(CallerEdge->Caller, Visited, CallerEdgeContextsForAlloc);
4182	removeNoneTypeCalleeEdges(Node: CallerEdge->Caller);
4183	// See if the recursive call to identifyClones moved the context ids to a
4184	// new edge from this node to a clone of caller, and switch to looking at
4185	// that new edge so that we clone Node for the new caller clone.
4186	bool UpdatedEdge = false;
4187	if (OrigIdCount > CallerEdge->getContextIds().size()) {
4188	for (auto E : Node->CallerEdges) {
4189	// Only interested in clones of the current edges caller.
4190	if (E->Caller->CloneOf != CallerEdge->Caller)
4191	continue;
4192	// See if this edge contains any of the context ids originally on the
4193	// current caller edge.
4194	auto CallerEdgeContextsForAllocNew =
4195	set_intersection(CallerEdgeContextsForAlloc, E->getContextIds());
4196	if (CallerEdgeContextsForAllocNew.empty())
4197	continue;
4198	// Make sure we don't pick a previously existing caller edge of this
4199	// Node, which would be processed on a different iteration of the
4200	// outer loop over the saved CallerEdges.
4201	if (llvm::is_contained(CallerEdges, E))
4202	continue;
4203	// The CallerAllocTypeForAlloc and CalleeEdgeAllocTypesForCallerEdge
4204	// are updated further below for all cases where we just invoked
4205	// identifyClones recursively.
4206	CallerEdgeContextsForAlloc.swap(CallerEdgeContextsForAllocNew);
4207	CallerEdge = E;
4208	UpdatedEdge = true;
4209	break;
4210	}
4211	}
4212	// If cloning removed this edge (and we didn't update it to a new edge
4213	// above), we're done with this edge. It's possible we moved all of the
4214	// context ids to an existing clone, in which case there's no need to do
4215	// further processing for them.
4216	if (CallerEdge->isRemoved())
4217	continue;
4218
4219	// Now we need to update the information used for the cloning decisions
4220	// further below, as we may have modified edges and their context ids.
4221
4222	// Note if we changed the CallerEdge above we would have already updated
4223	// the context ids.
4224	if (!UpdatedEdge) {
4225	CallerEdgeContextsForAlloc = set_intersection(
4226	CallerEdgeContextsForAlloc, CallerEdge->getContextIds());
4227	if (CallerEdgeContextsForAlloc.empty())
4228	continue;
4229	}
4230	// Update the other information that depends on the edges and on the now
4231	// updated CallerEdgeContextsForAlloc.
4232	CallerAllocTypeForAlloc = computeAllocType(ContextIds&: CallerEdgeContextsForAlloc);
4233	CalleeEdgeAllocTypesForCallerEdge.clear();
4234	for (auto &CalleeEdge : Node->CalleeEdges) {
4235	CalleeEdgeAllocTypesForCallerEdge.push_back(intersectAllocTypes(
4236	Node1Ids: CalleeEdge->getContextIds(), Node2Ids: CallerEdgeContextsForAlloc));
4237	}
4238	}
4239
4240	// First see if we can use an existing clone. Check each clone and its
4241	// callee edges for matching alloc types.
4242	ContextNode Clone = nullptr*;
4243	for (auto *CurClone : Node->Clones) {
4244	if (allocTypeToUse(CurClone->AllocTypes) !=
4245	allocTypeToUse(CallerAllocTypeForAlloc))
4246	continue;
4247
4248	bool BothSingleAlloc = hasSingleAllocType(CurClone->AllocTypes) &&
4249	hasSingleAllocType(CallerAllocTypeForAlloc);
4250	// The above check should mean that if both have single alloc types that
4251	// they should be equal.
4252	assert(!BothSingleAlloc \|\|
4253	CurClone->AllocTypes == CallerAllocTypeForAlloc);
4254
4255	// If either both have a single alloc type (which are the same), or if the
4256	// clone's callee edges have the same alloc types as those for the current
4257	// allocation on Node's callee edges (CalleeEdgeAllocTypesForCallerEdge),
4258	// then we can reuse this clone.
4259	if (BothSingleAlloc \|\| allocTypesMatchClone<DerivedCCG, FuncTy, CallTy>(
4260	CalleeEdgeAllocTypesForCallerEdge, CurClone)) {
4261	Clone = CurClone;
4262	break;
4263	}
4264	}
4265
4266	// The edge iterator is adjusted when we move the CallerEdge to the clone.
4267	if (Clone)
4268	moveEdgeToExistingCalleeClone(Edge: CallerEdge, NewCallee: Clone, /NewClone=/false,
4269	ContextIdsToMove: CallerEdgeContextsForAlloc);
4270	else
4271	Clone = moveEdgeToNewCalleeClone(Edge: CallerEdge, ContextIdsToMove: CallerEdgeContextsForAlloc);
4272
4273	// Sanity check that no alloc types on clone or its edges are None.
4274	assert(Clone->AllocTypes != (uint8_t)AllocationType::None);
4275	}
4276
4277	// We should still have some context ids on the original Node.
4278	assert(!Node->emptyContextIds());
4279
4280	// Sanity check that no alloc types on node or edges are None.
4281	assert(Node->AllocTypes != (uint8_t)AllocationType::None);
4282
4283	if (VerifyNodes)
4284	checkNode<DerivedCCG, FuncTy, CallTy>(Node, /CheckEdges=/false);
4285	}
4286
4287	void ModuleCallsiteContextGraph::updateAllocationCall(
4288	CallInfo &Call, AllocationType AllocType) {
4289	std::string AllocTypeString = getAllocTypeAttributeString(Type: AllocType);
4290	removeAnyExistingAmbiguousAttribute(CB: cast<CallBase>(Val: Call.call()));
4291	auto A = llvm::Attribute::get(Context&: Call.call()->getFunction()->getContext(),
4292	Kind: "memprof", Val: AllocTypeString);
4293	cast<CallBase>(Val: Call.call())->addFnAttr(Attr: A);
4294	OREGetter (Call.call()->getFunction())
4295	.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofAttribute", Call.call())
4296	<< ore::NV ("AllocationCall", Call.call()) << " in clone "
4297	<< ore::NV ("Caller", Call.call()->getFunction())
4298	<< " marked with memprof allocation attribute "
4299	<< ore::NV ("Attribute", AllocTypeString));
4300	}
4301
4302	void IndexCallsiteContextGraph::updateAllocationCall(CallInfo &Call,
4303	AllocationType AllocType) {
4304	auto AI = cast<AllocInfo >(Val: Call.call());
4305	assert(AI);
4306	assert(AI->Versions.size() > Call.cloneNo());
4307	AI->Versions [Call.cloneNo()] = (uint8_t)AllocType;
4308	}
4309
4310	AllocationType
4311	ModuleCallsiteContextGraph::getAllocationCallType(const CallInfo &Call) const {
4312	const auto *CB = cast<CallBase>(Val: Call.call());
4313	if (!CB->getAttributes().hasFnAttr(Kind: "memprof"))
4314	return AllocationType::None;
4315	return CB->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "cold"
4316	? AllocationType::Cold
4317	: AllocationType::NotCold;
4318	}
4319
4320	AllocationType
4321	IndexCallsiteContextGraph::getAllocationCallType(const CallInfo &Call) const {
4322	const auto AI = cast<AllocInfo >(Val: Call.call());
4323	assert(AI->Versions.size() > Call.cloneNo());
4324	return (AllocationType)AI->Versions [Call.cloneNo()];
4325	}
4326
4327	void ModuleCallsiteContextGraph::updateCall(CallInfo &CallerCall,
4328	FuncInfo CalleeFunc) {
4329	auto *CurF = getCalleeFunc(Call: CallerCall.call());
4330	auto NewCalleeCloneNo = CalleeFunc.cloneNo();
4331	if (isMemProfClone(F: *CurF)) {
4332	// If we already assigned this callsite to call a specific non-default
4333	// clone (i.e. not the original function which is clone 0), ensure that we
4334	// aren't trying to now update it to call a different clone, which is
4335	// indicative of a bug in the graph or function assignment.
4336	auto CurCalleeCloneNo = getMemProfCloneNum(F: *CurF);
4337	if (CurCalleeCloneNo != NewCalleeCloneNo) {
4338	LLVM_DEBUG(dbgs() << "Mismatch in call clone assignment: was "
4339	<< CurCalleeCloneNo << " now " << NewCalleeCloneNo
4340	<< "\n");
4341	MismatchedCloneAssignments ++;
4342	}
4343	}
4344	if (NewCalleeCloneNo > `0`)
4345	cast<CallBase>(Val: CallerCall.call())->setCalledFunction(CalleeFunc.func());
4346	OREGetter (CallerCall.call()->getFunction())
4347	.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofCall", CallerCall.call())
4348	<< ore::NV ("Call", CallerCall.call()) << " in clone "
4349	<< ore::NV ("Caller", CallerCall.call()->getFunction())
4350	<< " assigned to call function clone "
4351	<< ore::NV ("Callee", CalleeFunc.func()));
4352	}
4353
4354	void IndexCallsiteContextGraph::updateCall(CallInfo &CallerCall,
4355	FuncInfo CalleeFunc) {
4356	auto CI = cast<CallsiteInfo >(Val: CallerCall.call());
4357	assert(CI &&
4358	"Caller cannot be an allocation which should not have profiled calls");
4359	assert(CI->Clones.size() > CallerCall.cloneNo());
4360	auto NewCalleeCloneNo = CalleeFunc.cloneNo();
4361	auto &CurCalleeCloneNo = CI->Clones [CallerCall.cloneNo()];
4362	// If we already assigned this callsite to call a specific non-default
4363	// clone (i.e. not the original function which is clone 0), ensure that we
4364	// aren't trying to now update it to call a different clone, which is
4365	// indicative of a bug in the graph or function assignment.
4366	if (CurCalleeCloneNo != `0` && CurCalleeCloneNo != NewCalleeCloneNo) {
4367	LLVM_DEBUG(dbgs() << "Mismatch in call clone assignment: was "
4368	<< CurCalleeCloneNo << " now " << NewCalleeCloneNo
4369	<< "\n");
4370	MismatchedCloneAssignments ++;
4371	}
4372	CurCalleeCloneNo = NewCalleeCloneNo;
4373	}
4374
4375	// Update the debug information attached to NewFunc to use the clone Name. Note
4376	// this needs to be done for both any existing DISubprogram for the definition,
4377	// as well as any separate declaration DISubprogram.
4378	static void updateSubprogramLinkageName(Function *NewFunc, StringRef Name) {
4379	assert(Name == NewFunc->getName());
4380	auto *SP = NewFunc->getSubprogram();
4381	if (!SP)
4382	return;
4383	auto *MDName = MDString::get(Context&: NewFunc->getParent()->getContext(), Str: Name);
4384	SP->replaceLinkageName(LN: MDName);
4385	DISubprogram *Decl = SP->getDeclaration();
4386	if (!Decl)
4387	return;
4388	TempDISubprogram NewDecl = Decl->clone();
4389	NewDecl ->replaceLinkageName(LN: MDName);
4390	SP->replaceDeclaration(Decl: MDNode::replaceWithUniqued(N: std::move(NewDecl)));
4391	}
4392
4393	CallsiteContextGraph<ModuleCallsiteContextGraph, Function,
4394	Instruction *>::FuncInfo
4395	ModuleCallsiteContextGraph::cloneFunctionForCallsite(
4396	FuncInfo &Func, CallInfo &Call, DenseMap<CallInfo, CallInfo> &CallMap,
4397	std::vector<CallInfo> &CallsWithMetadataInFunc, unsigned CloneNo) {
4398	// Use existing LLVM facilities for cloning and obtaining Call in clone
4399	ValueToValueMapTy VMap;
4400	auto *NewFunc = CloneFunction(F: Func.func(), VMap);
4401	std::string Name = getMemProfFuncName(Base: Func.func()->getName(), CloneNo);
4402	assert(!Func.func()->getParent()->getFunction(Name));
4403	NewFunc->setName(Name);
4404	updateSubprogramLinkageName(NewFunc, Name);
4405	for (auto &Inst : CallsWithMetadataInFunc) {
4406	// This map always has the initial version in it.
4407	assert(Inst.cloneNo() == `0`);
4408	CallMap [Inst] = {cast<Instruction>(Val&: VMap [Inst.call()]), CloneNo};
4409	}
4410	OREGetter (Func.func())
4411	.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofClone", Func.func())
4412	<< "created clone " << ore::NV ("NewFunction", NewFunc));
4413	return {NewFunc, CloneNo};
4414	}
4415
4416	CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
4417	IndexCall>::FuncInfo
4418	IndexCallsiteContextGraph::cloneFunctionForCallsite(
4419	FuncInfo &Func, CallInfo &Call, DenseMap<CallInfo, CallInfo> &CallMap,
4420	std::vector<CallInfo> &CallsWithMetadataInFunc, unsigned CloneNo) {
4421	// Check how many clones we have of Call (and therefore function).
4422	// The next clone number is the current size of versions array.
4423	// Confirm this matches the CloneNo provided by the caller, which is based on
4424	// the number of function clones we have.
4425	assert(CloneNo == (isa<AllocInfo *>(Call.call())
4426	? cast<AllocInfo *>(Call.call())->Versions.size()
4427	: cast<CallsiteInfo *>(Call.call())->Clones.size()));
4428	// Walk all the instructions in this function. Create a new version for
4429	// each (by adding an entry to the Versions/Clones summary array), and copy
4430	// over the version being called for the function clone being cloned here.
4431	// Additionally, add an entry to the CallMap for the new function clone,
4432	// mapping the original call (clone 0, what is in CallsWithMetadataInFunc)
4433	// to the new call clone.
4434	for (auto &Inst : CallsWithMetadataInFunc) {
4435	// This map always has the initial version in it.
4436	assert(Inst.cloneNo() == `0`);
4437	if (auto AI = dyn_cast<AllocInfo >(Val: Inst.call())) {
4438	assert(AI->Versions.size() == CloneNo);
4439	// We assign the allocation type later (in updateAllocationCall), just add
4440	// an entry for it here.
4441	AI->Versions.push_back(Elt: `0`);
4442	} else {
4443	auto CI = cast<CallsiteInfo >(Val: Inst.call());
4444	assert(CI && CI->Clones.size() == CloneNo);
4445	// We assign the clone number later (in updateCall), just add an entry for
4446	// it here.
4447	CI->Clones.push_back(Elt: `0`);
4448	}
4449	CallMap [Inst] = {Inst.call(), CloneNo};
4450	}
4451	return {Func.func(), CloneNo};
4452	}
4453
4454	// We perform cloning for each allocation node separately. However, this
4455	// sometimes results in a situation where the same node calls multiple
4456	// clones of the same callee, created for different allocations. This
4457	// causes issues when assigning functions to these clones, as each node can
4458	// in reality only call a single callee clone.
4459	//
4460	// To address this, before assigning functions, merge callee clone nodes as
4461	// needed using a post order traversal from the allocations. We attempt to
4462	// use existing clones as the merge node when legal, and to share them
4463	// among callers with the same properties (callers calling the same set of
4464	// callee clone nodes for the same allocations).
4465	//
4466	// Without this fix, in some cases incorrect function assignment will lead
4467	// to calling the wrong allocation clone.
4468	template <typename DerivedCCG, typename FuncTy, typename CallTy>
4469	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::mergeClones() {
4470	if (!MergeClones)
4471	return;
4472
4473	// Generate a map from context id to the associated allocation node for use
4474	// when merging clones.
4475	DenseMap<uint32_t, ContextNode *> ContextIdToAllocationNode;
4476	for (auto &Entry : AllocationCallToContextNodeMap) {
4477	auto *Node = Entry.second;
4478	for (auto Id : Node->getContextIds())
4479	ContextIdToAllocationNode[Id] = Node->getOrigNode();
4480	for (auto *Clone : Node->Clones) {
4481	for (auto Id : Clone->getContextIds())
4482	ContextIdToAllocationNode[Id] = Clone->getOrigNode();
4483	}
4484	}
4485
4486	// Post order traversal starting from allocations to ensure each callsite
4487	// calls a single clone of its callee. Callee nodes that are clones of each
4488	// other are merged (via new merge nodes if needed) to achieve this.
4489	DenseSet<const ContextNode *> Visited;
4490	for (auto &Entry : AllocationCallToContextNodeMap) {
4491	auto *Node = Entry.second;
4492
4493	mergeClones(Node, Visited, ContextIdToAllocationNode);
4494
4495	// Make a copy so the recursive post order traversal that may create new
4496	// clones doesn't mess up iteration. Note that the recursive traversal
4497	// itself does not call mergeClones on any of these nodes, which are all
4498	// (clones of) allocations.
4499	auto Clones = Node->Clones;
4500	for (auto *Clone : Clones)
4501	mergeClones(Clone, Visited, ContextIdToAllocationNode);
4502	}
4503
4504	if (DumpCCG) {
4505	dbgs() << "CCG after merging:\n";
4506	dbgs() << *this;
4507	}
4508	if (ExportToDot)
4509	exportToDot(Label: "aftermerge");
4510
4511	if (VerifyCCG) {
4512	check();
4513	}
4514	}
4515
4516	// Recursive helper for above mergeClones method.
4517	template <typename DerivedCCG, typename FuncTy, typename CallTy>
4518	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::mergeClones(
4519	ContextNode Node, DenseSet<const* ContextNode *> &Visited,
4520	DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode) {
4521	auto Inserted = Visited.insert(Node);
4522	if (!Inserted.second)
4523	return;
4524
4525	// Iteratively perform merging on this node to handle new caller nodes created
4526	// during the recursive traversal. We could do something more elegant such as
4527	// maintain a worklist, but this is a simple approach that doesn't cause a
4528	// measureable compile time effect, as most nodes don't have many caller
4529	// edges to check.
4530	bool FoundUnvisited = true;
4531	unsigned Iters = `0`;
4532	while (FoundUnvisited) {
4533	Iters++;
4534	FoundUnvisited = false;
4535	// Make a copy since the recursive call may move a caller edge to a new
4536	// callee, messing up the iterator.
4537	auto CallerEdges = Node->CallerEdges;
4538	for (auto CallerEdge : CallerEdges) {
4539	// Skip any caller edge moved onto a different callee during recursion.
4540	if (CallerEdge->Callee != Node)
4541	continue;
4542	// If we found an unvisited caller, note that we should check the caller
4543	// edges again as mergeClones may add or change caller nodes.
4544	if (DoMergeIteration && !Visited.contains(CallerEdge->Caller))
4545	FoundUnvisited = true;
4546	mergeClones(CallerEdge->Caller, Visited, ContextIdToAllocationNode);
4547	}
4548	}
4549
4550	TotalMergeInvokes ++;
4551	TotalMergeIters += Iters;
4552	if (Iters > MaxMergeIters)
4553	MaxMergeIters = Iters;
4554
4555	// Merge for this node after we handle its callers.
4556	mergeNodeCalleeClones(Node, Visited, ContextIdToAllocationNode);
4557	}
4558
4559	template <typename DerivedCCG, typename FuncTy, typename CallTy>
4560	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::mergeNodeCalleeClones(
4561	ContextNode Node, DenseSet<const* ContextNode *> &Visited,
4562	DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode) {
4563	// Ignore Node if we moved all of its contexts to clones.
4564	if (Node->emptyContextIds())
4565	return;
4566
4567	// First identify groups of clones among Node's callee edges, by building
4568	// a map from each callee base node to the associated callee edges from Node.
4569	MapVector<ContextNode *, std::vector<std::shared_ptr<ContextEdge>>>
4570	OrigNodeToCloneEdges;
4571	for (const auto &E : Node->CalleeEdges) {
4572	auto *Callee = E->Callee;
4573	if (!Callee->CloneOf && Callee->Clones.empty())
4574	continue;
4575	ContextNode *Base = Callee->getOrigNode();
4576	OrigNodeToCloneEdges[Base].push_back(E);
4577	}
4578
4579	// Helper for callee edge sorting below. Return true if A's callee has fewer
4580	// caller edges than B, or if A is a clone and B is not, or if A's callee
4581	// node id is smaller than B's.
4582	auto CalleeCallerEdgeLessThan = [](const std::shared_ptr<ContextEdge> &A,
4583	const std::shared_ptr<ContextEdge> &B) {
4584	if (A->Callee->CallerEdges.size() != B->Callee->CallerEdges.size())
4585	return A->Callee->CallerEdges.size() < B->Callee->CallerEdges.size();
4586	if (A->Callee->CloneOf && !B->Callee->CloneOf)
4587	return true;
4588	else if (!A->Callee->CloneOf && B->Callee->CloneOf)
4589	return false;
4590	// Use the callee node id as a deterministic tie-breaker.
4591	return A->Callee->NodeId < B->Callee->NodeId;
4592	};
4593
4594	// Process each set of callee clones called by Node, performing the needed
4595	// merging.
4596	for (auto Entry : OrigNodeToCloneEdges) {
4597	// CalleeEdges is the set of edges from Node reaching callees that are
4598	// mutual clones of each other.
4599	auto &CalleeEdges = Entry.second;
4600	auto NumCalleeClones = CalleeEdges.size();
4601	// A single edge means there is no merging needed.
4602	if (NumCalleeClones == `1`)
4603	continue;
4604	// Sort the CalleeEdges calling this group of clones in ascending order of
4605	// their caller edge counts, putting the original non-clone node first in
4606	// cases of a tie. This simplifies finding an existing node to use as the
4607	// merge node.
4608	llvm::stable_sort(CalleeEdges, CalleeCallerEdgeLessThan);
4609
4610	/// Find other callers of the given set of callee edges that can
4611	/// share the same callee merge node. See the comments at this method
4612	/// definition for details.
4613	DenseSet<ContextNode *> OtherCallersToShareMerge;
4614	findOtherCallersToShareMerge(Node, CalleeEdges, ContextIdToAllocationNode,
4615	OtherCallersToShareMerge);
4616
4617	// Now do the actual merging. Identify existing or create a new MergeNode
4618	// during the first iteration. Move each callee over, along with edges from
4619	// other callers we've determined above can share the same merge node.
4620	ContextNode MergeNode = nullptr*;
4621	DenseMap<ContextNode , unsigned*> CallerToMoveCount;
4622	for (auto CalleeEdge : CalleeEdges) {
4623	auto *OrigCallee = CalleeEdge->Callee;
4624	// If we don't have a MergeNode yet (only happens on the first iteration,
4625	// as a new one will be created when we go to move the first callee edge
4626	// over as needed), see if we can use this callee.
4627	if (!MergeNode) {
4628	// If there are no other callers, simply use this callee.
4629	if (CalleeEdge->Callee->CallerEdges.size() == `1`) {
4630	MergeNode = OrigCallee;
4631	NonNewMergedNodes ++;
4632	continue;
4633	}
4634	// Otherwise, if we have identified other caller nodes that can share
4635	// the merge node with Node, see if all of OrigCallee's callers are
4636	// going to share the same merge node. In that case we can use callee
4637	// (since all of its callers would move to the new merge node).
4638	if (!OtherCallersToShareMerge.empty()) {
4639	bool MoveAllCallerEdges = true;
4640	for (auto CalleeCallerE : OrigCallee->CallerEdges) {
4641	if (CalleeCallerE == CalleeEdge)
4642	continue;
4643	if (!OtherCallersToShareMerge.contains(CalleeCallerE->Caller)) {
4644	MoveAllCallerEdges = false;
4645	break;
4646	}
4647	}
4648	// If we are going to move all callers over, we can use this callee as
4649	// the MergeNode.
4650	if (MoveAllCallerEdges) {
4651	MergeNode = OrigCallee;
4652	NonNewMergedNodes ++;
4653	continue;
4654	}
4655	}
4656	}
4657	// Move this callee edge, creating a new merge node if necessary.
4658	if (MergeNode) {
4659	assert(MergeNode != OrigCallee);
4660	moveEdgeToExistingCalleeClone(Edge: CalleeEdge, NewCallee: MergeNode,
4661	/NewClone/ false);
4662	} else {
4663	MergeNode = moveEdgeToNewCalleeClone(Edge: CalleeEdge);
4664	NewMergedNodes ++;
4665	}
4666	// Now move all identified edges from other callers over to the merge node
4667	// as well.
4668	if (!OtherCallersToShareMerge.empty()) {
4669	// Make and iterate over a copy of OrigCallee's caller edges because
4670	// some of these will be moved off of the OrigCallee and that would mess
4671	// up the iteration from OrigCallee.
4672	auto OrigCalleeCallerEdges = OrigCallee->CallerEdges;
4673	for (auto &CalleeCallerE : OrigCalleeCallerEdges) {
4674	if (CalleeCallerE == CalleeEdge)
4675	continue;
4676	if (!OtherCallersToShareMerge.contains(CalleeCallerE->Caller))
4677	continue;
4678	CallerToMoveCount[CalleeCallerE->Caller]++;
4679	moveEdgeToExistingCalleeClone(Edge: CalleeCallerE, NewCallee: MergeNode,
4680	/NewClone/ false);
4681	}
4682	}
4683	removeNoneTypeCalleeEdges(Node: OrigCallee);
4684	removeNoneTypeCalleeEdges(Node: MergeNode);
4685	}
4686	}
4687	}
4688
4689	// Look for other nodes that have edges to the same set of callee
4690	// clones as the current Node. Those can share the eventual merge node
4691	// (reducing cloning and binary size overhead) iff:
4692	// - they have edges to the same set of callee clones
4693	// - each callee edge reaches a subset of the same allocations as Node's
4694	// corresponding edge to the same callee clone.
4695	// The second requirement is to ensure that we don't undo any of the
4696	// necessary cloning to distinguish contexts with different allocation
4697	// behavior.
4698	// FIXME: This is somewhat conservative, as we really just need to ensure
4699	// that they don't reach the same allocations as contexts on edges from Node
4700	// going to any of the other* callee clones being merged. However, that*
4701	// requires more tracking and checking to get right.
4702	template <typename DerivedCCG, typename FuncTy, typename CallTy>
4703	void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
4704	findOtherCallersToShareMerge(
4705	ContextNode *Node,
4706	std::vector<std::shared_ptr<ContextEdge>> &CalleeEdges,
4707	DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode,
4708	DenseSet<ContextNode *> &OtherCallersToShareMerge) {
4709	auto NumCalleeClones = CalleeEdges.size();
4710	// This map counts how many edges to the same callee clone exist for other
4711	// caller nodes of each callee clone.
4712	DenseMap<ContextNode , unsigned*> OtherCallersToSharedCalleeEdgeCount;
4713	// Counts the number of other caller nodes that have edges to all callee
4714	// clones that don't violate the allocation context checking.
4715	unsigned PossibleOtherCallerNodes = `0`;
4716
4717	// We only need to look at other Caller nodes if the first callee edge has
4718	// multiple callers (recall they are sorted in ascending order above).
4719	if (CalleeEdges[`0`]->Callee->CallerEdges.size() < `2`)
4720	return;
4721
4722	// For each callee edge:
4723	// - Collect the count of other caller nodes calling the same callees.
4724	// - Collect the alloc nodes reached by contexts on each callee edge.
4725	DenseMap<ContextEdge , DenseSet<ContextNode >> CalleeEdgeToAllocNodes;
4726	for (auto CalleeEdge : CalleeEdges) {
4727	assert(CalleeEdge->Callee->CallerEdges.size() > `1`);
4728	// For each other caller of the same callee, increment the count of
4729	// edges reaching the same callee clone.
4730	for (auto CalleeCallerEdges : CalleeEdge->Callee->CallerEdges) {
4731	if (CalleeCallerEdges->Caller == Node) {
4732	assert(CalleeCallerEdges == CalleeEdge);
4733	continue;
4734	}
4735	OtherCallersToSharedCalleeEdgeCount[CalleeCallerEdges->Caller]++;
4736	// If this caller edge now reaches all of the same callee clones,
4737	// increment the count of candidate other caller nodes.
4738	if (OtherCallersToSharedCalleeEdgeCount[CalleeCallerEdges->Caller] ==
4739	NumCalleeClones)
4740	PossibleOtherCallerNodes++;
4741	}
4742	// Collect the alloc nodes reached by contexts on each callee edge, for
4743	// later analysis.
4744	for (auto Id : CalleeEdge->getContextIds()) {
4745	auto *Alloc = ContextIdToAllocationNode.lookup(Id);
4746	if (!Alloc) {
4747	// FIXME: unclear why this happens occasionally, presumably
4748	// imperfect graph updates possibly with recursion.
4749	MissingAllocForContextId ++;
4750	continue;
4751	}
4752	CalleeEdgeToAllocNodes[CalleeEdge.get()].insert(Alloc);
4753	}
4754	}
4755
4756	// Now walk the callee edges again, and make sure that for each candidate
4757	// caller node all of its edges to the callees reach the same allocs (or
4758	// a subset) as those along the corresponding callee edge from Node.
4759	for (auto CalleeEdge : CalleeEdges) {
4760	assert(CalleeEdge->Callee->CallerEdges.size() > `1`);
4761	// Stop if we do not have any (more) candidate other caller nodes.
4762	if (!PossibleOtherCallerNodes)
4763	break;
4764	auto &CurCalleeAllocNodes = CalleeEdgeToAllocNodes[CalleeEdge.get()];
4765	// Check each other caller of this callee clone.
4766	for (auto &CalleeCallerE : CalleeEdge->Callee->CallerEdges) {
4767	// Not interested in the callee edge from Node itself.
4768	if (CalleeCallerE == CalleeEdge)
4769	continue;
4770	// Skip any callers that didn't have callee edges to all the same
4771	// callee clones.
4772	if (OtherCallersToSharedCalleeEdgeCount[CalleeCallerE->Caller] !=
4773	NumCalleeClones)
4774	continue;
4775	// Make sure that each context along edge from candidate caller node
4776	// reaches an allocation also reached by this callee edge from Node.
4777	for (auto Id : CalleeCallerE->getContextIds()) {
4778	auto *Alloc = ContextIdToAllocationNode.lookup(Id);
4779	if (!Alloc)
4780	continue;
4781	// If not, simply reset the map entry to 0 so caller is ignored, and
4782	// reduce the count of candidate other caller nodes.
4783	if (!CurCalleeAllocNodes.contains(Alloc)) {
4784	OtherCallersToSharedCalleeEdgeCount[CalleeCallerE->Caller] = `0`;
4785	PossibleOtherCallerNodes--;
4786	break;
4787	}
4788	}
4789	}
4790	}
4791
4792	if (!PossibleOtherCallerNodes)
4793	return;
4794
4795	// Build the set of other caller nodes that can use the same callee merge
4796	// node.
4797	for (auto &[OtherCaller, Count] : OtherCallersToSharedCalleeEdgeCount) {
4798	if (Count != NumCalleeClones)
4799	continue;
4800	OtherCallersToShareMerge.insert(OtherCaller);
4801	}
4802	}
4803
4804	// This method assigns cloned callsites to functions, cloning the functions as
4805	// needed. The assignment is greedy and proceeds roughly as follows:
4806	//
4807	// For each function Func:
4808	// For each call with graph Node having clones:
4809	// Initialize ClonesWorklist to Node and its clones
4810	// Initialize NodeCloneCount to 0
4811	// While ClonesWorklist is not empty:
4812	// Clone = pop front ClonesWorklist
4813	// NodeCloneCount++
4814	// If Func has been cloned less than NodeCloneCount times:
4815	// If NodeCloneCount is 1:
4816	// Assign Clone to original Func
4817	// Continue
4818	// Create a new function clone
4819	// If other callers not assigned to call a function clone yet:
4820	// Assign them to call new function clone
4821	// Continue
4822	// Assign any other caller calling the cloned version to new clone
4823	//
4824	// For each caller of Clone:
4825	// If caller is assigned to call a specific function clone:
4826	// If we cannot assign Clone to that function clone:
4827	// Create new callsite Clone NewClone
4828	// Add NewClone to ClonesWorklist
4829	// Continue
4830	// Assign Clone to existing caller's called function clone
4831	// Else:
4832	// If Clone not already assigned to a function clone:
4833	// Assign to first function clone without assignment
4834	// Assign caller to selected function clone
4835	// For each call with graph Node having clones:
4836	// If number func clones > number call's callsite Node clones:
4837	// Record func CallInfo clones without Node clone in UnassignedCallClones
4838	// For callsite Nodes in DFS order from allocations:
4839	// If IsAllocation:
4840	// Update allocation with alloc type
4841	// Else:
4842	// For Call, all MatchingCalls, and associated UnnassignedCallClones:
4843	// Update call to call recorded callee clone
4844	//
4845	template <typename DerivedCCG, typename FuncTy, typename CallTy>
4846	bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::assignFunctions() {
4847	bool Changed = false;
4848
4849	mergeClones();
4850
4851	// Keep track of the assignment of nodes (callsites) to function clones they
4852	// call.
4853	DenseMap<ContextNode *, FuncInfo> CallsiteToCalleeFuncCloneMap;
4854
4855	// Update caller node to call function version CalleeFunc, by recording the
4856	// assignment in CallsiteToCalleeFuncCloneMap.
4857	auto RecordCalleeFuncOfCallsite = [&](ContextNode *Caller,
4858	const FuncInfo &CalleeFunc) {
4859	assert(Caller->hasCall());
4860	CallsiteToCalleeFuncCloneMap[Caller] = CalleeFunc;
4861	};
4862
4863	// Information for a single clone of this Func.
4864	struct FuncCloneInfo {
4865	// The function clone.
4866	FuncInfo FuncClone;
4867	// Remappings of each call of interest (from original uncloned call to the
4868	// corresponding cloned call in this function clone).
4869	DenseMap<CallInfo, CallInfo> CallMap;
4870	};
4871
4872	// Map to keep track of information needed to update calls in function clones
4873	// when their corresponding callsite node was not itself cloned for that
4874	// function clone. Because of call context pruning (i.e. we only keep as much
4875	// caller information as needed to distinguish hot vs cold), we may not have
4876	// caller edges coming to each callsite node from all possible function
4877	// callers. A function clone may get created for other callsites in the
4878	// function for which there are caller edges that were not pruned. Any other
4879	// callsites in that function clone, which were not themselved cloned for
4880	// that function clone, should get updated the same way as the corresponding
4881	// callsite in the original function (which may call a clone of its callee).
4882	//
4883	// We build this map after completing function cloning for each function, so
4884	// that we can record the information from its call maps before they are
4885	// destructed. The map will be used as we update calls to update any still
4886	// unassigned call clones. Note that we may create new node clones as we clone
4887	// other functions, so later on we check which node clones were still not
4888	// created. To this end, the inner map is a map from function clone number to
4889	// the list of calls cloned for that function (can be more than one due to the
4890	// Node's MatchingCalls array).
4891	//
4892	// The alternative is creating new callsite clone nodes below as we clone the
4893	// function, but that is tricker to get right and likely more overhead.
4894	//
4895	// Inner map is a std::map so sorted by key (clone number), in order to get
4896	// ordered remarks in the full LTO case.
4897	DenseMap<const ContextNode , std::map<unsigned*, SmallVector<CallInfo, `0`>>>
4898	UnassignedCallClones;
4899
4900	// Walk all functions for which we saw calls with memprof metadata, and handle
4901	// cloning for each of its calls.
4902	for (auto &[Func, CallsWithMetadata] : FuncToCallsWithMetadata) {
4903	FuncInfo OrigFunc(Func);
4904	// Map from each clone number of OrigFunc to information about that function
4905	// clone (the function clone FuncInfo and call remappings). The index into
4906	// the vector is the clone number, as function clones are created and
4907	// numbered sequentially.
4908	std::vector<FuncCloneInfo> FuncCloneInfos;
4909	for (auto &Call : CallsWithMetadata) {
4910	ContextNode *Node = getNodeForInst(C: Call);
4911	// Skip call if we do not have a node for it (all uses of its stack ids
4912	// were either on inlined chains or pruned from the MIBs), or if we did
4913	// not create any clones for it.
4914	if (!Node \|\| Node->Clones.empty())
4915	continue;
4916	assert(Node->hasCall() &&
4917	"Not having a call should have prevented cloning");
4918
4919	// Track the assignment of function clones to clones of the current
4920	// callsite Node being handled.
4921	std::map<FuncInfo, ContextNode *> FuncCloneToCurNodeCloneMap;
4922
4923	// Assign callsite version CallsiteClone to function version FuncClone,
4924	// and also assign (possibly cloned) Call to CallsiteClone.
4925	auto AssignCallsiteCloneToFuncClone = [&](const FuncInfo &FuncClone,
4926	CallInfo &Call,
4927	ContextNode *CallsiteClone,
4928	bool IsAlloc) {
4929	// Record the clone of callsite node assigned to this function clone.
4930	FuncCloneToCurNodeCloneMap[FuncClone] = CallsiteClone;
4931
4932	assert(FuncCloneInfos.size() > FuncClone.cloneNo());
4933	DenseMap<CallInfo, CallInfo> &CallMap =
4934	FuncCloneInfos[FuncClone.cloneNo()].CallMap;
4935	CallInfo CallClone(Call);
4936	if (auto It = CallMap.find(Call); It != CallMap.end())
4937	CallClone = It->second;
4938	CallsiteClone->setCall(CallClone);
4939	// Need to do the same for all matching calls.
4940	for (auto &MatchingCall : Node->MatchingCalls) {
4941	CallInfo CallClone(MatchingCall);
4942	if (auto It = CallMap.find(MatchingCall); It != CallMap.end())
4943	CallClone = It->second;
4944	// Updates the call in the list.
4945	MatchingCall = CallClone;
4946	}
4947	};
4948
4949	// Invokes moveEdgeToNewCalleeClone which creates a new clone, and then
4950	// performs the necessary fixups (removing none type edges, and
4951	// importantly, propagating any function call assignment of the original
4952	// node to the new clone).
4953	auto MoveEdgeToNewCalleeCloneAndSetUp =
4954	[&](const std::shared_ptr<ContextEdge> &Edge) {
4955	ContextNode *OrigCallee = Edge->Callee;
4956	ContextNode *NewClone = moveEdgeToNewCalleeClone(Edge);
4957	removeNoneTypeCalleeEdges(Node: NewClone);
4958	assert(NewClone->AllocTypes != (uint8_t)AllocationType::None);
4959	// If the original Callee was already assigned to call a specific
4960	// function version, make sure its new clone is assigned to call
4961	// that same function clone.
4962	if (CallsiteToCalleeFuncCloneMap.count(OrigCallee))
4963	RecordCalleeFuncOfCallsite(
4964	NewClone, CallsiteToCalleeFuncCloneMap[OrigCallee]);
4965	return NewClone;
4966	};
4967
4968	// Keep track of the clones of callsite Node that need to be assigned to
4969	// function clones. This list may be expanded in the loop body below if we
4970	// find additional cloning is required.
4971	std::deque<ContextNode *> ClonesWorklist;
4972	// Ignore original Node if we moved all of its contexts to clones.
4973	if (!Node->emptyContextIds())
4974	ClonesWorklist.push_back(Node);
4975	llvm::append_range(ClonesWorklist, Node->Clones);
4976
4977	// Now walk through all of the clones of this callsite Node that we need,
4978	// and determine the assignment to a corresponding clone of the current
4979	// function (creating new function clones as needed).
4980	unsigned NodeCloneCount = `0`;
4981	while (!ClonesWorklist.empty()) {
4982	ContextNode *Clone = ClonesWorklist.front();
4983	ClonesWorklist.pop_front();
4984	NodeCloneCount++;
4985	if (VerifyNodes)
4986	checkNode<DerivedCCG, FuncTy, CallTy>(Clone);
4987
4988	// Need to create a new function clone if we have more callsite clones
4989	// than existing function clones, which would have been assigned to an
4990	// earlier clone in the list (we assign callsite clones to function
4991	// clones greedily).
4992	if (FuncCloneInfos.size() < NodeCloneCount) {
4993	// If this is the first callsite copy, assign to original function.
4994	if (NodeCloneCount == `1`) {
4995	// Since FuncCloneInfos is empty in this case, no clones have
4996	// been created for this function yet, and no callers should have
4997	// been assigned a function clone for this callee node yet.
4998	assert(llvm::none_of(
4999	Clone->CallerEdges, [&](const std::shared_ptr<ContextEdge> &E) {
5000	return CallsiteToCalleeFuncCloneMap.count(E->Caller);
5001	}));
5002	// Initialize with empty call map, assign Clone to original function
5003	// and its callers, and skip to the next clone.
5004	FuncCloneInfos.push_back(
5005	{OrigFunc, DenseMap<CallInfo, CallInfo>()});
5006	AssignCallsiteCloneToFuncClone(
5007	OrigFunc, Call, Clone,
5008	AllocationCallToContextNodeMap.count(Call));
5009	for (auto &CE : Clone->CallerEdges) {
5010	// Ignore any caller that does not have a recorded callsite Call.
5011	if (!CE->Caller->hasCall())
5012	continue;
5013	RecordCalleeFuncOfCallsite(CE->Caller, OrigFunc);
5014	}
5015	continue;
5016	}
5017
5018	// First locate which copy of OrigFunc to clone again. If a caller
5019	// of this callsite clone was already assigned to call a particular
5020	// function clone, we need to redirect all of those callers to the
5021	// new function clone, and update their other callees within this
5022	// function.
5023	FuncInfo PreviousAssignedFuncClone;
5024	auto EI = llvm::find_if(
5025	Clone->CallerEdges, [&](const std::shared_ptr<ContextEdge> &E) {
5026	return CallsiteToCalleeFuncCloneMap.count(E->Caller);
5027	});
5028	bool CallerAssignedToCloneOfFunc = false;
5029	if (EI != Clone->CallerEdges.end()) {
5030	const std::shared_ptr<ContextEdge> &Edge = *EI;
5031	PreviousAssignedFuncClone =
5032	CallsiteToCalleeFuncCloneMap[Edge->Caller];
5033	CallerAssignedToCloneOfFunc = true;
5034	}
5035
5036	// Clone function and save it along with the CallInfo map created
5037	// during cloning in the FuncCloneInfos.
5038	DenseMap<CallInfo, CallInfo> NewCallMap;
5039	unsigned CloneNo = FuncCloneInfos.size();
5040	assert(CloneNo > `0` && "Clone 0 is the original function, which "
5041	"should already exist in the map");
5042	FuncInfo NewFuncClone = cloneFunctionForCallsite(
5043	Func&: OrigFunc, Call, CallMap&: NewCallMap, CallsWithMetadataInFunc&: CallsWithMetadata, CloneNo);
5044	FuncCloneInfos.push_back({NewFuncClone, std::move(NewCallMap)});
5045	FunctionClonesAnalysis ++;
5046	Changed = true;
5047
5048	// If no caller callsites were already assigned to a clone of this
5049	// function, we can simply assign this clone to the new func clone
5050	// and update all callers to it, then skip to the next clone.
5051	if (!CallerAssignedToCloneOfFunc) {
5052	AssignCallsiteCloneToFuncClone(
5053	NewFuncClone, Call, Clone,
5054	AllocationCallToContextNodeMap.count(Call));
5055	for (auto &CE : Clone->CallerEdges) {
5056	// Ignore any caller that does not have a recorded callsite Call.
5057	if (!CE->Caller->hasCall())
5058	continue;
5059	RecordCalleeFuncOfCallsite(CE->Caller, NewFuncClone);
5060	}
5061	continue;
5062	}
5063
5064	// We may need to do additional node cloning in this case.
5065	// Reset the CallsiteToCalleeFuncCloneMap entry for any callers
5066	// that were previously assigned to call PreviousAssignedFuncClone,
5067	// to record that they now call NewFuncClone.
5068	// The none type edge removal may remove some of this Clone's caller
5069	// edges, if it is reached via another of its caller's callees.
5070	// Iterate over a copy and skip any that were removed.
5071	auto CallerEdges = Clone->CallerEdges;
5072	for (auto CE : CallerEdges) {
5073	// Skip any that have been removed on an earlier iteration.
5074	if (CE->isRemoved()) {
5075	assert(!is_contained(Clone->CallerEdges, CE));
5076	continue;
5077	}
5078	assert(CE);
5079	// Ignore any caller that does not have a recorded callsite Call.
5080	if (!CE->Caller->hasCall())
5081	continue;
5082
5083	if (!CallsiteToCalleeFuncCloneMap.count(CE->Caller) \|\|
5084	// We subsequently fall through to later handling that
5085	// will perform any additional cloning required for
5086	// callers that were calling other function clones.
5087	CallsiteToCalleeFuncCloneMap[CE->Caller] !=
5088	PreviousAssignedFuncClone)
5089	continue;
5090
5091	RecordCalleeFuncOfCallsite(CE->Caller, NewFuncClone);
5092
5093	// If we are cloning a function that was already assigned to some
5094	// callers, then essentially we are creating new callsite clones
5095	// of the other callsites in that function that are reached by those
5096	// callers. Clone the other callees of the current callsite's caller
5097	// that were already assigned to PreviousAssignedFuncClone
5098	// accordingly. This is important since we subsequently update the
5099	// calls from the nodes in the graph and their assignments to callee
5100	// functions recorded in CallsiteToCalleeFuncCloneMap.
5101	// The none type edge removal may remove some of this caller's
5102	// callee edges, if it is reached via another of its callees.
5103	// Iterate over a copy and skip any that were removed.
5104	auto CalleeEdges = CE->Caller->CalleeEdges;
5105	for (auto CalleeEdge : CalleeEdges) {
5106	// Skip any that have been removed on an earlier iteration when
5107	// cleaning up newly None type callee edges.
5108	if (CalleeEdge->isRemoved()) {
5109	assert(!is_contained(CE->Caller->CalleeEdges, CalleeEdge));
5110	continue;
5111	}
5112	assert(CalleeEdge);
5113	ContextNode *Callee = CalleeEdge->Callee;
5114	// Skip the current callsite, we are looking for other
5115	// callsites Caller calls, as well as any that does not have a
5116	// recorded callsite Call.
5117	if (Callee == Clone \|\| !Callee->hasCall())
5118	continue;
5119	// Skip direct recursive calls. We don't need/want to clone the
5120	// caller node again, and this loop will not behave as expected if
5121	// we tried.
5122	if (Callee == CalleeEdge->Caller)
5123	continue;
5124	ContextNode *NewClone =
5125	MoveEdgeToNewCalleeCloneAndSetUp(CalleeEdge);
5126	// Moving the edge may have resulted in some none type
5127	// callee edges on the original Callee.
5128	removeNoneTypeCalleeEdges(Node: Callee);
5129	// Update NewClone with the new Call clone of this callsite's Call
5130	// created for the new function clone created earlier.
5131	// Recall that we have already ensured when building the graph
5132	// that each caller can only call callsites within the same
5133	// function, so we are guaranteed that Callee Call is in the
5134	// current OrigFunc.
5135	// CallMap is set up as indexed by original Call at clone 0.
5136	CallInfo OrigCall(Callee->getOrigNode()->Call);
5137	OrigCall.setCloneNo(`0`);
5138	DenseMap<CallInfo, CallInfo> &CallMap =
5139	FuncCloneInfos[NewFuncClone.cloneNo()].CallMap;
5140	assert(CallMap.count(OrigCall));
5141	CallInfo NewCall(CallMap[OrigCall]);
5142	assert(NewCall);
5143	NewClone->setCall(NewCall);
5144	// Need to do the same for all matching calls.
5145	for (auto &MatchingCall : NewClone->MatchingCalls) {
5146	CallInfo OrigMatchingCall(MatchingCall);
5147	OrigMatchingCall.setCloneNo(`0`);
5148	assert(CallMap.count(OrigMatchingCall));
5149	CallInfo NewCall(CallMap[OrigMatchingCall]);
5150	assert(NewCall);
5151	// Updates the call in the list.
5152	MatchingCall = NewCall;
5153	}
5154	}
5155	}
5156	// Fall through to handling below to perform the recording of the
5157	// function for this callsite clone. This enables handling of cases
5158	// where the callers were assigned to different clones of a function.
5159	}
5160
5161	auto FindFirstAvailFuncClone = [&]() {
5162	// Find first function in FuncCloneInfos without an assigned
5163	// clone of this callsite Node. We should always have one
5164	// available at this point due to the earlier cloning when the
5165	// FuncCloneInfos size was smaller than the clone number.
5166	for (auto &CF : FuncCloneInfos) {
5167	if (!FuncCloneToCurNodeCloneMap.count(CF.FuncClone))
5168	return CF.FuncClone;
5169	}
5170	llvm_unreachable(
5171	"Expected an available func clone for this callsite clone");
5172	};
5173
5174	// See if we can use existing function clone. Walk through
5175	// all caller edges to see if any have already been assigned to
5176	// a clone of this callsite's function. If we can use it, do so. If not,
5177	// because that function clone is already assigned to a different clone
5178	// of this callsite, then we need to clone again.
5179	// Basically, this checking is needed to handle the case where different
5180	// caller functions/callsites may need versions of this function
5181	// containing different mixes of callsite clones across the different
5182	// callsites within the function. If that happens, we need to create
5183	// additional function clones to handle the various combinations.
5184	//
5185	// Keep track of any new clones of this callsite created by the
5186	// following loop, as well as any existing clone that we decided to
5187	// assign this clone to.
5188	std::map<FuncInfo, ContextNode *> FuncCloneToNewCallsiteCloneMap;
5189	FuncInfo FuncCloneAssignedToCurCallsiteClone;
5190	// Iterate over a copy of Clone's caller edges, since we may need to
5191	// remove edges in the moveEdgeTo methods, and this simplifies the*
5192	// handling and makes it less error-prone.
5193	auto CloneCallerEdges = Clone->CallerEdges;
5194	for (auto &Edge : CloneCallerEdges) {
5195	// Skip removed edges (due to direct recursive edges updated when
5196	// updating callee edges when moving an edge and subsequently
5197	// removed by call to removeNoneTypeCalleeEdges on the Clone).
5198	if (Edge->isRemoved())
5199	continue;
5200	// Ignore any caller that does not have a recorded callsite Call.
5201	if (!Edge->Caller->hasCall())
5202	continue;
5203	// If this caller already assigned to call a version of OrigFunc, need
5204	// to ensure we can assign this callsite clone to that function clone.
5205	if (CallsiteToCalleeFuncCloneMap.count(Edge->Caller)) {
5206	FuncInfo FuncCloneCalledByCaller =
5207	CallsiteToCalleeFuncCloneMap[Edge->Caller];
5208	// First we need to confirm that this function clone is available
5209	// for use by this callsite node clone.
5210	//
5211	// While FuncCloneToCurNodeCloneMap is built only for this Node and
5212	// its callsite clones, one of those callsite clones X could have
5213	// been assigned to the same function clone called by Edge's caller
5214	// - if Edge's caller calls another callsite within Node's original
5215	// function, and that callsite has another caller reaching clone X.
5216	// We need to clone Node again in this case.
5217	if ((FuncCloneToCurNodeCloneMap.count(FuncCloneCalledByCaller) &&
5218	FuncCloneToCurNodeCloneMap[FuncCloneCalledByCaller] !=
5219	Clone) \|\|
5220	// Detect when we have multiple callers of this callsite that
5221	// have already been assigned to specific, and different, clones
5222	// of OrigFunc (due to other unrelated callsites in Func they
5223	// reach via call contexts). Is this Clone of callsite Node
5224	// assigned to a different clone of OrigFunc? If so, clone Node
5225	// again.
5226	(FuncCloneAssignedToCurCallsiteClone &&
5227	FuncCloneAssignedToCurCallsiteClone !=
5228	FuncCloneCalledByCaller)) {
5229	// We need to use a different newly created callsite clone, in
5230	// order to assign it to another new function clone on a
5231	// subsequent iteration over the Clones array (adjusted below).
5232	// Note we specifically do not reset the
5233	// CallsiteToCalleeFuncCloneMap entry for this caller, so that
5234	// when this new clone is processed later we know which version of
5235	// the function to copy (so that other callsite clones we have
5236	// assigned to that function clone are properly cloned over). See
5237	// comments in the function cloning handling earlier.
5238
5239	// Check if we already have cloned this callsite again while
5240	// walking through caller edges, for a caller calling the same
5241	// function clone. If so, we can move this edge to that new clone
5242	// rather than creating yet another new clone.
5243	if (FuncCloneToNewCallsiteCloneMap.count(
5244	FuncCloneCalledByCaller)) {
5245	ContextNode *NewClone =
5246	FuncCloneToNewCallsiteCloneMap[FuncCloneCalledByCaller];
5247	moveEdgeToExistingCalleeClone(Edge, NewCallee: NewClone);
5248	// Cleanup any none type edges cloned over.
5249	removeNoneTypeCalleeEdges(Node: NewClone);
5250	} else {
5251	// Create a new callsite clone.
5252	ContextNode *NewClone = MoveEdgeToNewCalleeCloneAndSetUp(Edge);
5253	FuncCloneToNewCallsiteCloneMap[FuncCloneCalledByCaller] =
5254	NewClone;
5255	// Add to list of clones and process later.
5256	ClonesWorklist.push_back(NewClone);
5257	}
5258	// Moving the caller edge may have resulted in some none type
5259	// callee edges.
5260	removeNoneTypeCalleeEdges(Node: Clone);
5261	// We will handle the newly created callsite clone in a subsequent
5262	// iteration over this Node's Clones.
5263	continue;
5264	}
5265
5266	// Otherwise, we can use the function clone already assigned to this
5267	// caller.
5268	if (!FuncCloneAssignedToCurCallsiteClone) {
5269	FuncCloneAssignedToCurCallsiteClone = FuncCloneCalledByCaller;
5270	// Assign Clone to FuncCloneCalledByCaller
5271	AssignCallsiteCloneToFuncClone(
5272	FuncCloneCalledByCaller, Call, Clone,
5273	AllocationCallToContextNodeMap.count(Call));
5274	} else
5275	// Don't need to do anything - callsite is already calling this
5276	// function clone.
5277	assert(FuncCloneAssignedToCurCallsiteClone ==
5278	FuncCloneCalledByCaller);
5279
5280	} else {
5281	// We have not already assigned this caller to a version of
5282	// OrigFunc. Do the assignment now.
5283
5284	// First check if we have already assigned this callsite clone to a
5285	// clone of OrigFunc for another caller during this iteration over
5286	// its caller edges.
5287	if (!FuncCloneAssignedToCurCallsiteClone) {
5288	FuncCloneAssignedToCurCallsiteClone = FindFirstAvailFuncClone();
5289	assert(FuncCloneAssignedToCurCallsiteClone);
5290	// Assign Clone to FuncCloneAssignedToCurCallsiteClone
5291	AssignCallsiteCloneToFuncClone(
5292	FuncCloneAssignedToCurCallsiteClone, Call, Clone,
5293	AllocationCallToContextNodeMap.count(Call));
5294	} else
5295	assert(FuncCloneToCurNodeCloneMap
5296	[FuncCloneAssignedToCurCallsiteClone] == Clone);
5297	// Update callers to record function version called.
5298	RecordCalleeFuncOfCallsite(Edge->Caller,
5299	FuncCloneAssignedToCurCallsiteClone);
5300	}
5301	}
5302	// If we didn't assign a function clone to this callsite clone yet, e.g.
5303	// none of its callers has a non-null call, do the assignment here.
5304	// We want to ensure that every callsite clone is assigned to some
5305	// function clone, so that the call updates below work as expected.
5306	// In particular if this is the original callsite, we want to ensure it
5307	// is assigned to the original function, otherwise the original function
5308	// will appear available for assignment to other callsite clones,
5309	// leading to unintended effects. For one, the unknown and not updated
5310	// callers will call into cloned paths leading to the wrong hints,
5311	// because they still call the original function (clone 0). Also,
5312	// because all callsites start out as being clone 0 by default, we can't
5313	// easily distinguish between callsites explicitly assigned to clone 0
5314	// vs those never assigned, which can lead to multiple updates of the
5315	// calls when invoking updateCall below, with mismatched clone values.
5316	// TODO: Add a flag to the callsite nodes or some other mechanism to
5317	// better distinguish and identify callsite clones that are not getting
5318	// assigned to function clones as expected.
5319	if (!FuncCloneAssignedToCurCallsiteClone) {
5320	FuncCloneAssignedToCurCallsiteClone = FindFirstAvailFuncClone();
5321	assert(FuncCloneAssignedToCurCallsiteClone &&
5322	"No available func clone for this callsite clone");
5323	AssignCallsiteCloneToFuncClone(
5324	FuncCloneAssignedToCurCallsiteClone, Call, Clone,
5325	/IsAlloc=/AllocationCallToContextNodeMap.contains(Call));
5326	}
5327	}
5328	if (VerifyCCG) {
5329	checkNode<DerivedCCG, FuncTy, CallTy>(Node);
5330	for (const auto &PE : Node->CalleeEdges)
5331	checkNode<DerivedCCG, FuncTy, CallTy>(PE->Callee);
5332	for (const auto &CE : Node->CallerEdges)
5333	checkNode<DerivedCCG, FuncTy, CallTy>(CE->Caller);
5334	for (auto *Clone : Node->Clones) {
5335	checkNode<DerivedCCG, FuncTy, CallTy>(Clone);
5336	for (const auto &PE : Clone->CalleeEdges)
5337	checkNode<DerivedCCG, FuncTy, CallTy>(PE->Callee);
5338	for (const auto &CE : Clone->CallerEdges)
5339	checkNode<DerivedCCG, FuncTy, CallTy>(CE->Caller);
5340	}
5341	}
5342	}
5343
5344	if (FuncCloneInfos.size() < `2`)
5345	continue;
5346
5347	// In this case there is more than just the original function copy.
5348	// Record call clones of any callsite nodes in the function that did not
5349	// themselves get cloned for all of the function clones.
5350	for (auto &Call : CallsWithMetadata) {
5351	ContextNode *Node = getNodeForInst(C: Call);
5352	if (!Node \|\| !Node->hasCall() \|\| Node->emptyContextIds())
5353	continue;
5354	// If Node has enough clones already to cover all function clones, we can
5355	// skip it. Need to add one for the original copy.
5356	// Use >= in case there were clones that were skipped due to having empty
5357	// context ids
5358	if (Node->Clones.size() + `1` >= FuncCloneInfos.size())
5359	continue;
5360	// First collect all function clones we cloned this callsite node for.
5361	// They may not be sequential due to empty clones e.g.
5362	DenseSet<unsigned> NodeCallClones;
5363	for (auto *C : Node->Clones)
5364	NodeCallClones.insert(C->Call.cloneNo());
5365	unsigned I = `0`;
5366	// Now check all the function clones.
5367	for (auto &FC : FuncCloneInfos) {
5368	// Function clones should be sequential.
5369	assert(FC.FuncClone.cloneNo() == I);
5370	// Skip the first clone which got the original call.
5371	// Also skip any other clones created for this Node.
5372	if (++I == `1` \|\| NodeCallClones.contains(V: I)) {
5373	continue;
5374	}
5375	// Record the call clones created for this callsite in this function
5376	// clone.
5377	auto &CallVector = UnassignedCallClones[Node][I];
5378	DenseMap<CallInfo, CallInfo> &CallMap = FC.CallMap;
5379	if (auto It = CallMap.find(Call); It != CallMap.end()) {
5380	CallInfo CallClone = It->second;
5381	CallVector.push_back(CallClone);
5382	} else {
5383	// All but the original clone (skipped earlier) should have an entry
5384	// for all calls.
5385	assert(false && "Expected to find call in CallMap");
5386	}
5387	// Need to do the same for all matching calls.
5388	for (auto &MatchingCall : Node->MatchingCalls) {
5389	if (auto It = CallMap.find(MatchingCall); It != CallMap.end()) {
5390	CallInfo CallClone = It->second;
5391	CallVector.push_back(CallClone);
5392	} else {
5393	// All but the original clone (skipped earlier) should have an entry
5394	// for all calls.
5395	assert(false && "Expected to find call in CallMap");
5396	}
5397	}
5398	}
5399	}
5400	}
5401
5402	uint8_t BothTypes =
5403	(uint8_t)AllocationType::Cold \| (uint8_t)AllocationType::NotCold;
5404
5405	auto UpdateCalls = [&](ContextNode *Node,
5406	DenseSet<const ContextNode *> &Visited,
5407	auto &&UpdateCalls) {
5408	auto Inserted = Visited.insert(Node);
5409	if (!Inserted.second)
5410	return;
5411
5412	for (auto *Clone : Node->Clones)
5413	UpdateCalls(Clone, Visited, UpdateCalls);
5414
5415	for (auto &Edge : Node->CallerEdges)
5416	UpdateCalls(Edge->Caller, Visited, UpdateCalls);
5417
5418	// Skip if either no call to update, or if we ended up with no context ids
5419	// (we moved all edges onto other clones).
5420	if (!Node->hasCall() \|\| Node->emptyContextIds())
5421	return;
5422
5423	if (Node->IsAllocation) {
5424	auto AT = allocTypeToUse(Node->AllocTypes);
5425	// If the allocation type is ambiguous, and more aggressive hinting
5426	// has been enabled via the MinClonedColdBytePercent flag, see if this
5427	// allocation should be hinted cold anyway because its fraction cold bytes
5428	// allocated is at least the given threshold.
5429	if (Node->AllocTypes == BothTypes && MinClonedColdBytePercent < `100` &&
5430	!ContextIdToContextSizeInfos.empty()) {
5431	uint64_t TotalCold = `0`;
5432	uint64_t Total = `0`;
5433	for (auto Id : Node->getContextIds()) {
5434	auto TypeI = ContextIdToAllocationType.find(Id);
5435	assert(TypeI != ContextIdToAllocationType.end());
5436	auto CSI = ContextIdToContextSizeInfos.find(Id);
5437	if (CSI != ContextIdToContextSizeInfos.end()) {
5438	for (auto &Info : CSI->second) {
5439	Total += Info.TotalSize;
5440	if (TypeI->second == AllocationType::Cold)
5441	TotalCold += Info.TotalSize;
5442	}
5443	}
5444	}
5445	if (TotalCold * `100` >= Total * MinClonedColdBytePercent)
5446	AT = AllocationType::Cold;
5447	}
5448	updateAllocationCall(Call&: Node->Call, AllocType: AT);
5449	assert(Node->MatchingCalls.empty());
5450	return;
5451	}
5452
5453	if (!CallsiteToCalleeFuncCloneMap.count(Node))
5454	return;
5455
5456	auto CalleeFunc = CallsiteToCalleeFuncCloneMap[Node];
5457	updateCall(CallerCall&: Node->Call, CalleeFunc);
5458	// Update all the matching calls as well.
5459	for (auto &Call : Node->MatchingCalls)
5460	updateCall(CallerCall&: Call, CalleeFunc);
5461
5462	// Now update all calls recorded earlier that are still in function clones
5463	// which don't have a clone of this callsite node.
5464	if (!UnassignedCallClones.contains(Node))
5465	return;
5466	DenseSet<unsigned> NodeCallClones;
5467	for (auto *C : Node->Clones)
5468	NodeCallClones.insert(C->Call.cloneNo());
5469	// Note that we already confirmed Node is in this map a few lines above.
5470	auto &ClonedCalls = UnassignedCallClones[Node];
5471	for (auto &[CloneNo, CallVector] : ClonedCalls) {
5472	// Should start at 1 as we never create an entry for original node.
5473	assert(CloneNo > `0`);
5474	// If we subsequently created a clone, skip this one.
5475	if (NodeCallClones.contains(V: CloneNo))
5476	continue;
5477	// Use the original Node's CalleeFunc.
5478	for (auto &Call : CallVector)
5479	updateCall(CallerCall&: Call, CalleeFunc);
5480	}
5481	};
5482
5483	// Performs DFS traversal starting from allocation nodes to update calls to
5484	// reflect cloning decisions recorded earlier. For regular LTO this will
5485	// update the actual calls in the IR to call the appropriate function clone
5486	// (and add attributes to allocation calls), whereas for ThinLTO the decisions
5487	// are recorded in the summary entries.
5488	DenseSet<const ContextNode *> Visited;
5489	for (auto &Entry : AllocationCallToContextNodeMap)
5490	UpdateCalls(Entry.second, Visited, UpdateCalls);
5491
5492	return Changed;
5493	}
5494
5495	// Compute a SHA1 hash of the callsite and alloc version information of clone I
5496	// in the summary, to use in detection of duplicate clones.
5497	uint64_t ComputeHash(const FunctionSummary FS, unsigned* I) {
5498	SHA1 Hasher;
5499	// Update hash with any callsites that call non-default (non-zero) callee
5500	// versions.
5501	for (auto &SN : FS->callsites()) {
5502	// In theory all callsites and allocs in this function should have the same
5503	// number of clone entries, but handle any discrepancies gracefully below
5504	// for NDEBUG builds.
5505	assert(
5506	SN.Clones.size() > I &&
5507	"Callsite summary has fewer entries than other summaries in function");
5508	if (SN.Clones.size() <= I \|\| !SN.Clones [I])
5509	continue;
5510	uint8_t Data[sizeof(SN.Clones [I])];
5511	support::endian::write32le(P: Data, V: SN.Clones [I]);
5512	Hasher.update(Data);
5513	}
5514	// Update hash with any allocs that have non-default (non-None) hints.
5515	for (auto &AN : FS->allocs()) {
5516	// In theory all callsites and allocs in this function should have the same
5517	// number of clone entries, but handle any discrepancies gracefully below
5518	// for NDEBUG builds.
5519	assert(AN.Versions.size() > I &&
5520	"Alloc summary has fewer entries than other summaries in function");
5521	if (AN.Versions.size() <= I \|\|
5522	(AllocationType)AN.Versions [I] == AllocationType::None)
5523	continue;
5524	Hasher.update(Data: ArrayRef<uint8_t>(&AN.Versions [I], `1`));
5525	}
5526	return support::endian::read64le(P: Hasher.result().data());
5527	}
5528
5529	static SmallVector<std::unique_ptr<ValueToValueMapTy>, `4`> createFunctionClones(
5530	Function &F, unsigned NumClones, Module &M, OptimizationRemarkEmitter &ORE,
5531	std::map<const Function , SmallPtrSet<const* GlobalAlias *, `1`>>
5532	&FuncToAliasMap,
5533	FunctionSummary *FS) {
5534	auto TakeDeclNameAndReplace = [](GlobalValue DeclGV, GlobalValue NewGV) {
5535	// We might have created this when adjusting callsite in another
5536	// function. It should be a declaration.
5537	assert(DeclGV->isDeclaration());
5538	NewGV->takeName(V: DeclGV);
5539	DeclGV->replaceAllUsesWith(V: NewGV);
5540	DeclGV->eraseFromParent();
5541	};
5542
5543	// Handle aliases to this function, and create analogous alias clones to the
5544	// provided clone of this function.
5545	auto CloneFuncAliases = [&](Function NewF, unsigned* I) {
5546	if (!FuncToAliasMap.count(x: &F))
5547	return;
5548	for (auto *A : FuncToAliasMap [&F]) {
5549	std::string AliasName = getMemProfFuncName(Base: A->getName(), CloneNo: I);
5550	auto *PrevA = M.getNamedAlias(Name: AliasName);
5551	auto *NewA = GlobalAlias::create(Ty: A->getValueType(),
5552	AddressSpace: A->getType()->getPointerAddressSpace(),
5553	Linkage: A->getLinkage(), Name: AliasName, Aliasee: NewF);
5554	NewA->copyAttributesFrom(Src: A);
5555	if (PrevA)
5556	TakeDeclNameAndReplace (PrevA, NewA);
5557	}
5558	};
5559
5560	// The first "clone" is the original copy, we should only call this if we
5561	// needed to create new clones.
5562	assert(NumClones > `1`);
5563	SmallVector<std::unique_ptr<ValueToValueMapTy>, `4`> VMaps;
5564	VMaps.reserve(N: NumClones - `1`);
5565	FunctionsClonedThinBackend ++;
5566
5567	// Map of hash of callsite/alloc versions to the instantiated function clone
5568	// (possibly the original) implementing those calls. Used to avoid
5569	// instantiating duplicate function clones.
5570	// FIXME: Ideally the thin link would not generate such duplicate clones to
5571	// start with, but right now it happens due to phase ordering in the function
5572	// assignment and possible new clones that produces. We simply make each
5573	// duplicate an alias to the matching instantiated clone recorded in the map
5574	// (except for available_externally which are made declarations as they would
5575	// be aliases in the prevailing module, and available_externally aliases are
5576	// not well supported right now).
5577	DenseMap<uint64_t, Function *> HashToFunc;
5578
5579	// Save the hash of the original function version.
5580	HashToFunc [ComputeHash(FS, I: `0`)] = &F;
5581
5582	for (unsigned I = `1`; I < NumClones; I++) {
5583	VMaps.emplace_back(Args: std::make_unique<ValueToValueMapTy>());
5584	std::string Name = getMemProfFuncName(Base: F.getName(), CloneNo: I);
5585	auto Hash = ComputeHash(FS, I);
5586	// If this clone would duplicate a previously seen clone, don't generate the
5587	// duplicate clone body, just make an alias to satisfy any (potentially
5588	// cross-module) references.
5589	if (HashToFunc.contains(Val: Hash)) {
5590	FunctionCloneDuplicatesThinBackend ++;
5591	auto *Func = HashToFunc [Hash];
5592	if (Func->hasAvailableExternallyLinkage()) {
5593	// Skip these as EliminateAvailableExternallyPass does not handle
5594	// available_externally aliases correctly and we end up with an
5595	// available_externally alias to a declaration. Just create a
5596	// declaration for now as we know we will have a definition in another
5597	// module.
5598	auto Decl = M.getOrInsertFunction(Name, T: Func->getFunctionType());
5599	ORE.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofClone", &F)
5600	<< "created clone decl " << ore::NV ("Decl", Decl.getCallee()));
5601	continue;
5602	}
5603	auto *PrevF = M.getFunction(Name);
5604	auto *Alias = GlobalAlias::create(Name, Aliasee: Func);
5605	if (PrevF)
5606	TakeDeclNameAndReplace (PrevF, Alias);
5607	ORE.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofClone", &F)
5608	<< "created clone alias " << ore::NV ("Alias", Alias));
5609
5610	// Now handle aliases to this function, and clone those as well.
5611	CloneFuncAliases (Func, I);
5612	continue;
5613	}
5614	auto NewF = CloneFunction(F: &F, VMap&: VMaps.back());
5615	HashToFunc [Hash] = NewF;
5616	FunctionClonesThinBackend ++;
5617	// Strip memprof and callsite metadata from clone as they are no longer
5618	// needed.
5619	for (auto &BB : *NewF) {
5620	for (auto &Inst : BB) {
5621	Inst.setMetadata(KindID: LLVMContext::MD_memprof, Node: nullptr);
5622	Inst.setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
5623	}
5624	}
5625	auto *PrevF = M.getFunction(Name);
5626	if (PrevF)
5627	TakeDeclNameAndReplace (PrevF, NewF);
5628	else
5629	NewF->setName(Name);
5630	updateSubprogramLinkageName(NewFunc: NewF, Name);
5631	ORE.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofClone", &F)
5632	<< "created clone " << ore::NV ("NewFunction", NewF));
5633
5634	// Now handle aliases to this function, and clone those as well.
5635	CloneFuncAliases (NewF, I);
5636	}
5637	return VMaps;
5638	}
5639
5640	// Locate the summary for F. This is complicated by the fact that it might
5641	// have been internalized or promoted.
5642	static ValueInfo findValueInfoForFunc(const Function &F, const Module &M,
5643	const ModuleSummaryIndex *ImportSummary,
5644	const Function CallingFunc = nullptr*) {
5645	// FIXME: Ideally we would retain the original GUID in some fashion on the
5646	// function (e.g. as metadata), but for now do our best to locate the
5647	// summary without that information.
5648	ValueInfo TheFnVI = ImportSummary->getValueInfo(GUID: F.getGUID());
5649	if (!TheFnVI)
5650	// See if theFn was internalized, by checking index directly with
5651	// original name (this avoids the name adjustment done by getGUID() for
5652	// internal symbols).
5653	TheFnVI = ImportSummary->getValueInfo(
5654	GUID: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: F.getName()));
5655	if (TheFnVI)
5656	return TheFnVI;
5657	// Now query with the original name before any promotion was performed.
5658	StringRef OrigName =
5659	ModuleSummaryIndex::getOriginalNameBeforePromote(Name: F.getName());
5660	// When this pass is enabled, we always add thinlto_src_file provenance
5661	// metadata to imported function definitions, which allows us to recreate the
5662	// original internal symbol's GUID.
5663	auto SrcFileMD = F.getMetadata(Kind: "thinlto_src_file");
5664	// If this is a call to an imported/promoted local for which we didn't import
5665	// the definition, the metadata will not exist on the declaration. However,
5666	// since we are doing this early, before any inlining in the LTO backend, we
5667	// can simply look at the metadata on the calling function which must have
5668	// been from the same module if F was an internal symbol originally.
5669	if (!SrcFileMD && F.isDeclaration()) {
5670	// We would only call this for a declaration for a direct callsite, in which
5671	// case the caller would have provided the calling function pointer.
5672	assert(CallingFunc);
5673	SrcFileMD = CallingFunc->getMetadata(Kind: "thinlto_src_file");
5674	// If this is a promoted local (OrigName != F.getName()), since this is a
5675	// declaration, it must be imported from a different module and therefore we
5676	// should always find the metadata on its calling function. Any call to a
5677	// promoted local that came from this module should still be a definition.
5678	assert(SrcFileMD \|\| OrigName == F.getName());
5679	}
5680	StringRef SrcFile = M.getSourceFileName();
5681	if (SrcFileMD)
5682	SrcFile = dyn_cast<MDString>(Val: SrcFileMD->getOperand(I: `0`))->getString();
5683	std::string OrigId = GlobalValue::getGlobalIdentifier(
5684	Name: OrigName, Linkage: GlobalValue::InternalLinkage, FileName: SrcFile);
5685	TheFnVI = ImportSummary->getValueInfo(
5686	GUID: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigId));
5687	// Internal func in original module may have gotten a numbered suffix if we
5688	// imported an external function with the same name. This happens
5689	// automatically during IR linking for naming conflicts. It would have to
5690	// still be internal in that case (otherwise it would have been renamed on
5691	// promotion in which case we wouldn't have a naming conflict).
5692	if (!TheFnVI && OrigName == F.getName() && F.hasLocalLinkage() &&
5693	F.getName().contains(C: `'.'`)) {
5694	OrigName = F.getName().rsplit(Separator: `'.'`).first;
5695	OrigId = GlobalValue::getGlobalIdentifier(
5696	Name: OrigName, Linkage: GlobalValue::InternalLinkage, FileName: SrcFile);
5697	TheFnVI = ImportSummary->getValueInfo(
5698	GUID: GlobalValue::getGUIDAssumingExternalLinkage(GlobalName: OrigId));
5699	}
5700	// The only way we may not have a VI is if this is a declaration created for
5701	// an imported reference. For distributed ThinLTO we may not have a VI for
5702	// such declarations in the distributed summary.
5703	assert(TheFnVI \|\| F.isDeclaration());
5704	return TheFnVI;
5705	}
5706
5707	bool MemProfContextDisambiguation::initializeIndirectCallPromotionInfo(
5708	Module &M) {
5709	ICallAnalysis = std::make_unique<ICallPromotionAnalysis>();
5710	Symtab = std::make_unique<InstrProfSymtab>();
5711	// Don't add canonical names, to avoid multiple functions to the symtab
5712	// when they both have the same root name with "." suffixes stripped.
5713	// If we pick the wrong one then this could lead to incorrect ICP and calling
5714	// a memprof clone that we don't actually create (resulting in linker unsats).
5715	// What this means is that the GUID of the function (or its PGOFuncName
5716	// metadata) must* match that in the VP metadata to allow promotion.*
5717	// In practice this should not be a limitation, since local functions should
5718	// have PGOFuncName metadata and global function names shouldn't need any
5719	// special handling (they should not get the ".llvm." suffix that the*
5720	// canonicalization handling is attempting to strip).
5721	if (Error E = Symtab ->create(M, /InLTO=/true, /AddCanonical=/false)) {
5722	std::string SymtabFailure = toString(E: std::move(E));
5723	M.getContext().emitError(ErrorStr: "Failed to create symtab: " + SymtabFailure);
5724	return false;
5725	}
5726	return true;
5727	}
5728
5729	#ifndef NDEBUG
5730	// Sanity check that the MIB stack ids match between the summary and
5731	// instruction metadata.
5732	static void checkAllocContextIds(
5733	const AllocInfo &AllocNode, const MDNode *MemProfMD,
5734	const CallStack<MDNode, MDNode::op_iterator> &CallsiteContext,
5735	const ModuleSummaryIndex *ImportSummary) {
5736	auto MIBIter = AllocNode.MIBs.begin();
5737	for (auto &MDOp : MemProfMD->operands()) {
5738	assert(MIBIter != AllocNode.MIBs.end());
5739	auto StackIdIndexIter = MIBIter->StackIdIndices.begin();
5740	auto MIBMD = cast<const* MDNode>(MDOp);
5741	MDNode *StackMDNode = getMIBStackNode(MIBMD);
5742	assert(StackMDNode);
5743	CallStack<MDNode, MDNode::op_iterator> StackContext(StackMDNode);
5744	auto ContextIterBegin =
5745	StackContext.beginAfterSharedPrefix(CallsiteContext);
5746	// Skip the checking on the first iteration.
5747	uint64_t LastStackContextId =
5748	(ContextIterBegin != StackContext.end() && *ContextIterBegin == `0`) ? `1`
5749	: `0`;
5750	for (auto ContextIter = ContextIterBegin; ContextIter != StackContext.end();
5751	++ContextIter) {
5752	// If this is a direct recursion, simply skip the duplicate
5753	// entries, to be consistent with how the summary ids were
5754	// generated during ModuleSummaryAnalysis.
5755	if (LastStackContextId == *ContextIter)
5756	continue;
5757	LastStackContextId = *ContextIter;
5758	assert(StackIdIndexIter != MIBIter->StackIdIndices.end());
5759	assert(ImportSummary->getStackIdAtIndex(*StackIdIndexIter) ==
5760	*ContextIter);
5761	StackIdIndexIter++;
5762	}
5763	MIBIter++;
5764	}
5765	}
5766	#endif
5767
5768	bool MemProfContextDisambiguation::applyImport(Module &M) {
5769	assert(ImportSummary);
5770	bool Changed = false;
5771
5772	// We also need to clone any aliases that reference cloned functions, because
5773	// the modified callsites may invoke via the alias. Keep track of the aliases
5774	// for each function.
5775	std::map<const Function , SmallPtrSet<const* GlobalAlias *, `1`>>
5776	FuncToAliasMap;
5777	for (auto &A : M.aliases()) {
5778	auto *Aliasee = A.getAliaseeObject();
5779	if (auto *F = dyn_cast<Function>(Val: Aliasee))
5780	FuncToAliasMap [F].insert(Ptr: &A);
5781	}
5782
5783	if (!initializeIndirectCallPromotionInfo(M))
5784	return false;
5785
5786	for (auto &F : M) {
5787	if (F.isDeclaration() \|\| isMemProfClone(F))
5788	continue;
5789
5790	OptimizationRemarkEmitter ORE(&F);
5791
5792	SmallVector<std::unique_ptr<ValueToValueMapTy>, `4`> VMaps;
5793	bool ClonesCreated = false;
5794	unsigned NumClonesCreated = `0`;
5795	auto CloneFuncIfNeeded = [&](unsigned NumClones, FunctionSummary *FS) {
5796	// We should at least have version 0 which is the original copy.
5797	assert(NumClones > `0`);
5798	// If only one copy needed use original.
5799	if (NumClones == `1`)
5800	return;
5801	// If we already performed cloning of this function, confirm that the
5802	// requested number of clones matches (the thin link should ensure the
5803	// number of clones for each constituent callsite is consistent within
5804	// each function), before returning.
5805	if (ClonesCreated) {
5806	assert(NumClonesCreated == NumClones);
5807	return;
5808	}
5809	VMaps = createFunctionClones(F, NumClones, M, ORE, FuncToAliasMap, FS);
5810	// The first "clone" is the original copy, which doesn't have a VMap.
5811	assert(VMaps.size() == NumClones - `1`);
5812	Changed = true;
5813	ClonesCreated = true;
5814	NumClonesCreated = NumClones;
5815	};
5816
5817	auto CloneCallsite = [&](const CallsiteInfo &StackNode, CallBase *CB,
5818	Function CalledFunction, FunctionSummary FS) {
5819	// Perform cloning if not yet done.
5820	CloneFuncIfNeeded (/NumClones=/StackNode.Clones.size(), FS);
5821
5822	assert(!isMemProfClone(*CalledFunction));
5823
5824	// Because we update the cloned calls by calling setCalledOperand (see
5825	// comment below), out of an abundance of caution make sure the called
5826	// function was actually the called operand (or its aliasee). We also
5827	// strip pointer casts when looking for calls (to match behavior during
5828	// summary generation), however, with opaque pointers in theory this
5829	// should not be an issue. Note we still clone the current function
5830	// (containing this call) above, as that could be needed for its callers.
5831	auto *GA = dyn_cast_or_null<GlobalAlias>(Val: CB->getCalledOperand());
5832	if (CalledFunction != CB->getCalledOperand() &&
5833	(!GA \|\| CalledFunction != GA->getAliaseeObject())) {
5834	SkippedCallsCloning ++;
5835	return;
5836	}
5837	// Update the calls per the summary info.
5838	// Save orig name since it gets updated in the first iteration
5839	// below.
5840	auto CalleeOrigName = CalledFunction->getName();
5841	for (unsigned J = `0`; J < StackNode.Clones.size(); J++) {
5842	// If the VMap is empty, this clone was a duplicate of another and was
5843	// created as an alias or a declaration.
5844	if (J > `0` && VMaps [J - `1`]->empty())
5845	continue;
5846	// Do nothing if this version calls the original version of its
5847	// callee.
5848	if (!StackNode.Clones [J])
5849	continue;
5850	auto NewF = M.getOrInsertFunction(
5851	Name: getMemProfFuncName(Base: CalleeOrigName, CloneNo: StackNode.Clones [J]),
5852	T: CalledFunction->getFunctionType());
5853	CallBase *CBClone;
5854	// Copy 0 is the original function.
5855	if (!J)
5856	CBClone = CB;
5857	else
5858	CBClone = cast<CallBase>(Val&: (*VMaps [J - `1`])[CB]);
5859	// Set the called operand directly instead of calling setCalledFunction,
5860	// as the latter mutates the function type on the call. In rare cases
5861	// we may have a slightly different type on a callee function
5862	// declaration due to it being imported from a different module with
5863	// incomplete types. We really just want to change the name of the
5864	// function to the clone, and not make any type changes.
5865	CBClone->setCalledOperand(NewF.getCallee());
5866	ORE.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofCall", CBClone)
5867	<< ore::NV ("Call", CBClone) << " in clone "
5868	<< ore::NV ("Caller", CBClone->getFunction())
5869	<< " assigned to call function clone "
5870	<< ore::NV ("Callee", NewF.getCallee()));
5871	}
5872	};
5873
5874	// Locate the summary for F.
5875	ValueInfo TheFnVI = findValueInfoForFunc(F, M, ImportSummary);
5876	// If not found, this could be an imported local (see comment in
5877	// findValueInfoForFunc). Skip for now as it will be cloned in its original
5878	// module (where it would have been promoted to global scope so should
5879	// satisfy any reference in this module).
5880	if (!TheFnVI)
5881	continue;
5882
5883	auto *GVSummary =
5884	ImportSummary->findSummaryInModule(VI: TheFnVI, ModuleId: M.getModuleIdentifier());
5885	if (!GVSummary) {
5886	// Must have been imported, use the summary which matches the definition。
5887	// (might be multiple if this was a linkonce_odr).
5888	auto SrcModuleMD = F.getMetadata(Kind: "thinlto_src_module");
5889	assert(SrcModuleMD &&
5890	"enable-import-metadata is needed to emit thinlto_src_module");
5891	StringRef SrcModule =
5892	dyn_cast<MDString>(Val: SrcModuleMD->getOperand(I: `0`))->getString();
5893	for (auto &GVS : TheFnVI.getSummaryList()) {
5894	if (GVS ->modulePath() == SrcModule) {
5895	GVSummary = GVS.get();
5896	break;
5897	}
5898	}
5899	// TODO: Put back the assert once we have metadata on imported copies of
5900	// aliases linking them back to the original alias GUID, which would allow
5901	// us to locate the alias summary here.
5902	// assert(GVSummary && GVSummary->modulePath() == SrcModule);
5903	}
5904
5905	// GVSummary can be null if this is a function imported as a copy of an
5906	// alias, and we don't have the aliasee's summary in our distributed index.
5907	// TODO: Once we can locate the original GUID for imported aliases (e.g. via
5908	// TBD additional metadata), we should find the alias summary instead, and
5909	// we can remove this check and fall back to the original check below.
5910	if (!GVSummary)
5911	continue;
5912
5913	// If this was an imported alias skip it as we won't have the function
5914	// summary, and it should be cloned in the original module.
5915	if (isa<AliasSummary>(Val: GVSummary))
5916	continue;
5917
5918	auto *FS = cast<FunctionSummary>(Val: GVSummary->getBaseObject());
5919
5920	if (FS->allocs().empty() && FS->callsites().empty())
5921	continue;
5922
5923	auto SI = FS->callsites().begin();
5924	auto AI = FS->allocs().begin();
5925
5926	// To handle callsite infos synthesized for tail calls which have missing
5927	// frames in the profiled context, map callee VI to the synthesized callsite
5928	// info.
5929	DenseMap<ValueInfo, CallsiteInfo> MapTailCallCalleeVIToCallsite;
5930	// Iterate the callsites for this function in reverse, since we place all
5931	// those synthesized for tail calls at the end.
5932	for (auto CallsiteIt = FS->callsites().rbegin();
5933	CallsiteIt != FS->callsites().rend(); CallsiteIt ++) {
5934	auto &Callsite = *CallsiteIt;
5935	// Stop as soon as we see a non-synthesized callsite info (see comment
5936	// above loop). All the entries added for discovered tail calls have empty
5937	// stack ids.
5938	if (!Callsite.StackIdIndices.empty())
5939	break;
5940	MapTailCallCalleeVIToCallsite.insert(KV: {Callsite.Callee, Callsite});
5941	}
5942
5943	// Keeps track of needed ICP for the function.
5944	SmallVector<ICallAnalysisData> ICallAnalysisInfo;
5945
5946	// Assume for now that the instructions are in the exact same order
5947	// as when the summary was created, but confirm this is correct by
5948	// matching the stack ids.
5949	for (auto &BB : F) {
5950	for (auto &I : BB) {
5951	auto *CB = dyn_cast<CallBase>(Val: &I);
5952	// Same handling as when creating module summary.
5953	if (!mayHaveMemprofSummary(CB))
5954	continue;
5955
5956	auto *CalledValue = CB->getCalledOperand();
5957	auto *CalledFunction = CB->getCalledFunction();
5958	if (CalledValue && !CalledFunction) {
5959	CalledValue = CalledValue->stripPointerCasts();
5960	// Stripping pointer casts can reveal a called function.
5961	CalledFunction = dyn_cast<Function>(Val: CalledValue);
5962	}
5963	// Check if this is an alias to a function. If so, get the
5964	// called aliasee for the checks below.
5965	if (auto *GA = dyn_cast<GlobalAlias>(Val: CalledValue)) {
5966	assert(!CalledFunction &&
5967	"Expected null called function in callsite for alias");
5968	CalledFunction = dyn_cast<Function>(Val: GA->getAliaseeObject());
5969	}
5970
5971	CallStack<MDNode, MDNode::op_iterator> CallsiteContext(
5972	I.getMetadata(KindID: LLVMContext::MD_callsite));
5973	auto *MemProfMD = I.getMetadata(KindID: LLVMContext::MD_memprof);
5974
5975	// Include allocs that were already assigned a memprof function
5976	// attribute in the statistics. Only do this for those that do not have
5977	// memprof metadata, since we add an "ambiguous" memprof attribute by
5978	// default.
5979	if (CB->getAttributes().hasFnAttr(Kind: "memprof") && !MemProfMD) {
5980	CB->getAttributes().getFnAttr(Kind: "memprof").getValueAsString() == "cold"
5981	? AllocTypeColdThinBackend ++
5982	: AllocTypeNotColdThinBackend ++;
5983	OrigAllocsThinBackend ++;
5984	AllocVersionsThinBackend ++;
5985	if (!MaxAllocVersionsThinBackend)
5986	MaxAllocVersionsThinBackend = `1`;
5987	continue;
5988	}
5989
5990	if (MemProfMD) {
5991	// Consult the next alloc node.
5992	assert(AI != FS->allocs().end());
5993	auto &AllocNode = *(AI++);
5994
5995	#ifndef NDEBUG
5996	checkAllocContextIds(AllocNode, MemProfMD, CallsiteContext,
5997	ImportSummary);
5998	#endif
5999
6000	// Perform cloning if not yet done.
6001	CloneFuncIfNeeded (/NumClones=/AllocNode.Versions.size(), FS);
6002
6003	OrigAllocsThinBackend ++;
6004	AllocVersionsThinBackend += AllocNode.Versions.size();
6005	if (MaxAllocVersionsThinBackend < AllocNode.Versions.size())
6006	MaxAllocVersionsThinBackend = AllocNode.Versions.size();
6007
6008	// If there is only one version that means we didn't end up
6009	// considering this function for cloning, and in that case the alloc
6010	// will still be none type or should have gotten the default NotCold.
6011	// Skip that after calling clone helper since that does some sanity
6012	// checks that confirm we haven't decided yet that we need cloning.
6013	// We might have a single version that is cold due to the
6014	// MinClonedColdBytePercent heuristic, make sure we don't skip in that
6015	// case.
6016	if (AllocNode.Versions.size() == `1` &&
6017	(AllocationType)AllocNode.Versions [`0`] != AllocationType::Cold) {
6018	assert((AllocationType)AllocNode.Versions[`0`] ==
6019	AllocationType::NotCold \|\|
6020	(AllocationType)AllocNode.Versions[`0`] ==
6021	AllocationType::None);
6022	UnclonableAllocsThinBackend ++;
6023	continue;
6024	}
6025
6026	// All versions should have a singular allocation type.
6027	assert(llvm::none_of(AllocNode.Versions, [](uint8_t Type) {
6028	return Type == ((uint8_t)AllocationType::NotCold \|
6029	(uint8_t)AllocationType::Cold);
6030	}));
6031
6032	// Update the allocation types per the summary info.
6033	for (unsigned J = `0`; J < AllocNode.Versions.size(); J++) {
6034	// If the VMap is empty, this clone was a duplicate of another and
6035	// was created as an alias or a declaration.
6036	if (J > `0` && VMaps [J - `1`]->empty())
6037	continue;
6038	// Ignore any that didn't get an assigned allocation type.
6039	if (AllocNode.Versions [J] == (uint8_t)AllocationType::None)
6040	continue;
6041	AllocationType AllocTy = (AllocationType)AllocNode.Versions [J];
6042	AllocTy == AllocationType::Cold ? AllocTypeColdThinBackend ++
6043	: AllocTypeNotColdThinBackend ++;
6044	std::string AllocTypeString = getAllocTypeAttributeString(Type: AllocTy);
6045	auto A = llvm::Attribute::get(Context&: F.getContext(), Kind: "memprof",
6046	Val: AllocTypeString);
6047	CallBase *CBClone;
6048	// Copy 0 is the original function.
6049	if (!J)
6050	CBClone = CB;
6051	else
6052	// Since VMaps are only created for new clones, we index with
6053	// clone J-1 (J==0 is the original clone and does not have a VMaps
6054	// entry).
6055	CBClone = cast<CallBase>(Val&: (*VMaps [J - `1`])[CB]);
6056	removeAnyExistingAmbiguousAttribute(CB: CBClone);
6057	CBClone->addFnAttr(Attr: A);
6058	ORE.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofAttribute", CBClone)
6059	<< ore::NV ("AllocationCall", CBClone) << " in clone "
6060	<< ore::NV ("Caller", CBClone->getFunction())
6061	<< " marked with memprof allocation attribute "
6062	<< ore::NV ("Attribute", AllocTypeString));
6063	}
6064	} else if (!CallsiteContext.empty()) {
6065	if (!CalledFunction) {
6066	#ifndef NDEBUG
6067	// We should have skipped inline assembly calls.
6068	auto *CI = dyn_cast<CallInst>(CB);
6069	assert(!CI \|\| !CI->isInlineAsm());
6070	#endif
6071	// We should have skipped direct calls via a Constant.
6072	assert(CalledValue && !isa<Constant>(CalledValue));
6073
6074	// This is an indirect call, see if we have profile information and
6075	// whether any clones were recorded for the profiled targets (that
6076	// we synthesized CallsiteInfo summary records for when building the
6077	// index).
6078	auto NumClones =
6079	recordICPInfo(CB, AllCallsites: FS->callsites(), SI, ICallAnalysisInfo);
6080
6081	// Perform cloning if not yet done. This is done here in case
6082	// we don't need to do ICP, but might need to clone this
6083	// function as it is the target of other cloned calls.
6084	if (NumClones)
6085	CloneFuncIfNeeded (NumClones, FS);
6086	}
6087
6088	else {
6089	// Consult the next callsite node.
6090	assert(SI != FS->callsites().end());
6091	auto &StackNode = *(SI++);
6092
6093	#ifndef NDEBUG
6094	// Sanity check that the stack ids match between the summary and
6095	// instruction metadata.
6096	auto StackIdIndexIter = StackNode.StackIdIndices.begin();
6097	for (auto StackId : CallsiteContext) {
6098	assert(StackIdIndexIter != StackNode.StackIdIndices.end());
6099	assert(ImportSummary->getStackIdAtIndex(*StackIdIndexIter) ==
6100	StackId);
6101	StackIdIndexIter++;
6102	}
6103	#endif
6104
6105	CloneCallsite (StackNode, CB, CalledFunction, FS);
6106	}
6107	} else if (CB->isTailCall() && CalledFunction) {
6108	// Locate the synthesized callsite info for the callee VI, if any was
6109	// created, and use that for cloning.
6110	ValueInfo CalleeVI =
6111	findValueInfoForFunc(F: *CalledFunction, M, ImportSummary, CallingFunc: &F);
6112	if (CalleeVI && MapTailCallCalleeVIToCallsite.count(Val: CalleeVI)) {
6113	auto Callsite = MapTailCallCalleeVIToCallsite.find(Val: CalleeVI);
6114	assert(Callsite != MapTailCallCalleeVIToCallsite.end());
6115	CloneCallsite (Callsite ->second, CB, CalledFunction, FS);
6116	}
6117	}
6118	}
6119	}
6120
6121	// Now do any promotion required for cloning.
6122	performICP(M, AllCallsites: FS->callsites(), VMaps, ICallAnalysisInfo, ORE);
6123	}
6124
6125	// We skip some of the functions and instructions above, so remove all the
6126	// metadata in a single sweep here.
6127	for (auto &F : M) {
6128	// We can skip memprof clones because createFunctionClones already strips
6129	// the metadata from the newly created clones.
6130	if (F.isDeclaration() \|\| isMemProfClone(F))
6131	continue;
6132	for (auto &BB : F) {
6133	for (auto &I : BB) {
6134	if (!isa<CallBase>(Val: I))
6135	continue;
6136	I.setMetadata(KindID: LLVMContext::MD_memprof, Node: nullptr);
6137	I.setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
6138	}
6139	}
6140	}
6141
6142	return Changed;
6143	}
6144
6145	unsigned MemProfContextDisambiguation::recordICPInfo(
6146	CallBase *CB, ArrayRef<CallsiteInfo> AllCallsites,
6147	ArrayRef<CallsiteInfo>::iterator &SI,
6148	SmallVector<ICallAnalysisData> &ICallAnalysisInfo) {
6149	// First see if we have profile information for this indirect call.
6150	uint32_t NumCandidates;
6151	uint64_t TotalCount;
6152	auto CandidateProfileData =
6153	ICallAnalysis ->getPromotionCandidatesForInstruction(
6154	I: CB, TotalCount, NumCandidates, MaxNumValueData: MaxSummaryIndirectEdges);
6155	if (CandidateProfileData.empty())
6156	return `0`;
6157
6158	// Iterate through all of the candidate profiled targets along with the
6159	// CallsiteInfo summary records synthesized for them when building the index,
6160	// and see if any are cloned and/or refer to clones.
6161	bool ICPNeeded = false;
6162	unsigned NumClones = `0`;
6163	size_t CallsiteInfoStartIndex = std::distance(first: AllCallsites.begin(), last: SI);
6164	for (const auto &Candidate : CandidateProfileData) {
6165	#ifndef NDEBUG
6166	auto CalleeValueInfo =
6167	#endif
6168	ImportSummary->getValueInfo(GUID: Candidate.Value);
6169	// We might not have a ValueInfo if this is a distributed
6170	// ThinLTO backend and decided not to import that function.
6171	assert(!CalleeValueInfo \|\| SI->Callee == CalleeValueInfo);
6172	assert(SI != AllCallsites.end());
6173	auto &StackNode = *(SI++);
6174	// See if any of the clones of the indirect callsite for this
6175	// profiled target should call a cloned version of the profiled
6176	// target. We only need to do the ICP here if so.
6177	ICPNeeded \|= llvm::any_of(Range: StackNode.Clones,
6178	P: [](unsigned CloneNo) { return CloneNo != `0`; });
6179	// Every callsite in the same function should have been cloned the same
6180	// number of times.
6181	assert(!NumClones \|\| NumClones == StackNode.Clones.size());
6182	NumClones = StackNode.Clones.size();
6183	}
6184	if (!ICPNeeded)
6185	return NumClones;
6186	// Save information for ICP, which is performed later to avoid messing up the
6187	// current function traversal.
6188	ICallAnalysisInfo.push_back(Elt: {.CB: CB, .CandidateProfileData: CandidateProfileData.vec(), .NumCandidates: NumCandidates,
6189	.TotalCount: TotalCount, .CallsiteInfoStartIndex: CallsiteInfoStartIndex});
6190	return NumClones;
6191	}
6192
6193	void MemProfContextDisambiguation::performICP(
6194	Module &M, ArrayRef<CallsiteInfo> AllCallsites,
6195	ArrayRef<std::unique_ptr<ValueToValueMapTy>> VMaps,
6196	ArrayRef<ICallAnalysisData> ICallAnalysisInfo,
6197	OptimizationRemarkEmitter &ORE) {
6198	// Now do any promotion required for cloning. Specifically, for each
6199	// recorded ICP candidate (which was only recorded because one clone of that
6200	// candidate should call a cloned target), we perform ICP (speculative
6201	// devirtualization) for each clone of the callsite, and update its callee
6202	// to the appropriate clone. Note that the ICP compares against the original
6203	// version of the target, which is what is in the vtable.
6204	for (auto &Info : ICallAnalysisInfo) {
6205	auto *CB = Info.CB;
6206	auto CallsiteIndex = Info.CallsiteInfoStartIndex;
6207	auto TotalCount = Info.TotalCount;
6208	unsigned NumClones = `0`;
6209	SmallVector<InstrProfValueData, `8`> RemainingCandidates;
6210
6211	for (auto &Candidate : Info.CandidateProfileData) {
6212	auto &StackNode = AllCallsites [CallsiteIndex++];
6213
6214	// All calls in the same function must have the same number of clones.
6215	assert(!NumClones \|\| NumClones == StackNode.Clones.size());
6216	NumClones = StackNode.Clones.size();
6217
6218	// See if the target is in the module. If it wasn't imported, it is
6219	// possible that this profile could have been collected on a different
6220	// target (or version of the code), and we need to be conservative
6221	// (similar to what is done in the ICP pass).
6222	Function *TargetFunction = Symtab ->getFunction(FuncMD5Hash: Candidate.Value);
6223	if (TargetFunction == nullptr \|\|
6224	// Any ThinLTO global dead symbol removal should have already
6225	// occurred, so it should be safe to promote when the target is a
6226	// declaration.
6227	// TODO: Remove internal option once more fully tested.
6228	(MemProfRequireDefinitionForPromotion &&
6229	TargetFunction->isDeclaration())) {
6230	ORE.emit(RemarkBuilder: [&]() {
6231	return OptimizationRemarkMissed (DEBUG_TYPE, "UnableToFindTarget", CB)
6232	<< "Memprof cannot promote indirect call: target with md5sum "
6233	<< ore::NV ("target md5sum", Candidate.Value) << " not found";
6234	});
6235	// FIXME: See if we can use the new declaration importing support to
6236	// at least get the declarations imported for this case. Hot indirect
6237	// targets should have been imported normally, however.
6238	RemainingCandidates.push_back(Elt: Candidate);
6239	continue;
6240	}
6241
6242	// Check if legal to promote
6243	const char Reason = nullptr*;
6244	if (!isLegalToPromote(CB: *CB, Callee: TargetFunction, FailureReason: &Reason)) {
6245	ORE.emit(RemarkBuilder: [&]() {
6246	return OptimizationRemarkMissed (DEBUG_TYPE, "UnableToPromote", CB)
6247	<< "Memprof cannot promote indirect call to "
6248	<< ore::NV ("TargetFunction", TargetFunction)
6249	<< " with count of " << ore::NV ("TotalCount", TotalCount)
6250	<< ": " << Reason;
6251	});
6252	RemainingCandidates.push_back(Elt: Candidate);
6253	continue;
6254	}
6255
6256	assert(!isMemProfClone(*TargetFunction));
6257
6258	// Handle each call clone, applying ICP so that each clone directly
6259	// calls the specified callee clone, guarded by the appropriate ICP
6260	// check.
6261	CallBase *CBClone = CB;
6262	for (unsigned J = `0`; J < NumClones; J++) {
6263	// If the VMap is empty, this clone was a duplicate of another and was
6264	// created as an alias or a declaration.
6265	if (J > `0` && VMaps [J - `1`]->empty())
6266	continue;
6267	// Copy 0 is the original function.
6268	if (J > `0`)
6269	CBClone = cast<CallBase>(Val&: (*VMaps [J - `1`])[CB]);
6270	// We do the promotion using the original name, so that the comparison
6271	// is against the name in the vtable. Then just below, change the new
6272	// direct call to call the cloned function.
6273	auto &DirectCall =
6274	pgo::promoteIndirectCall(CB&: *CBClone, F: TargetFunction, Count: Candidate.Count,
6275	TotalCount, AttachProfToDirectCall: isSamplePGO, ORE: &ORE);
6276	auto *TargetToUse = TargetFunction;
6277	// Call original if this version calls the original version of its
6278	// callee.
6279	if (StackNode.Clones [J]) {
6280	TargetToUse =
6281	cast<Function>(Val: M.getOrInsertFunction(
6282	Name: getMemProfFuncName(Base: TargetFunction->getName(),
6283	CloneNo: StackNode.Clones [J]),
6284	T: TargetFunction->getFunctionType())
6285	.getCallee());
6286	}
6287	DirectCall.setCalledFunction(TargetToUse);
6288	// During matching we generate synthetic VP metadata for indirect calls
6289	// not already having any, from the memprof profile's callee GUIDs. If
6290	// we subsequently promote and inline those callees, we currently lose
6291	// the ability to generate this synthetic VP metadata. Optionally apply
6292	// a noinline attribute to promoted direct calls, where the threshold is
6293	// set to capture synthetic VP metadata targets which get a count of 1.
6294	if (MemProfICPNoInlineThreshold &&
6295	Candidate.Count < MemProfICPNoInlineThreshold)
6296	DirectCall.setIsNoInline();
6297	ORE.emit(OptDiag: OptimizationRemark (DEBUG_TYPE, "MemprofCall", CBClone)
6298	<< ore::NV ("Call", CBClone) << " in clone "
6299	<< ore::NV ("Caller", CBClone->getFunction())
6300	<< " promoted and assigned to call function clone "
6301	<< ore::NV ("Callee", TargetToUse));
6302	}
6303
6304	// Update TotalCount (all clones should get same count above)
6305	TotalCount -= Candidate.Count;
6306	}
6307	// Adjust the MD.prof metadata for all clones, now that we have the new
6308	// TotalCount and the remaining candidates.
6309	CallBase *CBClone = CB;
6310	for (unsigned J = `0`; J < NumClones; J++) {
6311	// If the VMap is empty, this clone was a duplicate of another and was
6312	// created as an alias or a declaration.
6313	if (J > `0` && VMaps [J - `1`]->empty())
6314	continue;
6315	// Copy 0 is the original function.
6316	if (J > `0`)
6317	CBClone = cast<CallBase>(Val&: (*VMaps [J - `1`])[CB]);
6318	// First delete the old one.
6319	CBClone->setMetadata(KindID: LLVMContext::MD_prof, Node: nullptr);
6320	// If all promoted, we don't need the MD.prof metadata.
6321	// Otherwise we need update with the un-promoted records back.
6322	if (TotalCount != `0`)
6323	annotateValueSite(M, Inst&: *CBClone, VDs: RemainingCandidates, Sum: TotalCount,
6324	ValueKind: IPVK_IndirectCallTarget, MaxMDCount: Info.NumCandidates);
6325	}
6326	}
6327	}
6328
6329	template <typename DerivedCCG, typename FuncTy, typename CallTy>
6330	bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::process(
6331	function_ref<void(StringRef, StringRef, const Twine &)> EmitRemark,
6332	bool AllowExtraAnalysis) {
6333	if (DumpCCG) {
6334	dbgs() << "CCG before cloning:\n";
6335	dbgs() << *this;
6336	}
6337	if (ExportToDot)
6338	exportToDot(Label: "postbuild");
6339
6340	if (VerifyCCG) {
6341	check();
6342	}
6343
6344	identifyClones();
6345
6346	if (VerifyCCG) {
6347	check();
6348	}
6349
6350	if (DumpCCG) {
6351	dbgs() << "CCG after cloning:\n";
6352	dbgs() << *this;
6353	}
6354	if (ExportToDot)
6355	exportToDot(Label: "cloned");
6356
6357	bool Changed = assignFunctions();
6358
6359	if (DumpCCG) {
6360	dbgs() << "CCG after assigning function clones:\n";
6361	dbgs() << *this;
6362	}
6363	if (ExportToDot)
6364	exportToDot(Label: "clonefuncassign");
6365
6366	if (MemProfReportHintedSizes \|\| AllowExtraAnalysis)
6367	printTotalSizes(OS&: errs(), EmitRemark);
6368
6369	return Changed;
6370	}
6371
6372	bool MemProfContextDisambiguation::processModule(
6373	Module &M,
6374	llvm::function_ref<OptimizationRemarkEmitter &(Function *)> OREGetter) {
6375
6376	// If we have an import summary, then the cloning decisions were made during
6377	// the thin link on the index. Apply them and return.
6378	if (ImportSummary)
6379	return applyImport(M);
6380
6381	// TODO: If/when other types of memprof cloning are enabled beyond just for
6382	// hot and cold, we will need to change this to individually control the
6383	// AllocationType passed to addStackNodesForMIB during CCG construction.
6384	// Note that we specifically check this after applying imports above, so that
6385	// the option isn't needed to be passed to distributed ThinLTO backend
6386	// clang processes, which won't necessarily have visibility into the linker
6387	// dependences. Instead the information is communicated from the LTO link to
6388	// the backends via the combined summary index.
6389	if (!SupportsHotColdNew)
6390	return false;
6391
6392	ModuleCallsiteContextGraph CCG(M, OREGetter);
6393	// TODO: Set up remarks for regular LTO. We need to decide what function to
6394	// use in the callback.
6395	return CCG.process();
6396	}
6397
6398	MemProfContextDisambiguation::MemProfContextDisambiguation(
6399	const ModuleSummaryIndex Summary, bool* isSamplePGO)
6400	: ImportSummary(Summary), isSamplePGO(isSamplePGO) {
6401	// Check the dot graph printing options once here, to make sure we have valid
6402	// and expected combinations.
6403	if (DotGraphScope == DotScope::Alloc && !AllocIdForDot.getNumOccurrences())
6404	llvm::report_fatal_error(
6405	reason: "-memprof-dot-scope=alloc requires -memprof-dot-alloc-id");
6406	if (DotGraphScope == DotScope::Context &&
6407	!ContextIdForDot.getNumOccurrences())
6408	llvm::report_fatal_error(
6409	reason: "-memprof-dot-scope=context requires -memprof-dot-context-id");
6410	if (DotGraphScope == DotScope::All && AllocIdForDot.getNumOccurrences() &&
6411	ContextIdForDot.getNumOccurrences())
6412	llvm::report_fatal_error(
6413	reason: "-memprof-dot-scope=all can't have both -memprof-dot-alloc-id and "
6414	"-memprof-dot-context-id");
6415	if (ImportSummary) {
6416	// The MemProfImportSummary should only be used for testing ThinLTO
6417	// distributed backend handling via opt, in which case we don't have a
6418	// summary from the pass pipeline.
6419	assert(MemProfImportSummary.empty());
6420	return;
6421	}
6422	if (MemProfImportSummary.empty())
6423	return;
6424
6425	auto ReadSummaryFile =
6426	errorOrToExpected(EO: MemoryBuffer::getFile(Filename: MemProfImportSummary));
6427	if (!ReadSummaryFile) {
6428	logAllUnhandledErrors(E: ReadSummaryFile.takeError(), OS&: errs(),
6429	ErrorBanner: "Error loading file '" + MemProfImportSummary +
6430	"': ");
6431	return;
6432	}
6433	auto ImportSummaryForTestingOrErr = getModuleSummaryIndex(Buffer: **ReadSummaryFile);
6434	if (!ImportSummaryForTestingOrErr) {
6435	logAllUnhandledErrors(E: ImportSummaryForTestingOrErr.takeError(), OS&: errs(),
6436	ErrorBanner: "Error parsing file '" + MemProfImportSummary +
6437	"': ");
6438	return;
6439	}
6440	ImportSummaryForTesting = std::move(*ImportSummaryForTestingOrErr);
6441	ImportSummary = ImportSummaryForTesting.get();
6442	}
6443
6444	PreservedAnalyses MemProfContextDisambiguation::run(Module &M,
6445	ModuleAnalysisManager &AM) {
6446	auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
6447	auto OREGetter = [&](Function *F) -> OptimizationRemarkEmitter & {
6448	return FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: *F);
6449	};
6450	if (!processModule(M, OREGetter))
6451	return PreservedAnalyses::all();
6452	return PreservedAnalyses::none();
6453	}
6454
6455	void MemProfContextDisambiguation::run(
6456	ModuleSummaryIndex &Index,
6457	llvm::function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
6458	isPrevailing,
6459	LLVMContext &Ctx,
6460	function_ref<void(StringRef, StringRef, const Twine &)> EmitRemark) {
6461	// TODO: If/when other types of memprof cloning are enabled beyond just for
6462	// hot and cold, we will need to change this to individually control the
6463	// AllocationType passed to addStackNodesForMIB during CCG construction.
6464	// The index was set from the option, so these should be in sync.
6465	assert(Index.withSupportsHotColdNew() == SupportsHotColdNew);
6466	if (!SupportsHotColdNew)
6467	return;
6468
6469	bool AllowExtraAnalysis =
6470	OptimizationRemarkEmitter::allowExtraAnalysis(Ctx, DEBUG_TYPE);
6471
6472	IndexCallsiteContextGraph CCG(Index, isPrevailing);
6473	CCG.process(EmitRemark, AllowExtraAnalysis);
6474	}
6475
6476	// Strips MemProf attributes and metadata. Can be invoked by the pass pipeline
6477	// when we don't have an index that has recorded that we are linking with
6478	// allocation libraries containing the necessary APIs for downstream
6479	// transformations.
6480	PreservedAnalyses MemProfRemoveInfo::run(Module &M, ModuleAnalysisManager &AM) {
6481	// The profile matcher applies hotness attributes directly for allocations,
6482	// and those will cause us to generate calls to the hot/cold interfaces
6483	// unconditionally. If supports-hot-cold-new was not enabled in the LTO
6484	// link then assume we don't want these calls (e.g. not linking with
6485	// the appropriate library, or otherwise trying to disable this behavior).
6486	bool Changed = false;
6487	for (auto &F : M) {
6488	for (auto &BB : F) {
6489	for (auto &I : BB) {
6490	auto *CI = dyn_cast<CallBase>(Val: &I);
6491	if (!CI)
6492	continue;
6493	if (CI->hasFnAttr(Kind: "memprof")) {
6494	CI->removeFnAttr(Kind: "memprof");
6495	Changed = true;
6496	}
6497	if (!CI->hasMetadata(KindID: LLVMContext::MD_callsite)) {
6498	assert(!CI->hasMetadata(LLVMContext::MD_memprof));
6499	continue;
6500	}
6501	// Strip off all memprof metadata as it is no longer needed.
6502	// Importantly, this avoids the addition of new memprof attributes
6503	// after inlining propagation.
6504	CI->setMetadata(KindID: LLVMContext::MD_memprof, Node: nullptr);
6505	CI->setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
6506	Changed = true;
6507	}
6508	}
6509	}
6510	if (!Changed)
6511	return PreservedAnalyses::all();
6512	return PreservedAnalyses::none();
6513	}
6514

Browse the source code of llvm_projects/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp