1//===-- MemoryProfileInfo.cpp - memory profile info ------------------------==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains utilities to analyze memory profile information for
10// profile-guided memory optimization (MemPGO).
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Analysis/MemoryProfileInfo.h"
15#include "llvm/Analysis/OptimizationRemarkEmitter.h"
16#include "llvm/IR/Constants.h"
17#include "llvm/Support/CommandLine.h"
18#include "llvm/Support/Compiler.h"
19#include "llvm/Support/Format.h"
20
21using namespace llvm;
22using namespace llvm::memprof;
23
24#define DEBUG_TYPE "memory-profile-info"
25
26namespace llvm {
27
28cl::opt<bool> MemProfReportHintedSizes(
29 "memprof-report-hinted-sizes", cl::init(Val: false), cl::Hidden,
30 cl::desc("Report total allocation sizes of hinted allocations"));
31
32// This is useful if we have enabled reporting of hinted sizes, and want to get
33// information from the indexing step for all contexts (especially for testing),
34// or have specified a value less than 100% for -memprof-cloning-cold-threshold.
35LLVM_ABI cl::opt<bool> MemProfKeepAllNotColdContexts(
36 "memprof-keep-all-not-cold-contexts", cl::init(Val: false), cl::Hidden,
37 cl::desc("Keep all non-cold contexts (increases cloning overheads)"));
38
39cl::opt<unsigned> MinClonedColdBytePercent(
40 "memprof-cloning-cold-threshold", cl::init(Val: 100), cl::Hidden,
41 cl::desc("Min percent of cold bytes to hint alloc cold during cloning"));
42
43// Discard non-cold contexts if they overlap with much larger cold contexts,
44// specifically, if all contexts reaching a given callsite are at least this
45// percent cold byte allocations. This reduces the amount of cloning required
46// to expose the cold contexts when they greatly dominate non-cold contexts.
47cl::opt<unsigned> MinCallsiteColdBytePercent(
48 "memprof-callsite-cold-threshold", cl::init(Val: 100), cl::Hidden,
49 cl::desc("Min percent of cold bytes at a callsite to discard non-cold "
50 "contexts"));
51
52// Enable saving context size information for largest cold contexts, which can
53// be used to flag contexts for more aggressive cloning and reporting.
54cl::opt<unsigned> MinPercentMaxColdSize(
55 "memprof-min-percent-max-cold-size", cl::init(Val: 100), cl::Hidden,
56 cl::desc("Min percent of max cold bytes for critical cold context"));
57
58// Use this to keep the context size information in the memprof metadata for use
59// in remarks.
60cl::opt<bool> MemProfKeepContextSizeInfo(
61 "memprof-keep-context-size-info", cl::init(Val: false), cl::Hidden,
62 cl::desc("Keep context size information in memprof metadata"));
63
64LLVM_ABI cl::opt<bool> MemProfUseAmbiguousAttributes(
65 "memprof-ambiguous-attributes", cl::init(Val: true), cl::Hidden,
66 cl::desc("Apply ambiguous memprof attribute to ambiguous allocations"));
67
68} // end namespace llvm
69
70bool llvm::memprof::metadataIncludesAllContextSizeInfo() {
71 return MemProfReportHintedSizes || MemProfKeepContextSizeInfo ||
72 MinClonedColdBytePercent < 100;
73}
74
75bool llvm::memprof::metadataMayIncludeContextSizeInfo() {
76 return metadataIncludesAllContextSizeInfo() || MinPercentMaxColdSize < 100;
77}
78
79bool llvm::memprof::recordContextSizeInfoForAnalysis() {
80 return metadataMayIncludeContextSizeInfo() ||
81 MinCallsiteColdBytePercent < 100;
82}
83
84MDNode *llvm::memprof::buildCallstackMetadata(ArrayRef<uint64_t> CallStack,
85 LLVMContext &Ctx) {
86 SmallVector<Metadata *, 8> StackVals;
87 StackVals.reserve(N: CallStack.size());
88 for (auto Id : CallStack) {
89 auto *StackValMD =
90 ValueAsMetadata::get(V: ConstantInt::get(Ty: Type::getInt64Ty(C&: Ctx), V: Id));
91 StackVals.push_back(Elt: StackValMD);
92 }
93 return MDNode::get(Context&: Ctx, MDs: StackVals);
94}
95
96MDNode *llvm::memprof::getMIBStackNode(const MDNode *MIB) {
97 assert(MIB->getNumOperands() >= 2);
98 // The stack metadata is the first operand of each memprof MIB metadata.
99 return cast<MDNode>(Val: MIB->getOperand(I: 0));
100}
101
102AllocationType llvm::memprof::getMIBAllocType(const MDNode *MIB) {
103 assert(MIB->getNumOperands() >= 2);
104 // The allocation type is currently the second operand of each memprof
105 // MIB metadata. This will need to change as we add additional allocation
106 // types that can be applied based on the allocation profile data.
107 auto *MDS = dyn_cast<MDString>(Val: MIB->getOperand(I: 1));
108 assert(MDS);
109 if (MDS->getString() == "cold") {
110 return AllocationType::Cold;
111 } else if (MDS->getString() == "hot") {
112 return AllocationType::Hot;
113 }
114 return AllocationType::NotCold;
115}
116
117std::string llvm::memprof::getAllocTypeAttributeString(AllocationType Type) {
118 if (isAmbiguousAllocType(AllocTypes: (uint8_t)Type))
119 return "ambiguous";
120 switch (Type) {
121 case AllocationType::NotCold:
122 return "notcold";
123 break;
124 case AllocationType::Cold:
125 return "cold";
126 break;
127 case AllocationType::Hot:
128 return "hot";
129 break;
130 default:
131 assert(false && "Unexpected alloc type");
132 }
133 llvm_unreachable("invalid alloc type");
134}
135
136bool llvm::memprof::hasSingleAllocType(uint8_t AllocTypes) {
137 const unsigned NumAllocTypes = llvm::popcount(Value: AllocTypes);
138 assert(NumAllocTypes != 0);
139 return NumAllocTypes == 1;
140}
141
142bool llvm::memprof::isAmbiguousAllocType(uint8_t AllocTypes) {
143 const uint8_t BothTypes =
144 (uint8_t)AllocationType::NotCold | (uint8_t)AllocationType::Cold;
145 return (AllocTypes & BothTypes) == BothTypes;
146}
147
148void llvm::memprof::removeAnyExistingAmbiguousAttribute(CallBase *CB) {
149 if (!CB->hasFnAttr(Kind: "memprof"))
150 return;
151 assert(CB->getFnAttr("memprof").getValueAsString() == "ambiguous");
152 CB->removeFnAttr(Kind: "memprof");
153}
154
155void llvm::memprof::addAmbiguousAttribute(CallBase *CB) {
156 if (!MemProfUseAmbiguousAttributes)
157 return;
158 // We may have an existing ambiguous attribute if we are reanalyzing
159 // after inlining.
160 if (CB->hasFnAttr(Kind: "memprof")) {
161 assert(CB->getFnAttr("memprof").getValueAsString() == "ambiguous");
162 } else {
163 auto A = llvm::Attribute::get(Context&: CB->getContext(), Kind: "memprof", Val: "ambiguous");
164 CB->addFnAttr(Attr: A);
165 }
166}
167
168void CallStackTrie::addCallStack(
169 AllocationType AllocType, ArrayRef<uint64_t> StackIds,
170 std::vector<ContextTotalSize> ContextSizeInfo) {
171 bool First = true;
172 CallStackTrieNode *Curr = nullptr;
173 for (auto StackId : StackIds) {
174 // If this is the first stack frame, add or update alloc node.
175 if (First) {
176 First = false;
177 if (Alloc) {
178 assert(AllocStackId == StackId);
179 Alloc->addAllocType(AllocType);
180 } else {
181 AllocStackId = StackId;
182 Alloc = new CallStackTrieNode(AllocType);
183 }
184 Curr = Alloc;
185 continue;
186 }
187 // Update existing caller node if it exists.
188 auto [Next, Inserted] = Curr->Callers.try_emplace(k: StackId);
189 if (!Inserted) {
190 Curr = Next->second;
191 Curr->addAllocType(AllocType);
192 continue;
193 }
194 // Otherwise add a new caller node.
195 auto *New = new CallStackTrieNode(AllocType);
196 Next->second = New;
197 Curr = New;
198 }
199 assert(Curr);
200 // Append all of the ContextSizeInfo, along with their original AllocType.
201 llvm::append_range(C&: Curr->ContextInfo,
202 R: llvm::map_range(C&: ContextSizeInfo,
203 F: [AllocType](const ContextTotalSize &CTS) {
204 return ContextSizeTypePair(CTS,
205 AllocType);
206 }));
207}
208
209void CallStackTrie::addCallStack(MDNode *MIB) {
210 // Note that we are building this from existing MD_memprof metadata.
211 BuiltFromExistingMetadata = true;
212 MDNode *StackMD = getMIBStackNode(MIB);
213 assert(StackMD);
214 std::vector<uint64_t> CallStack;
215 CallStack.reserve(n: StackMD->getNumOperands());
216 for (const auto &MIBStackIter : StackMD->operands()) {
217 auto *StackId = mdconst::dyn_extract<ConstantInt>(MD: MIBStackIter);
218 assert(StackId);
219 CallStack.push_back(x: StackId->getZExtValue());
220 }
221 std::vector<ContextTotalSize> ContextSizeInfo;
222 // Collect the context size information if it exists.
223 if (MIB->getNumOperands() > 2) {
224 for (unsigned I = 2; I < MIB->getNumOperands(); I++) {
225 MDNode *ContextSizePair = dyn_cast<MDNode>(Val: MIB->getOperand(I));
226 assert(ContextSizePair->getNumOperands() == 2);
227 uint64_t FullStackId =
228 mdconst::dyn_extract<ConstantInt>(MD: ContextSizePair->getOperand(I: 0))
229 ->getZExtValue();
230 uint64_t TotalSize =
231 mdconst::dyn_extract<ConstantInt>(MD: ContextSizePair->getOperand(I: 1))
232 ->getZExtValue();
233 ContextSizeInfo.push_back(x: {.FullStackId: FullStackId, .TotalSize: TotalSize});
234 }
235 }
236 addCallStack(AllocType: getMIBAllocType(MIB), StackIds: CallStack, ContextSizeInfo: std::move(ContextSizeInfo));
237}
238
239static MDNode *createMIBNode(LLVMContext &Ctx, ArrayRef<uint64_t> MIBCallStack,
240 AllocationType AllocType,
241 ArrayRef<ContextSizeTypePair> ContextInfo,
242 const uint64_t MaxColdSize,
243 bool BuiltFromExistingMetadata,
244 uint64_t &TotalBytes, uint64_t &ColdBytes) {
245 SmallVector<Metadata *> MIBPayload(
246 {buildCallstackMetadata(CallStack: MIBCallStack, Ctx)});
247 MIBPayload.push_back(
248 Elt: MDString::get(Context&: Ctx, Str: getAllocTypeAttributeString(Type: AllocType)));
249
250 if (ContextInfo.empty()) {
251 // The profile matcher should have provided context size info if there was a
252 // MinCallsiteColdBytePercent < 100. Here we check >=100 to gracefully
253 // handle a user-provided percent larger than 100. However, we may not have
254 // this information if we built the Trie from existing MD_memprof metadata.
255 assert(BuiltFromExistingMetadata || MinCallsiteColdBytePercent >= 100);
256 return MDNode::get(Context&: Ctx, MDs: MIBPayload);
257 }
258
259 for (const auto &[CSI, AT] : ContextInfo) {
260 const auto &[FullStackId, TotalSize] = CSI;
261 TotalBytes += TotalSize;
262 bool LargeColdContext = false;
263 if (AllocType == AllocationType::Cold) {
264 ColdBytes += TotalSize;
265 // If we have the max cold context size from summary information and have
266 // requested identification of contexts above a percentage of the max, see
267 // if this context qualifies. We should assume this is large if we rebuilt
268 // the trie from existing metadata (i.e. to update after inlining), in
269 // which case we don't have a MaxSize from the profile - we assume any
270 // context size info in existence on the metadata should be propagated.
271 if (BuiltFromExistingMetadata ||
272 (MaxColdSize > 0 && MinPercentMaxColdSize < 100 &&
273 TotalSize * 100 >= MaxColdSize * MinPercentMaxColdSize))
274 LargeColdContext = true;
275 }
276 // Only add the context size info as metadata if we need it in the thin
277 // link (currently if reporting of hinted sizes is enabled, we have
278 // specified a threshold for marking allocations cold after cloning, or we
279 // have identified this as a large cold context of interest above).
280 if (metadataIncludesAllContextSizeInfo() || LargeColdContext) {
281 auto *FullStackIdMD = ValueAsMetadata::get(
282 V: ConstantInt::get(Ty: Type::getInt64Ty(C&: Ctx), V: FullStackId));
283 auto *TotalSizeMD = ValueAsMetadata::get(
284 V: ConstantInt::get(Ty: Type::getInt64Ty(C&: Ctx), V: TotalSize));
285 auto *ContextSizeMD = MDNode::get(Context&: Ctx, MDs: {FullStackIdMD, TotalSizeMD});
286 MIBPayload.push_back(Elt: ContextSizeMD);
287 }
288 }
289 assert(TotalBytes > 0);
290 return MDNode::get(Context&: Ctx, MDs: MIBPayload);
291}
292
293void CallStackTrie::collectContextInfo(
294 CallStackTrieNode *Node, std::vector<ContextSizeTypePair> &ContextInfo) {
295 llvm::append_range(C&: ContextInfo, R&: Node->ContextInfo);
296 for (auto &Caller : Node->Callers)
297 collectContextInfo(Node: Caller.second, ContextInfo);
298}
299
300void CallStackTrie::convertHotToNotCold(CallStackTrieNode *Node) {
301 if (Node->hasAllocType(AllocType: AllocationType::Hot)) {
302 Node->removeAllocType(AllocType: AllocationType::Hot);
303 Node->addAllocType(AllocType: AllocationType::NotCold);
304 }
305 for (auto &Caller : Node->Callers)
306 convertHotToNotCold(Node: Caller.second);
307}
308
309// Helper to emit messages for non-cold contexts that are ignored for various
310// reasons when reporting of hinted bytes is enabled.
311static void emitIgnoredNonColdContextMessage(StringRef Tag,
312 uint64_t FullStackId,
313 StringRef Extra,
314 uint64_t TotalSize) {
315 errs() << "MemProf hinting: Total size for " << Tag
316 << " non-cold full allocation context hash " << FullStackId << Extra
317 << ": " << TotalSize << "\n";
318}
319
320// Copy over some or all of NewMIBNodes to the SavedMIBNodes vector, depending
321// on options that enable filtering out some NotCold contexts.
322static void saveFilteredNewMIBNodes(std::vector<Metadata *> &NewMIBNodes,
323 std::vector<Metadata *> &SavedMIBNodes,
324 unsigned CallerContextLength,
325 uint64_t TotalBytes, uint64_t ColdBytes,
326 bool BuiltFromExistingMetadata) {
327 const bool MostlyCold =
328 // If we have built the Trie from existing MD_memprof metadata, we may or
329 // may not have context size information (in which case ColdBytes and
330 // TotalBytes are 0, which is not also guarded against below). Even if we
331 // do have some context size information from the the metadata, we have
332 // already gone through a round of discarding of small non-cold contexts
333 // during matching, and it would be overly aggressive to do it again, and
334 // we also want to maintain the same behavior with and without reporting
335 // of hinted bytes enabled.
336 !BuiltFromExistingMetadata && MinCallsiteColdBytePercent < 100 &&
337 ColdBytes > 0 &&
338 ColdBytes * 100 >= MinCallsiteColdBytePercent * TotalBytes;
339
340 // In the simplest case, with pruning disabled, keep all the new MIB nodes.
341 if (MemProfKeepAllNotColdContexts && !MostlyCold) {
342 append_range(C&: SavedMIBNodes, R&: NewMIBNodes);
343 return;
344 }
345
346 auto EmitMessageForRemovedContexts = [](const MDNode *MIBMD, StringRef Tag,
347 StringRef Extra) {
348 assert(MIBMD->getNumOperands() > 2);
349 for (unsigned I = 2; I < MIBMD->getNumOperands(); I++) {
350 MDNode *ContextSizePair = dyn_cast<MDNode>(Val: MIBMD->getOperand(I));
351 assert(ContextSizePair->getNumOperands() == 2);
352 uint64_t FullStackId =
353 mdconst::dyn_extract<ConstantInt>(MD: ContextSizePair->getOperand(I: 0))
354 ->getZExtValue();
355 uint64_t TS =
356 mdconst::dyn_extract<ConstantInt>(MD: ContextSizePair->getOperand(I: 1))
357 ->getZExtValue();
358 emitIgnoredNonColdContextMessage(Tag, FullStackId, Extra, TotalSize: TS);
359 }
360 };
361
362 // If the cold bytes at the current callsite exceed the given threshold, we
363 // discard all non-cold contexts so do not need any of the later pruning
364 // handling. We can simply copy over all the cold contexts and return early.
365 if (MostlyCold) {
366 auto NewColdMIBNodes =
367 make_filter_range(Range&: NewMIBNodes, Pred: [&](const Metadata *M) {
368 auto MIBMD = cast<MDNode>(Val: M);
369 // Only append cold contexts.
370 if (getMIBAllocType(MIB: MIBMD) == AllocationType::Cold)
371 return true;
372 if (MemProfReportHintedSizes) {
373 const float PercentCold = ColdBytes * 100.0 / TotalBytes;
374 std::string PercentStr;
375 llvm::raw_string_ostream OS(PercentStr);
376 OS << format(Fmt: " for %5.2f%% cold bytes", Vals: PercentCold);
377 EmitMessageForRemovedContexts(MIBMD, "discarded", OS.str());
378 }
379 return false;
380 });
381 for (auto *M : NewColdMIBNodes)
382 SavedMIBNodes.push_back(x: M);
383 return;
384 }
385
386 // Prune unneeded NotCold contexts, taking advantage of the fact
387 // that we later will only clone Cold contexts, as NotCold is the allocation
388 // default. We only need to keep as metadata the NotCold contexts that
389 // overlap the longest with Cold allocations, so that we know how deeply we
390 // need to clone. For example, assume we add the following contexts to the
391 // trie:
392 // 1 3 (notcold)
393 // 1 2 4 (cold)
394 // 1 2 5 (notcold)
395 // 1 2 6 (notcold)
396 // the trie looks like:
397 // 1
398 // / \
399 // 2 3
400 // /|\
401 // 4 5 6
402 //
403 // It is sufficient to prune all but one not-cold contexts (either 1,2,5 or
404 // 1,2,6, we arbitrarily keep the first one we encounter which will be
405 // 1,2,5).
406 //
407 // To do this pruning, we first check if there were any not-cold
408 // contexts kept for a deeper caller, which will have a context length larger
409 // than the CallerContextLength being handled here (i.e. kept by a deeper
410 // recursion step). If so, none of the not-cold MIB nodes added for the
411 // immediate callers need to be kept. If not, we keep the first (created
412 // for the immediate caller) not-cold MIB node.
413 bool LongerNotColdContextKept = false;
414 for (auto *MIB : NewMIBNodes) {
415 auto MIBMD = cast<MDNode>(Val: MIB);
416 if (getMIBAllocType(MIB: MIBMD) == AllocationType::Cold)
417 continue;
418 MDNode *StackMD = getMIBStackNode(MIB: MIBMD);
419 assert(StackMD);
420 if (StackMD->getNumOperands() > CallerContextLength) {
421 LongerNotColdContextKept = true;
422 break;
423 }
424 }
425 // Don't need to emit any for the immediate caller if we already have
426 // longer overlapping contexts;
427 bool KeepFirstNewNotCold = !LongerNotColdContextKept;
428 auto NewColdMIBNodes = make_filter_range(Range&: NewMIBNodes, Pred: [&](const Metadata *M) {
429 auto MIBMD = cast<MDNode>(Val: M);
430 // Only keep cold contexts and first (longest non-cold context).
431 if (getMIBAllocType(MIB: MIBMD) != AllocationType::Cold) {
432 MDNode *StackMD = getMIBStackNode(MIB: MIBMD);
433 assert(StackMD);
434 // Keep any already kept for longer contexts.
435 if (StackMD->getNumOperands() > CallerContextLength)
436 return true;
437 // Otherwise keep the first one added by the immediate caller if there
438 // were no longer contexts.
439 if (KeepFirstNewNotCold) {
440 KeepFirstNewNotCold = false;
441 return true;
442 }
443 if (MemProfReportHintedSizes)
444 EmitMessageForRemovedContexts(MIBMD, "pruned", "");
445 return false;
446 }
447 return true;
448 });
449 for (auto *M : NewColdMIBNodes)
450 SavedMIBNodes.push_back(x: M);
451}
452
453// Recursive helper to trim contexts and create metadata nodes.
454// Caller should have pushed Node's loc to MIBCallStack. Doing this in the
455// caller makes it simpler to handle the many early returns in this method.
456// Updates the total and cold profiled bytes in the subtrie rooted at this node.
457bool CallStackTrie::buildMIBNodes(CallStackTrieNode *Node, LLVMContext &Ctx,
458 std::vector<uint64_t> &MIBCallStack,
459 std::vector<Metadata *> &MIBNodes,
460 bool CalleeHasAmbiguousCallerContext,
461 uint64_t &TotalBytes, uint64_t &ColdBytes) {
462 // Trim context below the first node in a prefix with a single alloc type.
463 // Add an MIB record for the current call stack prefix.
464 if (hasSingleAllocType(AllocTypes: Node->AllocTypes)) {
465 std::vector<ContextSizeTypePair> ContextInfo;
466 collectContextInfo(Node, ContextInfo);
467 MIBNodes.push_back(x: createMIBNode(
468 Ctx, MIBCallStack, AllocType: (AllocationType)Node->AllocTypes, ContextInfo,
469 MaxColdSize, BuiltFromExistingMetadata, TotalBytes, ColdBytes));
470 return true;
471 }
472
473 // We don't have a single allocation for all the contexts sharing this prefix,
474 // so recursively descend into callers in trie.
475 if (!Node->Callers.empty()) {
476 bool NodeHasAmbiguousCallerContext = Node->Callers.size() > 1;
477 bool AddedMIBNodesForAllCallerContexts = true;
478 // Accumulate all new MIB nodes by the recursive calls below into a vector
479 // that will later be filtered before adding to the caller's MIBNodes
480 // vector.
481 std::vector<Metadata *> NewMIBNodes;
482 // Determine the total and cold byte counts for all callers, then add to the
483 // caller's counts further below.
484 uint64_t CallerTotalBytes = 0;
485 uint64_t CallerColdBytes = 0;
486 for (auto &Caller : Node->Callers) {
487 MIBCallStack.push_back(x: Caller.first);
488 AddedMIBNodesForAllCallerContexts &= buildMIBNodes(
489 Node: Caller.second, Ctx, MIBCallStack, MIBNodes&: NewMIBNodes,
490 CalleeHasAmbiguousCallerContext: NodeHasAmbiguousCallerContext, TotalBytes&: CallerTotalBytes, ColdBytes&: CallerColdBytes);
491 // Remove Caller.
492 MIBCallStack.pop_back();
493 }
494 // Pass in the stack length of the MIB nodes added for the immediate caller,
495 // which is the current stack length plus 1.
496 saveFilteredNewMIBNodes(NewMIBNodes, SavedMIBNodes&: MIBNodes, CallerContextLength: MIBCallStack.size() + 1,
497 TotalBytes: CallerTotalBytes, ColdBytes: CallerColdBytes,
498 BuiltFromExistingMetadata);
499 TotalBytes += CallerTotalBytes;
500 ColdBytes += CallerColdBytes;
501
502 if (AddedMIBNodesForAllCallerContexts)
503 return true;
504 // We expect that the callers should be forced to add MIBs to disambiguate
505 // the context in this case (see below).
506 assert(!NodeHasAmbiguousCallerContext);
507 }
508
509 // If we reached here, then this node does not have a single allocation type,
510 // and we didn't add metadata for a longer call stack prefix including any of
511 // Node's callers. That means we never hit a single allocation type along all
512 // call stacks with this prefix. This can happen due to recursion collapsing
513 // or the stack being deeper than tracked by the profiler runtime, leading to
514 // contexts with different allocation types being merged. In that case, we
515 // trim the context just below the deepest context split, which is this
516 // node if the callee has an ambiguous caller context (multiple callers),
517 // since the recursive calls above returned false. Conservatively give it
518 // non-cold allocation type.
519 if (!CalleeHasAmbiguousCallerContext)
520 return false;
521 std::vector<ContextSizeTypePair> ContextInfo;
522 collectContextInfo(Node, ContextInfo);
523 MIBNodes.push_back(x: createMIBNode(
524 Ctx, MIBCallStack, AllocType: AllocationType::NotCold, ContextInfo, MaxColdSize,
525 BuiltFromExistingMetadata, TotalBytes, ColdBytes));
526 return true;
527}
528
529void CallStackTrie::addSingleAllocTypeAttribute(CallBase *CI, AllocationType AT,
530 StringRef Descriptor) {
531 auto AllocTypeString = getAllocTypeAttributeString(Type: AT);
532 auto A = llvm::Attribute::get(Context&: CI->getContext(), Kind: "memprof", Val: AllocTypeString);
533 // After inlining we may be able to convert an existing ambiguous allocation
534 // to an unambiguous one.
535 removeAnyExistingAmbiguousAttribute(CB: CI);
536 CI->addFnAttr(Attr: A);
537
538 std::vector<ContextSizeTypePair> ContextInfo;
539 collectContextInfo(Node: Alloc, ContextInfo);
540
541 // If we don't have context size info, just emit a single remark for this
542 // allocation.
543 if (ContextInfo.empty()) {
544 if (ORE)
545 ORE->emit(OptDiag: OptimizationRemark(DEBUG_TYPE, "MemprofAttribute", CI)
546 << ore::NV("AllocationCall", CI) << " in function "
547 << ore::NV("Caller", CI->getFunction())
548 << " marked with memprof allocation attribute "
549 << ore::NV("Attribute", AllocTypeString));
550 return;
551 }
552
553 // Emit remarks or stderr reporting if requested.
554 for (const auto &[CSI, OrigAT] : ContextInfo) {
555 const auto &[FullStackId, TotalSize] = CSI;
556 // If the original alloc type is not the one being applied as the hint,
557 // then don't report that it was hinted. Optionally report that we ignored
558 // this context.
559 if (AT != OrigAT) {
560 if (MemProfReportHintedSizes)
561 emitIgnoredNonColdContextMessage(Tag: "ignored", FullStackId, Extra: "", TotalSize);
562 continue;
563 }
564 if (MemProfReportHintedSizes)
565 errs() << "MemProf hinting: Total size for full allocation context hash "
566 << FullStackId << " and " << Descriptor << " alloc type "
567 << getAllocTypeAttributeString(Type: AT) << ": " << TotalSize << "\n";
568 if (ORE)
569 ORE->emit(OptDiag: OptimizationRemark(DEBUG_TYPE, "MemprofAttribute", CI)
570 << ore::NV("AllocationCall", CI) << " in function "
571 << ore::NV("Caller", CI->getFunction())
572 << " marked with memprof allocation attribute "
573 << ore::NV("Attribute", AllocTypeString)
574 << " for full allocation context hash "
575 << ore::NV("FullStackId", FullStackId) << " with total size "
576 << ore::NV("TotalSize", TotalSize));
577 }
578}
579
580// Build and attach the minimal necessary MIB metadata. If the alloc has a
581// single allocation type, add a function attribute instead. Returns true if
582// memprof metadata attached, false if not (attribute added).
583bool CallStackTrie::buildAndAttachMIBMetadata(CallBase *CI) {
584 if (hasSingleAllocType(AllocTypes: Alloc->AllocTypes)) {
585 addSingleAllocTypeAttribute(CI, AT: (AllocationType)Alloc->AllocTypes,
586 Descriptor: "single");
587 return false;
588 }
589 // If there were any hot allocation contexts, the Alloc trie node would have
590 // the Hot type set. If so, because we don't currently support cloning for hot
591 // contexts, they should be converted to NotCold. This happens in the cloning
592 // support anyway, however, doing this now enables more aggressive context
593 // trimming when building the MIB metadata (and possibly may make the
594 // allocation have a single NotCold allocation type), greatly reducing
595 // overheads in bitcode, cloning memory and cloning time.
596 if (Alloc->hasAllocType(AllocType: AllocationType::Hot)) {
597 convertHotToNotCold(Node: Alloc);
598 // Check whether we now have a single alloc type.
599 if (hasSingleAllocType(AllocTypes: Alloc->AllocTypes)) {
600 addSingleAllocTypeAttribute(CI, AT: (AllocationType)Alloc->AllocTypes,
601 Descriptor: "single");
602 return false;
603 }
604 }
605 auto &Ctx = CI->getContext();
606 std::vector<uint64_t> MIBCallStack;
607 MIBCallStack.push_back(x: AllocStackId);
608 std::vector<Metadata *> MIBNodes;
609 uint64_t TotalBytes = 0;
610 uint64_t ColdBytes = 0;
611 assert(!Alloc->Callers.empty() && "addCallStack has not been called yet");
612 // The CalleeHasAmbiguousCallerContext flag is meant to say whether the
613 // callee of the given node has more than one caller. Here the node being
614 // passed in is the alloc and it has no callees. So it's false.
615 if (buildMIBNodes(Node: Alloc, Ctx, MIBCallStack, MIBNodes,
616 /*CalleeHasAmbiguousCallerContext=*/false, TotalBytes,
617 ColdBytes)) {
618 assert(MIBCallStack.size() == 1 &&
619 "Should only be left with Alloc's location in stack");
620 CI->setMetadata(KindID: LLVMContext::MD_memprof, Node: MDNode::get(Context&: Ctx, MDs: MIBNodes));
621 addAmbiguousAttribute(CB: CI);
622 return true;
623 }
624 // If there exists corner case that CallStackTrie has one chain to leaf
625 // and all node in the chain have multi alloc type, conservatively give
626 // it non-cold allocation type.
627 // FIXME: Avoid this case before memory profile created. Alternatively, select
628 // hint based on fraction cold.
629 addSingleAllocTypeAttribute(CI, AT: AllocationType::NotCold, Descriptor: "indistinguishable");
630 return false;
631}
632
633template <>
634CallStack<MDNode, MDNode::op_iterator>::CallStackIterator::CallStackIterator(
635 const MDNode *N, bool End)
636 : N(N) {
637 if (!N)
638 return;
639 Iter = End ? N->op_end() : N->op_begin();
640}
641
642template <>
643uint64_t
644CallStack<MDNode, MDNode::op_iterator>::CallStackIterator::operator*() {
645 assert(Iter != N->op_end());
646 ConstantInt *StackIdCInt = mdconst::dyn_extract<ConstantInt>(MD: *Iter);
647 assert(StackIdCInt);
648 return StackIdCInt->getZExtValue();
649}
650
651template <> uint64_t CallStack<MDNode, MDNode::op_iterator>::back() const {
652 assert(N);
653 return mdconst::dyn_extract<ConstantInt>(MD: N->operands().back())
654 ->getZExtValue();
655}
656
657MDNode *MDNode::getMergedMemProfMetadata(MDNode *A, MDNode *B) {
658 // TODO: Support more sophisticated merging, such as selecting the one with
659 // more bytes allocated, or implement support for carrying multiple allocation
660 // leaf contexts. For now, keep the first one.
661 if (A)
662 return A;
663 return B;
664}
665
666MDNode *MDNode::getMergedCallsiteMetadata(MDNode *A, MDNode *B) {
667 // TODO: Support more sophisticated merging, which will require support for
668 // carrying multiple contexts. For now, keep the first one.
669 if (A)
670 return A;
671 return B;
672}
673