1//===- MLInlineAdvisor.cpp - machine learned InlineAdvisor ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the interface between the inliner and a learned model.
10// It delegates model evaluation to either the AOT compiled model (the
11// 'release' mode) or a runtime-loaded model (the 'development' case).
12//
13//===----------------------------------------------------------------------===//
14#include "llvm/Analysis/MLInlineAdvisor.h"
15#include "AnalysisOptions.h"
16#include "llvm/ADT/SCCIterator.h"
17#include "llvm/Analysis/AssumptionCache.h"
18#include "llvm/Analysis/BlockFrequencyInfo.h"
19#include "llvm/Analysis/CallGraph.h"
20#include "llvm/Analysis/FunctionPropertiesAnalysis.h"
21#include "llvm/Analysis/InlineCost.h"
22#include "llvm/Analysis/InlineModelFeatureMaps.h"
23#include "llvm/Analysis/LazyCallGraph.h"
24#include "llvm/Analysis/LoopInfo.h"
25#include "llvm/Analysis/MLModelRunner.h"
26#include "llvm/Analysis/OptimizationRemarkEmitter.h"
27#include "llvm/Analysis/ProfileSummaryInfo.h"
28#include "llvm/Analysis/ReleaseModeModelRunner.h"
29#include "llvm/Analysis/TargetTransformInfo.h"
30#include "llvm/Analysis/TensorSpec.h"
31#include "llvm/Analysis/Utils/MLGOUtils.h"
32#include "llvm/IR/Dominators.h"
33#include "llvm/IR/InstIterator.h"
34#include "llvm/IR/Module.h"
35#include "llvm/IR/PassManager.h"
36#include "llvm/Support/CommandLine.h"
37
38using namespace llvm;
39
40#if defined(LLVM_HAVE_TF_AOT_INLINERSIZEMODEL)
41// codegen-ed file
42#include "InlinerSizeModel.h" // NOLINT
43using CompiledModelType = llvm::InlinerSizeModel;
44#else
45using CompiledModelType = NoopSavedModelImpl;
46#endif
47
48#include "llvm/Analysis/EmitCModelRunner.h"
49#include "llvm/Analysis/InlinerModels.h"
50
51enum class EmitCModelChoice {
52 Default,
53#define MLGO_MODEL(CLASS_NAME, CLI_FLAG) CLASS_NAME,
54#include "llvm/Analysis/InlinerModels.def"
55};
56
57static llvm::cl::opt<EmitCModelChoice> SelectedMLGOModel(
58 "mlgo-model", llvm::cl::desc("Select the MLGO model to execute:"),
59 llvm::cl::init(Val: EmitCModelChoice::Default),
60 llvm::cl::values(clEnumValN(EmitCModelChoice::Default, "default",
61 "Use standard heuristic")
62#define MLGO_MODEL(CLASS_NAME, CLI_FLAG) \
63 , clEnumValN(EmitCModelChoice::CLASS_NAME, CLI_FLAG, \
64 "Use the " CLI_FLAG " MLGO model")
65#include "llvm/Analysis/InlinerModels.def"
66 ));
67
68static std::unique_ptr<MLModelRunner>
69createEmitCModelRunner(LLVMContext &Ctx,
70 const std::vector<TensorSpec> &InputFeatures) {
71 switch (SelectedMLGOModel) {
72 case EmitCModelChoice::Default:
73 return nullptr;
74#define MLGO_MODEL(CLASS_NAME, CLI_FLAG) \
75 case EmitCModelChoice::CLASS_NAME: \
76 return std::make_unique<EmitCModelRunner<CLASS_NAME>>(Ctx, InputFeatures);
77#include "llvm/Analysis/InlinerModels.def"
78 }
79 llvm_unreachable("Unknown MLGO model type!");
80}
81
82std::unique_ptr<InlineAdvisor>
83llvm::getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM,
84 std::function<bool(CallBase &)> GetDefaultAdvice) {
85 const AnalysisOptions &Opts = getAnalysisOptions(M);
86 if (!isReleaseModelValid<CompiledModelType>(
87 InteractiveChannelBaseName: Opts.inliner_interactive_channel_base, SelectedModel: SelectedMLGOModel))
88 return nullptr;
89 auto RunnerFactory = [&](const std::vector<TensorSpec> &InputFeatures)
90 -> std::unique_ptr<MLModelRunner> {
91 return createReleaseModeModelRunner<CompiledModelType>(
92 Ctx&: M.getContext(), InputFeatures, DecisionName,
93 InteractiveChannelBaseName: Opts.inliner_interactive_channel_base.str(), InteractiveDecisionSpec: InlineDecisionSpec,
94 CreateEmitCModelRunner&: createEmitCModelRunner,
95 Options: EmbeddedModelRunnerOptions().setModelSelector(
96 Opts.ml_inliner_model_selector));
97 };
98 return std::make_unique<MLInlineAdvisor>(args&: M, args&: MAM, args&: RunnerFactory,
99 args&: GetDefaultAdvice);
100}
101
102#define DEBUG_TYPE "inline-ml"
103
104const std::vector<TensorSpec> &MLInlineAdvisor::getInitialFeatureMap() {
105 // clang-format off
106static std::vector<TensorSpec> FeatureMap{
107#define POPULATE_NAMES(DTYPE, SHAPE, NAME, __) TensorSpec::createSpec<DTYPE>(#NAME, SHAPE),
108// InlineCost features - these must come first
109 INLINE_COST_FEATURE_ITERATOR(POPULATE_NAMES)
110
111// Non-cost features
112 INLINE_FEATURE_ITERATOR(POPULATE_NAMES)
113#undef POPULATE_NAMES
114};
115 // clang-format on
116 return FeatureMap;
117}
118
119const char *const llvm::DecisionName = "inlining_decision";
120const TensorSpec llvm::InlineDecisionSpec =
121 TensorSpec::createSpec<int64_t>(Name: DecisionName, Shape: {1});
122const char *const llvm::DefaultDecisionName = "inlining_default";
123const TensorSpec llvm::DefaultDecisionSpec =
124 TensorSpec::createSpec<int64_t>(Name: DefaultDecisionName, Shape: {1});
125const char *const llvm::RewardName = "delta_size";
126
127CallBase *getInlinableCS(Instruction &I) {
128 if (auto *CS = dyn_cast<CallBase>(Val: &I))
129 if (Function *Callee = CS->getCalledFunction()) {
130 if (!Callee->isDeclaration()) {
131 return CS;
132 }
133 }
134 return nullptr;
135}
136
137MLInlineAdvisor::MLInlineAdvisor(
138 Module &M, ModuleAnalysisManager &MAM,
139 std::function<
140 std::unique_ptr<MLModelRunner>(const std::vector<TensorSpec> &)>
141 GetModelRunner,
142 std::function<bool(CallBase &)> GetDefaultAdvice)
143 : InlineAdvisor(
144 M, MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager()),
145 Opts(getAnalysisOptions(M)), GetDefaultAdvice(GetDefaultAdvice),
146 FeatureMap(getInitialFeatureMap()),
147 CG(MAM.getResult<LazyCallGraphAnalysis>(IR&: M)),
148 UseIR2Vec(MAM.getCachedResult<IR2VecVocabAnalysis>(IR&: M) != nullptr),
149 InitialIRSize(getModuleIRSize()), CurrentIRSize(InitialIRSize),
150 PSI(MAM.getResult<ProfileSummaryAnalysis>(IR&: M)) {
151 // Extract the 'call site height' feature - the position of a call site
152 // relative to the farthest statically reachable SCC node. We don't mutate
153 // this value while inlining happens. Empirically, this feature proved
154 // critical in behavioral cloning - i.e. training a model to mimic the manual
155 // heuristic's decisions - and, thus, equally important for training for
156 // improvement.
157 CallGraph CGraph(M);
158 for (auto I = scc_begin(G: &CGraph); !I.isAtEnd(); ++I) {
159 const std::vector<CallGraphNode *> &CGNodes = *I;
160 unsigned Level = 0;
161 for (auto *CGNode : CGNodes) {
162 Function *F = CGNode->getFunction();
163 if (!F || F->isDeclaration())
164 continue;
165 for (auto &I : instructions(F)) {
166 if (auto *CS = getInlinableCS(I)) {
167 auto *Called = CS->getCalledFunction();
168 auto Pos = FunctionLevels.find(x: &CG.get(F&: *Called));
169 // In bottom up traversal, an inlinable callee is either in the
170 // same SCC, or to a function in a visited SCC. So not finding its
171 // level means we haven't visited it yet, meaning it's in this SCC.
172 if (Pos == FunctionLevels.end())
173 continue;
174 Level = std::max(a: Level, b: Pos->second + 1);
175 }
176 }
177 }
178 for (auto *CGNode : CGNodes) {
179 Function *F = CGNode->getFunction();
180 if (F && !F->isDeclaration())
181 FunctionLevels[&CG.get(F&: *F)] = Level;
182 }
183 }
184 for (auto KVP : FunctionLevels) {
185 AllNodes.insert(V: KVP.first);
186 EdgeCount += getLocalCalls(F&: KVP.first->getFunction());
187 }
188 NodeCount = AllNodes.size();
189
190 if (auto *IR2VecVocabResult = MAM.getCachedResult<IR2VecVocabAnalysis>(IR&: M)) {
191 if (!IR2VecVocabResult->isValid()) {
192 M.getContext().emitError(ErrorStr: "IR2VecVocabAnalysis is not valid");
193 return;
194 }
195 // Add the IR2Vec features to the feature map
196 auto IR2VecDim = IR2VecVocabResult->getDimension();
197 FeatureMap.push_back(
198 x: TensorSpec::createSpec<float>(Name: "callee_embedding", Shape: {IR2VecDim}));
199 FeatureMap.push_back(
200 x: TensorSpec::createSpec<float>(Name: "caller_embedding", Shape: {IR2VecDim}));
201 }
202 if (Opts.inliner_interactive_include_default)
203 FeatureMap.push_back(x: DefaultDecisionSpec);
204
205 ModelRunner = GetModelRunner(getFeatureMap());
206 if (!ModelRunner) {
207 M.getContext().emitError(ErrorStr: "Could not create model runner");
208 return;
209 }
210 ModelRunner->switchContext(Name: "");
211 ForceStop = Opts.ml_inliner_stop_immediately;
212}
213
214unsigned MLInlineAdvisor::getInitialFunctionLevel(const Function &F) const {
215 return CG.lookup(F) ? FunctionLevels.at(k: CG.lookup(F)) : 0;
216}
217
218void MLInlineAdvisor::onPassEntry(LazyCallGraph::SCC *CurSCC) {
219 if (!CurSCC || ForceStop)
220 return;
221 FPICache.clear();
222 // Function passes executed between InlinerPass runs may have changed the
223 // module-wide features.
224 // The cgscc pass manager rules are such that:
225 // - if a pass leads to merging SCCs, then the pipeline is restarted on the
226 // merged SCC
227 // - if a pass leads to splitting the SCC, then we continue with one of the
228 // splits
229 // This means that the NodesInLastSCC is a superset (not strict) of the nodes
230 // that subsequent passes would have processed
231 // - in addition, if new Nodes were created by a pass (e.g. CoroSplit),
232 // they'd be adjacent to Nodes in the last SCC. So we just need to check the
233 // boundary of Nodes in NodesInLastSCC for Nodes we haven't seen. We don't
234 // care about the nature of the Edge (call or ref). `FunctionLevels`-wise, we
235 // record them at the same level as the original node (this is a choice, may
236 // need revisiting).
237 // - nodes are only deleted at the end of a call graph walk where they are
238 // batch deleted, so we shouldn't see any dead nodes here.
239 while (!NodesInLastSCC.empty()) {
240 const auto *N = *NodesInLastSCC.begin();
241 assert(!N->isDead());
242 NodesInLastSCC.erase(Ptr: N);
243 EdgeCount += getLocalCalls(F&: N->getFunction());
244 const auto NLevel = FunctionLevels.at(k: N);
245 for (const auto &E : *(*N)) {
246 const auto *AdjNode = &E.getNode();
247 assert(!AdjNode->isDead() && !AdjNode->getFunction().isDeclaration());
248 auto I = AllNodes.insert(V: AdjNode);
249 // We've discovered a new function.
250 if (I.second) {
251 ++NodeCount;
252 NodesInLastSCC.insert(Ptr: AdjNode);
253 FunctionLevels[AdjNode] = NLevel;
254 }
255 }
256 }
257
258 EdgeCount -= EdgesOfLastSeenNodes;
259 EdgesOfLastSeenNodes = 0;
260
261 // (Re)use NodesInLastSCC to remember the nodes in the SCC right now,
262 // in case the SCC is split before onPassExit and some nodes are split out
263 assert(NodesInLastSCC.empty());
264 for (const auto &N : *CurSCC)
265 NodesInLastSCC.insert(Ptr: &N);
266}
267
268void MLInlineAdvisor::onPassExit(LazyCallGraph::SCC *CurSCC) {
269 // No need to keep this around - function passes will invalidate it.
270 if (!Opts.ml_advisor_keep_fpi_cache)
271 FPICache.clear();
272 if (!CurSCC || ForceStop)
273 return;
274 // Keep track of the nodes and edges we last saw. Then, in onPassEntry,
275 // we update the node count and edge count from the subset of these nodes that
276 // survived.
277 EdgesOfLastSeenNodes = 0;
278
279 // Check on nodes that were in SCC onPassEntry
280 for (const LazyCallGraph::Node *N : NodesInLastSCC) {
281 assert(!N->isDead());
282 EdgesOfLastSeenNodes += getLocalCalls(F&: N->getFunction());
283 }
284
285 // Check on nodes that may have got added to SCC
286 for (const auto &N : *CurSCC) {
287 assert(!N.isDead());
288 auto I = NodesInLastSCC.insert(Ptr: &N);
289 if (I.second)
290 EdgesOfLastSeenNodes += getLocalCalls(F&: N.getFunction());
291 }
292 assert(NodeCount >= NodesInLastSCC.size());
293 assert(EdgeCount >= EdgesOfLastSeenNodes);
294}
295
296int64_t MLInlineAdvisor::getLocalCalls(Function &F) {
297 return getCachedFPI(F).DirectCallsToDefinedFunctions;
298}
299
300// Update the internal state of the advisor, and force invalidate feature
301// analysis. Currently, we maintain minimal (and very simple) global state - the
302// number of functions and the number of static calls. We also keep track of the
303// total IR size in this module, to stop misbehaving policies at a certain bloat
304// factor (-ml-advisor-size-increase-threshold)
305void MLInlineAdvisor::onSuccessfulInlining(const MLInlineAdvice &Advice,
306 bool CalleeWasDeleted) {
307 assert(!ForceStop);
308 Function *Caller = Advice.getCaller();
309 Function *Callee = Advice.getCallee();
310 // The caller features aren't valid anymore.
311 {
312 PreservedAnalyses PA = PreservedAnalyses::all();
313 PA.abandon<FunctionPropertiesAnalysis>();
314 PA.abandon<LoopAnalysis>();
315 FAM.invalidate(IR&: *Caller, PA);
316 }
317 Advice.updateCachedCallerFPI(FAM);
318 if (Caller == Callee) {
319 assert(!CalleeWasDeleted);
320 // We double-counted CallerAndCalleeEdges - since the caller and callee
321 // would be the same
322 assert(Advice.CallerAndCalleeEdges % 2 == 0);
323 CurrentIRSize += getIRSize(F&: *Caller) - Advice.CallerIRSize;
324 EdgeCount += getCachedFPI(*Caller).DirectCallsToDefinedFunctions -
325 Advice.CallerAndCalleeEdges / 2;
326 // The NodeCount would stay the same.
327 } else {
328 int64_t IRSizeAfter =
329 getIRSize(F&: *Caller) + (CalleeWasDeleted ? 0 : Advice.CalleeIRSize);
330 CurrentIRSize += IRSizeAfter - (Advice.CallerIRSize + Advice.CalleeIRSize);
331
332 // We can delta-update module-wide features. We know the inlining only
333 // changed the caller, and maybe the callee (by deleting the latter). Nodes
334 // are simple to update. For edges, we 'forget' the edges that the caller
335 // and callee used to have before inlining, and add back what they currently
336 // have together.
337 int64_t NewCallerAndCalleeEdges =
338 getCachedFPI(*Caller).DirectCallsToDefinedFunctions;
339
340 // A dead function's node is not actually removed from the call graph until
341 // the end of the call graph walk, but the node no longer belongs to any
342 // valid SCC.
343 if (CalleeWasDeleted) {
344 --NodeCount;
345 NodesInLastSCC.erase(Ptr: CG.lookup(F: *Callee));
346 DeadFunctions.insert(V: Callee);
347 } else {
348 NewCallerAndCalleeEdges +=
349 getCachedFPI(*Callee).DirectCallsToDefinedFunctions;
350 }
351 EdgeCount += (NewCallerAndCalleeEdges - Advice.CallerAndCalleeEdges);
352 }
353 if (CurrentIRSize > Opts.ml_advisor_size_increase_threshold * InitialIRSize)
354 ForceStop = true;
355
356 assert(CurrentIRSize >= 0 && EdgeCount >= 0 && NodeCount >= 0);
357}
358
359int64_t MLInlineAdvisor::getModuleIRSize() const {
360 int64_t Ret = 0;
361 for (auto &F : M)
362 if (!F.isDeclaration())
363 Ret += getIRSize(F);
364 return Ret;
365}
366
367FunctionPropertiesInfo &MLInlineAdvisor::getCachedFPI(Function &F) const {
368 auto InsertPair = FPICache.try_emplace(k: &F);
369 if (!InsertPair.second)
370 return InsertPair.first->second;
371 InsertPair.first->second = FAM.getResult<FunctionPropertiesAnalysis>(IR&: F);
372 return InsertPair.first->second;
373}
374
375std::unique_ptr<InlineAdvice> MLInlineAdvisor::getAdviceImpl(CallBase &CB) {
376 if (auto Skip = getSkipAdviceIfUnreachableCallsite(CB))
377 return Skip;
378
379 auto &Caller = *CB.getCaller();
380 auto &Callee = *CB.getCalledFunction();
381
382 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
383 return FAM.getResult<AssumptionAnalysis>(IR&: F);
384 };
385 auto &TIR = FAM.getResult<TargetIRAnalysis>(IR&: Caller);
386 auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: Caller);
387
388 if (Opts.ml_inliner_skip_policy == SkipMLPolicyCriteria::IfCallerIsNotCold) {
389 if (!PSI.isFunctionEntryCold(F: &Caller)) {
390 // Return a MLInlineAdvice, despite delegating to the default advice,
391 // because we need to keep track of the internal state. This is different
392 // from the other instances where we return a "default" InlineAdvice,
393 // which happen at points we won't come back to the MLAdvisor for
394 // decisions requiring that state.
395 return ForceStop ? std::make_unique<InlineAdvice>(args: this, args&: CB, args&: ORE,
396 args: GetDefaultAdvice(CB))
397 : std::make_unique<MLInlineAdvice>(args: this, args&: CB, args&: ORE,
398 args: GetDefaultAdvice(CB));
399 }
400 }
401 auto MandatoryKind = InlineAdvisor::getMandatoryKind(CB, FAM, ORE);
402 // If this is a "never inline" case, there won't be any changes to internal
403 // state we need to track, so we can just return the base InlineAdvice, which
404 // will do nothing interesting.
405 // Same thing if this is a recursive case.
406 if (MandatoryKind == InlineAdvisor::MandatoryInliningKind::Never ||
407 &Caller == &Callee)
408 return getMandatoryAdvice(CB, Advice: false);
409
410 bool Mandatory =
411 MandatoryKind == InlineAdvisor::MandatoryInliningKind::Always;
412
413 // If we need to stop, we won't want to track anymore any state changes, so
414 // we just return the base InlineAdvice, which acts as a noop.
415 if (ForceStop) {
416 ORE.emit(RemarkBuilder: [&] {
417 return OptimizationRemarkMissed(DEBUG_TYPE, "ForceStop", &CB)
418 << "Won't attempt inlining because module size grew too much.";
419 });
420 return std::make_unique<InlineAdvice>(args: this, args&: CB, args&: ORE, args&: Mandatory);
421 }
422
423 int CostEstimate = 0;
424 if (!Mandatory) {
425 auto IsCallSiteInlinable =
426 llvm::getInliningCostEstimate(Call&: CB, CallerTTI&: TIR, GetAssumptionCache);
427 if (!IsCallSiteInlinable) {
428 // We can't inline this for correctness reasons, so return the base
429 // InlineAdvice, as we don't care about tracking any state changes (which
430 // won't happen).
431 return std::make_unique<InlineAdvice>(args: this, args&: CB, args&: ORE, args: false);
432 }
433 CostEstimate = *IsCallSiteInlinable;
434 }
435
436 const auto CostFeatures =
437 llvm::getInliningCostFeatures(Call&: CB, CallerTTI&: TIR, GetAssumptionCache);
438 if (!CostFeatures) {
439 return std::make_unique<InlineAdvice>(args: this, args&: CB, args&: ORE, args: false);
440 }
441
442 if (Mandatory)
443 return getMandatoryAdvice(CB, Advice: true);
444
445 auto NumCtantParams = 0;
446 for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
447 NumCtantParams += (isa<Constant>(Val: *I));
448 }
449
450 auto &CallerBefore = getCachedFPI(F&: Caller);
451 auto &CalleeBefore = getCachedFPI(F&: Callee);
452
453 *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::callee_basic_block_count) =
454 CalleeBefore.BasicBlockCount;
455 *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::callsite_height) =
456 getInitialFunctionLevel(F: Caller);
457 *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::node_count) = NodeCount;
458 *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::nr_ctant_params) =
459 NumCtantParams;
460 *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::edge_count) = EdgeCount;
461 *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::caller_users) =
462 CallerBefore.Uses;
463 *ModelRunner->getTensor<int64_t>(
464 FeatureID: FeatureIndex::caller_conditionally_executed_blocks) =
465 CallerBefore.BlocksReachedFromConditionalInstruction;
466 *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::caller_basic_block_count) =
467 CallerBefore.BasicBlockCount;
468 *ModelRunner->getTensor<int64_t>(
469 FeatureID: FeatureIndex::callee_conditionally_executed_blocks) =
470 CalleeBefore.BlocksReachedFromConditionalInstruction;
471 *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::callee_users) =
472 CalleeBefore.Uses;
473 *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::cost_estimate) = CostEstimate;
474 *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::is_callee_avail_external) =
475 Callee.hasAvailableExternallyLinkage();
476 *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::is_caller_avail_external) =
477 Caller.hasAvailableExternallyLinkage();
478
479 if (UseIR2Vec) {
480 // Python side expects float embeddings. The IR2Vec embeddings are doubles
481 // as of now due to the restriction of fromJSON method used by the
482 // readVocabulary method in ir2vec::Embeddings.
483 auto setEmbedding = [&](const ir2vec::Embedding &Embedding,
484 FeatureIndex Index) {
485 llvm::transform(Range: Embedding, d_first: ModelRunner->getTensor<float>(FeatureID: Index),
486 F: [](double Val) { return static_cast<float>(Val); });
487 };
488
489 setEmbedding(CalleeBefore.getFunctionEmbedding(),
490 FeatureIndex::callee_embedding);
491 setEmbedding(CallerBefore.getFunctionEmbedding(),
492 FeatureIndex::caller_embedding);
493 }
494
495 // Add the cost features
496 for (size_t I = 0;
497 I < static_cast<size_t>(InlineCostFeatureIndex::NumberOfFeatures); ++I) {
498 *ModelRunner->getTensor<int64_t>(FeatureID: inlineCostFeatureToMlFeature(
499 Feature: static_cast<InlineCostFeatureIndex>(I))) = CostFeatures->at(n: I);
500 }
501 // This one would have been set up to be right at the end.
502 if (!Opts.inliner_interactive_channel_base.empty() &&
503 Opts.inliner_interactive_include_default)
504 *ModelRunner->getTensor<int64_t>(FeatureID: getFeatureMap().size() - 1) =
505 GetDefaultAdvice(CB);
506 return getAdviceFromModel(CB, ORE);
507}
508
509std::unique_ptr<MLInlineAdvice>
510MLInlineAdvisor::getAdviceFromModel(CallBase &CB,
511 OptimizationRemarkEmitter &ORE) {
512 return std::make_unique<MLInlineAdvice>(
513 args: this, args&: CB, args&: ORE, args: static_cast<bool>(ModelRunner->evaluate<int64_t>()));
514}
515
516std::unique_ptr<InlineAdvice>
517MLInlineAdvisor::getSkipAdviceIfUnreachableCallsite(CallBase &CB) {
518 if (!FAM.getResult<DominatorTreeAnalysis>(IR&: *CB.getCaller())
519 .isReachableFromEntry(A: CB.getParent()))
520 return std::make_unique<InlineAdvice>(args: this, args&: CB, args&: getCallerORE(CB), args: false);
521 return nullptr;
522}
523
524std::unique_ptr<InlineAdvice> MLInlineAdvisor::getMandatoryAdvice(CallBase &CB,
525 bool Advice) {
526 // Make sure we track inlinings in all cases - mandatory or not.
527 if (auto Skip = getSkipAdviceIfUnreachableCallsite(CB))
528 return Skip;
529 if (Advice && !ForceStop)
530 return getMandatoryAdviceImpl(CB);
531
532 // If this is a "never inline" case, there won't be any changes to internal
533 // state we need to track, so we can just return the base InlineAdvice, which
534 // will do nothing interesting.
535 // Same if we are forced to stop - we don't track anymore.
536 return std::make_unique<InlineAdvice>(args: this, args&: CB, args&: getCallerORE(CB), args&: Advice);
537}
538
539std::unique_ptr<MLInlineAdvice>
540MLInlineAdvisor::getMandatoryAdviceImpl(CallBase &CB) {
541 return std::make_unique<MLInlineAdvice>(args: this, args&: CB, args&: getCallerORE(CB), args: true);
542}
543
544void MLInlineAdvisor::print(raw_ostream &OS) const {
545 OS << "[MLInlineAdvisor] Nodes: " << NodeCount << " Edges: " << EdgeCount
546 << " EdgesOfLastSeenNodes: " << EdgesOfLastSeenNodes << "\n";
547 OS << "[MLInlineAdvisor] FPI:\n";
548 for (auto I : FPICache) {
549 OS << I.first->getName() << ":\n";
550 I.second.print(OS);
551 OS << "\n";
552 }
553 OS << "\n";
554 OS << "[MLInlineAdvisor] FuncLevels:\n";
555 for (auto I : FunctionLevels)
556 OS << (DeadFunctions.contains(V: &I.first->getFunction())
557 ? "<deleted>"
558 : I.first->getFunction().getName())
559 << " : " << I.second << "\n";
560
561 OS << "\n";
562}
563
564MLInlineAdvice::MLInlineAdvice(MLInlineAdvisor *Advisor, CallBase &CB,
565 OptimizationRemarkEmitter &ORE,
566 bool Recommendation)
567 : InlineAdvice(Advisor, CB, ORE, Recommendation),
568 CallerIRSize(Advisor->isForcedToStop() ? 0 : Advisor->getIRSize(F&: *Caller)),
569 CalleeIRSize(Advisor->isForcedToStop() ? 0 : Advisor->getIRSize(F&: *Callee)),
570 CallerAndCalleeEdges(Advisor->isForcedToStop()
571 ? 0
572 : (Advisor->getLocalCalls(F&: *Caller) +
573 Advisor->getLocalCalls(F&: *Callee))),
574 PreInlineCallerFPI(Advisor->getCachedFPI(F&: *Caller)) {
575 if (Recommendation)
576 FPU.emplace(args&: Advisor->getCachedFPI(F&: *getCaller()), args&: CB);
577}
578
579void MLInlineAdvice::reportContextForRemark(
580 DiagnosticInfoOptimizationBase &OR) {
581 using namespace ore;
582 OR << NV("Callee", Callee->getName());
583 for (size_t I = 0; I < getAdvisor()->getFeatureMap().size(); ++I)
584 OR << NV(getAdvisor()->getFeatureMap()[I].name(),
585 *getAdvisor()->getModelRunner().getTensor<int64_t>(FeatureID: I));
586 OR << NV("ShouldInline", isInliningRecommended());
587}
588
589void MLInlineAdvice::updateCachedCallerFPI(FunctionAnalysisManager &FAM) const {
590 FPU->finish(FAM);
591}
592
593void MLInlineAdvice::recordInliningImpl() {
594 ORE.emit(RemarkBuilder: [&]() {
595 OptimizationRemark R(DEBUG_TYPE, "InliningSuccess", DLoc, Block);
596 reportContextForRemark(OR&: R);
597 return R;
598 });
599 getAdvisor()->onSuccessfulInlining(Advice: *this, /*CalleeWasDeleted*/ false);
600}
601
602void MLInlineAdvice::recordInliningWithCalleeDeletedImpl() {
603 ORE.emit(RemarkBuilder: [&]() {
604 OptimizationRemark R(DEBUG_TYPE, "InliningSuccessWithCalleeDeleted", DLoc,
605 Block);
606 reportContextForRemark(OR&: R);
607 return R;
608 });
609 getAdvisor()->onSuccessfulInlining(Advice: *this, /*CalleeWasDeleted*/ true);
610}
611
612void MLInlineAdvice::recordUnsuccessfulInliningImpl(
613 const InlineResult &Result) {
614 getAdvisor()->getCachedFPI(F&: *Caller) = PreInlineCallerFPI;
615 ORE.emit(RemarkBuilder: [&]() {
616 OptimizationRemarkMissed R(DEBUG_TYPE, "InliningAttemptedAndUnsuccessful",
617 DLoc, Block);
618 reportContextForRemark(OR&: R);
619 return R;
620 });
621}
622void MLInlineAdvice::recordUnattemptedInliningImpl() {
623 assert(!FPU);
624 ORE.emit(RemarkBuilder: [&]() {
625 OptimizationRemarkMissed R(DEBUG_TYPE, "IniningNotAttempted", DLoc, Block);
626 reportContextForRemark(OR&: R);
627 return R;
628 });
629}
630