1//===- InlineAdvisor.cpp - analysis pass implementation -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements InlineAdvisorAnalysis and DefaultInlineAdvisor, and
10// related types.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Analysis/InlineAdvisor.h"
15#include "AnalysisOptions.h"
16#include "llvm/ADT/Statistic.h"
17#include "llvm/ADT/StringExtras.h"
18#include "llvm/Analysis/AssumptionCache.h"
19#include "llvm/Analysis/EphemeralValuesCache.h"
20#include "llvm/Analysis/IR2Vec.h"
21#include "llvm/Analysis/InlineCost.h"
22#include "llvm/Analysis/OptimizationRemarkEmitter.h"
23#include "llvm/Analysis/ProfileSummaryInfo.h"
24#include "llvm/Analysis/ReplayInlineAdvisor.h"
25#include "llvm/Analysis/TargetLibraryInfo.h"
26#include "llvm/Analysis/TargetTransformInfo.h"
27#include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h"
28#include "llvm/IR/DebugInfoMetadata.h"
29#include "llvm/IR/Module.h"
30#include "llvm/IR/PassManager.h"
31#include "llvm/Support/raw_ostream.h"
32
33using namespace llvm;
34#define DEBUG_TYPE "inline"
35#ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL
36#define LLVM_HAVE_TF_AOT
37#endif
38
39// This weirdly named statistic tracks the number of times that, when attempting
40// to inline a function A into B, we analyze the callers of B in order to see
41// if those would be more profitable and blocked inline steps.
42STATISTIC(NumCallerCallersAnalyzed, "Number of caller-callers analyzed");
43
44namespace {
45using namespace llvm::ore;
46class MandatoryInlineAdvice : public InlineAdvice {
47public:
48 MandatoryInlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
49 OptimizationRemarkEmitter &ORE,
50 bool IsInliningMandatory)
51 : InlineAdvice(Advisor, CB, ORE, IsInliningMandatory) {}
52
53private:
54 void recordInliningWithCalleeDeletedImpl() override { recordInliningImpl(); }
55
56 void recordInliningImpl() override {
57 if (IsInliningRecommended)
58 emitInlinedInto(ORE, DLoc, Block, Callee: *Callee, Caller: *Caller, IsMandatory: IsInliningRecommended,
59 ExtraContext: [&](OptimizationRemark &Remark) {
60 Remark << ": always inline attribute";
61 });
62 }
63
64 void recordUnsuccessfulInliningImpl(const InlineResult &Result) override {
65 if (IsInliningRecommended)
66 ORE.emit(RemarkBuilder: [&]() {
67 return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(),
68 "NotInlined", DLoc, Block)
69 << "'" << NV("Callee", Callee) << "' is not AlwaysInline into '"
70 << NV("Caller", Caller)
71 << "': " << NV("Reason", Result.getFailureReason());
72 });
73 }
74
75 void recordUnattemptedInliningImpl() override {
76 assert(!IsInliningRecommended && "Expected to attempt inlining");
77 }
78};
79} // namespace
80
81void DefaultInlineAdvice::recordUnsuccessfulInliningImpl(
82 const InlineResult &Result) {
83 using namespace ore;
84 llvm::setInlineRemark(CB&: *OriginalCB, Message: std::string(Result.getFailureReason()) +
85 "; " + inlineCostStr(IC: *OIC));
86 ORE.emit(RemarkBuilder: [&]() {
87 return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(),
88 "NotInlined", DLoc, Block)
89 << "'" << NV("Callee", Callee) << "' is not inlined into '"
90 << NV("Caller", Caller)
91 << "': " << NV("Reason", Result.getFailureReason());
92 });
93}
94
95void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() {
96 if (EmitRemarks)
97 emitInlinedIntoBasedOnCost(ORE, DLoc, Block, Callee: *Callee, Caller: *Caller, IC: *OIC,
98 /* ForProfileContext= */ false,
99 PassName: Advisor->getAnnotatedInlinePassName());
100}
101
102void DefaultInlineAdvice::recordInliningImpl() {
103 if (EmitRemarks)
104 emitInlinedIntoBasedOnCost(ORE, DLoc, Block, Callee: *Callee, Caller: *Caller, IC: *OIC,
105 /* ForProfileContext= */ false,
106 PassName: Advisor->getAnnotatedInlinePassName());
107}
108
109std::optional<llvm::InlineCost> static getDefaultInlineAdvice(
110 CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params) {
111 Function &Caller = *CB.getCaller();
112 ProfileSummaryInfo *PSI =
113 FAM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: Caller)
114 .getCachedResult<ProfileSummaryAnalysis>(
115 IR&: *CB.getParent()->getParent()->getParent());
116
117 auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: Caller);
118 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
119 return FAM.getResult<AssumptionAnalysis>(IR&: F);
120 };
121 auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
122 return FAM.getResult<BlockFrequencyAnalysis>(IR&: F);
123 };
124 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
125 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
126 };
127 auto GetEphValuesCache =
128 [&](Function &F) -> EphemeralValuesAnalysis::Result & {
129 return FAM.getResult<EphemeralValuesAnalysis>(IR&: F);
130 };
131
132 Function &Callee = *CB.getCalledFunction();
133 auto &CallerTTI = FAM.getResult<TargetIRAnalysis>(IR&: Caller);
134 auto GetInlineCost = [&](CallBase &CB) {
135 bool RemarksEnabled =
136 Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
137 DEBUG_TYPE);
138 return getInlineCost(Call&: CB, Params, CallerTTI, GetAssumptionCache, GetTLI,
139 GetBFI, PSI, ORE: RemarksEnabled ? &ORE : nullptr,
140 GetEphValuesCache);
141 };
142 return llvm::shouldInline(CB, CallerTTI, GetInlineCost, ORE,
143 EnableDeferral: Params.EnableDeferral
144 ? *Params.EnableDeferral
145 : getAnalysisOptions(F: Caller).inline_deferral);
146}
147
148std::unique_ptr<InlineAdvice>
149DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) {
150 auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
151 return std::make_unique<DefaultInlineAdvice>(
152 args: this, args&: CB, args&: OIC,
153 args&: FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: *CB.getCaller()));
154}
155
156InlineAdvice::InlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
157 OptimizationRemarkEmitter &ORE,
158 bool IsInliningRecommended)
159 : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()),
160 DLoc(CB.getDebugLoc()), Block(CB.getParent()), ORE(ORE),
161 IsInliningRecommended(IsInliningRecommended) {}
162
163void InlineAdvice::recordInlineStatsIfNeeded() {
164 if (Advisor->ImportedFunctionsStats)
165 Advisor->ImportedFunctionsStats->recordInline(Caller: *Caller, Callee: *Callee);
166}
167
168void InlineAdvice::recordInlining() {
169 markRecorded();
170 recordInlineStatsIfNeeded();
171 recordInliningImpl();
172}
173
174void InlineAdvice::recordInliningWithCalleeDeleted() {
175 markRecorded();
176 recordInlineStatsIfNeeded();
177 recordInliningWithCalleeDeletedImpl();
178}
179
180AnalysisKey InlineAdvisorAnalysis::Key;
181AnalysisKey PluginInlineAdvisorAnalysis::Key;
182
183bool InlineAdvisorAnalysis::initializeIR2VecVocabIfRequested(
184 Module &M, ModuleAnalysisManager &MAM) {
185 if (!getAnalysisOptions(M).ml_inliner_ir2vec_vocab_file.empty()) {
186 auto &IR2VecVocabResult = MAM.getResult<IR2VecVocabAnalysis>(IR&: M);
187 if (!IR2VecVocabResult.isValid()) {
188 M.getContext().emitError(ErrorStr: "Failed to load IR2Vec vocabulary");
189 return false;
190 }
191 }
192 // No vocab file specified is OK; We just don't use IR2Vec
193 // embeddings.
194 return true;
195}
196
197bool InlineAdvisorAnalysis::Result::tryCreate(
198 InlineParams Params, InliningAdvisorMode Mode,
199 const ReplayInlinerSettings &ReplaySettings, InlineContext IC) {
200 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
201 if (MAM.isPassRegistered<PluginInlineAdvisorAnalysis>()) {
202 auto &DA = MAM.getResult<PluginInlineAdvisorAnalysis>(IR&: M);
203 Advisor.reset(p: DA.Factory(M, FAM, Params, IC));
204 return !!Advisor;
205 }
206 auto GetDefaultAdvice = [&FAM, Params](CallBase &CB) {
207 auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
208 return OIC.has_value();
209 };
210 switch (Mode) {
211 case InliningAdvisorMode::Default:
212 LLVM_DEBUG(dbgs() << "Using default inliner heuristic.\n");
213 Advisor.reset(p: new DefaultInlineAdvisor(M, FAM, Params, IC));
214 // Restrict replay to default advisor, ML advisors are stateful so
215 // replay will need augmentations to interleave with them correctly.
216 if (!ReplaySettings.ReplayFile.empty()) {
217 Advisor = llvm::getReplayInlineAdvisor(M, FAM, Context&: M.getContext(),
218 OriginalAdvisor: std::move(Advisor), ReplaySettings,
219 /* EmitRemarks =*/true, IC);
220 }
221 break;
222 // Run IR2VecVocabAnalysis once per module to get the vocabulary.
223 // We run it here because it is immutable and we want to avoid running it
224 // multiple times.
225 case InliningAdvisorMode::Development:
226#ifdef LLVM_HAVE_TFLITE
227 LLVM_DEBUG(dbgs() << "Using development-mode inliner policy.\n");
228 if (!InlineAdvisorAnalysis::initializeIR2VecVocabIfRequested(M, MAM))
229 return false;
230 Advisor = llvm::getDevelopmentModeAdvisor(M, MAM, GetDefaultAdvice);
231#endif
232 break;
233 case InliningAdvisorMode::Release:
234 LLVM_DEBUG(dbgs() << "Using release-mode inliner policy.\n");
235 if (!InlineAdvisorAnalysis::initializeIR2VecVocabIfRequested(M, MAM))
236 return false;
237 Advisor = llvm::getReleaseModeAdvisor(M, MAM, GetDefaultAdvice);
238 break;
239 }
240
241 return !!Advisor;
242}
243
244/// Return true if inlining of CB can block the caller from being
245/// inlined which is proved to be more beneficial. \p IC is the
246/// estimated inline cost associated with callsite \p CB.
247/// \p TotalSecondaryCost will be set to the estimated cost of inlining the
248/// caller if \p CB is suppressed for inlining.
249static bool
250shouldBeDeferred(Function *Caller, TargetTransformInfo &CallerTTI,
251 InlineCost IC, int &TotalSecondaryCost,
252 function_ref<InlineCost(CallBase &CB)> GetInlineCost) {
253 // For now we only handle local or inline functions.
254 if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
255 return false;
256 // If the cost of inlining CB is non-positive, it is not going to prevent the
257 // caller from being inlined into its callers and hence we don't need to
258 // defer.
259 if (IC.getCost() <= 0)
260 return false;
261 // Try to detect the case where the current inlining candidate caller (call
262 // it B) is a static or linkonce-ODR function and is an inlining candidate
263 // elsewhere, and the current candidate callee (call it C) is large enough
264 // that inlining it into B would make B too big to inline later. In these
265 // circumstances it may be best not to inline C into B, but to inline B into
266 // its callers.
267 //
268 // This only applies to static and linkonce-ODR functions because those are
269 // expected to be available for inlining in the translation units where they
270 // are used. Thus we will always have the opportunity to make local inlining
271 // decisions. Importantly the linkonce-ODR linkage covers inline functions
272 // and templates in C++.
273 //
274 // FIXME: All of this logic should be sunk into getInlineCost. It relies on
275 // the internal implementation of the inline cost metrics rather than
276 // treating them as truly abstract units etc.
277 TotalSecondaryCost = 0;
278 // The candidate cost to be imposed upon the current function.
279 int CandidateCost = IC.getCost() - 1;
280 // If the caller has local linkage and can be inlined to all its callers, we
281 // can apply a huge negative bonus to TotalSecondaryCost.
282 bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse();
283 // This bool tracks what happens if we DO inline C into B.
284 bool InliningPreventsSomeOuterInline = false;
285 unsigned NumCallerUsers = 0;
286 for (User *U : Caller->users()) {
287 CallBase *CS2 = dyn_cast<CallBase>(Val: U);
288
289 // If this isn't a call to Caller (it could be some other sort
290 // of reference) skip it. Such references will prevent the caller
291 // from being removed.
292 if (!CS2 || CS2->getCalledFunction() != Caller) {
293 ApplyLastCallBonus = false;
294 continue;
295 }
296
297 InlineCost IC2 = GetInlineCost(*CS2);
298 ++NumCallerCallersAnalyzed;
299 if (!IC2) {
300 ApplyLastCallBonus = false;
301 continue;
302 }
303 if (IC2.isAlways())
304 continue;
305
306 // See if inlining of the original callsite would erase the cost delta of
307 // this callsite. We subtract off the penalty for the call instruction,
308 // which we would be deleting.
309 if (IC2.getCostDelta() <= CandidateCost) {
310 InliningPreventsSomeOuterInline = true;
311 TotalSecondaryCost += IC2.getCost();
312 NumCallerUsers++;
313 }
314 }
315
316 if (!InliningPreventsSomeOuterInline)
317 return false;
318
319 // If all outer calls to Caller would get inlined, the cost for the last
320 // one is set very low by getInlineCost, in anticipation that Caller will
321 // be removed entirely. We did not account for this above unless there
322 // is only one caller of Caller.
323 if (ApplyLastCallBonus)
324 TotalSecondaryCost -= CallerTTI.getInliningLastCallToStaticBonus();
325
326 // If InlineDeferralScale is negative, then ignore the cost of primary
327 // inlining -- IC.getCost() multiplied by the number of callers to Caller.
328 int InlineDeferralScale = getAnalysisOptions(F: *Caller).inline_deferral_scale;
329 if (InlineDeferralScale < 0)
330 return TotalSecondaryCost < IC.getCost();
331
332 int TotalCost = TotalSecondaryCost + IC.getCost() * NumCallerUsers;
333 int Allowance = IC.getCost() * InlineDeferralScale;
334 return TotalCost < Allowance;
335}
336
337namespace llvm {
338static raw_ostream &operator<<(raw_ostream &R, const ore::NV &Arg) {
339 return R << Arg.Val;
340}
341
342template <class RemarkT>
343decltype(auto) operator<<(RemarkT &&R, const InlineCost &IC) {
344 using namespace ore;
345 if (IC.isAlways()) {
346 R << "(cost=always)";
347 } else if (IC.isNever()) {
348 R << "(cost=never)";
349 } else {
350 R << "(cost=" << ore::NV("Cost", IC.getCost())
351 << ", threshold=" << ore::NV("Threshold", IC.getThreshold()) << ")";
352 }
353 if (const char *Reason = IC.getReason())
354 R << ": " << ore::NV("Reason", Reason);
355 return std::forward<RemarkT>(R);
356}
357} // namespace llvm
358
359std::string llvm::inlineCostStr(const InlineCost &IC) {
360 std::string Buffer;
361 raw_string_ostream Remark(Buffer);
362 Remark << IC;
363 return Remark.str();
364}
365
366void llvm::setInlineRemark(CallBase &CB, StringRef Message) {
367 if (!getAnalysisOptions(C: CB.getContext()).inline_remark_attribute)
368 return;
369
370 Attribute Attr = Attribute::get(Context&: CB.getContext(), Kind: "inline-remark", Val: Message);
371 CB.addFnAttr(Attr);
372}
373
374/// Return the cost only if the inliner should attempt to inline at the given
375/// CallSite. If we return the cost, we will emit an optimisation remark later
376/// using that cost, so we won't do so from this function. Return std::nullopt
377/// if inlining should not be attempted.
378std::optional<InlineCost>
379llvm::shouldInline(CallBase &CB, TargetTransformInfo &CallerTTI,
380 function_ref<InlineCost(CallBase &CB)> GetInlineCost,
381 OptimizationRemarkEmitter &ORE, bool EnableDeferral) {
382 using namespace ore;
383
384 InlineCost IC = GetInlineCost(CB);
385 Instruction *Call = &CB;
386 Function *Callee = CB.getCalledFunction();
387 Function *Caller = CB.getCaller();
388
389 if (IC.isAlways()) {
390 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC)
391 << ", Call: " << CB << "\n");
392 return IC;
393 }
394
395 if (!IC) {
396 LLVM_DEBUG(dbgs() << " NOT Inlining " << inlineCostStr(IC)
397 << ", Call: " << CB << "\n");
398 if (IC.isNever()) {
399 ORE.emit(RemarkBuilder: [&]() {
400 return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", Call)
401 << "'" << NV("Callee", Callee) << "' not inlined into '"
402 << NV("Caller", Caller)
403 << "' because it should never be inlined " << IC;
404 });
405 } else {
406 ORE.emit(RemarkBuilder: [&]() {
407 return OptimizationRemarkMissed(DEBUG_TYPE, "TooCostly", Call)
408 << "'" << NV("Callee", Callee) << "' not inlined into '"
409 << NV("Caller", Caller) << "' because too costly to inline "
410 << IC;
411 });
412 }
413 setInlineRemark(CB, Message: inlineCostStr(IC));
414 return std::nullopt;
415 }
416
417 int TotalSecondaryCost = 0;
418 if (EnableDeferral && shouldBeDeferred(Caller, CallerTTI, IC,
419 TotalSecondaryCost, GetInlineCost)) {
420 LLVM_DEBUG(dbgs() << " NOT Inlining: " << CB
421 << " Cost = " << IC.getCost()
422 << ", outer Cost = " << TotalSecondaryCost << '\n');
423 ORE.emit(RemarkBuilder: [&]() {
424 return OptimizationRemarkMissed(DEBUG_TYPE, "IncreaseCostInOtherContexts",
425 Call)
426 << "Not inlining. Cost of inlining '" << NV("Callee", Callee)
427 << "' increases the cost of inlining '" << NV("Caller", Caller)
428 << "' in other contexts";
429 });
430 setInlineRemark(CB, Message: "deferred");
431 return std::nullopt;
432 }
433
434 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) << ", Call: " << CB
435 << '\n');
436 return IC;
437}
438
439std::string llvm::formatCallSiteLocation(DebugLoc DLoc,
440 const CallSiteFormat &Format) {
441 std::string Buffer;
442 raw_string_ostream CallSiteLoc(Buffer);
443 ListSeparator LS(" @ ");
444 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
445 CallSiteLoc << LS;
446 // Note that negative line offset is actually possible, but we use
447 // unsigned int to match line offset representation in remarks so
448 // it's directly consumable by relay advisor.
449 uint32_t Offset =
450 DIL->getLine() - DIL->getScope()->getSubprogram()->getLine();
451 uint32_t Discriminator = DIL->getBaseDiscriminator();
452 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
453 if (Name.empty())
454 Name = DIL->getScope()->getSubprogram()->getName();
455 CallSiteLoc << Name.str() << ":" << llvm::utostr(X: Offset);
456 if (Format.outputColumn())
457 CallSiteLoc << ":" << llvm::utostr(X: DIL->getColumn());
458 if (Format.outputDiscriminator() && Discriminator)
459 CallSiteLoc << "." << llvm::utostr(X: Discriminator);
460 }
461
462 return CallSiteLoc.str();
463}
464
465void llvm::addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc) {
466 if (!DLoc)
467 return;
468
469 bool First = true;
470 Remark << " at callsite ";
471 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
472 if (!First)
473 Remark << " @ ";
474 unsigned int Offset = DIL->getLine();
475 Offset -= DIL->getScope()->getSubprogram()->getLine();
476 unsigned int Discriminator = DIL->getBaseDiscriminator();
477 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
478 if (Name.empty())
479 Name = DIL->getScope()->getSubprogram()->getName();
480 Remark << Name << ":" << ore::NV("Line", Offset) << ":"
481 << ore::NV("Column", DIL->getColumn());
482 if (Discriminator)
483 Remark << "." << ore::NV("Disc", Discriminator);
484 First = false;
485 }
486
487 Remark << ";";
488}
489
490void llvm::emitInlinedInto(
491 OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block,
492 const Function &Callee, const Function &Caller, bool AlwaysInline,
493 function_ref<void(OptimizationRemark &)> ExtraContext,
494 const char *PassName) {
495 ORE.emit(RemarkBuilder: [&]() {
496 StringRef RemarkName = AlwaysInline ? "AlwaysInline" : "Inlined";
497 OptimizationRemark Remark(PassName ? PassName : DEBUG_TYPE, RemarkName,
498 DLoc, Block);
499 Remark << "'" << ore::NV("Callee", &Callee) << "' inlined into '"
500 << ore::NV("Caller", &Caller) << "'";
501 if (ExtraContext)
502 ExtraContext(Remark);
503 addLocationToRemarks(Remark, DLoc);
504 return Remark;
505 });
506}
507
508void llvm::emitInlinedIntoBasedOnCost(
509 OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block,
510 const Function &Callee, const Function &Caller, const InlineCost &IC,
511 bool ForProfileContext, const char *PassName) {
512 llvm::emitInlinedInto(
513 ORE, DLoc, Block, Callee, Caller, AlwaysInline: IC.isAlways(),
514 ExtraContext: [&](OptimizationRemark &Remark) {
515 if (ForProfileContext)
516 Remark << " to match profiling context";
517 Remark << " with " << IC;
518 },
519 PassName);
520}
521
522InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM,
523 std::optional<InlineContext> IC)
524 : M(M), FAM(FAM), IC(IC),
525 AnnotatedInlinePassName(
526 (IC && getAnalysisOptions(M).annotate_inline_phase)
527 ? llvm::AnnotateInlinePassName(IC: *IC)
528 : DEBUG_TYPE) {
529 if (getAnalysisOptions(M).inliner_function_import_stats !=
530 InlinerFunctionImportStatsOpts::No) {
531 ImportedFunctionsStats =
532 std::make_unique<ImportedFunctionsInliningStatistics>();
533 ImportedFunctionsStats->setModuleInfo(M);
534 }
535}
536
537InlineAdvisor::~InlineAdvisor() {
538 if (ImportedFunctionsStats)
539 ImportedFunctionsStats->dump(
540 Verbose: getAnalysisOptions(M).inliner_function_import_stats ==
541 InlinerFunctionImportStatsOpts::Verbose);
542}
543
544std::unique_ptr<InlineAdvice> InlineAdvisor::getMandatoryAdvice(CallBase &CB,
545 bool Advice) {
546 return std::make_unique<MandatoryInlineAdvice>(args: this, args&: CB, args&: getCallerORE(CB),
547 args&: Advice);
548}
549
550static inline const char *getLTOPhase(ThinOrFullLTOPhase LTOPhase) {
551 switch (LTOPhase) {
552 case (ThinOrFullLTOPhase::None):
553 return "main";
554 case (ThinOrFullLTOPhase::ThinLTOPreLink):
555 case (ThinOrFullLTOPhase::FullLTOPreLink):
556 return "prelink";
557 case (ThinOrFullLTOPhase::ThinLTOPostLink):
558 case (ThinOrFullLTOPhase::FullLTOPostLink):
559 return "postlink";
560 }
561 llvm_unreachable("unreachable");
562}
563
564static inline const char *getInlineAdvisorContext(InlinePass IP) {
565 switch (IP) {
566 case (InlinePass::AlwaysInliner):
567 return "always-inline";
568 case (InlinePass::CGSCCInliner):
569 return "cgscc-inline";
570 case (InlinePass::EarlyInliner):
571 return "early-inline";
572 case (InlinePass::MLInliner):
573 return "ml-inline";
574 case (InlinePass::ModuleInliner):
575 return "module-inline";
576 case (InlinePass::ReplayCGSCCInliner):
577 return "replay-cgscc-inline";
578 case (InlinePass::ReplaySampleProfileInliner):
579 return "replay-sample-profile-inline";
580 case (InlinePass::SampleProfileInliner):
581 return "sample-profile-inline";
582 }
583
584 llvm_unreachable("unreachable");
585}
586
587std::string llvm::AnnotateInlinePassName(InlineContext IC) {
588 return std::string(getLTOPhase(LTOPhase: IC.LTOPhase)) + "-" +
589 std::string(getInlineAdvisorContext(IP: IC.Pass));
590}
591
592InlineAdvisor::MandatoryInliningKind
593InlineAdvisor::getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM,
594 OptimizationRemarkEmitter &ORE) {
595 auto &Callee = *CB.getCalledFunction();
596
597 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
598 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
599 };
600
601 auto &TIR = FAM.getResult<TargetIRAnalysis>(IR&: *CB.getCaller());
602
603 auto TrivialDecision =
604 llvm::getAttributeBasedInliningDecision(Call&: CB, Callee: &Callee, CallerTTI&: TIR, GetTLI);
605
606 if (TrivialDecision) {
607 if (TrivialDecision->isSuccess())
608 return MandatoryInliningKind::Always;
609 else
610 return MandatoryInliningKind::Never;
611 }
612 return MandatoryInliningKind::NotMandatory;
613}
614
615std::unique_ptr<InlineAdvice> InlineAdvisor::getAdvice(CallBase &CB,
616 bool MandatoryOnly) {
617 if (!MandatoryOnly)
618 return getAdviceImpl(CB);
619 bool Advice = CB.getCaller() != CB.getCalledFunction() &&
620 MandatoryInliningKind::Always ==
621 getMandatoryKind(CB, FAM, ORE&: getCallerORE(CB));
622 return getMandatoryAdvice(CB, Advice);
623}
624
625OptimizationRemarkEmitter &InlineAdvisor::getCallerORE(CallBase &CB) {
626 return FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: *CB.getCaller());
627}
628
629PreservedAnalyses
630InlineAdvisorAnalysisPrinterPass::run(Module &M, ModuleAnalysisManager &MAM) {
631 const auto *IA = MAM.getCachedResult<InlineAdvisorAnalysis>(IR&: M);
632 if (!IA)
633 OS << "No Inline Advisor\n";
634 else
635 IA->getAdvisor()->print(OS);
636 return PreservedAnalyses::all();
637}
638
639PreservedAnalyses InlineAdvisorAnalysisPrinterPass::run(
640 LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM, LazyCallGraph &CG,
641 CGSCCUpdateResult &UR) {
642 const auto &MAMProxy =
643 AM.getResult<ModuleAnalysisManagerCGSCCProxy>(IR&: InitialC, ExtraArgs&: CG);
644
645 if (InitialC.size() == 0) {
646 OS << "SCC is empty!\n";
647 return PreservedAnalyses::all();
648 }
649 Module &M = *InitialC.begin()->getFunction().getParent();
650 const auto *IA = MAMProxy.getCachedResult<InlineAdvisorAnalysis>(IR&: M);
651 if (!IA)
652 OS << "No Inline Advisor\n";
653 else
654 IA->getAdvisor()->print(OS);
655 return PreservedAnalyses::all();
656}
657