1//===- InlineAdvisor.cpp - analysis pass implementation -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements InlineAdvisorAnalysis and DefaultInlineAdvisor, and
10// related types.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Analysis/InlineAdvisor.h"
15#include "llvm/ADT/Statistic.h"
16#include "llvm/ADT/StringExtras.h"
17#include "llvm/Analysis/AssumptionCache.h"
18#include "llvm/Analysis/InlineCost.h"
19#include "llvm/Analysis/OptimizationRemarkEmitter.h"
20#include "llvm/Analysis/ProfileSummaryInfo.h"
21#include "llvm/Analysis/ReplayInlineAdvisor.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Analysis/TargetTransformInfo.h"
24#include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h"
25#include "llvm/IR/DebugInfoMetadata.h"
26#include "llvm/IR/Module.h"
27#include "llvm/IR/PassManager.h"
28#include "llvm/Support/CommandLine.h"
29#include "llvm/Support/raw_ostream.h"
30
31using namespace llvm;
32#define DEBUG_TYPE "inline"
33#ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL
34#define LLVM_HAVE_TF_AOT
35#endif
36
37// This weirdly named statistic tracks the number of times that, when attempting
38// to inline a function A into B, we analyze the callers of B in order to see
39// if those would be more profitable and blocked inline steps.
40STATISTIC(NumCallerCallersAnalyzed, "Number of caller-callers analyzed");
41
42/// Flag to add inline messages as callsite attributes 'inline-remark'.
43static cl::opt<bool>
44 InlineRemarkAttribute("inline-remark-attribute", cl::init(Val: false),
45 cl::Hidden,
46 cl::desc("Enable adding inline-remark attribute to"
47 " callsites processed by inliner but decided"
48 " to be not inlined"));
49
50static cl::opt<bool> EnableInlineDeferral("inline-deferral", cl::init(Val: false),
51 cl::Hidden,
52 cl::desc("Enable deferred inlining"));
53
54// An integer used to limit the cost of inline deferral. The default negative
55// number tells shouldBeDeferred to only take the secondary cost into account.
56static cl::opt<int>
57 InlineDeferralScale("inline-deferral-scale",
58 cl::desc("Scale to limit the cost of inline deferral"),
59 cl::init(Val: 2), cl::Hidden);
60
61static cl::opt<bool>
62 AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(Val: false),
63 cl::desc("If true, annotate inline advisor remarks "
64 "with LTO and pass information."));
65
66namespace llvm {
67extern cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats;
68} // namespace llvm
69
70namespace {
71using namespace llvm::ore;
72class MandatoryInlineAdvice : public InlineAdvice {
73public:
74 MandatoryInlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
75 OptimizationRemarkEmitter &ORE,
76 bool IsInliningMandatory)
77 : InlineAdvice(Advisor, CB, ORE, IsInliningMandatory) {}
78
79private:
80 void recordInliningWithCalleeDeletedImpl() override { recordInliningImpl(); }
81
82 void recordInliningImpl() override {
83 if (IsInliningRecommended)
84 emitInlinedInto(ORE, DLoc, Block, Callee: *Callee, Caller: *Caller, IsMandatory: IsInliningRecommended,
85 ExtraContext: [&](OptimizationRemark &Remark) {
86 Remark << ": always inline attribute";
87 });
88 }
89
90 void recordUnsuccessfulInliningImpl(const InlineResult &Result) override {
91 if (IsInliningRecommended)
92 ORE.emit(RemarkBuilder: [&]() {
93 return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(),
94 "NotInlined", DLoc, Block)
95 << "'" << NV("Callee", Callee) << "' is not AlwaysInline into '"
96 << NV("Caller", Caller)
97 << "': " << NV("Reason", Result.getFailureReason());
98 });
99 }
100
101 void recordUnattemptedInliningImpl() override {
102 assert(!IsInliningRecommended && "Expected to attempt inlining");
103 }
104};
105} // namespace
106
107void DefaultInlineAdvice::recordUnsuccessfulInliningImpl(
108 const InlineResult &Result) {
109 using namespace ore;
110 llvm::setInlineRemark(CB&: *OriginalCB, Message: std::string(Result.getFailureReason()) +
111 "; " + inlineCostStr(IC: *OIC));
112 ORE.emit(RemarkBuilder: [&]() {
113 return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(),
114 "NotInlined", DLoc, Block)
115 << "'" << NV("Callee", Callee) << "' is not inlined into '"
116 << NV("Caller", Caller)
117 << "': " << NV("Reason", Result.getFailureReason());
118 });
119}
120
121void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() {
122 if (EmitRemarks)
123 emitInlinedIntoBasedOnCost(ORE, DLoc, Block, Callee: *Callee, Caller: *Caller, IC: *OIC,
124 /* ForProfileContext= */ false,
125 PassName: Advisor->getAnnotatedInlinePassName());
126}
127
128void DefaultInlineAdvice::recordInliningImpl() {
129 if (EmitRemarks)
130 emitInlinedIntoBasedOnCost(ORE, DLoc, Block, Callee: *Callee, Caller: *Caller, IC: *OIC,
131 /* ForProfileContext= */ false,
132 PassName: Advisor->getAnnotatedInlinePassName());
133}
134
135std::optional<llvm::InlineCost> static getDefaultInlineAdvice(
136 CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params) {
137 Function &Caller = *CB.getCaller();
138 ProfileSummaryInfo *PSI =
139 FAM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: Caller)
140 .getCachedResult<ProfileSummaryAnalysis>(
141 IR&: *CB.getParent()->getParent()->getParent());
142
143 auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: Caller);
144 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
145 return FAM.getResult<AssumptionAnalysis>(IR&: F);
146 };
147 auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
148 return FAM.getResult<BlockFrequencyAnalysis>(IR&: F);
149 };
150 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
151 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
152 };
153
154 auto GetInlineCost = [&](CallBase &CB) {
155 Function &Callee = *CB.getCalledFunction();
156 auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(IR&: Callee);
157 bool RemarksEnabled =
158 Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
159 DEBUG_TYPE);
160 return getInlineCost(Call&: CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
161 GetBFI, PSI, ORE: RemarksEnabled ? &ORE : nullptr);
162 };
163 return llvm::shouldInline(
164 CB, GetInlineCost, ORE,
165 EnableDeferral: Params.EnableDeferral.value_or(u&: EnableInlineDeferral));
166}
167
168std::unique_ptr<InlineAdvice>
169DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) {
170 auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
171 return std::make_unique<DefaultInlineAdvice>(
172 args: this, args&: CB, args&: OIC,
173 args&: FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: *CB.getCaller()));
174}
175
176InlineAdvice::InlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
177 OptimizationRemarkEmitter &ORE,
178 bool IsInliningRecommended)
179 : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()),
180 DLoc(CB.getDebugLoc()), Block(CB.getParent()), ORE(ORE),
181 IsInliningRecommended(IsInliningRecommended) {}
182
183void InlineAdvice::recordInlineStatsIfNeeded() {
184 if (Advisor->ImportedFunctionsStats)
185 Advisor->ImportedFunctionsStats->recordInline(Caller: *Caller, Callee: *Callee);
186}
187
188void InlineAdvice::recordInlining() {
189 markRecorded();
190 recordInlineStatsIfNeeded();
191 recordInliningImpl();
192}
193
194void InlineAdvice::recordInliningWithCalleeDeleted() {
195 markRecorded();
196 recordInlineStatsIfNeeded();
197 recordInliningWithCalleeDeletedImpl();
198}
199
200AnalysisKey InlineAdvisorAnalysis::Key;
201AnalysisKey PluginInlineAdvisorAnalysis::Key;
202bool PluginInlineAdvisorAnalysis::HasBeenRegistered = false;
203
204bool InlineAdvisorAnalysis::Result::tryCreate(
205 InlineParams Params, InliningAdvisorMode Mode,
206 const ReplayInlinerSettings &ReplaySettings, InlineContext IC) {
207 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
208 if (PluginInlineAdvisorAnalysis::HasBeenRegistered) {
209 auto &DA = MAM.getResult<PluginInlineAdvisorAnalysis>(IR&: M);
210 Advisor.reset(p: DA.Factory(M, FAM, Params, IC));
211 return !!Advisor;
212 }
213 auto GetDefaultAdvice = [&FAM, Params](CallBase &CB) {
214 auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
215 return OIC.has_value();
216 };
217 switch (Mode) {
218 case InliningAdvisorMode::Default:
219 LLVM_DEBUG(dbgs() << "Using default inliner heuristic.\n");
220 Advisor.reset(p: new DefaultInlineAdvisor(M, FAM, Params, IC));
221 // Restrict replay to default advisor, ML advisors are stateful so
222 // replay will need augmentations to interleave with them correctly.
223 if (!ReplaySettings.ReplayFile.empty()) {
224 Advisor = llvm::getReplayInlineAdvisor(M, FAM, Context&: M.getContext(),
225 OriginalAdvisor: std::move(Advisor), ReplaySettings,
226 /* EmitRemarks =*/true, IC);
227 }
228 break;
229 case InliningAdvisorMode::Development:
230#ifdef LLVM_HAVE_TFLITE
231 LLVM_DEBUG(dbgs() << "Using development-mode inliner policy.\n");
232 Advisor = llvm::getDevelopmentModeAdvisor(M, MAM, GetDefaultAdvice);
233#endif
234 break;
235 case InliningAdvisorMode::Release:
236 LLVM_DEBUG(dbgs() << "Using release-mode inliner policy.\n");
237 Advisor = llvm::getReleaseModeAdvisor(M, MAM, GetDefaultAdvice);
238 break;
239 }
240
241 return !!Advisor;
242}
243
244/// Return true if inlining of CB can block the caller from being
245/// inlined which is proved to be more beneficial. \p IC is the
246/// estimated inline cost associated with callsite \p CB.
247/// \p TotalSecondaryCost will be set to the estimated cost of inlining the
248/// caller if \p CB is suppressed for inlining.
249static bool
250shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost,
251 function_ref<InlineCost(CallBase &CB)> GetInlineCost) {
252 // For now we only handle local or inline functions.
253 if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
254 return false;
255 // If the cost of inlining CB is non-positive, it is not going to prevent the
256 // caller from being inlined into its callers and hence we don't need to
257 // defer.
258 if (IC.getCost() <= 0)
259 return false;
260 // Try to detect the case where the current inlining candidate caller (call
261 // it B) is a static or linkonce-ODR function and is an inlining candidate
262 // elsewhere, and the current candidate callee (call it C) is large enough
263 // that inlining it into B would make B too big to inline later. In these
264 // circumstances it may be best not to inline C into B, but to inline B into
265 // its callers.
266 //
267 // This only applies to static and linkonce-ODR functions because those are
268 // expected to be available for inlining in the translation units where they
269 // are used. Thus we will always have the opportunity to make local inlining
270 // decisions. Importantly the linkonce-ODR linkage covers inline functions
271 // and templates in C++.
272 //
273 // FIXME: All of this logic should be sunk into getInlineCost. It relies on
274 // the internal implementation of the inline cost metrics rather than
275 // treating them as truly abstract units etc.
276 TotalSecondaryCost = 0;
277 // The candidate cost to be imposed upon the current function.
278 int CandidateCost = IC.getCost() - 1;
279 // If the caller has local linkage and can be inlined to all its callers, we
280 // can apply a huge negative bonus to TotalSecondaryCost.
281 bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse();
282 // This bool tracks what happens if we DO inline C into B.
283 bool InliningPreventsSomeOuterInline = false;
284 unsigned NumCallerUsers = 0;
285 for (User *U : Caller->users()) {
286 CallBase *CS2 = dyn_cast<CallBase>(Val: U);
287
288 // If this isn't a call to Caller (it could be some other sort
289 // of reference) skip it. Such references will prevent the caller
290 // from being removed.
291 if (!CS2 || CS2->getCalledFunction() != Caller) {
292 ApplyLastCallBonus = false;
293 continue;
294 }
295
296 InlineCost IC2 = GetInlineCost(*CS2);
297 ++NumCallerCallersAnalyzed;
298 if (!IC2) {
299 ApplyLastCallBonus = false;
300 continue;
301 }
302 if (IC2.isAlways())
303 continue;
304
305 // See if inlining of the original callsite would erase the cost delta of
306 // this callsite. We subtract off the penalty for the call instruction,
307 // which we would be deleting.
308 if (IC2.getCostDelta() <= CandidateCost) {
309 InliningPreventsSomeOuterInline = true;
310 TotalSecondaryCost += IC2.getCost();
311 NumCallerUsers++;
312 }
313 }
314
315 if (!InliningPreventsSomeOuterInline)
316 return false;
317
318 // If all outer calls to Caller would get inlined, the cost for the last
319 // one is set very low by getInlineCost, in anticipation that Caller will
320 // be removed entirely. We did not account for this above unless there
321 // is only one caller of Caller.
322 if (ApplyLastCallBonus)
323 TotalSecondaryCost -= InlineConstants::LastCallToStaticBonus;
324
325 // If InlineDeferralScale is negative, then ignore the cost of primary
326 // inlining -- IC.getCost() multiplied by the number of callers to Caller.
327 if (InlineDeferralScale < 0)
328 return TotalSecondaryCost < IC.getCost();
329
330 int TotalCost = TotalSecondaryCost + IC.getCost() * NumCallerUsers;
331 int Allowance = IC.getCost() * InlineDeferralScale;
332 return TotalCost < Allowance;
333}
334
335namespace llvm {
336static raw_ostream &operator<<(raw_ostream &R, const ore::NV &Arg) {
337 return R << Arg.Val;
338}
339
340template <class RemarkT>
341RemarkT &operator<<(RemarkT &&R, const InlineCost &IC) {
342 using namespace ore;
343 if (IC.isAlways()) {
344 R << "(cost=always)";
345 } else if (IC.isNever()) {
346 R << "(cost=never)";
347 } else {
348 R << "(cost=" << ore::NV("Cost", IC.getCost())
349 << ", threshold=" << ore::NV("Threshold", IC.getThreshold()) << ")";
350 }
351 if (const char *Reason = IC.getReason())
352 R << ": " << ore::NV("Reason", Reason);
353 return R;
354}
355} // namespace llvm
356
357std::string llvm::inlineCostStr(const InlineCost &IC) {
358 std::string Buffer;
359 raw_string_ostream Remark(Buffer);
360 Remark << IC;
361 return Remark.str();
362}
363
364void llvm::setInlineRemark(CallBase &CB, StringRef Message) {
365 if (!InlineRemarkAttribute)
366 return;
367
368 Attribute Attr = Attribute::get(Context&: CB.getContext(), Kind: "inline-remark", Val: Message);
369 CB.addFnAttr(Attr);
370}
371
372/// Return the cost only if the inliner should attempt to inline at the given
373/// CallSite. If we return the cost, we will emit an optimisation remark later
374/// using that cost, so we won't do so from this function. Return std::nullopt
375/// if inlining should not be attempted.
376std::optional<InlineCost>
377llvm::shouldInline(CallBase &CB,
378 function_ref<InlineCost(CallBase &CB)> GetInlineCost,
379 OptimizationRemarkEmitter &ORE, bool EnableDeferral) {
380 using namespace ore;
381
382 InlineCost IC = GetInlineCost(CB);
383 Instruction *Call = &CB;
384 Function *Callee = CB.getCalledFunction();
385 Function *Caller = CB.getCaller();
386
387 if (IC.isAlways()) {
388 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC)
389 << ", Call: " << CB << "\n");
390 return IC;
391 }
392
393 if (!IC) {
394 LLVM_DEBUG(dbgs() << " NOT Inlining " << inlineCostStr(IC)
395 << ", Call: " << CB << "\n");
396 if (IC.isNever()) {
397 ORE.emit(RemarkBuilder: [&]() {
398 return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", Call)
399 << "'" << NV("Callee", Callee) << "' not inlined into '"
400 << NV("Caller", Caller)
401 << "' because it should never be inlined " << IC;
402 });
403 } else {
404 ORE.emit(RemarkBuilder: [&]() {
405 return OptimizationRemarkMissed(DEBUG_TYPE, "TooCostly", Call)
406 << "'" << NV("Callee", Callee) << "' not inlined into '"
407 << NV("Caller", Caller) << "' because too costly to inline "
408 << IC;
409 });
410 }
411 setInlineRemark(CB, Message: inlineCostStr(IC));
412 return std::nullopt;
413 }
414
415 int TotalSecondaryCost = 0;
416 if (EnableDeferral &&
417 shouldBeDeferred(Caller, IC, TotalSecondaryCost, GetInlineCost)) {
418 LLVM_DEBUG(dbgs() << " NOT Inlining: " << CB
419 << " Cost = " << IC.getCost()
420 << ", outer Cost = " << TotalSecondaryCost << '\n');
421 ORE.emit(RemarkBuilder: [&]() {
422 return OptimizationRemarkMissed(DEBUG_TYPE, "IncreaseCostInOtherContexts",
423 Call)
424 << "Not inlining. Cost of inlining '" << NV("Callee", Callee)
425 << "' increases the cost of inlining '" << NV("Caller", Caller)
426 << "' in other contexts";
427 });
428 setInlineRemark(CB, Message: "deferred");
429 return std::nullopt;
430 }
431
432 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) << ", Call: " << CB
433 << '\n');
434 return IC;
435}
436
437std::string llvm::formatCallSiteLocation(DebugLoc DLoc,
438 const CallSiteFormat &Format) {
439 std::string Buffer;
440 raw_string_ostream CallSiteLoc(Buffer);
441 bool First = true;
442 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
443 if (!First)
444 CallSiteLoc << " @ ";
445 // Note that negative line offset is actually possible, but we use
446 // unsigned int to match line offset representation in remarks so
447 // it's directly consumable by relay advisor.
448 uint32_t Offset =
449 DIL->getLine() - DIL->getScope()->getSubprogram()->getLine();
450 uint32_t Discriminator = DIL->getBaseDiscriminator();
451 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
452 if (Name.empty())
453 Name = DIL->getScope()->getSubprogram()->getName();
454 CallSiteLoc << Name.str() << ":" << llvm::utostr(X: Offset);
455 if (Format.outputColumn())
456 CallSiteLoc << ":" << llvm::utostr(X: DIL->getColumn());
457 if (Format.outputDiscriminator() && Discriminator)
458 CallSiteLoc << "." << llvm::utostr(X: Discriminator);
459 First = false;
460 }
461
462 return CallSiteLoc.str();
463}
464
465void llvm::addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc) {
466 if (!DLoc) {
467 return;
468 }
469
470 bool First = true;
471 Remark << " at callsite ";
472 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
473 if (!First)
474 Remark << " @ ";
475 unsigned int Offset = DIL->getLine();
476 Offset -= DIL->getScope()->getSubprogram()->getLine();
477 unsigned int Discriminator = DIL->getBaseDiscriminator();
478 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
479 if (Name.empty())
480 Name = DIL->getScope()->getSubprogram()->getName();
481 Remark << Name << ":" << ore::NV("Line", Offset) << ":"
482 << ore::NV("Column", DIL->getColumn());
483 if (Discriminator)
484 Remark << "." << ore::NV("Disc", Discriminator);
485 First = false;
486 }
487
488 Remark << ";";
489}
490
491void llvm::emitInlinedInto(
492 OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block,
493 const Function &Callee, const Function &Caller, bool AlwaysInline,
494 function_ref<void(OptimizationRemark &)> ExtraContext,
495 const char *PassName) {
496 ORE.emit(RemarkBuilder: [&]() {
497 StringRef RemarkName = AlwaysInline ? "AlwaysInline" : "Inlined";
498 OptimizationRemark Remark(PassName ? PassName : DEBUG_TYPE, RemarkName,
499 DLoc, Block);
500 Remark << "'" << ore::NV("Callee", &Callee) << "' inlined into '"
501 << ore::NV("Caller", &Caller) << "'";
502 if (ExtraContext)
503 ExtraContext(Remark);
504 addLocationToRemarks(Remark, DLoc);
505 return Remark;
506 });
507}
508
509void llvm::emitInlinedIntoBasedOnCost(
510 OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block,
511 const Function &Callee, const Function &Caller, const InlineCost &IC,
512 bool ForProfileContext, const char *PassName) {
513 llvm::emitInlinedInto(
514 ORE, DLoc, Block, Callee, Caller, AlwaysInline: IC.isAlways(),
515 ExtraContext: [&](OptimizationRemark &Remark) {
516 if (ForProfileContext)
517 Remark << " to match profiling context";
518 Remark << " with " << IC;
519 },
520 PassName);
521}
522
523InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM,
524 std::optional<InlineContext> IC)
525 : M(M), FAM(FAM), IC(IC),
526 AnnotatedInlinePassName((IC && AnnotateInlinePhase)
527 ? llvm::AnnotateInlinePassName(IC: *IC)
528 : DEBUG_TYPE) {
529 if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) {
530 ImportedFunctionsStats =
531 std::make_unique<ImportedFunctionsInliningStatistics>();
532 ImportedFunctionsStats->setModuleInfo(M);
533 }
534}
535
536InlineAdvisor::~InlineAdvisor() {
537 if (ImportedFunctionsStats) {
538 assert(InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No);
539 ImportedFunctionsStats->dump(Verbose: InlinerFunctionImportStats ==
540 InlinerFunctionImportStatsOpts::Verbose);
541 }
542}
543
544std::unique_ptr<InlineAdvice> InlineAdvisor::getMandatoryAdvice(CallBase &CB,
545 bool Advice) {
546 return std::make_unique<MandatoryInlineAdvice>(args: this, args&: CB, args&: getCallerORE(CB),
547 args&: Advice);
548}
549
550static inline const char *getLTOPhase(ThinOrFullLTOPhase LTOPhase) {
551 switch (LTOPhase) {
552 case (ThinOrFullLTOPhase::None):
553 return "main";
554 case (ThinOrFullLTOPhase::ThinLTOPreLink):
555 case (ThinOrFullLTOPhase::FullLTOPreLink):
556 return "prelink";
557 case (ThinOrFullLTOPhase::ThinLTOPostLink):
558 case (ThinOrFullLTOPhase::FullLTOPostLink):
559 return "postlink";
560 }
561 llvm_unreachable("unreachable");
562}
563
564static inline const char *getInlineAdvisorContext(InlinePass IP) {
565 switch (IP) {
566 case (InlinePass::AlwaysInliner):
567 return "always-inline";
568 case (InlinePass::CGSCCInliner):
569 return "cgscc-inline";
570 case (InlinePass::EarlyInliner):
571 return "early-inline";
572 case (InlinePass::MLInliner):
573 return "ml-inline";
574 case (InlinePass::ModuleInliner):
575 return "module-inline";
576 case (InlinePass::ReplayCGSCCInliner):
577 return "replay-cgscc-inline";
578 case (InlinePass::ReplaySampleProfileInliner):
579 return "replay-sample-profile-inline";
580 case (InlinePass::SampleProfileInliner):
581 return "sample-profile-inline";
582 }
583
584 llvm_unreachable("unreachable");
585}
586
587std::string llvm::AnnotateInlinePassName(InlineContext IC) {
588 return std::string(getLTOPhase(LTOPhase: IC.LTOPhase)) + "-" +
589 std::string(getInlineAdvisorContext(IP: IC.Pass));
590}
591
592InlineAdvisor::MandatoryInliningKind
593InlineAdvisor::getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM,
594 OptimizationRemarkEmitter &ORE) {
595 auto &Callee = *CB.getCalledFunction();
596
597 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
598 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
599 };
600
601 auto &TIR = FAM.getResult<TargetIRAnalysis>(IR&: Callee);
602
603 auto TrivialDecision =
604 llvm::getAttributeBasedInliningDecision(Call&: CB, Callee: &Callee, CalleeTTI&: TIR, GetTLI);
605
606 if (TrivialDecision) {
607 if (TrivialDecision->isSuccess())
608 return MandatoryInliningKind::Always;
609 else
610 return MandatoryInliningKind::Never;
611 }
612 return MandatoryInliningKind::NotMandatory;
613}
614
615std::unique_ptr<InlineAdvice> InlineAdvisor::getAdvice(CallBase &CB,
616 bool MandatoryOnly) {
617 if (!MandatoryOnly)
618 return getAdviceImpl(CB);
619 bool Advice = CB.getCaller() != CB.getCalledFunction() &&
620 MandatoryInliningKind::Always ==
621 getMandatoryKind(CB, FAM, ORE&: getCallerORE(CB));
622 return getMandatoryAdvice(CB, Advice);
623}
624
625OptimizationRemarkEmitter &InlineAdvisor::getCallerORE(CallBase &CB) {
626 return FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: *CB.getCaller());
627}
628
629PreservedAnalyses
630InlineAdvisorAnalysisPrinterPass::run(Module &M, ModuleAnalysisManager &MAM) {
631 const auto *IA = MAM.getCachedResult<InlineAdvisorAnalysis>(IR&: M);
632 if (!IA)
633 OS << "No Inline Advisor\n";
634 else
635 IA->getAdvisor()->print(OS);
636 return PreservedAnalyses::all();
637}
638
639PreservedAnalyses InlineAdvisorAnalysisPrinterPass::run(
640 LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM, LazyCallGraph &CG,
641 CGSCCUpdateResult &UR) {
642 const auto &MAMProxy =
643 AM.getResult<ModuleAnalysisManagerCGSCCProxy>(IR&: InitialC, ExtraArgs&: CG);
644
645 if (InitialC.size() == 0) {
646 OS << "SCC is empty!\n";
647 return PreservedAnalyses::all();
648 }
649 Module &M = *InitialC.begin()->getFunction().getParent();
650 const auto *IA = MAMProxy.getCachedResult<InlineAdvisorAnalysis>(IR&: M);
651 if (!IA)
652 OS << "No Inline Advisor\n";
653 else
654 IA->getAdvisor()->print(OS);
655 return PreservedAnalyses::all();
656}
657