1//===- InlineAdvisor.cpp - analysis pass implementation -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements InlineAdvisorAnalysis and DefaultInlineAdvisor, and
10// related types.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Analysis/InlineAdvisor.h"
15#include "llvm/ADT/Statistic.h"
16#include "llvm/ADT/StringExtras.h"
17#include "llvm/Analysis/AssumptionCache.h"
18#include "llvm/Analysis/EphemeralValuesCache.h"
19#include "llvm/Analysis/IR2Vec.h"
20#include "llvm/Analysis/InlineCost.h"
21#include "llvm/Analysis/OptimizationRemarkEmitter.h"
22#include "llvm/Analysis/ProfileSummaryInfo.h"
23#include "llvm/Analysis/ReplayInlineAdvisor.h"
24#include "llvm/Analysis/TargetLibraryInfo.h"
25#include "llvm/Analysis/TargetTransformInfo.h"
26#include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h"
27#include "llvm/IR/DebugInfoMetadata.h"
28#include "llvm/IR/Module.h"
29#include "llvm/IR/PassManager.h"
30#include "llvm/Support/CommandLine.h"
31#include "llvm/Support/raw_ostream.h"
32
33using namespace llvm;
34#define DEBUG_TYPE "inline"
35#ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL
36#define LLVM_HAVE_TF_AOT
37#endif
38
39// This weirdly named statistic tracks the number of times that, when attempting
40// to inline a function A into B, we analyze the callers of B in order to see
41// if those would be more profitable and blocked inline steps.
42STATISTIC(NumCallerCallersAnalyzed, "Number of caller-callers analyzed");
43
44/// Flag to add inline messages as callsite attributes 'inline-remark'.
45static cl::opt<bool>
46 InlineRemarkAttribute("inline-remark-attribute", cl::init(Val: false),
47 cl::Hidden,
48 cl::desc("Enable adding inline-remark attribute to"
49 " callsites processed by inliner but decided"
50 " to be not inlined"));
51
52static cl::opt<bool> EnableInlineDeferral("inline-deferral", cl::init(Val: false),
53 cl::Hidden,
54 cl::desc("Enable deferred inlining"));
55
56// An integer used to limit the cost of inline deferral. The default negative
57// number tells shouldBeDeferred to only take the secondary cost into account.
58static cl::opt<int>
59 InlineDeferralScale("inline-deferral-scale",
60 cl::desc("Scale to limit the cost of inline deferral"),
61 cl::init(Val: 2), cl::Hidden);
62
63static cl::opt<bool>
64 AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(Val: false),
65 cl::desc("If true, annotate inline advisor remarks "
66 "with LTO and pass information."));
67
68// This flag is used to enable IR2Vec embeddings in the ML inliner; Only valid
69// with ML inliner. The vocab file is used to initialize the embeddings.
70static cl::opt<std::string> IR2VecVocabFile(
71 "ml-inliner-ir2vec-vocab-file", cl::Hidden,
72 cl::desc("Vocab file for IR2Vec; Setting this enables "
73 "configuring the model to use IR2Vec embeddings."));
74
75namespace llvm {
76extern cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats;
77} // namespace llvm
78
79namespace {
80using namespace llvm::ore;
81class MandatoryInlineAdvice : public InlineAdvice {
82public:
83 MandatoryInlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
84 OptimizationRemarkEmitter &ORE,
85 bool IsInliningMandatory)
86 : InlineAdvice(Advisor, CB, ORE, IsInliningMandatory) {}
87
88private:
89 void recordInliningWithCalleeDeletedImpl() override { recordInliningImpl(); }
90
91 void recordInliningImpl() override {
92 if (IsInliningRecommended)
93 emitInlinedInto(ORE, DLoc, Block, Callee: *Callee, Caller: *Caller, IsMandatory: IsInliningRecommended,
94 ExtraContext: [&](OptimizationRemark &Remark) {
95 Remark << ": always inline attribute";
96 });
97 }
98
99 void recordUnsuccessfulInliningImpl(const InlineResult &Result) override {
100 if (IsInliningRecommended)
101 ORE.emit(RemarkBuilder: [&]() {
102 return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(),
103 "NotInlined", DLoc, Block)
104 << "'" << NV("Callee", Callee) << "' is not AlwaysInline into '"
105 << NV("Caller", Caller)
106 << "': " << NV("Reason", Result.getFailureReason());
107 });
108 }
109
110 void recordUnattemptedInliningImpl() override {
111 assert(!IsInliningRecommended && "Expected to attempt inlining");
112 }
113};
114} // namespace
115
116void DefaultInlineAdvice::recordUnsuccessfulInliningImpl(
117 const InlineResult &Result) {
118 using namespace ore;
119 llvm::setInlineRemark(CB&: *OriginalCB, Message: std::string(Result.getFailureReason()) +
120 "; " + inlineCostStr(IC: *OIC));
121 ORE.emit(RemarkBuilder: [&]() {
122 return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(),
123 "NotInlined", DLoc, Block)
124 << "'" << NV("Callee", Callee) << "' is not inlined into '"
125 << NV("Caller", Caller)
126 << "': " << NV("Reason", Result.getFailureReason());
127 });
128}
129
130void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() {
131 if (EmitRemarks)
132 emitInlinedIntoBasedOnCost(ORE, DLoc, Block, Callee: *Callee, Caller: *Caller, IC: *OIC,
133 /* ForProfileContext= */ false,
134 PassName: Advisor->getAnnotatedInlinePassName());
135}
136
137void DefaultInlineAdvice::recordInliningImpl() {
138 if (EmitRemarks)
139 emitInlinedIntoBasedOnCost(ORE, DLoc, Block, Callee: *Callee, Caller: *Caller, IC: *OIC,
140 /* ForProfileContext= */ false,
141 PassName: Advisor->getAnnotatedInlinePassName());
142}
143
144std::optional<llvm::InlineCost> static getDefaultInlineAdvice(
145 CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params) {
146 Function &Caller = *CB.getCaller();
147 ProfileSummaryInfo *PSI =
148 FAM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: Caller)
149 .getCachedResult<ProfileSummaryAnalysis>(
150 IR&: *CB.getParent()->getParent()->getParent());
151
152 auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: Caller);
153 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
154 return FAM.getResult<AssumptionAnalysis>(IR&: F);
155 };
156 auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
157 return FAM.getResult<BlockFrequencyAnalysis>(IR&: F);
158 };
159 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
160 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
161 };
162 auto GetEphValuesCache =
163 [&](Function &F) -> EphemeralValuesAnalysis::Result & {
164 return FAM.getResult<EphemeralValuesAnalysis>(IR&: F);
165 };
166
167 Function &Callee = *CB.getCalledFunction();
168 auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(IR&: Callee);
169 auto GetInlineCost = [&](CallBase &CB) {
170 bool RemarksEnabled =
171 Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
172 DEBUG_TYPE);
173 return getInlineCost(Call&: CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
174 GetBFI, PSI, ORE: RemarksEnabled ? &ORE : nullptr,
175 GetEphValuesCache);
176 };
177 return llvm::shouldInline(
178 CB, CalleeTTI, GetInlineCost, ORE,
179 EnableDeferral: Params.EnableDeferral.value_or(u&: EnableInlineDeferral));
180}
181
182std::unique_ptr<InlineAdvice>
183DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) {
184 auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
185 return std::make_unique<DefaultInlineAdvice>(
186 args: this, args&: CB, args&: OIC,
187 args&: FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: *CB.getCaller()));
188}
189
190InlineAdvice::InlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
191 OptimizationRemarkEmitter &ORE,
192 bool IsInliningRecommended)
193 : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()),
194 DLoc(CB.getDebugLoc()), Block(CB.getParent()), ORE(ORE),
195 IsInliningRecommended(IsInliningRecommended) {}
196
197void InlineAdvice::recordInlineStatsIfNeeded() {
198 if (Advisor->ImportedFunctionsStats)
199 Advisor->ImportedFunctionsStats->recordInline(Caller: *Caller, Callee: *Callee);
200}
201
202void InlineAdvice::recordInlining() {
203 markRecorded();
204 recordInlineStatsIfNeeded();
205 recordInliningImpl();
206}
207
208void InlineAdvice::recordInliningWithCalleeDeleted() {
209 markRecorded();
210 recordInlineStatsIfNeeded();
211 recordInliningWithCalleeDeletedImpl();
212}
213
214AnalysisKey InlineAdvisorAnalysis::Key;
215AnalysisKey PluginInlineAdvisorAnalysis::Key;
216
217bool InlineAdvisorAnalysis::initializeIR2VecVocabIfRequested(
218 Module &M, ModuleAnalysisManager &MAM) {
219 if (!IR2VecVocabFile.empty()) {
220 auto IR2VecVocabResult = MAM.getResult<IR2VecVocabAnalysis>(IR&: M);
221 if (!IR2VecVocabResult.isValid()) {
222 M.getContext().emitError(ErrorStr: "Failed to load IR2Vec vocabulary");
223 return false;
224 }
225 }
226 // No vocab file specified is OK; We just don't use IR2Vec
227 // embeddings.
228 return true;
229}
230
231bool InlineAdvisorAnalysis::Result::tryCreate(
232 InlineParams Params, InliningAdvisorMode Mode,
233 const ReplayInlinerSettings &ReplaySettings, InlineContext IC) {
234 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
235 if (MAM.isPassRegistered<PluginInlineAdvisorAnalysis>()) {
236 auto &DA = MAM.getResult<PluginInlineAdvisorAnalysis>(IR&: M);
237 Advisor.reset(p: DA.Factory(M, FAM, Params, IC));
238 return !!Advisor;
239 }
240 auto GetDefaultAdvice = [&FAM, Params](CallBase &CB) {
241 auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
242 return OIC.has_value();
243 };
244 switch (Mode) {
245 case InliningAdvisorMode::Default:
246 LLVM_DEBUG(dbgs() << "Using default inliner heuristic.\n");
247 Advisor.reset(p: new DefaultInlineAdvisor(M, FAM, Params, IC));
248 // Restrict replay to default advisor, ML advisors are stateful so
249 // replay will need augmentations to interleave with them correctly.
250 if (!ReplaySettings.ReplayFile.empty()) {
251 Advisor = llvm::getReplayInlineAdvisor(M, FAM, Context&: M.getContext(),
252 OriginalAdvisor: std::move(Advisor), ReplaySettings,
253 /* EmitRemarks =*/true, IC);
254 }
255 break;
256 // Run IR2VecVocabAnalysis once per module to get the vocabulary.
257 // We run it here because it is immutable and we want to avoid running it
258 // multiple times.
259 case InliningAdvisorMode::Development:
260#ifdef LLVM_HAVE_TFLITE
261 LLVM_DEBUG(dbgs() << "Using development-mode inliner policy.\n");
262 if (!InlineAdvisorAnalysis::initializeIR2VecVocabIfRequested(M, MAM))
263 return false;
264 Advisor = llvm::getDevelopmentModeAdvisor(M, MAM, GetDefaultAdvice);
265#endif
266 break;
267 case InliningAdvisorMode::Release:
268 LLVM_DEBUG(dbgs() << "Using release-mode inliner policy.\n");
269 if (!InlineAdvisorAnalysis::initializeIR2VecVocabIfRequested(M, MAM))
270 return false;
271 Advisor = llvm::getReleaseModeAdvisor(M, MAM, GetDefaultAdvice);
272 break;
273 }
274
275 return !!Advisor;
276}
277
278/// Return true if inlining of CB can block the caller from being
279/// inlined which is proved to be more beneficial. \p IC is the
280/// estimated inline cost associated with callsite \p CB.
281/// \p TotalSecondaryCost will be set to the estimated cost of inlining the
282/// caller if \p CB is suppressed for inlining.
283static bool
284shouldBeDeferred(Function *Caller, TargetTransformInfo &CalleeTTI,
285 InlineCost IC, int &TotalSecondaryCost,
286 function_ref<InlineCost(CallBase &CB)> GetInlineCost) {
287 // For now we only handle local or inline functions.
288 if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
289 return false;
290 // If the cost of inlining CB is non-positive, it is not going to prevent the
291 // caller from being inlined into its callers and hence we don't need to
292 // defer.
293 if (IC.getCost() <= 0)
294 return false;
295 // Try to detect the case where the current inlining candidate caller (call
296 // it B) is a static or linkonce-ODR function and is an inlining candidate
297 // elsewhere, and the current candidate callee (call it C) is large enough
298 // that inlining it into B would make B too big to inline later. In these
299 // circumstances it may be best not to inline C into B, but to inline B into
300 // its callers.
301 //
302 // This only applies to static and linkonce-ODR functions because those are
303 // expected to be available for inlining in the translation units where they
304 // are used. Thus we will always have the opportunity to make local inlining
305 // decisions. Importantly the linkonce-ODR linkage covers inline functions
306 // and templates in C++.
307 //
308 // FIXME: All of this logic should be sunk into getInlineCost. It relies on
309 // the internal implementation of the inline cost metrics rather than
310 // treating them as truly abstract units etc.
311 TotalSecondaryCost = 0;
312 // The candidate cost to be imposed upon the current function.
313 int CandidateCost = IC.getCost() - 1;
314 // If the caller has local linkage and can be inlined to all its callers, we
315 // can apply a huge negative bonus to TotalSecondaryCost.
316 bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse();
317 // This bool tracks what happens if we DO inline C into B.
318 bool InliningPreventsSomeOuterInline = false;
319 unsigned NumCallerUsers = 0;
320 for (User *U : Caller->users()) {
321 CallBase *CS2 = dyn_cast<CallBase>(Val: U);
322
323 // If this isn't a call to Caller (it could be some other sort
324 // of reference) skip it. Such references will prevent the caller
325 // from being removed.
326 if (!CS2 || CS2->getCalledFunction() != Caller) {
327 ApplyLastCallBonus = false;
328 continue;
329 }
330
331 InlineCost IC2 = GetInlineCost(*CS2);
332 ++NumCallerCallersAnalyzed;
333 if (!IC2) {
334 ApplyLastCallBonus = false;
335 continue;
336 }
337 if (IC2.isAlways())
338 continue;
339
340 // See if inlining of the original callsite would erase the cost delta of
341 // this callsite. We subtract off the penalty for the call instruction,
342 // which we would be deleting.
343 if (IC2.getCostDelta() <= CandidateCost) {
344 InliningPreventsSomeOuterInline = true;
345 TotalSecondaryCost += IC2.getCost();
346 NumCallerUsers++;
347 }
348 }
349
350 if (!InliningPreventsSomeOuterInline)
351 return false;
352
353 // If all outer calls to Caller would get inlined, the cost for the last
354 // one is set very low by getInlineCost, in anticipation that Caller will
355 // be removed entirely. We did not account for this above unless there
356 // is only one caller of Caller.
357 if (ApplyLastCallBonus)
358 TotalSecondaryCost -= CalleeTTI.getInliningLastCallToStaticBonus();
359
360 // If InlineDeferralScale is negative, then ignore the cost of primary
361 // inlining -- IC.getCost() multiplied by the number of callers to Caller.
362 if (InlineDeferralScale < 0)
363 return TotalSecondaryCost < IC.getCost();
364
365 int TotalCost = TotalSecondaryCost + IC.getCost() * NumCallerUsers;
366 int Allowance = IC.getCost() * InlineDeferralScale;
367 return TotalCost < Allowance;
368}
369
370namespace llvm {
371static raw_ostream &operator<<(raw_ostream &R, const ore::NV &Arg) {
372 return R << Arg.Val;
373}
374
375template <class RemarkT>
376decltype(auto) operator<<(RemarkT &&R, const InlineCost &IC) {
377 using namespace ore;
378 if (IC.isAlways()) {
379 R << "(cost=always)";
380 } else if (IC.isNever()) {
381 R << "(cost=never)";
382 } else {
383 R << "(cost=" << ore::NV("Cost", IC.getCost())
384 << ", threshold=" << ore::NV("Threshold", IC.getThreshold()) << ")";
385 }
386 if (const char *Reason = IC.getReason())
387 R << ": " << ore::NV("Reason", Reason);
388 return std::forward<RemarkT>(R);
389}
390} // namespace llvm
391
392std::string llvm::inlineCostStr(const InlineCost &IC) {
393 std::string Buffer;
394 raw_string_ostream Remark(Buffer);
395 Remark << IC;
396 return Remark.str();
397}
398
399void llvm::setInlineRemark(CallBase &CB, StringRef Message) {
400 if (!InlineRemarkAttribute)
401 return;
402
403 Attribute Attr = Attribute::get(Context&: CB.getContext(), Kind: "inline-remark", Val: Message);
404 CB.addFnAttr(Attr);
405}
406
407/// Return the cost only if the inliner should attempt to inline at the given
408/// CallSite. If we return the cost, we will emit an optimisation remark later
409/// using that cost, so we won't do so from this function. Return std::nullopt
410/// if inlining should not be attempted.
411std::optional<InlineCost>
412llvm::shouldInline(CallBase &CB, TargetTransformInfo &CalleeTTI,
413 function_ref<InlineCost(CallBase &CB)> GetInlineCost,
414 OptimizationRemarkEmitter &ORE, bool EnableDeferral) {
415 using namespace ore;
416
417 InlineCost IC = GetInlineCost(CB);
418 Instruction *Call = &CB;
419 Function *Callee = CB.getCalledFunction();
420 Function *Caller = CB.getCaller();
421
422 if (IC.isAlways()) {
423 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC)
424 << ", Call: " << CB << "\n");
425 return IC;
426 }
427
428 if (!IC) {
429 LLVM_DEBUG(dbgs() << " NOT Inlining " << inlineCostStr(IC)
430 << ", Call: " << CB << "\n");
431 if (IC.isNever()) {
432 ORE.emit(RemarkBuilder: [&]() {
433 return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", Call)
434 << "'" << NV("Callee", Callee) << "' not inlined into '"
435 << NV("Caller", Caller)
436 << "' because it should never be inlined " << IC;
437 });
438 } else {
439 ORE.emit(RemarkBuilder: [&]() {
440 return OptimizationRemarkMissed(DEBUG_TYPE, "TooCostly", Call)
441 << "'" << NV("Callee", Callee) << "' not inlined into '"
442 << NV("Caller", Caller) << "' because too costly to inline "
443 << IC;
444 });
445 }
446 setInlineRemark(CB, Message: inlineCostStr(IC));
447 return std::nullopt;
448 }
449
450 int TotalSecondaryCost = 0;
451 if (EnableDeferral && shouldBeDeferred(Caller, CalleeTTI, IC,
452 TotalSecondaryCost, GetInlineCost)) {
453 LLVM_DEBUG(dbgs() << " NOT Inlining: " << CB
454 << " Cost = " << IC.getCost()
455 << ", outer Cost = " << TotalSecondaryCost << '\n');
456 ORE.emit(RemarkBuilder: [&]() {
457 return OptimizationRemarkMissed(DEBUG_TYPE, "IncreaseCostInOtherContexts",
458 Call)
459 << "Not inlining. Cost of inlining '" << NV("Callee", Callee)
460 << "' increases the cost of inlining '" << NV("Caller", Caller)
461 << "' in other contexts";
462 });
463 setInlineRemark(CB, Message: "deferred");
464 return std::nullopt;
465 }
466
467 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) << ", Call: " << CB
468 << '\n');
469 return IC;
470}
471
472std::string llvm::formatCallSiteLocation(DebugLoc DLoc,
473 const CallSiteFormat &Format) {
474 std::string Buffer;
475 raw_string_ostream CallSiteLoc(Buffer);
476 bool First = true;
477 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
478 if (!First)
479 CallSiteLoc << " @ ";
480 // Note that negative line offset is actually possible, but we use
481 // unsigned int to match line offset representation in remarks so
482 // it's directly consumable by relay advisor.
483 uint32_t Offset =
484 DIL->getLine() - DIL->getScope()->getSubprogram()->getLine();
485 uint32_t Discriminator = DIL->getBaseDiscriminator();
486 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
487 if (Name.empty())
488 Name = DIL->getScope()->getSubprogram()->getName();
489 CallSiteLoc << Name.str() << ":" << llvm::utostr(X: Offset);
490 if (Format.outputColumn())
491 CallSiteLoc << ":" << llvm::utostr(X: DIL->getColumn());
492 if (Format.outputDiscriminator() && Discriminator)
493 CallSiteLoc << "." << llvm::utostr(X: Discriminator);
494 First = false;
495 }
496
497 return CallSiteLoc.str();
498}
499
500void llvm::addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc) {
501 if (!DLoc) {
502 return;
503 }
504
505 bool First = true;
506 Remark << " at callsite ";
507 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
508 if (!First)
509 Remark << " @ ";
510 unsigned int Offset = DIL->getLine();
511 Offset -= DIL->getScope()->getSubprogram()->getLine();
512 unsigned int Discriminator = DIL->getBaseDiscriminator();
513 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
514 if (Name.empty())
515 Name = DIL->getScope()->getSubprogram()->getName();
516 Remark << Name << ":" << ore::NV("Line", Offset) << ":"
517 << ore::NV("Column", DIL->getColumn());
518 if (Discriminator)
519 Remark << "." << ore::NV("Disc", Discriminator);
520 First = false;
521 }
522
523 Remark << ";";
524}
525
526void llvm::emitInlinedInto(
527 OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block,
528 const Function &Callee, const Function &Caller, bool AlwaysInline,
529 function_ref<void(OptimizationRemark &)> ExtraContext,
530 const char *PassName) {
531 ORE.emit(RemarkBuilder: [&]() {
532 StringRef RemarkName = AlwaysInline ? "AlwaysInline" : "Inlined";
533 OptimizationRemark Remark(PassName ? PassName : DEBUG_TYPE, RemarkName,
534 DLoc, Block);
535 Remark << "'" << ore::NV("Callee", &Callee) << "' inlined into '"
536 << ore::NV("Caller", &Caller) << "'";
537 if (ExtraContext)
538 ExtraContext(Remark);
539 addLocationToRemarks(Remark, DLoc);
540 return Remark;
541 });
542}
543
544void llvm::emitInlinedIntoBasedOnCost(
545 OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block,
546 const Function &Callee, const Function &Caller, const InlineCost &IC,
547 bool ForProfileContext, const char *PassName) {
548 llvm::emitInlinedInto(
549 ORE, DLoc, Block, Callee, Caller, AlwaysInline: IC.isAlways(),
550 ExtraContext: [&](OptimizationRemark &Remark) {
551 if (ForProfileContext)
552 Remark << " to match profiling context";
553 Remark << " with " << IC;
554 },
555 PassName);
556}
557
558InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM,
559 std::optional<InlineContext> IC)
560 : M(M), FAM(FAM), IC(IC),
561 AnnotatedInlinePassName((IC && AnnotateInlinePhase)
562 ? llvm::AnnotateInlinePassName(IC: *IC)
563 : DEBUG_TYPE) {
564 if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) {
565 ImportedFunctionsStats =
566 std::make_unique<ImportedFunctionsInliningStatistics>();
567 ImportedFunctionsStats->setModuleInfo(M);
568 }
569}
570
571InlineAdvisor::~InlineAdvisor() {
572 if (ImportedFunctionsStats) {
573 assert(InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No);
574 ImportedFunctionsStats->dump(Verbose: InlinerFunctionImportStats ==
575 InlinerFunctionImportStatsOpts::Verbose);
576 }
577}
578
579std::unique_ptr<InlineAdvice> InlineAdvisor::getMandatoryAdvice(CallBase &CB,
580 bool Advice) {
581 return std::make_unique<MandatoryInlineAdvice>(args: this, args&: CB, args&: getCallerORE(CB),
582 args&: Advice);
583}
584
585static inline const char *getLTOPhase(ThinOrFullLTOPhase LTOPhase) {
586 switch (LTOPhase) {
587 case (ThinOrFullLTOPhase::None):
588 return "main";
589 case (ThinOrFullLTOPhase::ThinLTOPreLink):
590 case (ThinOrFullLTOPhase::FullLTOPreLink):
591 return "prelink";
592 case (ThinOrFullLTOPhase::ThinLTOPostLink):
593 case (ThinOrFullLTOPhase::FullLTOPostLink):
594 return "postlink";
595 }
596 llvm_unreachable("unreachable");
597}
598
599static inline const char *getInlineAdvisorContext(InlinePass IP) {
600 switch (IP) {
601 case (InlinePass::AlwaysInliner):
602 return "always-inline";
603 case (InlinePass::CGSCCInliner):
604 return "cgscc-inline";
605 case (InlinePass::EarlyInliner):
606 return "early-inline";
607 case (InlinePass::MLInliner):
608 return "ml-inline";
609 case (InlinePass::ModuleInliner):
610 return "module-inline";
611 case (InlinePass::ReplayCGSCCInliner):
612 return "replay-cgscc-inline";
613 case (InlinePass::ReplaySampleProfileInliner):
614 return "replay-sample-profile-inline";
615 case (InlinePass::SampleProfileInliner):
616 return "sample-profile-inline";
617 }
618
619 llvm_unreachable("unreachable");
620}
621
622std::string llvm::AnnotateInlinePassName(InlineContext IC) {
623 return std::string(getLTOPhase(LTOPhase: IC.LTOPhase)) + "-" +
624 std::string(getInlineAdvisorContext(IP: IC.Pass));
625}
626
627InlineAdvisor::MandatoryInliningKind
628InlineAdvisor::getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM,
629 OptimizationRemarkEmitter &ORE) {
630 auto &Callee = *CB.getCalledFunction();
631
632 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
633 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
634 };
635
636 auto &TIR = FAM.getResult<TargetIRAnalysis>(IR&: Callee);
637
638 auto TrivialDecision =
639 llvm::getAttributeBasedInliningDecision(Call&: CB, Callee: &Callee, CalleeTTI&: TIR, GetTLI);
640
641 if (TrivialDecision) {
642 if (TrivialDecision->isSuccess())
643 return MandatoryInliningKind::Always;
644 else
645 return MandatoryInliningKind::Never;
646 }
647 return MandatoryInliningKind::NotMandatory;
648}
649
650std::unique_ptr<InlineAdvice> InlineAdvisor::getAdvice(CallBase &CB,
651 bool MandatoryOnly) {
652 if (!MandatoryOnly)
653 return getAdviceImpl(CB);
654 bool Advice = CB.getCaller() != CB.getCalledFunction() &&
655 MandatoryInliningKind::Always ==
656 getMandatoryKind(CB, FAM, ORE&: getCallerORE(CB));
657 return getMandatoryAdvice(CB, Advice);
658}
659
660OptimizationRemarkEmitter &InlineAdvisor::getCallerORE(CallBase &CB) {
661 return FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: *CB.getCaller());
662}
663
664PreservedAnalyses
665InlineAdvisorAnalysisPrinterPass::run(Module &M, ModuleAnalysisManager &MAM) {
666 const auto *IA = MAM.getCachedResult<InlineAdvisorAnalysis>(IR&: M);
667 if (!IA)
668 OS << "No Inline Advisor\n";
669 else
670 IA->getAdvisor()->print(OS);
671 return PreservedAnalyses::all();
672}
673
674PreservedAnalyses InlineAdvisorAnalysisPrinterPass::run(
675 LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM, LazyCallGraph &CG,
676 CGSCCUpdateResult &UR) {
677 const auto &MAMProxy =
678 AM.getResult<ModuleAnalysisManagerCGSCCProxy>(IR&: InitialC, ExtraArgs&: CG);
679
680 if (InitialC.size() == 0) {
681 OS << "SCC is empty!\n";
682 return PreservedAnalyses::all();
683 }
684 Module &M = *InitialC.begin()->getFunction().getParent();
685 const auto *IA = MAMProxy.getCachedResult<InlineAdvisorAnalysis>(IR&: M);
686 if (!IA)
687 OS << "No Inline Advisor\n";
688 else
689 IA->getAdvisor()->print(OS);
690 return PreservedAnalyses::all();
691}
692