1//===- AlwaysInliner.cpp - Code to inline always_inline functions ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a custom inliner that handles only functions that
10// are marked as "always inline".
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/IPO/AlwaysInliner.h"
15#include "llvm/ADT/SetVector.h"
16#include "llvm/Analysis/AliasAnalysis.h"
17#include "llvm/Analysis/AssumptionCache.h"
18#include "llvm/Analysis/InlineAdvisor.h"
19#include "llvm/Analysis/InlineCost.h"
20#include "llvm/Analysis/OptimizationRemarkEmitter.h"
21#include "llvm/Analysis/ProfileSummaryInfo.h"
22#include "llvm/IR/Module.h"
23#include "llvm/InitializePasses.h"
24#include "llvm/Transforms/Utils/Cloning.h"
25#include "llvm/Transforms/Utils/ModuleUtils.h"
26
27using namespace llvm;
28
29#define DEBUG_TYPE "inline"
30
31namespace {
32
33bool AlwaysInlineImpl(
34 Module &M, bool InsertLifetime, ProfileSummaryInfo &PSI,
35 function_ref<AssumptionCache &(Function &)> GetAssumptionCache,
36 function_ref<AAResults &(Function &)> GetAAR,
37 function_ref<BlockFrequencyInfo &(Function &)> GetBFI) {
38 SmallSetVector<CallBase *, 16> Calls;
39 bool Changed = false;
40 SmallVector<Function *, 16> InlinedComdatFunctions;
41
42 for (Function &F : make_early_inc_range(Range&: M)) {
43 if (F.isPresplitCoroutine())
44 continue;
45
46 if (F.isDeclaration() || !isInlineViable(Callee&: F).isSuccess())
47 continue;
48
49 Calls.clear();
50
51 for (User *U : F.users())
52 if (auto *CB = dyn_cast<CallBase>(Val: U))
53 if (CB->getCalledFunction() == &F &&
54 CB->hasFnAttr(Kind: Attribute::AlwaysInline) &&
55 !CB->getAttributes().hasFnAttr(Kind: Attribute::NoInline))
56 Calls.insert(X: CB);
57
58 for (CallBase *CB : Calls) {
59 Function *Caller = CB->getCaller();
60 OptimizationRemarkEmitter ORE(Caller);
61 DebugLoc DLoc = CB->getDebugLoc();
62 BasicBlock *Block = CB->getParent();
63
64 InlineFunctionInfo IFI(GetAssumptionCache, &PSI,
65 GetBFI ? &GetBFI(*Caller) : nullptr,
66 GetBFI ? &GetBFI(F) : nullptr);
67
68 InlineResult Res = InlineFunction(CB&: *CB, IFI, /*MergeAttributes=*/true,
69 CalleeAAR: &GetAAR(F), InsertLifetime);
70 if (!Res.isSuccess()) {
71 ORE.emit(RemarkBuilder: [&]() {
72 return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block)
73 << "'" << ore::NV("Callee", &F) << "' is not inlined into '"
74 << ore::NV("Caller", Caller)
75 << "': " << ore::NV("Reason", Res.getFailureReason());
76 });
77 continue;
78 }
79
80 emitInlinedIntoBasedOnCost(
81 ORE, DLoc, Block, Callee: F, Caller: *Caller,
82 IC: InlineCost::getAlways(Reason: "always inline attribute"),
83 /*ForProfileContext=*/false, DEBUG_TYPE);
84
85 Changed = true;
86 }
87
88 F.removeDeadConstantUsers();
89 if (F.hasFnAttribute(Kind: Attribute::AlwaysInline) && F.isDefTriviallyDead()) {
90 // Remember to try and delete this function afterward. This allows to call
91 // filterDeadComdatFunctions() only once.
92 if (F.hasComdat()) {
93 InlinedComdatFunctions.push_back(Elt: &F);
94 } else {
95 M.getFunctionList().erase(IT&: F);
96 Changed = true;
97 }
98 }
99 }
100
101 if (!InlinedComdatFunctions.empty()) {
102 // Now we just have the comdat functions. Filter out the ones whose comdats
103 // are not actually dead.
104 filterDeadComdatFunctions(DeadComdatFunctions&: InlinedComdatFunctions);
105 // The remaining functions are actually dead.
106 for (Function *F : InlinedComdatFunctions) {
107 M.getFunctionList().erase(IT: F);
108 Changed = true;
109 }
110 }
111
112 return Changed;
113}
114
115struct AlwaysInlinerLegacyPass : public ModulePass {
116 bool InsertLifetime;
117
118 AlwaysInlinerLegacyPass()
119 : AlwaysInlinerLegacyPass(/*InsertLifetime*/ true) {}
120
121 AlwaysInlinerLegacyPass(bool InsertLifetime)
122 : ModulePass(ID), InsertLifetime(InsertLifetime) {
123 initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry());
124 }
125
126 /// Main run interface method. We override here to avoid calling skipSCC().
127 bool runOnModule(Module &M) override {
128
129 auto &PSI = getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
130 auto GetAAR = [&](Function &F) -> AAResults & {
131 return getAnalysis<AAResultsWrapperPass>(F).getAAResults();
132 };
133 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
134 return getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
135 };
136
137 return AlwaysInlineImpl(M, InsertLifetime, PSI, GetAssumptionCache, GetAAR,
138 /*GetBFI*/ nullptr);
139 }
140
141 static char ID; // Pass identification, replacement for typeid
142
143 void getAnalysisUsage(AnalysisUsage &AU) const override {
144 AU.addRequired<AssumptionCacheTracker>();
145 AU.addRequired<AAResultsWrapperPass>();
146 AU.addRequired<ProfileSummaryInfoWrapperPass>();
147 }
148};
149
150} // namespace
151
152char AlwaysInlinerLegacyPass::ID = 0;
153INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline",
154 "Inliner for always_inline functions", false, false)
155INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
156INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
157INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
158INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline",
159 "Inliner for always_inline functions", false, false)
160
161Pass *llvm::createAlwaysInlinerLegacyPass(bool InsertLifetime) {
162 return new AlwaysInlinerLegacyPass(InsertLifetime);
163}
164
165PreservedAnalyses AlwaysInlinerPass::run(Module &M,
166 ModuleAnalysisManager &MAM) {
167 FunctionAnalysisManager &FAM =
168 MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
169 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
170 return FAM.getResult<AssumptionAnalysis>(IR&: F);
171 };
172 auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
173 return FAM.getResult<BlockFrequencyAnalysis>(IR&: F);
174 };
175 auto GetAAR = [&](Function &F) -> AAResults & {
176 return FAM.getResult<AAManager>(IR&: F);
177 };
178 auto &PSI = MAM.getResult<ProfileSummaryAnalysis>(IR&: M);
179
180 bool Changed = AlwaysInlineImpl(M, InsertLifetime, PSI, GetAssumptionCache,
181 GetAAR, GetBFI);
182
183 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
184}
185