1//===- AlwaysInliner.cpp - Code to inline always_inline functions ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a custom inliner that handles only functions that
10// are marked as "always inline".
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/IPO/AlwaysInliner.h"
15#include "llvm/ADT/SetVector.h"
16#include "llvm/Analysis/AliasAnalysis.h"
17#include "llvm/Analysis/AssumptionCache.h"
18#include "llvm/Analysis/InlineAdvisor.h"
19#include "llvm/Analysis/InlineCost.h"
20#include "llvm/Analysis/OptimizationRemarkEmitter.h"
21#include "llvm/Analysis/ProfileSummaryInfo.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Analysis/TargetTransformInfo.h"
24#include "llvm/IR/Module.h"
25#include "llvm/InitializePasses.h"
26#include "llvm/Transforms/Utils/Cloning.h"
27#include "llvm/Transforms/Utils/ModuleUtils.h"
28
29using namespace llvm;
30
31#define DEBUG_TYPE "inline"
32
33namespace {
34
35bool AlwaysInlineImpl(
36 Module &M, bool InsertLifetime, ProfileSummaryInfo &PSI,
37 FunctionAnalysisManager *FAM,
38 function_ref<AssumptionCache &(Function &)> GetAssumptionCache,
39 function_ref<AAResults &(Function &)> GetAAR,
40 function_ref<TargetTransformInfo &(Function &)> GetTTI,
41 function_ref<const TargetLibraryInfo &(Function &)> GetTLI) {
42 SmallSetVector<CallBase *, 16> Calls;
43 bool Changed = false;
44 SmallVector<Function *, 16> InlinedComdatFunctions;
45 SmallVector<Function *, 4> NeedFlattening;
46
47 auto TryInline = [&](CallBase &CB, Function &Callee,
48 OptimizationRemarkEmitter &ORE, const char *InlineReason,
49 SmallVectorImpl<CallBase *> *NewCallSites =
50 nullptr) -> bool {
51 Function *Caller = CB.getCaller();
52 DebugLoc DLoc = CB.getDebugLoc();
53 BasicBlock *Block = CB.getParent();
54
55 InlineFunctionInfo IFI(GetAssumptionCache, &PSI);
56 InlineResult Res = InlineFunction(CB, IFI, /*MergeAttributes=*/true,
57 CalleeAAR: &GetAAR(Callee), InsertLifetime);
58 if (!Res.isSuccess()) {
59 ORE.emit(RemarkBuilder: [&]() {
60 return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block)
61 << "'" << ore::NV("Callee", &Callee) << "' is not inlined into '"
62 << ore::NV("Caller", Caller)
63 << "': " << ore::NV("Reason", Res.getFailureReason());
64 });
65 return false;
66 }
67
68 emitInlinedIntoBasedOnCost(ORE, DLoc, Block, Callee, Caller: *Caller,
69 IC: InlineCost::getAlways(Reason: InlineReason),
70 /*ForProfileContext=*/false, DEBUG_TYPE);
71 if (FAM)
72 FAM->invalidate(IR&: *Caller, PA: PreservedAnalyses::none());
73 if (NewCallSites)
74 *NewCallSites = std::move(IFI.InlinedCallSites);
75 return true;
76 };
77
78 for (Function &F : make_early_inc_range(Range&: M)) {
79 if (F.hasFnAttribute(Kind: Attribute::Flatten))
80 NeedFlattening.push_back(Elt: &F);
81
82 if (F.isPresplitCoroutine())
83 continue;
84
85 if (F.isDeclaration() || !isInlineViable(Callee&: F).isSuccess())
86 continue;
87
88 Calls.clear();
89
90 for (User *U : F.users())
91 if (auto *CB = dyn_cast<CallBase>(Val: U))
92 if (CB->getCalledFunction() == &F &&
93 CB->hasFnAttr(Kind: Attribute::AlwaysInline) &&
94 !CB->getAttributes().hasFnAttr(Kind: Attribute::NoInline))
95 Calls.insert(X: CB);
96
97 for (CallBase *CB : Calls) {
98 OptimizationRemarkEmitter ORE(CB->getCaller());
99 Changed |= TryInline(*CB, F, ORE, "always inline attribute");
100 }
101
102 F.removeDeadConstantUsers();
103 if (F.hasFnAttribute(Kind: Attribute::AlwaysInline) && F.isDefTriviallyDead()) {
104 if (F.hasComdat()) {
105 InlinedComdatFunctions.push_back(Elt: &F);
106 } else {
107 if (FAM)
108 FAM->clear(IR&: F, Name: F.getName());
109 M.getFunctionList().erase(IT&: F);
110 Changed = true;
111 }
112 }
113 }
114
115 // Flatten functions with the flatten attribute using a local worklist.
116 for (Function *F : NeedFlattening) {
117 SmallVector<std::pair<CallBase *, int>, 16> Worklist;
118 SmallVector<std::pair<Function *, int>, 16> InlineHistory;
119 SmallVector<CallBase *> NewCallSites;
120 OptimizationRemarkEmitter ORE(F);
121
122 // Collect initial calls.
123 for (BasicBlock &BB : *F) {
124 for (Instruction &I : BB) {
125 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
126 Function *Callee = CB->getCalledFunction();
127 if (!Callee || Callee->isDeclaration())
128 continue;
129 Worklist.push_back(Elt: {CB, -1});
130 }
131 }
132 }
133
134 while (!Worklist.empty()) {
135 auto Item = Worklist.pop_back_val();
136 CallBase *CB = Item.first;
137 int InlineHistoryID = Item.second;
138 Function *Callee = CB->getCalledFunction();
139 if (!Callee)
140 continue;
141
142 // Detect recursion.
143 if (Callee == F ||
144 inlineHistoryIncludes(F: Callee, InlineHistoryID, InlineHistory)) {
145 ORE.emit(RemarkBuilder: [&]() {
146 return OptimizationRemarkMissed("inline", "NotInlined",
147 CB->getDebugLoc(), CB->getParent())
148 << "'" << ore::NV("Callee", Callee)
149 << "' is not inlined into '"
150 << ore::NV("Caller", CB->getCaller())
151 << "': recursive call during flattening";
152 });
153 continue;
154 }
155
156 // Use getAttributeBasedInliningDecision for all attribute-based checks
157 // including TTI/TLI compatibility and isInlineViable.
158 TargetTransformInfo &CalleeTTI = GetTTI(*Callee);
159 auto Decision =
160 getAttributeBasedInliningDecision(Call&: *CB, Callee, CalleeTTI, GetTLI);
161 if (!Decision || !Decision->isSuccess())
162 continue;
163
164 if (!TryInline(*CB, *Callee, ORE, "flatten attribute", &NewCallSites))
165 continue;
166
167 Changed = true;
168
169 // Add new call sites from the inlined function to the worklist.
170 if (!NewCallSites.empty()) {
171 int NewHistoryID = InlineHistory.size();
172 InlineHistory.push_back(Elt: {Callee, InlineHistoryID});
173 for (CallBase *NewCB : NewCallSites) {
174 Function *NewCallee = NewCB->getCalledFunction();
175 if (NewCallee && !NewCallee->isDeclaration())
176 Worklist.push_back(Elt: {NewCB, NewHistoryID});
177 }
178 }
179 }
180 }
181
182 if (!InlinedComdatFunctions.empty()) {
183 // Now we just have the comdat functions. Filter out the ones whose comdats
184 // are not actually dead.
185 filterDeadComdatFunctions(DeadComdatFunctions&: InlinedComdatFunctions);
186 // The remaining functions are actually dead.
187 for (Function *F : InlinedComdatFunctions) {
188 if (FAM)
189 FAM->clear(IR&: *F, Name: F->getName());
190 M.getFunctionList().erase(IT: F);
191 Changed = true;
192 }
193 }
194
195 return Changed;
196}
197
198struct AlwaysInlinerLegacyPass : public ModulePass {
199 bool InsertLifetime;
200
201 AlwaysInlinerLegacyPass()
202 : AlwaysInlinerLegacyPass(/*InsertLifetime*/ true) {}
203
204 AlwaysInlinerLegacyPass(bool InsertLifetime)
205 : ModulePass(ID), InsertLifetime(InsertLifetime) {}
206
207 /// Main run interface method.
208 bool runOnModule(Module &M) override {
209
210 auto &PSI = getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
211 auto GetAAR = [&](Function &F) -> AAResults & {
212 return getAnalysis<AAResultsWrapperPass>(F).getAAResults();
213 };
214 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
215 return getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
216 };
217 auto GetTTI = [&](Function &F) -> TargetTransformInfo & {
218 return getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
219 };
220 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
221 return getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
222 };
223
224 return AlwaysInlineImpl(M, InsertLifetime, PSI, /*FAM=*/nullptr,
225 GetAssumptionCache, GetAAR, GetTTI, GetTLI);
226 }
227
228 static char ID; // Pass identification, replacement for typeid
229
230 void getAnalysisUsage(AnalysisUsage &AU) const override {
231 AU.addRequired<AssumptionCacheTracker>();
232 AU.addRequired<AAResultsWrapperPass>();
233 AU.addRequired<ProfileSummaryInfoWrapperPass>();
234 AU.addRequired<TargetLibraryInfoWrapperPass>();
235 AU.addRequired<TargetTransformInfoWrapperPass>();
236 }
237};
238
239} // namespace
240
241char AlwaysInlinerLegacyPass::ID = 0;
242INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline",
243 "Inliner for always_inline functions", false, false)
244INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
245INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
246INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
247INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
248INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
249INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline",
250 "Inliner for always_inline functions", false, false)
251
252Pass *llvm::createAlwaysInlinerLegacyPass(bool InsertLifetime) {
253 return new AlwaysInlinerLegacyPass(InsertLifetime);
254}
255
256PreservedAnalyses AlwaysInlinerPass::run(Module &M,
257 ModuleAnalysisManager &MAM) {
258 FunctionAnalysisManager &FAM =
259 MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
260 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
261 return FAM.getResult<AssumptionAnalysis>(IR&: F);
262 };
263 auto GetAAR = [&](Function &F) -> AAResults & {
264 return FAM.getResult<AAManager>(IR&: F);
265 };
266 auto GetTTI = [&](Function &F) -> TargetTransformInfo & {
267 return FAM.getResult<TargetIRAnalysis>(IR&: F);
268 };
269 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
270 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
271 };
272 auto &PSI = MAM.getResult<ProfileSummaryAnalysis>(IR&: M);
273
274 bool Changed = AlwaysInlineImpl(M, InsertLifetime, PSI, FAM: &FAM,
275 GetAssumptionCache, GetAAR, GetTTI, GetTLI);
276 if (!Changed)
277 return PreservedAnalyses::all();
278
279 PreservedAnalyses PA;
280 // We have already invalidated all analyses on modified functions.
281 PA.preserveSet<AllAnalysesOn<Function>>();
282 return PA;
283}
284