1//===- AlwaysInliner.cpp - Code to inline always_inline functions ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a custom inliner that handles only functions that
10// are marked as "always inline".
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/IPO/AlwaysInliner.h"
15#include "llvm/ADT/SetVector.h"
16#include "llvm/Analysis/AliasAnalysis.h"
17#include "llvm/Analysis/AssumptionCache.h"
18#include "llvm/Analysis/InlineAdvisor.h"
19#include "llvm/Analysis/InlineCost.h"
20#include "llvm/Analysis/OptimizationRemarkEmitter.h"
21#include "llvm/Analysis/ProfileSummaryInfo.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Analysis/TargetTransformInfo.h"
24#include "llvm/IR/DiagnosticInfo.h"
25#include "llvm/IR/Module.h"
26#include "llvm/InitializePasses.h"
27#include "llvm/Transforms/Utils/Cloning.h"
28#include "llvm/Transforms/Utils/ModuleUtils.h"
29
30using namespace llvm;
31
32#define DEBUG_TYPE "inline"
33
34namespace {
35
36bool AlwaysInlineImpl(
37 Module &M, bool InsertLifetime, ProfileSummaryInfo &PSI,
38 FunctionAnalysisManager *FAM,
39 function_ref<AssumptionCache &(Function &)> GetAssumptionCache,
40 function_ref<AAResults &(Function &)> GetAAR,
41 function_ref<TargetTransformInfo &(Function &)> GetTTI,
42 function_ref<const TargetLibraryInfo &(Function &)> GetTLI) {
43 SmallSetVector<CallBase *, 16> Calls;
44 bool Changed = false;
45 SmallVector<Function *, 16> InlinedComdatFunctions;
46 SmallVector<Function *, 4> NeedFlattening;
47
48 auto TryInline = [&](CallBase &CB, Function &Callee,
49 OptimizationRemarkEmitter &ORE, const char *InlineReason,
50 SmallVectorImpl<CallBase *> *NewCallSites =
51 nullptr) -> bool {
52 Function *Caller = CB.getCaller();
53 DebugLoc DLoc = CB.getDebugLoc();
54 BasicBlock *Block = CB.getParent();
55
56 TargetTransformInfo &CalleeTTI = GetTTI(Callee);
57 std::optional<InlineResult> CanInlineWithAttributes =
58 getAttributeBasedInliningDecision(Call&: CB, Callee: &Callee, CalleeTTI, GetTLI);
59 if (!CanInlineWithAttributes || !CanInlineWithAttributes->isSuccess()) {
60 ORE.emit(RemarkBuilder: [&]() {
61 return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block)
62 << "'" << ore::NV("Callee", &Callee) << ", is not inlined into"
63 << ore::NV("Caller", Caller) << "': "
64 << ore::NV("Reason",
65 CanInlineWithAttributes.has_value()
66 ? CanInlineWithAttributes->getFailureReason()
67 : "due to incompatible function attributes");
68 });
69 return false;
70 }
71
72 InlineFunctionInfo IFI(GetAssumptionCache, &PSI);
73 InlineResult Res = InlineFunction(
74 CB, IFI, /*MergeAttributes=*/true, CalleeAAR: &GetAAR(Callee), InsertLifetime,
75 /*TrackInlineHistory=*/NewCallSites != nullptr);
76 if (!Res.isSuccess()) {
77 ORE.emit(RemarkBuilder: [&]() {
78 return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block)
79 << "'" << ore::NV("Callee", &Callee) << "' is not inlined into '"
80 << ore::NV("Caller", Caller)
81 << "': " << ore::NV("Reason", Res.getFailureReason());
82 });
83 return false;
84 }
85
86 emitInlinedIntoBasedOnCost(ORE, DLoc, Block, Callee, Caller: *Caller,
87 IC: InlineCost::getAlways(Reason: InlineReason),
88 /*ForProfileContext=*/false, DEBUG_TYPE);
89 if (FAM)
90 FAM->invalidate(IR&: *Caller, PA: PreservedAnalyses::none());
91 if (NewCallSites)
92 *NewCallSites = std::move(IFI.InlinedCallSites);
93 return true;
94 };
95
96 for (Function &F : make_early_inc_range(Range&: M)) {
97 if (F.hasFnAttribute(Kind: Attribute::Flatten))
98 NeedFlattening.push_back(Elt: &F);
99
100 if (F.isPresplitCoroutine())
101 continue;
102
103 if (F.isDeclaration() || !isInlineViable(Callee&: F).isSuccess())
104 continue;
105
106 Calls.clear();
107
108 for (User *U : F.users())
109 if (auto *CB = dyn_cast<CallBase>(Val: U))
110 if (CB->getCalledFunction() == &F &&
111 CB->hasFnAttr(Kind: Attribute::AlwaysInline) &&
112 !CB->getAttributes().hasFnAttr(Kind: Attribute::NoInline))
113 Calls.insert(X: CB);
114
115 for (CallBase *CB : Calls) {
116 OptimizationRemarkEmitter ORE(CB->getCaller());
117 Changed |= TryInline(*CB, F, ORE, "always inline attribute");
118 }
119
120 F.removeDeadConstantUsers();
121 if (F.hasFnAttribute(Kind: Attribute::AlwaysInline) && F.isDefTriviallyDead()) {
122 if (F.hasComdat()) {
123 InlinedComdatFunctions.push_back(Elt: &F);
124 } else {
125 if (FAM)
126 FAM->clear(IR&: F, Name: F.getName());
127 M.getFunctionList().erase(IT&: F);
128 Changed = true;
129 }
130 }
131 }
132
133 // Flatten functions with the flatten attribute using a local worklist.
134 for (Function *F : NeedFlattening) {
135 SmallVector<std::pair<CallBase *, int>, 16> Worklist;
136 SmallVector<std::pair<Function *, int>, 16> InlineHistory;
137 SmallVector<CallBase *> NewCallSites;
138 OptimizationRemarkEmitter ORE(F);
139
140 // Collect initial calls.
141 for (BasicBlock &BB : *F) {
142 for (Instruction &I : BB) {
143 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
144 Function *Callee = CB->getCalledFunction();
145 if (!Callee || Callee->isDeclaration())
146 continue;
147 Worklist.push_back(Elt: {CB, -1});
148 }
149 }
150 }
151
152 while (!Worklist.empty()) {
153 auto Item = Worklist.pop_back_val();
154 CallBase *CB = Item.first;
155 int InlineHistoryID = Item.second;
156 Function *Callee = CB->getCalledFunction();
157 if (!Callee)
158 continue;
159
160 // Detect recursion.
161 if (Callee == F) {
162 ORE.emit(RemarkBuilder: [&]() {
163 return OptimizationRemarkMissed("inline", "NotInlined",
164 CB->getDebugLoc(), CB->getParent())
165 << "'" << ore::NV("Callee", Callee)
166 << "' is not inlined into '"
167 << ore::NV("Caller", CB->getCaller())
168 << "': recursive call during flattening";
169 });
170 continue;
171 }
172
173 // Use getAttributeBasedInliningDecision for all attribute-based checks
174 // including TTI/TLI compatibility and isInlineViable.
175 TargetTransformInfo &CalleeTTI = GetTTI(*Callee);
176 auto Decision =
177 getAttributeBasedInliningDecision(Call&: *CB, Callee, CalleeTTI, GetTLI);
178 if (!Decision || !Decision->isSuccess())
179 continue;
180
181 if (!TryInline(*CB, *Callee, ORE, "flatten attribute", &NewCallSites))
182 continue;
183
184 Changed = true;
185
186 // Add new call sites from the inlined function to the worklist.
187 if (!NewCallSites.empty()) {
188 int NewHistoryID = InlineHistory.size();
189 InlineHistory.push_back(Elt: {Callee, InlineHistoryID});
190 for (CallBase *NewCB : NewCallSites) {
191 Function *NewCallee = NewCB->getCalledFunction();
192 if (NewCallee && !NewCallee->isDeclaration())
193 Worklist.push_back(Elt: {NewCB, NewHistoryID});
194 }
195 }
196 }
197 }
198
199 if (!InlinedComdatFunctions.empty()) {
200 // Now we just have the comdat functions. Filter out the ones whose comdats
201 // are not actually dead.
202 filterDeadComdatFunctions(DeadComdatFunctions&: InlinedComdatFunctions);
203 // The remaining functions are actually dead.
204 for (Function *F : InlinedComdatFunctions) {
205 if (FAM)
206 FAM->clear(IR&: *F, Name: F->getName());
207 M.getFunctionList().erase(IT: F);
208 Changed = true;
209 }
210 }
211
212 return Changed;
213}
214
215struct AlwaysInlinerLegacyPass : public ModulePass {
216 bool InsertLifetime;
217
218 AlwaysInlinerLegacyPass()
219 : AlwaysInlinerLegacyPass(/*InsertLifetime*/ true) {}
220
221 AlwaysInlinerLegacyPass(bool InsertLifetime)
222 : ModulePass(ID), InsertLifetime(InsertLifetime) {}
223
224 /// Main run interface method.
225 bool runOnModule(Module &M) override {
226
227 auto &PSI = getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
228 auto GetAAR = [&](Function &F) -> AAResults & {
229 return getAnalysis<AAResultsWrapperPass>(F).getAAResults();
230 };
231 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
232 return getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
233 };
234 auto GetTTI = [&](Function &F) -> TargetTransformInfo & {
235 return getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
236 };
237 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
238 return getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
239 };
240
241 return AlwaysInlineImpl(M, InsertLifetime, PSI, /*FAM=*/nullptr,
242 GetAssumptionCache, GetAAR, GetTTI, GetTLI);
243 }
244
245 static char ID; // Pass identification, replacement for typeid
246
247 void getAnalysisUsage(AnalysisUsage &AU) const override {
248 AU.addRequired<AssumptionCacheTracker>();
249 AU.addRequired<AAResultsWrapperPass>();
250 AU.addRequired<ProfileSummaryInfoWrapperPass>();
251 AU.addRequired<TargetLibraryInfoWrapperPass>();
252 AU.addRequired<TargetTransformInfoWrapperPass>();
253 }
254};
255
256} // namespace
257
258char AlwaysInlinerLegacyPass::ID = 0;
259INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline",
260 "Inliner for always_inline functions", false, false)
261INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
262INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
263INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
264INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
265INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
266INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline",
267 "Inliner for always_inline functions", false, false)
268
269Pass *llvm::createAlwaysInlinerLegacyPass(bool InsertLifetime) {
270 return new AlwaysInlinerLegacyPass(InsertLifetime);
271}
272
273PreservedAnalyses AlwaysInlinerPass::run(Module &M,
274 ModuleAnalysisManager &MAM) {
275 FunctionAnalysisManager &FAM =
276 MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
277 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
278 return FAM.getResult<AssumptionAnalysis>(IR&: F);
279 };
280 auto GetAAR = [&](Function &F) -> AAResults & {
281 return FAM.getResult<AAManager>(IR&: F);
282 };
283 auto GetTTI = [&](Function &F) -> TargetTransformInfo & {
284 return FAM.getResult<TargetIRAnalysis>(IR&: F);
285 };
286 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
287 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
288 };
289 auto &PSI = MAM.getResult<ProfileSummaryAnalysis>(IR&: M);
290
291 bool Changed = AlwaysInlineImpl(M, InsertLifetime, PSI, FAM: &FAM,
292 GetAssumptionCache, GetAAR, GetTTI, GetTLI);
293 if (!Changed)
294 return PreservedAnalyses::all();
295
296 PreservedAnalyses PA;
297 // We have already invalidated all analyses on modified functions.
298 PA.preserveSet<AllAnalysesOn<Function>>();
299 return PA;
300}
301