1//===-- AMDGPUAlwaysInlinePass.cpp - Promote Allocas ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass marks all internal functions as always_inline and creates
11/// duplicates of all other functions and marks the duplicates as always_inline.
12//
13//===----------------------------------------------------------------------===//
14
15#include "AMDGPU.h"
16#include "AMDGPUTargetMachine.h"
17#include "Utils/AMDGPUBaseInfo.h"
18#include "llvm/CodeGen/CommandFlags.h"
19#include "llvm/IR/Module.h"
20#include "llvm/Pass.h"
21
22using namespace llvm;
23
24namespace {
25
26class AMDGPUAlwaysInline : public ModulePass {
27 bool GlobalOpt;
28
29public:
30 static char ID;
31
32 AMDGPUAlwaysInline(bool GlobalOpt = false) :
33 ModulePass(ID), GlobalOpt(GlobalOpt) { }
34 bool runOnModule(Module &M) override;
35
36 void getAnalysisUsage(AnalysisUsage &AU) const override {
37 AU.setPreservesAll();
38 }
39};
40
41} // End anonymous namespace
42
43INITIALIZE_PASS(AMDGPUAlwaysInline, "amdgpu-always-inline",
44 "AMDGPU Inline All Functions", false, false)
45
46char AMDGPUAlwaysInline::ID = 0;
47
48static void
49recursivelyVisitUsers(GlobalValue &GV,
50 SmallPtrSetImpl<Function *> &FuncsToAlwaysInline) {
51 SmallVector<User *, 16> Stack(GV.users());
52
53 SmallPtrSet<const Value *, 8> Visited;
54
55 while (!Stack.empty()) {
56 User *U = Stack.pop_back_val();
57 if (!Visited.insert(Ptr: U).second)
58 continue;
59
60 if (Instruction *I = dyn_cast<Instruction>(Val: U)) {
61 Function *F = I->getFunction();
62 if (!AMDGPU::isEntryFunctionCC(CC: F->getCallingConv())) {
63 // FIXME: This is a horrible hack. We should always respect noinline,
64 // and just let us hit the error when we can't handle this.
65 //
66 // Unfortunately, clang adds noinline to all functions at -O0. We have
67 // to override this here until that's fixed.
68 F->removeFnAttr(Kind: Attribute::NoInline);
69
70 FuncsToAlwaysInline.insert(Ptr: F);
71 Stack.push_back(Elt: F);
72 }
73
74 // No need to look at further users, but we do need to inline any callers.
75 continue;
76 }
77
78 append_range(C&: Stack, R: U->users());
79 }
80}
81
82static bool alwaysInlineImpl(Module &M, bool GlobalOpt) {
83 bool StressCalls = M.getContext()
84 .getLibraryOptions()
85 .get<AMDGPUOptions>()
86 .stress_function_calls;
87 std::vector<GlobalAlias*> AliasesToRemove;
88
89 bool Changed = false;
90 SmallPtrSet<Function *, 8> FuncsToAlwaysInline;
91 SmallPtrSet<Function *, 8> FuncsToNoInline;
92 Triple TT(M.getTargetTriple());
93
94 for (GlobalAlias &A : M.aliases()) {
95 if (Function* F = dyn_cast<Function>(Val: A.getAliasee())) {
96 if (TT.isAMDGCN() && A.getLinkage() != GlobalValue::InternalLinkage)
97 continue;
98 Changed = true;
99 A.replaceAllUsesWith(V: F);
100 AliasesToRemove.push_back(x: &A);
101 }
102
103 // FIXME: If the aliasee isn't a function, it's some kind of constant expr
104 // cast that won't be inlined through.
105 }
106
107 if (GlobalOpt) {
108 for (GlobalAlias* A : AliasesToRemove) {
109 A->eraseFromParent();
110 }
111 }
112
113 // Always force inlining of any function that uses an LDS global address. This
114 // is something of a workaround because we don't have a way of supporting LDS
115 // objects defined in functions. LDS is always allocated by a kernel, and it
116 // is difficult to manage LDS usage if a function may be used by multiple
117 // kernels.
118 //
119 // OpenCL doesn't allow declaring LDS in non-kernels, so in practice this
120 // should only appear when IPO passes manages to move LDs defined in a kernel
121 // into a single user function.
122
123 for (GlobalVariable &GV : M.globals()) {
124 // TODO: Region address
125 unsigned AS = GV.getAddressSpace();
126 if ((AS == AMDGPUAS::REGION_ADDRESS) ||
127 (AS == AMDGPUAS::LOCAL_ADDRESS &&
128 (!AMDGPUTargetMachine::EnableLowerModuleLDS)))
129 recursivelyVisitUsers(GV, FuncsToAlwaysInline);
130 }
131
132 if (!AMDGPUTargetMachine::EnableFunctionCalls || StressCalls) {
133 auto IncompatAttr
134 = StressCalls ? Attribute::AlwaysInline : Attribute::NoInline;
135
136 for (Function &F : M) {
137 if (!F.isDeclaration() && !F.use_empty() &&
138 !F.hasFnAttribute(Kind: IncompatAttr)) {
139 if (StressCalls) {
140 if (!FuncsToAlwaysInline.count(Ptr: &F))
141 FuncsToNoInline.insert(Ptr: &F);
142 } else
143 FuncsToAlwaysInline.insert(Ptr: &F);
144 }
145 }
146 }
147
148 for (Function *F : FuncsToAlwaysInline)
149 F->addFnAttr(Kind: Attribute::AlwaysInline);
150
151 for (Function *F : FuncsToNoInline)
152 F->addFnAttr(Kind: Attribute::NoInline);
153
154 return Changed || !FuncsToAlwaysInline.empty() || !FuncsToNoInline.empty();
155}
156
157bool AMDGPUAlwaysInline::runOnModule(Module &M) {
158 return alwaysInlineImpl(M, GlobalOpt);
159}
160
161ModulePass *llvm::createAMDGPUAlwaysInlinePass(bool GlobalOpt) {
162 return new AMDGPUAlwaysInline(GlobalOpt);
163}
164
165PreservedAnalyses AMDGPUAlwaysInlinePass::run(Module &M,
166 ModuleAnalysisManager &AM) {
167 const bool Changed = alwaysInlineImpl(M, GlobalOpt);
168 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
169}
170