1//===-- AMDGPUAlwaysInlinePass.cpp - Promote Allocas ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass marks all internal functions as always_inline and creates
11/// duplicates of all other functions and marks the duplicates as always_inline.
12//
13//===----------------------------------------------------------------------===//
14
15#include "AMDGPU.h"
16#include "AMDGPUTargetMachine.h"
17#include "Utils/AMDGPUBaseInfo.h"
18#include "llvm/CodeGen/CommandFlags.h"
19#include "llvm/IR/Module.h"
20#include "llvm/Pass.h"
21#include "llvm/Support/CommandLine.h"
22
23using namespace llvm;
24
25namespace {
26
27static cl::opt<bool> StressCalls(
28 "amdgpu-stress-function-calls",
29 cl::Hidden,
30 cl::desc("Force all functions to be noinline"),
31 cl::init(Val: false));
32
33class AMDGPUAlwaysInline : public ModulePass {
34 bool GlobalOpt;
35
36public:
37 static char ID;
38
39 AMDGPUAlwaysInline(bool GlobalOpt = false) :
40 ModulePass(ID), GlobalOpt(GlobalOpt) { }
41 bool runOnModule(Module &M) override;
42
43 void getAnalysisUsage(AnalysisUsage &AU) const override {
44 AU.setPreservesAll();
45 }
46};
47
48} // End anonymous namespace
49
50INITIALIZE_PASS(AMDGPUAlwaysInline, "amdgpu-always-inline",
51 "AMDGPU Inline All Functions", false, false)
52
53char AMDGPUAlwaysInline::ID = 0;
54
55static void
56recursivelyVisitUsers(GlobalValue &GV,
57 SmallPtrSetImpl<Function *> &FuncsToAlwaysInline) {
58 SmallVector<User *, 16> Stack(GV.users());
59
60 SmallPtrSet<const Value *, 8> Visited;
61
62 while (!Stack.empty()) {
63 User *U = Stack.pop_back_val();
64 if (!Visited.insert(Ptr: U).second)
65 continue;
66
67 if (Instruction *I = dyn_cast<Instruction>(Val: U)) {
68 Function *F = I->getParent()->getParent();
69 if (!AMDGPU::isEntryFunctionCC(CC: F->getCallingConv())) {
70 // FIXME: This is a horrible hack. We should always respect noinline,
71 // and just let us hit the error when we can't handle this.
72 //
73 // Unfortunately, clang adds noinline to all functions at -O0. We have
74 // to override this here until that's fixed.
75 F->removeFnAttr(Kind: Attribute::NoInline);
76
77 FuncsToAlwaysInline.insert(Ptr: F);
78 Stack.push_back(Elt: F);
79 }
80
81 // No need to look at further users, but we do need to inline any callers.
82 continue;
83 }
84
85 append_range(C&: Stack, R: U->users());
86 }
87}
88
89static bool alwaysInlineImpl(Module &M, bool GlobalOpt) {
90 std::vector<GlobalAlias*> AliasesToRemove;
91
92 bool Changed = false;
93 SmallPtrSet<Function *, 8> FuncsToAlwaysInline;
94 SmallPtrSet<Function *, 8> FuncsToNoInline;
95 Triple TT(M.getTargetTriple());
96
97 for (GlobalAlias &A : M.aliases()) {
98 if (Function* F = dyn_cast<Function>(Val: A.getAliasee())) {
99 if (TT.isAMDGCN() && A.getLinkage() != GlobalValue::InternalLinkage)
100 continue;
101 Changed = true;
102 A.replaceAllUsesWith(V: F);
103 AliasesToRemove.push_back(x: &A);
104 }
105
106 // FIXME: If the aliasee isn't a function, it's some kind of constant expr
107 // cast that won't be inlined through.
108 }
109
110 if (GlobalOpt) {
111 for (GlobalAlias* A : AliasesToRemove) {
112 A->eraseFromParent();
113 }
114 }
115
116 // Always force inlining of any function that uses an LDS global address. This
117 // is something of a workaround because we don't have a way of supporting LDS
118 // objects defined in functions. LDS is always allocated by a kernel, and it
119 // is difficult to manage LDS usage if a function may be used by multiple
120 // kernels.
121 //
122 // OpenCL doesn't allow declaring LDS in non-kernels, so in practice this
123 // should only appear when IPO passes manages to move LDs defined in a kernel
124 // into a single user function.
125
126 for (GlobalVariable &GV : M.globals()) {
127 // TODO: Region address
128 unsigned AS = GV.getAddressSpace();
129 if ((AS == AMDGPUAS::REGION_ADDRESS) ||
130 (AS == AMDGPUAS::LOCAL_ADDRESS &&
131 (!AMDGPUTargetMachine::EnableLowerModuleLDS)))
132 recursivelyVisitUsers(GV, FuncsToAlwaysInline);
133 }
134
135 if (!AMDGPUTargetMachine::EnableFunctionCalls || StressCalls) {
136 auto IncompatAttr
137 = StressCalls ? Attribute::AlwaysInline : Attribute::NoInline;
138
139 for (Function &F : M) {
140 if (!F.isDeclaration() && !F.use_empty() &&
141 !F.hasFnAttribute(Kind: IncompatAttr)) {
142 if (StressCalls) {
143 if (!FuncsToAlwaysInline.count(Ptr: &F))
144 FuncsToNoInline.insert(Ptr: &F);
145 } else
146 FuncsToAlwaysInline.insert(Ptr: &F);
147 }
148 }
149 }
150
151 for (Function *F : FuncsToAlwaysInline)
152 F->addFnAttr(Kind: Attribute::AlwaysInline);
153
154 for (Function *F : FuncsToNoInline)
155 F->addFnAttr(Kind: Attribute::NoInline);
156
157 return Changed || !FuncsToAlwaysInline.empty() || !FuncsToNoInline.empty();
158}
159
160bool AMDGPUAlwaysInline::runOnModule(Module &M) {
161 return alwaysInlineImpl(M, GlobalOpt);
162}
163
164ModulePass *llvm::createAMDGPUAlwaysInlinePass(bool GlobalOpt) {
165 return new AMDGPUAlwaysInline(GlobalOpt);
166}
167
168PreservedAnalyses AMDGPUAlwaysInlinePass::run(Module &M,
169 ModuleAnalysisManager &AM) {
170 const bool Changed = alwaysInlineImpl(M, GlobalOpt);
171 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
172}
173