1//===- SplitModule.cpp - Split a module into partitions -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the function llvm::SplitModule, which splits a module
10// into multiple linkable partitions. It can be used to implement parallel code
11// generation for link-time optimization.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Transforms/Utils/SplitModule.h"
16#include "llvm/ADT/DenseMap.h"
17#include "llvm/ADT/EquivalenceClasses.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/ADT/SmallVector.h"
20#include "llvm/ADT/StringRef.h"
21#include "llvm/IR/Comdat.h"
22#include "llvm/IR/Constant.h"
23#include "llvm/IR/Constants.h"
24#include "llvm/IR/Function.h"
25#include "llvm/IR/GlobalAlias.h"
26#include "llvm/IR/GlobalObject.h"
27#include "llvm/IR/GlobalValue.h"
28#include "llvm/IR/GlobalVariable.h"
29#include "llvm/IR/Instruction.h"
30#include "llvm/IR/Module.h"
31#include "llvm/IR/User.h"
32#include "llvm/IR/Value.h"
33#include "llvm/Support/Casting.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/MD5.h"
37#include "llvm/Support/raw_ostream.h"
38#include "llvm/Transforms/Utils/Cloning.h"
39#include "llvm/Transforms/Utils/ValueMapper.h"
40#include <cassert>
41#include <iterator>
42#include <memory>
43#include <queue>
44#include <utility>
45#include <vector>
46
47using namespace llvm;
48
49#define DEBUG_TYPE "split-module"
50
51namespace {
52
53using ClusterMapType = EquivalenceClasses<const GlobalValue *>;
54using ComdatMembersType = DenseMap<const Comdat *, const GlobalValue *>;
55using ClusterIDMapType = DenseMap<const GlobalValue *, unsigned>;
56
57bool compareClusters(const std::pair<unsigned, unsigned> &A,
58 const std::pair<unsigned, unsigned> &B) {
59 if (A.second || B.second)
60 return A.second > B.second;
61 return A.first > B.first;
62}
63
64using BalancingQueueType =
65 std::priority_queue<std::pair<unsigned, unsigned>,
66 std::vector<std::pair<unsigned, unsigned>>,
67 decltype(compareClusters) *>;
68
69} // end anonymous namespace
70
71static void addNonConstUser(ClusterMapType &GVtoClusterMap,
72 const GlobalValue *GV, const User *U) {
73 assert((!isa<Constant>(U) || isa<GlobalValue>(U)) && "Bad user");
74
75 if (const Instruction *I = dyn_cast<Instruction>(Val: U)) {
76 const GlobalValue *F = I->getParent()->getParent();
77 GVtoClusterMap.unionSets(V1: GV, V2: F);
78 } else if (const GlobalValue *GVU = dyn_cast<GlobalValue>(Val: U)) {
79 GVtoClusterMap.unionSets(V1: GV, V2: GVU);
80 } else {
81 llvm_unreachable("Underimplemented use case");
82 }
83}
84
85// Adds all GlobalValue users of V to the same cluster as GV.
86static void addAllGlobalValueUsers(ClusterMapType &GVtoClusterMap,
87 const GlobalValue *GV, const Value *V) {
88 for (const auto *U : V->users()) {
89 SmallVector<const User *, 4> Worklist;
90 Worklist.push_back(Elt: U);
91 while (!Worklist.empty()) {
92 const User *UU = Worklist.pop_back_val();
93 // For each constant that is not a GV (a pure const) recurse.
94 if (isa<Constant>(Val: UU) && !isa<GlobalValue>(Val: UU)) {
95 Worklist.append(in_start: UU->user_begin(), in_end: UU->user_end());
96 continue;
97 }
98 addNonConstUser(GVtoClusterMap, GV, U: UU);
99 }
100 }
101}
102
103static const GlobalObject *getGVPartitioningRoot(const GlobalValue *GV) {
104 const GlobalObject *GO = GV->getAliaseeObject();
105 if (const auto *GI = dyn_cast_or_null<GlobalIFunc>(Val: GO))
106 GO = GI->getResolverFunction();
107 return GO;
108}
109
110// Find partitions for module in the way that no locals need to be
111// globalized.
112// Try to balance pack those partitions into N files since this roughly equals
113// thread balancing for the backend codegen step.
114static void findPartitions(Module &M, ClusterIDMapType &ClusterIDMap,
115 unsigned N) {
116 // At this point module should have the proper mix of globals and locals.
117 // As we attempt to partition this module, we must not change any
118 // locals to globals.
119 LLVM_DEBUG(dbgs() << "Partition module with (" << M.size()
120 << ") functions\n");
121 ClusterMapType GVtoClusterMap;
122 ComdatMembersType ComdatMembers;
123
124 auto recordGVSet = [&GVtoClusterMap, &ComdatMembers](GlobalValue &GV) {
125 if (GV.isDeclaration())
126 return;
127
128 GV.nameUnnamed();
129
130 // Comdat groups must not be partitioned. For comdat groups that contain
131 // locals, record all their members here so we can keep them together.
132 // Comdat groups that only contain external globals are already handled by
133 // the MD5-based partitioning.
134 if (const Comdat *C = GV.getComdat()) {
135 auto &Member = ComdatMembers[C];
136 if (Member)
137 GVtoClusterMap.unionSets(V1: Member, V2: &GV);
138 else
139 Member = &GV;
140 }
141
142 // Aliases should not be separated from their aliasees and ifuncs should
143 // not be separated from their resolvers regardless of linkage.
144 if (const GlobalObject *Root = getGVPartitioningRoot(GV: &GV))
145 if (&GV != Root)
146 GVtoClusterMap.unionSets(V1: &GV, V2: Root);
147
148 if (const Function *F = dyn_cast<Function>(Val: &GV)) {
149 for (const BasicBlock &BB : *F) {
150 BlockAddress *BA = BlockAddress::lookup(BB: &BB);
151 if (!BA || !BA->isConstantUsed())
152 continue;
153 addAllGlobalValueUsers(GVtoClusterMap, GV: F, V: BA);
154 }
155 }
156
157 if (GV.hasLocalLinkage())
158 addAllGlobalValueUsers(GVtoClusterMap, GV: &GV, V: &GV);
159 };
160
161 llvm::for_each(Range: M.functions(), F: recordGVSet);
162 llvm::for_each(Range: M.globals(), F: recordGVSet);
163 llvm::for_each(Range: M.aliases(), F: recordGVSet);
164 llvm::for_each(Range: M.ifuncs(), F: recordGVSet);
165
166 // Assigned all GVs to merged clusters while balancing number of objects in
167 // each.
168 BalancingQueueType BalancingQueue(compareClusters);
169 // Pre-populate priority queue with N slot blanks.
170 for (unsigned i = 0; i < N; ++i)
171 BalancingQueue.push(x: std::make_pair(x&: i, y: 0));
172
173 SmallPtrSet<const GlobalValue *, 32> Visited;
174
175 // To guarantee determinism, we have to sort SCC according to size.
176 // When size is the same, use leader's name.
177 for (const auto &C : GVtoClusterMap) {
178 if (!C->isLeader())
179 continue;
180
181 unsigned CurrentClusterID = BalancingQueue.top().first;
182 unsigned CurrentClusterSize = BalancingQueue.top().second;
183 BalancingQueue.pop();
184
185 LLVM_DEBUG(dbgs() << "Root[" << CurrentClusterID << "] cluster_size("
186 << std::distance(GVtoClusterMap.member_begin(*C),
187 GVtoClusterMap.member_end())
188 << ") ----> " << C->getData()->getName() << "\n");
189
190 for (ClusterMapType::member_iterator MI = GVtoClusterMap.findLeader(ECV: *C);
191 MI != GVtoClusterMap.member_end(); ++MI) {
192 if (!Visited.insert(Ptr: *MI).second)
193 continue;
194 LLVM_DEBUG(dbgs() << "----> " << (*MI)->getName()
195 << ((*MI)->hasLocalLinkage() ? " l " : " e ") << "\n");
196 Visited.insert(Ptr: *MI);
197 ClusterIDMap[*MI] = CurrentClusterID;
198 CurrentClusterSize++;
199 }
200 // Add this set size to the number of entries in this cluster.
201 BalancingQueue.push(x: std::make_pair(x&: CurrentClusterID, y&: CurrentClusterSize));
202 }
203}
204
205// Returns whether GV should be in partition (0-based) I of N.
206static bool isInPartition(const GlobalValue *GV, unsigned I, unsigned N) {
207 if (const GlobalObject *Root = getGVPartitioningRoot(GV))
208 GV = Root;
209
210 StringRef Name;
211 if (const Comdat *C = GV->getComdat())
212 Name = C->getName();
213 else
214 Name = GV->getName();
215
216 // Partition by MD5 hash. We only need a few bits for evenness as the number
217 // of partitions will generally be in the 1-2 figure range; the low 16 bits
218 // are enough.
219 MD5 H;
220 MD5::MD5Result R;
221 H.update(Str: Name);
222 H.final(Result&: R);
223 return (R[0] | (R[1] << 8)) % N == I;
224}
225
226void llvm::SplitModule(
227 Module &M, unsigned N,
228 function_ref<void(std::unique_ptr<Module> MPart)> ModuleCallback,
229 bool PreserveLocals, bool RoundRobin) {
230 if (!PreserveLocals) {
231 for (Function &F : M)
232 F.externalize();
233 for (GlobalVariable &GV : M.globals())
234 GV.externalize();
235 for (GlobalAlias &GA : M.aliases())
236 GA.externalize();
237 for (GlobalIFunc &GIF : M.ifuncs())
238 GIF.externalize();
239 }
240
241 // This performs splitting without a need for externalization, which might not
242 // always be possible.
243 ClusterIDMapType ClusterIDMap;
244 findPartitions(M, ClusterIDMap, N);
245
246 // Find functions not mapped to modules in ClusterIDMap and count functions
247 // per module. Map unmapped functions using round-robin so that they skip
248 // being distributed by isInPartition() based on function name hashes below.
249 // This provides better uniformity of distribution of functions to modules
250 // in some cases - for example when the number of functions equals to N.
251 if (RoundRobin) {
252 DenseMap<unsigned, unsigned> ModuleFunctionCount;
253 SmallVector<const GlobalValue *> UnmappedFunctions;
254 for (const auto &F : M.functions()) {
255 if (F.isDeclaration() ||
256 F.getLinkage() != GlobalValue::LinkageTypes::ExternalLinkage)
257 continue;
258 auto It = ClusterIDMap.find(Val: &F);
259 if (It == ClusterIDMap.end())
260 UnmappedFunctions.push_back(Elt: &F);
261 else
262 ++ModuleFunctionCount[It->second];
263 }
264 BalancingQueueType BalancingQueue(compareClusters);
265 for (unsigned I = 0; I < N; ++I) {
266 if (auto It = ModuleFunctionCount.find(Val: I);
267 It != ModuleFunctionCount.end())
268 BalancingQueue.push(x: *It);
269 else
270 BalancingQueue.push(x: {I, 0});
271 }
272 for (const auto *const F : UnmappedFunctions) {
273 const unsigned I = BalancingQueue.top().first;
274 const unsigned Count = BalancingQueue.top().second;
275 BalancingQueue.pop();
276 ClusterIDMap.insert(KV: {F, I});
277 BalancingQueue.push(x: {I, Count + 1});
278 }
279 }
280
281 // FIXME: We should be able to reuse M as the last partition instead of
282 // cloning it. Note that the callers at the moment expect the module to
283 // be preserved, so will need some adjustments as well.
284 for (unsigned I = 0; I < N; ++I) {
285 ValueToValueMapTy VMap;
286 std::unique_ptr<Module> MPart(
287 CloneModule(M, VMap, ShouldCloneDefinition: [&](const GlobalValue *GV) {
288 if (auto It = ClusterIDMap.find(Val: GV); It != ClusterIDMap.end())
289 return It->second == I;
290 else
291 return isInPartition(GV, I, N);
292 }));
293 if (I != 0)
294 MPart->removeModuleInlineAsm();
295 ModuleCallback(std::move(MPart));
296 }
297}
298