1 | //===- lib/Transforms/Utils/FunctionImportUtils.cpp - Importing utilities -===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file implements the FunctionImportGlobalProcessing class, used |
10 | // to perform the necessary global value handling for function importing. |
11 | // |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #include "llvm/Transforms/Utils/FunctionImportUtils.h" |
15 | #include "llvm/Support/CommandLine.h" |
16 | using namespace llvm; |
17 | |
18 | /// Uses the "source_filename" instead of a Module hash ID for the suffix of |
19 | /// promoted locals during LTO. NOTE: This requires that the source filename |
20 | /// has a unique name / path to avoid name collisions. |
21 | static cl::opt<bool> UseSourceFilenameForPromotedLocals( |
22 | "use-source-filename-for-promoted-locals" , cl::Hidden, |
23 | cl::desc("Uses the source file name instead of the Module hash. " |
24 | "This requires that the source filename has a unique name / " |
25 | "path to avoid name collisions." )); |
26 | |
27 | /// Checks if we should import SGV as a definition, otherwise import as a |
28 | /// declaration. |
29 | bool FunctionImportGlobalProcessing::doImportAsDefinition( |
30 | const GlobalValue *SGV) { |
31 | if (!isPerformingImport()) |
32 | return false; |
33 | |
34 | // Only import the globals requested for importing. |
35 | if (!GlobalsToImport->count(key: const_cast<GlobalValue *>(SGV))) |
36 | return false; |
37 | |
38 | assert(!isa<GlobalAlias>(SGV) && |
39 | "Unexpected global alias in the import list." ); |
40 | |
41 | // Otherwise yes. |
42 | return true; |
43 | } |
44 | |
45 | bool FunctionImportGlobalProcessing::shouldPromoteLocalToGlobal( |
46 | const GlobalValue *SGV, ValueInfo VI) { |
47 | assert(SGV->hasLocalLinkage()); |
48 | |
49 | // Ifuncs and ifunc alias does not have summary. |
50 | if (isa<GlobalIFunc>(Val: SGV) || |
51 | (isa<GlobalAlias>(Val: SGV) && |
52 | isa<GlobalIFunc>(Val: cast<GlobalAlias>(Val: SGV)->getAliaseeObject()))) |
53 | return false; |
54 | |
55 | // Both the imported references and the original local variable must |
56 | // be promoted. |
57 | if (!isPerformingImport() && !isModuleExporting()) |
58 | return false; |
59 | |
60 | if (isPerformingImport()) { |
61 | assert((!GlobalsToImport->count(const_cast<GlobalValue *>(SGV)) || |
62 | !isNonRenamableLocal(*SGV)) && |
63 | "Attempting to promote non-renamable local" ); |
64 | // We don't know for sure yet if we are importing this value (as either |
65 | // a reference or a def), since we are simply walking all values in the |
66 | // module. But by necessity if we end up importing it and it is local, |
67 | // it must be promoted, so unconditionally promote all values in the |
68 | // importing module. |
69 | return true; |
70 | } |
71 | |
72 | // When exporting, consult the index. We can have more than one local |
73 | // with the same GUID, in the case of same-named locals in different but |
74 | // same-named source files that were compiled in their respective directories |
75 | // (so the source file name and resulting GUID is the same). Find the one |
76 | // in this module. |
77 | auto Summary = ImportIndex.findSummaryInModule( |
78 | VI, ModuleId: SGV->getParent()->getModuleIdentifier()); |
79 | assert(Summary && "Missing summary for global value when exporting" ); |
80 | auto Linkage = Summary->linkage(); |
81 | if (!GlobalValue::isLocalLinkage(Linkage)) { |
82 | assert(!isNonRenamableLocal(*SGV) && |
83 | "Attempting to promote non-renamable local" ); |
84 | return true; |
85 | } |
86 | |
87 | return false; |
88 | } |
89 | |
90 | #ifndef NDEBUG |
91 | bool FunctionImportGlobalProcessing::isNonRenamableLocal( |
92 | const GlobalValue &GV) const { |
93 | if (!GV.hasLocalLinkage()) |
94 | return false; |
95 | // This needs to stay in sync with the logic in buildModuleSummaryIndex. |
96 | if (GV.hasSection()) |
97 | return true; |
98 | if (Used.count(const_cast<GlobalValue *>(&GV))) |
99 | return true; |
100 | return false; |
101 | } |
102 | #endif |
103 | |
104 | std::string |
105 | FunctionImportGlobalProcessing::getPromotedName(const GlobalValue *SGV) { |
106 | assert(SGV->hasLocalLinkage()); |
107 | |
108 | // For locals that must be promoted to global scope, ensure that |
109 | // the promoted name uniquely identifies the copy in the original module, |
110 | // using the ID assigned during combined index creation. |
111 | if (UseSourceFilenameForPromotedLocals && |
112 | !SGV->getParent()->getSourceFileName().empty()) { |
113 | SmallString<256> Suffix(SGV->getParent()->getSourceFileName()); |
114 | std::replace_if(first: std::begin(cont&: Suffix), last: std::end(cont&: Suffix), |
115 | pred: [&](char ch) { return !isAlnum(C: ch); }, new_value: '_'); |
116 | return ModuleSummaryIndex::getGlobalNameForLocal( |
117 | Name: SGV->getName(), Suffix); |
118 | } |
119 | |
120 | return ModuleSummaryIndex::getGlobalNameForLocal( |
121 | Name: SGV->getName(), |
122 | ModHash: ImportIndex.getModuleHash(ModPath: SGV->getParent()->getModuleIdentifier())); |
123 | } |
124 | |
125 | GlobalValue::LinkageTypes |
126 | FunctionImportGlobalProcessing::getLinkage(const GlobalValue *SGV, |
127 | bool DoPromote) { |
128 | // Any local variable that is referenced by an exported function needs |
129 | // to be promoted to global scope. Since we don't currently know which |
130 | // functions reference which local variables/functions, we must treat |
131 | // all as potentially exported if this module is exporting anything. |
132 | if (isModuleExporting()) { |
133 | if (SGV->hasLocalLinkage() && DoPromote) |
134 | return GlobalValue::ExternalLinkage; |
135 | return SGV->getLinkage(); |
136 | } |
137 | |
138 | // Otherwise, if we aren't importing, no linkage change is needed. |
139 | if (!isPerformingImport()) |
140 | return SGV->getLinkage(); |
141 | |
142 | switch (SGV->getLinkage()) { |
143 | case GlobalValue::LinkOnceODRLinkage: |
144 | case GlobalValue::ExternalLinkage: |
145 | // External and linkonce definitions are converted to available_externally |
146 | // definitions upon import, so that they are available for inlining |
147 | // and/or optimization, but are turned into declarations later |
148 | // during the EliminateAvailableExternally pass. |
149 | if (doImportAsDefinition(SGV) && !isa<GlobalAlias>(Val: SGV)) |
150 | return GlobalValue::AvailableExternallyLinkage; |
151 | // An imported external declaration stays external. |
152 | return SGV->getLinkage(); |
153 | |
154 | case GlobalValue::AvailableExternallyLinkage: |
155 | // An imported available_externally definition converts |
156 | // to external if imported as a declaration. |
157 | if (!doImportAsDefinition(SGV)) |
158 | return GlobalValue::ExternalLinkage; |
159 | // An imported available_externally declaration stays that way. |
160 | return SGV->getLinkage(); |
161 | |
162 | case GlobalValue::LinkOnceAnyLinkage: |
163 | case GlobalValue::WeakAnyLinkage: |
164 | // Can't import linkonce_any/weak_any definitions correctly, or we might |
165 | // change the program semantics, since the linker will pick the first |
166 | // linkonce_any/weak_any definition and importing would change the order |
167 | // they are seen by the linker. The module linking caller needs to enforce |
168 | // this. |
169 | assert(!doImportAsDefinition(SGV)); |
170 | // If imported as a declaration, it becomes external_weak. |
171 | return SGV->getLinkage(); |
172 | |
173 | case GlobalValue::WeakODRLinkage: |
174 | // For weak_odr linkage, there is a guarantee that all copies will be |
175 | // equivalent, so the issue described above for weak_any does not exist, |
176 | // and the definition can be imported. It can be treated similarly |
177 | // to an imported externally visible global value. |
178 | if (doImportAsDefinition(SGV) && !isa<GlobalAlias>(Val: SGV)) |
179 | return GlobalValue::AvailableExternallyLinkage; |
180 | else |
181 | return GlobalValue::ExternalLinkage; |
182 | |
183 | case GlobalValue::AppendingLinkage: |
184 | // It would be incorrect to import an appending linkage variable, |
185 | // since it would cause global constructors/destructors to be |
186 | // executed multiple times. This should have already been handled |
187 | // by linkIfNeeded, and we will assert in shouldLinkFromSource |
188 | // if we try to import, so we simply return AppendingLinkage. |
189 | return GlobalValue::AppendingLinkage; |
190 | |
191 | case GlobalValue::InternalLinkage: |
192 | case GlobalValue::PrivateLinkage: |
193 | // If we are promoting the local to global scope, it is handled |
194 | // similarly to a normal externally visible global. |
195 | if (DoPromote) { |
196 | if (doImportAsDefinition(SGV) && !isa<GlobalAlias>(Val: SGV)) |
197 | return GlobalValue::AvailableExternallyLinkage; |
198 | else |
199 | return GlobalValue::ExternalLinkage; |
200 | } |
201 | // A non-promoted imported local definition stays local. |
202 | // The ThinLTO pass will eventually force-import their definitions. |
203 | return SGV->getLinkage(); |
204 | |
205 | case GlobalValue::ExternalWeakLinkage: |
206 | // External weak doesn't apply to definitions, must be a declaration. |
207 | assert(!doImportAsDefinition(SGV)); |
208 | // Linkage stays external_weak. |
209 | return SGV->getLinkage(); |
210 | |
211 | case GlobalValue::CommonLinkage: |
212 | // Linkage stays common on definitions. |
213 | // The ThinLTO pass will eventually force-import their definitions. |
214 | return SGV->getLinkage(); |
215 | } |
216 | |
217 | llvm_unreachable("unknown linkage type" ); |
218 | } |
219 | |
220 | void FunctionImportGlobalProcessing::processGlobalForThinLTO(GlobalValue &GV) { |
221 | |
222 | ValueInfo VI; |
223 | if (GV.hasName()) { |
224 | VI = ImportIndex.getValueInfo(GUID: GV.getGUID()); |
225 | // Set synthetic function entry counts. |
226 | if (VI && ImportIndex.hasSyntheticEntryCounts()) { |
227 | if (Function *F = dyn_cast<Function>(Val: &GV)) { |
228 | if (!F->isDeclaration()) { |
229 | for (const auto &S : VI.getSummaryList()) { |
230 | auto *FS = cast<FunctionSummary>(Val: S->getBaseObject()); |
231 | if (FS->modulePath() == M.getModuleIdentifier()) { |
232 | F->setEntryCount(Count: Function::ProfileCount(FS->entryCount(), |
233 | Function::PCT_Synthetic)); |
234 | break; |
235 | } |
236 | } |
237 | } |
238 | } |
239 | } |
240 | } |
241 | |
242 | // We should always have a ValueInfo (i.e. GV in index) for definitions when |
243 | // we are exporting, and also when importing that value. |
244 | assert(VI || GV.isDeclaration() || |
245 | (isPerformingImport() && !doImportAsDefinition(&GV))); |
246 | |
247 | // Mark read/write-only variables which can be imported with specific |
248 | // attribute. We can't internalize them now because IRMover will fail |
249 | // to link variable definitions to their external declarations during |
250 | // ThinLTO import. We'll internalize read-only variables later, after |
251 | // import is finished. See internalizeGVsAfterImport. |
252 | // |
253 | // If global value dead stripping is not enabled in summary then |
254 | // propagateConstants hasn't been run. We can't internalize GV |
255 | // in such case. |
256 | if (!GV.isDeclaration() && VI && ImportIndex.withAttributePropagation()) { |
257 | if (GlobalVariable *V = dyn_cast<GlobalVariable>(Val: &GV)) { |
258 | // We can have more than one local with the same GUID, in the case of |
259 | // same-named locals in different but same-named source files that were |
260 | // compiled in their respective directories (so the source file name |
261 | // and resulting GUID is the same). Find the one in this module. |
262 | // Handle the case where there is no summary found in this module. That |
263 | // can happen in the distributed ThinLTO backend, because the index only |
264 | // contains summaries from the source modules if they are being imported. |
265 | // We might have a non-null VI and get here even in that case if the name |
266 | // matches one in this module (e.g. weak or appending linkage). |
267 | auto *GVS = dyn_cast_or_null<GlobalVarSummary>( |
268 | Val: ImportIndex.findSummaryInModule(VI, ModuleId: M.getModuleIdentifier())); |
269 | if (GVS && |
270 | (ImportIndex.isReadOnly(GVS) || ImportIndex.isWriteOnly(GVS))) { |
271 | V->addAttribute(Kind: "thinlto-internalize" ); |
272 | // Objects referenced by writeonly GV initializer should not be |
273 | // promoted, because there is no any kind of read access to them |
274 | // on behalf of this writeonly GV. To avoid promotion we convert |
275 | // GV initializer to 'zeroinitializer'. This effectively drops |
276 | // references in IR module (not in combined index), so we can |
277 | // ignore them when computing import. We do not export references |
278 | // of writeonly object. See computeImportForReferencedGlobals |
279 | if (ImportIndex.isWriteOnly(GVS)) |
280 | V->setInitializer(Constant::getNullValue(Ty: V->getValueType())); |
281 | } |
282 | } |
283 | } |
284 | |
285 | if (GV.hasLocalLinkage() && shouldPromoteLocalToGlobal(SGV: &GV, VI)) { |
286 | // Save the original name string before we rename GV below. |
287 | auto Name = GV.getName().str(); |
288 | GV.setName(getPromotedName(SGV: &GV)); |
289 | GV.setLinkage(getLinkage(SGV: &GV, /* DoPromote */ true)); |
290 | assert(!GV.hasLocalLinkage()); |
291 | GV.setVisibility(GlobalValue::HiddenVisibility); |
292 | |
293 | // If we are renaming a COMDAT leader, ensure that we record the COMDAT |
294 | // for later renaming as well. This is required for COFF. |
295 | if (const auto *C = GV.getComdat()) |
296 | if (C->getName() == Name) |
297 | RenamedComdats.try_emplace(Key: C, Args: M.getOrInsertComdat(Name: GV.getName())); |
298 | } else |
299 | GV.setLinkage(getLinkage(SGV: &GV, /* DoPromote */ false)); |
300 | |
301 | // When ClearDSOLocalOnDeclarations is true, clear dso_local if GV is |
302 | // converted to a declaration, to disable direct access. Don't do this if GV |
303 | // is implicitly dso_local due to a non-default visibility. |
304 | if (ClearDSOLocalOnDeclarations && |
305 | (GV.isDeclarationForLinker() || |
306 | (isPerformingImport() && !doImportAsDefinition(SGV: &GV))) && |
307 | !GV.isImplicitDSOLocal()) { |
308 | GV.setDSOLocal(false); |
309 | } else if (VI && VI.isDSOLocal(WithDSOLocalPropagation: ImportIndex.withDSOLocalPropagation())) { |
310 | // If all summaries are dso_local, symbol gets resolved to a known local |
311 | // definition. |
312 | GV.setDSOLocal(true); |
313 | if (GV.hasDLLImportStorageClass()) |
314 | GV.setDLLStorageClass(GlobalValue::DefaultStorageClass); |
315 | } |
316 | |
317 | // Remove functions imported as available externally defs from comdats, |
318 | // as this is a declaration for the linker, and will be dropped eventually. |
319 | // It is illegal for comdats to contain declarations. |
320 | auto *GO = dyn_cast<GlobalObject>(Val: &GV); |
321 | if (GO && GO->isDeclarationForLinker() && GO->hasComdat()) { |
322 | // The IRMover should not have placed any imported declarations in |
323 | // a comdat, so the only declaration that should be in a comdat |
324 | // at this point would be a definition imported as available_externally. |
325 | assert(GO->hasAvailableExternallyLinkage() && |
326 | "Expected comdat on definition (possibly available external)" ); |
327 | GO->setComdat(nullptr); |
328 | } |
329 | } |
330 | |
331 | void FunctionImportGlobalProcessing::processGlobalsForThinLTO() { |
332 | for (GlobalVariable &GV : M.globals()) |
333 | processGlobalForThinLTO(GV); |
334 | for (Function &SF : M) |
335 | processGlobalForThinLTO(GV&: SF); |
336 | for (GlobalAlias &GA : M.aliases()) |
337 | processGlobalForThinLTO(GV&: GA); |
338 | |
339 | // Replace any COMDATS that required renaming (because the COMDAT leader was |
340 | // promoted and renamed). |
341 | if (!RenamedComdats.empty()) |
342 | for (auto &GO : M.global_objects()) |
343 | if (auto *C = GO.getComdat()) { |
344 | auto Replacement = RenamedComdats.find(Val: C); |
345 | if (Replacement != RenamedComdats.end()) |
346 | GO.setComdat(Replacement->second); |
347 | } |
348 | } |
349 | |
350 | bool FunctionImportGlobalProcessing::run() { |
351 | processGlobalsForThinLTO(); |
352 | return false; |
353 | } |
354 | |
355 | bool llvm::renameModuleForThinLTO(Module &M, const ModuleSummaryIndex &Index, |
356 | bool ClearDSOLocalOnDeclarations, |
357 | SetVector<GlobalValue *> *GlobalsToImport) { |
358 | FunctionImportGlobalProcessing ThinLTOProcessing(M, Index, GlobalsToImport, |
359 | ClearDSOLocalOnDeclarations); |
360 | return ThinLTOProcessing.run(); |
361 | } |
362 | |