1//===- LTO.cpp ------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "LTO.h"
10#include "Config.h"
11#include "Driver.h"
12#include "InputFiles.h"
13#include "SymbolTable.h"
14#include "Symbols.h"
15#include "lld/Common/ErrorHandler.h"
16#include "lld/Common/Filesystem.h"
17#include "lld/Common/Strings.h"
18#include "lld/Common/TargetOptionsCommandFlags.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/ADT/Twine.h"
21#include "llvm/BinaryFormat/ELF.h"
22#include "llvm/Bitcode/BitcodeWriter.h"
23#include "llvm/DTLTO/DTLTO.h"
24#include "llvm/LTO/Config.h"
25#include "llvm/LTO/LTO.h"
26#include "llvm/Support/Caching.h"
27#include "llvm/Support/CodeGen.h"
28#include "llvm/Support/MemoryBuffer.h"
29#include "llvm/Support/Path.h"
30#include <cstddef>
31#include <memory>
32#include <string>
33#include <system_error>
34#include <vector>
35
36using namespace llvm;
37using namespace llvm::object;
38using namespace llvm::ELF;
39using namespace lld;
40using namespace lld::elf;
41
42static AddBufferFn
43createAddBufferFn(std::vector<std::unique_ptr<MemoryBuffer>> &files,
44 SmallVectorImpl<std::string> &filenames) {
45 return [&files, &filenames](unsigned task, const Twine &moduleName,
46 std::unique_ptr<MemoryBuffer> mb) {
47 files[task] = std::move(mb);
48 filenames[task] = moduleName.str();
49 };
50}
51
52static std::string getThinLTOOutputFile(Ctx &ctx, StringRef modulePath) {
53 return lto::getThinLTOOutputFile(Path: modulePath, OldPrefix: ctx.arg.thinLTOPrefixReplaceOld,
54 NewPrefix: ctx.arg.thinLTOPrefixReplaceNew);
55}
56
57static lto::Config createConfig(Ctx &ctx) {
58 lto::Config c;
59 c.LibraryOptions = &ctx.arg.libraryOptions;
60
61 // LLD supports the new relocations and address-significance tables.
62 c.Options = getTargetOptions(libraryOptions: ctx.arg.libraryOptions);
63 c.Options.EmitAddrsig = true;
64 for (StringRef C : ctx.arg.mllvmOpts)
65 c.MllvmArgs.emplace_back(args: C.str());
66
67 // Always emit a section per function/datum with LTO.
68 c.Options.FunctionSections = true;
69 c.Options.DataSections = true;
70
71 // Check if basic block sections must be used.
72 // Allowed values for --lto-basic-block-sections are "all",
73 // "<file name specifying basic block ids>", or none. This is the equivalent
74 // of -fbasic-block-sections= flag in clang.
75 if (!ctx.arg.ltoBasicBlockSections.empty()) {
76 if (ctx.arg.ltoBasicBlockSections == "all") {
77 c.Options.BBSections = BasicBlockSection::All;
78 } else if (ctx.arg.ltoBasicBlockSections == "labels") {
79 c.Options.BBAddrMap = true;
80 Warn(ctx)
81 << "'--lto-basic-block-sections=labels' is deprecated; Please use "
82 "'--lto-basic-block-address-map' instead";
83 } else if (ctx.arg.ltoBasicBlockSections == "none") {
84 c.Options.BBSections = BasicBlockSection::None;
85 } else {
86 ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr =
87 MemoryBuffer::getFile(Filename: ctx.arg.ltoBasicBlockSections.str());
88 if (!MBOrErr) {
89 ErrAlways(ctx) << "cannot open " << ctx.arg.ltoBasicBlockSections << ":"
90 << MBOrErr.getError().message();
91 } else {
92 c.Options.BBSectionsFuncListBuf = std::move(*MBOrErr);
93 c.Options.BBSections = BasicBlockSection::List;
94 }
95 }
96 }
97
98 c.Options.BBAddrMap = ctx.arg.ltoBBAddrMap;
99
100 c.Options.UniqueBasicBlockSectionNames =
101 ctx.arg.ltoUniqueBasicBlockSectionNames;
102
103 if (auto relocModel = getRelocModelFromCMModel(libraryOptions: ctx.arg.libraryOptions))
104 c.RelocModel = *relocModel;
105 else if (ctx.arg.relocatable)
106 c.RelocModel = std::nullopt;
107 else if (ctx.arg.isPic)
108 c.RelocModel = Reloc::PIC_;
109 else
110 c.RelocModel = Reloc::Static;
111
112 c.CodeModel = getCodeModelFromCMModel(libraryOptions: ctx.arg.libraryOptions);
113 c.DisableVerify = ctx.arg.disableVerify;
114 c.DiagHandler = diagnosticHandler;
115 c.OptLevel = ctx.arg.ltoo;
116 c.CPU = getCPUStr(libraryOptions: ctx.arg.libraryOptions);
117 c.MAttrs = getMAttrs(libraryOptions: ctx.arg.libraryOptions);
118 c.CGOptLevel = ctx.arg.ltoCgo;
119
120 c.PTO.LoopVectorization = c.OptLevel > 1;
121 c.PTO.SLPVectorization = c.OptLevel > 1;
122
123 // Set up a custom pipeline if we've been asked to.
124 c.OptPipeline = std::string(ctx.arg.ltoNewPmPasses);
125 c.AAPipeline = std::string(ctx.arg.ltoAAPipeline);
126
127 // Set up optimization remarks if we've been asked to.
128 c.RemarksFilename = std::string(ctx.arg.optRemarksFilename);
129 c.RemarksPasses = std::string(ctx.arg.optRemarksPasses);
130 c.RemarksWithHotness = ctx.arg.optRemarksWithHotness;
131 c.RemarksHotnessThreshold = ctx.arg.optRemarksHotnessThreshold;
132 c.RemarksFormat = std::string(ctx.arg.optRemarksFormat);
133
134 // Set up output file to emit statistics.
135 c.StatsFile = std::string(ctx.arg.optStatsFilename);
136
137 c.SampleProfile = std::string(ctx.arg.ltoSampleProfile);
138 for (StringRef pluginFn : ctx.arg.passPlugins)
139 c.PassPluginFilenames.push_back(x: std::string(pluginFn));
140 c.DebugPassManager = ctx.arg.ltoDebugPassManager;
141 c.DwoDir = std::string(ctx.arg.dwoDir);
142
143 c.HasWholeProgramVisibility = ctx.arg.ltoWholeProgramVisibility;
144 c.ValidateAllVtablesHaveTypeInfos =
145 ctx.arg.ltoValidateAllVtablesHaveTypeInfos;
146 c.AllVtablesHaveTypeInfos = ctx.ltoAllVtablesHaveTypeInfos;
147 c.AlwaysEmitRegularLTOObj = !ctx.arg.ltoObjPath.empty();
148 c.KeepSymbolNameCopies = false;
149
150 for (const llvm::StringRef &name : ctx.arg.thinLTOModulesToCompile)
151 c.ThinLTOModulesToCompile.emplace_back(args: name);
152
153 c.TimeTraceEnabled = ctx.arg.timeTraceEnabled;
154 c.TimeTraceGranularity = ctx.arg.timeTraceGranularity;
155
156 c.CSIRProfile = std::string(ctx.arg.ltoCSProfileFile);
157 c.RunCSIRInstr = ctx.arg.ltoCSProfileGenerate;
158 c.PGOWarnMismatch = ctx.arg.ltoPGOWarnMismatch;
159
160 if (ctx.arg.emitLLVM) {
161 c.PreCodeGenModuleHook = [&ctx](size_t task, const Module &m) {
162 if (std::unique_ptr<raw_fd_ostream> os =
163 openLTOOutputFile(file: ctx.arg.outputFile))
164 WriteBitcodeToFile(M: m, Out&: *os, ShouldPreserveUseListOrder: false);
165 return false;
166 };
167 }
168
169 if (ctx.arg.ltoEmitAsm) {
170 c.CGFileType = CodeGenFileType::AssemblyFile;
171 c.Options.MCOptions.AsmVerbose = true;
172 }
173
174 if (ctx.arg.thinLTOIndexOnly) {
175 c.PostRegularLTOLinkHook = [&ctx]() {
176 markBuffersAsDontNeed(ctx, /*skipLinkedOutput=*/true);
177 };
178 }
179
180 if (!ctx.arg.saveTempsArgs.empty())
181 checkError(eh&: ctx.e, e: c.addSaveTemps(OutputFileName: ctx.arg.outputFile.str() + ".",
182 /*UseInputModulePath*/ true,
183 SaveTempsArgs: ctx.arg.saveTempsArgs));
184 return c;
185}
186
187BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) {
188 // Initialize indexFile.
189 if (!ctx.arg.thinLTOIndexOnlyArg.empty())
190 indexFile = openFile(file: ctx.arg.thinLTOIndexOnlyArg);
191
192 // Initialize ltoObj.
193 lto::ThinBackend backend;
194 auto onIndexWrite = [&](StringRef s) { thinIndices.erase(V: s); };
195 if (ctx.arg.thinLTOIndexOnly) {
196 backend = lto::createWriteIndexesThinBackend(
197 Parallelism: llvm::hardware_concurrency(Num: ctx.arg.thinLTOJobs),
198 OldPrefix: std::string(ctx.arg.thinLTOPrefixReplaceOld),
199 NewPrefix: std::string(ctx.arg.thinLTOPrefixReplaceNew),
200 NativeObjectPrefix: std::string(ctx.arg.thinLTOPrefixReplaceNativeObject),
201 ShouldEmitImportsFiles: ctx.arg.thinLTOEmitImportsFiles, LinkedObjectsFile: indexFile.get(), OnWrite: onIndexWrite);
202 } else {
203 backend = lto::createInProcessThinBackend(
204 Parallelism: llvm::heavyweight_hardware_concurrency(Num: ctx.arg.thinLTOJobs),
205 OnWrite: onIndexWrite, ShouldEmitIndexFiles: ctx.arg.thinLTOEmitIndexFiles,
206 ShouldEmitImportsFiles: ctx.arg.thinLTOEmitImportsFiles);
207 }
208
209 constexpr llvm::lto::LTO::LTOKind ltoModes[3] = {
210 llvm::lto::LTO::LTOKind::LTOK_UnifiedThin,
211 llvm::lto::LTO::LTOKind::LTOK_UnifiedRegular,
212 llvm::lto::LTO::LTOKind::LTOK_Default};
213
214 if (ctx.arg.dtltoDistributor.empty())
215 ltoObj = std::make_unique<lto::LTO>(args: createConfig(ctx), args&: backend,
216 args&: ctx.arg.ltoPartitions,
217 args: ltoModes[ctx.arg.ltoKind]);
218 else
219 ltoObj = std::make_unique<lto::DTLTO>(
220 args: createConfig(ctx), args&: ctx.arg.ltoPartitions, args: ltoModes[ctx.arg.ltoKind],
221 args&: onIndexWrite, args&: ctx.arg.thinLTOEmitIndexFiles,
222 args&: ctx.arg.thinLTOEmitImportsFiles, args&: ctx.arg.outputFile,
223 args&: ctx.arg.dtltoDistributor, args&: ctx.arg.dtltoDistributorArgs,
224 args&: ctx.arg.dtltoCompiler, args&: ctx.arg.dtltoCompilerPrependArgs,
225 args&: ctx.arg.dtltoCompilerArgs, args: createAddBufferFn(files, filenames),
226 args: !ctx.arg.saveTempsArgs.empty());
227 // Initialize usedStartStop.
228 if (ctx.bitcodeFiles.empty())
229 return;
230 for (Symbol *sym : ctx.symtab->getSymbols()) {
231 if (sym->isPlaceholder())
232 continue;
233 StringRef s = sym->getName();
234 for (StringRef prefix : {"__start_", "__stop_"})
235 if (s.starts_with(Prefix: prefix))
236 usedStartStop.insert(V: s.substr(Start: prefix.size()));
237 }
238}
239
240BitcodeCompiler::~BitcodeCompiler() = default;
241
242void BitcodeCompiler::waitForLTOCleanup() {
243 if (ltoObj)
244 ltoObj->waitForCleanup();
245}
246
247void BitcodeCompiler::add(BitcodeFile &f) {
248 lto::InputFile &obj = *f.obj;
249 bool isExec = !ctx.arg.shared && !ctx.arg.relocatable;
250
251 if (ctx.arg.thinLTOEmitIndexFiles)
252 thinIndices.insert(V: obj.getName());
253
254 ArrayRef<Symbol *> syms = f.getSymbols();
255 ArrayRef<lto::InputFile::Symbol> objSyms = obj.symbols();
256 std::vector<lto::SymbolResolution> resols(syms.size());
257
258 // Provide a resolution to the LTO API for each symbol.
259 for (size_t i = 0, e = syms.size(); i != e; ++i) {
260 Symbol *sym = syms[i];
261 const lto::InputFile::Symbol &objSym = objSyms[i];
262 lto::SymbolResolution &r = resols[i];
263
264 // Ideally we shouldn't check for SF_Undefined but currently IRObjectFile
265 // reports two symbols for module ASM defined. Without this check, lld
266 // flags an undefined in IR with a definition in ASM as prevailing.
267 // Once IRObjectFile is fixed to report only one symbol this hack can
268 // be removed.
269 r.Prevailing = !objSym.isUndefined() && sym->file == &f;
270
271 // We ask LTO to preserve following global symbols:
272 // 1) All symbols when doing relocatable link, so that them can be used
273 // for doing final link.
274 // 2) Symbols that are used in regular objects.
275 // 3) C named sections if we have corresponding __start_/__stop_ symbol.
276 // 4) Symbols that are defined in bitcode files and used for dynamic
277 // linking.
278 // 5) Symbols that will be referenced after linker wrapping is performed.
279 r.VisibleToRegularObj = ctx.arg.relocatable || sym->isUsedInRegularObj ||
280 sym->referencedAfterWrap ||
281 (r.Prevailing && sym->isExported) ||
282 usedStartStop.contains(V: objSym.getSectionName());
283 // Identify symbols exported dynamically, and that therefore could be
284 // referenced by a shared library not visible to the linker.
285 r.ExportDynamic = sym->computeBinding(ctx) != STB_LOCAL &&
286 (ctx.arg.exportDynamic || sym->isExported);
287 const auto *dr = dyn_cast<Defined>(Val: sym);
288 r.FinalDefinitionInLinkageUnit =
289 (isExec || sym->visibility() != STV_DEFAULT) && dr &&
290 // Skip absolute symbols from ELF objects, otherwise PC-rel relocations
291 // will be generated by for them, triggering linker errors.
292 // Symbol section is always null for bitcode symbols, hence the check
293 // for isElf(). Skip linker script defined symbols as well: they have
294 // no File defined.
295 !(dr->section == nullptr &&
296 (sym->file->isInternal() || sym->file->isElf()));
297
298 if (r.Prevailing)
299 Undefined(ctx.internalFile, StringRef(), STB_GLOBAL, STV_DEFAULT,
300 sym->type)
301 .overwrite(sym&: *sym);
302
303 // We tell LTO to not apply interprocedural optimization for wrapped
304 // (with --wrap) symbols because otherwise LTO would inline them while
305 // their values are still not final.
306 r.LinkerRedefined = sym->scriptDefined;
307 }
308 checkError(eh&: ctx.e, e: ltoObj->add(Obj: std::move(f.obj), Res: resols));
309}
310
311// If LazyObjFile has not been added to link, emit empty index files.
312// This is needed because this is what GNU gold plugin does and we have a
313// distributed build system that depends on that behavior.
314static void thinLTOCreateEmptyIndexFiles(Ctx &ctx) {
315 DenseSet<StringRef> linkedBitCodeFiles;
316 for (BitcodeFile *f : ctx.bitcodeFiles)
317 linkedBitCodeFiles.insert(V: f->getName());
318
319 for (BitcodeFile *f : ctx.lazyBitcodeFiles) {
320 if (!f->lazy)
321 continue;
322 if (linkedBitCodeFiles.contains(V: f->getName()))
323 continue;
324 std::string path =
325 replaceThinLTOSuffix(ctx, path: getThinLTOOutputFile(ctx, modulePath: f->obj->getName()));
326 std::unique_ptr<raw_fd_ostream> os = openFile(file: path + ".thinlto.bc");
327 if (!os)
328 continue;
329
330 ModuleSummaryIndex m(/*HaveGVs*/ false);
331 m.setSkipModuleByDistributedBackend();
332 writeIndexToFile(Index: m, Out&: *os);
333 if (ctx.arg.thinLTOEmitImportsFiles)
334 openFile(file: path + ".imports");
335 }
336}
337
338// Merge all the bitcode files we have seen, codegen the result
339// and return the resulting ObjectFile(s).
340SmallVector<std::unique_ptr<InputFile>, 0> BitcodeCompiler::compile() {
341 unsigned maxTasks = ltoObj->getMaxTasks();
342 buf.resize(N: maxTasks);
343 files.resize(new_size: maxTasks);
344 filenames.resize(N: maxTasks);
345
346 // The --thinlto-cache-dir option specifies the path to a directory in which
347 // to cache native object files for ThinLTO incremental builds. If a path was
348 // specified, configure LTO to use it as the cache directory.
349 FileCache cache;
350 if (!ctx.arg.thinLTOCacheDir.empty())
351 cache = check(e: localCache(CacheNameRef: "ThinLTO", TempFilePrefixRef: "Thin", CacheDirectoryPathRef: ctx.arg.thinLTOCacheDir,
352 AddBuffer: createAddBufferFn(files, filenames),
353 CacheFileRename: !ctx.arg.dtltoDistributor.empty()));
354
355 if (!ctx.bitcodeFiles.empty())
356 checkError(eh&: ctx.e, e: ltoObj->run(
357 AddStream: [&](size_t task, const Twine &moduleName) {
358 buf[task].first = moduleName.str();
359 return std::make_unique<CachedFileStream>(
360 args: std::make_unique<raw_svector_ostream>(
361 args&: buf[task].second));
362 },
363 Cache: cache));
364
365 // Emit empty index files for non-indexed files but not in single-module mode.
366 if (ctx.arg.thinLTOModulesToCompile.empty()) {
367 for (StringRef s : thinIndices) {
368 std::string path = getThinLTOOutputFile(ctx, modulePath: s);
369 openFile(file: path + ".thinlto.bc");
370 if (ctx.arg.thinLTOEmitImportsFiles)
371 openFile(file: path + ".imports");
372 }
373 }
374
375 if (ctx.arg.thinLTOEmitIndexFiles)
376 thinLTOCreateEmptyIndexFiles(ctx);
377
378 if (ctx.arg.thinLTOIndexOnly) {
379 if (!ctx.arg.ltoObjPath.empty())
380 saveBuffer(buffer: buf[0].second, path: ctx.arg.ltoObjPath);
381
382 // ThinLTO with index only option is required to generate only the index
383 // files. After that, we exit from linker and ThinLTO backend runs in a
384 // distributed environment.
385 if (indexFile)
386 indexFile->close();
387 return {};
388 }
389
390 if (!ctx.arg.thinLTOCacheDir.empty())
391 check(
392 e: pruneCache(Path: ctx.arg.thinLTOCacheDir, Policy: ctx.arg.thinLTOCachePolicy, Files: files));
393
394 bool savePrelink = ctx.arg.saveTempsArgs.contains(V: "prelink");
395 SmallVector<std::unique_ptr<InputFile>, 0> ret;
396 const char *ext = ctx.arg.ltoEmitAsm ? ".s" : ".o";
397 for (unsigned i = 0; i != maxTasks; ++i) {
398 StringRef bitcodeFilePath;
399 StringRef objBuf;
400 if (files[i]) {
401 // When files[i] is not null, it holds a native relocatable file provided
402 // as a MemoryBuffer, for example from the cache or from an external DTLTO
403 // backend compilation. filenames[i] contains the original BitcodeFile's
404 // identifier.
405 objBuf = files[i]->getBuffer();
406 bitcodeFilePath = filenames[i];
407 } else {
408 // Get the native relocatable file after in-process LTO compilation.
409 objBuf = buf[i].second;
410 bitcodeFilePath = buf[i].first;
411 }
412 if (!ctx.arg.ltoObjPath.empty())
413 saveBuffer(buffer: objBuf, path: ctx.arg.ltoObjPath + (i == 0 ? "" : Twine(i)));
414 if (objBuf.empty())
415 continue;
416
417 // If the input bitcode file is path/to/x.o and -o specifies a.out, the
418 // corresponding native relocatable file path will look like:
419 // path/to/a.out.lto.x.o.
420 StringRef ltoObjName;
421 if (bitcodeFilePath == "ld-temp.o") {
422 ltoObjName =
423 ctx.saver.save(S: Twine(ctx.arg.outputFile) + ".lto" +
424 (i == 0 ? Twine("") : Twine('.') + Twine(i)) + ext);
425 } else {
426 StringRef directory = sys::path::parent_path(path: bitcodeFilePath);
427 // For an archive member, which has an identifier like "d/a.a(coll.o at
428 // 8)" (see BitcodeFile::BitcodeFile), use the filename; otherwise, use
429 // the stem (d/a.o => a).
430 StringRef baseName = bitcodeFilePath.ends_with(Suffix: ")")
431 ? sys::path::filename(path: bitcodeFilePath)
432 : sys::path::stem(path: bitcodeFilePath);
433 StringRef outputFileBaseName = sys::path::filename(path: ctx.arg.outputFile);
434 SmallString<256> path;
435 sys::path::append(path, a: directory,
436 b: outputFileBaseName + ".lto." + baseName + ext);
437 sys::path::remove_dots(path, remove_dot_dot: true);
438 ltoObjName = ctx.saver.save(S: path.str());
439 }
440 if (savePrelink || ctx.arg.ltoEmitAsm)
441 saveBuffer(buffer: buf[i].second, path: ltoObjName);
442 if (!ctx.arg.ltoEmitAsm)
443 ret.push_back(Elt: createObjFile(ctx, mb: MemoryBufferRef(objBuf, ltoObjName)));
444 }
445 return ret;
446}
447
448void BitcodeCompiler::setBitcodeLibFuncs(ArrayRef<StringRef> bitcodeLibFuncs) {
449 ltoObj->setBitcodeLibFuncs(bitcodeLibFuncs);
450}
451