1//===- Bitcode/Writer/BitcodeWriter.cpp - Bitcode Writer ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Bitcode writer implementation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Bitcode/BitcodeWriter.h"
14#include "ValueEnumerator.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/DenseMap.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SetVector.h"
21#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/SmallString.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringMap.h"
25#include "llvm/ADT/StringRef.h"
26#include "llvm/Analysis/MemoryProfileInfo.h"
27#include "llvm/BinaryFormat/Dwarf.h"
28#include "llvm/Bitcode/BitcodeCommon.h"
29#include "llvm/Bitcode/BitcodeReader.h"
30#include "llvm/Bitcode/LLVMBitCodes.h"
31#include "llvm/Bitstream/BitCodes.h"
32#include "llvm/Bitstream/BitstreamWriter.h"
33#include "llvm/Config/llvm-config.h"
34#include "llvm/IR/Attributes.h"
35#include "llvm/IR/BasicBlock.h"
36#include "llvm/IR/Comdat.h"
37#include "llvm/IR/Constant.h"
38#include "llvm/IR/ConstantRangeList.h"
39#include "llvm/IR/Constants.h"
40#include "llvm/IR/DebugInfoMetadata.h"
41#include "llvm/IR/DebugLoc.h"
42#include "llvm/IR/DerivedTypes.h"
43#include "llvm/IR/Function.h"
44#include "llvm/IR/GlobalAlias.h"
45#include "llvm/IR/GlobalIFunc.h"
46#include "llvm/IR/GlobalObject.h"
47#include "llvm/IR/GlobalValue.h"
48#include "llvm/IR/GlobalVariable.h"
49#include "llvm/IR/InlineAsm.h"
50#include "llvm/IR/InstrTypes.h"
51#include "llvm/IR/Instruction.h"
52#include "llvm/IR/Instructions.h"
53#include "llvm/IR/LLVMContext.h"
54#include "llvm/IR/Metadata.h"
55#include "llvm/IR/Module.h"
56#include "llvm/IR/ModuleSummaryIndex.h"
57#include "llvm/IR/Operator.h"
58#include "llvm/IR/Type.h"
59#include "llvm/IR/UseListOrder.h"
60#include "llvm/IR/Value.h"
61#include "llvm/IR/ValueSymbolTable.h"
62#include "llvm/MC/StringTableBuilder.h"
63#include "llvm/MC/TargetRegistry.h"
64#include "llvm/Object/IRSymtab.h"
65#include "llvm/ProfileData/MemProf.h"
66#include "llvm/ProfileData/MemProfRadixTree.h"
67#include "llvm/Support/AtomicOrdering.h"
68#include "llvm/Support/Casting.h"
69#include "llvm/Support/CommandLine.h"
70#include "llvm/Support/Compiler.h"
71#include "llvm/Support/Endian.h"
72#include "llvm/Support/Error.h"
73#include "llvm/Support/ErrorHandling.h"
74#include "llvm/Support/MathExtras.h"
75#include "llvm/Support/SHA1.h"
76#include "llvm/Support/raw_ostream.h"
77#include "llvm/TargetParser/Triple.h"
78#include <algorithm>
79#include <cassert>
80#include <cstddef>
81#include <cstdint>
82#include <iterator>
83#include <map>
84#include <memory>
85#include <optional>
86#include <string>
87#include <utility>
88#include <vector>
89
90using namespace llvm;
91using namespace llvm::memprof;
92
93static cl::opt<unsigned>
94 IndexThreshold("bitcode-mdindex-threshold", cl::Hidden, cl::init(Val: 25),
95 cl::desc("Number of metadatas above which we emit an index "
96 "to enable lazy-loading"));
97static cl::opt<uint32_t> FlushThreshold(
98 "bitcode-flush-threshold", cl::Hidden, cl::init(Val: 512),
99 cl::desc("The threshold (unit M) for flushing LLVM bitcode."));
100
101static cl::opt<bool> WriteRelBFToSummary(
102 "write-relbf-to-summary", cl::Hidden, cl::init(Val: false),
103 cl::desc("Write relative block frequency to function summary "));
104
105// Since we only use the context information in the memprof summary records in
106// the LTO backends to do assertion checking, save time and space by only
107// serializing the context for non-NDEBUG builds.
108// TODO: Currently this controls writing context of the allocation info records,
109// which are larger and more expensive, but we should do this for the callsite
110// records as well.
111// FIXME: Convert to a const once this has undergone more sigificant testing.
112static cl::opt<bool>
113 CombinedIndexMemProfContext("combined-index-memprof-context", cl::Hidden,
114#ifdef NDEBUG
115 cl::init(Val: false),
116#else
117 cl::init(true),
118#endif
119 cl::desc(""));
120
121namespace llvm {
122extern FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold;
123}
124
125namespace {
126
127/// These are manifest constants used by the bitcode writer. They do not need to
128/// be kept in sync with the reader, but need to be consistent within this file.
129enum {
130 // VALUE_SYMTAB_BLOCK abbrev id's.
131 VST_ENTRY_8_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
132 VST_ENTRY_7_ABBREV,
133 VST_ENTRY_6_ABBREV,
134 VST_BBENTRY_6_ABBREV,
135
136 // CONSTANTS_BLOCK abbrev id's.
137 CONSTANTS_SETTYPE_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
138 CONSTANTS_INTEGER_ABBREV,
139 CONSTANTS_CE_CAST_Abbrev,
140 CONSTANTS_NULL_Abbrev,
141
142 // FUNCTION_BLOCK abbrev id's.
143 FUNCTION_INST_LOAD_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
144 FUNCTION_INST_STORE_ABBREV,
145 FUNCTION_INST_UNOP_ABBREV,
146 FUNCTION_INST_UNOP_FLAGS_ABBREV,
147 FUNCTION_INST_BINOP_ABBREV,
148 FUNCTION_INST_BINOP_FLAGS_ABBREV,
149 FUNCTION_INST_CAST_ABBREV,
150 FUNCTION_INST_CAST_FLAGS_ABBREV,
151 FUNCTION_INST_RET_VOID_ABBREV,
152 FUNCTION_INST_RET_VAL_ABBREV,
153 FUNCTION_INST_BR_UNCOND_ABBREV,
154 FUNCTION_INST_BR_COND_ABBREV,
155 FUNCTION_INST_UNREACHABLE_ABBREV,
156 FUNCTION_INST_GEP_ABBREV,
157 FUNCTION_INST_CMP_ABBREV,
158 FUNCTION_INST_CMP_FLAGS_ABBREV,
159 FUNCTION_DEBUG_RECORD_VALUE_ABBREV,
160};
161
162/// Abstract class to manage the bitcode writing, subclassed for each bitcode
163/// file type.
164class BitcodeWriterBase {
165protected:
166 /// The stream created and owned by the client.
167 BitstreamWriter &Stream;
168
169 StringTableBuilder &StrtabBuilder;
170
171public:
172 /// Constructs a BitcodeWriterBase object that writes to the provided
173 /// \p Stream.
174 BitcodeWriterBase(BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder)
175 : Stream(Stream), StrtabBuilder(StrtabBuilder) {}
176
177protected:
178 void writeModuleVersion();
179};
180
181void BitcodeWriterBase::writeModuleVersion() {
182 // VERSION: [version#]
183 Stream.EmitRecord(Code: bitc::MODULE_CODE_VERSION, Vals: ArrayRef<uint64_t>{2});
184}
185
186/// Base class to manage the module bitcode writing, currently subclassed for
187/// ModuleBitcodeWriter and ThinLinkBitcodeWriter.
188class ModuleBitcodeWriterBase : public BitcodeWriterBase {
189protected:
190 /// The Module to write to bitcode.
191 const Module &M;
192
193 /// Enumerates ids for all values in the module.
194 ValueEnumerator VE;
195
196 /// Optional per-module index to write for ThinLTO.
197 const ModuleSummaryIndex *Index;
198
199 /// Map that holds the correspondence between GUIDs in the summary index,
200 /// that came from indirect call profiles, and a value id generated by this
201 /// class to use in the VST and summary block records.
202 std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
203
204 /// Tracks the last value id recorded in the GUIDToValueMap.
205 unsigned GlobalValueId;
206
207 /// Saves the offset of the VSTOffset record that must eventually be
208 /// backpatched with the offset of the actual VST.
209 uint64_t VSTOffsetPlaceholder = 0;
210
211public:
212 /// Constructs a ModuleBitcodeWriterBase object for the given Module,
213 /// writing to the provided \p Buffer.
214 ModuleBitcodeWriterBase(const Module &M, StringTableBuilder &StrtabBuilder,
215 BitstreamWriter &Stream,
216 bool ShouldPreserveUseListOrder,
217 const ModuleSummaryIndex *Index)
218 : BitcodeWriterBase(Stream, StrtabBuilder), M(M),
219 VE(M, ShouldPreserveUseListOrder), Index(Index) {
220 // Assign ValueIds to any callee values in the index that came from
221 // indirect call profiles and were recorded as a GUID not a Value*
222 // (which would have been assigned an ID by the ValueEnumerator).
223 // The starting ValueId is just after the number of values in the
224 // ValueEnumerator, so that they can be emitted in the VST.
225 GlobalValueId = VE.getValues().size();
226 if (!Index)
227 return;
228 for (const auto &GUIDSummaryLists : *Index)
229 // Examine all summaries for this GUID.
230 for (auto &Summary : GUIDSummaryLists.second.SummaryList)
231 if (auto FS = dyn_cast<FunctionSummary>(Val: Summary.get())) {
232 // For each call in the function summary, see if the call
233 // is to a GUID (which means it is for an indirect call,
234 // otherwise we would have a Value for it). If so, synthesize
235 // a value id.
236 for (auto &CallEdge : FS->calls())
237 if (!CallEdge.first.haveGVs() || !CallEdge.first.getValue())
238 assignValueId(ValGUID: CallEdge.first.getGUID());
239
240 // For each referenced variables in the function summary, see if the
241 // variable is represented by a GUID (as opposed to a symbol to
242 // declarations or definitions in the module). If so, synthesize a
243 // value id.
244 for (auto &RefEdge : FS->refs())
245 if (!RefEdge.haveGVs() || !RefEdge.getValue())
246 assignValueId(ValGUID: RefEdge.getGUID());
247 }
248 }
249
250protected:
251 void writePerModuleGlobalValueSummary();
252
253private:
254 void writePerModuleFunctionSummaryRecord(
255 SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
256 unsigned ValueID, unsigned FSCallsAbbrev, unsigned FSCallsProfileAbbrev,
257 unsigned CallsiteAbbrev, unsigned AllocAbbrev, unsigned ContextIdAbbvId,
258 const Function &F, DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
259 CallStackId &CallStackCount);
260 void writeModuleLevelReferences(const GlobalVariable &V,
261 SmallVector<uint64_t, 64> &NameVals,
262 unsigned FSModRefsAbbrev,
263 unsigned FSModVTableRefsAbbrev);
264
265 void assignValueId(GlobalValue::GUID ValGUID) {
266 GUIDToValueIdMap[ValGUID] = ++GlobalValueId;
267 }
268
269 unsigned getValueId(GlobalValue::GUID ValGUID) {
270 const auto &VMI = GUIDToValueIdMap.find(x: ValGUID);
271 // Expect that any GUID value had a value Id assigned by an
272 // earlier call to assignValueId.
273 assert(VMI != GUIDToValueIdMap.end() &&
274 "GUID does not have assigned value Id");
275 return VMI->second;
276 }
277
278 // Helper to get the valueId for the type of value recorded in VI.
279 unsigned getValueId(ValueInfo VI) {
280 if (!VI.haveGVs() || !VI.getValue())
281 return getValueId(ValGUID: VI.getGUID());
282 return VE.getValueID(V: VI.getValue());
283 }
284
285 std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
286};
287
288/// Class to manage the bitcode writing for a module.
289class ModuleBitcodeWriter : public ModuleBitcodeWriterBase {
290 /// True if a module hash record should be written.
291 bool GenerateHash;
292
293 /// If non-null, when GenerateHash is true, the resulting hash is written
294 /// into ModHash.
295 ModuleHash *ModHash;
296
297 SHA1 Hasher;
298
299 /// The start bit of the identification block.
300 uint64_t BitcodeStartBit;
301
302public:
303 /// Constructs a ModuleBitcodeWriter object for the given Module,
304 /// writing to the provided \p Buffer.
305 ModuleBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
306 BitstreamWriter &Stream, bool ShouldPreserveUseListOrder,
307 const ModuleSummaryIndex *Index, bool GenerateHash,
308 ModuleHash *ModHash = nullptr)
309 : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
310 ShouldPreserveUseListOrder, Index),
311 GenerateHash(GenerateHash), ModHash(ModHash),
312 BitcodeStartBit(Stream.GetCurrentBitNo()) {}
313
314 /// Emit the current module to the bitstream.
315 void write();
316
317private:
318 uint64_t bitcodeStartBit() { return BitcodeStartBit; }
319
320 size_t addToStrtab(StringRef Str);
321
322 void writeAttributeGroupTable();
323 void writeAttributeTable();
324 void writeTypeTable();
325 void writeComdats();
326 void writeValueSymbolTableForwardDecl();
327 void writeModuleInfo();
328 void writeValueAsMetadata(const ValueAsMetadata *MD,
329 SmallVectorImpl<uint64_t> &Record);
330 void writeMDTuple(const MDTuple *N, SmallVectorImpl<uint64_t> &Record,
331 unsigned Abbrev);
332 unsigned createDILocationAbbrev();
333 void writeDILocation(const DILocation *N, SmallVectorImpl<uint64_t> &Record,
334 unsigned &Abbrev);
335 unsigned createGenericDINodeAbbrev();
336 void writeGenericDINode(const GenericDINode *N,
337 SmallVectorImpl<uint64_t> &Record, unsigned &Abbrev);
338 void writeDISubrange(const DISubrange *N, SmallVectorImpl<uint64_t> &Record,
339 unsigned Abbrev);
340 void writeDIGenericSubrange(const DIGenericSubrange *N,
341 SmallVectorImpl<uint64_t> &Record,
342 unsigned Abbrev);
343 void writeDIEnumerator(const DIEnumerator *N,
344 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
345 void writeDIBasicType(const DIBasicType *N, SmallVectorImpl<uint64_t> &Record,
346 unsigned Abbrev);
347 void writeDIFixedPointType(const DIFixedPointType *N,
348 SmallVectorImpl<uint64_t> &Record,
349 unsigned Abbrev);
350 void writeDIStringType(const DIStringType *N,
351 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
352 void writeDIDerivedType(const DIDerivedType *N,
353 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
354 void writeDISubrangeType(const DISubrangeType *N,
355 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
356 void writeDICompositeType(const DICompositeType *N,
357 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
358 void writeDISubroutineType(const DISubroutineType *N,
359 SmallVectorImpl<uint64_t> &Record,
360 unsigned Abbrev);
361 void writeDIFile(const DIFile *N, SmallVectorImpl<uint64_t> &Record,
362 unsigned Abbrev);
363 void writeDICompileUnit(const DICompileUnit *N,
364 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
365 void writeDISubprogram(const DISubprogram *N,
366 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
367 void writeDILexicalBlock(const DILexicalBlock *N,
368 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
369 void writeDILexicalBlockFile(const DILexicalBlockFile *N,
370 SmallVectorImpl<uint64_t> &Record,
371 unsigned Abbrev);
372 void writeDICommonBlock(const DICommonBlock *N,
373 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
374 void writeDINamespace(const DINamespace *N, SmallVectorImpl<uint64_t> &Record,
375 unsigned Abbrev);
376 void writeDIMacro(const DIMacro *N, SmallVectorImpl<uint64_t> &Record,
377 unsigned Abbrev);
378 void writeDIMacroFile(const DIMacroFile *N, SmallVectorImpl<uint64_t> &Record,
379 unsigned Abbrev);
380 void writeDIArgList(const DIArgList *N, SmallVectorImpl<uint64_t> &Record);
381 void writeDIModule(const DIModule *N, SmallVectorImpl<uint64_t> &Record,
382 unsigned Abbrev);
383 void writeDIAssignID(const DIAssignID *N, SmallVectorImpl<uint64_t> &Record,
384 unsigned Abbrev);
385 void writeDITemplateTypeParameter(const DITemplateTypeParameter *N,
386 SmallVectorImpl<uint64_t> &Record,
387 unsigned Abbrev);
388 void writeDITemplateValueParameter(const DITemplateValueParameter *N,
389 SmallVectorImpl<uint64_t> &Record,
390 unsigned Abbrev);
391 void writeDIGlobalVariable(const DIGlobalVariable *N,
392 SmallVectorImpl<uint64_t> &Record,
393 unsigned Abbrev);
394 void writeDILocalVariable(const DILocalVariable *N,
395 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
396 void writeDILabel(const DILabel *N,
397 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
398 void writeDIExpression(const DIExpression *N,
399 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
400 void writeDIGlobalVariableExpression(const DIGlobalVariableExpression *N,
401 SmallVectorImpl<uint64_t> &Record,
402 unsigned Abbrev);
403 void writeDIObjCProperty(const DIObjCProperty *N,
404 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
405 void writeDIImportedEntity(const DIImportedEntity *N,
406 SmallVectorImpl<uint64_t> &Record,
407 unsigned Abbrev);
408 unsigned createNamedMetadataAbbrev();
409 void writeNamedMetadata(SmallVectorImpl<uint64_t> &Record);
410 unsigned createMetadataStringsAbbrev();
411 void writeMetadataStrings(ArrayRef<const Metadata *> Strings,
412 SmallVectorImpl<uint64_t> &Record);
413 void writeMetadataRecords(ArrayRef<const Metadata *> MDs,
414 SmallVectorImpl<uint64_t> &Record,
415 std::vector<unsigned> *MDAbbrevs = nullptr,
416 std::vector<uint64_t> *IndexPos = nullptr);
417 void writeModuleMetadata();
418 void writeFunctionMetadata(const Function &F);
419 void writeFunctionMetadataAttachment(const Function &F);
420 void pushGlobalMetadataAttachment(SmallVectorImpl<uint64_t> &Record,
421 const GlobalObject &GO);
422 void writeModuleMetadataKinds();
423 void writeOperandBundleTags();
424 void writeSyncScopeNames();
425 void writeConstants(unsigned FirstVal, unsigned LastVal, bool isGlobal);
426 void writeModuleConstants();
427 bool pushValueAndType(const Value *V, unsigned InstID,
428 SmallVectorImpl<unsigned> &Vals);
429 bool pushValueOrMetadata(const Value *V, unsigned InstID,
430 SmallVectorImpl<unsigned> &Vals);
431 void writeOperandBundles(const CallBase &CB, unsigned InstID);
432 void pushValue(const Value *V, unsigned InstID,
433 SmallVectorImpl<unsigned> &Vals);
434 void pushValueSigned(const Value *V, unsigned InstID,
435 SmallVectorImpl<uint64_t> &Vals);
436 void writeInstruction(const Instruction &I, unsigned InstID,
437 SmallVectorImpl<unsigned> &Vals);
438 void writeFunctionLevelValueSymbolTable(const ValueSymbolTable &VST);
439 void writeGlobalValueSymbolTable(
440 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
441 void writeUseList(UseListOrder &&Order);
442 void writeUseListBlock(const Function *F);
443 void
444 writeFunction(const Function &F,
445 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
446 void writeBlockInfo();
447 void writeModuleHash(StringRef View);
448
449 unsigned getEncodedSyncScopeID(SyncScope::ID SSID) {
450 return unsigned(SSID);
451 }
452
453 unsigned getEncodedAlign(MaybeAlign Alignment) { return encode(A: Alignment); }
454};
455
456/// Class to manage the bitcode writing for a combined index.
457class IndexBitcodeWriter : public BitcodeWriterBase {
458 /// The combined index to write to bitcode.
459 const ModuleSummaryIndex &Index;
460
461 /// When writing combined summaries, provides the set of global value
462 /// summaries for which the value (function, function alias, etc) should be
463 /// imported as a declaration.
464 const GVSummaryPtrSet *DecSummaries = nullptr;
465
466 /// When writing a subset of the index for distributed backends, client
467 /// provides a map of modules to the corresponding GUIDs/summaries to write.
468 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex;
469
470 /// Map that holds the correspondence between the GUID used in the combined
471 /// index and a value id generated by this class to use in references.
472 std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
473
474 // The stack ids used by this index, which will be a subset of those in
475 // the full index in the case of distributed indexes.
476 std::vector<uint64_t> StackIds;
477
478 // Keep a map of the stack id indices used by records being written for this
479 // index to the index of the corresponding stack id in the above StackIds
480 // vector. Ensures we write each referenced stack id once.
481 DenseMap<unsigned, unsigned> StackIdIndicesToIndex;
482
483 /// Tracks the last value id recorded in the GUIDToValueMap.
484 unsigned GlobalValueId = 0;
485
486 /// Tracks the assignment of module paths in the module path string table to
487 /// an id assigned for use in summary references to the module path.
488 DenseMap<StringRef, uint64_t> ModuleIdMap;
489
490public:
491 /// Constructs a IndexBitcodeWriter object for the given combined index,
492 /// writing to the provided \p Buffer. When writing a subset of the index
493 /// for a distributed backend, provide a \p ModuleToSummariesForIndex map.
494 /// If provided, \p DecSummaries specifies the set of summaries for which
495 /// the corresponding functions or aliased functions should be imported as a
496 /// declaration (but not definition) for each module.
497 IndexBitcodeWriter(
498 BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder,
499 const ModuleSummaryIndex &Index,
500 const GVSummaryPtrSet *DecSummaries = nullptr,
501 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex = nullptr)
502 : BitcodeWriterBase(Stream, StrtabBuilder), Index(Index),
503 DecSummaries(DecSummaries),
504 ModuleToSummariesForIndex(ModuleToSummariesForIndex) {
505
506 // See if the StackIdIndex was already added to the StackId map and
507 // vector. If not, record it.
508 auto RecordStackIdReference = [&](unsigned StackIdIndex) {
509 // If the StackIdIndex is not yet in the map, the below insert ensures
510 // that it will point to the new StackIds vector entry we push to just
511 // below.
512 auto Inserted =
513 StackIdIndicesToIndex.insert(KV: {StackIdIndex, StackIds.size()});
514 if (Inserted.second)
515 StackIds.push_back(x: Index.getStackIdAtIndex(Index: StackIdIndex));
516 };
517
518 // Assign unique value ids to all summaries to be written, for use
519 // in writing out the call graph edges. Save the mapping from GUID
520 // to the new global value id to use when writing those edges, which
521 // are currently saved in the index in terms of GUID.
522 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
523 GUIDToValueIdMap[I.first] = ++GlobalValueId;
524 // If this is invoked for an aliasee, we want to record the above mapping,
525 // but not the information needed for its summary entry (if the aliasee is
526 // to be imported, we will invoke this separately with IsAliasee=false).
527 if (IsAliasee)
528 return;
529 auto *FS = dyn_cast<FunctionSummary>(Val: I.second);
530 if (!FS)
531 return;
532 // Record all stack id indices actually used in the summary entries being
533 // written, so that we can compact them in the case of distributed ThinLTO
534 // indexes.
535 for (auto &CI : FS->callsites()) {
536 // If the stack id list is empty, this callsite info was synthesized for
537 // a missing tail call frame. Ensure that the callee's GUID gets a value
538 // id. Normally we only generate these for defined summaries, which in
539 // the case of distributed ThinLTO is only the functions already defined
540 // in the module or that we want to import. We don't bother to include
541 // all the callee symbols as they aren't normally needed in the backend.
542 // However, for the synthesized callsite infos we do need the callee
543 // GUID in the backend so that we can correlate the identified callee
544 // with this callsite info (which for non-tail calls is done by the
545 // ordering of the callsite infos and verified via stack ids).
546 if (CI.StackIdIndices.empty()) {
547 GUIDToValueIdMap[CI.Callee.getGUID()] = ++GlobalValueId;
548 continue;
549 }
550 for (auto Idx : CI.StackIdIndices)
551 RecordStackIdReference(Idx);
552 }
553 if (CombinedIndexMemProfContext) {
554 for (auto &AI : FS->allocs())
555 for (auto &MIB : AI.MIBs)
556 for (auto Idx : MIB.StackIdIndices)
557 RecordStackIdReference(Idx);
558 }
559 });
560 }
561
562 /// The below iterator returns the GUID and associated summary.
563 using GVInfo = std::pair<GlobalValue::GUID, GlobalValueSummary *>;
564
565 /// Calls the callback for each value GUID and summary to be written to
566 /// bitcode. This hides the details of whether they are being pulled from the
567 /// entire index or just those in a provided ModuleToSummariesForIndex map.
568 template<typename Functor>
569 void forEachSummary(Functor Callback) {
570 if (ModuleToSummariesForIndex) {
571 for (auto &M : *ModuleToSummariesForIndex)
572 for (auto &Summary : M.second) {
573 Callback(Summary, false);
574 // Ensure aliasee is handled, e.g. for assigning a valueId,
575 // even if we are not importing the aliasee directly (the
576 // imported alias will contain a copy of aliasee).
577 if (auto *AS = dyn_cast<AliasSummary>(Val: Summary.getSecond()))
578 Callback({AS->getAliaseeGUID(), &AS->getAliasee()}, true);
579 }
580 } else {
581 for (auto &Summaries : Index)
582 for (auto &Summary : Summaries.second.SummaryList)
583 Callback({Summaries.first, Summary.get()}, false);
584 }
585 }
586
587 /// Calls the callback for each entry in the modulePaths StringMap that
588 /// should be written to the module path string table. This hides the details
589 /// of whether they are being pulled from the entire index or just those in a
590 /// provided ModuleToSummariesForIndex map.
591 template <typename Functor> void forEachModule(Functor Callback) {
592 if (ModuleToSummariesForIndex) {
593 for (const auto &M : *ModuleToSummariesForIndex) {
594 const auto &MPI = Index.modulePaths().find(Key: M.first);
595 if (MPI == Index.modulePaths().end()) {
596 // This should only happen if the bitcode file was empty, in which
597 // case we shouldn't be importing (the ModuleToSummariesForIndex
598 // would only include the module we are writing and index for).
599 assert(ModuleToSummariesForIndex->size() == 1);
600 continue;
601 }
602 Callback(*MPI);
603 }
604 } else {
605 // Since StringMap iteration order isn't guaranteed, order by path string
606 // first.
607 // FIXME: Make this a vector of StringMapEntry instead to avoid the later
608 // map lookup.
609 std::vector<StringRef> ModulePaths;
610 for (auto &[ModPath, _] : Index.modulePaths())
611 ModulePaths.push_back(x: ModPath);
612 llvm::sort(Start: ModulePaths.begin(), End: ModulePaths.end());
613 for (auto &ModPath : ModulePaths)
614 Callback(*Index.modulePaths().find(Key: ModPath));
615 }
616 }
617
618 /// Main entry point for writing a combined index to bitcode.
619 void write();
620
621private:
622 void writeModStrings();
623 void writeCombinedGlobalValueSummary();
624
625 std::optional<unsigned> getValueId(GlobalValue::GUID ValGUID) {
626 auto VMI = GUIDToValueIdMap.find(x: ValGUID);
627 if (VMI == GUIDToValueIdMap.end())
628 return std::nullopt;
629 return VMI->second;
630 }
631
632 std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
633};
634
635} // end anonymous namespace
636
637static unsigned getEncodedCastOpcode(unsigned Opcode) {
638 switch (Opcode) {
639 default: llvm_unreachable("Unknown cast instruction!");
640 case Instruction::Trunc : return bitc::CAST_TRUNC;
641 case Instruction::ZExt : return bitc::CAST_ZEXT;
642 case Instruction::SExt : return bitc::CAST_SEXT;
643 case Instruction::FPToUI : return bitc::CAST_FPTOUI;
644 case Instruction::FPToSI : return bitc::CAST_FPTOSI;
645 case Instruction::UIToFP : return bitc::CAST_UITOFP;
646 case Instruction::SIToFP : return bitc::CAST_SITOFP;
647 case Instruction::FPTrunc : return bitc::CAST_FPTRUNC;
648 case Instruction::FPExt : return bitc::CAST_FPEXT;
649 case Instruction::PtrToInt: return bitc::CAST_PTRTOINT;
650 case Instruction::IntToPtr: return bitc::CAST_INTTOPTR;
651 case Instruction::BitCast : return bitc::CAST_BITCAST;
652 case Instruction::AddrSpaceCast: return bitc::CAST_ADDRSPACECAST;
653 }
654}
655
656static unsigned getEncodedUnaryOpcode(unsigned Opcode) {
657 switch (Opcode) {
658 default: llvm_unreachable("Unknown binary instruction!");
659 case Instruction::FNeg: return bitc::UNOP_FNEG;
660 }
661}
662
663static unsigned getEncodedBinaryOpcode(unsigned Opcode) {
664 switch (Opcode) {
665 default: llvm_unreachable("Unknown binary instruction!");
666 case Instruction::Add:
667 case Instruction::FAdd: return bitc::BINOP_ADD;
668 case Instruction::Sub:
669 case Instruction::FSub: return bitc::BINOP_SUB;
670 case Instruction::Mul:
671 case Instruction::FMul: return bitc::BINOP_MUL;
672 case Instruction::UDiv: return bitc::BINOP_UDIV;
673 case Instruction::FDiv:
674 case Instruction::SDiv: return bitc::BINOP_SDIV;
675 case Instruction::URem: return bitc::BINOP_UREM;
676 case Instruction::FRem:
677 case Instruction::SRem: return bitc::BINOP_SREM;
678 case Instruction::Shl: return bitc::BINOP_SHL;
679 case Instruction::LShr: return bitc::BINOP_LSHR;
680 case Instruction::AShr: return bitc::BINOP_ASHR;
681 case Instruction::And: return bitc::BINOP_AND;
682 case Instruction::Or: return bitc::BINOP_OR;
683 case Instruction::Xor: return bitc::BINOP_XOR;
684 }
685}
686
687static unsigned getEncodedRMWOperation(AtomicRMWInst::BinOp Op) {
688 switch (Op) {
689 default: llvm_unreachable("Unknown RMW operation!");
690 case AtomicRMWInst::Xchg: return bitc::RMW_XCHG;
691 case AtomicRMWInst::Add: return bitc::RMW_ADD;
692 case AtomicRMWInst::Sub: return bitc::RMW_SUB;
693 case AtomicRMWInst::And: return bitc::RMW_AND;
694 case AtomicRMWInst::Nand: return bitc::RMW_NAND;
695 case AtomicRMWInst::Or: return bitc::RMW_OR;
696 case AtomicRMWInst::Xor: return bitc::RMW_XOR;
697 case AtomicRMWInst::Max: return bitc::RMW_MAX;
698 case AtomicRMWInst::Min: return bitc::RMW_MIN;
699 case AtomicRMWInst::UMax: return bitc::RMW_UMAX;
700 case AtomicRMWInst::UMin: return bitc::RMW_UMIN;
701 case AtomicRMWInst::FAdd: return bitc::RMW_FADD;
702 case AtomicRMWInst::FSub: return bitc::RMW_FSUB;
703 case AtomicRMWInst::FMax: return bitc::RMW_FMAX;
704 case AtomicRMWInst::FMin: return bitc::RMW_FMIN;
705 case AtomicRMWInst::FMaximum:
706 return bitc::RMW_FMAXIMUM;
707 case AtomicRMWInst::FMinimum:
708 return bitc::RMW_FMINIMUM;
709 case AtomicRMWInst::UIncWrap:
710 return bitc::RMW_UINC_WRAP;
711 case AtomicRMWInst::UDecWrap:
712 return bitc::RMW_UDEC_WRAP;
713 case AtomicRMWInst::USubCond:
714 return bitc::RMW_USUB_COND;
715 case AtomicRMWInst::USubSat:
716 return bitc::RMW_USUB_SAT;
717 }
718}
719
720static unsigned getEncodedOrdering(AtomicOrdering Ordering) {
721 switch (Ordering) {
722 case AtomicOrdering::NotAtomic: return bitc::ORDERING_NOTATOMIC;
723 case AtomicOrdering::Unordered: return bitc::ORDERING_UNORDERED;
724 case AtomicOrdering::Monotonic: return bitc::ORDERING_MONOTONIC;
725 case AtomicOrdering::Acquire: return bitc::ORDERING_ACQUIRE;
726 case AtomicOrdering::Release: return bitc::ORDERING_RELEASE;
727 case AtomicOrdering::AcquireRelease: return bitc::ORDERING_ACQREL;
728 case AtomicOrdering::SequentiallyConsistent: return bitc::ORDERING_SEQCST;
729 }
730 llvm_unreachable("Invalid ordering");
731}
732
733static void writeStringRecord(BitstreamWriter &Stream, unsigned Code,
734 StringRef Str, unsigned AbbrevToUse) {
735 SmallVector<unsigned, 64> Vals;
736
737 // Code: [strchar x N]
738 for (char C : Str) {
739 if (AbbrevToUse && !BitCodeAbbrevOp::isChar6(C))
740 AbbrevToUse = 0;
741 Vals.push_back(Elt: C);
742 }
743
744 // Emit the finished record.
745 Stream.EmitRecord(Code, Vals, Abbrev: AbbrevToUse);
746}
747
748static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
749 switch (Kind) {
750 case Attribute::Alignment:
751 return bitc::ATTR_KIND_ALIGNMENT;
752 case Attribute::AllocAlign:
753 return bitc::ATTR_KIND_ALLOC_ALIGN;
754 case Attribute::AllocSize:
755 return bitc::ATTR_KIND_ALLOC_SIZE;
756 case Attribute::AlwaysInline:
757 return bitc::ATTR_KIND_ALWAYS_INLINE;
758 case Attribute::Builtin:
759 return bitc::ATTR_KIND_BUILTIN;
760 case Attribute::ByVal:
761 return bitc::ATTR_KIND_BY_VAL;
762 case Attribute::Convergent:
763 return bitc::ATTR_KIND_CONVERGENT;
764 case Attribute::InAlloca:
765 return bitc::ATTR_KIND_IN_ALLOCA;
766 case Attribute::Cold:
767 return bitc::ATTR_KIND_COLD;
768 case Attribute::DisableSanitizerInstrumentation:
769 return bitc::ATTR_KIND_DISABLE_SANITIZER_INSTRUMENTATION;
770 case Attribute::FnRetThunkExtern:
771 return bitc::ATTR_KIND_FNRETTHUNK_EXTERN;
772 case Attribute::Hot:
773 return bitc::ATTR_KIND_HOT;
774 case Attribute::ElementType:
775 return bitc::ATTR_KIND_ELEMENTTYPE;
776 case Attribute::HybridPatchable:
777 return bitc::ATTR_KIND_HYBRID_PATCHABLE;
778 case Attribute::InlineHint:
779 return bitc::ATTR_KIND_INLINE_HINT;
780 case Attribute::InReg:
781 return bitc::ATTR_KIND_IN_REG;
782 case Attribute::JumpTable:
783 return bitc::ATTR_KIND_JUMP_TABLE;
784 case Attribute::MinSize:
785 return bitc::ATTR_KIND_MIN_SIZE;
786 case Attribute::AllocatedPointer:
787 return bitc::ATTR_KIND_ALLOCATED_POINTER;
788 case Attribute::AllocKind:
789 return bitc::ATTR_KIND_ALLOC_KIND;
790 case Attribute::Memory:
791 return bitc::ATTR_KIND_MEMORY;
792 case Attribute::NoFPClass:
793 return bitc::ATTR_KIND_NOFPCLASS;
794 case Attribute::Naked:
795 return bitc::ATTR_KIND_NAKED;
796 case Attribute::Nest:
797 return bitc::ATTR_KIND_NEST;
798 case Attribute::NoAlias:
799 return bitc::ATTR_KIND_NO_ALIAS;
800 case Attribute::NoBuiltin:
801 return bitc::ATTR_KIND_NO_BUILTIN;
802 case Attribute::NoCallback:
803 return bitc::ATTR_KIND_NO_CALLBACK;
804 case Attribute::NoDivergenceSource:
805 return bitc::ATTR_KIND_NO_DIVERGENCE_SOURCE;
806 case Attribute::NoDuplicate:
807 return bitc::ATTR_KIND_NO_DUPLICATE;
808 case Attribute::NoFree:
809 return bitc::ATTR_KIND_NOFREE;
810 case Attribute::NoImplicitFloat:
811 return bitc::ATTR_KIND_NO_IMPLICIT_FLOAT;
812 case Attribute::NoInline:
813 return bitc::ATTR_KIND_NO_INLINE;
814 case Attribute::NoRecurse:
815 return bitc::ATTR_KIND_NO_RECURSE;
816 case Attribute::NoMerge:
817 return bitc::ATTR_KIND_NO_MERGE;
818 case Attribute::NonLazyBind:
819 return bitc::ATTR_KIND_NON_LAZY_BIND;
820 case Attribute::NonNull:
821 return bitc::ATTR_KIND_NON_NULL;
822 case Attribute::Dereferenceable:
823 return bitc::ATTR_KIND_DEREFERENCEABLE;
824 case Attribute::DereferenceableOrNull:
825 return bitc::ATTR_KIND_DEREFERENCEABLE_OR_NULL;
826 case Attribute::NoRedZone:
827 return bitc::ATTR_KIND_NO_RED_ZONE;
828 case Attribute::NoReturn:
829 return bitc::ATTR_KIND_NO_RETURN;
830 case Attribute::NoSync:
831 return bitc::ATTR_KIND_NOSYNC;
832 case Attribute::NoCfCheck:
833 return bitc::ATTR_KIND_NOCF_CHECK;
834 case Attribute::NoProfile:
835 return bitc::ATTR_KIND_NO_PROFILE;
836 case Attribute::SkipProfile:
837 return bitc::ATTR_KIND_SKIP_PROFILE;
838 case Attribute::NoUnwind:
839 return bitc::ATTR_KIND_NO_UNWIND;
840 case Attribute::NoSanitizeBounds:
841 return bitc::ATTR_KIND_NO_SANITIZE_BOUNDS;
842 case Attribute::NoSanitizeCoverage:
843 return bitc::ATTR_KIND_NO_SANITIZE_COVERAGE;
844 case Attribute::NullPointerIsValid:
845 return bitc::ATTR_KIND_NULL_POINTER_IS_VALID;
846 case Attribute::OptimizeForDebugging:
847 return bitc::ATTR_KIND_OPTIMIZE_FOR_DEBUGGING;
848 case Attribute::OptForFuzzing:
849 return bitc::ATTR_KIND_OPT_FOR_FUZZING;
850 case Attribute::OptimizeForSize:
851 return bitc::ATTR_KIND_OPTIMIZE_FOR_SIZE;
852 case Attribute::OptimizeNone:
853 return bitc::ATTR_KIND_OPTIMIZE_NONE;
854 case Attribute::ReadNone:
855 return bitc::ATTR_KIND_READ_NONE;
856 case Attribute::ReadOnly:
857 return bitc::ATTR_KIND_READ_ONLY;
858 case Attribute::Returned:
859 return bitc::ATTR_KIND_RETURNED;
860 case Attribute::ReturnsTwice:
861 return bitc::ATTR_KIND_RETURNS_TWICE;
862 case Attribute::SExt:
863 return bitc::ATTR_KIND_S_EXT;
864 case Attribute::Speculatable:
865 return bitc::ATTR_KIND_SPECULATABLE;
866 case Attribute::StackAlignment:
867 return bitc::ATTR_KIND_STACK_ALIGNMENT;
868 case Attribute::StackProtect:
869 return bitc::ATTR_KIND_STACK_PROTECT;
870 case Attribute::StackProtectReq:
871 return bitc::ATTR_KIND_STACK_PROTECT_REQ;
872 case Attribute::StackProtectStrong:
873 return bitc::ATTR_KIND_STACK_PROTECT_STRONG;
874 case Attribute::SafeStack:
875 return bitc::ATTR_KIND_SAFESTACK;
876 case Attribute::ShadowCallStack:
877 return bitc::ATTR_KIND_SHADOWCALLSTACK;
878 case Attribute::StrictFP:
879 return bitc::ATTR_KIND_STRICT_FP;
880 case Attribute::StructRet:
881 return bitc::ATTR_KIND_STRUCT_RET;
882 case Attribute::SanitizeAddress:
883 return bitc::ATTR_KIND_SANITIZE_ADDRESS;
884 case Attribute::SanitizeHWAddress:
885 return bitc::ATTR_KIND_SANITIZE_HWADDRESS;
886 case Attribute::SanitizeThread:
887 return bitc::ATTR_KIND_SANITIZE_THREAD;
888 case Attribute::SanitizeType:
889 return bitc::ATTR_KIND_SANITIZE_TYPE;
890 case Attribute::SanitizeMemory:
891 return bitc::ATTR_KIND_SANITIZE_MEMORY;
892 case Attribute::SanitizeNumericalStability:
893 return bitc::ATTR_KIND_SANITIZE_NUMERICAL_STABILITY;
894 case Attribute::SanitizeRealtime:
895 return bitc::ATTR_KIND_SANITIZE_REALTIME;
896 case Attribute::SanitizeRealtimeBlocking:
897 return bitc::ATTR_KIND_SANITIZE_REALTIME_BLOCKING;
898 case Attribute::SpeculativeLoadHardening:
899 return bitc::ATTR_KIND_SPECULATIVE_LOAD_HARDENING;
900 case Attribute::SwiftError:
901 return bitc::ATTR_KIND_SWIFT_ERROR;
902 case Attribute::SwiftSelf:
903 return bitc::ATTR_KIND_SWIFT_SELF;
904 case Attribute::SwiftAsync:
905 return bitc::ATTR_KIND_SWIFT_ASYNC;
906 case Attribute::UWTable:
907 return bitc::ATTR_KIND_UW_TABLE;
908 case Attribute::VScaleRange:
909 return bitc::ATTR_KIND_VSCALE_RANGE;
910 case Attribute::WillReturn:
911 return bitc::ATTR_KIND_WILLRETURN;
912 case Attribute::WriteOnly:
913 return bitc::ATTR_KIND_WRITEONLY;
914 case Attribute::ZExt:
915 return bitc::ATTR_KIND_Z_EXT;
916 case Attribute::ImmArg:
917 return bitc::ATTR_KIND_IMMARG;
918 case Attribute::SanitizeMemTag:
919 return bitc::ATTR_KIND_SANITIZE_MEMTAG;
920 case Attribute::Preallocated:
921 return bitc::ATTR_KIND_PREALLOCATED;
922 case Attribute::NoUndef:
923 return bitc::ATTR_KIND_NOUNDEF;
924 case Attribute::ByRef:
925 return bitc::ATTR_KIND_BYREF;
926 case Attribute::MustProgress:
927 return bitc::ATTR_KIND_MUSTPROGRESS;
928 case Attribute::PresplitCoroutine:
929 return bitc::ATTR_KIND_PRESPLIT_COROUTINE;
930 case Attribute::Writable:
931 return bitc::ATTR_KIND_WRITABLE;
932 case Attribute::CoroDestroyOnlyWhenComplete:
933 return bitc::ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE;
934 case Attribute::CoroElideSafe:
935 return bitc::ATTR_KIND_CORO_ELIDE_SAFE;
936 case Attribute::DeadOnUnwind:
937 return bitc::ATTR_KIND_DEAD_ON_UNWIND;
938 case Attribute::Range:
939 return bitc::ATTR_KIND_RANGE;
940 case Attribute::Initializes:
941 return bitc::ATTR_KIND_INITIALIZES;
942 case Attribute::NoExt:
943 return bitc::ATTR_KIND_NO_EXT;
944 case Attribute::Captures:
945 return bitc::ATTR_KIND_CAPTURES;
946 case Attribute::DeadOnReturn:
947 return bitc::ATTR_KIND_DEAD_ON_RETURN;
948 case Attribute::EndAttrKinds:
949 llvm_unreachable("Can not encode end-attribute kinds marker.");
950 case Attribute::None:
951 llvm_unreachable("Can not encode none-attribute.");
952 case Attribute::EmptyKey:
953 case Attribute::TombstoneKey:
954 llvm_unreachable("Trying to encode EmptyKey/TombstoneKey");
955 }
956
957 llvm_unreachable("Trying to encode unknown attribute");
958}
959
960static void emitSignedInt64(SmallVectorImpl<uint64_t> &Vals, uint64_t V) {
961 if ((int64_t)V >= 0)
962 Vals.push_back(Elt: V << 1);
963 else
964 Vals.push_back(Elt: (-V << 1) | 1);
965}
966
967static void emitWideAPInt(SmallVectorImpl<uint64_t> &Vals, const APInt &A) {
968 // We have an arbitrary precision integer value to write whose
969 // bit width is > 64. However, in canonical unsigned integer
970 // format it is likely that the high bits are going to be zero.
971 // So, we only write the number of active words.
972 unsigned NumWords = A.getActiveWords();
973 const uint64_t *RawData = A.getRawData();
974 for (unsigned i = 0; i < NumWords; i++)
975 emitSignedInt64(Vals, V: RawData[i]);
976}
977
978static void emitConstantRange(SmallVectorImpl<uint64_t> &Record,
979 const ConstantRange &CR, bool EmitBitWidth) {
980 unsigned BitWidth = CR.getBitWidth();
981 if (EmitBitWidth)
982 Record.push_back(Elt: BitWidth);
983 if (BitWidth > 64) {
984 Record.push_back(Elt: CR.getLower().getActiveWords() |
985 (uint64_t(CR.getUpper().getActiveWords()) << 32));
986 emitWideAPInt(Vals&: Record, A: CR.getLower());
987 emitWideAPInt(Vals&: Record, A: CR.getUpper());
988 } else {
989 emitSignedInt64(Vals&: Record, V: CR.getLower().getSExtValue());
990 emitSignedInt64(Vals&: Record, V: CR.getUpper().getSExtValue());
991 }
992}
993
994void ModuleBitcodeWriter::writeAttributeGroupTable() {
995 const std::vector<ValueEnumerator::IndexAndAttrSet> &AttrGrps =
996 VE.getAttributeGroups();
997 if (AttrGrps.empty()) return;
998
999 Stream.EnterSubblock(BlockID: bitc::PARAMATTR_GROUP_BLOCK_ID, CodeLen: 3);
1000
1001 SmallVector<uint64_t, 64> Record;
1002 for (ValueEnumerator::IndexAndAttrSet Pair : AttrGrps) {
1003 unsigned AttrListIndex = Pair.first;
1004 AttributeSet AS = Pair.second;
1005 Record.push_back(Elt: VE.getAttributeGroupID(Group: Pair));
1006 Record.push_back(Elt: AttrListIndex);
1007
1008 for (Attribute Attr : AS) {
1009 if (Attr.isEnumAttribute()) {
1010 Record.push_back(Elt: 0);
1011 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1012 } else if (Attr.isIntAttribute()) {
1013 Record.push_back(Elt: 1);
1014 Attribute::AttrKind Kind = Attr.getKindAsEnum();
1015 Record.push_back(Elt: getAttrKindEncoding(Kind));
1016 if (Kind == Attribute::Memory) {
1017 // Version field for upgrading old memory effects.
1018 const uint64_t Version = 1;
1019 Record.push_back(Elt: (Version << 56) | Attr.getValueAsInt());
1020 } else {
1021 Record.push_back(Elt: Attr.getValueAsInt());
1022 }
1023 } else if (Attr.isStringAttribute()) {
1024 StringRef Kind = Attr.getKindAsString();
1025 StringRef Val = Attr.getValueAsString();
1026
1027 Record.push_back(Elt: Val.empty() ? 3 : 4);
1028 Record.append(in_start: Kind.begin(), in_end: Kind.end());
1029 Record.push_back(Elt: 0);
1030 if (!Val.empty()) {
1031 Record.append(in_start: Val.begin(), in_end: Val.end());
1032 Record.push_back(Elt: 0);
1033 }
1034 } else if (Attr.isTypeAttribute()) {
1035 Type *Ty = Attr.getValueAsType();
1036 Record.push_back(Elt: Ty ? 6 : 5);
1037 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1038 if (Ty)
1039 Record.push_back(Elt: VE.getTypeID(T: Attr.getValueAsType()));
1040 } else if (Attr.isConstantRangeAttribute()) {
1041 Record.push_back(Elt: 7);
1042 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1043 emitConstantRange(Record, CR: Attr.getValueAsConstantRange(),
1044 /*EmitBitWidth=*/true);
1045 } else {
1046 assert(Attr.isConstantRangeListAttribute());
1047 Record.push_back(Elt: 8);
1048 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1049 ArrayRef<ConstantRange> Val = Attr.getValueAsConstantRangeList();
1050 Record.push_back(Elt: Val.size());
1051 Record.push_back(Elt: Val[0].getBitWidth());
1052 for (auto &CR : Val)
1053 emitConstantRange(Record, CR, /*EmitBitWidth=*/false);
1054 }
1055 }
1056
1057 Stream.EmitRecord(Code: bitc::PARAMATTR_GRP_CODE_ENTRY, Vals: Record);
1058 Record.clear();
1059 }
1060
1061 Stream.ExitBlock();
1062}
1063
1064void ModuleBitcodeWriter::writeAttributeTable() {
1065 const std::vector<AttributeList> &Attrs = VE.getAttributeLists();
1066 if (Attrs.empty()) return;
1067
1068 Stream.EnterSubblock(BlockID: bitc::PARAMATTR_BLOCK_ID, CodeLen: 3);
1069
1070 SmallVector<uint64_t, 64> Record;
1071 for (const AttributeList &AL : Attrs) {
1072 for (unsigned i : AL.indexes()) {
1073 AttributeSet AS = AL.getAttributes(Index: i);
1074 if (AS.hasAttributes())
1075 Record.push_back(Elt: VE.getAttributeGroupID(Group: {i, AS}));
1076 }
1077
1078 Stream.EmitRecord(Code: bitc::PARAMATTR_CODE_ENTRY, Vals: Record);
1079 Record.clear();
1080 }
1081
1082 Stream.ExitBlock();
1083}
1084
1085/// WriteTypeTable - Write out the type table for a module.
1086void ModuleBitcodeWriter::writeTypeTable() {
1087 const ValueEnumerator::TypeList &TypeList = VE.getTypes();
1088
1089 Stream.EnterSubblock(BlockID: bitc::TYPE_BLOCK_ID_NEW, CodeLen: 4 /*count from # abbrevs */);
1090 SmallVector<uint64_t, 64> TypeVals;
1091
1092 uint64_t NumBits = VE.computeBitsRequiredForTypeIndices();
1093
1094 // Abbrev for TYPE_CODE_OPAQUE_POINTER.
1095 auto Abbv = std::make_shared<BitCodeAbbrev>();
1096 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_OPAQUE_POINTER));
1097 Abbv->Add(OpInfo: BitCodeAbbrevOp(0)); // Addrspace = 0
1098 unsigned OpaquePtrAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1099
1100 // Abbrev for TYPE_CODE_FUNCTION.
1101 Abbv = std::make_shared<BitCodeAbbrev>();
1102 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_FUNCTION));
1103 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isvararg
1104 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1105 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1106 unsigned FunctionAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1107
1108 // Abbrev for TYPE_CODE_STRUCT_ANON.
1109 Abbv = std::make_shared<BitCodeAbbrev>();
1110 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_ANON));
1111 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked
1112 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1113 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1114 unsigned StructAnonAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1115
1116 // Abbrev for TYPE_CODE_STRUCT_NAME.
1117 Abbv = std::make_shared<BitCodeAbbrev>();
1118 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAME));
1119 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1120 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
1121 unsigned StructNameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1122
1123 // Abbrev for TYPE_CODE_STRUCT_NAMED.
1124 Abbv = std::make_shared<BitCodeAbbrev>();
1125 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAMED));
1126 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked
1127 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1128 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1129 unsigned StructNamedAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1130
1131 // Abbrev for TYPE_CODE_ARRAY.
1132 Abbv = std::make_shared<BitCodeAbbrev>();
1133 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_ARRAY));
1134 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // size
1135 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1136 unsigned ArrayAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1137
1138 // Emit an entry count so the reader can reserve space.
1139 TypeVals.push_back(Elt: TypeList.size());
1140 Stream.EmitRecord(Code: bitc::TYPE_CODE_NUMENTRY, Vals: TypeVals);
1141 TypeVals.clear();
1142
1143 // Loop over all of the types, emitting each in turn.
1144 for (Type *T : TypeList) {
1145 int AbbrevToUse = 0;
1146 unsigned Code = 0;
1147
1148 switch (T->getTypeID()) {
1149 case Type::VoidTyID: Code = bitc::TYPE_CODE_VOID; break;
1150 case Type::HalfTyID: Code = bitc::TYPE_CODE_HALF; break;
1151 case Type::BFloatTyID: Code = bitc::TYPE_CODE_BFLOAT; break;
1152 case Type::FloatTyID: Code = bitc::TYPE_CODE_FLOAT; break;
1153 case Type::DoubleTyID: Code = bitc::TYPE_CODE_DOUBLE; break;
1154 case Type::X86_FP80TyID: Code = bitc::TYPE_CODE_X86_FP80; break;
1155 case Type::FP128TyID: Code = bitc::TYPE_CODE_FP128; break;
1156 case Type::PPC_FP128TyID: Code = bitc::TYPE_CODE_PPC_FP128; break;
1157 case Type::LabelTyID: Code = bitc::TYPE_CODE_LABEL; break;
1158 case Type::MetadataTyID:
1159 Code = bitc::TYPE_CODE_METADATA;
1160 break;
1161 case Type::X86_AMXTyID: Code = bitc::TYPE_CODE_X86_AMX; break;
1162 case Type::TokenTyID: Code = bitc::TYPE_CODE_TOKEN; break;
1163 case Type::IntegerTyID:
1164 // INTEGER: [width]
1165 Code = bitc::TYPE_CODE_INTEGER;
1166 TypeVals.push_back(Elt: cast<IntegerType>(Val: T)->getBitWidth());
1167 break;
1168 case Type::PointerTyID: {
1169 PointerType *PTy = cast<PointerType>(Val: T);
1170 unsigned AddressSpace = PTy->getAddressSpace();
1171 // OPAQUE_POINTER: [address space]
1172 Code = bitc::TYPE_CODE_OPAQUE_POINTER;
1173 TypeVals.push_back(Elt: AddressSpace);
1174 if (AddressSpace == 0)
1175 AbbrevToUse = OpaquePtrAbbrev;
1176 break;
1177 }
1178 case Type::FunctionTyID: {
1179 FunctionType *FT = cast<FunctionType>(Val: T);
1180 // FUNCTION: [isvararg, retty, paramty x N]
1181 Code = bitc::TYPE_CODE_FUNCTION;
1182 TypeVals.push_back(Elt: FT->isVarArg());
1183 TypeVals.push_back(Elt: VE.getTypeID(T: FT->getReturnType()));
1184 for (unsigned i = 0, e = FT->getNumParams(); i != e; ++i)
1185 TypeVals.push_back(Elt: VE.getTypeID(T: FT->getParamType(i)));
1186 AbbrevToUse = FunctionAbbrev;
1187 break;
1188 }
1189 case Type::StructTyID: {
1190 StructType *ST = cast<StructType>(Val: T);
1191 // STRUCT: [ispacked, eltty x N]
1192 TypeVals.push_back(Elt: ST->isPacked());
1193 // Output all of the element types.
1194 for (Type *ET : ST->elements())
1195 TypeVals.push_back(Elt: VE.getTypeID(T: ET));
1196
1197 if (ST->isLiteral()) {
1198 Code = bitc::TYPE_CODE_STRUCT_ANON;
1199 AbbrevToUse = StructAnonAbbrev;
1200 } else {
1201 if (ST->isOpaque()) {
1202 Code = bitc::TYPE_CODE_OPAQUE;
1203 } else {
1204 Code = bitc::TYPE_CODE_STRUCT_NAMED;
1205 AbbrevToUse = StructNamedAbbrev;
1206 }
1207
1208 // Emit the name if it is present.
1209 if (!ST->getName().empty())
1210 writeStringRecord(Stream, Code: bitc::TYPE_CODE_STRUCT_NAME, Str: ST->getName(),
1211 AbbrevToUse: StructNameAbbrev);
1212 }
1213 break;
1214 }
1215 case Type::ArrayTyID: {
1216 ArrayType *AT = cast<ArrayType>(Val: T);
1217 // ARRAY: [numelts, eltty]
1218 Code = bitc::TYPE_CODE_ARRAY;
1219 TypeVals.push_back(Elt: AT->getNumElements());
1220 TypeVals.push_back(Elt: VE.getTypeID(T: AT->getElementType()));
1221 AbbrevToUse = ArrayAbbrev;
1222 break;
1223 }
1224 case Type::FixedVectorTyID:
1225 case Type::ScalableVectorTyID: {
1226 VectorType *VT = cast<VectorType>(Val: T);
1227 // VECTOR [numelts, eltty] or
1228 // [numelts, eltty, scalable]
1229 Code = bitc::TYPE_CODE_VECTOR;
1230 TypeVals.push_back(Elt: VT->getElementCount().getKnownMinValue());
1231 TypeVals.push_back(Elt: VE.getTypeID(T: VT->getElementType()));
1232 if (isa<ScalableVectorType>(Val: VT))
1233 TypeVals.push_back(Elt: true);
1234 break;
1235 }
1236 case Type::TargetExtTyID: {
1237 TargetExtType *TET = cast<TargetExtType>(Val: T);
1238 Code = bitc::TYPE_CODE_TARGET_TYPE;
1239 writeStringRecord(Stream, Code: bitc::TYPE_CODE_STRUCT_NAME, Str: TET->getName(),
1240 AbbrevToUse: StructNameAbbrev);
1241 TypeVals.push_back(Elt: TET->getNumTypeParameters());
1242 for (Type *InnerTy : TET->type_params())
1243 TypeVals.push_back(Elt: VE.getTypeID(T: InnerTy));
1244 llvm::append_range(C&: TypeVals, R: TET->int_params());
1245 break;
1246 }
1247 case Type::TypedPointerTyID:
1248 llvm_unreachable("Typed pointers cannot be added to IR modules");
1249 }
1250
1251 // Emit the finished record.
1252 Stream.EmitRecord(Code, Vals: TypeVals, Abbrev: AbbrevToUse);
1253 TypeVals.clear();
1254 }
1255
1256 Stream.ExitBlock();
1257}
1258
1259static unsigned getEncodedLinkage(const GlobalValue::LinkageTypes Linkage) {
1260 switch (Linkage) {
1261 case GlobalValue::ExternalLinkage:
1262 return 0;
1263 case GlobalValue::WeakAnyLinkage:
1264 return 16;
1265 case GlobalValue::AppendingLinkage:
1266 return 2;
1267 case GlobalValue::InternalLinkage:
1268 return 3;
1269 case GlobalValue::LinkOnceAnyLinkage:
1270 return 18;
1271 case GlobalValue::ExternalWeakLinkage:
1272 return 7;
1273 case GlobalValue::CommonLinkage:
1274 return 8;
1275 case GlobalValue::PrivateLinkage:
1276 return 9;
1277 case GlobalValue::WeakODRLinkage:
1278 return 17;
1279 case GlobalValue::LinkOnceODRLinkage:
1280 return 19;
1281 case GlobalValue::AvailableExternallyLinkage:
1282 return 12;
1283 }
1284 llvm_unreachable("Invalid linkage");
1285}
1286
1287static unsigned getEncodedLinkage(const GlobalValue &GV) {
1288 return getEncodedLinkage(Linkage: GV.getLinkage());
1289}
1290
1291static uint64_t getEncodedFFlags(FunctionSummary::FFlags Flags) {
1292 uint64_t RawFlags = 0;
1293 RawFlags |= Flags.ReadNone;
1294 RawFlags |= (Flags.ReadOnly << 1);
1295 RawFlags |= (Flags.NoRecurse << 2);
1296 RawFlags |= (Flags.ReturnDoesNotAlias << 3);
1297 RawFlags |= (Flags.NoInline << 4);
1298 RawFlags |= (Flags.AlwaysInline << 5);
1299 RawFlags |= (Flags.NoUnwind << 6);
1300 RawFlags |= (Flags.MayThrow << 7);
1301 RawFlags |= (Flags.HasUnknownCall << 8);
1302 RawFlags |= (Flags.MustBeUnreachable << 9);
1303 return RawFlags;
1304}
1305
1306// Decode the flags for GlobalValue in the summary. See getDecodedGVSummaryFlags
1307// in BitcodeReader.cpp.
1308static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags,
1309 bool ImportAsDecl = false) {
1310 uint64_t RawFlags = 0;
1311
1312 RawFlags |= Flags.NotEligibleToImport; // bool
1313 RawFlags |= (Flags.Live << 1);
1314 RawFlags |= (Flags.DSOLocal << 2);
1315 RawFlags |= (Flags.CanAutoHide << 3);
1316
1317 // Linkage don't need to be remapped at that time for the summary. Any future
1318 // change to the getEncodedLinkage() function will need to be taken into
1319 // account here as well.
1320 RawFlags = (RawFlags << 4) | Flags.Linkage; // 4 bits
1321
1322 RawFlags |= (Flags.Visibility << 8); // 2 bits
1323
1324 unsigned ImportType = Flags.ImportType | ImportAsDecl;
1325 RawFlags |= (ImportType << 10); // 1 bit
1326
1327 return RawFlags;
1328}
1329
1330static uint64_t getEncodedGVarFlags(GlobalVarSummary::GVarFlags Flags) {
1331 uint64_t RawFlags = Flags.MaybeReadOnly | (Flags.MaybeWriteOnly << 1) |
1332 (Flags.Constant << 2) | Flags.VCallVisibility << 3;
1333 return RawFlags;
1334}
1335
1336static uint64_t getEncodedHotnessCallEdgeInfo(const CalleeInfo &CI) {
1337 uint64_t RawFlags = 0;
1338
1339 RawFlags |= CI.Hotness; // 3 bits
1340 RawFlags |= (CI.HasTailCall << 3); // 1 bit
1341
1342 return RawFlags;
1343}
1344
1345static uint64_t getEncodedRelBFCallEdgeInfo(const CalleeInfo &CI) {
1346 uint64_t RawFlags = 0;
1347
1348 RawFlags |= CI.RelBlockFreq; // CalleeInfo::RelBlockFreqBits bits
1349 RawFlags |= (CI.HasTailCall << CalleeInfo::RelBlockFreqBits); // 1 bit
1350
1351 return RawFlags;
1352}
1353
1354static unsigned getEncodedVisibility(const GlobalValue &GV) {
1355 switch (GV.getVisibility()) {
1356 case GlobalValue::DefaultVisibility: return 0;
1357 case GlobalValue::HiddenVisibility: return 1;
1358 case GlobalValue::ProtectedVisibility: return 2;
1359 }
1360 llvm_unreachable("Invalid visibility");
1361}
1362
1363static unsigned getEncodedDLLStorageClass(const GlobalValue &GV) {
1364 switch (GV.getDLLStorageClass()) {
1365 case GlobalValue::DefaultStorageClass: return 0;
1366 case GlobalValue::DLLImportStorageClass: return 1;
1367 case GlobalValue::DLLExportStorageClass: return 2;
1368 }
1369 llvm_unreachable("Invalid DLL storage class");
1370}
1371
1372static unsigned getEncodedThreadLocalMode(const GlobalValue &GV) {
1373 switch (GV.getThreadLocalMode()) {
1374 case GlobalVariable::NotThreadLocal: return 0;
1375 case GlobalVariable::GeneralDynamicTLSModel: return 1;
1376 case GlobalVariable::LocalDynamicTLSModel: return 2;
1377 case GlobalVariable::InitialExecTLSModel: return 3;
1378 case GlobalVariable::LocalExecTLSModel: return 4;
1379 }
1380 llvm_unreachable("Invalid TLS model");
1381}
1382
1383static unsigned getEncodedComdatSelectionKind(const Comdat &C) {
1384 switch (C.getSelectionKind()) {
1385 case Comdat::Any:
1386 return bitc::COMDAT_SELECTION_KIND_ANY;
1387 case Comdat::ExactMatch:
1388 return bitc::COMDAT_SELECTION_KIND_EXACT_MATCH;
1389 case Comdat::Largest:
1390 return bitc::COMDAT_SELECTION_KIND_LARGEST;
1391 case Comdat::NoDeduplicate:
1392 return bitc::COMDAT_SELECTION_KIND_NO_DUPLICATES;
1393 case Comdat::SameSize:
1394 return bitc::COMDAT_SELECTION_KIND_SAME_SIZE;
1395 }
1396 llvm_unreachable("Invalid selection kind");
1397}
1398
1399static unsigned getEncodedUnnamedAddr(const GlobalValue &GV) {
1400 switch (GV.getUnnamedAddr()) {
1401 case GlobalValue::UnnamedAddr::None: return 0;
1402 case GlobalValue::UnnamedAddr::Local: return 2;
1403 case GlobalValue::UnnamedAddr::Global: return 1;
1404 }
1405 llvm_unreachable("Invalid unnamed_addr");
1406}
1407
1408size_t ModuleBitcodeWriter::addToStrtab(StringRef Str) {
1409 if (GenerateHash)
1410 Hasher.update(Str);
1411 return StrtabBuilder.add(S: Str);
1412}
1413
1414void ModuleBitcodeWriter::writeComdats() {
1415 SmallVector<unsigned, 64> Vals;
1416 for (const Comdat *C : VE.getComdats()) {
1417 // COMDAT: [strtab offset, strtab size, selection_kind]
1418 Vals.push_back(Elt: addToStrtab(Str: C->getName()));
1419 Vals.push_back(Elt: C->getName().size());
1420 Vals.push_back(Elt: getEncodedComdatSelectionKind(C: *C));
1421 Stream.EmitRecord(Code: bitc::MODULE_CODE_COMDAT, Vals, /*AbbrevToUse=*/Abbrev: 0);
1422 Vals.clear();
1423 }
1424}
1425
1426/// Write a record that will eventually hold the word offset of the
1427/// module-level VST. For now the offset is 0, which will be backpatched
1428/// after the real VST is written. Saves the bit offset to backpatch.
1429void ModuleBitcodeWriter::writeValueSymbolTableForwardDecl() {
1430 // Write a placeholder value in for the offset of the real VST,
1431 // which is written after the function blocks so that it can include
1432 // the offset of each function. The placeholder offset will be
1433 // updated when the real VST is written.
1434 auto Abbv = std::make_shared<BitCodeAbbrev>();
1435 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_VSTOFFSET));
1436 // Blocks are 32-bit aligned, so we can use a 32-bit word offset to
1437 // hold the real VST offset. Must use fixed instead of VBR as we don't
1438 // know how many VBR chunks to reserve ahead of time.
1439 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
1440 unsigned VSTOffsetAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1441
1442 // Emit the placeholder
1443 uint64_t Vals[] = {bitc::MODULE_CODE_VSTOFFSET, 0};
1444 Stream.EmitRecordWithAbbrev(Abbrev: VSTOffsetAbbrev, Vals);
1445
1446 // Compute and save the bit offset to the placeholder, which will be
1447 // patched when the real VST is written. We can simply subtract the 32-bit
1448 // fixed size from the current bit number to get the location to backpatch.
1449 VSTOffsetPlaceholder = Stream.GetCurrentBitNo() - 32;
1450}
1451
1452enum StringEncoding { SE_Char6, SE_Fixed7, SE_Fixed8 };
1453
1454/// Determine the encoding to use for the given string name and length.
1455static StringEncoding getStringEncoding(StringRef Str) {
1456 bool isChar6 = true;
1457 for (char C : Str) {
1458 if (isChar6)
1459 isChar6 = BitCodeAbbrevOp::isChar6(C);
1460 if ((unsigned char)C & 128)
1461 // don't bother scanning the rest.
1462 return SE_Fixed8;
1463 }
1464 if (isChar6)
1465 return SE_Char6;
1466 return SE_Fixed7;
1467}
1468
1469static_assert(sizeof(GlobalValue::SanitizerMetadata) <= sizeof(unsigned),
1470 "Sanitizer Metadata is too large for naive serialization.");
1471static unsigned
1472serializeSanitizerMetadata(const GlobalValue::SanitizerMetadata &Meta) {
1473 return Meta.NoAddress | (Meta.NoHWAddress << 1) |
1474 (Meta.Memtag << 2) | (Meta.IsDynInit << 3);
1475}
1476
1477/// Emit top-level description of module, including target triple, inline asm,
1478/// descriptors for global variables, and function prototype info.
1479/// Returns the bit offset to backpatch with the location of the real VST.
1480void ModuleBitcodeWriter::writeModuleInfo() {
1481 // Emit various pieces of data attached to a module.
1482 if (!M.getTargetTriple().empty())
1483 writeStringRecord(Stream, Code: bitc::MODULE_CODE_TRIPLE,
1484 Str: M.getTargetTriple().str(), AbbrevToUse: 0 /*TODO*/);
1485 const std::string &DL = M.getDataLayoutStr();
1486 if (!DL.empty())
1487 writeStringRecord(Stream, Code: bitc::MODULE_CODE_DATALAYOUT, Str: DL, AbbrevToUse: 0 /*TODO*/);
1488 if (!M.getModuleInlineAsm().empty())
1489 writeStringRecord(Stream, Code: bitc::MODULE_CODE_ASM, Str: M.getModuleInlineAsm(),
1490 AbbrevToUse: 0 /*TODO*/);
1491
1492 // Emit information about sections and GC, computing how many there are. Also
1493 // compute the maximum alignment value.
1494 std::map<std::string, unsigned> SectionMap;
1495 std::map<std::string, unsigned> GCMap;
1496 MaybeAlign MaxAlignment;
1497 unsigned MaxGlobalType = 0;
1498 const auto UpdateMaxAlignment = [&MaxAlignment](const MaybeAlign A) {
1499 if (A)
1500 MaxAlignment = !MaxAlignment ? *A : std::max(a: *MaxAlignment, b: *A);
1501 };
1502 for (const GlobalVariable &GV : M.globals()) {
1503 UpdateMaxAlignment(GV.getAlign());
1504 MaxGlobalType = std::max(a: MaxGlobalType, b: VE.getTypeID(T: GV.getValueType()));
1505 if (GV.hasSection()) {
1506 // Give section names unique ID's.
1507 unsigned &Entry = SectionMap[std::string(GV.getSection())];
1508 if (!Entry) {
1509 writeStringRecord(Stream, Code: bitc::MODULE_CODE_SECTIONNAME, Str: GV.getSection(),
1510 AbbrevToUse: 0 /*TODO*/);
1511 Entry = SectionMap.size();
1512 }
1513 }
1514 }
1515 for (const Function &F : M) {
1516 UpdateMaxAlignment(F.getAlign());
1517 if (F.hasSection()) {
1518 // Give section names unique ID's.
1519 unsigned &Entry = SectionMap[std::string(F.getSection())];
1520 if (!Entry) {
1521 writeStringRecord(Stream, Code: bitc::MODULE_CODE_SECTIONNAME, Str: F.getSection(),
1522 AbbrevToUse: 0 /*TODO*/);
1523 Entry = SectionMap.size();
1524 }
1525 }
1526 if (F.hasGC()) {
1527 // Same for GC names.
1528 unsigned &Entry = GCMap[F.getGC()];
1529 if (!Entry) {
1530 writeStringRecord(Stream, Code: bitc::MODULE_CODE_GCNAME, Str: F.getGC(),
1531 AbbrevToUse: 0 /*TODO*/);
1532 Entry = GCMap.size();
1533 }
1534 }
1535 }
1536
1537 // Emit abbrev for globals, now that we know # sections and max alignment.
1538 unsigned SimpleGVarAbbrev = 0;
1539 if (!M.global_empty()) {
1540 // Add an abbrev for common globals with no visibility or thread localness.
1541 auto Abbv = std::make_shared<BitCodeAbbrev>();
1542 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_GLOBALVAR));
1543 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1544 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1545 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1546 Log2_32_Ceil(Value: MaxGlobalType+1)));
1547 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // AddrSpace << 2
1548 //| explicitType << 1
1549 //| constant
1550 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Initializer.
1551 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // Linkage.
1552 if (!MaxAlignment) // Alignment.
1553 Abbv->Add(OpInfo: BitCodeAbbrevOp(0));
1554 else {
1555 unsigned MaxEncAlignment = getEncodedAlign(Alignment: MaxAlignment);
1556 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1557 Log2_32_Ceil(Value: MaxEncAlignment+1)));
1558 }
1559 if (SectionMap.empty()) // Section.
1560 Abbv->Add(OpInfo: BitCodeAbbrevOp(0));
1561 else
1562 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1563 Log2_32_Ceil(Value: SectionMap.size()+1)));
1564 // Don't bother emitting vis + thread local.
1565 SimpleGVarAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1566 }
1567
1568 SmallVector<unsigned, 64> Vals;
1569 // Emit the module's source file name.
1570 {
1571 StringEncoding Bits = getStringEncoding(Str: M.getSourceFileName());
1572 BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
1573 if (Bits == SE_Char6)
1574 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
1575 else if (Bits == SE_Fixed7)
1576 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
1577
1578 // MODULE_CODE_SOURCE_FILENAME: [namechar x N]
1579 auto Abbv = std::make_shared<BitCodeAbbrev>();
1580 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
1581 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1582 Abbv->Add(OpInfo: AbbrevOpToUse);
1583 unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1584
1585 for (const auto P : M.getSourceFileName())
1586 Vals.push_back(Elt: (unsigned char)P);
1587
1588 // Emit the finished record.
1589 Stream.EmitRecord(Code: bitc::MODULE_CODE_SOURCE_FILENAME, Vals, Abbrev: FilenameAbbrev);
1590 Vals.clear();
1591 }
1592
1593 // Emit the global variable information.
1594 for (const GlobalVariable &GV : M.globals()) {
1595 unsigned AbbrevToUse = 0;
1596
1597 // GLOBALVAR: [strtab offset, strtab size, type, isconst, initid,
1598 // linkage, alignment, section, visibility, threadlocal,
1599 // unnamed_addr, externally_initialized, dllstorageclass,
1600 // comdat, attributes, DSO_Local, GlobalSanitizer, code_model]
1601 Vals.push_back(Elt: addToStrtab(Str: GV.getName()));
1602 Vals.push_back(Elt: GV.getName().size());
1603 Vals.push_back(Elt: VE.getTypeID(T: GV.getValueType()));
1604 Vals.push_back(Elt: GV.getType()->getAddressSpace() << 2 | 2 | GV.isConstant());
1605 Vals.push_back(Elt: GV.isDeclaration() ? 0 :
1606 (VE.getValueID(V: GV.getInitializer()) + 1));
1607 Vals.push_back(Elt: getEncodedLinkage(GV));
1608 Vals.push_back(Elt: getEncodedAlign(Alignment: GV.getAlign()));
1609 Vals.push_back(Elt: GV.hasSection() ? SectionMap[std::string(GV.getSection())]
1610 : 0);
1611 if (GV.isThreadLocal() ||
1612 GV.getVisibility() != GlobalValue::DefaultVisibility ||
1613 GV.getUnnamedAddr() != GlobalValue::UnnamedAddr::None ||
1614 GV.isExternallyInitialized() ||
1615 GV.getDLLStorageClass() != GlobalValue::DefaultStorageClass ||
1616 GV.hasComdat() || GV.hasAttributes() || GV.isDSOLocal() ||
1617 GV.hasPartition() || GV.hasSanitizerMetadata() || GV.getCodeModel()) {
1618 Vals.push_back(Elt: getEncodedVisibility(GV));
1619 Vals.push_back(Elt: getEncodedThreadLocalMode(GV));
1620 Vals.push_back(Elt: getEncodedUnnamedAddr(GV));
1621 Vals.push_back(Elt: GV.isExternallyInitialized());
1622 Vals.push_back(Elt: getEncodedDLLStorageClass(GV));
1623 Vals.push_back(Elt: GV.hasComdat() ? VE.getComdatID(C: GV.getComdat()) : 0);
1624
1625 auto AL = GV.getAttributesAsList(index: AttributeList::FunctionIndex);
1626 Vals.push_back(Elt: VE.getAttributeListID(PAL: AL));
1627
1628 Vals.push_back(Elt: GV.isDSOLocal());
1629 Vals.push_back(Elt: addToStrtab(Str: GV.getPartition()));
1630 Vals.push_back(Elt: GV.getPartition().size());
1631
1632 Vals.push_back(Elt: (GV.hasSanitizerMetadata() ? serializeSanitizerMetadata(
1633 Meta: GV.getSanitizerMetadata())
1634 : 0));
1635 Vals.push_back(Elt: GV.getCodeModelRaw());
1636 } else {
1637 AbbrevToUse = SimpleGVarAbbrev;
1638 }
1639
1640 Stream.EmitRecord(Code: bitc::MODULE_CODE_GLOBALVAR, Vals, Abbrev: AbbrevToUse);
1641 Vals.clear();
1642 }
1643
1644 // Emit the function proto information.
1645 for (const Function &F : M) {
1646 // FUNCTION: [strtab offset, strtab size, type, callingconv, isproto,
1647 // linkage, paramattrs, alignment, section, visibility, gc,
1648 // unnamed_addr, prologuedata, dllstorageclass, comdat,
1649 // prefixdata, personalityfn, DSO_Local, addrspace]
1650 Vals.push_back(Elt: addToStrtab(Str: F.getName()));
1651 Vals.push_back(Elt: F.getName().size());
1652 Vals.push_back(Elt: VE.getTypeID(T: F.getFunctionType()));
1653 Vals.push_back(Elt: F.getCallingConv());
1654 Vals.push_back(Elt: F.isDeclaration());
1655 Vals.push_back(Elt: getEncodedLinkage(GV: F));
1656 Vals.push_back(Elt: VE.getAttributeListID(PAL: F.getAttributes()));
1657 Vals.push_back(Elt: getEncodedAlign(Alignment: F.getAlign()));
1658 Vals.push_back(Elt: F.hasSection() ? SectionMap[std::string(F.getSection())]
1659 : 0);
1660 Vals.push_back(Elt: getEncodedVisibility(GV: F));
1661 Vals.push_back(Elt: F.hasGC() ? GCMap[F.getGC()] : 0);
1662 Vals.push_back(Elt: getEncodedUnnamedAddr(GV: F));
1663 Vals.push_back(Elt: F.hasPrologueData() ? (VE.getValueID(V: F.getPrologueData()) + 1)
1664 : 0);
1665 Vals.push_back(Elt: getEncodedDLLStorageClass(GV: F));
1666 Vals.push_back(Elt: F.hasComdat() ? VE.getComdatID(C: F.getComdat()) : 0);
1667 Vals.push_back(Elt: F.hasPrefixData() ? (VE.getValueID(V: F.getPrefixData()) + 1)
1668 : 0);
1669 Vals.push_back(
1670 Elt: F.hasPersonalityFn() ? (VE.getValueID(V: F.getPersonalityFn()) + 1) : 0);
1671
1672 Vals.push_back(Elt: F.isDSOLocal());
1673 Vals.push_back(Elt: F.getAddressSpace());
1674 Vals.push_back(Elt: addToStrtab(Str: F.getPartition()));
1675 Vals.push_back(Elt: F.getPartition().size());
1676
1677 unsigned AbbrevToUse = 0;
1678 Stream.EmitRecord(Code: bitc::MODULE_CODE_FUNCTION, Vals, Abbrev: AbbrevToUse);
1679 Vals.clear();
1680 }
1681
1682 // Emit the alias information.
1683 for (const GlobalAlias &A : M.aliases()) {
1684 // ALIAS: [strtab offset, strtab size, alias type, aliasee val#, linkage,
1685 // visibility, dllstorageclass, threadlocal, unnamed_addr,
1686 // DSO_Local]
1687 Vals.push_back(Elt: addToStrtab(Str: A.getName()));
1688 Vals.push_back(Elt: A.getName().size());
1689 Vals.push_back(Elt: VE.getTypeID(T: A.getValueType()));
1690 Vals.push_back(Elt: A.getType()->getAddressSpace());
1691 Vals.push_back(Elt: VE.getValueID(V: A.getAliasee()));
1692 Vals.push_back(Elt: getEncodedLinkage(GV: A));
1693 Vals.push_back(Elt: getEncodedVisibility(GV: A));
1694 Vals.push_back(Elt: getEncodedDLLStorageClass(GV: A));
1695 Vals.push_back(Elt: getEncodedThreadLocalMode(GV: A));
1696 Vals.push_back(Elt: getEncodedUnnamedAddr(GV: A));
1697 Vals.push_back(Elt: A.isDSOLocal());
1698 Vals.push_back(Elt: addToStrtab(Str: A.getPartition()));
1699 Vals.push_back(Elt: A.getPartition().size());
1700
1701 unsigned AbbrevToUse = 0;
1702 Stream.EmitRecord(Code: bitc::MODULE_CODE_ALIAS, Vals, Abbrev: AbbrevToUse);
1703 Vals.clear();
1704 }
1705
1706 // Emit the ifunc information.
1707 for (const GlobalIFunc &I : M.ifuncs()) {
1708 // IFUNC: [strtab offset, strtab size, ifunc type, address space, resolver
1709 // val#, linkage, visibility, DSO_Local]
1710 Vals.push_back(Elt: addToStrtab(Str: I.getName()));
1711 Vals.push_back(Elt: I.getName().size());
1712 Vals.push_back(Elt: VE.getTypeID(T: I.getValueType()));
1713 Vals.push_back(Elt: I.getType()->getAddressSpace());
1714 Vals.push_back(Elt: VE.getValueID(V: I.getResolver()));
1715 Vals.push_back(Elt: getEncodedLinkage(GV: I));
1716 Vals.push_back(Elt: getEncodedVisibility(GV: I));
1717 Vals.push_back(Elt: I.isDSOLocal());
1718 Vals.push_back(Elt: addToStrtab(Str: I.getPartition()));
1719 Vals.push_back(Elt: I.getPartition().size());
1720 Stream.EmitRecord(Code: bitc::MODULE_CODE_IFUNC, Vals);
1721 Vals.clear();
1722 }
1723
1724 writeValueSymbolTableForwardDecl();
1725}
1726
1727static uint64_t getOptimizationFlags(const Value *V) {
1728 uint64_t Flags = 0;
1729
1730 if (const auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V)) {
1731 if (OBO->hasNoSignedWrap())
1732 Flags |= 1 << bitc::OBO_NO_SIGNED_WRAP;
1733 if (OBO->hasNoUnsignedWrap())
1734 Flags |= 1 << bitc::OBO_NO_UNSIGNED_WRAP;
1735 } else if (const auto *PEO = dyn_cast<PossiblyExactOperator>(Val: V)) {
1736 if (PEO->isExact())
1737 Flags |= 1 << bitc::PEO_EXACT;
1738 } else if (const auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: V)) {
1739 if (PDI->isDisjoint())
1740 Flags |= 1 << bitc::PDI_DISJOINT;
1741 } else if (const auto *FPMO = dyn_cast<FPMathOperator>(Val: V)) {
1742 if (FPMO->hasAllowReassoc())
1743 Flags |= bitc::AllowReassoc;
1744 if (FPMO->hasNoNaNs())
1745 Flags |= bitc::NoNaNs;
1746 if (FPMO->hasNoInfs())
1747 Flags |= bitc::NoInfs;
1748 if (FPMO->hasNoSignedZeros())
1749 Flags |= bitc::NoSignedZeros;
1750 if (FPMO->hasAllowReciprocal())
1751 Flags |= bitc::AllowReciprocal;
1752 if (FPMO->hasAllowContract())
1753 Flags |= bitc::AllowContract;
1754 if (FPMO->hasApproxFunc())
1755 Flags |= bitc::ApproxFunc;
1756 } else if (const auto *NNI = dyn_cast<PossiblyNonNegInst>(Val: V)) {
1757 if (NNI->hasNonNeg())
1758 Flags |= 1 << bitc::PNNI_NON_NEG;
1759 } else if (const auto *TI = dyn_cast<TruncInst>(Val: V)) {
1760 if (TI->hasNoSignedWrap())
1761 Flags |= 1 << bitc::TIO_NO_SIGNED_WRAP;
1762 if (TI->hasNoUnsignedWrap())
1763 Flags |= 1 << bitc::TIO_NO_UNSIGNED_WRAP;
1764 } else if (const auto *GEP = dyn_cast<GEPOperator>(Val: V)) {
1765 if (GEP->isInBounds())
1766 Flags |= 1 << bitc::GEP_INBOUNDS;
1767 if (GEP->hasNoUnsignedSignedWrap())
1768 Flags |= 1 << bitc::GEP_NUSW;
1769 if (GEP->hasNoUnsignedWrap())
1770 Flags |= 1 << bitc::GEP_NUW;
1771 } else if (const auto *ICmp = dyn_cast<ICmpInst>(Val: V)) {
1772 if (ICmp->hasSameSign())
1773 Flags |= 1 << bitc::ICMP_SAME_SIGN;
1774 }
1775
1776 return Flags;
1777}
1778
1779void ModuleBitcodeWriter::writeValueAsMetadata(
1780 const ValueAsMetadata *MD, SmallVectorImpl<uint64_t> &Record) {
1781 // Mimic an MDNode with a value as one operand.
1782 Value *V = MD->getValue();
1783 Record.push_back(Elt: VE.getTypeID(T: V->getType()));
1784 Record.push_back(Elt: VE.getValueID(V));
1785 Stream.EmitRecord(Code: bitc::METADATA_VALUE, Vals: Record, Abbrev: 0);
1786 Record.clear();
1787}
1788
1789void ModuleBitcodeWriter::writeMDTuple(const MDTuple *N,
1790 SmallVectorImpl<uint64_t> &Record,
1791 unsigned Abbrev) {
1792 for (const MDOperand &MDO : N->operands()) {
1793 Metadata *MD = MDO;
1794 assert(!(MD && isa<LocalAsMetadata>(MD)) &&
1795 "Unexpected function-local metadata");
1796 Record.push_back(Elt: VE.getMetadataOrNullID(MD));
1797 }
1798 Stream.EmitRecord(Code: N->isDistinct() ? bitc::METADATA_DISTINCT_NODE
1799 : bitc::METADATA_NODE,
1800 Vals: Record, Abbrev);
1801 Record.clear();
1802}
1803
1804unsigned ModuleBitcodeWriter::createDILocationAbbrev() {
1805 // Assume the column is usually under 128, and always output the inlined-at
1806 // location (it's never more expensive than building an array size 1).
1807 auto Abbv = std::make_shared<BitCodeAbbrev>();
1808 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_LOCATION));
1809 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1810 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1811 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1812 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1813 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1814 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1815 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
1816}
1817
1818void ModuleBitcodeWriter::writeDILocation(const DILocation *N,
1819 SmallVectorImpl<uint64_t> &Record,
1820 unsigned &Abbrev) {
1821 if (!Abbrev)
1822 Abbrev = createDILocationAbbrev();
1823
1824 Record.push_back(Elt: N->isDistinct());
1825 Record.push_back(Elt: N->getLine());
1826 Record.push_back(Elt: N->getColumn());
1827 Record.push_back(Elt: VE.getMetadataID(MD: N->getScope()));
1828 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getInlinedAt()));
1829 Record.push_back(Elt: N->isImplicitCode());
1830
1831 Stream.EmitRecord(Code: bitc::METADATA_LOCATION, Vals: Record, Abbrev);
1832 Record.clear();
1833}
1834
1835unsigned ModuleBitcodeWriter::createGenericDINodeAbbrev() {
1836 // Assume the column is usually under 128, and always output the inlined-at
1837 // location (it's never more expensive than building an array size 1).
1838 auto Abbv = std::make_shared<BitCodeAbbrev>();
1839 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_GENERIC_DEBUG));
1840 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1841 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1842 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1843 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1844 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1845 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1846 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
1847}
1848
1849void ModuleBitcodeWriter::writeGenericDINode(const GenericDINode *N,
1850 SmallVectorImpl<uint64_t> &Record,
1851 unsigned &Abbrev) {
1852 if (!Abbrev)
1853 Abbrev = createGenericDINodeAbbrev();
1854
1855 Record.push_back(Elt: N->isDistinct());
1856 Record.push_back(Elt: N->getTag());
1857 Record.push_back(Elt: 0); // Per-tag version field; unused for now.
1858
1859 for (auto &I : N->operands())
1860 Record.push_back(Elt: VE.getMetadataOrNullID(MD: I));
1861
1862 Stream.EmitRecord(Code: bitc::METADATA_GENERIC_DEBUG, Vals: Record, Abbrev);
1863 Record.clear();
1864}
1865
1866void ModuleBitcodeWriter::writeDISubrange(const DISubrange *N,
1867 SmallVectorImpl<uint64_t> &Record,
1868 unsigned Abbrev) {
1869 const uint64_t Version = 2 << 1;
1870 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
1871 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawCountNode()));
1872 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
1873 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
1874 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
1875
1876 Stream.EmitRecord(Code: bitc::METADATA_SUBRANGE, Vals: Record, Abbrev);
1877 Record.clear();
1878}
1879
1880void ModuleBitcodeWriter::writeDIGenericSubrange(
1881 const DIGenericSubrange *N, SmallVectorImpl<uint64_t> &Record,
1882 unsigned Abbrev) {
1883 Record.push_back(Elt: (uint64_t)N->isDistinct());
1884 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawCountNode()));
1885 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
1886 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
1887 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
1888
1889 Stream.EmitRecord(Code: bitc::METADATA_GENERIC_SUBRANGE, Vals: Record, Abbrev);
1890 Record.clear();
1891}
1892
1893void ModuleBitcodeWriter::writeDIEnumerator(const DIEnumerator *N,
1894 SmallVectorImpl<uint64_t> &Record,
1895 unsigned Abbrev) {
1896 const uint64_t IsBigInt = 1 << 2;
1897 Record.push_back(Elt: IsBigInt | (N->isUnsigned() << 1) | N->isDistinct());
1898 Record.push_back(Elt: N->getValue().getBitWidth());
1899 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1900 emitWideAPInt(Vals&: Record, A: N->getValue());
1901
1902 Stream.EmitRecord(Code: bitc::METADATA_ENUMERATOR, Vals: Record, Abbrev);
1903 Record.clear();
1904}
1905
1906void ModuleBitcodeWriter::writeDIBasicType(const DIBasicType *N,
1907 SmallVectorImpl<uint64_t> &Record,
1908 unsigned Abbrev) {
1909 const unsigned SizeIsMetadata = 0x2;
1910 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
1911 Record.push_back(Elt: N->getTag());
1912 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1913 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
1914 Record.push_back(Elt: N->getAlignInBits());
1915 Record.push_back(Elt: N->getEncoding());
1916 Record.push_back(Elt: N->getFlags());
1917 Record.push_back(Elt: N->getNumExtraInhabitants());
1918
1919 Stream.EmitRecord(Code: bitc::METADATA_BASIC_TYPE, Vals: Record, Abbrev);
1920 Record.clear();
1921}
1922
1923void ModuleBitcodeWriter::writeDIFixedPointType(
1924 const DIFixedPointType *N, SmallVectorImpl<uint64_t> &Record,
1925 unsigned Abbrev) {
1926 const unsigned SizeIsMetadata = 0x2;
1927 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
1928 Record.push_back(Elt: N->getTag());
1929 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1930 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
1931 Record.push_back(Elt: N->getAlignInBits());
1932 Record.push_back(Elt: N->getEncoding());
1933 Record.push_back(Elt: N->getFlags());
1934 Record.push_back(Elt: N->getKind());
1935 Record.push_back(Elt: N->getFactorRaw());
1936
1937 auto WriteWideInt = [&](const APInt &Value) {
1938 // Write an encoded word that holds the number of active words and
1939 // the number of bits.
1940 uint64_t NumWords = Value.getActiveWords();
1941 uint64_t Encoded = (NumWords << 32) | Value.getBitWidth();
1942 Record.push_back(Elt: Encoded);
1943 emitWideAPInt(Vals&: Record, A: Value);
1944 };
1945
1946 WriteWideInt(N->getNumeratorRaw());
1947 WriteWideInt(N->getDenominatorRaw());
1948
1949 Stream.EmitRecord(Code: bitc::METADATA_FIXED_POINT_TYPE, Vals: Record, Abbrev);
1950 Record.clear();
1951}
1952
1953void ModuleBitcodeWriter::writeDIStringType(const DIStringType *N,
1954 SmallVectorImpl<uint64_t> &Record,
1955 unsigned Abbrev) {
1956 const unsigned SizeIsMetadata = 0x2;
1957 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
1958 Record.push_back(Elt: N->getTag());
1959 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1960 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLength()));
1961 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLengthExp()));
1962 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLocationExp()));
1963 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
1964 Record.push_back(Elt: N->getAlignInBits());
1965 Record.push_back(Elt: N->getEncoding());
1966
1967 Stream.EmitRecord(Code: bitc::METADATA_STRING_TYPE, Vals: Record, Abbrev);
1968 Record.clear();
1969}
1970
1971void ModuleBitcodeWriter::writeDIDerivedType(const DIDerivedType *N,
1972 SmallVectorImpl<uint64_t> &Record,
1973 unsigned Abbrev) {
1974 const unsigned SizeIsMetadata = 0x2;
1975 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
1976 Record.push_back(Elt: N->getTag());
1977 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1978 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
1979 Record.push_back(Elt: N->getLine());
1980 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
1981 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
1982 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
1983 Record.push_back(Elt: N->getAlignInBits());
1984 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawOffsetInBits()));
1985 Record.push_back(Elt: N->getFlags());
1986 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getExtraData()));
1987
1988 // DWARF address space is encoded as N->getDWARFAddressSpace() + 1. 0 means
1989 // that there is no DWARF address space associated with DIDerivedType.
1990 if (const auto &DWARFAddressSpace = N->getDWARFAddressSpace())
1991 Record.push_back(Elt: *DWARFAddressSpace + 1);
1992 else
1993 Record.push_back(Elt: 0);
1994
1995 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
1996
1997 if (auto PtrAuthData = N->getPtrAuthData())
1998 Record.push_back(Elt: PtrAuthData->RawData);
1999 else
2000 Record.push_back(Elt: 0);
2001
2002 Stream.EmitRecord(Code: bitc::METADATA_DERIVED_TYPE, Vals: Record, Abbrev);
2003 Record.clear();
2004}
2005
2006void ModuleBitcodeWriter::writeDISubrangeType(const DISubrangeType *N,
2007 SmallVectorImpl<uint64_t> &Record,
2008 unsigned Abbrev) {
2009 const unsigned SizeIsMetadata = 0x2;
2010 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2011 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2012 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2013 Record.push_back(Elt: N->getLine());
2014 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2015 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2016 Record.push_back(Elt: N->getAlignInBits());
2017 Record.push_back(Elt: N->getFlags());
2018 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2019 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
2020 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
2021 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
2022 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawBias()));
2023
2024 Stream.EmitRecord(Code: bitc::METADATA_SUBRANGE_TYPE, Vals: Record, Abbrev);
2025 Record.clear();
2026}
2027
2028void ModuleBitcodeWriter::writeDICompositeType(
2029 const DICompositeType *N, SmallVectorImpl<uint64_t> &Record,
2030 unsigned Abbrev) {
2031 const unsigned IsNotUsedInOldTypeRef = 0x2;
2032 const unsigned SizeIsMetadata = 0x4;
2033 Record.push_back(Elt: SizeIsMetadata | IsNotUsedInOldTypeRef |
2034 (unsigned)N->isDistinct());
2035 Record.push_back(Elt: N->getTag());
2036 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2037 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2038 Record.push_back(Elt: N->getLine());
2039 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2040 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2041 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2042 Record.push_back(Elt: N->getAlignInBits());
2043 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawOffsetInBits()));
2044 Record.push_back(Elt: N->getFlags());
2045 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2046 Record.push_back(Elt: N->getRuntimeLang());
2047 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getVTableHolder()));
2048 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams().get()));
2049 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawIdentifier()));
2050 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDiscriminator()));
2051 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawDataLocation()));
2052 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawAssociated()));
2053 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawAllocated()));
2054 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawRank()));
2055 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2056 Record.push_back(Elt: N->getNumExtraInhabitants());
2057 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSpecification()));
2058 Record.push_back(
2059 Elt: N->getEnumKind().value_or(u: dwarf::DW_APPLE_ENUM_KIND_invalid));
2060 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawBitStride()));
2061
2062 Stream.EmitRecord(Code: bitc::METADATA_COMPOSITE_TYPE, Vals: Record, Abbrev);
2063 Record.clear();
2064}
2065
2066void ModuleBitcodeWriter::writeDISubroutineType(
2067 const DISubroutineType *N, SmallVectorImpl<uint64_t> &Record,
2068 unsigned Abbrev) {
2069 const unsigned HasNoOldTypeRefs = 0x2;
2070 Record.push_back(Elt: HasNoOldTypeRefs | (unsigned)N->isDistinct());
2071 Record.push_back(Elt: N->getFlags());
2072 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTypeArray().get()));
2073 Record.push_back(Elt: N->getCC());
2074
2075 Stream.EmitRecord(Code: bitc::METADATA_SUBROUTINE_TYPE, Vals: Record, Abbrev);
2076 Record.clear();
2077}
2078
2079void ModuleBitcodeWriter::writeDIFile(const DIFile *N,
2080 SmallVectorImpl<uint64_t> &Record,
2081 unsigned Abbrev) {
2082 Record.push_back(Elt: N->isDistinct());
2083 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFilename()));
2084 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawDirectory()));
2085 if (N->getRawChecksum()) {
2086 Record.push_back(Elt: N->getRawChecksum()->Kind);
2087 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawChecksum()->Value));
2088 } else {
2089 // Maintain backwards compatibility with the old internal representation of
2090 // CSK_None in ChecksumKind by writing nulls here when Checksum is None.
2091 Record.push_back(Elt: 0);
2092 Record.push_back(Elt: VE.getMetadataOrNullID(MD: nullptr));
2093 }
2094 auto Source = N->getRawSource();
2095 if (Source)
2096 Record.push_back(Elt: VE.getMetadataOrNullID(MD: Source));
2097
2098 Stream.EmitRecord(Code: bitc::METADATA_FILE, Vals: Record, Abbrev);
2099 Record.clear();
2100}
2101
2102void ModuleBitcodeWriter::writeDICompileUnit(const DICompileUnit *N,
2103 SmallVectorImpl<uint64_t> &Record,
2104 unsigned Abbrev) {
2105 assert(N->isDistinct() && "Expected distinct compile units");
2106 Record.push_back(/* IsDistinct */ Elt: true);
2107 Record.push_back(Elt: N->getSourceLanguage());
2108 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2109 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawProducer()));
2110 Record.push_back(Elt: N->isOptimized());
2111 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFlags()));
2112 Record.push_back(Elt: N->getRuntimeVersion());
2113 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSplitDebugFilename()));
2114 Record.push_back(Elt: N->getEmissionKind());
2115 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getEnumTypes().get()));
2116 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRetainedTypes().get()));
2117 Record.push_back(/* subprograms */ Elt: 0);
2118 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getGlobalVariables().get()));
2119 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getImportedEntities().get()));
2120 Record.push_back(Elt: N->getDWOId());
2121 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getMacros().get()));
2122 Record.push_back(Elt: N->getSplitDebugInlining());
2123 Record.push_back(Elt: N->getDebugInfoForProfiling());
2124 Record.push_back(Elt: (unsigned)N->getNameTableKind());
2125 Record.push_back(Elt: N->getRangesBaseAddress());
2126 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSysRoot()));
2127 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSDK()));
2128
2129 Stream.EmitRecord(Code: bitc::METADATA_COMPILE_UNIT, Vals: Record, Abbrev);
2130 Record.clear();
2131}
2132
2133void ModuleBitcodeWriter::writeDISubprogram(const DISubprogram *N,
2134 SmallVectorImpl<uint64_t> &Record,
2135 unsigned Abbrev) {
2136 const uint64_t HasUnitFlag = 1 << 1;
2137 const uint64_t HasSPFlagsFlag = 1 << 2;
2138 Record.push_back(Elt: uint64_t(N->isDistinct()) | HasUnitFlag | HasSPFlagsFlag);
2139 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2140 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2141 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLinkageName()));
2142 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2143 Record.push_back(Elt: N->getLine());
2144 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2145 Record.push_back(Elt: N->getScopeLine());
2146 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getContainingType()));
2147 Record.push_back(Elt: N->getSPFlags());
2148 Record.push_back(Elt: N->getVirtualIndex());
2149 Record.push_back(Elt: N->getFlags());
2150 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUnit()));
2151 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams().get()));
2152 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDeclaration()));
2153 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRetainedNodes().get()));
2154 Record.push_back(Elt: N->getThisAdjustment());
2155 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getThrownTypes().get()));
2156 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2157 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawTargetFuncName()));
2158
2159 Stream.EmitRecord(Code: bitc::METADATA_SUBPROGRAM, Vals: Record, Abbrev);
2160 Record.clear();
2161}
2162
2163void ModuleBitcodeWriter::writeDILexicalBlock(const DILexicalBlock *N,
2164 SmallVectorImpl<uint64_t> &Record,
2165 unsigned Abbrev) {
2166 Record.push_back(Elt: N->isDistinct());
2167 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2168 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2169 Record.push_back(Elt: N->getLine());
2170 Record.push_back(Elt: N->getColumn());
2171
2172 Stream.EmitRecord(Code: bitc::METADATA_LEXICAL_BLOCK, Vals: Record, Abbrev);
2173 Record.clear();
2174}
2175
2176void ModuleBitcodeWriter::writeDILexicalBlockFile(
2177 const DILexicalBlockFile *N, SmallVectorImpl<uint64_t> &Record,
2178 unsigned Abbrev) {
2179 Record.push_back(Elt: N->isDistinct());
2180 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2181 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2182 Record.push_back(Elt: N->getDiscriminator());
2183
2184 Stream.EmitRecord(Code: bitc::METADATA_LEXICAL_BLOCK_FILE, Vals: Record, Abbrev);
2185 Record.clear();
2186}
2187
2188void ModuleBitcodeWriter::writeDICommonBlock(const DICommonBlock *N,
2189 SmallVectorImpl<uint64_t> &Record,
2190 unsigned Abbrev) {
2191 Record.push_back(Elt: N->isDistinct());
2192 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2193 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDecl()));
2194 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2195 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2196 Record.push_back(Elt: N->getLineNo());
2197
2198 Stream.EmitRecord(Code: bitc::METADATA_COMMON_BLOCK, Vals: Record, Abbrev);
2199 Record.clear();
2200}
2201
2202void ModuleBitcodeWriter::writeDINamespace(const DINamespace *N,
2203 SmallVectorImpl<uint64_t> &Record,
2204 unsigned Abbrev) {
2205 Record.push_back(Elt: N->isDistinct() | N->getExportSymbols() << 1);
2206 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2207 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2208
2209 Stream.EmitRecord(Code: bitc::METADATA_NAMESPACE, Vals: Record, Abbrev);
2210 Record.clear();
2211}
2212
2213void ModuleBitcodeWriter::writeDIMacro(const DIMacro *N,
2214 SmallVectorImpl<uint64_t> &Record,
2215 unsigned Abbrev) {
2216 Record.push_back(Elt: N->isDistinct());
2217 Record.push_back(Elt: N->getMacinfoType());
2218 Record.push_back(Elt: N->getLine());
2219 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2220 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawValue()));
2221
2222 Stream.EmitRecord(Code: bitc::METADATA_MACRO, Vals: Record, Abbrev);
2223 Record.clear();
2224}
2225
2226void ModuleBitcodeWriter::writeDIMacroFile(const DIMacroFile *N,
2227 SmallVectorImpl<uint64_t> &Record,
2228 unsigned Abbrev) {
2229 Record.push_back(Elt: N->isDistinct());
2230 Record.push_back(Elt: N->getMacinfoType());
2231 Record.push_back(Elt: N->getLine());
2232 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2233 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2234
2235 Stream.EmitRecord(Code: bitc::METADATA_MACRO_FILE, Vals: Record, Abbrev);
2236 Record.clear();
2237}
2238
2239void ModuleBitcodeWriter::writeDIArgList(const DIArgList *N,
2240 SmallVectorImpl<uint64_t> &Record) {
2241 Record.reserve(N: N->getArgs().size());
2242 for (ValueAsMetadata *MD : N->getArgs())
2243 Record.push_back(Elt: VE.getMetadataID(MD));
2244
2245 Stream.EmitRecord(Code: bitc::METADATA_ARG_LIST, Vals: Record);
2246 Record.clear();
2247}
2248
2249void ModuleBitcodeWriter::writeDIModule(const DIModule *N,
2250 SmallVectorImpl<uint64_t> &Record,
2251 unsigned Abbrev) {
2252 Record.push_back(Elt: N->isDistinct());
2253 for (auto &I : N->operands())
2254 Record.push_back(Elt: VE.getMetadataOrNullID(MD: I));
2255 Record.push_back(Elt: N->getLineNo());
2256 Record.push_back(Elt: N->getIsDecl());
2257
2258 Stream.EmitRecord(Code: bitc::METADATA_MODULE, Vals: Record, Abbrev);
2259 Record.clear();
2260}
2261
2262void ModuleBitcodeWriter::writeDIAssignID(const DIAssignID *N,
2263 SmallVectorImpl<uint64_t> &Record,
2264 unsigned Abbrev) {
2265 // There are no arguments for this metadata type.
2266 Record.push_back(Elt: N->isDistinct());
2267 Stream.EmitRecord(Code: bitc::METADATA_ASSIGN_ID, Vals: Record, Abbrev);
2268 Record.clear();
2269}
2270
2271void ModuleBitcodeWriter::writeDITemplateTypeParameter(
2272 const DITemplateTypeParameter *N, SmallVectorImpl<uint64_t> &Record,
2273 unsigned Abbrev) {
2274 Record.push_back(Elt: N->isDistinct());
2275 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2276 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2277 Record.push_back(Elt: N->isDefault());
2278
2279 Stream.EmitRecord(Code: bitc::METADATA_TEMPLATE_TYPE, Vals: Record, Abbrev);
2280 Record.clear();
2281}
2282
2283void ModuleBitcodeWriter::writeDITemplateValueParameter(
2284 const DITemplateValueParameter *N, SmallVectorImpl<uint64_t> &Record,
2285 unsigned Abbrev) {
2286 Record.push_back(Elt: N->isDistinct());
2287 Record.push_back(Elt: N->getTag());
2288 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2289 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2290 Record.push_back(Elt: N->isDefault());
2291 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getValue()));
2292
2293 Stream.EmitRecord(Code: bitc::METADATA_TEMPLATE_VALUE, Vals: Record, Abbrev);
2294 Record.clear();
2295}
2296
2297void ModuleBitcodeWriter::writeDIGlobalVariable(
2298 const DIGlobalVariable *N, SmallVectorImpl<uint64_t> &Record,
2299 unsigned Abbrev) {
2300 const uint64_t Version = 2 << 1;
2301 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
2302 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2303 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2304 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLinkageName()));
2305 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2306 Record.push_back(Elt: N->getLine());
2307 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2308 Record.push_back(Elt: N->isLocalToUnit());
2309 Record.push_back(Elt: N->isDefinition());
2310 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStaticDataMemberDeclaration()));
2311 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams()));
2312 Record.push_back(Elt: N->getAlignInBits());
2313 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2314
2315 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_VAR, Vals: Record, Abbrev);
2316 Record.clear();
2317}
2318
2319void ModuleBitcodeWriter::writeDILocalVariable(
2320 const DILocalVariable *N, SmallVectorImpl<uint64_t> &Record,
2321 unsigned Abbrev) {
2322 // In order to support all possible bitcode formats in BitcodeReader we need
2323 // to distinguish the following cases:
2324 // 1) Record has no artificial tag (Record[1]),
2325 // has no obsolete inlinedAt field (Record[9]).
2326 // In this case Record size will be 8, HasAlignment flag is false.
2327 // 2) Record has artificial tag (Record[1]),
2328 // has no obsolete inlignedAt field (Record[9]).
2329 // In this case Record size will be 9, HasAlignment flag is false.
2330 // 3) Record has both artificial tag (Record[1]) and
2331 // obsolete inlignedAt field (Record[9]).
2332 // In this case Record size will be 10, HasAlignment flag is false.
2333 // 4) Record has neither artificial tag, nor inlignedAt field, but
2334 // HasAlignment flag is true and Record[8] contains alignment value.
2335 const uint64_t HasAlignmentFlag = 1 << 1;
2336 Record.push_back(Elt: (uint64_t)N->isDistinct() | HasAlignmentFlag);
2337 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2338 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2339 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2340 Record.push_back(Elt: N->getLine());
2341 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2342 Record.push_back(Elt: N->getArg());
2343 Record.push_back(Elt: N->getFlags());
2344 Record.push_back(Elt: N->getAlignInBits());
2345 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2346
2347 Stream.EmitRecord(Code: bitc::METADATA_LOCAL_VAR, Vals: Record, Abbrev);
2348 Record.clear();
2349}
2350
2351void ModuleBitcodeWriter::writeDILabel(
2352 const DILabel *N, SmallVectorImpl<uint64_t> &Record,
2353 unsigned Abbrev) {
2354 uint64_t IsArtificialFlag = uint64_t(N->isArtificial()) << 1;
2355 Record.push_back(Elt: (uint64_t)N->isDistinct() | IsArtificialFlag);
2356 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2357 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2358 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2359 Record.push_back(Elt: N->getLine());
2360 Record.push_back(Elt: N->getColumn());
2361 Record.push_back(Elt: N->getCoroSuspendIdx().has_value()
2362 ? (uint64_t)N->getCoroSuspendIdx().value()
2363 : std::numeric_limits<uint64_t>::max());
2364
2365 Stream.EmitRecord(Code: bitc::METADATA_LABEL, Vals: Record, Abbrev);
2366 Record.clear();
2367}
2368
2369void ModuleBitcodeWriter::writeDIExpression(const DIExpression *N,
2370 SmallVectorImpl<uint64_t> &Record,
2371 unsigned Abbrev) {
2372 Record.reserve(N: N->getElements().size() + 1);
2373 const uint64_t Version = 3 << 1;
2374 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
2375 Record.append(in_start: N->elements_begin(), in_end: N->elements_end());
2376
2377 Stream.EmitRecord(Code: bitc::METADATA_EXPRESSION, Vals: Record, Abbrev);
2378 Record.clear();
2379}
2380
2381void ModuleBitcodeWriter::writeDIGlobalVariableExpression(
2382 const DIGlobalVariableExpression *N, SmallVectorImpl<uint64_t> &Record,
2383 unsigned Abbrev) {
2384 Record.push_back(Elt: N->isDistinct());
2385 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getVariable()));
2386 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getExpression()));
2387
2388 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_VAR_EXPR, Vals: Record, Abbrev);
2389 Record.clear();
2390}
2391
2392void ModuleBitcodeWriter::writeDIObjCProperty(const DIObjCProperty *N,
2393 SmallVectorImpl<uint64_t> &Record,
2394 unsigned Abbrev) {
2395 Record.push_back(Elt: N->isDistinct());
2396 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2397 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2398 Record.push_back(Elt: N->getLine());
2399 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSetterName()));
2400 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawGetterName()));
2401 Record.push_back(Elt: N->getAttributes());
2402 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2403
2404 Stream.EmitRecord(Code: bitc::METADATA_OBJC_PROPERTY, Vals: Record, Abbrev);
2405 Record.clear();
2406}
2407
2408void ModuleBitcodeWriter::writeDIImportedEntity(
2409 const DIImportedEntity *N, SmallVectorImpl<uint64_t> &Record,
2410 unsigned Abbrev) {
2411 Record.push_back(Elt: N->isDistinct());
2412 Record.push_back(Elt: N->getTag());
2413 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2414 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getEntity()));
2415 Record.push_back(Elt: N->getLine());
2416 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2417 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFile()));
2418 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2419
2420 Stream.EmitRecord(Code: bitc::METADATA_IMPORTED_ENTITY, Vals: Record, Abbrev);
2421 Record.clear();
2422}
2423
2424unsigned ModuleBitcodeWriter::createNamedMetadataAbbrev() {
2425 auto Abbv = std::make_shared<BitCodeAbbrev>();
2426 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_NAME));
2427 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2428 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2429 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
2430}
2431
2432void ModuleBitcodeWriter::writeNamedMetadata(
2433 SmallVectorImpl<uint64_t> &Record) {
2434 if (M.named_metadata_empty())
2435 return;
2436
2437 unsigned Abbrev = createNamedMetadataAbbrev();
2438 for (const NamedMDNode &NMD : M.named_metadata()) {
2439 // Write name.
2440 StringRef Str = NMD.getName();
2441 Record.append(in_start: Str.bytes_begin(), in_end: Str.bytes_end());
2442 Stream.EmitRecord(Code: bitc::METADATA_NAME, Vals: Record, Abbrev);
2443 Record.clear();
2444
2445 // Write named metadata operands.
2446 for (const MDNode *N : NMD.operands())
2447 Record.push_back(Elt: VE.getMetadataID(MD: N));
2448 Stream.EmitRecord(Code: bitc::METADATA_NAMED_NODE, Vals: Record, Abbrev: 0);
2449 Record.clear();
2450 }
2451}
2452
2453unsigned ModuleBitcodeWriter::createMetadataStringsAbbrev() {
2454 auto Abbv = std::make_shared<BitCodeAbbrev>();
2455 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_STRINGS));
2456 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // # of strings
2457 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // offset to chars
2458 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
2459 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
2460}
2461
2462/// Write out a record for MDString.
2463///
2464/// All the metadata strings in a metadata block are emitted in a single
2465/// record. The sizes and strings themselves are shoved into a blob.
2466void ModuleBitcodeWriter::writeMetadataStrings(
2467 ArrayRef<const Metadata *> Strings, SmallVectorImpl<uint64_t> &Record) {
2468 if (Strings.empty())
2469 return;
2470
2471 // Start the record with the number of strings.
2472 Record.push_back(Elt: bitc::METADATA_STRINGS);
2473 Record.push_back(Elt: Strings.size());
2474
2475 // Emit the sizes of the strings in the blob.
2476 SmallString<256> Blob;
2477 {
2478 BitstreamWriter W(Blob);
2479 for (const Metadata *MD : Strings)
2480 W.EmitVBR(Val: cast<MDString>(Val: MD)->getLength(), NumBits: 6);
2481 W.FlushToWord();
2482 }
2483
2484 // Add the offset to the strings to the record.
2485 Record.push_back(Elt: Blob.size());
2486
2487 // Add the strings to the blob.
2488 for (const Metadata *MD : Strings)
2489 Blob.append(RHS: cast<MDString>(Val: MD)->getString());
2490
2491 // Emit the final record.
2492 Stream.EmitRecordWithBlob(Abbrev: createMetadataStringsAbbrev(), Vals: Record, Blob);
2493 Record.clear();
2494}
2495
2496// Generates an enum to use as an index in the Abbrev array of Metadata record.
2497enum MetadataAbbrev : unsigned {
2498#define HANDLE_MDNODE_LEAF(CLASS) CLASS##AbbrevID,
2499#include "llvm/IR/Metadata.def"
2500 LastPlusOne
2501};
2502
2503void ModuleBitcodeWriter::writeMetadataRecords(
2504 ArrayRef<const Metadata *> MDs, SmallVectorImpl<uint64_t> &Record,
2505 std::vector<unsigned> *MDAbbrevs, std::vector<uint64_t> *IndexPos) {
2506 if (MDs.empty())
2507 return;
2508
2509 // Initialize MDNode abbreviations.
2510#define HANDLE_MDNODE_LEAF(CLASS) unsigned CLASS##Abbrev = 0;
2511#include "llvm/IR/Metadata.def"
2512
2513 for (const Metadata *MD : MDs) {
2514 if (IndexPos)
2515 IndexPos->push_back(x: Stream.GetCurrentBitNo());
2516 if (const MDNode *N = dyn_cast<MDNode>(Val: MD)) {
2517 assert(N->isResolved() && "Expected forward references to be resolved");
2518
2519 switch (N->getMetadataID()) {
2520 default:
2521 llvm_unreachable("Invalid MDNode subclass");
2522#define HANDLE_MDNODE_LEAF(CLASS) \
2523 case Metadata::CLASS##Kind: \
2524 if (MDAbbrevs) \
2525 write##CLASS(cast<CLASS>(N), Record, \
2526 (*MDAbbrevs)[MetadataAbbrev::CLASS##AbbrevID]); \
2527 else \
2528 write##CLASS(cast<CLASS>(N), Record, CLASS##Abbrev); \
2529 continue;
2530#include "llvm/IR/Metadata.def"
2531 }
2532 }
2533 if (auto *AL = dyn_cast<DIArgList>(Val: MD)) {
2534 writeDIArgList(N: AL, Record);
2535 continue;
2536 }
2537 writeValueAsMetadata(MD: cast<ValueAsMetadata>(Val: MD), Record);
2538 }
2539}
2540
2541void ModuleBitcodeWriter::writeModuleMetadata() {
2542 if (!VE.hasMDs() && M.named_metadata_empty())
2543 return;
2544
2545 Stream.EnterSubblock(BlockID: bitc::METADATA_BLOCK_ID, CodeLen: 4);
2546 SmallVector<uint64_t, 64> Record;
2547
2548 // Emit all abbrevs upfront, so that the reader can jump in the middle of the
2549 // block and load any metadata.
2550 std::vector<unsigned> MDAbbrevs;
2551
2552 MDAbbrevs.resize(new_size: MetadataAbbrev::LastPlusOne);
2553 MDAbbrevs[MetadataAbbrev::DILocationAbbrevID] = createDILocationAbbrev();
2554 MDAbbrevs[MetadataAbbrev::GenericDINodeAbbrevID] =
2555 createGenericDINodeAbbrev();
2556
2557 auto Abbv = std::make_shared<BitCodeAbbrev>();
2558 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_INDEX_OFFSET));
2559 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2560 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2561 unsigned OffsetAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2562
2563 Abbv = std::make_shared<BitCodeAbbrev>();
2564 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_INDEX));
2565 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2566 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
2567 unsigned IndexAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2568
2569 // Emit MDStrings together upfront.
2570 writeMetadataStrings(Strings: VE.getMDStrings(), Record);
2571
2572 // We only emit an index for the metadata record if we have more than a given
2573 // (naive) threshold of metadatas, otherwise it is not worth it.
2574 if (VE.getNonMDStrings().size() > IndexThreshold) {
2575 // Write a placeholder value in for the offset of the metadata index,
2576 // which is written after the records, so that it can include
2577 // the offset of each entry. The placeholder offset will be
2578 // updated after all records are emitted.
2579 uint64_t Vals[] = {0, 0};
2580 Stream.EmitRecord(Code: bitc::METADATA_INDEX_OFFSET, Vals, Abbrev: OffsetAbbrev);
2581 }
2582
2583 // Compute and save the bit offset to the current position, which will be
2584 // patched when we emit the index later. We can simply subtract the 64-bit
2585 // fixed size from the current bit number to get the location to backpatch.
2586 uint64_t IndexOffsetRecordBitPos = Stream.GetCurrentBitNo();
2587
2588 // This index will contain the bitpos for each individual record.
2589 std::vector<uint64_t> IndexPos;
2590 IndexPos.reserve(n: VE.getNonMDStrings().size());
2591
2592 // Write all the records
2593 writeMetadataRecords(MDs: VE.getNonMDStrings(), Record, MDAbbrevs: &MDAbbrevs, IndexPos: &IndexPos);
2594
2595 if (VE.getNonMDStrings().size() > IndexThreshold) {
2596 // Now that we have emitted all the records we will emit the index. But
2597 // first
2598 // backpatch the forward reference so that the reader can skip the records
2599 // efficiently.
2600 Stream.BackpatchWord64(BitNo: IndexOffsetRecordBitPos - 64,
2601 Val: Stream.GetCurrentBitNo() - IndexOffsetRecordBitPos);
2602
2603 // Delta encode the index.
2604 uint64_t PreviousValue = IndexOffsetRecordBitPos;
2605 for (auto &Elt : IndexPos) {
2606 auto EltDelta = Elt - PreviousValue;
2607 PreviousValue = Elt;
2608 Elt = EltDelta;
2609 }
2610 // Emit the index record.
2611 Stream.EmitRecord(Code: bitc::METADATA_INDEX, Vals: IndexPos, Abbrev: IndexAbbrev);
2612 IndexPos.clear();
2613 }
2614
2615 // Write the named metadata now.
2616 writeNamedMetadata(Record);
2617
2618 auto AddDeclAttachedMetadata = [&](const GlobalObject &GO) {
2619 SmallVector<uint64_t, 4> Record;
2620 Record.push_back(Elt: VE.getValueID(V: &GO));
2621 pushGlobalMetadataAttachment(Record, GO);
2622 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_DECL_ATTACHMENT, Vals: Record);
2623 };
2624 for (const Function &F : M)
2625 if (F.isDeclaration() && F.hasMetadata())
2626 AddDeclAttachedMetadata(F);
2627 // FIXME: Only store metadata for declarations here, and move data for global
2628 // variable definitions to a separate block (PR28134).
2629 for (const GlobalVariable &GV : M.globals())
2630 if (GV.hasMetadata())
2631 AddDeclAttachedMetadata(GV);
2632
2633 Stream.ExitBlock();
2634}
2635
2636void ModuleBitcodeWriter::writeFunctionMetadata(const Function &F) {
2637 if (!VE.hasMDs())
2638 return;
2639
2640 Stream.EnterSubblock(BlockID: bitc::METADATA_BLOCK_ID, CodeLen: 3);
2641 SmallVector<uint64_t, 64> Record;
2642 writeMetadataStrings(Strings: VE.getMDStrings(), Record);
2643 writeMetadataRecords(MDs: VE.getNonMDStrings(), Record);
2644 Stream.ExitBlock();
2645}
2646
2647void ModuleBitcodeWriter::pushGlobalMetadataAttachment(
2648 SmallVectorImpl<uint64_t> &Record, const GlobalObject &GO) {
2649 // [n x [id, mdnode]]
2650 SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2651 GO.getAllMetadata(MDs);
2652 for (const auto &I : MDs) {
2653 Record.push_back(Elt: I.first);
2654 Record.push_back(Elt: VE.getMetadataID(MD: I.second));
2655 }
2656}
2657
2658void ModuleBitcodeWriter::writeFunctionMetadataAttachment(const Function &F) {
2659 Stream.EnterSubblock(BlockID: bitc::METADATA_ATTACHMENT_ID, CodeLen: 3);
2660
2661 SmallVector<uint64_t, 64> Record;
2662
2663 if (F.hasMetadata()) {
2664 pushGlobalMetadataAttachment(Record, GO: F);
2665 Stream.EmitRecord(Code: bitc::METADATA_ATTACHMENT, Vals: Record, Abbrev: 0);
2666 Record.clear();
2667 }
2668
2669 // Write metadata attachments
2670 // METADATA_ATTACHMENT - [m x [value, [n x [id, mdnode]]]
2671 SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2672 for (const BasicBlock &BB : F)
2673 for (const Instruction &I : BB) {
2674 MDs.clear();
2675 I.getAllMetadataOtherThanDebugLoc(MDs);
2676
2677 // If no metadata, ignore instruction.
2678 if (MDs.empty()) continue;
2679
2680 Record.push_back(Elt: VE.getInstructionID(I: &I));
2681
2682 for (const auto &[ID, MD] : MDs) {
2683 Record.push_back(Elt: ID);
2684 Record.push_back(Elt: VE.getMetadataID(MD));
2685 }
2686 Stream.EmitRecord(Code: bitc::METADATA_ATTACHMENT, Vals: Record, Abbrev: 0);
2687 Record.clear();
2688 }
2689
2690 Stream.ExitBlock();
2691}
2692
2693void ModuleBitcodeWriter::writeModuleMetadataKinds() {
2694 SmallVector<uint64_t, 64> Record;
2695
2696 // Write metadata kinds
2697 // METADATA_KIND - [n x [id, name]]
2698 SmallVector<StringRef, 8> Names;
2699 M.getMDKindNames(Result&: Names);
2700
2701 if (Names.empty()) return;
2702
2703 Stream.EnterSubblock(BlockID: bitc::METADATA_KIND_BLOCK_ID, CodeLen: 3);
2704
2705 for (unsigned MDKindID = 0, e = Names.size(); MDKindID != e; ++MDKindID) {
2706 Record.push_back(Elt: MDKindID);
2707 StringRef KName = Names[MDKindID];
2708 Record.append(in_start: KName.begin(), in_end: KName.end());
2709
2710 Stream.EmitRecord(Code: bitc::METADATA_KIND, Vals: Record, Abbrev: 0);
2711 Record.clear();
2712 }
2713
2714 Stream.ExitBlock();
2715}
2716
2717void ModuleBitcodeWriter::writeOperandBundleTags() {
2718 // Write metadata kinds
2719 //
2720 // OPERAND_BUNDLE_TAGS_BLOCK_ID : N x OPERAND_BUNDLE_TAG
2721 //
2722 // OPERAND_BUNDLE_TAG - [strchr x N]
2723
2724 SmallVector<StringRef, 8> Tags;
2725 M.getOperandBundleTags(Result&: Tags);
2726
2727 if (Tags.empty())
2728 return;
2729
2730 Stream.EnterSubblock(BlockID: bitc::OPERAND_BUNDLE_TAGS_BLOCK_ID, CodeLen: 3);
2731
2732 SmallVector<uint64_t, 64> Record;
2733
2734 for (auto Tag : Tags) {
2735 Record.append(in_start: Tag.begin(), in_end: Tag.end());
2736
2737 Stream.EmitRecord(Code: bitc::OPERAND_BUNDLE_TAG, Vals: Record, Abbrev: 0);
2738 Record.clear();
2739 }
2740
2741 Stream.ExitBlock();
2742}
2743
2744void ModuleBitcodeWriter::writeSyncScopeNames() {
2745 SmallVector<StringRef, 8> SSNs;
2746 M.getContext().getSyncScopeNames(SSNs);
2747 if (SSNs.empty())
2748 return;
2749
2750 Stream.EnterSubblock(BlockID: bitc::SYNC_SCOPE_NAMES_BLOCK_ID, CodeLen: 2);
2751
2752 SmallVector<uint64_t, 64> Record;
2753 for (auto SSN : SSNs) {
2754 Record.append(in_start: SSN.begin(), in_end: SSN.end());
2755 Stream.EmitRecord(Code: bitc::SYNC_SCOPE_NAME, Vals: Record, Abbrev: 0);
2756 Record.clear();
2757 }
2758
2759 Stream.ExitBlock();
2760}
2761
2762void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal,
2763 bool isGlobal) {
2764 if (FirstVal == LastVal) return;
2765
2766 Stream.EnterSubblock(BlockID: bitc::CONSTANTS_BLOCK_ID, CodeLen: 4);
2767
2768 unsigned AggregateAbbrev = 0;
2769 unsigned String8Abbrev = 0;
2770 unsigned CString7Abbrev = 0;
2771 unsigned CString6Abbrev = 0;
2772 // If this is a constant pool for the module, emit module-specific abbrevs.
2773 if (isGlobal) {
2774 // Abbrev for CST_CODE_AGGREGATE.
2775 auto Abbv = std::make_shared<BitCodeAbbrev>();
2776 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_AGGREGATE));
2777 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2778 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(Value: LastVal+1)));
2779 AggregateAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2780
2781 // Abbrev for CST_CODE_STRING.
2782 Abbv = std::make_shared<BitCodeAbbrev>();
2783 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_STRING));
2784 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2785 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2786 String8Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2787 // Abbrev for CST_CODE_CSTRING.
2788 Abbv = std::make_shared<BitCodeAbbrev>();
2789 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2790 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2791 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
2792 CString7Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2793 // Abbrev for CST_CODE_CSTRING.
2794 Abbv = std::make_shared<BitCodeAbbrev>();
2795 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2796 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2797 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
2798 CString6Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2799 }
2800
2801 SmallVector<uint64_t, 64> Record;
2802
2803 const ValueEnumerator::ValueList &Vals = VE.getValues();
2804 Type *LastTy = nullptr;
2805 for (unsigned i = FirstVal; i != LastVal; ++i) {
2806 const Value *V = Vals[i].first;
2807 // If we need to switch types, do so now.
2808 if (V->getType() != LastTy) {
2809 LastTy = V->getType();
2810 Record.push_back(Elt: VE.getTypeID(T: LastTy));
2811 Stream.EmitRecord(Code: bitc::CST_CODE_SETTYPE, Vals: Record,
2812 Abbrev: CONSTANTS_SETTYPE_ABBREV);
2813 Record.clear();
2814 }
2815
2816 if (const InlineAsm *IA = dyn_cast<InlineAsm>(Val: V)) {
2817 Record.push_back(Elt: VE.getTypeID(T: IA->getFunctionType()));
2818 Record.push_back(
2819 Elt: unsigned(IA->hasSideEffects()) | unsigned(IA->isAlignStack()) << 1 |
2820 unsigned(IA->getDialect() & 1) << 2 | unsigned(IA->canThrow()) << 3);
2821
2822 // Add the asm string.
2823 StringRef AsmStr = IA->getAsmString();
2824 Record.push_back(Elt: AsmStr.size());
2825 Record.append(in_start: AsmStr.begin(), in_end: AsmStr.end());
2826
2827 // Add the constraint string.
2828 StringRef ConstraintStr = IA->getConstraintString();
2829 Record.push_back(Elt: ConstraintStr.size());
2830 Record.append(in_start: ConstraintStr.begin(), in_end: ConstraintStr.end());
2831 Stream.EmitRecord(Code: bitc::CST_CODE_INLINEASM, Vals: Record);
2832 Record.clear();
2833 continue;
2834 }
2835 const Constant *C = cast<Constant>(Val: V);
2836 unsigned Code = -1U;
2837 unsigned AbbrevToUse = 0;
2838 if (C->isNullValue()) {
2839 Code = bitc::CST_CODE_NULL;
2840 } else if (isa<PoisonValue>(Val: C)) {
2841 Code = bitc::CST_CODE_POISON;
2842 } else if (isa<UndefValue>(Val: C)) {
2843 Code = bitc::CST_CODE_UNDEF;
2844 } else if (const ConstantInt *IV = dyn_cast<ConstantInt>(Val: C)) {
2845 if (IV->getBitWidth() <= 64) {
2846 uint64_t V = IV->getSExtValue();
2847 emitSignedInt64(Vals&: Record, V);
2848 Code = bitc::CST_CODE_INTEGER;
2849 AbbrevToUse = CONSTANTS_INTEGER_ABBREV;
2850 } else { // Wide integers, > 64 bits in size.
2851 emitWideAPInt(Vals&: Record, A: IV->getValue());
2852 Code = bitc::CST_CODE_WIDE_INTEGER;
2853 }
2854 } else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
2855 Code = bitc::CST_CODE_FLOAT;
2856 Type *Ty = CFP->getType()->getScalarType();
2857 if (Ty->isHalfTy() || Ty->isBFloatTy() || Ty->isFloatTy() ||
2858 Ty->isDoubleTy()) {
2859 Record.push_back(Elt: CFP->getValueAPF().bitcastToAPInt().getZExtValue());
2860 } else if (Ty->isX86_FP80Ty()) {
2861 // api needed to prevent premature destruction
2862 // bits are not in the same order as a normal i80 APInt, compensate.
2863 APInt api = CFP->getValueAPF().bitcastToAPInt();
2864 const uint64_t *p = api.getRawData();
2865 Record.push_back(Elt: (p[1] << 48) | (p[0] >> 16));
2866 Record.push_back(Elt: p[0] & 0xffffLL);
2867 } else if (Ty->isFP128Ty() || Ty->isPPC_FP128Ty()) {
2868 APInt api = CFP->getValueAPF().bitcastToAPInt();
2869 const uint64_t *p = api.getRawData();
2870 Record.push_back(Elt: p[0]);
2871 Record.push_back(Elt: p[1]);
2872 } else {
2873 assert(0 && "Unknown FP type!");
2874 }
2875 } else if (isa<ConstantDataSequential>(Val: C) &&
2876 cast<ConstantDataSequential>(Val: C)->isString()) {
2877 const ConstantDataSequential *Str = cast<ConstantDataSequential>(Val: C);
2878 // Emit constant strings specially.
2879 uint64_t NumElts = Str->getNumElements();
2880 // If this is a null-terminated string, use the denser CSTRING encoding.
2881 if (Str->isCString()) {
2882 Code = bitc::CST_CODE_CSTRING;
2883 --NumElts; // Don't encode the null, which isn't allowed by char6.
2884 } else {
2885 Code = bitc::CST_CODE_STRING;
2886 AbbrevToUse = String8Abbrev;
2887 }
2888 bool isCStr7 = Code == bitc::CST_CODE_CSTRING;
2889 bool isCStrChar6 = Code == bitc::CST_CODE_CSTRING;
2890 for (uint64_t i = 0; i != NumElts; ++i) {
2891 unsigned char V = Str->getElementAsInteger(i);
2892 Record.push_back(Elt: V);
2893 isCStr7 &= (V & 128) == 0;
2894 if (isCStrChar6)
2895 isCStrChar6 = BitCodeAbbrevOp::isChar6(C: V);
2896 }
2897
2898 if (isCStrChar6)
2899 AbbrevToUse = CString6Abbrev;
2900 else if (isCStr7)
2901 AbbrevToUse = CString7Abbrev;
2902 } else if (const ConstantDataSequential *CDS =
2903 dyn_cast<ConstantDataSequential>(Val: C)) {
2904 Code = bitc::CST_CODE_DATA;
2905 Type *EltTy = CDS->getElementType();
2906 if (isa<IntegerType>(Val: EltTy)) {
2907 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i)
2908 Record.push_back(Elt: CDS->getElementAsInteger(i));
2909 } else {
2910 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i)
2911 Record.push_back(
2912 Elt: CDS->getElementAsAPFloat(i).bitcastToAPInt().getLimitedValue());
2913 }
2914 } else if (isa<ConstantAggregate>(Val: C)) {
2915 Code = bitc::CST_CODE_AGGREGATE;
2916 for (const Value *Op : C->operands())
2917 Record.push_back(Elt: VE.getValueID(V: Op));
2918 AbbrevToUse = AggregateAbbrev;
2919 } else if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: C)) {
2920 switch (CE->getOpcode()) {
2921 default:
2922 if (Instruction::isCast(Opcode: CE->getOpcode())) {
2923 Code = bitc::CST_CODE_CE_CAST;
2924 Record.push_back(Elt: getEncodedCastOpcode(Opcode: CE->getOpcode()));
2925 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
2926 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2927 AbbrevToUse = CONSTANTS_CE_CAST_Abbrev;
2928 } else {
2929 assert(CE->getNumOperands() == 2 && "Unknown constant expr!");
2930 Code = bitc::CST_CODE_CE_BINOP;
2931 Record.push_back(Elt: getEncodedBinaryOpcode(Opcode: CE->getOpcode()));
2932 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2933 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
2934 uint64_t Flags = getOptimizationFlags(V: CE);
2935 if (Flags != 0)
2936 Record.push_back(Elt: Flags);
2937 }
2938 break;
2939 case Instruction::FNeg: {
2940 assert(CE->getNumOperands() == 1 && "Unknown constant expr!");
2941 Code = bitc::CST_CODE_CE_UNOP;
2942 Record.push_back(Elt: getEncodedUnaryOpcode(Opcode: CE->getOpcode()));
2943 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2944 uint64_t Flags = getOptimizationFlags(V: CE);
2945 if (Flags != 0)
2946 Record.push_back(Elt: Flags);
2947 break;
2948 }
2949 case Instruction::GetElementPtr: {
2950 Code = bitc::CST_CODE_CE_GEP;
2951 const auto *GO = cast<GEPOperator>(Val: C);
2952 Record.push_back(Elt: VE.getTypeID(T: GO->getSourceElementType()));
2953 Record.push_back(Elt: getOptimizationFlags(V: GO));
2954 if (std::optional<ConstantRange> Range = GO->getInRange()) {
2955 Code = bitc::CST_CODE_CE_GEP_WITH_INRANGE;
2956 emitConstantRange(Record, CR: *Range, /*EmitBitWidth=*/true);
2957 }
2958 for (const Value *Op : CE->operands()) {
2959 Record.push_back(Elt: VE.getTypeID(T: Op->getType()));
2960 Record.push_back(Elt: VE.getValueID(V: Op));
2961 }
2962 break;
2963 }
2964 case Instruction::ExtractElement:
2965 Code = bitc::CST_CODE_CE_EXTRACTELT;
2966 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
2967 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2968 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 1)->getType()));
2969 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
2970 break;
2971 case Instruction::InsertElement:
2972 Code = bitc::CST_CODE_CE_INSERTELT;
2973 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2974 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
2975 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 2)->getType()));
2976 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 2)));
2977 break;
2978 case Instruction::ShuffleVector:
2979 // If the return type and argument types are the same, this is a
2980 // standard shufflevector instruction. If the types are different,
2981 // then the shuffle is widening or truncating the input vectors, and
2982 // the argument type must also be encoded.
2983 if (C->getType() == C->getOperand(i: 0)->getType()) {
2984 Code = bitc::CST_CODE_CE_SHUFFLEVEC;
2985 } else {
2986 Code = bitc::CST_CODE_CE_SHUFVEC_EX;
2987 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
2988 }
2989 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2990 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
2991 Record.push_back(Elt: VE.getValueID(V: CE->getShuffleMaskForBitcode()));
2992 break;
2993 }
2994 } else if (const BlockAddress *BA = dyn_cast<BlockAddress>(Val: C)) {
2995 Code = bitc::CST_CODE_BLOCKADDRESS;
2996 Record.push_back(Elt: VE.getTypeID(T: BA->getFunction()->getType()));
2997 Record.push_back(Elt: VE.getValueID(V: BA->getFunction()));
2998 Record.push_back(Elt: VE.getGlobalBasicBlockID(BB: BA->getBasicBlock()));
2999 } else if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(Val: C)) {
3000 Code = bitc::CST_CODE_DSO_LOCAL_EQUIVALENT;
3001 Record.push_back(Elt: VE.getTypeID(T: Equiv->getGlobalValue()->getType()));
3002 Record.push_back(Elt: VE.getValueID(V: Equiv->getGlobalValue()));
3003 } else if (const auto *NC = dyn_cast<NoCFIValue>(Val: C)) {
3004 Code = bitc::CST_CODE_NO_CFI_VALUE;
3005 Record.push_back(Elt: VE.getTypeID(T: NC->getGlobalValue()->getType()));
3006 Record.push_back(Elt: VE.getValueID(V: NC->getGlobalValue()));
3007 } else if (const auto *CPA = dyn_cast<ConstantPtrAuth>(Val: C)) {
3008 Code = bitc::CST_CODE_PTRAUTH;
3009 Record.push_back(Elt: VE.getValueID(V: CPA->getPointer()));
3010 Record.push_back(Elt: VE.getValueID(V: CPA->getKey()));
3011 Record.push_back(Elt: VE.getValueID(V: CPA->getDiscriminator()));
3012 Record.push_back(Elt: VE.getValueID(V: CPA->getAddrDiscriminator()));
3013 } else {
3014#ifndef NDEBUG
3015 C->dump();
3016#endif
3017 llvm_unreachable("Unknown constant!");
3018 }
3019 Stream.EmitRecord(Code, Vals: Record, Abbrev: AbbrevToUse);
3020 Record.clear();
3021 }
3022
3023 Stream.ExitBlock();
3024}
3025
3026void ModuleBitcodeWriter::writeModuleConstants() {
3027 const ValueEnumerator::ValueList &Vals = VE.getValues();
3028
3029 // Find the first constant to emit, which is the first non-globalvalue value.
3030 // We know globalvalues have been emitted by WriteModuleInfo.
3031 for (unsigned i = 0, e = Vals.size(); i != e; ++i) {
3032 if (!isa<GlobalValue>(Val: Vals[i].first)) {
3033 writeConstants(FirstVal: i, LastVal: Vals.size(), isGlobal: true);
3034 return;
3035 }
3036 }
3037}
3038
3039/// pushValueAndType - The file has to encode both the value and type id for
3040/// many values, because we need to know what type to create for forward
3041/// references. However, most operands are not forward references, so this type
3042/// field is not needed.
3043///
3044/// This function adds V's value ID to Vals. If the value ID is higher than the
3045/// instruction ID, then it is a forward reference, and it also includes the
3046/// type ID. The value ID that is written is encoded relative to the InstID.
3047bool ModuleBitcodeWriter::pushValueAndType(const Value *V, unsigned InstID,
3048 SmallVectorImpl<unsigned> &Vals) {
3049 unsigned ValID = VE.getValueID(V);
3050 // Make encoding relative to the InstID.
3051 Vals.push_back(Elt: InstID - ValID);
3052 if (ValID >= InstID) {
3053 Vals.push_back(Elt: VE.getTypeID(T: V->getType()));
3054 return true;
3055 }
3056 return false;
3057}
3058
3059bool ModuleBitcodeWriter::pushValueOrMetadata(const Value *V, unsigned InstID,
3060 SmallVectorImpl<unsigned> &Vals) {
3061 bool IsMetadata = V->getType()->isMetadataTy();
3062 if (IsMetadata) {
3063 Vals.push_back(Elt: bitc::OB_METADATA);
3064 Metadata *MD = cast<MetadataAsValue>(Val: V)->getMetadata();
3065 unsigned ValID = VE.getMetadataID(MD);
3066 Vals.push_back(Elt: InstID - ValID);
3067 return false;
3068 }
3069 return pushValueAndType(V, InstID, Vals);
3070}
3071
3072void ModuleBitcodeWriter::writeOperandBundles(const CallBase &CS,
3073 unsigned InstID) {
3074 SmallVector<unsigned, 64> Record;
3075 LLVMContext &C = CS.getContext();
3076
3077 for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i) {
3078 const auto &Bundle = CS.getOperandBundleAt(Index: i);
3079 Record.push_back(Elt: C.getOperandBundleTagID(Tag: Bundle.getTagName()));
3080
3081 for (auto &Input : Bundle.Inputs)
3082 pushValueOrMetadata(V: Input, InstID, Vals&: Record);
3083
3084 Stream.EmitRecord(Code: bitc::FUNC_CODE_OPERAND_BUNDLE, Vals: Record);
3085 Record.clear();
3086 }
3087}
3088
3089/// pushValue - Like pushValueAndType, but where the type of the value is
3090/// omitted (perhaps it was already encoded in an earlier operand).
3091void ModuleBitcodeWriter::pushValue(const Value *V, unsigned InstID,
3092 SmallVectorImpl<unsigned> &Vals) {
3093 unsigned ValID = VE.getValueID(V);
3094 Vals.push_back(Elt: InstID - ValID);
3095}
3096
3097void ModuleBitcodeWriter::pushValueSigned(const Value *V, unsigned InstID,
3098 SmallVectorImpl<uint64_t> &Vals) {
3099 unsigned ValID = VE.getValueID(V);
3100 int64_t diff = ((int32_t)InstID - (int32_t)ValID);
3101 emitSignedInt64(Vals, V: diff);
3102}
3103
3104/// WriteInstruction - Emit an instruction to the specified stream.
3105void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
3106 unsigned InstID,
3107 SmallVectorImpl<unsigned> &Vals) {
3108 unsigned Code = 0;
3109 unsigned AbbrevToUse = 0;
3110 VE.setInstructionID(&I);
3111 switch (I.getOpcode()) {
3112 default:
3113 if (Instruction::isCast(Opcode: I.getOpcode())) {
3114 Code = bitc::FUNC_CODE_INST_CAST;
3115 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3116 AbbrevToUse = FUNCTION_INST_CAST_ABBREV;
3117 Vals.push_back(Elt: VE.getTypeID(T: I.getType()));
3118 Vals.push_back(Elt: getEncodedCastOpcode(Opcode: I.getOpcode()));
3119 uint64_t Flags = getOptimizationFlags(V: &I);
3120 if (Flags != 0) {
3121 if (AbbrevToUse == FUNCTION_INST_CAST_ABBREV)
3122 AbbrevToUse = FUNCTION_INST_CAST_FLAGS_ABBREV;
3123 Vals.push_back(Elt: Flags);
3124 }
3125 } else {
3126 assert(isa<BinaryOperator>(I) && "Unknown instruction!");
3127 Code = bitc::FUNC_CODE_INST_BINOP;
3128 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3129 AbbrevToUse = FUNCTION_INST_BINOP_ABBREV;
3130 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3131 Vals.push_back(Elt: getEncodedBinaryOpcode(Opcode: I.getOpcode()));
3132 uint64_t Flags = getOptimizationFlags(V: &I);
3133 if (Flags != 0) {
3134 if (AbbrevToUse == FUNCTION_INST_BINOP_ABBREV)
3135 AbbrevToUse = FUNCTION_INST_BINOP_FLAGS_ABBREV;
3136 Vals.push_back(Elt: Flags);
3137 }
3138 }
3139 break;
3140 case Instruction::FNeg: {
3141 Code = bitc::FUNC_CODE_INST_UNOP;
3142 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3143 AbbrevToUse = FUNCTION_INST_UNOP_ABBREV;
3144 Vals.push_back(Elt: getEncodedUnaryOpcode(Opcode: I.getOpcode()));
3145 uint64_t Flags = getOptimizationFlags(V: &I);
3146 if (Flags != 0) {
3147 if (AbbrevToUse == FUNCTION_INST_UNOP_ABBREV)
3148 AbbrevToUse = FUNCTION_INST_UNOP_FLAGS_ABBREV;
3149 Vals.push_back(Elt: Flags);
3150 }
3151 break;
3152 }
3153 case Instruction::GetElementPtr: {
3154 Code = bitc::FUNC_CODE_INST_GEP;
3155 AbbrevToUse = FUNCTION_INST_GEP_ABBREV;
3156 auto &GEPInst = cast<GetElementPtrInst>(Val: I);
3157 Vals.push_back(Elt: getOptimizationFlags(V: &I));
3158 Vals.push_back(Elt: VE.getTypeID(T: GEPInst.getSourceElementType()));
3159 for (const Value *Op : I.operands())
3160 pushValueAndType(V: Op, InstID, Vals);
3161 break;
3162 }
3163 case Instruction::ExtractValue: {
3164 Code = bitc::FUNC_CODE_INST_EXTRACTVAL;
3165 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3166 const ExtractValueInst *EVI = cast<ExtractValueInst>(Val: &I);
3167 Vals.append(in_start: EVI->idx_begin(), in_end: EVI->idx_end());
3168 break;
3169 }
3170 case Instruction::InsertValue: {
3171 Code = bitc::FUNC_CODE_INST_INSERTVAL;
3172 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3173 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3174 const InsertValueInst *IVI = cast<InsertValueInst>(Val: &I);
3175 Vals.append(in_start: IVI->idx_begin(), in_end: IVI->idx_end());
3176 break;
3177 }
3178 case Instruction::Select: {
3179 Code = bitc::FUNC_CODE_INST_VSELECT;
3180 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3181 pushValue(V: I.getOperand(i: 2), InstID, Vals);
3182 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3183 uint64_t Flags = getOptimizationFlags(V: &I);
3184 if (Flags != 0)
3185 Vals.push_back(Elt: Flags);
3186 break;
3187 }
3188 case Instruction::ExtractElement:
3189 Code = bitc::FUNC_CODE_INST_EXTRACTELT;
3190 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3191 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3192 break;
3193 case Instruction::InsertElement:
3194 Code = bitc::FUNC_CODE_INST_INSERTELT;
3195 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3196 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3197 pushValueAndType(V: I.getOperand(i: 2), InstID, Vals);
3198 break;
3199 case Instruction::ShuffleVector:
3200 Code = bitc::FUNC_CODE_INST_SHUFFLEVEC;
3201 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3202 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3203 pushValue(V: cast<ShuffleVectorInst>(Val: I).getShuffleMaskForBitcode(), InstID,
3204 Vals);
3205 break;
3206 case Instruction::ICmp:
3207 case Instruction::FCmp: {
3208 // compare returning Int1Ty or vector of Int1Ty
3209 Code = bitc::FUNC_CODE_INST_CMP2;
3210 AbbrevToUse = FUNCTION_INST_CMP_ABBREV;
3211 if (pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3212 AbbrevToUse = 0;
3213 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3214 Vals.push_back(Elt: cast<CmpInst>(Val: I).getPredicate());
3215 uint64_t Flags = getOptimizationFlags(V: &I);
3216 if (Flags != 0) {
3217 Vals.push_back(Elt: Flags);
3218 if (AbbrevToUse)
3219 AbbrevToUse = FUNCTION_INST_CMP_FLAGS_ABBREV;
3220 }
3221 break;
3222 }
3223
3224 case Instruction::Ret:
3225 {
3226 Code = bitc::FUNC_CODE_INST_RET;
3227 unsigned NumOperands = I.getNumOperands();
3228 if (NumOperands == 0)
3229 AbbrevToUse = FUNCTION_INST_RET_VOID_ABBREV;
3230 else if (NumOperands == 1) {
3231 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3232 AbbrevToUse = FUNCTION_INST_RET_VAL_ABBREV;
3233 } else {
3234 for (const Value *Op : I.operands())
3235 pushValueAndType(V: Op, InstID, Vals);
3236 }
3237 }
3238 break;
3239 case Instruction::Br:
3240 {
3241 Code = bitc::FUNC_CODE_INST_BR;
3242 AbbrevToUse = FUNCTION_INST_BR_UNCOND_ABBREV;
3243 const BranchInst &II = cast<BranchInst>(Val: I);
3244 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 0)));
3245 if (II.isConditional()) {
3246 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 1)));
3247 pushValue(V: II.getCondition(), InstID, Vals);
3248 AbbrevToUse = FUNCTION_INST_BR_COND_ABBREV;
3249 }
3250 }
3251 break;
3252 case Instruction::Switch:
3253 {
3254 Code = bitc::FUNC_CODE_INST_SWITCH;
3255 const SwitchInst &SI = cast<SwitchInst>(Val: I);
3256 Vals.push_back(Elt: VE.getTypeID(T: SI.getCondition()->getType()));
3257 pushValue(V: SI.getCondition(), InstID, Vals);
3258 Vals.push_back(Elt: VE.getValueID(V: SI.getDefaultDest()));
3259 for (auto Case : SI.cases()) {
3260 Vals.push_back(Elt: VE.getValueID(V: Case.getCaseValue()));
3261 Vals.push_back(Elt: VE.getValueID(V: Case.getCaseSuccessor()));
3262 }
3263 }
3264 break;
3265 case Instruction::IndirectBr:
3266 Code = bitc::FUNC_CODE_INST_INDIRECTBR;
3267 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType()));
3268 // Encode the address operand as relative, but not the basic blocks.
3269 pushValue(V: I.getOperand(i: 0), InstID, Vals);
3270 for (const Value *Op : drop_begin(RangeOrContainer: I.operands()))
3271 Vals.push_back(Elt: VE.getValueID(V: Op));
3272 break;
3273
3274 case Instruction::Invoke: {
3275 const InvokeInst *II = cast<InvokeInst>(Val: &I);
3276 const Value *Callee = II->getCalledOperand();
3277 FunctionType *FTy = II->getFunctionType();
3278
3279 if (II->hasOperandBundles())
3280 writeOperandBundles(CS: *II, InstID);
3281
3282 Code = bitc::FUNC_CODE_INST_INVOKE;
3283
3284 Vals.push_back(Elt: VE.getAttributeListID(PAL: II->getAttributes()));
3285 Vals.push_back(Elt: II->getCallingConv() | 1 << 13);
3286 Vals.push_back(Elt: VE.getValueID(V: II->getNormalDest()));
3287 Vals.push_back(Elt: VE.getValueID(V: II->getUnwindDest()));
3288 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3289 pushValueAndType(V: Callee, InstID, Vals);
3290
3291 // Emit value #'s for the fixed parameters.
3292 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3293 pushValue(V: I.getOperand(i), InstID, Vals); // fixed param.
3294
3295 // Emit type/value pairs for varargs params.
3296 if (FTy->isVarArg()) {
3297 for (unsigned i = FTy->getNumParams(), e = II->arg_size(); i != e; ++i)
3298 pushValueAndType(V: I.getOperand(i), InstID, Vals); // vararg
3299 }
3300 break;
3301 }
3302 case Instruction::Resume:
3303 Code = bitc::FUNC_CODE_INST_RESUME;
3304 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3305 break;
3306 case Instruction::CleanupRet: {
3307 Code = bitc::FUNC_CODE_INST_CLEANUPRET;
3308 const auto &CRI = cast<CleanupReturnInst>(Val: I);
3309 pushValue(V: CRI.getCleanupPad(), InstID, Vals);
3310 if (CRI.hasUnwindDest())
3311 Vals.push_back(Elt: VE.getValueID(V: CRI.getUnwindDest()));
3312 break;
3313 }
3314 case Instruction::CatchRet: {
3315 Code = bitc::FUNC_CODE_INST_CATCHRET;
3316 const auto &CRI = cast<CatchReturnInst>(Val: I);
3317 pushValue(V: CRI.getCatchPad(), InstID, Vals);
3318 Vals.push_back(Elt: VE.getValueID(V: CRI.getSuccessor()));
3319 break;
3320 }
3321 case Instruction::CleanupPad:
3322 case Instruction::CatchPad: {
3323 const auto &FuncletPad = cast<FuncletPadInst>(Val: I);
3324 Code = isa<CatchPadInst>(Val: FuncletPad) ? bitc::FUNC_CODE_INST_CATCHPAD
3325 : bitc::FUNC_CODE_INST_CLEANUPPAD;
3326 pushValue(V: FuncletPad.getParentPad(), InstID, Vals);
3327
3328 unsigned NumArgOperands = FuncletPad.arg_size();
3329 Vals.push_back(Elt: NumArgOperands);
3330 for (unsigned Op = 0; Op != NumArgOperands; ++Op)
3331 pushValueAndType(V: FuncletPad.getArgOperand(i: Op), InstID, Vals);
3332 break;
3333 }
3334 case Instruction::CatchSwitch: {
3335 Code = bitc::FUNC_CODE_INST_CATCHSWITCH;
3336 const auto &CatchSwitch = cast<CatchSwitchInst>(Val: I);
3337
3338 pushValue(V: CatchSwitch.getParentPad(), InstID, Vals);
3339
3340 unsigned NumHandlers = CatchSwitch.getNumHandlers();
3341 Vals.push_back(Elt: NumHandlers);
3342 for (const BasicBlock *CatchPadBB : CatchSwitch.handlers())
3343 Vals.push_back(Elt: VE.getValueID(V: CatchPadBB));
3344
3345 if (CatchSwitch.hasUnwindDest())
3346 Vals.push_back(Elt: VE.getValueID(V: CatchSwitch.getUnwindDest()));
3347 break;
3348 }
3349 case Instruction::CallBr: {
3350 const CallBrInst *CBI = cast<CallBrInst>(Val: &I);
3351 const Value *Callee = CBI->getCalledOperand();
3352 FunctionType *FTy = CBI->getFunctionType();
3353
3354 if (CBI->hasOperandBundles())
3355 writeOperandBundles(CS: *CBI, InstID);
3356
3357 Code = bitc::FUNC_CODE_INST_CALLBR;
3358
3359 Vals.push_back(Elt: VE.getAttributeListID(PAL: CBI->getAttributes()));
3360
3361 Vals.push_back(Elt: CBI->getCallingConv() << bitc::CALL_CCONV |
3362 1 << bitc::CALL_EXPLICIT_TYPE);
3363
3364 Vals.push_back(Elt: VE.getValueID(V: CBI->getDefaultDest()));
3365 Vals.push_back(Elt: CBI->getNumIndirectDests());
3366 for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i)
3367 Vals.push_back(Elt: VE.getValueID(V: CBI->getIndirectDest(i)));
3368
3369 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3370 pushValueAndType(V: Callee, InstID, Vals);
3371
3372 // Emit value #'s for the fixed parameters.
3373 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3374 pushValue(V: I.getOperand(i), InstID, Vals); // fixed param.
3375
3376 // Emit type/value pairs for varargs params.
3377 if (FTy->isVarArg()) {
3378 for (unsigned i = FTy->getNumParams(), e = CBI->arg_size(); i != e; ++i)
3379 pushValueAndType(V: I.getOperand(i), InstID, Vals); // vararg
3380 }
3381 break;
3382 }
3383 case Instruction::Unreachable:
3384 Code = bitc::FUNC_CODE_INST_UNREACHABLE;
3385 AbbrevToUse = FUNCTION_INST_UNREACHABLE_ABBREV;
3386 break;
3387
3388 case Instruction::PHI: {
3389 const PHINode &PN = cast<PHINode>(Val: I);
3390 Code = bitc::FUNC_CODE_INST_PHI;
3391 // With the newer instruction encoding, forward references could give
3392 // negative valued IDs. This is most common for PHIs, so we use
3393 // signed VBRs.
3394 SmallVector<uint64_t, 128> Vals64;
3395 Vals64.push_back(Elt: VE.getTypeID(T: PN.getType()));
3396 for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) {
3397 pushValueSigned(V: PN.getIncomingValue(i), InstID, Vals&: Vals64);
3398 Vals64.push_back(Elt: VE.getValueID(V: PN.getIncomingBlock(i)));
3399 }
3400
3401 uint64_t Flags = getOptimizationFlags(V: &I);
3402 if (Flags != 0)
3403 Vals64.push_back(Elt: Flags);
3404
3405 // Emit a Vals64 vector and exit.
3406 Stream.EmitRecord(Code, Vals: Vals64, Abbrev: AbbrevToUse);
3407 Vals64.clear();
3408 return;
3409 }
3410
3411 case Instruction::LandingPad: {
3412 const LandingPadInst &LP = cast<LandingPadInst>(Val: I);
3413 Code = bitc::FUNC_CODE_INST_LANDINGPAD;
3414 Vals.push_back(Elt: VE.getTypeID(T: LP.getType()));
3415 Vals.push_back(Elt: LP.isCleanup());
3416 Vals.push_back(Elt: LP.getNumClauses());
3417 for (unsigned I = 0, E = LP.getNumClauses(); I != E; ++I) {
3418 if (LP.isCatch(Idx: I))
3419 Vals.push_back(Elt: LandingPadInst::Catch);
3420 else
3421 Vals.push_back(Elt: LandingPadInst::Filter);
3422 pushValueAndType(V: LP.getClause(Idx: I), InstID, Vals);
3423 }
3424 break;
3425 }
3426
3427 case Instruction::Alloca: {
3428 Code = bitc::FUNC_CODE_INST_ALLOCA;
3429 const AllocaInst &AI = cast<AllocaInst>(Val: I);
3430 Vals.push_back(Elt: VE.getTypeID(T: AI.getAllocatedType()));
3431 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType()));
3432 Vals.push_back(Elt: VE.getValueID(V: I.getOperand(i: 0))); // size.
3433 using APV = AllocaPackedValues;
3434 unsigned Record = 0;
3435 unsigned EncodedAlign = getEncodedAlign(Alignment: AI.getAlign());
3436 Bitfield::set<APV::AlignLower>(
3437 Packed&: Record, Value: EncodedAlign & ((1 << APV::AlignLower::Bits) - 1));
3438 Bitfield::set<APV::AlignUpper>(Packed&: Record,
3439 Value: EncodedAlign >> APV::AlignLower::Bits);
3440 Bitfield::set<APV::UsedWithInAlloca>(Packed&: Record, Value: AI.isUsedWithInAlloca());
3441 Bitfield::set<APV::ExplicitType>(Packed&: Record, Value: true);
3442 Bitfield::set<APV::SwiftError>(Packed&: Record, Value: AI.isSwiftError());
3443 Vals.push_back(Elt: Record);
3444
3445 unsigned AS = AI.getAddressSpace();
3446 if (AS != M.getDataLayout().getAllocaAddrSpace())
3447 Vals.push_back(Elt: AS);
3448 break;
3449 }
3450
3451 case Instruction::Load:
3452 if (cast<LoadInst>(Val: I).isAtomic()) {
3453 Code = bitc::FUNC_CODE_INST_LOADATOMIC;
3454 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3455 } else {
3456 Code = bitc::FUNC_CODE_INST_LOAD;
3457 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals)) // ptr
3458 AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
3459 }
3460 Vals.push_back(Elt: VE.getTypeID(T: I.getType()));
3461 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<LoadInst>(Val: I).getAlign()));
3462 Vals.push_back(Elt: cast<LoadInst>(Val: I).isVolatile());
3463 if (cast<LoadInst>(Val: I).isAtomic()) {
3464 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<LoadInst>(Val: I).getOrdering()));
3465 Vals.push_back(Elt: getEncodedSyncScopeID(SSID: cast<LoadInst>(Val: I).getSyncScopeID()));
3466 }
3467 break;
3468 case Instruction::Store:
3469 if (cast<StoreInst>(Val: I).isAtomic()) {
3470 Code = bitc::FUNC_CODE_INST_STOREATOMIC;
3471 } else {
3472 Code = bitc::FUNC_CODE_INST_STORE;
3473 AbbrevToUse = FUNCTION_INST_STORE_ABBREV;
3474 }
3475 if (pushValueAndType(V: I.getOperand(i: 1), InstID, Vals)) // ptrty + ptr
3476 AbbrevToUse = 0;
3477 if (pushValueAndType(V: I.getOperand(i: 0), InstID, Vals)) // valty + val
3478 AbbrevToUse = 0;
3479 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<StoreInst>(Val: I).getAlign()));
3480 Vals.push_back(Elt: cast<StoreInst>(Val: I).isVolatile());
3481 if (cast<StoreInst>(Val: I).isAtomic()) {
3482 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<StoreInst>(Val: I).getOrdering()));
3483 Vals.push_back(
3484 Elt: getEncodedSyncScopeID(SSID: cast<StoreInst>(Val: I).getSyncScopeID()));
3485 }
3486 break;
3487 case Instruction::AtomicCmpXchg:
3488 Code = bitc::FUNC_CODE_INST_CMPXCHG;
3489 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals); // ptrty + ptr
3490 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals); // cmp.
3491 pushValue(V: I.getOperand(i: 2), InstID, Vals); // newval.
3492 Vals.push_back(Elt: cast<AtomicCmpXchgInst>(Val: I).isVolatile());
3493 Vals.push_back(
3494 Elt: getEncodedOrdering(Ordering: cast<AtomicCmpXchgInst>(Val: I).getSuccessOrdering()));
3495 Vals.push_back(
3496 Elt: getEncodedSyncScopeID(SSID: cast<AtomicCmpXchgInst>(Val: I).getSyncScopeID()));
3497 Vals.push_back(
3498 Elt: getEncodedOrdering(Ordering: cast<AtomicCmpXchgInst>(Val: I).getFailureOrdering()));
3499 Vals.push_back(Elt: cast<AtomicCmpXchgInst>(Val: I).isWeak());
3500 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<AtomicCmpXchgInst>(Val: I).getAlign()));
3501 break;
3502 case Instruction::AtomicRMW:
3503 Code = bitc::FUNC_CODE_INST_ATOMICRMW;
3504 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals); // ptrty + ptr
3505 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals); // valty + val
3506 Vals.push_back(
3507 Elt: getEncodedRMWOperation(Op: cast<AtomicRMWInst>(Val: I).getOperation()));
3508 Vals.push_back(Elt: cast<AtomicRMWInst>(Val: I).isVolatile());
3509 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<AtomicRMWInst>(Val: I).getOrdering()));
3510 Vals.push_back(
3511 Elt: getEncodedSyncScopeID(SSID: cast<AtomicRMWInst>(Val: I).getSyncScopeID()));
3512 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<AtomicRMWInst>(Val: I).getAlign()));
3513 break;
3514 case Instruction::Fence:
3515 Code = bitc::FUNC_CODE_INST_FENCE;
3516 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<FenceInst>(Val: I).getOrdering()));
3517 Vals.push_back(Elt: getEncodedSyncScopeID(SSID: cast<FenceInst>(Val: I).getSyncScopeID()));
3518 break;
3519 case Instruction::Call: {
3520 const CallInst &CI = cast<CallInst>(Val: I);
3521 FunctionType *FTy = CI.getFunctionType();
3522
3523 if (CI.hasOperandBundles())
3524 writeOperandBundles(CS: CI, InstID);
3525
3526 Code = bitc::FUNC_CODE_INST_CALL;
3527
3528 Vals.push_back(Elt: VE.getAttributeListID(PAL: CI.getAttributes()));
3529
3530 unsigned Flags = getOptimizationFlags(V: &I);
3531 Vals.push_back(Elt: CI.getCallingConv() << bitc::CALL_CCONV |
3532 unsigned(CI.isTailCall()) << bitc::CALL_TAIL |
3533 unsigned(CI.isMustTailCall()) << bitc::CALL_MUSTTAIL |
3534 1 << bitc::CALL_EXPLICIT_TYPE |
3535 unsigned(CI.isNoTailCall()) << bitc::CALL_NOTAIL |
3536 unsigned(Flags != 0) << bitc::CALL_FMF);
3537 if (Flags != 0)
3538 Vals.push_back(Elt: Flags);
3539
3540 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3541 pushValueAndType(V: CI.getCalledOperand(), InstID, Vals); // Callee
3542
3543 // Emit value #'s for the fixed parameters.
3544 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3545 pushValue(V: CI.getArgOperand(i), InstID, Vals); // fixed param.
3546
3547 // Emit type/value pairs for varargs params.
3548 if (FTy->isVarArg()) {
3549 for (unsigned i = FTy->getNumParams(), e = CI.arg_size(); i != e; ++i)
3550 pushValueAndType(V: CI.getArgOperand(i), InstID, Vals); // varargs
3551 }
3552 break;
3553 }
3554 case Instruction::VAArg:
3555 Code = bitc::FUNC_CODE_INST_VAARG;
3556 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType())); // valistty
3557 pushValue(V: I.getOperand(i: 0), InstID, Vals); // valist.
3558 Vals.push_back(Elt: VE.getTypeID(T: I.getType())); // restype.
3559 break;
3560 case Instruction::Freeze:
3561 Code = bitc::FUNC_CODE_INST_FREEZE;
3562 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3563 break;
3564 }
3565
3566 Stream.EmitRecord(Code, Vals, Abbrev: AbbrevToUse);
3567 Vals.clear();
3568}
3569
3570/// Write a GlobalValue VST to the module. The purpose of this data structure is
3571/// to allow clients to efficiently find the function body.
3572void ModuleBitcodeWriter::writeGlobalValueSymbolTable(
3573 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3574 // Get the offset of the VST we are writing, and backpatch it into
3575 // the VST forward declaration record.
3576 uint64_t VSTOffset = Stream.GetCurrentBitNo();
3577 // The BitcodeStartBit was the stream offset of the identification block.
3578 VSTOffset -= bitcodeStartBit();
3579 assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned");
3580 // Note that we add 1 here because the offset is relative to one word
3581 // before the start of the identification block, which was historically
3582 // always the start of the regular bitcode header.
3583 Stream.BackpatchWord(BitNo: VSTOffsetPlaceholder, Val: VSTOffset / 32 + 1);
3584
3585 Stream.EnterSubblock(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, CodeLen: 4);
3586
3587 auto Abbv = std::make_shared<BitCodeAbbrev>();
3588 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY));
3589 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
3590 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset
3591 unsigned FnEntryAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
3592
3593 for (const Function &F : M) {
3594 uint64_t Record[2];
3595
3596 if (F.isDeclaration())
3597 continue;
3598
3599 Record[0] = VE.getValueID(V: &F);
3600
3601 // Save the word offset of the function (from the start of the
3602 // actual bitcode written to the stream).
3603 uint64_t BitcodeIndex = FunctionToBitcodeIndex[&F] - bitcodeStartBit();
3604 assert((BitcodeIndex & 31) == 0 && "function block not 32-bit aligned");
3605 // Note that we add 1 here because the offset is relative to one word
3606 // before the start of the identification block, which was historically
3607 // always the start of the regular bitcode header.
3608 Record[1] = BitcodeIndex / 32 + 1;
3609
3610 Stream.EmitRecord(Code: bitc::VST_CODE_FNENTRY, Vals: Record, Abbrev: FnEntryAbbrev);
3611 }
3612
3613 Stream.ExitBlock();
3614}
3615
3616/// Emit names for arguments, instructions and basic blocks in a function.
3617void ModuleBitcodeWriter::writeFunctionLevelValueSymbolTable(
3618 const ValueSymbolTable &VST) {
3619 if (VST.empty())
3620 return;
3621
3622 Stream.EnterSubblock(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, CodeLen: 4);
3623
3624 // FIXME: Set up the abbrev, we know how many values there are!
3625 // FIXME: We know if the type names can use 7-bit ascii.
3626 SmallVector<uint64_t, 64> NameVals;
3627
3628 for (const ValueName &Name : VST) {
3629 // Figure out the encoding to use for the name.
3630 StringEncoding Bits = getStringEncoding(Str: Name.getKey());
3631
3632 unsigned AbbrevToUse = VST_ENTRY_8_ABBREV;
3633 NameVals.push_back(Elt: VE.getValueID(V: Name.getValue()));
3634
3635 // VST_CODE_ENTRY: [valueid, namechar x N]
3636 // VST_CODE_BBENTRY: [bbid, namechar x N]
3637 unsigned Code;
3638 if (isa<BasicBlock>(Val: Name.getValue())) {
3639 Code = bitc::VST_CODE_BBENTRY;
3640 if (Bits == SE_Char6)
3641 AbbrevToUse = VST_BBENTRY_6_ABBREV;
3642 } else {
3643 Code = bitc::VST_CODE_ENTRY;
3644 if (Bits == SE_Char6)
3645 AbbrevToUse = VST_ENTRY_6_ABBREV;
3646 else if (Bits == SE_Fixed7)
3647 AbbrevToUse = VST_ENTRY_7_ABBREV;
3648 }
3649
3650 for (const auto P : Name.getKey())
3651 NameVals.push_back(Elt: (unsigned char)P);
3652
3653 // Emit the finished record.
3654 Stream.EmitRecord(Code, Vals: NameVals, Abbrev: AbbrevToUse);
3655 NameVals.clear();
3656 }
3657
3658 Stream.ExitBlock();
3659}
3660
3661void ModuleBitcodeWriter::writeUseList(UseListOrder &&Order) {
3662 assert(Order.Shuffle.size() >= 2 && "Shuffle too small");
3663 unsigned Code;
3664 if (isa<BasicBlock>(Val: Order.V))
3665 Code = bitc::USELIST_CODE_BB;
3666 else
3667 Code = bitc::USELIST_CODE_DEFAULT;
3668
3669 SmallVector<uint64_t, 64> Record(Order.Shuffle.begin(), Order.Shuffle.end());
3670 Record.push_back(Elt: VE.getValueID(V: Order.V));
3671 Stream.EmitRecord(Code, Vals: Record);
3672}
3673
3674void ModuleBitcodeWriter::writeUseListBlock(const Function *F) {
3675 assert(VE.shouldPreserveUseListOrder() &&
3676 "Expected to be preserving use-list order");
3677
3678 auto hasMore = [&]() {
3679 return !VE.UseListOrders.empty() && VE.UseListOrders.back().F == F;
3680 };
3681 if (!hasMore())
3682 // Nothing to do.
3683 return;
3684
3685 Stream.EnterSubblock(BlockID: bitc::USELIST_BLOCK_ID, CodeLen: 3);
3686 while (hasMore()) {
3687 writeUseList(Order: std::move(VE.UseListOrders.back()));
3688 VE.UseListOrders.pop_back();
3689 }
3690 Stream.ExitBlock();
3691}
3692
3693/// Emit a function body to the module stream.
3694void ModuleBitcodeWriter::writeFunction(
3695 const Function &F,
3696 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3697 // Save the bitcode index of the start of this function block for recording
3698 // in the VST.
3699 FunctionToBitcodeIndex[&F] = Stream.GetCurrentBitNo();
3700
3701 Stream.EnterSubblock(BlockID: bitc::FUNCTION_BLOCK_ID, CodeLen: 5);
3702 VE.incorporateFunction(F);
3703
3704 SmallVector<unsigned, 64> Vals;
3705
3706 // Emit the number of basic blocks, so the reader can create them ahead of
3707 // time.
3708 Vals.push_back(Elt: VE.getBasicBlocks().size());
3709 Stream.EmitRecord(Code: bitc::FUNC_CODE_DECLAREBLOCKS, Vals);
3710 Vals.clear();
3711
3712 // If there are function-local constants, emit them now.
3713 unsigned CstStart, CstEnd;
3714 VE.getFunctionConstantRange(Start&: CstStart, End&: CstEnd);
3715 writeConstants(FirstVal: CstStart, LastVal: CstEnd, isGlobal: false);
3716
3717 // If there is function-local metadata, emit it now.
3718 writeFunctionMetadata(F);
3719
3720 // Keep a running idea of what the instruction ID is.
3721 unsigned InstID = CstEnd;
3722
3723 bool NeedsMetadataAttachment = F.hasMetadata();
3724
3725 DILocation *LastDL = nullptr;
3726 SmallSetVector<Function *, 4> BlockAddressUsers;
3727
3728 // Finally, emit all the instructions, in order.
3729 for (const BasicBlock &BB : F) {
3730 for (const Instruction &I : BB) {
3731 writeInstruction(I, InstID, Vals);
3732
3733 if (!I.getType()->isVoidTy())
3734 ++InstID;
3735
3736 // If the instruction has metadata, write a metadata attachment later.
3737 NeedsMetadataAttachment |= I.hasMetadataOtherThanDebugLoc();
3738
3739 // If the instruction has a debug location, emit it.
3740 if (DILocation *DL = I.getDebugLoc()) {
3741 if (DL == LastDL) {
3742 // Just repeat the same debug loc as last time.
3743 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_LOC_AGAIN, Vals);
3744 } else {
3745 Vals.push_back(Elt: DL->getLine());
3746 Vals.push_back(Elt: DL->getColumn());
3747 Vals.push_back(Elt: VE.getMetadataOrNullID(MD: DL->getScope()));
3748 Vals.push_back(Elt: VE.getMetadataOrNullID(MD: DL->getInlinedAt()));
3749 Vals.push_back(Elt: DL->isImplicitCode());
3750 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_LOC, Vals);
3751 Vals.clear();
3752 LastDL = DL;
3753 }
3754 }
3755
3756 // If the instruction has DbgRecords attached to it, emit them. Note that
3757 // they come after the instruction so that it's easy to attach them again
3758 // when reading the bitcode, even though conceptually the debug locations
3759 // start "before" the instruction.
3760 if (I.hasDbgRecords()) {
3761 /// Try to push the value only (unwrapped), otherwise push the
3762 /// metadata wrapped value. Returns true if the value was pushed
3763 /// without the ValueAsMetadata wrapper.
3764 auto PushValueOrMetadata = [&Vals, InstID,
3765 this](Metadata *RawLocation) {
3766 assert(RawLocation &&
3767 "RawLocation unexpectedly null in DbgVariableRecord");
3768 if (ValueAsMetadata *VAM = dyn_cast<ValueAsMetadata>(Val: RawLocation)) {
3769 SmallVector<unsigned, 2> ValAndType;
3770 // If the value is a fwd-ref the type is also pushed. We don't
3771 // want the type, so fwd-refs are kept wrapped (pushValueAndType
3772 // returns false if the value is pushed without type).
3773 if (!pushValueAndType(V: VAM->getValue(), InstID, Vals&: ValAndType)) {
3774 Vals.push_back(Elt: ValAndType[0]);
3775 return true;
3776 }
3777 }
3778 // The metadata is a DIArgList, or ValueAsMetadata wrapping a
3779 // fwd-ref. Push the metadata ID.
3780 Vals.push_back(Elt: VE.getMetadataID(MD: RawLocation));
3781 return false;
3782 };
3783
3784 // Write out non-instruction debug information attached to this
3785 // instruction. Write it after the instruction so that it's easy to
3786 // re-attach to the instruction reading the records in.
3787 for (DbgRecord &DR : I.DebugMarker->getDbgRecordRange()) {
3788 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
3789 Vals.push_back(Elt: VE.getMetadataID(MD: &*DLR->getDebugLoc()));
3790 Vals.push_back(Elt: VE.getMetadataID(MD: DLR->getLabel()));
3791 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_LABEL, Vals);
3792 Vals.clear();
3793 continue;
3794 }
3795
3796 // First 3 fields are common to all kinds:
3797 // DILocation, DILocalVariable, DIExpression
3798 // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE)
3799 // ..., LocationMetadata
3800 // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE - abbrev'd)
3801 // ..., Value
3802 // dbg_declare (FUNC_CODE_DEBUG_RECORD_DECLARE)
3803 // ..., LocationMetadata
3804 // dbg_assign (FUNC_CODE_DEBUG_RECORD_ASSIGN)
3805 // ..., LocationMetadata, DIAssignID, DIExpression, LocationMetadata
3806 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
3807 Vals.push_back(Elt: VE.getMetadataID(MD: &*DVR.getDebugLoc()));
3808 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getVariable()));
3809 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getExpression()));
3810 if (DVR.isDbgValue()) {
3811 if (PushValueOrMetadata(DVR.getRawLocation()))
3812 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE, Vals,
3813 Abbrev: FUNCTION_DEBUG_RECORD_VALUE_ABBREV);
3814 else
3815 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_VALUE, Vals);
3816 } else if (DVR.isDbgDeclare()) {
3817 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3818 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_DECLARE, Vals);
3819 } else {
3820 assert(DVR.isDbgAssign() && "Unexpected DbgRecord kind");
3821 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3822 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getAssignID()));
3823 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getAddressExpression()));
3824 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawAddress()));
3825 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_ASSIGN, Vals);
3826 }
3827 Vals.clear();
3828 }
3829 }
3830 }
3831
3832 if (BlockAddress *BA = BlockAddress::lookup(BB: &BB)) {
3833 SmallVector<Value *> Worklist{BA};
3834 SmallPtrSet<Value *, 8> Visited{BA};
3835 while (!Worklist.empty()) {
3836 Value *V = Worklist.pop_back_val();
3837 for (User *U : V->users()) {
3838 if (auto *I = dyn_cast<Instruction>(Val: U)) {
3839 Function *P = I->getFunction();
3840 if (P != &F)
3841 BlockAddressUsers.insert(X: P);
3842 } else if (isa<Constant>(Val: U) && !isa<GlobalValue>(Val: U) &&
3843 Visited.insert(Ptr: U).second)
3844 Worklist.push_back(Elt: U);
3845 }
3846 }
3847 }
3848 }
3849
3850 if (!BlockAddressUsers.empty()) {
3851 Vals.resize(N: BlockAddressUsers.size());
3852 for (auto I : llvm::enumerate(First&: BlockAddressUsers))
3853 Vals[I.index()] = VE.getValueID(V: I.value());
3854 Stream.EmitRecord(Code: bitc::FUNC_CODE_BLOCKADDR_USERS, Vals);
3855 Vals.clear();
3856 }
3857
3858 // Emit names for all the instructions etc.
3859 if (auto *Symtab = F.getValueSymbolTable())
3860 writeFunctionLevelValueSymbolTable(VST: *Symtab);
3861
3862 if (NeedsMetadataAttachment)
3863 writeFunctionMetadataAttachment(F);
3864 if (VE.shouldPreserveUseListOrder())
3865 writeUseListBlock(F: &F);
3866 VE.purgeFunction();
3867 Stream.ExitBlock();
3868}
3869
3870// Emit blockinfo, which defines the standard abbreviations etc.
3871void ModuleBitcodeWriter::writeBlockInfo() {
3872 // We only want to emit block info records for blocks that have multiple
3873 // instances: CONSTANTS_BLOCK, FUNCTION_BLOCK and VALUE_SYMTAB_BLOCK.
3874 // Other blocks can define their abbrevs inline.
3875 Stream.EnterBlockInfoBlock();
3876
3877 // Encode type indices using fixed size based on number of types.
3878 BitCodeAbbrevOp TypeAbbrevOp(BitCodeAbbrevOp::Fixed,
3879 VE.computeBitsRequiredForTypeIndices());
3880 // Encode value indices as 6-bit VBR.
3881 BitCodeAbbrevOp ValAbbrevOp(BitCodeAbbrevOp::VBR, 6);
3882
3883 { // 8-bit fixed-width VST_CODE_ENTRY/VST_CODE_BBENTRY strings.
3884 auto Abbv = std::make_shared<BitCodeAbbrev>();
3885 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3));
3886 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3887 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3888 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
3889 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3890 VST_ENTRY_8_ABBREV)
3891 llvm_unreachable("Unexpected abbrev ordering!");
3892 }
3893
3894 { // 7-bit fixed width VST_CODE_ENTRY strings.
3895 auto Abbv = std::make_shared<BitCodeAbbrev>();
3896 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
3897 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3898 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3899 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
3900 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3901 VST_ENTRY_7_ABBREV)
3902 llvm_unreachable("Unexpected abbrev ordering!");
3903 }
3904 { // 6-bit char6 VST_CODE_ENTRY strings.
3905 auto Abbv = std::make_shared<BitCodeAbbrev>();
3906 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
3907 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3908 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3909 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
3910 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3911 VST_ENTRY_6_ABBREV)
3912 llvm_unreachable("Unexpected abbrev ordering!");
3913 }
3914 { // 6-bit char6 VST_CODE_BBENTRY strings.
3915 auto Abbv = std::make_shared<BitCodeAbbrev>();
3916 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_BBENTRY));
3917 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3918 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3919 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
3920 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3921 VST_BBENTRY_6_ABBREV)
3922 llvm_unreachable("Unexpected abbrev ordering!");
3923 }
3924
3925 { // SETTYPE abbrev for CONSTANTS_BLOCK.
3926 auto Abbv = std::make_shared<BitCodeAbbrev>();
3927 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_SETTYPE));
3928 Abbv->Add(OpInfo: TypeAbbrevOp);
3929 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3930 CONSTANTS_SETTYPE_ABBREV)
3931 llvm_unreachable("Unexpected abbrev ordering!");
3932 }
3933
3934 { // INTEGER abbrev for CONSTANTS_BLOCK.
3935 auto Abbv = std::make_shared<BitCodeAbbrev>();
3936 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_INTEGER));
3937 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3938 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3939 CONSTANTS_INTEGER_ABBREV)
3940 llvm_unreachable("Unexpected abbrev ordering!");
3941 }
3942
3943 { // CE_CAST abbrev for CONSTANTS_BLOCK.
3944 auto Abbv = std::make_shared<BitCodeAbbrev>();
3945 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CE_CAST));
3946 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // cast opc
3947 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // typeid
3948 VE.computeBitsRequiredForTypeIndices()));
3949 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
3950
3951 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3952 CONSTANTS_CE_CAST_Abbrev)
3953 llvm_unreachable("Unexpected abbrev ordering!");
3954 }
3955 { // NULL abbrev for CONSTANTS_BLOCK.
3956 auto Abbv = std::make_shared<BitCodeAbbrev>();
3957 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_NULL));
3958 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3959 CONSTANTS_NULL_Abbrev)
3960 llvm_unreachable("Unexpected abbrev ordering!");
3961 }
3962
3963 // FIXME: This should only use space for first class types!
3964
3965 { // INST_LOAD abbrev for FUNCTION_BLOCK.
3966 auto Abbv = std::make_shared<BitCodeAbbrev>();
3967 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_LOAD));
3968 Abbv->Add(OpInfo: ValAbbrevOp); // Ptr
3969 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
3970 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Align
3971 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
3972 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
3973 FUNCTION_INST_LOAD_ABBREV)
3974 llvm_unreachable("Unexpected abbrev ordering!");
3975 }
3976 {
3977 auto Abbv = std::make_shared<BitCodeAbbrev>();
3978 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_STORE));
3979 Abbv->Add(OpInfo: ValAbbrevOp); // op1
3980 Abbv->Add(OpInfo: ValAbbrevOp); // op0
3981 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // align
3982 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
3983 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
3984 FUNCTION_INST_STORE_ABBREV)
3985 llvm_unreachable("Unexpected abbrev ordering!");
3986 }
3987 { // INST_UNOP abbrev for FUNCTION_BLOCK.
3988 auto Abbv = std::make_shared<BitCodeAbbrev>();
3989 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
3990 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
3991 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
3992 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
3993 FUNCTION_INST_UNOP_ABBREV)
3994 llvm_unreachable("Unexpected abbrev ordering!");
3995 }
3996 { // INST_UNOP_FLAGS abbrev for FUNCTION_BLOCK.
3997 auto Abbv = std::make_shared<BitCodeAbbrev>();
3998 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
3999 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4000 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4001 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4002 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4003 FUNCTION_INST_UNOP_FLAGS_ABBREV)
4004 llvm_unreachable("Unexpected abbrev ordering!");
4005 }
4006 { // INST_BINOP abbrev for FUNCTION_BLOCK.
4007 auto Abbv = std::make_shared<BitCodeAbbrev>();
4008 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
4009 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4010 Abbv->Add(OpInfo: ValAbbrevOp); // RHS
4011 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4012 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4013 FUNCTION_INST_BINOP_ABBREV)
4014 llvm_unreachable("Unexpected abbrev ordering!");
4015 }
4016 { // INST_BINOP_FLAGS abbrev for FUNCTION_BLOCK.
4017 auto Abbv = std::make_shared<BitCodeAbbrev>();
4018 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
4019 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4020 Abbv->Add(OpInfo: ValAbbrevOp); // RHS
4021 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4022 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4023 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4024 FUNCTION_INST_BINOP_FLAGS_ABBREV)
4025 llvm_unreachable("Unexpected abbrev ordering!");
4026 }
4027 { // INST_CAST abbrev for FUNCTION_BLOCK.
4028 auto Abbv = std::make_shared<BitCodeAbbrev>();
4029 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
4030 Abbv->Add(OpInfo: ValAbbrevOp); // OpVal
4031 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4032 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4033 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4034 FUNCTION_INST_CAST_ABBREV)
4035 llvm_unreachable("Unexpected abbrev ordering!");
4036 }
4037 { // INST_CAST_FLAGS abbrev for FUNCTION_BLOCK.
4038 auto Abbv = std::make_shared<BitCodeAbbrev>();
4039 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
4040 Abbv->Add(OpInfo: ValAbbrevOp); // OpVal
4041 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4042 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4043 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4044 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4045 FUNCTION_INST_CAST_FLAGS_ABBREV)
4046 llvm_unreachable("Unexpected abbrev ordering!");
4047 }
4048
4049 { // INST_RET abbrev for FUNCTION_BLOCK.
4050 auto Abbv = std::make_shared<BitCodeAbbrev>();
4051 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
4052 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4053 FUNCTION_INST_RET_VOID_ABBREV)
4054 llvm_unreachable("Unexpected abbrev ordering!");
4055 }
4056 { // INST_RET abbrev for FUNCTION_BLOCK.
4057 auto Abbv = std::make_shared<BitCodeAbbrev>();
4058 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
4059 Abbv->Add(OpInfo: ValAbbrevOp);
4060 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4061 FUNCTION_INST_RET_VAL_ABBREV)
4062 llvm_unreachable("Unexpected abbrev ordering!");
4063 }
4064 {
4065 auto Abbv = std::make_shared<BitCodeAbbrev>();
4066 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BR));
4067 // TODO: Use different abbrev for absolute value reference (succ0)?
4068 Abbv->Add(OpInfo: ValAbbrevOp); // succ0
4069 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4070 FUNCTION_INST_BR_UNCOND_ABBREV)
4071 llvm_unreachable("Unexpected abbrev ordering!");
4072 }
4073 {
4074 auto Abbv = std::make_shared<BitCodeAbbrev>();
4075 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BR));
4076 // TODO: Use different abbrev for absolute value references (succ0, succ1)?
4077 Abbv->Add(OpInfo: ValAbbrevOp); // succ0
4078 Abbv->Add(OpInfo: ValAbbrevOp); // succ1
4079 Abbv->Add(OpInfo: ValAbbrevOp); // cond
4080 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4081 FUNCTION_INST_BR_COND_ABBREV)
4082 llvm_unreachable("Unexpected abbrev ordering!");
4083 }
4084 { // INST_UNREACHABLE abbrev for FUNCTION_BLOCK.
4085 auto Abbv = std::make_shared<BitCodeAbbrev>();
4086 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNREACHABLE));
4087 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4088 FUNCTION_INST_UNREACHABLE_ABBREV)
4089 llvm_unreachable("Unexpected abbrev ordering!");
4090 }
4091 {
4092 auto Abbv = std::make_shared<BitCodeAbbrev>();
4093 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_GEP));
4094 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // flags
4095 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4096 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4097 Abbv->Add(OpInfo: ValAbbrevOp);
4098 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4099 FUNCTION_INST_GEP_ABBREV)
4100 llvm_unreachable("Unexpected abbrev ordering!");
4101 }
4102 {
4103 auto Abbv = std::make_shared<BitCodeAbbrev>();
4104 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CMP2));
4105 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4106 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4107 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // pred
4108 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4109 FUNCTION_INST_CMP_ABBREV)
4110 llvm_unreachable("Unexpected abbrev ordering!");
4111 }
4112 {
4113 auto Abbv = std::make_shared<BitCodeAbbrev>();
4114 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CMP2));
4115 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4116 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4117 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // pred
4118 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4119 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4120 FUNCTION_INST_CMP_FLAGS_ABBREV)
4121 llvm_unreachable("Unexpected abbrev ordering!");
4122 }
4123 {
4124 auto Abbv = std::make_shared<BitCodeAbbrev>();
4125 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE));
4126 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // dbgloc
4127 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // var
4128 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // expr
4129 Abbv->Add(OpInfo: ValAbbrevOp); // val
4130 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4131 FUNCTION_DEBUG_RECORD_VALUE_ABBREV)
4132 llvm_unreachable("Unexpected abbrev ordering! 1");
4133 }
4134 Stream.ExitBlock();
4135}
4136
4137/// Write the module path strings, currently only used when generating
4138/// a combined index file.
4139void IndexBitcodeWriter::writeModStrings() {
4140 Stream.EnterSubblock(BlockID: bitc::MODULE_STRTAB_BLOCK_ID, CodeLen: 3);
4141
4142 // TODO: See which abbrev sizes we actually need to emit
4143
4144 // 8-bit fixed-width MST_ENTRY strings.
4145 auto Abbv = std::make_shared<BitCodeAbbrev>();
4146 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4147 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4148 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4149 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
4150 unsigned Abbrev8Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4151
4152 // 7-bit fixed width MST_ENTRY strings.
4153 Abbv = std::make_shared<BitCodeAbbrev>();
4154 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4155 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4156 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4157 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
4158 unsigned Abbrev7Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4159
4160 // 6-bit char6 MST_ENTRY strings.
4161 Abbv = std::make_shared<BitCodeAbbrev>();
4162 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4163 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4164 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4165 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
4166 unsigned Abbrev6Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4167
4168 // Module Hash, 160 bits SHA1. Optionally, emitted after each MST_CODE_ENTRY.
4169 Abbv = std::make_shared<BitCodeAbbrev>();
4170 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_HASH));
4171 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4172 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4173 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4174 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4175 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4176 unsigned AbbrevHash = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4177
4178 SmallVector<unsigned, 64> Vals;
4179 forEachModule(Callback: [&](const StringMapEntry<ModuleHash> &MPSE) {
4180 StringRef Key = MPSE.getKey();
4181 const auto &Hash = MPSE.getValue();
4182 StringEncoding Bits = getStringEncoding(Str: Key);
4183 unsigned AbbrevToUse = Abbrev8Bit;
4184 if (Bits == SE_Char6)
4185 AbbrevToUse = Abbrev6Bit;
4186 else if (Bits == SE_Fixed7)
4187 AbbrevToUse = Abbrev7Bit;
4188
4189 auto ModuleId = ModuleIdMap.size();
4190 ModuleIdMap[Key] = ModuleId;
4191 Vals.push_back(Elt: ModuleId);
4192 Vals.append(in_start: Key.begin(), in_end: Key.end());
4193
4194 // Emit the finished record.
4195 Stream.EmitRecord(Code: bitc::MST_CODE_ENTRY, Vals, Abbrev: AbbrevToUse);
4196
4197 // Emit an optional hash for the module now
4198 if (llvm::any_of(Range: Hash, P: [](uint32_t H) { return H; })) {
4199 Vals.assign(in_start: Hash.begin(), in_end: Hash.end());
4200 // Emit the hash record.
4201 Stream.EmitRecord(Code: bitc::MST_CODE_HASH, Vals, Abbrev: AbbrevHash);
4202 }
4203
4204 Vals.clear();
4205 });
4206 Stream.ExitBlock();
4207}
4208
4209/// Write the function type metadata related records that need to appear before
4210/// a function summary entry (whether per-module or combined).
4211template <typename Fn>
4212static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream,
4213 FunctionSummary *FS,
4214 Fn GetValueID) {
4215 if (!FS->type_tests().empty())
4216 Stream.EmitRecord(Code: bitc::FS_TYPE_TESTS, Vals: FS->type_tests());
4217
4218 SmallVector<uint64_t, 64> Record;
4219
4220 auto WriteVFuncIdVec = [&](uint64_t Ty,
4221 ArrayRef<FunctionSummary::VFuncId> VFs) {
4222 if (VFs.empty())
4223 return;
4224 Record.clear();
4225 for (auto &VF : VFs) {
4226 Record.push_back(Elt: VF.GUID);
4227 Record.push_back(Elt: VF.Offset);
4228 }
4229 Stream.EmitRecord(Code: Ty, Vals: Record);
4230 };
4231
4232 WriteVFuncIdVec(bitc::FS_TYPE_TEST_ASSUME_VCALLS,
4233 FS->type_test_assume_vcalls());
4234 WriteVFuncIdVec(bitc::FS_TYPE_CHECKED_LOAD_VCALLS,
4235 FS->type_checked_load_vcalls());
4236
4237 auto WriteConstVCallVec = [&](uint64_t Ty,
4238 ArrayRef<FunctionSummary::ConstVCall> VCs) {
4239 for (auto &VC : VCs) {
4240 Record.clear();
4241 Record.push_back(Elt: VC.VFunc.GUID);
4242 Record.push_back(Elt: VC.VFunc.Offset);
4243 llvm::append_range(C&: Record, R: VC.Args);
4244 Stream.EmitRecord(Code: Ty, Vals: Record);
4245 }
4246 };
4247
4248 WriteConstVCallVec(bitc::FS_TYPE_TEST_ASSUME_CONST_VCALL,
4249 FS->type_test_assume_const_vcalls());
4250 WriteConstVCallVec(bitc::FS_TYPE_CHECKED_LOAD_CONST_VCALL,
4251 FS->type_checked_load_const_vcalls());
4252
4253 auto WriteRange = [&](ConstantRange Range) {
4254 Range = Range.sextOrTrunc(BitWidth: FunctionSummary::ParamAccess::RangeWidth);
4255 assert(Range.getLower().getNumWords() == 1);
4256 assert(Range.getUpper().getNumWords() == 1);
4257 emitSignedInt64(Vals&: Record, V: *Range.getLower().getRawData());
4258 emitSignedInt64(Vals&: Record, V: *Range.getUpper().getRawData());
4259 };
4260
4261 if (!FS->paramAccesses().empty()) {
4262 Record.clear();
4263 for (auto &Arg : FS->paramAccesses()) {
4264 size_t UndoSize = Record.size();
4265 Record.push_back(Elt: Arg.ParamNo);
4266 WriteRange(Arg.Use);
4267 Record.push_back(Elt: Arg.Calls.size());
4268 for (auto &Call : Arg.Calls) {
4269 Record.push_back(Elt: Call.ParamNo);
4270 std::optional<unsigned> ValueID = GetValueID(Call.Callee);
4271 if (!ValueID) {
4272 // If ValueID is unknown we can't drop just this call, we must drop
4273 // entire parameter.
4274 Record.resize(N: UndoSize);
4275 break;
4276 }
4277 Record.push_back(Elt: *ValueID);
4278 WriteRange(Call.Offsets);
4279 }
4280 }
4281 if (!Record.empty())
4282 Stream.EmitRecord(Code: bitc::FS_PARAM_ACCESS, Vals: Record);
4283 }
4284}
4285
4286/// Collect type IDs from type tests used by function.
4287static void
4288getReferencedTypeIds(FunctionSummary *FS,
4289 std::set<GlobalValue::GUID> &ReferencedTypeIds) {
4290 if (!FS->type_tests().empty())
4291 for (auto &TT : FS->type_tests())
4292 ReferencedTypeIds.insert(x: TT);
4293
4294 auto GetReferencedTypesFromVFuncIdVec =
4295 [&](ArrayRef<FunctionSummary::VFuncId> VFs) {
4296 for (auto &VF : VFs)
4297 ReferencedTypeIds.insert(x: VF.GUID);
4298 };
4299
4300 GetReferencedTypesFromVFuncIdVec(FS->type_test_assume_vcalls());
4301 GetReferencedTypesFromVFuncIdVec(FS->type_checked_load_vcalls());
4302
4303 auto GetReferencedTypesFromConstVCallVec =
4304 [&](ArrayRef<FunctionSummary::ConstVCall> VCs) {
4305 for (auto &VC : VCs)
4306 ReferencedTypeIds.insert(x: VC.VFunc.GUID);
4307 };
4308
4309 GetReferencedTypesFromConstVCallVec(FS->type_test_assume_const_vcalls());
4310 GetReferencedTypesFromConstVCallVec(FS->type_checked_load_const_vcalls());
4311}
4312
4313static void writeWholeProgramDevirtResolutionByArg(
4314 SmallVector<uint64_t, 64> &NameVals, const std::vector<uint64_t> &args,
4315 const WholeProgramDevirtResolution::ByArg &ByArg) {
4316 NameVals.push_back(Elt: args.size());
4317 llvm::append_range(C&: NameVals, R: args);
4318
4319 NameVals.push_back(Elt: ByArg.TheKind);
4320 NameVals.push_back(Elt: ByArg.Info);
4321 NameVals.push_back(Elt: ByArg.Byte);
4322 NameVals.push_back(Elt: ByArg.Bit);
4323}
4324
4325static void writeWholeProgramDevirtResolution(
4326 SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
4327 uint64_t Id, const WholeProgramDevirtResolution &Wpd) {
4328 NameVals.push_back(Elt: Id);
4329
4330 NameVals.push_back(Elt: Wpd.TheKind);
4331 NameVals.push_back(Elt: StrtabBuilder.add(S: Wpd.SingleImplName));
4332 NameVals.push_back(Elt: Wpd.SingleImplName.size());
4333
4334 NameVals.push_back(Elt: Wpd.ResByArg.size());
4335 for (auto &A : Wpd.ResByArg)
4336 writeWholeProgramDevirtResolutionByArg(NameVals, args: A.first, ByArg: A.second);
4337}
4338
4339static void writeTypeIdSummaryRecord(SmallVector<uint64_t, 64> &NameVals,
4340 StringTableBuilder &StrtabBuilder,
4341 StringRef Id,
4342 const TypeIdSummary &Summary) {
4343 NameVals.push_back(Elt: StrtabBuilder.add(S: Id));
4344 NameVals.push_back(Elt: Id.size());
4345
4346 NameVals.push_back(Elt: Summary.TTRes.TheKind);
4347 NameVals.push_back(Elt: Summary.TTRes.SizeM1BitWidth);
4348 NameVals.push_back(Elt: Summary.TTRes.AlignLog2);
4349 NameVals.push_back(Elt: Summary.TTRes.SizeM1);
4350 NameVals.push_back(Elt: Summary.TTRes.BitMask);
4351 NameVals.push_back(Elt: Summary.TTRes.InlineBits);
4352
4353 for (auto &W : Summary.WPDRes)
4354 writeWholeProgramDevirtResolution(NameVals, StrtabBuilder, Id: W.first,
4355 Wpd: W.second);
4356}
4357
4358static void writeTypeIdCompatibleVtableSummaryRecord(
4359 SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
4360 StringRef Id, const TypeIdCompatibleVtableInfo &Summary,
4361 ValueEnumerator &VE) {
4362 NameVals.push_back(Elt: StrtabBuilder.add(S: Id));
4363 NameVals.push_back(Elt: Id.size());
4364
4365 for (auto &P : Summary) {
4366 NameVals.push_back(Elt: P.AddressPointOffset);
4367 NameVals.push_back(Elt: VE.getValueID(V: P.VTableVI.getValue()));
4368 }
4369}
4370
4371// Adds the allocation contexts to the CallStacks map. We simply use the
4372// size at the time the context was added as the CallStackId. This works because
4373// when we look up the call stacks later on we process the function summaries
4374// and their allocation records in the same exact order.
4375static void collectMemProfCallStacks(
4376 FunctionSummary *FS, std::function<LinearFrameId(unsigned)> GetStackIndex,
4377 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> &CallStacks) {
4378 // The interfaces in ProfileData/MemProf.h use a type alias for a stack frame
4379 // id offset into the index of the full stack frames. The ModuleSummaryIndex
4380 // currently uses unsigned. Make sure these stay in sync.
4381 static_assert(std::is_same_v<LinearFrameId, unsigned>);
4382 for (auto &AI : FS->allocs()) {
4383 for (auto &MIB : AI.MIBs) {
4384 SmallVector<unsigned> StackIdIndices;
4385 StackIdIndices.reserve(N: MIB.StackIdIndices.size());
4386 for (auto Id : MIB.StackIdIndices)
4387 StackIdIndices.push_back(Elt: GetStackIndex(Id));
4388 // The CallStackId is the size at the time this context was inserted.
4389 CallStacks.insert(KV: {CallStacks.size(), StackIdIndices});
4390 }
4391 }
4392}
4393
4394// Build the radix tree from the accumulated CallStacks, write out the resulting
4395// linearized radix tree array, and return the map of call stack positions into
4396// this array for use when writing the allocation records. The returned map is
4397// indexed by a CallStackId which in this case is implicitly determined by the
4398// order of function summaries and their allocation infos being written.
4399static DenseMap<CallStackId, LinearCallStackId> writeMemoryProfileRadixTree(
4400 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> &&CallStacks,
4401 BitstreamWriter &Stream, unsigned RadixAbbrev) {
4402 assert(!CallStacks.empty());
4403 DenseMap<unsigned, FrameStat> FrameHistogram =
4404 computeFrameHistogram<LinearFrameId>(MemProfCallStackData&: CallStacks);
4405 CallStackRadixTreeBuilder<LinearFrameId> Builder;
4406 // We don't need a MemProfFrameIndexes map as we have already converted the
4407 // full stack id hash to a linear offset into the StackIds array.
4408 Builder.build(MemProfCallStackData: std::move(CallStacks), /*MemProfFrameIndexes=*/nullptr,
4409 FrameHistogram);
4410 Stream.EmitRecord(Code: bitc::FS_CONTEXT_RADIX_TREE_ARRAY, Vals: Builder.getRadixArray(),
4411 Abbrev: RadixAbbrev);
4412 return Builder.takeCallStackPos();
4413}
4414
4415static void writeFunctionHeapProfileRecords(
4416 BitstreamWriter &Stream, FunctionSummary *FS, unsigned CallsiteAbbrev,
4417 unsigned AllocAbbrev, unsigned ContextIdAbbvId, bool PerModule,
4418 std::function<unsigned(const ValueInfo &VI)> GetValueID,
4419 std::function<unsigned(unsigned)> GetStackIndex,
4420 bool WriteContextSizeInfoIndex,
4421 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
4422 CallStackId &CallStackCount) {
4423 SmallVector<uint64_t> Record;
4424
4425 for (auto &CI : FS->callsites()) {
4426 Record.clear();
4427 // Per module callsite clones should always have a single entry of
4428 // value 0.
4429 assert(!PerModule || (CI.Clones.size() == 1 && CI.Clones[0] == 0));
4430 Record.push_back(Elt: GetValueID(CI.Callee));
4431 if (!PerModule) {
4432 Record.push_back(Elt: CI.StackIdIndices.size());
4433 Record.push_back(Elt: CI.Clones.size());
4434 }
4435 for (auto Id : CI.StackIdIndices)
4436 Record.push_back(Elt: GetStackIndex(Id));
4437 if (!PerModule)
4438 llvm::append_range(C&: Record, R: CI.Clones);
4439 Stream.EmitRecord(Code: PerModule ? bitc::FS_PERMODULE_CALLSITE_INFO
4440 : bitc::FS_COMBINED_CALLSITE_INFO,
4441 Vals: Record, Abbrev: CallsiteAbbrev);
4442 }
4443
4444 for (auto &AI : FS->allocs()) {
4445 Record.clear();
4446 // Per module alloc versions should always have a single entry of
4447 // value 0.
4448 assert(!PerModule || (AI.Versions.size() == 1 && AI.Versions[0] == 0));
4449 Record.push_back(Elt: AI.MIBs.size());
4450 if (!PerModule)
4451 Record.push_back(Elt: AI.Versions.size());
4452 for (auto &MIB : AI.MIBs) {
4453 Record.push_back(Elt: (uint8_t)MIB.AllocType);
4454 // The per-module summary always needs to include the alloc context, as we
4455 // use it during the thin link. For the combined index it is optional (see
4456 // comments where CombinedIndexMemProfContext is defined).
4457 if (PerModule || CombinedIndexMemProfContext) {
4458 // Record the index into the radix tree array for this context.
4459 assert(CallStackCount <= CallStackPos.size());
4460 Record.push_back(Elt: CallStackPos[CallStackCount++]);
4461 }
4462 }
4463 if (!PerModule)
4464 llvm::append_range(C&: Record, R: AI.Versions);
4465 assert(AI.ContextSizeInfos.empty() ||
4466 AI.ContextSizeInfos.size() == AI.MIBs.size());
4467 // Optionally emit the context size information if it exists.
4468 if (WriteContextSizeInfoIndex && !AI.ContextSizeInfos.empty()) {
4469 // The abbreviation id for the context ids record should have been created
4470 // if we are emitting the per-module index, which is where we write this
4471 // info.
4472 assert(ContextIdAbbvId);
4473 SmallVector<uint32_t> ContextIds;
4474 // At least one context id per ContextSizeInfos entry (MIB), broken into 2
4475 // halves.
4476 ContextIds.reserve(N: AI.ContextSizeInfos.size() * 2);
4477 for (auto &Infos : AI.ContextSizeInfos) {
4478 Record.push_back(Elt: Infos.size());
4479 for (auto [FullStackId, TotalSize] : Infos) {
4480 // The context ids are emitted separately as a fixed width array,
4481 // which is more efficient than a VBR given that these hashes are
4482 // typically close to 64-bits. The max fixed width entry is 32 bits so
4483 // it is split into 2.
4484 ContextIds.push_back(Elt: static_cast<uint32_t>(FullStackId >> 32));
4485 ContextIds.push_back(Elt: static_cast<uint32_t>(FullStackId));
4486 Record.push_back(Elt: TotalSize);
4487 }
4488 }
4489 // The context ids are expected by the reader to immediately precede the
4490 // associated alloc info record.
4491 Stream.EmitRecord(Code: bitc::FS_ALLOC_CONTEXT_IDS, Vals: ContextIds,
4492 Abbrev: ContextIdAbbvId);
4493 }
4494 Stream.EmitRecord(Code: PerModule
4495 ? bitc::FS_PERMODULE_ALLOC_INFO
4496 : (CombinedIndexMemProfContext
4497 ? bitc::FS_COMBINED_ALLOC_INFO
4498 : bitc::FS_COMBINED_ALLOC_INFO_NO_CONTEXT),
4499 Vals: Record, Abbrev: AllocAbbrev);
4500 }
4501}
4502
4503// Helper to emit a single function summary record.
4504void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
4505 SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
4506 unsigned ValueID, unsigned FSCallsRelBFAbbrev,
4507 unsigned FSCallsProfileAbbrev, unsigned CallsiteAbbrev,
4508 unsigned AllocAbbrev, unsigned ContextIdAbbvId, const Function &F,
4509 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
4510 CallStackId &CallStackCount) {
4511 NameVals.push_back(Elt: ValueID);
4512
4513 FunctionSummary *FS = cast<FunctionSummary>(Val: Summary);
4514
4515 writeFunctionTypeMetadataRecords(
4516 Stream, FS, GetValueID: [&](const ValueInfo &VI) -> std::optional<unsigned> {
4517 return {VE.getValueID(V: VI.getValue())};
4518 });
4519
4520 writeFunctionHeapProfileRecords(
4521 Stream, FS, CallsiteAbbrev, AllocAbbrev, ContextIdAbbvId,
4522 /*PerModule*/ true,
4523 /*GetValueId*/ GetValueID: [&](const ValueInfo &VI) { return getValueId(VI); },
4524 /*GetStackIndex*/ [&](unsigned I) { return I; },
4525 /*WriteContextSizeInfoIndex*/ true, CallStackPos, CallStackCount);
4526
4527 auto SpecialRefCnts = FS->specialRefCounts();
4528 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: FS->flags()));
4529 NameVals.push_back(Elt: FS->instCount());
4530 NameVals.push_back(Elt: getEncodedFFlags(Flags: FS->fflags()));
4531 NameVals.push_back(Elt: FS->refs().size());
4532 NameVals.push_back(Elt: SpecialRefCnts.first); // rorefcnt
4533 NameVals.push_back(Elt: SpecialRefCnts.second); // worefcnt
4534
4535 for (auto &RI : FS->refs())
4536 NameVals.push_back(Elt: getValueId(VI: RI));
4537
4538 const bool UseRelBFRecord =
4539 WriteRelBFToSummary && !F.hasProfileData() &&
4540 ForceSummaryEdgesCold == FunctionSummary::FSHT_None;
4541 for (auto &ECI : FS->calls()) {
4542 NameVals.push_back(Elt: getValueId(VI: ECI.first));
4543 if (UseRelBFRecord)
4544 NameVals.push_back(Elt: getEncodedRelBFCallEdgeInfo(CI: ECI.second));
4545 else
4546 NameVals.push_back(Elt: getEncodedHotnessCallEdgeInfo(CI: ECI.second));
4547 }
4548
4549 unsigned FSAbbrev =
4550 (UseRelBFRecord ? FSCallsRelBFAbbrev : FSCallsProfileAbbrev);
4551 unsigned Code =
4552 (UseRelBFRecord ? bitc::FS_PERMODULE_RELBF : bitc::FS_PERMODULE_PROFILE);
4553
4554 // Emit the finished record.
4555 Stream.EmitRecord(Code, Vals: NameVals, Abbrev: FSAbbrev);
4556 NameVals.clear();
4557}
4558
4559// Collect the global value references in the given variable's initializer,
4560// and emit them in a summary record.
4561void ModuleBitcodeWriterBase::writeModuleLevelReferences(
4562 const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals,
4563 unsigned FSModRefsAbbrev, unsigned FSModVTableRefsAbbrev) {
4564 auto VI = Index->getValueInfo(GUID: V.getGUID());
4565 if (!VI || VI.getSummaryList().empty()) {
4566 // Only declarations should not have a summary (a declaration might however
4567 // have a summary if the def was in module level asm).
4568 assert(V.isDeclaration());
4569 return;
4570 }
4571 auto *Summary = VI.getSummaryList()[0].get();
4572 NameVals.push_back(Elt: VE.getValueID(V: &V));
4573 GlobalVarSummary *VS = cast<GlobalVarSummary>(Val: Summary);
4574 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: VS->flags()));
4575 NameVals.push_back(Elt: getEncodedGVarFlags(Flags: VS->varflags()));
4576
4577 auto VTableFuncs = VS->vTableFuncs();
4578 if (!VTableFuncs.empty())
4579 NameVals.push_back(Elt: VS->refs().size());
4580
4581 unsigned SizeBeforeRefs = NameVals.size();
4582 for (auto &RI : VS->refs())
4583 NameVals.push_back(Elt: VE.getValueID(V: RI.getValue()));
4584 // Sort the refs for determinism output, the vector returned by FS->refs() has
4585 // been initialized from a DenseSet.
4586 llvm::sort(C: drop_begin(RangeOrContainer&: NameVals, N: SizeBeforeRefs));
4587
4588 if (VTableFuncs.empty())
4589 Stream.EmitRecord(Code: bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, Vals: NameVals,
4590 Abbrev: FSModRefsAbbrev);
4591 else {
4592 // VTableFuncs pairs should already be sorted by offset.
4593 for (auto &P : VTableFuncs) {
4594 NameVals.push_back(Elt: VE.getValueID(V: P.FuncVI.getValue()));
4595 NameVals.push_back(Elt: P.VTableOffset);
4596 }
4597
4598 Stream.EmitRecord(Code: bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS, Vals: NameVals,
4599 Abbrev: FSModVTableRefsAbbrev);
4600 }
4601 NameVals.clear();
4602}
4603
4604/// Emit the per-module summary section alongside the rest of
4605/// the module's bitcode.
4606void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
4607 // By default we compile with ThinLTO if the module has a summary, but the
4608 // client can request full LTO with a module flag.
4609 bool IsThinLTO = true;
4610 if (auto *MD =
4611 mdconst::extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: "ThinLTO")))
4612 IsThinLTO = MD->getZExtValue();
4613 Stream.EnterSubblock(BlockID: IsThinLTO ? bitc::GLOBALVAL_SUMMARY_BLOCK_ID
4614 : bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID,
4615 CodeLen: 4);
4616
4617 Stream.EmitRecord(
4618 Code: bitc::FS_VERSION,
4619 Vals: ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
4620
4621 // Write the index flags.
4622 uint64_t Flags = 0;
4623 // Bits 1-3 are set only in the combined index, skip them.
4624 if (Index->enableSplitLTOUnit())
4625 Flags |= 0x8;
4626 if (Index->hasUnifiedLTO())
4627 Flags |= 0x200;
4628
4629 Stream.EmitRecord(Code: bitc::FS_FLAGS, Vals: ArrayRef<uint64_t>{Flags});
4630
4631 if (Index->begin() == Index->end()) {
4632 Stream.ExitBlock();
4633 return;
4634 }
4635
4636 auto Abbv = std::make_shared<BitCodeAbbrev>();
4637 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_VALUE_GUID));
4638 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4639 // GUIDS often use up most of 64-bits, so encode as two Fixed 32.
4640 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4641 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4642 unsigned ValueGuidAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4643
4644 for (const auto &GVI : valueIds()) {
4645 Stream.EmitRecord(Code: bitc::FS_VALUE_GUID,
4646 Vals: ArrayRef<uint32_t>{GVI.second,
4647 static_cast<uint32_t>(GVI.first >> 32),
4648 static_cast<uint32_t>(GVI.first)},
4649 Abbrev: ValueGuidAbbrev);
4650 }
4651
4652 if (!Index->stackIds().empty()) {
4653 auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4654 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_STACK_IDS));
4655 // numids x stackid
4656 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4657 // The stack ids are hashes that are close to 64 bits in size, so emitting
4658 // as a pair of 32-bit fixed-width values is more efficient than a VBR.
4659 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4660 unsigned StackIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(StackIdAbbv));
4661 SmallVector<uint32_t> Vals;
4662 Vals.reserve(N: Index->stackIds().size() * 2);
4663 for (auto Id : Index->stackIds()) {
4664 Vals.push_back(Elt: static_cast<uint32_t>(Id >> 32));
4665 Vals.push_back(Elt: static_cast<uint32_t>(Id));
4666 }
4667 Stream.EmitRecord(Code: bitc::FS_STACK_IDS, Vals, Abbrev: StackIdAbbvId);
4668 }
4669
4670 unsigned ContextIdAbbvId = 0;
4671 if (metadataMayIncludeContextSizeInfo()) {
4672 // n x context id
4673 auto ContextIdAbbv = std::make_shared<BitCodeAbbrev>();
4674 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_ALLOC_CONTEXT_IDS));
4675 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4676 // The context ids are hashes that are close to 64 bits in size, so emitting
4677 // as a pair of 32-bit fixed-width values is more efficient than a VBR if we
4678 // are emitting them for all MIBs. Otherwise we use VBR to better compress 0
4679 // values that are expected to more frequently occur in an alloc's memprof
4680 // summary.
4681 if (metadataIncludesAllContextSizeInfo())
4682 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4683 else
4684 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4685 ContextIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(ContextIdAbbv));
4686 }
4687
4688 // Abbrev for FS_PERMODULE_PROFILE.
4689 Abbv = std::make_shared<BitCodeAbbrev>();
4690 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE));
4691 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4692 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // flags
4693 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
4694 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
4695 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4696 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
4697 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
4698 // numrefs x valueid, n x (valueid, hotness+tailcall flags)
4699 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4700 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4701 unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4702
4703 // Abbrev for FS_PERMODULE_RELBF.
4704 Abbv = std::make_shared<BitCodeAbbrev>();
4705 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_RELBF));
4706 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4707 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4708 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
4709 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
4710 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4711 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
4712 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
4713 // numrefs x valueid, n x (valueid, rel_block_freq+tailcall])
4714 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4715 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4716 unsigned FSCallsRelBFAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4717
4718 // Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS.
4719 Abbv = std::make_shared<BitCodeAbbrev>();
4720 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS));
4721 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4722 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4723 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
4724 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4725 unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4726
4727 // Abbrev for FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS.
4728 Abbv = std::make_shared<BitCodeAbbrev>();
4729 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS));
4730 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4731 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4732 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4733 // numrefs x valueid, n x (valueid , offset)
4734 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4735 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4736 unsigned FSModVTableRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4737
4738 // Abbrev for FS_ALIAS.
4739 Abbv = std::make_shared<BitCodeAbbrev>();
4740 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_ALIAS));
4741 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4742 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4743 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4744 unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4745
4746 // Abbrev for FS_TYPE_ID_METADATA
4747 Abbv = std::make_shared<BitCodeAbbrev>();
4748 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_TYPE_ID_METADATA));
4749 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid strtab index
4750 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid length
4751 // n x (valueid , offset)
4752 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4753 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4754 unsigned TypeIdCompatibleVtableAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4755
4756 Abbv = std::make_shared<BitCodeAbbrev>();
4757 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_CALLSITE_INFO));
4758 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4759 // n x stackidindex
4760 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4761 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4762 unsigned CallsiteAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4763
4764 Abbv = std::make_shared<BitCodeAbbrev>();
4765 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_ALLOC_INFO));
4766 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib
4767 // n x (alloc type, context radix tree index)
4768 // optional: nummib x (numcontext x total size)
4769 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4770 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4771 unsigned AllocAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4772
4773 Abbv = std::make_shared<BitCodeAbbrev>();
4774 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_CONTEXT_RADIX_TREE_ARRAY));
4775 // n x entry
4776 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4777 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4778 unsigned RadixAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4779
4780 // First walk through all the functions and collect the allocation contexts in
4781 // their associated summaries, for use in constructing a radix tree of
4782 // contexts. Note that we need to do this in the same order as the functions
4783 // are processed further below since the call stack positions in the resulting
4784 // radix tree array are identified based on this order.
4785 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> CallStacks;
4786 for (const Function &F : M) {
4787 // Summary emission does not support anonymous functions, they have to be
4788 // renamed using the anonymous function renaming pass.
4789 if (!F.hasName())
4790 report_fatal_error(reason: "Unexpected anonymous function when writing summary");
4791
4792 ValueInfo VI = Index->getValueInfo(GUID: F.getGUID());
4793 if (!VI || VI.getSummaryList().empty()) {
4794 // Only declarations should not have a summary (a declaration might
4795 // however have a summary if the def was in module level asm).
4796 assert(F.isDeclaration());
4797 continue;
4798 }
4799 auto *Summary = VI.getSummaryList()[0].get();
4800 FunctionSummary *FS = cast<FunctionSummary>(Val: Summary);
4801 collectMemProfCallStacks(
4802 FS, /*GetStackIndex*/ [](unsigned I) { return I; }, CallStacks);
4803 }
4804 // Finalize the radix tree, write it out, and get the map of positions in the
4805 // linearized tree array.
4806 DenseMap<CallStackId, LinearCallStackId> CallStackPos;
4807 if (!CallStacks.empty()) {
4808 CallStackPos =
4809 writeMemoryProfileRadixTree(CallStacks: std::move(CallStacks), Stream, RadixAbbrev);
4810 }
4811
4812 // Keep track of the current index into the CallStackPos map.
4813 CallStackId CallStackCount = 0;
4814
4815 SmallVector<uint64_t, 64> NameVals;
4816 // Iterate over the list of functions instead of the Index to
4817 // ensure the ordering is stable.
4818 for (const Function &F : M) {
4819 // Summary emission does not support anonymous functions, they have to
4820 // renamed using the anonymous function renaming pass.
4821 if (!F.hasName())
4822 report_fatal_error(reason: "Unexpected anonymous function when writing summary");
4823
4824 ValueInfo VI = Index->getValueInfo(GUID: F.getGUID());
4825 if (!VI || VI.getSummaryList().empty()) {
4826 // Only declarations should not have a summary (a declaration might
4827 // however have a summary if the def was in module level asm).
4828 assert(F.isDeclaration());
4829 continue;
4830 }
4831 auto *Summary = VI.getSummaryList()[0].get();
4832 writePerModuleFunctionSummaryRecord(
4833 NameVals, Summary, ValueID: VE.getValueID(V: &F), FSCallsRelBFAbbrev,
4834 FSCallsProfileAbbrev, CallsiteAbbrev, AllocAbbrev, ContextIdAbbvId, F,
4835 CallStackPos, CallStackCount);
4836 }
4837
4838 // Capture references from GlobalVariable initializers, which are outside
4839 // of a function scope.
4840 for (const GlobalVariable &G : M.globals())
4841 writeModuleLevelReferences(V: G, NameVals, FSModRefsAbbrev,
4842 FSModVTableRefsAbbrev);
4843
4844 for (const GlobalAlias &A : M.aliases()) {
4845 auto *Aliasee = A.getAliaseeObject();
4846 // Skip ifunc and nameless functions which don't have an entry in the
4847 // summary.
4848 if (!Aliasee->hasName() || isa<GlobalIFunc>(Val: Aliasee))
4849 continue;
4850 auto AliasId = VE.getValueID(V: &A);
4851 auto AliaseeId = VE.getValueID(V: Aliasee);
4852 NameVals.push_back(Elt: AliasId);
4853 auto *Summary = Index->getGlobalValueSummary(GV: A);
4854 AliasSummary *AS = cast<AliasSummary>(Val: Summary);
4855 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: AS->flags()));
4856 NameVals.push_back(Elt: AliaseeId);
4857 Stream.EmitRecord(Code: bitc::FS_ALIAS, Vals: NameVals, Abbrev: FSAliasAbbrev);
4858 NameVals.clear();
4859 }
4860
4861 for (auto &S : Index->typeIdCompatibleVtableMap()) {
4862 writeTypeIdCompatibleVtableSummaryRecord(NameVals, StrtabBuilder, Id: S.first,
4863 Summary: S.second, VE);
4864 Stream.EmitRecord(Code: bitc::FS_TYPE_ID_METADATA, Vals: NameVals,
4865 Abbrev: TypeIdCompatibleVtableAbbrev);
4866 NameVals.clear();
4867 }
4868
4869 if (Index->getBlockCount())
4870 Stream.EmitRecord(Code: bitc::FS_BLOCK_COUNT,
4871 Vals: ArrayRef<uint64_t>{Index->getBlockCount()});
4872
4873 Stream.ExitBlock();
4874}
4875
4876/// Emit the combined summary section into the combined index file.
4877void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
4878 Stream.EnterSubblock(BlockID: bitc::GLOBALVAL_SUMMARY_BLOCK_ID, CodeLen: 4);
4879 Stream.EmitRecord(
4880 Code: bitc::FS_VERSION,
4881 Vals: ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
4882
4883 // Write the index flags.
4884 Stream.EmitRecord(Code: bitc::FS_FLAGS, Vals: ArrayRef<uint64_t>{Index.getFlags()});
4885
4886 auto Abbv = std::make_shared<BitCodeAbbrev>();
4887 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_VALUE_GUID));
4888 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4889 // GUIDS often use up most of 64-bits, so encode as two Fixed 32.
4890 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4891 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4892 unsigned ValueGuidAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4893
4894 for (const auto &GVI : valueIds()) {
4895 Stream.EmitRecord(Code: bitc::FS_VALUE_GUID,
4896 Vals: ArrayRef<uint32_t>{GVI.second,
4897 static_cast<uint32_t>(GVI.first >> 32),
4898 static_cast<uint32_t>(GVI.first)},
4899 Abbrev: ValueGuidAbbrev);
4900 }
4901
4902 // Write the stack ids used by this index, which will be a subset of those in
4903 // the full index in the case of distributed indexes.
4904 if (!StackIds.empty()) {
4905 auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4906 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_STACK_IDS));
4907 // numids x stackid
4908 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4909 // The stack ids are hashes that are close to 64 bits in size, so emitting
4910 // as a pair of 32-bit fixed-width values is more efficient than a VBR.
4911 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4912 unsigned StackIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(StackIdAbbv));
4913 SmallVector<uint32_t> Vals;
4914 Vals.reserve(N: StackIds.size() * 2);
4915 for (auto Id : StackIds) {
4916 Vals.push_back(Elt: static_cast<uint32_t>(Id >> 32));
4917 Vals.push_back(Elt: static_cast<uint32_t>(Id));
4918 }
4919 Stream.EmitRecord(Code: bitc::FS_STACK_IDS, Vals, Abbrev: StackIdAbbvId);
4920 }
4921
4922 // Abbrev for FS_COMBINED_PROFILE.
4923 Abbv = std::make_shared<BitCodeAbbrev>();
4924 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_PROFILE));
4925 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4926 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
4927 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4928 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
4929 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
4930 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // entrycount
4931 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4932 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
4933 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
4934 // numrefs x valueid, n x (valueid, hotness+tailcall flags)
4935 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4936 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4937 unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4938
4939 // Abbrev for FS_COMBINED_GLOBALVAR_INIT_REFS.
4940 Abbv = std::make_shared<BitCodeAbbrev>();
4941 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS));
4942 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4943 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
4944 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4945 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
4946 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4947 unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4948
4949 // Abbrev for FS_COMBINED_ALIAS.
4950 Abbv = std::make_shared<BitCodeAbbrev>();
4951 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_ALIAS));
4952 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4953 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
4954 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4955 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4956 unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4957
4958 Abbv = std::make_shared<BitCodeAbbrev>();
4959 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_CALLSITE_INFO));
4960 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4961 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numstackindices
4962 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
4963 // numstackindices x stackidindex, numver x version
4964 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4965 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4966 unsigned CallsiteAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4967
4968 Abbv = std::make_shared<BitCodeAbbrev>();
4969 Abbv->Add(OpInfo: BitCodeAbbrevOp(CombinedIndexMemProfContext
4970 ? bitc::FS_COMBINED_ALLOC_INFO
4971 : bitc::FS_COMBINED_ALLOC_INFO_NO_CONTEXT));
4972 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib
4973 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
4974 // nummib x (alloc type, context radix tree index),
4975 // numver x version
4976 // optional: nummib x total size
4977 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4978 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4979 unsigned AllocAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4980
4981 auto shouldImportValueAsDecl = [&](GlobalValueSummary *GVS) -> bool {
4982 if (DecSummaries == nullptr)
4983 return false;
4984 return DecSummaries->count(x: GVS);
4985 };
4986
4987 // The aliases are emitted as a post-pass, and will point to the value
4988 // id of the aliasee. Save them in a vector for post-processing.
4989 SmallVector<AliasSummary *, 64> Aliases;
4990
4991 // Save the value id for each summary for alias emission.
4992 DenseMap<const GlobalValueSummary *, unsigned> SummaryToValueIdMap;
4993
4994 SmallVector<uint64_t, 64> NameVals;
4995
4996 // Set that will be populated during call to writeFunctionTypeMetadataRecords
4997 // with the type ids referenced by this index file.
4998 std::set<GlobalValue::GUID> ReferencedTypeIds;
4999
5000 // For local linkage, we also emit the original name separately
5001 // immediately after the record.
5002 auto MaybeEmitOriginalName = [&](GlobalValueSummary &S) {
5003 // We don't need to emit the original name if we are writing the index for
5004 // distributed backends (in which case ModuleToSummariesForIndex is
5005 // non-null). The original name is only needed during the thin link, since
5006 // for SamplePGO the indirect call targets for local functions have
5007 // have the original name annotated in profile.
5008 // Continue to emit it when writing out the entire combined index, which is
5009 // used in testing the thin link via llvm-lto.
5010 if (ModuleToSummariesForIndex || !GlobalValue::isLocalLinkage(Linkage: S.linkage()))
5011 return;
5012 NameVals.push_back(Elt: S.getOriginalName());
5013 Stream.EmitRecord(Code: bitc::FS_COMBINED_ORIGINAL_NAME, Vals: NameVals);
5014 NameVals.clear();
5015 };
5016
5017 DenseMap<CallStackId, LinearCallStackId> CallStackPos;
5018 if (CombinedIndexMemProfContext) {
5019 Abbv = std::make_shared<BitCodeAbbrev>();
5020 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_CONTEXT_RADIX_TREE_ARRAY));
5021 // n x entry
5022 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5023 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5024 unsigned RadixAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5025
5026 // First walk through all the functions and collect the allocation contexts
5027 // in their associated summaries, for use in constructing a radix tree of
5028 // contexts. Note that we need to do this in the same order as the functions
5029 // are processed further below since the call stack positions in the
5030 // resulting radix tree array are identified based on this order.
5031 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> CallStacks;
5032 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
5033 // Don't collect this when invoked for an aliasee, as it is not needed for
5034 // the alias summary. If the aliasee is to be imported, we will invoke
5035 // this separately with IsAliasee=false.
5036 if (IsAliasee)
5037 return;
5038 GlobalValueSummary *S = I.second;
5039 assert(S);
5040 auto *FS = dyn_cast<FunctionSummary>(Val: S);
5041 if (!FS)
5042 return;
5043 collectMemProfCallStacks(
5044 FS,
5045 /*GetStackIndex*/
5046 [&](unsigned I) {
5047 // Get the corresponding index into the list of StackIds actually
5048 // being written for this combined index (which may be a subset in
5049 // the case of distributed indexes).
5050 assert(StackIdIndicesToIndex.contains(I));
5051 return StackIdIndicesToIndex[I];
5052 },
5053 CallStacks);
5054 });
5055 // Finalize the radix tree, write it out, and get the map of positions in
5056 // the linearized tree array.
5057 if (!CallStacks.empty()) {
5058 CallStackPos = writeMemoryProfileRadixTree(CallStacks: std::move(CallStacks), Stream,
5059 RadixAbbrev);
5060 }
5061 }
5062
5063 // Keep track of the current index into the CallStackPos map. Not used if
5064 // CombinedIndexMemProfContext is false.
5065 CallStackId CallStackCount = 0;
5066
5067 DenseSet<GlobalValue::GUID> DefOrUseGUIDs;
5068 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
5069 GlobalValueSummary *S = I.second;
5070 assert(S);
5071 DefOrUseGUIDs.insert(V: I.first);
5072 for (const ValueInfo &VI : S->refs())
5073 DefOrUseGUIDs.insert(V: VI.getGUID());
5074
5075 auto ValueId = getValueId(ValGUID: I.first);
5076 assert(ValueId);
5077 SummaryToValueIdMap[S] = *ValueId;
5078
5079 // If this is invoked for an aliasee, we want to record the above
5080 // mapping, but then not emit a summary entry (if the aliasee is
5081 // to be imported, we will invoke this separately with IsAliasee=false).
5082 if (IsAliasee)
5083 return;
5084
5085 if (auto *AS = dyn_cast<AliasSummary>(Val: S)) {
5086 // Will process aliases as a post-pass because the reader wants all
5087 // global to be loaded first.
5088 Aliases.push_back(Elt: AS);
5089 return;
5090 }
5091
5092 if (auto *VS = dyn_cast<GlobalVarSummary>(Val: S)) {
5093 NameVals.push_back(Elt: *ValueId);
5094 assert(ModuleIdMap.count(VS->modulePath()));
5095 NameVals.push_back(Elt: ModuleIdMap[VS->modulePath()]);
5096 NameVals.push_back(
5097 Elt: getEncodedGVSummaryFlags(Flags: VS->flags(), ImportAsDecl: shouldImportValueAsDecl(VS)));
5098 NameVals.push_back(Elt: getEncodedGVarFlags(Flags: VS->varflags()));
5099 for (auto &RI : VS->refs()) {
5100 auto RefValueId = getValueId(ValGUID: RI.getGUID());
5101 if (!RefValueId)
5102 continue;
5103 NameVals.push_back(Elt: *RefValueId);
5104 }
5105
5106 // Emit the finished record.
5107 Stream.EmitRecord(Code: bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, Vals: NameVals,
5108 Abbrev: FSModRefsAbbrev);
5109 NameVals.clear();
5110 MaybeEmitOriginalName(*S);
5111 return;
5112 }
5113
5114 auto GetValueId = [&](const ValueInfo &VI) -> std::optional<unsigned> {
5115 if (!VI)
5116 return std::nullopt;
5117 return getValueId(ValGUID: VI.getGUID());
5118 };
5119
5120 auto *FS = cast<FunctionSummary>(Val: S);
5121 writeFunctionTypeMetadataRecords(Stream, FS, GetValueID: GetValueId);
5122 getReferencedTypeIds(FS, ReferencedTypeIds);
5123
5124 writeFunctionHeapProfileRecords(
5125 Stream, FS, CallsiteAbbrev, AllocAbbrev, /*ContextIdAbbvId*/ 0,
5126 /*PerModule*/ false,
5127 /*GetValueId*/
5128 GetValueID: [&](const ValueInfo &VI) -> unsigned {
5129 std::optional<unsigned> ValueID = GetValueId(VI);
5130 // This can happen in shared index files for distributed ThinLTO if
5131 // the callee function summary is not included. Record 0 which we
5132 // will have to deal with conservatively when doing any kind of
5133 // validation in the ThinLTO backends.
5134 if (!ValueID)
5135 return 0;
5136 return *ValueID;
5137 },
5138 /*GetStackIndex*/
5139 [&](unsigned I) {
5140 // Get the corresponding index into the list of StackIds actually
5141 // being written for this combined index (which may be a subset in
5142 // the case of distributed indexes).
5143 assert(StackIdIndicesToIndex.contains(I));
5144 return StackIdIndicesToIndex[I];
5145 },
5146 /*WriteContextSizeInfoIndex*/ false, CallStackPos, CallStackCount);
5147
5148 NameVals.push_back(Elt: *ValueId);
5149 assert(ModuleIdMap.count(FS->modulePath()));
5150 NameVals.push_back(Elt: ModuleIdMap[FS->modulePath()]);
5151 NameVals.push_back(
5152 Elt: getEncodedGVSummaryFlags(Flags: FS->flags(), ImportAsDecl: shouldImportValueAsDecl(FS)));
5153 NameVals.push_back(Elt: FS->instCount());
5154 NameVals.push_back(Elt: getEncodedFFlags(Flags: FS->fflags()));
5155 // TODO: Stop writing entry count and bump bitcode version.
5156 NameVals.push_back(Elt: 0 /* EntryCount */);
5157
5158 // Fill in below
5159 NameVals.push_back(Elt: 0); // numrefs
5160 NameVals.push_back(Elt: 0); // rorefcnt
5161 NameVals.push_back(Elt: 0); // worefcnt
5162
5163 unsigned Count = 0, RORefCnt = 0, WORefCnt = 0;
5164 for (auto &RI : FS->refs()) {
5165 auto RefValueId = getValueId(ValGUID: RI.getGUID());
5166 if (!RefValueId)
5167 continue;
5168 NameVals.push_back(Elt: *RefValueId);
5169 if (RI.isReadOnly())
5170 RORefCnt++;
5171 else if (RI.isWriteOnly())
5172 WORefCnt++;
5173 Count++;
5174 }
5175 NameVals[6] = Count;
5176 NameVals[7] = RORefCnt;
5177 NameVals[8] = WORefCnt;
5178
5179 for (auto &EI : FS->calls()) {
5180 // If this GUID doesn't have a value id, it doesn't have a function
5181 // summary and we don't need to record any calls to it.
5182 std::optional<unsigned> CallValueId = GetValueId(EI.first);
5183 if (!CallValueId)
5184 continue;
5185 NameVals.push_back(Elt: *CallValueId);
5186 NameVals.push_back(Elt: getEncodedHotnessCallEdgeInfo(CI: EI.second));
5187 }
5188
5189 // Emit the finished record.
5190 Stream.EmitRecord(Code: bitc::FS_COMBINED_PROFILE, Vals: NameVals,
5191 Abbrev: FSCallsProfileAbbrev);
5192 NameVals.clear();
5193 MaybeEmitOriginalName(*S);
5194 });
5195
5196 for (auto *AS : Aliases) {
5197 auto AliasValueId = SummaryToValueIdMap[AS];
5198 assert(AliasValueId);
5199 NameVals.push_back(Elt: AliasValueId);
5200 assert(ModuleIdMap.count(AS->modulePath()));
5201 NameVals.push_back(Elt: ModuleIdMap[AS->modulePath()]);
5202 NameVals.push_back(
5203 Elt: getEncodedGVSummaryFlags(Flags: AS->flags(), ImportAsDecl: shouldImportValueAsDecl(AS)));
5204 auto AliaseeValueId = SummaryToValueIdMap[&AS->getAliasee()];
5205 assert(AliaseeValueId);
5206 NameVals.push_back(Elt: AliaseeValueId);
5207
5208 // Emit the finished record.
5209 Stream.EmitRecord(Code: bitc::FS_COMBINED_ALIAS, Vals: NameVals, Abbrev: FSAliasAbbrev);
5210 NameVals.clear();
5211 MaybeEmitOriginalName(*AS);
5212
5213 if (auto *FS = dyn_cast<FunctionSummary>(Val: &AS->getAliasee()))
5214 getReferencedTypeIds(FS, ReferencedTypeIds);
5215 }
5216
5217 SmallVector<StringRef, 4> Functions;
5218 auto EmitCfiFunctions = [&](const CfiFunctionIndex &CfiIndex,
5219 bitc::GlobalValueSummarySymtabCodes Code) {
5220 if (CfiIndex.empty())
5221 return;
5222 for (GlobalValue::GUID GUID : DefOrUseGUIDs) {
5223 auto Defs = CfiIndex.forGuid(GUID);
5224 llvm::append_range(C&: Functions, R&: Defs);
5225 }
5226 if (Functions.empty())
5227 return;
5228 llvm::sort(C&: Functions);
5229 for (const auto &S : Functions) {
5230 NameVals.push_back(Elt: StrtabBuilder.add(S));
5231 NameVals.push_back(Elt: S.size());
5232 }
5233 Stream.EmitRecord(Code, Vals: NameVals);
5234 NameVals.clear();
5235 Functions.clear();
5236 };
5237
5238 EmitCfiFunctions(Index.cfiFunctionDefs(), bitc::FS_CFI_FUNCTION_DEFS);
5239 EmitCfiFunctions(Index.cfiFunctionDecls(), bitc::FS_CFI_FUNCTION_DECLS);
5240
5241 // Walk the GUIDs that were referenced, and write the
5242 // corresponding type id records.
5243 for (auto &T : ReferencedTypeIds) {
5244 auto TidIter = Index.typeIds().equal_range(x: T);
5245 for (const auto &[GUID, TypeIdPair] : make_range(p: TidIter)) {
5246 writeTypeIdSummaryRecord(NameVals, StrtabBuilder, Id: TypeIdPair.first,
5247 Summary: TypeIdPair.second);
5248 Stream.EmitRecord(Code: bitc::FS_TYPE_ID, Vals: NameVals);
5249 NameVals.clear();
5250 }
5251 }
5252
5253 if (Index.getBlockCount())
5254 Stream.EmitRecord(Code: bitc::FS_BLOCK_COUNT,
5255 Vals: ArrayRef<uint64_t>{Index.getBlockCount()});
5256
5257 Stream.ExitBlock();
5258}
5259
5260/// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the
5261/// current llvm version, and a record for the epoch number.
5262static void writeIdentificationBlock(BitstreamWriter &Stream) {
5263 Stream.EnterSubblock(BlockID: bitc::IDENTIFICATION_BLOCK_ID, CodeLen: 5);
5264
5265 // Write the "user readable" string identifying the bitcode producer
5266 auto Abbv = std::make_shared<BitCodeAbbrev>();
5267 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_STRING));
5268 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5269 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
5270 auto StringAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5271 writeStringRecord(Stream, Code: bitc::IDENTIFICATION_CODE_STRING,
5272 Str: "LLVM" LLVM_VERSION_STRING, AbbrevToUse: StringAbbrev);
5273
5274 // Write the epoch version
5275 Abbv = std::make_shared<BitCodeAbbrev>();
5276 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_EPOCH));
5277 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
5278 auto EpochAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5279 constexpr std::array<unsigned, 1> Vals = {._M_elems: {bitc::BITCODE_CURRENT_EPOCH}};
5280 Stream.EmitRecord(Code: bitc::IDENTIFICATION_CODE_EPOCH, Vals, Abbrev: EpochAbbrev);
5281 Stream.ExitBlock();
5282}
5283
5284void ModuleBitcodeWriter::writeModuleHash(StringRef View) {
5285 // Emit the module's hash.
5286 // MODULE_CODE_HASH: [5*i32]
5287 if (GenerateHash) {
5288 uint32_t Vals[5];
5289 Hasher.update(Data: ArrayRef<uint8_t>(
5290 reinterpret_cast<const uint8_t *>(View.data()), View.size()));
5291 std::array<uint8_t, 20> Hash = Hasher.result();
5292 for (int Pos = 0; Pos < 20; Pos += 4) {
5293 Vals[Pos / 4] = support::endian::read32be(P: Hash.data() + Pos);
5294 }
5295
5296 // Emit the finished record.
5297 Stream.EmitRecord(Code: bitc::MODULE_CODE_HASH, Vals);
5298
5299 if (ModHash)
5300 // Save the written hash value.
5301 llvm::copy(Range&: Vals, Out: std::begin(cont&: *ModHash));
5302 }
5303}
5304
5305void ModuleBitcodeWriter::write() {
5306 writeIdentificationBlock(Stream);
5307
5308 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5309 // We will want to write the module hash at this point. Block any flushing so
5310 // we can have access to the whole underlying data later.
5311 Stream.markAndBlockFlushing();
5312
5313 writeModuleVersion();
5314
5315 // Emit blockinfo, which defines the standard abbreviations etc.
5316 writeBlockInfo();
5317
5318 // Emit information describing all of the types in the module.
5319 writeTypeTable();
5320
5321 // Emit information about attribute groups.
5322 writeAttributeGroupTable();
5323
5324 // Emit information about parameter attributes.
5325 writeAttributeTable();
5326
5327 writeComdats();
5328
5329 // Emit top-level description of module, including target triple, inline asm,
5330 // descriptors for global variables, and function prototype info.
5331 writeModuleInfo();
5332
5333 // Emit constants.
5334 writeModuleConstants();
5335
5336 // Emit metadata kind names.
5337 writeModuleMetadataKinds();
5338
5339 // Emit metadata.
5340 writeModuleMetadata();
5341
5342 // Emit module-level use-lists.
5343 if (VE.shouldPreserveUseListOrder())
5344 writeUseListBlock(F: nullptr);
5345
5346 writeOperandBundleTags();
5347 writeSyncScopeNames();
5348
5349 // Emit function bodies.
5350 DenseMap<const Function *, uint64_t> FunctionToBitcodeIndex;
5351 for (const Function &F : M)
5352 if (!F.isDeclaration())
5353 writeFunction(F, FunctionToBitcodeIndex);
5354
5355 // Need to write after the above call to WriteFunction which populates
5356 // the summary information in the index.
5357 if (Index)
5358 writePerModuleGlobalValueSummary();
5359
5360 writeGlobalValueSymbolTable(FunctionToBitcodeIndex);
5361
5362 writeModuleHash(View: Stream.getMarkedBufferAndResumeFlushing());
5363
5364 Stream.ExitBlock();
5365}
5366
5367static void writeInt32ToBuffer(uint32_t Value, SmallVectorImpl<char> &Buffer,
5368 uint32_t &Position) {
5369 support::endian::write32le(P: &Buffer[Position], V: Value);
5370 Position += 4;
5371}
5372
5373/// If generating a bc file on darwin, we have to emit a
5374/// header and trailer to make it compatible with the system archiver. To do
5375/// this we emit the following header, and then emit a trailer that pads the
5376/// file out to be a multiple of 16 bytes.
5377///
5378/// struct bc_header {
5379/// uint32_t Magic; // 0x0B17C0DE
5380/// uint32_t Version; // Version, currently always 0.
5381/// uint32_t BitcodeOffset; // Offset to traditional bitcode file.
5382/// uint32_t BitcodeSize; // Size of traditional bitcode file.
5383/// uint32_t CPUType; // CPU specifier.
5384/// ... potentially more later ...
5385/// };
5386static void emitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer,
5387 const Triple &TT) {
5388 unsigned CPUType = ~0U;
5389
5390 // Match x86_64-*, i[3-9]86-*, powerpc-*, powerpc64-*, arm-*, thumb-*,
5391 // armv[0-9]-*, thumbv[0-9]-*, armv5te-*, or armv6t2-*. The CPUType is a magic
5392 // number from /usr/include/mach/machine.h. It is ok to reproduce the
5393 // specific constants here because they are implicitly part of the Darwin ABI.
5394 enum {
5395 DARWIN_CPU_ARCH_ABI64 = 0x01000000,
5396 DARWIN_CPU_TYPE_X86 = 7,
5397 DARWIN_CPU_TYPE_ARM = 12,
5398 DARWIN_CPU_TYPE_POWERPC = 18
5399 };
5400
5401 Triple::ArchType Arch = TT.getArch();
5402 if (Arch == Triple::x86_64)
5403 CPUType = DARWIN_CPU_TYPE_X86 | DARWIN_CPU_ARCH_ABI64;
5404 else if (Arch == Triple::x86)
5405 CPUType = DARWIN_CPU_TYPE_X86;
5406 else if (Arch == Triple::ppc)
5407 CPUType = DARWIN_CPU_TYPE_POWERPC;
5408 else if (Arch == Triple::ppc64)
5409 CPUType = DARWIN_CPU_TYPE_POWERPC | DARWIN_CPU_ARCH_ABI64;
5410 else if (Arch == Triple::arm || Arch == Triple::thumb)
5411 CPUType = DARWIN_CPU_TYPE_ARM;
5412
5413 // Traditional Bitcode starts after header.
5414 assert(Buffer.size() >= BWH_HeaderSize &&
5415 "Expected header size to be reserved");
5416 unsigned BCOffset = BWH_HeaderSize;
5417 unsigned BCSize = Buffer.size() - BWH_HeaderSize;
5418
5419 // Write the magic and version.
5420 unsigned Position = 0;
5421 writeInt32ToBuffer(Value: 0x0B17C0DE, Buffer, Position);
5422 writeInt32ToBuffer(Value: 0, Buffer, Position); // Version.
5423 writeInt32ToBuffer(Value: BCOffset, Buffer, Position);
5424 writeInt32ToBuffer(Value: BCSize, Buffer, Position);
5425 writeInt32ToBuffer(Value: CPUType, Buffer, Position);
5426
5427 // If the file is not a multiple of 16 bytes, insert dummy padding.
5428 while (Buffer.size() & 15)
5429 Buffer.push_back(Elt: 0);
5430}
5431
5432/// Helper to write the header common to all bitcode files.
5433static void writeBitcodeHeader(BitstreamWriter &Stream) {
5434 // Emit the file header.
5435 Stream.Emit(Val: (unsigned)'B', NumBits: 8);
5436 Stream.Emit(Val: (unsigned)'C', NumBits: 8);
5437 Stream.Emit(Val: 0x0, NumBits: 4);
5438 Stream.Emit(Val: 0xC, NumBits: 4);
5439 Stream.Emit(Val: 0xE, NumBits: 4);
5440 Stream.Emit(Val: 0xD, NumBits: 4);
5441}
5442
5443BitcodeWriter::BitcodeWriter(SmallVectorImpl<char> &Buffer)
5444 : Stream(new BitstreamWriter(Buffer)) {
5445 writeBitcodeHeader(Stream&: *Stream);
5446}
5447
5448BitcodeWriter::BitcodeWriter(raw_ostream &FS)
5449 : Stream(new BitstreamWriter(FS, FlushThreshold)) {
5450 writeBitcodeHeader(Stream&: *Stream);
5451}
5452
5453BitcodeWriter::~BitcodeWriter() { assert(WroteStrtab); }
5454
5455void BitcodeWriter::writeBlob(unsigned Block, unsigned Record, StringRef Blob) {
5456 Stream->EnterSubblock(BlockID: Block, CodeLen: 3);
5457
5458 auto Abbv = std::make_shared<BitCodeAbbrev>();
5459 Abbv->Add(OpInfo: BitCodeAbbrevOp(Record));
5460 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
5461 auto AbbrevNo = Stream->EmitAbbrev(Abbv: std::move(Abbv));
5462
5463 Stream->EmitRecordWithBlob(Abbrev: AbbrevNo, Vals: ArrayRef<uint64_t>{Record}, Blob);
5464
5465 Stream->ExitBlock();
5466}
5467
5468void BitcodeWriter::writeSymtab() {
5469 assert(!WroteStrtab && !WroteSymtab);
5470
5471 // If any module has module-level inline asm, we will require a registered asm
5472 // parser for the target so that we can create an accurate symbol table for
5473 // the module.
5474 for (Module *M : Mods) {
5475 if (M->getModuleInlineAsm().empty())
5476 continue;
5477
5478 std::string Err;
5479 const Triple TT(M->getTargetTriple());
5480 const Target *T = TargetRegistry::lookupTarget(TheTriple: TT, Error&: Err);
5481 if (!T || !T->hasMCAsmParser())
5482 return;
5483 }
5484
5485 WroteSymtab = true;
5486 SmallVector<char, 0> Symtab;
5487 // The irsymtab::build function may be unable to create a symbol table if the
5488 // module is malformed (e.g. it contains an invalid alias). Writing a symbol
5489 // table is not required for correctness, but we still want to be able to
5490 // write malformed modules to bitcode files, so swallow the error.
5491 if (Error E = irsymtab::build(Mods, Symtab, StrtabBuilder, Alloc)) {
5492 consumeError(Err: std::move(E));
5493 return;
5494 }
5495
5496 writeBlob(Block: bitc::SYMTAB_BLOCK_ID, Record: bitc::SYMTAB_BLOB,
5497 Blob: {Symtab.data(), Symtab.size()});
5498}
5499
5500void BitcodeWriter::writeStrtab() {
5501 assert(!WroteStrtab);
5502
5503 std::vector<char> Strtab;
5504 StrtabBuilder.finalizeInOrder();
5505 Strtab.resize(new_size: StrtabBuilder.getSize());
5506 StrtabBuilder.write(Buf: (uint8_t *)Strtab.data());
5507
5508 writeBlob(Block: bitc::STRTAB_BLOCK_ID, Record: bitc::STRTAB_BLOB,
5509 Blob: {Strtab.data(), Strtab.size()});
5510
5511 WroteStrtab = true;
5512}
5513
5514void BitcodeWriter::copyStrtab(StringRef Strtab) {
5515 writeBlob(Block: bitc::STRTAB_BLOCK_ID, Record: bitc::STRTAB_BLOB, Blob: Strtab);
5516 WroteStrtab = true;
5517}
5518
5519void BitcodeWriter::writeModule(const Module &M,
5520 bool ShouldPreserveUseListOrder,
5521 const ModuleSummaryIndex *Index,
5522 bool GenerateHash, ModuleHash *ModHash) {
5523 assert(!WroteStrtab);
5524
5525 // The Mods vector is used by irsymtab::build, which requires non-const
5526 // Modules in case it needs to materialize metadata. But the bitcode writer
5527 // requires that the module is materialized, so we can cast to non-const here,
5528 // after checking that it is in fact materialized.
5529 assert(M.isMaterialized());
5530 Mods.push_back(x: const_cast<Module *>(&M));
5531
5532 ModuleBitcodeWriter ModuleWriter(M, StrtabBuilder, *Stream,
5533 ShouldPreserveUseListOrder, Index,
5534 GenerateHash, ModHash);
5535 ModuleWriter.write();
5536}
5537
5538void BitcodeWriter::writeIndex(
5539 const ModuleSummaryIndex *Index,
5540 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex,
5541 const GVSummaryPtrSet *DecSummaries) {
5542 IndexBitcodeWriter IndexWriter(*Stream, StrtabBuilder, *Index, DecSummaries,
5543 ModuleToSummariesForIndex);
5544 IndexWriter.write();
5545}
5546
5547/// Write the specified module to the specified output stream.
5548void llvm::WriteBitcodeToFile(const Module &M, raw_ostream &Out,
5549 bool ShouldPreserveUseListOrder,
5550 const ModuleSummaryIndex *Index,
5551 bool GenerateHash, ModuleHash *ModHash) {
5552 auto Write = [&](BitcodeWriter &Writer) {
5553 Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash,
5554 ModHash);
5555 Writer.writeSymtab();
5556 Writer.writeStrtab();
5557 };
5558 Triple TT(M.getTargetTriple());
5559 if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) {
5560 // If this is darwin or another generic macho target, reserve space for the
5561 // header. Note that the header is computed *after* the output is known, so
5562 // we currently explicitly use a buffer, write to it, and then subsequently
5563 // flush to Out.
5564 SmallVector<char, 0> Buffer;
5565 Buffer.reserve(N: 256 * 1024);
5566 Buffer.insert(I: Buffer.begin(), NumToInsert: BWH_HeaderSize, Elt: 0);
5567 BitcodeWriter Writer(Buffer);
5568 Write(Writer);
5569 emitDarwinBCHeaderAndTrailer(Buffer, TT);
5570 Out.write(Ptr: Buffer.data(), Size: Buffer.size());
5571 } else {
5572 BitcodeWriter Writer(Out);
5573 Write(Writer);
5574 }
5575}
5576
5577void IndexBitcodeWriter::write() {
5578 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5579
5580 writeModuleVersion();
5581
5582 // Write the module paths in the combined index.
5583 writeModStrings();
5584
5585 // Write the summary combined index records.
5586 writeCombinedGlobalValueSummary();
5587
5588 Stream.ExitBlock();
5589}
5590
5591// Write the specified module summary index to the given raw output stream,
5592// where it will be written in a new bitcode block. This is used when
5593// writing the combined index file for ThinLTO. When writing a subset of the
5594// index for a distributed backend, provide a \p ModuleToSummariesForIndex map.
5595void llvm::writeIndexToFile(
5596 const ModuleSummaryIndex &Index, raw_ostream &Out,
5597 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex,
5598 const GVSummaryPtrSet *DecSummaries) {
5599 SmallVector<char, 0> Buffer;
5600 Buffer.reserve(N: 256 * 1024);
5601
5602 BitcodeWriter Writer(Buffer);
5603 Writer.writeIndex(Index: &Index, ModuleToSummariesForIndex, DecSummaries);
5604 Writer.writeStrtab();
5605
5606 Out.write(Ptr: (char *)&Buffer.front(), Size: Buffer.size());
5607}
5608
5609namespace {
5610
5611/// Class to manage the bitcode writing for a thin link bitcode file.
5612class ThinLinkBitcodeWriter : public ModuleBitcodeWriterBase {
5613 /// ModHash is for use in ThinLTO incremental build, generated while writing
5614 /// the module bitcode file.
5615 const ModuleHash *ModHash;
5616
5617public:
5618 ThinLinkBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
5619 BitstreamWriter &Stream,
5620 const ModuleSummaryIndex &Index,
5621 const ModuleHash &ModHash)
5622 : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
5623 /*ShouldPreserveUseListOrder=*/false, &Index),
5624 ModHash(&ModHash) {}
5625
5626 void write();
5627
5628private:
5629 void writeSimplifiedModuleInfo();
5630};
5631
5632} // end anonymous namespace
5633
5634// This function writes a simpilified module info for thin link bitcode file.
5635// It only contains the source file name along with the name(the offset and
5636// size in strtab) and linkage for global values. For the global value info
5637// entry, in order to keep linkage at offset 5, there are three zeros used
5638// as padding.
5639void ThinLinkBitcodeWriter::writeSimplifiedModuleInfo() {
5640 SmallVector<unsigned, 64> Vals;
5641 // Emit the module's source file name.
5642 {
5643 StringEncoding Bits = getStringEncoding(Str: M.getSourceFileName());
5644 BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
5645 if (Bits == SE_Char6)
5646 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
5647 else if (Bits == SE_Fixed7)
5648 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
5649
5650 // MODULE_CODE_SOURCE_FILENAME: [namechar x N]
5651 auto Abbv = std::make_shared<BitCodeAbbrev>();
5652 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
5653 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5654 Abbv->Add(OpInfo: AbbrevOpToUse);
5655 unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5656
5657 for (const auto P : M.getSourceFileName())
5658 Vals.push_back(Elt: (unsigned char)P);
5659
5660 Stream.EmitRecord(Code: bitc::MODULE_CODE_SOURCE_FILENAME, Vals, Abbrev: FilenameAbbrev);
5661 Vals.clear();
5662 }
5663
5664 // Emit the global variable information.
5665 for (const GlobalVariable &GV : M.globals()) {
5666 // GLOBALVAR: [strtab offset, strtab size, 0, 0, 0, linkage]
5667 Vals.push_back(Elt: StrtabBuilder.add(S: GV.getName()));
5668 Vals.push_back(Elt: GV.getName().size());
5669 Vals.push_back(Elt: 0);
5670 Vals.push_back(Elt: 0);
5671 Vals.push_back(Elt: 0);
5672 Vals.push_back(Elt: getEncodedLinkage(GV));
5673
5674 Stream.EmitRecord(Code: bitc::MODULE_CODE_GLOBALVAR, Vals);
5675 Vals.clear();
5676 }
5677
5678 // Emit the function proto information.
5679 for (const Function &F : M) {
5680 // FUNCTION: [strtab offset, strtab size, 0, 0, 0, linkage]
5681 Vals.push_back(Elt: StrtabBuilder.add(S: F.getName()));
5682 Vals.push_back(Elt: F.getName().size());
5683 Vals.push_back(Elt: 0);
5684 Vals.push_back(Elt: 0);
5685 Vals.push_back(Elt: 0);
5686 Vals.push_back(Elt: getEncodedLinkage(GV: F));
5687
5688 Stream.EmitRecord(Code: bitc::MODULE_CODE_FUNCTION, Vals);
5689 Vals.clear();
5690 }
5691
5692 // Emit the alias information.
5693 for (const GlobalAlias &A : M.aliases()) {
5694 // ALIAS: [strtab offset, strtab size, 0, 0, 0, linkage]
5695 Vals.push_back(Elt: StrtabBuilder.add(S: A.getName()));
5696 Vals.push_back(Elt: A.getName().size());
5697 Vals.push_back(Elt: 0);
5698 Vals.push_back(Elt: 0);
5699 Vals.push_back(Elt: 0);
5700 Vals.push_back(Elt: getEncodedLinkage(GV: A));
5701
5702 Stream.EmitRecord(Code: bitc::MODULE_CODE_ALIAS, Vals);
5703 Vals.clear();
5704 }
5705
5706 // Emit the ifunc information.
5707 for (const GlobalIFunc &I : M.ifuncs()) {
5708 // IFUNC: [strtab offset, strtab size, 0, 0, 0, linkage]
5709 Vals.push_back(Elt: StrtabBuilder.add(S: I.getName()));
5710 Vals.push_back(Elt: I.getName().size());
5711 Vals.push_back(Elt: 0);
5712 Vals.push_back(Elt: 0);
5713 Vals.push_back(Elt: 0);
5714 Vals.push_back(Elt: getEncodedLinkage(GV: I));
5715
5716 Stream.EmitRecord(Code: bitc::MODULE_CODE_IFUNC, Vals);
5717 Vals.clear();
5718 }
5719}
5720
5721void ThinLinkBitcodeWriter::write() {
5722 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5723
5724 writeModuleVersion();
5725
5726 writeSimplifiedModuleInfo();
5727
5728 writePerModuleGlobalValueSummary();
5729
5730 // Write module hash.
5731 Stream.EmitRecord(Code: bitc::MODULE_CODE_HASH, Vals: ArrayRef<uint32_t>(*ModHash));
5732
5733 Stream.ExitBlock();
5734}
5735
5736void BitcodeWriter::writeThinLinkBitcode(const Module &M,
5737 const ModuleSummaryIndex &Index,
5738 const ModuleHash &ModHash) {
5739 assert(!WroteStrtab);
5740
5741 // The Mods vector is used by irsymtab::build, which requires non-const
5742 // Modules in case it needs to materialize metadata. But the bitcode writer
5743 // requires that the module is materialized, so we can cast to non-const here,
5744 // after checking that it is in fact materialized.
5745 assert(M.isMaterialized());
5746 Mods.push_back(x: const_cast<Module *>(&M));
5747
5748 ThinLinkBitcodeWriter ThinLinkWriter(M, StrtabBuilder, *Stream, Index,
5749 ModHash);
5750 ThinLinkWriter.write();
5751}
5752
5753// Write the specified thin link bitcode file to the given raw output stream,
5754// where it will be written in a new bitcode block. This is used when
5755// writing the per-module index file for ThinLTO.
5756void llvm::writeThinLinkBitcodeToFile(const Module &M, raw_ostream &Out,
5757 const ModuleSummaryIndex &Index,
5758 const ModuleHash &ModHash) {
5759 SmallVector<char, 0> Buffer;
5760 Buffer.reserve(N: 256 * 1024);
5761
5762 BitcodeWriter Writer(Buffer);
5763 Writer.writeThinLinkBitcode(M, Index, ModHash);
5764 Writer.writeSymtab();
5765 Writer.writeStrtab();
5766
5767 Out.write(Ptr: (char *)&Buffer.front(), Size: Buffer.size());
5768}
5769
5770static const char *getSectionNameForBitcode(const Triple &T) {
5771 switch (T.getObjectFormat()) {
5772 case Triple::MachO:
5773 return "__LLVM,__bitcode";
5774 case Triple::COFF:
5775 case Triple::ELF:
5776 case Triple::Wasm:
5777 case Triple::UnknownObjectFormat:
5778 return ".llvmbc";
5779 case Triple::GOFF:
5780 llvm_unreachable("GOFF is not yet implemented");
5781 break;
5782 case Triple::SPIRV:
5783 if (T.getVendor() == Triple::AMD)
5784 return ".llvmbc";
5785 llvm_unreachable("SPIRV is not yet implemented");
5786 break;
5787 case Triple::XCOFF:
5788 llvm_unreachable("XCOFF is not yet implemented");
5789 break;
5790 case Triple::DXContainer:
5791 llvm_unreachable("DXContainer is not yet implemented");
5792 break;
5793 }
5794 llvm_unreachable("Unimplemented ObjectFormatType");
5795}
5796
5797static const char *getSectionNameForCommandline(const Triple &T) {
5798 switch (T.getObjectFormat()) {
5799 case Triple::MachO:
5800 return "__LLVM,__cmdline";
5801 case Triple::COFF:
5802 case Triple::ELF:
5803 case Triple::Wasm:
5804 case Triple::UnknownObjectFormat:
5805 return ".llvmcmd";
5806 case Triple::GOFF:
5807 llvm_unreachable("GOFF is not yet implemented");
5808 break;
5809 case Triple::SPIRV:
5810 if (T.getVendor() == Triple::AMD)
5811 return ".llvmcmd";
5812 llvm_unreachable("SPIRV is not yet implemented");
5813 break;
5814 case Triple::XCOFF:
5815 llvm_unreachable("XCOFF is not yet implemented");
5816 break;
5817 case Triple::DXContainer:
5818 llvm_unreachable("DXC is not yet implemented");
5819 break;
5820 }
5821 llvm_unreachable("Unimplemented ObjectFormatType");
5822}
5823
5824void llvm::embedBitcodeInModule(llvm::Module &M, llvm::MemoryBufferRef Buf,
5825 bool EmbedBitcode, bool EmbedCmdline,
5826 const std::vector<uint8_t> &CmdArgs) {
5827 // Save llvm.compiler.used and remove it.
5828 SmallVector<Constant *, 2> UsedArray;
5829 SmallVector<GlobalValue *, 4> UsedGlobals;
5830 GlobalVariable *Used = collectUsedGlobalVariables(M, Vec&: UsedGlobals, CompilerUsed: true);
5831 Type *UsedElementType = Used ? Used->getValueType()->getArrayElementType()
5832 : PointerType::getUnqual(C&: M.getContext());
5833 for (auto *GV : UsedGlobals) {
5834 if (GV->getName() != "llvm.embedded.module" &&
5835 GV->getName() != "llvm.cmdline")
5836 UsedArray.push_back(
5837 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
5838 }
5839 if (Used)
5840 Used->eraseFromParent();
5841
5842 // Embed the bitcode for the llvm module.
5843 std::string Data;
5844 ArrayRef<uint8_t> ModuleData;
5845 Triple T(M.getTargetTriple());
5846
5847 if (EmbedBitcode) {
5848 if (Buf.getBufferSize() == 0 ||
5849 !isBitcode(BufPtr: (const unsigned char *)Buf.getBufferStart(),
5850 BufEnd: (const unsigned char *)Buf.getBufferEnd())) {
5851 // If the input is LLVM Assembly, bitcode is produced by serializing
5852 // the module. Use-lists order need to be preserved in this case.
5853 llvm::raw_string_ostream OS(Data);
5854 llvm::WriteBitcodeToFile(M, Out&: OS, /* ShouldPreserveUseListOrder */ true);
5855 ModuleData =
5856 ArrayRef<uint8_t>((const uint8_t *)OS.str().data(), OS.str().size());
5857 } else
5858 // If the input is LLVM bitcode, write the input byte stream directly.
5859 ModuleData = ArrayRef<uint8_t>((const uint8_t *)Buf.getBufferStart(),
5860 Buf.getBufferSize());
5861 }
5862 llvm::Constant *ModuleConstant =
5863 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts: ModuleData);
5864 llvm::GlobalVariable *GV = new llvm::GlobalVariable(
5865 M, ModuleConstant->getType(), true, llvm::GlobalValue::PrivateLinkage,
5866 ModuleConstant);
5867 GV->setSection(getSectionNameForBitcode(T));
5868 // Set alignment to 1 to prevent padding between two contributions from input
5869 // sections after linking.
5870 GV->setAlignment(Align(1));
5871 UsedArray.push_back(
5872 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
5873 if (llvm::GlobalVariable *Old =
5874 M.getGlobalVariable(Name: "llvm.embedded.module", AllowInternal: true)) {
5875 assert(Old->hasZeroLiveUses() &&
5876 "llvm.embedded.module can only be used once in llvm.compiler.used");
5877 GV->takeName(V: Old);
5878 Old->eraseFromParent();
5879 } else {
5880 GV->setName("llvm.embedded.module");
5881 }
5882
5883 // Skip if only bitcode needs to be embedded.
5884 if (EmbedCmdline) {
5885 // Embed command-line options.
5886 ArrayRef<uint8_t> CmdData(const_cast<uint8_t *>(CmdArgs.data()),
5887 CmdArgs.size());
5888 llvm::Constant *CmdConstant =
5889 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts: CmdData);
5890 GV = new llvm::GlobalVariable(M, CmdConstant->getType(), true,
5891 llvm::GlobalValue::PrivateLinkage,
5892 CmdConstant);
5893 GV->setSection(getSectionNameForCommandline(T));
5894 GV->setAlignment(Align(1));
5895 UsedArray.push_back(
5896 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
5897 if (llvm::GlobalVariable *Old = M.getGlobalVariable(Name: "llvm.cmdline", AllowInternal: true)) {
5898 assert(Old->hasZeroLiveUses() &&
5899 "llvm.cmdline can only be used once in llvm.compiler.used");
5900 GV->takeName(V: Old);
5901 Old->eraseFromParent();
5902 } else {
5903 GV->setName("llvm.cmdline");
5904 }
5905 }
5906
5907 if (UsedArray.empty())
5908 return;
5909
5910 // Recreate llvm.compiler.used.
5911 ArrayType *ATy = ArrayType::get(ElementType: UsedElementType, NumElements: UsedArray.size());
5912 auto *NewUsed = new GlobalVariable(
5913 M, ATy, false, llvm::GlobalValue::AppendingLinkage,
5914 llvm::ConstantArray::get(T: ATy, V: UsedArray), "llvm.compiler.used");
5915 NewUsed->setSection("llvm.metadata");
5916}
5917