1//===- Bitcode/Writer/BitcodeWriter.cpp - Bitcode Writer ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Bitcode writer implementation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Bitcode/BitcodeWriter.h"
14#include "ValueEnumerator.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/DenseMap.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SetVector.h"
21#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/SmallString.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringMap.h"
25#include "llvm/ADT/StringRef.h"
26#include "llvm/Analysis/MemoryProfileInfo.h"
27#include "llvm/BinaryFormat/Dwarf.h"
28#include "llvm/Bitcode/BitcodeCommon.h"
29#include "llvm/Bitcode/BitcodeReader.h"
30#include "llvm/Bitcode/LLVMBitCodes.h"
31#include "llvm/Bitstream/BitCodes.h"
32#include "llvm/Bitstream/BitstreamWriter.h"
33#include "llvm/Config/llvm-config.h"
34#include "llvm/IR/Attributes.h"
35#include "llvm/IR/BasicBlock.h"
36#include "llvm/IR/Comdat.h"
37#include "llvm/IR/Constant.h"
38#include "llvm/IR/ConstantRangeList.h"
39#include "llvm/IR/Constants.h"
40#include "llvm/IR/DebugInfoMetadata.h"
41#include "llvm/IR/DebugLoc.h"
42#include "llvm/IR/DerivedTypes.h"
43#include "llvm/IR/Function.h"
44#include "llvm/IR/GlobalAlias.h"
45#include "llvm/IR/GlobalIFunc.h"
46#include "llvm/IR/GlobalObject.h"
47#include "llvm/IR/GlobalValue.h"
48#include "llvm/IR/GlobalVariable.h"
49#include "llvm/IR/InlineAsm.h"
50#include "llvm/IR/InstrTypes.h"
51#include "llvm/IR/Instruction.h"
52#include "llvm/IR/Instructions.h"
53#include "llvm/IR/LLVMContext.h"
54#include "llvm/IR/Metadata.h"
55#include "llvm/IR/Module.h"
56#include "llvm/IR/ModuleSummaryIndex.h"
57#include "llvm/IR/Operator.h"
58#include "llvm/IR/Type.h"
59#include "llvm/IR/UseListOrder.h"
60#include "llvm/IR/Value.h"
61#include "llvm/IR/ValueSymbolTable.h"
62#include "llvm/MC/StringTableBuilder.h"
63#include "llvm/MC/TargetRegistry.h"
64#include "llvm/Object/IRSymtab.h"
65#include "llvm/ProfileData/MemProf.h"
66#include "llvm/ProfileData/MemProfRadixTree.h"
67#include "llvm/Support/AtomicOrdering.h"
68#include "llvm/Support/Casting.h"
69#include "llvm/Support/CommandLine.h"
70#include "llvm/Support/Compiler.h"
71#include "llvm/Support/Endian.h"
72#include "llvm/Support/Error.h"
73#include "llvm/Support/ErrorHandling.h"
74#include "llvm/Support/MathExtras.h"
75#include "llvm/Support/SHA1.h"
76#include "llvm/Support/raw_ostream.h"
77#include "llvm/TargetParser/Triple.h"
78#include <algorithm>
79#include <cassert>
80#include <cstddef>
81#include <cstdint>
82#include <iterator>
83#include <map>
84#include <memory>
85#include <optional>
86#include <string>
87#include <utility>
88#include <vector>
89
90using namespace llvm;
91using namespace llvm::memprof;
92
93static cl::opt<unsigned>
94 IndexThreshold("bitcode-mdindex-threshold", cl::Hidden, cl::init(Val: 25),
95 cl::desc("Number of metadatas above which we emit an index "
96 "to enable lazy-loading"));
97static cl::opt<uint32_t> FlushThreshold(
98 "bitcode-flush-threshold", cl::Hidden, cl::init(Val: 512),
99 cl::desc("The threshold (unit M) for flushing LLVM bitcode."));
100
101// Since we only use the context information in the memprof summary records in
102// the LTO backends to do assertion checking, save time and space by only
103// serializing the context for non-NDEBUG builds.
104// TODO: Currently this controls writing context of the allocation info records,
105// which are larger and more expensive, but we should do this for the callsite
106// records as well.
107// FIXME: Convert to a const once this has undergone more sigificant testing.
108static cl::opt<bool>
109 CombinedIndexMemProfContext("combined-index-memprof-context", cl::Hidden,
110#ifdef NDEBUG
111 cl::init(Val: false),
112#else
113 cl::init(true),
114#endif
115 cl::desc(""));
116
117static cl::opt<bool> PreserveBitcodeUseListOrder(
118 "preserve-bc-uselistorder", cl::Hidden, cl::init(Val: true),
119 cl::desc("Preserve use-list order when writing LLVM bitcode."));
120
121namespace llvm {
122extern FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold;
123}
124
125namespace {
126
127/// These are manifest constants used by the bitcode writer. They do not need to
128/// be kept in sync with the reader, but need to be consistent within this file.
129enum {
130 // VALUE_SYMTAB_BLOCK abbrev id's.
131 VST_ENTRY_8_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
132 VST_ENTRY_7_ABBREV,
133 VST_ENTRY_6_ABBREV,
134 VST_BBENTRY_6_ABBREV,
135
136 // CONSTANTS_BLOCK abbrev id's.
137 CONSTANTS_SETTYPE_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
138 CONSTANTS_INTEGER_ABBREV,
139 CONSTANTS_BYTE_ABBREV,
140 CONSTANTS_CE_CAST_Abbrev,
141 CONSTANTS_NULL_Abbrev,
142
143 // FUNCTION_BLOCK abbrev id's.
144 FUNCTION_INST_LOAD_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
145 FUNCTION_INST_STORE_ABBREV,
146 FUNCTION_INST_UNOP_ABBREV,
147 FUNCTION_INST_UNOP_FLAGS_ABBREV,
148 FUNCTION_INST_BINOP_ABBREV,
149 FUNCTION_INST_BINOP_FLAGS_ABBREV,
150 FUNCTION_INST_CAST_ABBREV,
151 FUNCTION_INST_CAST_FLAGS_ABBREV,
152 FUNCTION_INST_RET_VOID_ABBREV,
153 FUNCTION_INST_RET_VAL_ABBREV,
154 FUNCTION_INST_BR_UNCOND_ABBREV,
155 FUNCTION_INST_BR_COND_ABBREV,
156 FUNCTION_INST_UNREACHABLE_ABBREV,
157 FUNCTION_INST_GEP_ABBREV,
158 FUNCTION_INST_CMP_ABBREV,
159 FUNCTION_INST_CMP_FLAGS_ABBREV,
160 FUNCTION_DEBUG_RECORD_VALUE_ABBREV,
161 FUNCTION_DEBUG_LOC_ABBREV,
162};
163
164/// Abstract class to manage the bitcode writing, subclassed for each bitcode
165/// file type.
166class BitcodeWriterBase {
167protected:
168 /// The stream created and owned by the client.
169 BitstreamWriter &Stream;
170
171 StringTableBuilder &StrtabBuilder;
172
173public:
174 /// Constructs a BitcodeWriterBase object that writes to the provided
175 /// \p Stream.
176 BitcodeWriterBase(BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder)
177 : Stream(Stream), StrtabBuilder(StrtabBuilder) {}
178
179protected:
180 void writeModuleVersion();
181};
182
183void BitcodeWriterBase::writeModuleVersion() {
184 // VERSION: [version#]
185 Stream.EmitRecord(Code: bitc::MODULE_CODE_VERSION, Vals: ArrayRef<uint64_t>{2});
186}
187
188/// Base class to manage the module bitcode writing, currently subclassed for
189/// ModuleBitcodeWriter and ThinLinkBitcodeWriter.
190class ModuleBitcodeWriterBase : public BitcodeWriterBase {
191protected:
192 /// The Module to write to bitcode.
193 const Module &M;
194
195 /// Enumerates ids for all values in the module.
196 ValueEnumerator VE;
197
198 /// Optional per-module index to write for ThinLTO.
199 const ModuleSummaryIndex *Index;
200
201 /// Map that holds the correspondence between GUIDs in the summary index,
202 /// that came from indirect call profiles, and a value id generated by this
203 /// class to use in the VST and summary block records.
204 std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
205
206 /// Tracks the last value id recorded in the GUIDToValueMap.
207 unsigned GlobalValueId;
208
209 /// Saves the offset of the VSTOffset record that must eventually be
210 /// backpatched with the offset of the actual VST.
211 uint64_t VSTOffsetPlaceholder = 0;
212
213public:
214 /// Constructs a ModuleBitcodeWriterBase object for the given Module,
215 /// writing to the provided \p Buffer.
216 ModuleBitcodeWriterBase(const Module &M, StringTableBuilder &StrtabBuilder,
217 BitstreamWriter &Stream,
218 bool ShouldPreserveUseListOrder,
219 const ModuleSummaryIndex *Index)
220 : BitcodeWriterBase(Stream, StrtabBuilder), M(M),
221 VE(M, PreserveBitcodeUseListOrder.getNumOccurrences()
222 ? PreserveBitcodeUseListOrder
223 : ShouldPreserveUseListOrder),
224 Index(Index) {
225 // Assign ValueIds to any callee values in the index that came from
226 // indirect call profiles and were recorded as a GUID not a Value*
227 // (which would have been assigned an ID by the ValueEnumerator).
228 // The starting ValueId is just after the number of values in the
229 // ValueEnumerator, so that they can be emitted in the VST.
230 GlobalValueId = VE.getValues().size();
231 if (!Index)
232 return;
233 // Sort by GUID for deterministic value ID assignment.
234 for (const auto &GUIDSummaryLists :
235 Index->sortedGlobalValueSummariesRange())
236 // Examine all summaries for this GUID.
237 for (auto &Summary : GUIDSummaryLists.second.getSummaryList())
238 if (auto *FS = dyn_cast<FunctionSummary>(Val: Summary.get())) {
239 // For each call in the function summary, see if the call
240 // is to a GUID (which means it is for an indirect call,
241 // otherwise we would have a Value for it). If so, synthesize
242 // a value id.
243 for (auto &CallEdge : FS->calls())
244 if (!CallEdge.first.haveGVs() || !CallEdge.first.getValue())
245 assignValueId(ValGUID: CallEdge.first.getGUID());
246
247 // For each referenced variables in the function summary, see if the
248 // variable is represented by a GUID (as opposed to a symbol to
249 // declarations or definitions in the module). If so, synthesize a
250 // value id.
251 for (auto &RefEdge : FS->refs())
252 if (!RefEdge.haveGVs() || !RefEdge.getValue())
253 assignValueId(ValGUID: RefEdge.getGUID());
254 }
255 }
256
257protected:
258 void writePerModuleGlobalValueSummary();
259 void writeGUIDList();
260
261private:
262 void writePerModuleFunctionSummaryRecord(
263 SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
264 unsigned ValueID, unsigned FSCallsProfileAbbrev, unsigned CallsiteAbbrev,
265 unsigned AllocAbbrev, unsigned ContextIdAbbvId, const Function &F,
266 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
267 CallStackId &CallStackCount);
268 void writeModuleLevelReferences(const GlobalVariable &V,
269 SmallVector<uint64_t, 64> &NameVals,
270 unsigned FSModRefsAbbrev,
271 unsigned FSModVTableRefsAbbrev);
272
273 void assignValueId(GlobalValue::GUID ValGUID) {
274 GUIDToValueIdMap[ValGUID] = ++GlobalValueId;
275 }
276
277 unsigned getValueId(GlobalValue::GUID ValGUID) {
278 const auto &VMI = GUIDToValueIdMap.find(x: ValGUID);
279 // Expect that any GUID value had a value Id assigned by an
280 // earlier call to assignValueId.
281 assert(VMI != GUIDToValueIdMap.end() &&
282 "GUID does not have assigned value Id");
283 return VMI->second;
284 }
285
286 // Helper to get the valueId for the type of value recorded in VI.
287 unsigned getValueId(ValueInfo VI) {
288 if (!VI.haveGVs() || !VI.getValue())
289 return getValueId(ValGUID: VI.getGUID());
290 return VE.getValueID(V: VI.getValue());
291 }
292
293 std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
294};
295
296/// Class to manage the bitcode writing for a module.
297class ModuleBitcodeWriter : public ModuleBitcodeWriterBase {
298 /// True if a module hash record should be written.
299 bool GenerateHash;
300
301 /// If non-null, when GenerateHash is true, the resulting hash is written
302 /// into ModHash.
303 ModuleHash *ModHash;
304
305 SHA1 Hasher;
306
307 /// The start bit of the identification block.
308 uint64_t BitcodeStartBit;
309
310public:
311 /// Constructs a ModuleBitcodeWriter object for the given Module,
312 /// writing to the provided \p Buffer.
313 ModuleBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
314 BitstreamWriter &Stream, bool ShouldPreserveUseListOrder,
315 const ModuleSummaryIndex *Index, bool GenerateHash,
316 ModuleHash *ModHash = nullptr)
317 : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
318 ShouldPreserveUseListOrder, Index),
319 GenerateHash(GenerateHash), ModHash(ModHash),
320 BitcodeStartBit(Stream.GetCurrentBitNo()) {}
321
322 /// Emit the current module to the bitstream.
323 void write();
324
325private:
326 uint64_t bitcodeStartBit() { return BitcodeStartBit; }
327
328 size_t addToStrtab(StringRef Str);
329
330 void writeAttributeGroupTable();
331 void writeAttributeTable();
332 void writeTypeTable();
333 void writeComdats();
334 void writeValueSymbolTableForwardDecl();
335 void writeModuleInfo();
336 void writeValueAsMetadata(const ValueAsMetadata *MD,
337 SmallVectorImpl<uint64_t> &Record);
338 void writeMDTuple(const MDTuple *N, SmallVectorImpl<uint64_t> &Record,
339 unsigned Abbrev);
340 unsigned createDILocationAbbrev();
341 void writeDILocation(const DILocation *N, SmallVectorImpl<uint64_t> &Record,
342 unsigned &Abbrev);
343 unsigned createGenericDINodeAbbrev();
344 void writeGenericDINode(const GenericDINode *N,
345 SmallVectorImpl<uint64_t> &Record, unsigned &Abbrev);
346 void writeDISubrange(const DISubrange *N, SmallVectorImpl<uint64_t> &Record,
347 unsigned Abbrev);
348 void writeDIGenericSubrange(const DIGenericSubrange *N,
349 SmallVectorImpl<uint64_t> &Record,
350 unsigned Abbrev);
351 void writeDIEnumerator(const DIEnumerator *N,
352 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
353 void writeDIBasicType(const DIBasicType *N, SmallVectorImpl<uint64_t> &Record,
354 unsigned Abbrev);
355 void writeDIFixedPointType(const DIFixedPointType *N,
356 SmallVectorImpl<uint64_t> &Record,
357 unsigned Abbrev);
358 void writeDIStringType(const DIStringType *N,
359 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
360 void writeDIDerivedType(const DIDerivedType *N,
361 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
362 void writeDISubrangeType(const DISubrangeType *N,
363 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
364 void writeDICompositeType(const DICompositeType *N,
365 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
366 void writeDISubroutineType(const DISubroutineType *N,
367 SmallVectorImpl<uint64_t> &Record,
368 unsigned Abbrev);
369 void writeDIFile(const DIFile *N, SmallVectorImpl<uint64_t> &Record,
370 unsigned Abbrev);
371 void writeDICompileUnit(const DICompileUnit *N,
372 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
373 void writeDISubprogram(const DISubprogram *N,
374 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
375 void writeDILexicalBlock(const DILexicalBlock *N,
376 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
377 void writeDILexicalBlockFile(const DILexicalBlockFile *N,
378 SmallVectorImpl<uint64_t> &Record,
379 unsigned Abbrev);
380 void writeDICommonBlock(const DICommonBlock *N,
381 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
382 void writeDINamespace(const DINamespace *N, SmallVectorImpl<uint64_t> &Record,
383 unsigned Abbrev);
384 void writeDIMacro(const DIMacro *N, SmallVectorImpl<uint64_t> &Record,
385 unsigned Abbrev);
386 void writeDIMacroFile(const DIMacroFile *N, SmallVectorImpl<uint64_t> &Record,
387 unsigned Abbrev);
388 void writeDIArgList(const DIArgList *N, SmallVectorImpl<uint64_t> &Record);
389 void writeDIModule(const DIModule *N, SmallVectorImpl<uint64_t> &Record,
390 unsigned Abbrev);
391 void writeDIAssignID(const DIAssignID *N, SmallVectorImpl<uint64_t> &Record,
392 unsigned Abbrev);
393 void writeDITemplateTypeParameter(const DITemplateTypeParameter *N,
394 SmallVectorImpl<uint64_t> &Record,
395 unsigned Abbrev);
396 void writeDITemplateValueParameter(const DITemplateValueParameter *N,
397 SmallVectorImpl<uint64_t> &Record,
398 unsigned Abbrev);
399 void writeDIGlobalVariable(const DIGlobalVariable *N,
400 SmallVectorImpl<uint64_t> &Record,
401 unsigned Abbrev);
402 void writeDILocalVariable(const DILocalVariable *N,
403 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
404 void writeDILabel(const DILabel *N,
405 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
406 void writeDIExpression(const DIExpression *N,
407 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
408 void writeDIGlobalVariableExpression(const DIGlobalVariableExpression *N,
409 SmallVectorImpl<uint64_t> &Record,
410 unsigned Abbrev);
411 void writeDIObjCProperty(const DIObjCProperty *N,
412 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
413 void writeDIProperty(const DIProperty *N, SmallVectorImpl<uint64_t> &Record,
414 unsigned Abbrev);
415 void writeDIImportedEntity(const DIImportedEntity *N,
416 SmallVectorImpl<uint64_t> &Record,
417 unsigned Abbrev);
418 unsigned createNamedMetadataAbbrev();
419 void writeNamedMetadata(SmallVectorImpl<uint64_t> &Record);
420 unsigned createMetadataStringsAbbrev();
421 void writeMetadataStrings(ArrayRef<const Metadata *> Strings,
422 SmallVectorImpl<uint64_t> &Record);
423 void writeMetadataRecords(ArrayRef<const Metadata *> MDs,
424 SmallVectorImpl<uint64_t> &Record,
425 std::vector<unsigned> *MDAbbrevs = nullptr,
426 std::vector<uint64_t> *IndexPos = nullptr);
427 void writeModuleMetadata();
428 void writeFunctionMetadata(const Function &F);
429 void writeFunctionMetadataAttachment(const Function &F);
430 void pushGlobalMetadataAttachment(SmallVectorImpl<uint64_t> &Record,
431 const GlobalObject &GO);
432 void writeModuleMetadataKinds();
433 void writeOperandBundleTags();
434 void writeSyncScopeNames();
435 void writeConstants(unsigned FirstVal, unsigned LastVal, bool isGlobal);
436 void writeModuleConstants();
437 bool pushValueAndType(const Value *V, unsigned InstID,
438 SmallVectorImpl<unsigned> &Vals);
439 bool pushValueOrMetadata(const Value *V, unsigned InstID,
440 SmallVectorImpl<unsigned> &Vals);
441 void writeOperandBundles(const CallBase &CB, unsigned InstID);
442 void pushValue(const Value *V, unsigned InstID,
443 SmallVectorImpl<unsigned> &Vals);
444 void pushValueSigned(const Value *V, unsigned InstID,
445 SmallVectorImpl<uint64_t> &Vals);
446 void writeInstruction(const Instruction &I, unsigned InstID,
447 SmallVectorImpl<unsigned> &Vals);
448 void writeFunctionLevelValueSymbolTable(const ValueSymbolTable &VST);
449 void writeGlobalValueSymbolTable(
450 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
451 void writeUseList(UseListOrder &&Order);
452 void writeUseListBlock(const Function *F);
453 void
454 writeFunction(const Function &F,
455 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
456 void writeBlockInfo();
457 void writeModuleHash(StringRef View);
458
459 unsigned getEncodedSyncScopeID(SyncScope::ID SSID) {
460 return unsigned(SSID);
461 }
462
463 unsigned getEncodedAlign(MaybeAlign Alignment) { return encode(A: Alignment); }
464};
465
466/// Class to manage the bitcode writing for a combined index.
467class IndexBitcodeWriter : public BitcodeWriterBase {
468 /// The combined index to write to bitcode.
469 const ModuleSummaryIndex &Index;
470
471 /// When writing combined summaries, provides the set of global value
472 /// summaries for which the value (function, function alias, etc) should be
473 /// imported as a declaration.
474 const GVSummaryPtrSet *DecSummaries = nullptr;
475
476 /// When writing a subset of the index for distributed backends, client
477 /// provides a map of modules to the corresponding GUIDs/summaries to write.
478 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex;
479
480 /// Map that holds the correspondence between the GUID used in the combined
481 /// index and a value id generated by this class to use in references.
482 std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
483
484 // The stack ids used by this index, which will be a subset of those in
485 // the full index in the case of distributed indexes.
486 std::vector<uint64_t> StackIds;
487
488 // Keep a map of the stack id indices used by records being written for this
489 // index to the index of the corresponding stack id in the above StackIds
490 // vector. Ensures we write each referenced stack id once.
491 DenseMap<unsigned, unsigned> StackIdIndicesToIndex;
492
493 /// Tracks the last value id recorded in the GUIDToValueMap.
494 unsigned GlobalValueId = 0;
495
496 /// Tracks the assignment of module paths in the module path string table to
497 /// an id assigned for use in summary references to the module path.
498 DenseMap<StringRef, uint64_t> ModuleIdMap;
499
500public:
501 /// Constructs a IndexBitcodeWriter object for the given combined index,
502 /// writing to the provided \p Buffer. When writing a subset of the index
503 /// for a distributed backend, provide a \p ModuleToSummariesForIndex map.
504 /// If provided, \p DecSummaries specifies the set of summaries for which
505 /// the corresponding functions or aliased functions should be imported as a
506 /// declaration (but not definition) for each module.
507 IndexBitcodeWriter(
508 BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder,
509 const ModuleSummaryIndex &Index,
510 const GVSummaryPtrSet *DecSummaries = nullptr,
511 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex = nullptr)
512 : BitcodeWriterBase(Stream, StrtabBuilder), Index(Index),
513 DecSummaries(DecSummaries),
514 ModuleToSummariesForIndex(ModuleToSummariesForIndex) {
515
516 // See if the StackIdIndex was already added to the StackId map and
517 // vector. If not, record it.
518 auto RecordStackIdReference = [&](unsigned StackIdIndex) {
519 // If the StackIdIndex is not yet in the map, the below insert ensures
520 // that it will point to the new StackIds vector entry we push to just
521 // below.
522 auto Inserted =
523 StackIdIndicesToIndex.insert(KV: {StackIdIndex, StackIds.size()});
524 if (Inserted.second)
525 StackIds.push_back(x: Index.getStackIdAtIndex(Index: StackIdIndex));
526 };
527
528 // Assign unique value ids to all summaries to be written, for use
529 // in writing out the call graph edges. Save the mapping from GUID
530 // to the new global value id to use when writing those edges, which
531 // are currently saved in the index in terms of GUID.
532 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
533 GUIDToValueIdMap[I.first] = ++GlobalValueId;
534 // If this is invoked for an aliasee, we want to record the above mapping,
535 // but not the information needed for its summary entry (if the aliasee is
536 // to be imported, we will invoke this separately with IsAliasee=false).
537 if (IsAliasee)
538 return;
539 auto *FS = dyn_cast<FunctionSummary>(Val: I.second);
540 if (!FS)
541 return;
542 // Record all stack id indices actually used in the summary entries being
543 // written, so that we can compact them in the case of distributed ThinLTO
544 // indexes.
545 for (auto &CI : FS->callsites()) {
546 // If the stack id list is empty, this callsite info was synthesized for
547 // a missing tail call frame. Ensure that the callee's GUID gets a value
548 // id. Normally we only generate these for defined summaries, which in
549 // the case of distributed ThinLTO is only the functions already defined
550 // in the module or that we want to import. We don't bother to include
551 // all the callee symbols as they aren't normally needed in the backend.
552 // However, for the synthesized callsite infos we do need the callee
553 // GUID in the backend so that we can correlate the identified callee
554 // with this callsite info (which for non-tail calls is done by the
555 // ordering of the callsite infos and verified via stack ids).
556 if (CI.StackIdIndices.empty()) {
557 GUIDToValueIdMap[CI.Callee.getGUID()] = ++GlobalValueId;
558 continue;
559 }
560 for (auto Idx : CI.StackIdIndices)
561 RecordStackIdReference(Idx);
562 }
563 if (CombinedIndexMemProfContext) {
564 for (auto &AI : FS->allocs())
565 for (auto &MIB : AI.MIBs)
566 for (auto Idx : MIB.StackIdIndices)
567 RecordStackIdReference(Idx);
568 }
569 });
570 }
571
572 /// The below iterator returns the GUID and associated summary.
573 using GVInfo = std::pair<GlobalValue::GUID, GlobalValueSummary *>;
574
575 /// Calls the callback for each value GUID and summary to be written to
576 /// bitcode. This hides the details of whether they are being pulled from the
577 /// entire index or just those in a provided ModuleToSummariesForIndex map.
578 template<typename Functor>
579 void forEachSummary(Functor Callback) {
580 if (ModuleToSummariesForIndex) {
581 for (auto &M : *ModuleToSummariesForIndex)
582 for (auto &[GUID, GVS] : M.second) {
583 Callback({GUID, GVS}, false);
584 // Ensure aliasee is handled, e.g. for assigning a valueId,
585 // even if we are not importing the aliasee directly (the
586 // imported alias will contain a copy of aliasee).
587 if (auto *AS = dyn_cast<AliasSummary>(Val: GVS))
588 Callback({AS->getAliaseeGUID(), &AS->getAliasee()}, true);
589 }
590 } else {
591 // Sort by GUID for deterministic output.
592 for (const auto &Summaries : Index.sortedGlobalValueSummariesRange())
593 for (auto &Summary : Summaries.second.getSummaryList())
594 Callback({Summaries.first, Summary.get()}, false);
595 }
596 }
597
598 /// Calls the callback for each entry in the modulePaths StringMap that
599 /// should be written to the module path string table. This hides the details
600 /// of whether they are being pulled from the entire index or just those in a
601 /// provided ModuleToSummariesForIndex map.
602 template <typename Functor> void forEachModule(Functor Callback) {
603 if (ModuleToSummariesForIndex) {
604 for (const auto &M : *ModuleToSummariesForIndex) {
605 const auto &MPI = Index.modulePaths().find(Key: M.first);
606 if (MPI == Index.modulePaths().end()) {
607 // This should only happen if the bitcode file was empty, in which
608 // case we shouldn't be importing (the ModuleToSummariesForIndex
609 // would only include the module we are writing and index for).
610 assert(ModuleToSummariesForIndex->size() == 1);
611 continue;
612 }
613 Callback(*MPI);
614 }
615 } else {
616 // Since StringMap iteration order isn't guaranteed, order by path string
617 // first.
618 // FIXME: Make this a vector of StringMapEntry instead to avoid the later
619 // map lookup.
620 std::vector<StringRef> ModulePaths;
621 for (auto &[ModPath, _] : Index.modulePaths())
622 ModulePaths.push_back(x: ModPath);
623 llvm::sort(C&: ModulePaths);
624 for (auto &ModPath : ModulePaths)
625 Callback(*Index.modulePaths().find(Key: ModPath));
626 }
627 }
628
629 /// Main entry point for writing a combined index to bitcode.
630 void write();
631
632private:
633 void writeModStrings();
634 void writeCombinedGlobalValueSummary();
635
636 std::optional<unsigned> getValueId(GlobalValue::GUID ValGUID) {
637 auto VMI = GUIDToValueIdMap.find(x: ValGUID);
638 if (VMI == GUIDToValueIdMap.end())
639 return std::nullopt;
640 return VMI->second;
641 }
642
643 std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
644};
645
646} // end anonymous namespace
647
648static unsigned getEncodedCastOpcode(unsigned Opcode) {
649 switch (Opcode) {
650 default: llvm_unreachable("Unknown cast instruction!");
651 case Instruction::Trunc : return bitc::CAST_TRUNC;
652 case Instruction::ZExt : return bitc::CAST_ZEXT;
653 case Instruction::SExt : return bitc::CAST_SEXT;
654 case Instruction::FPToUI : return bitc::CAST_FPTOUI;
655 case Instruction::FPToSI : return bitc::CAST_FPTOSI;
656 case Instruction::UIToFP : return bitc::CAST_UITOFP;
657 case Instruction::SIToFP : return bitc::CAST_SITOFP;
658 case Instruction::FPTrunc : return bitc::CAST_FPTRUNC;
659 case Instruction::FPExt : return bitc::CAST_FPEXT;
660 case Instruction::PtrToAddr: return bitc::CAST_PTRTOADDR;
661 case Instruction::PtrToInt: return bitc::CAST_PTRTOINT;
662 case Instruction::IntToPtr: return bitc::CAST_INTTOPTR;
663 case Instruction::BitCast : return bitc::CAST_BITCAST;
664 case Instruction::AddrSpaceCast: return bitc::CAST_ADDRSPACECAST;
665 }
666}
667
668static unsigned getEncodedUnaryOpcode(unsigned Opcode) {
669 switch (Opcode) {
670 default: llvm_unreachable("Unknown binary instruction!");
671 case Instruction::FNeg: return bitc::UNOP_FNEG;
672 }
673}
674
675static unsigned getEncodedBinaryOpcode(unsigned Opcode) {
676 switch (Opcode) {
677 default: llvm_unreachable("Unknown binary instruction!");
678 case Instruction::Add:
679 case Instruction::FAdd: return bitc::BINOP_ADD;
680 case Instruction::Sub:
681 case Instruction::FSub: return bitc::BINOP_SUB;
682 case Instruction::Mul:
683 case Instruction::FMul: return bitc::BINOP_MUL;
684 case Instruction::UDiv: return bitc::BINOP_UDIV;
685 case Instruction::FDiv:
686 case Instruction::SDiv: return bitc::BINOP_SDIV;
687 case Instruction::URem: return bitc::BINOP_UREM;
688 case Instruction::FRem:
689 case Instruction::SRem: return bitc::BINOP_SREM;
690 case Instruction::Shl: return bitc::BINOP_SHL;
691 case Instruction::LShr: return bitc::BINOP_LSHR;
692 case Instruction::AShr: return bitc::BINOP_ASHR;
693 case Instruction::And: return bitc::BINOP_AND;
694 case Instruction::Or: return bitc::BINOP_OR;
695 case Instruction::Xor: return bitc::BINOP_XOR;
696 }
697}
698
699static unsigned getEncodedRMWOperation(const AtomicRMWInst &I) {
700 unsigned Encoding = 0;
701 switch (I.getOperation()) {
702 default: llvm_unreachable("Unknown RMW operation!");
703 case AtomicRMWInst::Xchg:
704 Encoding = bitc::RMW_XCHG;
705 break;
706 case AtomicRMWInst::Add:
707 Encoding = bitc::RMW_ADD;
708 break;
709 case AtomicRMWInst::Sub:
710 Encoding = bitc::RMW_SUB;
711 break;
712 case AtomicRMWInst::And:
713 Encoding = bitc::RMW_AND;
714 break;
715 case AtomicRMWInst::Nand:
716 Encoding = bitc::RMW_NAND;
717 break;
718 case AtomicRMWInst::Or:
719 Encoding = bitc::RMW_OR;
720 break;
721 case AtomicRMWInst::Xor:
722 Encoding = bitc::RMW_XOR;
723 break;
724 case AtomicRMWInst::Max:
725 Encoding = bitc::RMW_MAX;
726 break;
727 case AtomicRMWInst::Min:
728 Encoding = bitc::RMW_MIN;
729 break;
730 case AtomicRMWInst::UMax:
731 Encoding = bitc::RMW_UMAX;
732 break;
733 case AtomicRMWInst::UMin:
734 Encoding = bitc::RMW_UMIN;
735 break;
736 case AtomicRMWInst::FAdd:
737 Encoding = bitc::RMW_FADD;
738 break;
739 case AtomicRMWInst::FSub:
740 Encoding = bitc::RMW_FSUB;
741 break;
742 case AtomicRMWInst::FMax:
743 Encoding = bitc::RMW_FMAX;
744 break;
745 case AtomicRMWInst::FMin:
746 Encoding = bitc::RMW_FMIN;
747 break;
748 case AtomicRMWInst::FMaximum:
749 Encoding = bitc::RMW_FMAXIMUM;
750 break;
751 case AtomicRMWInst::FMinimum:
752 Encoding = bitc::RMW_FMINIMUM;
753 break;
754 case AtomicRMWInst::FMaximumNum:
755 Encoding = bitc::RMW_FMAXIMUMNUM;
756 break;
757 case AtomicRMWInst::FMinimumNum:
758 Encoding = bitc::RMW_FMINIMUMNUM;
759 break;
760 case AtomicRMWInst::UIncWrap:
761 Encoding = bitc::RMW_UINC_WRAP;
762 break;
763 case AtomicRMWInst::UDecWrap:
764 Encoding = bitc::RMW_UDEC_WRAP;
765 break;
766 case AtomicRMWInst::USubCond:
767 Encoding = bitc::RMW_USUB_COND;
768 break;
769 case AtomicRMWInst::USubSat:
770 Encoding = bitc::RMW_USUB_SAT;
771 break;
772 }
773
774 if (I.isElementwise())
775 Encoding |= bitc::RMW_ELEMENTWISE_FLAG;
776 return Encoding;
777}
778
779static unsigned getEncodedOrdering(AtomicOrdering Ordering) {
780 switch (Ordering) {
781 case AtomicOrdering::NotAtomic: return bitc::ORDERING_NOTATOMIC;
782 case AtomicOrdering::Unordered: return bitc::ORDERING_UNORDERED;
783 case AtomicOrdering::Monotonic: return bitc::ORDERING_MONOTONIC;
784 case AtomicOrdering::Acquire: return bitc::ORDERING_ACQUIRE;
785 case AtomicOrdering::Release: return bitc::ORDERING_RELEASE;
786 case AtomicOrdering::AcquireRelease: return bitc::ORDERING_ACQREL;
787 case AtomicOrdering::SequentiallyConsistent: return bitc::ORDERING_SEQCST;
788 }
789 llvm_unreachable("Invalid ordering");
790}
791
792static void writeStringRecord(BitstreamWriter &Stream, unsigned Code,
793 StringRef Str, unsigned AbbrevToUse) {
794 SmallVector<unsigned, 64> Vals;
795
796 // Code: [strchar x N]
797 for (char C : Str) {
798 if (AbbrevToUse && !BitCodeAbbrevOp::isChar6(C))
799 AbbrevToUse = 0;
800 Vals.push_back(Elt: C);
801 }
802
803 // Emit the finished record.
804 Stream.EmitRecord(Code, Vals, Abbrev: AbbrevToUse);
805}
806
807static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
808 switch (Kind) {
809 case Attribute::Alignment:
810 return bitc::ATTR_KIND_ALIGNMENT;
811 case Attribute::AllocAlign:
812 return bitc::ATTR_KIND_ALLOC_ALIGN;
813 case Attribute::AllocSize:
814 return bitc::ATTR_KIND_ALLOC_SIZE;
815 case Attribute::AlwaysInline:
816 return bitc::ATTR_KIND_ALWAYS_INLINE;
817 case Attribute::Builtin:
818 return bitc::ATTR_KIND_BUILTIN;
819 case Attribute::ByVal:
820 return bitc::ATTR_KIND_BY_VAL;
821 case Attribute::Convergent:
822 return bitc::ATTR_KIND_CONVERGENT;
823 case Attribute::InAlloca:
824 return bitc::ATTR_KIND_IN_ALLOCA;
825 case Attribute::Cold:
826 return bitc::ATTR_KIND_COLD;
827 case Attribute::DisableSanitizerInstrumentation:
828 return bitc::ATTR_KIND_DISABLE_SANITIZER_INSTRUMENTATION;
829 case Attribute::FnRetThunkExtern:
830 return bitc::ATTR_KIND_FNRETTHUNK_EXTERN;
831 case Attribute::Flatten:
832 return bitc::ATTR_KIND_FLATTEN;
833 case Attribute::Hot:
834 return bitc::ATTR_KIND_HOT;
835 case Attribute::ElementType:
836 return bitc::ATTR_KIND_ELEMENTTYPE;
837 case Attribute::HybridPatchable:
838 return bitc::ATTR_KIND_HYBRID_PATCHABLE;
839 case Attribute::InlineHint:
840 return bitc::ATTR_KIND_INLINE_HINT;
841 case Attribute::InReg:
842 return bitc::ATTR_KIND_IN_REG;
843 case Attribute::JumpTable:
844 return bitc::ATTR_KIND_JUMP_TABLE;
845 case Attribute::MinSize:
846 return bitc::ATTR_KIND_MIN_SIZE;
847 case Attribute::AllocatedPointer:
848 return bitc::ATTR_KIND_ALLOCATED_POINTER;
849 case Attribute::AllocKind:
850 return bitc::ATTR_KIND_ALLOC_KIND;
851 case Attribute::Memory:
852 return bitc::ATTR_KIND_MEMORY;
853 case Attribute::NoFPClass:
854 return bitc::ATTR_KIND_NOFPCLASS;
855 case Attribute::Naked:
856 return bitc::ATTR_KIND_NAKED;
857 case Attribute::Nest:
858 return bitc::ATTR_KIND_NEST;
859 case Attribute::NoAlias:
860 return bitc::ATTR_KIND_NO_ALIAS;
861 case Attribute::NoBuiltin:
862 return bitc::ATTR_KIND_NO_BUILTIN;
863 case Attribute::NoCallback:
864 return bitc::ATTR_KIND_NO_CALLBACK;
865 case Attribute::NoDivergenceSource:
866 return bitc::ATTR_KIND_NO_DIVERGENCE_SOURCE;
867 case Attribute::NoDuplicate:
868 return bitc::ATTR_KIND_NO_DUPLICATE;
869 case Attribute::NoFree:
870 return bitc::ATTR_KIND_NOFREE;
871 case Attribute::NoFreeObj:
872 return bitc::ATTR_KIND_NOFREEOBJ;
873 case Attribute::NoImplicitFloat:
874 return bitc::ATTR_KIND_NO_IMPLICIT_FLOAT;
875 case Attribute::NoInline:
876 return bitc::ATTR_KIND_NO_INLINE;
877 case Attribute::NoRecurse:
878 return bitc::ATTR_KIND_NO_RECURSE;
879 case Attribute::NoMerge:
880 return bitc::ATTR_KIND_NO_MERGE;
881 case Attribute::NonLazyBind:
882 return bitc::ATTR_KIND_NON_LAZY_BIND;
883 case Attribute::NonNull:
884 return bitc::ATTR_KIND_NON_NULL;
885 case Attribute::Dereferenceable:
886 return bitc::ATTR_KIND_DEREFERENCEABLE;
887 case Attribute::DereferenceableOrNull:
888 return bitc::ATTR_KIND_DEREFERENCEABLE_OR_NULL;
889 case Attribute::NoRedZone:
890 return bitc::ATTR_KIND_NO_RED_ZONE;
891 case Attribute::NoReturn:
892 return bitc::ATTR_KIND_NO_RETURN;
893 case Attribute::NoSync:
894 return bitc::ATTR_KIND_NOSYNC;
895 case Attribute::NoCfCheck:
896 return bitc::ATTR_KIND_NOCF_CHECK;
897 case Attribute::NoProfile:
898 return bitc::ATTR_KIND_NO_PROFILE;
899 case Attribute::SkipProfile:
900 return bitc::ATTR_KIND_SKIP_PROFILE;
901 case Attribute::NoUnwind:
902 return bitc::ATTR_KIND_NO_UNWIND;
903 case Attribute::NoSanitizeBounds:
904 return bitc::ATTR_KIND_NO_SANITIZE_BOUNDS;
905 case Attribute::NoSanitizeCoverage:
906 return bitc::ATTR_KIND_NO_SANITIZE_COVERAGE;
907 case Attribute::NullPointerIsValid:
908 return bitc::ATTR_KIND_NULL_POINTER_IS_VALID;
909 case Attribute::OptimizeForDebugging:
910 return bitc::ATTR_KIND_OPTIMIZE_FOR_DEBUGGING;
911 case Attribute::OptForFuzzing:
912 return bitc::ATTR_KIND_OPT_FOR_FUZZING;
913 case Attribute::OptimizeForSize:
914 return bitc::ATTR_KIND_OPTIMIZE_FOR_SIZE;
915 case Attribute::OptimizeNone:
916 return bitc::ATTR_KIND_OPTIMIZE_NONE;
917 case Attribute::ReadNone:
918 return bitc::ATTR_KIND_READ_NONE;
919 case Attribute::ReadOnly:
920 return bitc::ATTR_KIND_READ_ONLY;
921 case Attribute::Returned:
922 return bitc::ATTR_KIND_RETURNED;
923 case Attribute::ReturnsTwice:
924 return bitc::ATTR_KIND_RETURNS_TWICE;
925 case Attribute::SExt:
926 return bitc::ATTR_KIND_S_EXT;
927 case Attribute::Speculatable:
928 return bitc::ATTR_KIND_SPECULATABLE;
929 case Attribute::StackAlignment:
930 return bitc::ATTR_KIND_STACK_ALIGNMENT;
931 case Attribute::StackProtect:
932 return bitc::ATTR_KIND_STACK_PROTECT;
933 case Attribute::StackProtectReq:
934 return bitc::ATTR_KIND_STACK_PROTECT_REQ;
935 case Attribute::StackProtectStrong:
936 return bitc::ATTR_KIND_STACK_PROTECT_STRONG;
937 case Attribute::SafeStack:
938 return bitc::ATTR_KIND_SAFESTACK;
939 case Attribute::ShadowCallStack:
940 return bitc::ATTR_KIND_SHADOWCALLSTACK;
941 case Attribute::StrictFP:
942 return bitc::ATTR_KIND_STRICT_FP;
943 case Attribute::StructRet:
944 return bitc::ATTR_KIND_STRUCT_RET;
945 case Attribute::SanitizeAddress:
946 return bitc::ATTR_KIND_SANITIZE_ADDRESS;
947 case Attribute::SanitizeAllocToken:
948 return bitc::ATTR_KIND_SANITIZE_ALLOC_TOKEN;
949 case Attribute::SanitizeHWAddress:
950 return bitc::ATTR_KIND_SANITIZE_HWADDRESS;
951 case Attribute::SanitizeThread:
952 return bitc::ATTR_KIND_SANITIZE_THREAD;
953 case Attribute::SanitizeType:
954 return bitc::ATTR_KIND_SANITIZE_TYPE;
955 case Attribute::SanitizeMemory:
956 return bitc::ATTR_KIND_SANITIZE_MEMORY;
957 case Attribute::SanitizeNumericalStability:
958 return bitc::ATTR_KIND_SANITIZE_NUMERICAL_STABILITY;
959 case Attribute::SanitizeRealtime:
960 return bitc::ATTR_KIND_SANITIZE_REALTIME;
961 case Attribute::SanitizeRealtimeBlocking:
962 return bitc::ATTR_KIND_SANITIZE_REALTIME_BLOCKING;
963 case Attribute::SpeculativeLoadHardening:
964 return bitc::ATTR_KIND_SPECULATIVE_LOAD_HARDENING;
965 case Attribute::SwiftError:
966 return bitc::ATTR_KIND_SWIFT_ERROR;
967 case Attribute::SwiftSelf:
968 return bitc::ATTR_KIND_SWIFT_SELF;
969 case Attribute::SwiftAsync:
970 return bitc::ATTR_KIND_SWIFT_ASYNC;
971 case Attribute::UWTable:
972 return bitc::ATTR_KIND_UW_TABLE;
973 case Attribute::VScaleRange:
974 return bitc::ATTR_KIND_VSCALE_RANGE;
975 case Attribute::WillReturn:
976 return bitc::ATTR_KIND_WILLRETURN;
977 case Attribute::WriteOnly:
978 return bitc::ATTR_KIND_WRITEONLY;
979 case Attribute::ZExt:
980 return bitc::ATTR_KIND_Z_EXT;
981 case Attribute::ImmArg:
982 return bitc::ATTR_KIND_IMMARG;
983 case Attribute::SanitizeMemTag:
984 return bitc::ATTR_KIND_SANITIZE_MEMTAG;
985 case Attribute::Preallocated:
986 return bitc::ATTR_KIND_PREALLOCATED;
987 case Attribute::NoUndef:
988 return bitc::ATTR_KIND_NOUNDEF;
989 case Attribute::ByRef:
990 return bitc::ATTR_KIND_BYREF;
991 case Attribute::MustProgress:
992 return bitc::ATTR_KIND_MUSTPROGRESS;
993 case Attribute::PresplitCoroutine:
994 return bitc::ATTR_KIND_PRESPLIT_COROUTINE;
995 case Attribute::Writable:
996 return bitc::ATTR_KIND_WRITABLE;
997 case Attribute::CoroDestroyOnlyWhenComplete:
998 return bitc::ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE;
999 case Attribute::CoroElideSafe:
1000 return bitc::ATTR_KIND_CORO_ELIDE_SAFE;
1001 case Attribute::DeadOnUnwind:
1002 return bitc::ATTR_KIND_DEAD_ON_UNWIND;
1003 case Attribute::Range:
1004 return bitc::ATTR_KIND_RANGE;
1005 case Attribute::Initializes:
1006 return bitc::ATTR_KIND_INITIALIZES;
1007 case Attribute::NoExt:
1008 return bitc::ATTR_KIND_NO_EXT;
1009 case Attribute::Captures:
1010 return bitc::ATTR_KIND_CAPTURES;
1011 case Attribute::DeadOnReturn:
1012 return bitc::ATTR_KIND_DEAD_ON_RETURN;
1013 case Attribute::NoCreateUndefOrPoison:
1014 return bitc::ATTR_KIND_NO_CREATE_UNDEF_OR_POISON;
1015 case Attribute::DenormalFPEnv:
1016 return bitc::ATTR_KIND_DENORMAL_FPENV;
1017 case Attribute::NoOutline:
1018 return bitc::ATTR_KIND_NOOUTLINE;
1019 case Attribute::NoIPA:
1020 return bitc::ATTR_KIND_NOIPA;
1021 case Attribute::EndAttrKinds:
1022 llvm_unreachable("Can not encode end-attribute kinds marker.");
1023 case Attribute::None:
1024 llvm_unreachable("Can not encode none-attribute.");
1025 case Attribute::EmptyKey:
1026 case Attribute::TombstoneKey:
1027 llvm_unreachable("Trying to encode EmptyKey/TombstoneKey");
1028 }
1029
1030 llvm_unreachable("Trying to encode unknown attribute");
1031}
1032
1033static void emitSignedInt64(SmallVectorImpl<uint64_t> &Vals, uint64_t V) {
1034 if ((int64_t)V >= 0)
1035 Vals.push_back(Elt: V << 1);
1036 else
1037 Vals.push_back(Elt: (-V << 1) | 1);
1038}
1039
1040static void emitWideAPInt(SmallVectorImpl<uint64_t> &Vals, const APInt &A) {
1041 // We have an arbitrary precision integer value to write whose
1042 // bit width is > 64. However, in canonical unsigned integer
1043 // format it is likely that the high bits are going to be zero.
1044 // So, we only write the number of active words.
1045 unsigned NumWords = A.getActiveWords();
1046 const uint64_t *RawData = A.getRawData();
1047 for (unsigned i = 0; i < NumWords; i++)
1048 emitSignedInt64(Vals, V: RawData[i]);
1049}
1050
1051static void emitConstantRange(SmallVectorImpl<uint64_t> &Record,
1052 const ConstantRange &CR, bool EmitBitWidth) {
1053 unsigned BitWidth = CR.getBitWidth();
1054 if (EmitBitWidth)
1055 Record.push_back(Elt: BitWidth);
1056 if (BitWidth > 64) {
1057 Record.push_back(Elt: CR.getLower().getActiveWords() |
1058 (uint64_t(CR.getUpper().getActiveWords()) << 32));
1059 emitWideAPInt(Vals&: Record, A: CR.getLower());
1060 emitWideAPInt(Vals&: Record, A: CR.getUpper());
1061 } else {
1062 emitSignedInt64(Vals&: Record, V: CR.getLower().getSExtValue());
1063 emitSignedInt64(Vals&: Record, V: CR.getUpper().getSExtValue());
1064 }
1065}
1066
1067void ModuleBitcodeWriter::writeAttributeGroupTable() {
1068 const std::vector<ValueEnumerator::IndexAndAttrSet> &AttrGrps =
1069 VE.getAttributeGroups();
1070 if (AttrGrps.empty()) return;
1071
1072 Stream.EnterSubblock(BlockID: bitc::PARAMATTR_GROUP_BLOCK_ID, CodeLen: 3);
1073
1074 SmallVector<uint64_t, 64> Record;
1075 for (ValueEnumerator::IndexAndAttrSet Pair : AttrGrps) {
1076 unsigned AttrListIndex = Pair.first;
1077 AttributeSet AS = Pair.second;
1078 Record.push_back(Elt: VE.getAttributeGroupID(Group: Pair));
1079 Record.push_back(Elt: AttrListIndex);
1080
1081 for (Attribute Attr : AS) {
1082 if (Attr.isEnumAttribute()) {
1083 Record.push_back(Elt: 0);
1084 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1085 } else if (Attr.isIntAttribute()) {
1086 Record.push_back(Elt: 1);
1087 Attribute::AttrKind Kind = Attr.getKindAsEnum();
1088 Record.push_back(Elt: getAttrKindEncoding(Kind));
1089 if (Kind == Attribute::Memory) {
1090 // Version field for upgrading old memory effects.
1091 const uint64_t Version = 2;
1092 Record.push_back(Elt: (Version << 56) | Attr.getValueAsInt());
1093 } else {
1094 Record.push_back(Elt: Attr.getValueAsInt());
1095 }
1096 } else if (Attr.isStringAttribute()) {
1097 StringRef Kind = Attr.getKindAsString();
1098 StringRef Val = Attr.getValueAsString();
1099
1100 Record.push_back(Elt: Val.empty() ? 3 : 4);
1101 Record.append(in_start: Kind.begin(), in_end: Kind.end());
1102 Record.push_back(Elt: 0);
1103 if (!Val.empty()) {
1104 Record.append(in_start: Val.begin(), in_end: Val.end());
1105 Record.push_back(Elt: 0);
1106 }
1107 } else if (Attr.isTypeAttribute()) {
1108 Type *Ty = Attr.getValueAsType();
1109 Record.push_back(Elt: Ty ? 6 : 5);
1110 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1111 if (Ty)
1112 Record.push_back(Elt: VE.getTypeID(T: Attr.getValueAsType()));
1113 } else if (Attr.isConstantRangeAttribute()) {
1114 Record.push_back(Elt: 7);
1115 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1116 emitConstantRange(Record, CR: Attr.getValueAsConstantRange(),
1117 /*EmitBitWidth=*/true);
1118 } else {
1119 assert(Attr.isConstantRangeListAttribute());
1120 Record.push_back(Elt: 8);
1121 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1122 ArrayRef<ConstantRange> Val = Attr.getValueAsConstantRangeList();
1123 Record.push_back(Elt: Val.size());
1124 Record.push_back(Elt: Val[0].getBitWidth());
1125 for (auto &CR : Val)
1126 emitConstantRange(Record, CR, /*EmitBitWidth=*/false);
1127 }
1128 }
1129
1130 Stream.EmitRecord(Code: bitc::PARAMATTR_GRP_CODE_ENTRY, Vals: Record);
1131 Record.clear();
1132 }
1133
1134 Stream.ExitBlock();
1135}
1136
1137void ModuleBitcodeWriter::writeAttributeTable() {
1138 const std::vector<AttributeList> &Attrs = VE.getAttributeLists();
1139 if (Attrs.empty()) return;
1140
1141 Stream.EnterSubblock(BlockID: bitc::PARAMATTR_BLOCK_ID, CodeLen: 3);
1142
1143 SmallVector<uint64_t, 64> Record;
1144 for (const AttributeList &AL : Attrs) {
1145 for (unsigned i : AL.indexes()) {
1146 AttributeSet AS = AL.getAttributes(Index: i);
1147 if (AS.hasAttributes())
1148 Record.push_back(Elt: VE.getAttributeGroupID(Group: {i, AS}));
1149 }
1150
1151 Stream.EmitRecord(Code: bitc::PARAMATTR_CODE_ENTRY, Vals: Record);
1152 Record.clear();
1153 }
1154
1155 Stream.ExitBlock();
1156}
1157
1158/// WriteTypeTable - Write out the type table for a module.
1159void ModuleBitcodeWriter::writeTypeTable() {
1160 const ValueEnumerator::TypeList &TypeList = VE.getTypes();
1161
1162 Stream.EnterSubblock(BlockID: bitc::TYPE_BLOCK_ID_NEW, CodeLen: 4 /*count from # abbrevs */);
1163 SmallVector<uint64_t, 64> TypeVals;
1164
1165 uint64_t NumBits = VE.computeBitsRequiredForTypeIndices();
1166
1167 // Abbrev for TYPE_CODE_OPAQUE_POINTER.
1168 auto Abbv = std::make_shared<BitCodeAbbrev>();
1169 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_OPAQUE_POINTER));
1170 Abbv->Add(OpInfo: BitCodeAbbrevOp(0)); // Addrspace = 0
1171 unsigned OpaquePtrAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1172
1173 // Abbrev for TYPE_CODE_FUNCTION.
1174 Abbv = std::make_shared<BitCodeAbbrev>();
1175 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_FUNCTION));
1176 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isvararg
1177 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1178 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1179 unsigned FunctionAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1180
1181 // Abbrev for TYPE_CODE_STRUCT_ANON.
1182 Abbv = std::make_shared<BitCodeAbbrev>();
1183 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_ANON));
1184 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked
1185 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1186 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1187 unsigned StructAnonAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1188
1189 // Abbrev for TYPE_CODE_STRUCT_NAME.
1190 Abbv = std::make_shared<BitCodeAbbrev>();
1191 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAME));
1192 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1193 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
1194 unsigned StructNameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1195
1196 // Abbrev for TYPE_CODE_STRUCT_NAMED.
1197 Abbv = std::make_shared<BitCodeAbbrev>();
1198 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAMED));
1199 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked
1200 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1201 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1202 unsigned StructNamedAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1203
1204 // Abbrev for TYPE_CODE_ARRAY.
1205 Abbv = std::make_shared<BitCodeAbbrev>();
1206 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_ARRAY));
1207 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // size
1208 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1209 unsigned ArrayAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1210
1211 // Emit an entry count so the reader can reserve space.
1212 TypeVals.push_back(Elt: TypeList.size());
1213 Stream.EmitRecord(Code: bitc::TYPE_CODE_NUMENTRY, Vals: TypeVals);
1214 TypeVals.clear();
1215
1216 // Loop over all of the types, emitting each in turn.
1217 for (Type *T : TypeList) {
1218 int AbbrevToUse = 0;
1219 unsigned Code = 0;
1220
1221 switch (T->getTypeID()) {
1222 case Type::VoidTyID: Code = bitc::TYPE_CODE_VOID; break;
1223 case Type::HalfTyID: Code = bitc::TYPE_CODE_HALF; break;
1224 case Type::BFloatTyID: Code = bitc::TYPE_CODE_BFLOAT; break;
1225 case Type::FloatTyID: Code = bitc::TYPE_CODE_FLOAT; break;
1226 case Type::DoubleTyID: Code = bitc::TYPE_CODE_DOUBLE; break;
1227 case Type::X86_FP80TyID: Code = bitc::TYPE_CODE_X86_FP80; break;
1228 case Type::FP128TyID: Code = bitc::TYPE_CODE_FP128; break;
1229 case Type::PPC_FP128TyID: Code = bitc::TYPE_CODE_PPC_FP128; break;
1230 case Type::LabelTyID: Code = bitc::TYPE_CODE_LABEL; break;
1231 case Type::MetadataTyID:
1232 Code = bitc::TYPE_CODE_METADATA;
1233 break;
1234 case Type::X86_AMXTyID: Code = bitc::TYPE_CODE_X86_AMX; break;
1235 case Type::TokenTyID: Code = bitc::TYPE_CODE_TOKEN; break;
1236 case Type::ByteTyID:
1237 // BYTE: [width]
1238 Code = bitc::TYPE_CODE_BYTE;
1239 TypeVals.push_back(Elt: T->getByteBitWidth());
1240 break;
1241 case Type::IntegerTyID:
1242 // INTEGER: [width]
1243 Code = bitc::TYPE_CODE_INTEGER;
1244 TypeVals.push_back(Elt: cast<IntegerType>(Val: T)->getBitWidth());
1245 break;
1246 case Type::PointerTyID: {
1247 PointerType *PTy = cast<PointerType>(Val: T);
1248 unsigned AddressSpace = PTy->getAddressSpace();
1249 // OPAQUE_POINTER: [address space]
1250 Code = bitc::TYPE_CODE_OPAQUE_POINTER;
1251 TypeVals.push_back(Elt: AddressSpace);
1252 if (AddressSpace == 0)
1253 AbbrevToUse = OpaquePtrAbbrev;
1254 break;
1255 }
1256 case Type::FunctionTyID: {
1257 FunctionType *FT = cast<FunctionType>(Val: T);
1258 // FUNCTION: [isvararg, retty, paramty x N]
1259 Code = bitc::TYPE_CODE_FUNCTION;
1260 TypeVals.push_back(Elt: FT->isVarArg());
1261 TypeVals.push_back(Elt: VE.getTypeID(T: FT->getReturnType()));
1262 for (unsigned i = 0, e = FT->getNumParams(); i != e; ++i)
1263 TypeVals.push_back(Elt: VE.getTypeID(T: FT->getParamType(i)));
1264 AbbrevToUse = FunctionAbbrev;
1265 break;
1266 }
1267 case Type::StructTyID: {
1268 StructType *ST = cast<StructType>(Val: T);
1269 // STRUCT: [ispacked, eltty x N]
1270 TypeVals.push_back(Elt: ST->isPacked());
1271 // Output all of the element types.
1272 for (Type *ET : ST->elements())
1273 TypeVals.push_back(Elt: VE.getTypeID(T: ET));
1274
1275 if (ST->isLiteral()) {
1276 Code = bitc::TYPE_CODE_STRUCT_ANON;
1277 AbbrevToUse = StructAnonAbbrev;
1278 } else {
1279 if (ST->isOpaque()) {
1280 Code = bitc::TYPE_CODE_OPAQUE;
1281 } else {
1282 Code = bitc::TYPE_CODE_STRUCT_NAMED;
1283 AbbrevToUse = StructNamedAbbrev;
1284 }
1285
1286 // Emit the name if it is present.
1287 if (!ST->getName().empty())
1288 writeStringRecord(Stream, Code: bitc::TYPE_CODE_STRUCT_NAME, Str: ST->getName(),
1289 AbbrevToUse: StructNameAbbrev);
1290 }
1291 break;
1292 }
1293 case Type::ArrayTyID: {
1294 ArrayType *AT = cast<ArrayType>(Val: T);
1295 // ARRAY: [numelts, eltty]
1296 Code = bitc::TYPE_CODE_ARRAY;
1297 TypeVals.push_back(Elt: AT->getNumElements());
1298 TypeVals.push_back(Elt: VE.getTypeID(T: AT->getElementType()));
1299 AbbrevToUse = ArrayAbbrev;
1300 break;
1301 }
1302 case Type::FixedVectorTyID:
1303 case Type::ScalableVectorTyID: {
1304 VectorType *VT = cast<VectorType>(Val: T);
1305 // VECTOR [numelts, eltty] or
1306 // [numelts, eltty, scalable]
1307 Code = bitc::TYPE_CODE_VECTOR;
1308 TypeVals.push_back(Elt: VT->getElementCount().getKnownMinValue());
1309 TypeVals.push_back(Elt: VE.getTypeID(T: VT->getElementType()));
1310 if (isa<ScalableVectorType>(Val: VT))
1311 TypeVals.push_back(Elt: true);
1312 break;
1313 }
1314 case Type::TargetExtTyID: {
1315 TargetExtType *TET = cast<TargetExtType>(Val: T);
1316 Code = bitc::TYPE_CODE_TARGET_TYPE;
1317 writeStringRecord(Stream, Code: bitc::TYPE_CODE_STRUCT_NAME, Str: TET->getName(),
1318 AbbrevToUse: StructNameAbbrev);
1319 TypeVals.push_back(Elt: TET->getNumTypeParameters());
1320 for (Type *InnerTy : TET->type_params())
1321 TypeVals.push_back(Elt: VE.getTypeID(T: InnerTy));
1322 llvm::append_range(C&: TypeVals, R: TET->int_params());
1323 break;
1324 }
1325 case Type::TypedPointerTyID:
1326 llvm_unreachable("Typed pointers cannot be added to IR modules");
1327 }
1328
1329 // Emit the finished record.
1330 Stream.EmitRecord(Code, Vals: TypeVals, Abbrev: AbbrevToUse);
1331 TypeVals.clear();
1332 }
1333
1334 Stream.ExitBlock();
1335}
1336
1337static unsigned getEncodedLinkage(const GlobalValue::LinkageTypes Linkage) {
1338 switch (Linkage) {
1339 case GlobalValue::ExternalLinkage:
1340 return 0;
1341 case GlobalValue::WeakAnyLinkage:
1342 return 16;
1343 case GlobalValue::AppendingLinkage:
1344 return 2;
1345 case GlobalValue::InternalLinkage:
1346 return 3;
1347 case GlobalValue::LinkOnceAnyLinkage:
1348 return 18;
1349 case GlobalValue::ExternalWeakLinkage:
1350 return 7;
1351 case GlobalValue::CommonLinkage:
1352 return 8;
1353 case GlobalValue::PrivateLinkage:
1354 return 9;
1355 case GlobalValue::WeakODRLinkage:
1356 return 17;
1357 case GlobalValue::LinkOnceODRLinkage:
1358 return 19;
1359 case GlobalValue::AvailableExternallyLinkage:
1360 return 12;
1361 }
1362 llvm_unreachable("Invalid linkage");
1363}
1364
1365static unsigned getEncodedLinkage(const GlobalValue &GV) {
1366 return getEncodedLinkage(Linkage: GV.getLinkage());
1367}
1368
1369static uint64_t getEncodedFFlags(FunctionSummary::FFlags Flags) {
1370 uint64_t RawFlags = 0;
1371 RawFlags |= Flags.ReadNone;
1372 RawFlags |= (Flags.ReadOnly << 1);
1373 RawFlags |= (Flags.NoRecurse << 2);
1374 RawFlags |= (Flags.ReturnDoesNotAlias << 3);
1375 RawFlags |= (Flags.NoInline << 4);
1376 RawFlags |= (Flags.AlwaysInline << 5);
1377 RawFlags |= (Flags.NoUnwind << 6);
1378 RawFlags |= (Flags.MayThrow << 7);
1379 RawFlags |= (Flags.HasUnknownCall << 8);
1380 RawFlags |= (Flags.MustBeUnreachable << 9);
1381 return RawFlags;
1382}
1383
1384// Decode the flags for GlobalValue in the summary. See getDecodedGVSummaryFlags
1385// in BitcodeReader.cpp.
1386static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags,
1387 bool ImportAsDecl = false) {
1388 uint64_t RawFlags = 0;
1389
1390 RawFlags |= Flags.NotEligibleToImport; // bool
1391 RawFlags |= (Flags.Live << 1);
1392 RawFlags |= (Flags.DSOLocal << 2);
1393 RawFlags |= (Flags.CanAutoHide << 3);
1394
1395 // Linkage don't need to be remapped at that time for the summary. Any future
1396 // change to the getEncodedLinkage() function will need to be taken into
1397 // account here as well.
1398 RawFlags = (RawFlags << 4) | Flags.Linkage; // 4 bits
1399
1400 RawFlags |= (Flags.Visibility << 8); // 2 bits
1401
1402 unsigned ImportType = Flags.ImportType | ImportAsDecl;
1403 RawFlags |= (ImportType << 10); // 1 bit
1404
1405 RawFlags |= (Flags.NoRenameOnPromotion << 11); // 1 bit
1406
1407 return RawFlags;
1408}
1409
1410static uint64_t getEncodedGVarFlags(GlobalVarSummary::GVarFlags Flags) {
1411 uint64_t RawFlags = Flags.MaybeReadOnly | (Flags.MaybeWriteOnly << 1) |
1412 (Flags.Constant << 2) | Flags.VCallVisibility << 3;
1413 return RawFlags;
1414}
1415
1416static uint64_t getEncodedHotnessCallEdgeInfo(const CalleeInfo &CI) {
1417 uint64_t RawFlags = 0;
1418
1419 RawFlags |= CI.Hotness; // 3 bits
1420 RawFlags |= (CI.HasTailCall << 3); // 1 bit
1421
1422 return RawFlags;
1423}
1424
1425static unsigned getEncodedVisibility(const GlobalValue &GV) {
1426 switch (GV.getVisibility()) {
1427 case GlobalValue::DefaultVisibility: return 0;
1428 case GlobalValue::HiddenVisibility: return 1;
1429 case GlobalValue::ProtectedVisibility: return 2;
1430 }
1431 llvm_unreachable("Invalid visibility");
1432}
1433
1434static unsigned getEncodedDLLStorageClass(const GlobalValue &GV) {
1435 switch (GV.getDLLStorageClass()) {
1436 case GlobalValue::DefaultStorageClass: return 0;
1437 case GlobalValue::DLLImportStorageClass: return 1;
1438 case GlobalValue::DLLExportStorageClass: return 2;
1439 }
1440 llvm_unreachable("Invalid DLL storage class");
1441}
1442
1443static unsigned getEncodedThreadLocalMode(const GlobalValue &GV) {
1444 switch (GV.getThreadLocalMode()) {
1445 case GlobalVariable::NotThreadLocal: return 0;
1446 case GlobalVariable::GeneralDynamicTLSModel: return 1;
1447 case GlobalVariable::LocalDynamicTLSModel: return 2;
1448 case GlobalVariable::InitialExecTLSModel: return 3;
1449 case GlobalVariable::LocalExecTLSModel: return 4;
1450 }
1451 llvm_unreachable("Invalid TLS model");
1452}
1453
1454static unsigned getEncodedComdatSelectionKind(const Comdat &C) {
1455 switch (C.getSelectionKind()) {
1456 case Comdat::Any:
1457 return bitc::COMDAT_SELECTION_KIND_ANY;
1458 case Comdat::ExactMatch:
1459 return bitc::COMDAT_SELECTION_KIND_EXACT_MATCH;
1460 case Comdat::Largest:
1461 return bitc::COMDAT_SELECTION_KIND_LARGEST;
1462 case Comdat::NoDeduplicate:
1463 return bitc::COMDAT_SELECTION_KIND_NO_DUPLICATES;
1464 case Comdat::SameSize:
1465 return bitc::COMDAT_SELECTION_KIND_SAME_SIZE;
1466 }
1467 llvm_unreachable("Invalid selection kind");
1468}
1469
1470static unsigned getEncodedUnnamedAddr(const GlobalValue &GV) {
1471 switch (GV.getUnnamedAddr()) {
1472 case GlobalValue::UnnamedAddr::None: return 0;
1473 case GlobalValue::UnnamedAddr::Local: return 2;
1474 case GlobalValue::UnnamedAddr::Global: return 1;
1475 }
1476 llvm_unreachable("Invalid unnamed_addr");
1477}
1478
1479size_t ModuleBitcodeWriter::addToStrtab(StringRef Str) {
1480 if (GenerateHash)
1481 Hasher.update(Str);
1482 return StrtabBuilder.add(S: Str);
1483}
1484
1485void ModuleBitcodeWriter::writeComdats() {
1486 SmallVector<unsigned, 64> Vals;
1487 for (const Comdat *C : VE.getComdats()) {
1488 // COMDAT: [strtab offset, strtab size, selection_kind]
1489 Vals.push_back(Elt: addToStrtab(Str: C->getName()));
1490 Vals.push_back(Elt: C->getName().size());
1491 Vals.push_back(Elt: getEncodedComdatSelectionKind(C: *C));
1492 Stream.EmitRecord(Code: bitc::MODULE_CODE_COMDAT, Vals, /*AbbrevToUse=*/Abbrev: 0);
1493 Vals.clear();
1494 }
1495}
1496
1497/// Write a record that will eventually hold the word offset of the
1498/// module-level VST. For now the offset is 0, which will be backpatched
1499/// after the real VST is written. Saves the bit offset to backpatch.
1500void ModuleBitcodeWriter::writeValueSymbolTableForwardDecl() {
1501 // Write a placeholder value in for the offset of the real VST,
1502 // which is written after the function blocks so that it can include
1503 // the offset of each function. The placeholder offset will be
1504 // updated when the real VST is written.
1505 auto Abbv = std::make_shared<BitCodeAbbrev>();
1506 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_VSTOFFSET));
1507 // Blocks are 32-bit aligned, so we can use a 32-bit word offset to
1508 // hold the real VST offset. Must use fixed instead of VBR as we don't
1509 // know how many VBR chunks to reserve ahead of time.
1510 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
1511 unsigned VSTOffsetAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1512
1513 // Emit the placeholder
1514 uint64_t Vals[] = {bitc::MODULE_CODE_VSTOFFSET, 0};
1515 Stream.EmitRecordWithAbbrev(Abbrev: VSTOffsetAbbrev, Vals);
1516
1517 // Compute and save the bit offset to the placeholder, which will be
1518 // patched when the real VST is written. We can simply subtract the 32-bit
1519 // fixed size from the current bit number to get the location to backpatch.
1520 VSTOffsetPlaceholder = Stream.GetCurrentBitNo() - 32;
1521}
1522
1523enum StringEncoding { SE_Char6, SE_Fixed7, SE_Fixed8 };
1524
1525/// Determine the encoding to use for the given string name and length.
1526static StringEncoding getStringEncoding(StringRef Str) {
1527 bool isChar6 = true;
1528 for (char C : Str) {
1529 if (isChar6)
1530 isChar6 = BitCodeAbbrevOp::isChar6(C);
1531 if ((unsigned char)C & 128)
1532 // don't bother scanning the rest.
1533 return SE_Fixed8;
1534 }
1535 if (isChar6)
1536 return SE_Char6;
1537 return SE_Fixed7;
1538}
1539
1540static_assert(sizeof(GlobalValue::SanitizerMetadata) <= sizeof(unsigned),
1541 "Sanitizer Metadata is too large for naive serialization.");
1542static unsigned
1543serializeSanitizerMetadata(const GlobalValue::SanitizerMetadata &Meta) {
1544 return Meta.NoAddress | (Meta.NoHWAddress << 1) |
1545 (Meta.Memtag << 2) | (Meta.IsDynInit << 3);
1546}
1547
1548/// Emit top-level description of module, including target triple, inline asm,
1549/// descriptors for global variables, and function prototype info.
1550/// Returns the bit offset to backpatch with the location of the real VST.
1551void ModuleBitcodeWriter::writeModuleInfo() {
1552 // Emit various pieces of data attached to a module.
1553 if (!M.getTargetTriple().empty())
1554 writeStringRecord(Stream, Code: bitc::MODULE_CODE_TRIPLE,
1555 Str: M.getTargetTriple().str(), AbbrevToUse: 0 /*TODO*/);
1556 const std::string &DL = M.getDataLayoutStr();
1557 if (!DL.empty())
1558 writeStringRecord(Stream, Code: bitc::MODULE_CODE_DATALAYOUT, Str: DL, AbbrevToUse: 0 /*TODO*/);
1559
1560 for (const Module::GlobalAsmFragment &Frag : M.getModuleInlineAsm()) {
1561 SmallVector<std::pair<StringRef, StringRef>> Props =
1562 Frag.Props.getAsStrings();
1563 for (auto [Key, Value] : Props) {
1564 SmallVector<unsigned, 64> Record;
1565 Record.append(in_start: Key.begin(), in_end: Key.end());
1566 Record.push_back(Elt: 0);
1567 Record.append(in_start: Value.begin(), in_end: Value.end());
1568 Stream.EmitRecord(Code: bitc::MODULE_CODE_ASM_PROPERTY, Vals: Record);
1569 }
1570 writeStringRecord(Stream, Code: bitc::MODULE_CODE_ASM, Str: Frag.Asm, AbbrevToUse: 0 /*TODO*/);
1571 }
1572
1573 // Emit information about sections and GC, computing how many there are. Also
1574 // compute the maximum alignment value.
1575 std::map<std::string, unsigned> SectionMap;
1576 std::map<std::string, unsigned> GCMap;
1577 MaybeAlign MaxGVarAlignment;
1578 unsigned MaxGlobalType = 0;
1579 for (const GlobalVariable &GV : M.globals()) {
1580 if (MaybeAlign A = GV.getAlign())
1581 MaxGVarAlignment = !MaxGVarAlignment ? *A : std::max(a: *MaxGVarAlignment, b: *A);
1582 MaxGlobalType = std::max(a: MaxGlobalType, b: VE.getTypeID(T: GV.getValueType()));
1583 if (GV.hasSection()) {
1584 // Give section names unique ID's.
1585 unsigned &Entry = SectionMap[std::string(GV.getSection())];
1586 if (!Entry) {
1587 writeStringRecord(Stream, Code: bitc::MODULE_CODE_SECTIONNAME, Str: GV.getSection(),
1588 AbbrevToUse: 0 /*TODO*/);
1589 Entry = SectionMap.size();
1590 }
1591 }
1592 }
1593 for (const Function &F : M) {
1594 if (F.hasSection()) {
1595 // Give section names unique ID's.
1596 unsigned &Entry = SectionMap[std::string(F.getSection())];
1597 if (!Entry) {
1598 writeStringRecord(Stream, Code: bitc::MODULE_CODE_SECTIONNAME, Str: F.getSection(),
1599 AbbrevToUse: 0 /*TODO*/);
1600 Entry = SectionMap.size();
1601 }
1602 }
1603 if (F.hasGC()) {
1604 // Same for GC names.
1605 unsigned &Entry = GCMap[F.getGC()];
1606 if (!Entry) {
1607 writeStringRecord(Stream, Code: bitc::MODULE_CODE_GCNAME, Str: F.getGC(),
1608 AbbrevToUse: 0 /*TODO*/);
1609 Entry = GCMap.size();
1610 }
1611 }
1612 }
1613
1614 // Emit abbrev for globals, now that we know # sections and max alignment.
1615 unsigned SimpleGVarAbbrev = 0;
1616 if (!M.global_empty()) {
1617 // Add an abbrev for common globals with no visibility or thread localness.
1618 auto Abbv = std::make_shared<BitCodeAbbrev>();
1619 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_GLOBALVAR));
1620 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1621 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1622 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1623 Log2_32_Ceil(Value: MaxGlobalType+1)));
1624 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // AddrSpace << 2
1625 //| explicitType << 1
1626 //| constant
1627 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Initializer.
1628 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // Linkage.
1629 if (!MaxGVarAlignment) // Alignment.
1630 Abbv->Add(OpInfo: BitCodeAbbrevOp(0));
1631 else {
1632 unsigned MaxEncAlignment = getEncodedAlign(Alignment: MaxGVarAlignment);
1633 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1634 Log2_32_Ceil(Value: MaxEncAlignment+1)));
1635 }
1636 if (SectionMap.empty()) // Section.
1637 Abbv->Add(OpInfo: BitCodeAbbrevOp(0));
1638 else
1639 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1640 Log2_32_Ceil(Value: SectionMap.size()+1)));
1641 // Don't bother emitting vis + thread local.
1642 SimpleGVarAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1643 }
1644
1645 SmallVector<unsigned, 64> Vals;
1646 // Emit the module's source file name.
1647 {
1648 StringEncoding Bits = getStringEncoding(Str: M.getSourceFileName());
1649 BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
1650 if (Bits == SE_Char6)
1651 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
1652 else if (Bits == SE_Fixed7)
1653 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
1654
1655 // MODULE_CODE_SOURCE_FILENAME: [namechar x N]
1656 auto Abbv = std::make_shared<BitCodeAbbrev>();
1657 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
1658 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1659 Abbv->Add(OpInfo: AbbrevOpToUse);
1660 unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1661
1662 for (const auto P : M.getSourceFileName())
1663 Vals.push_back(Elt: (unsigned char)P);
1664
1665 // Emit the finished record.
1666 Stream.EmitRecord(Code: bitc::MODULE_CODE_SOURCE_FILENAME, Vals, Abbrev: FilenameAbbrev);
1667 Vals.clear();
1668 }
1669
1670 writeGUIDList();
1671
1672 // Emit the global variable information.
1673 for (const GlobalVariable &GV : M.globals()) {
1674 unsigned AbbrevToUse = 0;
1675
1676 // GLOBALVAR: [strtab offset, strtab size, type, isconst, initid,
1677 // linkage, alignment, section, visibility, threadlocal,
1678 // unnamed_addr, externally_initialized, dllstorageclass,
1679 // comdat, attributes, DSO_Local, GlobalSanitizer, code_model]
1680 Vals.push_back(Elt: addToStrtab(Str: GV.getName()));
1681 Vals.push_back(Elt: GV.getName().size());
1682 Vals.push_back(Elt: VE.getTypeID(T: GV.getValueType()));
1683 Vals.push_back(Elt: GV.getType()->getAddressSpace() << 2 | 2 | GV.isConstant());
1684 Vals.push_back(Elt: GV.isDeclaration() ? 0 :
1685 (VE.getValueID(V: GV.getInitializer()) + 1));
1686 Vals.push_back(Elt: getEncodedLinkage(GV));
1687 Vals.push_back(Elt: getEncodedAlign(Alignment: GV.getAlign()));
1688 Vals.push_back(Elt: GV.hasSection() ? SectionMap[std::string(GV.getSection())]
1689 : 0);
1690 if (GV.isThreadLocal() ||
1691 GV.getVisibility() != GlobalValue::DefaultVisibility ||
1692 GV.getUnnamedAddr() != GlobalValue::UnnamedAddr::None ||
1693 GV.isExternallyInitialized() ||
1694 GV.getDLLStorageClass() != GlobalValue::DefaultStorageClass ||
1695 GV.hasComdat() || GV.hasAttributes() || GV.isDSOLocal() ||
1696 GV.hasPartition() || GV.hasSanitizerMetadata() || GV.getCodeModel()) {
1697 Vals.push_back(Elt: getEncodedVisibility(GV));
1698 Vals.push_back(Elt: getEncodedThreadLocalMode(GV));
1699 Vals.push_back(Elt: getEncodedUnnamedAddr(GV));
1700 Vals.push_back(Elt: GV.isExternallyInitialized());
1701 Vals.push_back(Elt: getEncodedDLLStorageClass(GV));
1702 Vals.push_back(Elt: GV.hasComdat() ? VE.getComdatID(C: GV.getComdat()) : 0);
1703
1704 auto AL = GV.getAttributesAsList(index: AttributeList::FunctionIndex);
1705 Vals.push_back(Elt: VE.getAttributeListID(PAL: AL));
1706
1707 Vals.push_back(Elt: GV.isDSOLocal());
1708 Vals.push_back(Elt: addToStrtab(Str: GV.getPartition()));
1709 Vals.push_back(Elt: GV.getPartition().size());
1710
1711 Vals.push_back(Elt: (GV.hasSanitizerMetadata() ? serializeSanitizerMetadata(
1712 Meta: GV.getSanitizerMetadata())
1713 : 0));
1714 Vals.push_back(Elt: GV.getCodeModelRaw());
1715 } else {
1716 AbbrevToUse = SimpleGVarAbbrev;
1717 }
1718
1719 Stream.EmitRecord(Code: bitc::MODULE_CODE_GLOBALVAR, Vals, Abbrev: AbbrevToUse);
1720 Vals.clear();
1721 }
1722
1723 // Emit the function proto information.
1724 for (const Function &F : M) {
1725 // FUNCTION: [strtab offset, strtab size, type, callingconv, isproto,
1726 // linkage, paramattrs, alignment, section, visibility, gc,
1727 // unnamed_addr, prologuedata, dllstorageclass, comdat,
1728 // prefixdata, personalityfn, DSO_Local, addrspace,
1729 // partition_strtab, partition_size, prefalign]
1730 Vals.push_back(Elt: addToStrtab(Str: F.getName()));
1731 Vals.push_back(Elt: F.getName().size());
1732 Vals.push_back(Elt: VE.getTypeID(T: F.getFunctionType()));
1733 Vals.push_back(Elt: F.getCallingConv());
1734 Vals.push_back(Elt: F.isDeclaration());
1735 Vals.push_back(Elt: getEncodedLinkage(GV: F));
1736 Vals.push_back(Elt: VE.getAttributeListID(PAL: F.getAttributes()));
1737 Vals.push_back(Elt: getEncodedAlign(Alignment: F.getAlign()));
1738 Vals.push_back(Elt: F.hasSection() ? SectionMap[std::string(F.getSection())]
1739 : 0);
1740 Vals.push_back(Elt: getEncodedVisibility(GV: F));
1741 Vals.push_back(Elt: F.hasGC() ? GCMap[F.getGC()] : 0);
1742 Vals.push_back(Elt: getEncodedUnnamedAddr(GV: F));
1743 Vals.push_back(Elt: F.hasPrologueData() ? (VE.getValueID(V: F.getPrologueData()) + 1)
1744 : 0);
1745 Vals.push_back(Elt: getEncodedDLLStorageClass(GV: F));
1746 Vals.push_back(Elt: F.hasComdat() ? VE.getComdatID(C: F.getComdat()) : 0);
1747 Vals.push_back(Elt: F.hasPrefixData() ? (VE.getValueID(V: F.getPrefixData()) + 1)
1748 : 0);
1749 Vals.push_back(
1750 Elt: F.hasPersonalityFn() ? (VE.getValueID(V: F.getPersonalityFn()) + 1) : 0);
1751
1752 Vals.push_back(Elt: F.isDSOLocal());
1753 Vals.push_back(Elt: F.getAddressSpace());
1754 Vals.push_back(Elt: addToStrtab(Str: F.getPartition()));
1755 Vals.push_back(Elt: F.getPartition().size());
1756 Vals.push_back(Elt: getEncodedAlign(Alignment: F.getPreferredAlignment()));
1757
1758 unsigned AbbrevToUse = 0;
1759 Stream.EmitRecord(Code: bitc::MODULE_CODE_FUNCTION, Vals, Abbrev: AbbrevToUse);
1760 Vals.clear();
1761 }
1762
1763 // Emit the alias information.
1764 for (const GlobalAlias &A : M.aliases()) {
1765 // ALIAS: [strtab offset, strtab size, alias type, aliasee val#, linkage,
1766 // visibility, dllstorageclass, threadlocal, unnamed_addr,
1767 // DSO_Local]
1768 Vals.push_back(Elt: addToStrtab(Str: A.getName()));
1769 Vals.push_back(Elt: A.getName().size());
1770 Vals.push_back(Elt: VE.getTypeID(T: A.getValueType()));
1771 Vals.push_back(Elt: A.getType()->getAddressSpace());
1772 Vals.push_back(Elt: VE.getValueID(V: A.getAliasee()));
1773 Vals.push_back(Elt: getEncodedLinkage(GV: A));
1774 Vals.push_back(Elt: getEncodedVisibility(GV: A));
1775 Vals.push_back(Elt: getEncodedDLLStorageClass(GV: A));
1776 Vals.push_back(Elt: getEncodedThreadLocalMode(GV: A));
1777 Vals.push_back(Elt: getEncodedUnnamedAddr(GV: A));
1778 Vals.push_back(Elt: A.isDSOLocal());
1779 Vals.push_back(Elt: addToStrtab(Str: A.getPartition()));
1780 Vals.push_back(Elt: A.getPartition().size());
1781
1782 unsigned AbbrevToUse = 0;
1783 Stream.EmitRecord(Code: bitc::MODULE_CODE_ALIAS, Vals, Abbrev: AbbrevToUse);
1784 Vals.clear();
1785 }
1786
1787 // Emit the ifunc information.
1788 for (const GlobalIFunc &I : M.ifuncs()) {
1789 // IFUNC: [strtab offset, strtab size, ifunc type, address space, resolver
1790 // val#, linkage, visibility, DSO_Local]
1791 Vals.push_back(Elt: addToStrtab(Str: I.getName()));
1792 Vals.push_back(Elt: I.getName().size());
1793 Vals.push_back(Elt: VE.getTypeID(T: I.getValueType()));
1794 Vals.push_back(Elt: I.getType()->getAddressSpace());
1795 Vals.push_back(Elt: VE.getValueID(V: I.getResolver()));
1796 Vals.push_back(Elt: getEncodedLinkage(GV: I));
1797 Vals.push_back(Elt: getEncodedVisibility(GV: I));
1798 Vals.push_back(Elt: I.isDSOLocal());
1799 Vals.push_back(Elt: addToStrtab(Str: I.getPartition()));
1800 Vals.push_back(Elt: I.getPartition().size());
1801 Stream.EmitRecord(Code: bitc::MODULE_CODE_IFUNC, Vals);
1802 Vals.clear();
1803 }
1804
1805 writeValueSymbolTableForwardDecl();
1806}
1807
1808static uint64_t getOptimizationFlags(const Value *V) {
1809 uint64_t Flags = 0;
1810
1811 if (const auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V)) {
1812 if (OBO->hasNoSignedWrap())
1813 Flags |= 1 << bitc::OBO_NO_SIGNED_WRAP;
1814 if (OBO->hasNoUnsignedWrap())
1815 Flags |= 1 << bitc::OBO_NO_UNSIGNED_WRAP;
1816 } else if (const auto *PEO = dyn_cast<PossiblyExactOperator>(Val: V)) {
1817 if (PEO->isExact())
1818 Flags |= 1 << bitc::PEO_EXACT;
1819 } else if (const auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: V)) {
1820 if (PDI->isDisjoint())
1821 Flags |= 1 << bitc::PDI_DISJOINT;
1822 } else if (const auto *FPMO = dyn_cast<FPMathOperator>(Val: V)) {
1823 if (FPMO->hasAllowReassoc())
1824 Flags |= bitc::AllowReassoc;
1825 if (FPMO->hasNoNaNs())
1826 Flags |= bitc::NoNaNs;
1827 if (FPMO->hasNoInfs())
1828 Flags |= bitc::NoInfs;
1829 if (FPMO->hasNoSignedZeros())
1830 Flags |= bitc::NoSignedZeros;
1831 if (FPMO->hasAllowReciprocal())
1832 Flags |= bitc::AllowReciprocal;
1833 if (FPMO->hasAllowContract())
1834 Flags |= bitc::AllowContract;
1835 if (FPMO->hasApproxFunc())
1836 Flags |= bitc::ApproxFunc;
1837
1838 // Handle uitofp.
1839 if (const auto *NNI = dyn_cast<PossiblyNonNegInst>(Val: V)) {
1840 Flags <<= 1;
1841 if (NNI->hasNonNeg())
1842 Flags |= 1 << bitc::PNNI_NON_NEG;
1843 }
1844 } else if (const auto *NNI = dyn_cast<PossiblyNonNegInst>(Val: V)) {
1845 if (NNI->hasNonNeg())
1846 Flags |= 1 << bitc::PNNI_NON_NEG;
1847 } else if (const auto *TI = dyn_cast<TruncInst>(Val: V)) {
1848 if (TI->hasNoSignedWrap())
1849 Flags |= 1 << bitc::TIO_NO_SIGNED_WRAP;
1850 if (TI->hasNoUnsignedWrap())
1851 Flags |= 1 << bitc::TIO_NO_UNSIGNED_WRAP;
1852 } else if (const auto *GEP = dyn_cast<GEPOperator>(Val: V)) {
1853 if (GEP->isInBounds())
1854 Flags |= 1 << bitc::GEP_INBOUNDS;
1855 if (GEP->hasNoUnsignedSignedWrap())
1856 Flags |= 1 << bitc::GEP_NUSW;
1857 if (GEP->hasNoUnsignedWrap())
1858 Flags |= 1 << bitc::GEP_NUW;
1859 } else if (const auto *ICmp = dyn_cast<ICmpInst>(Val: V)) {
1860 if (ICmp->hasSameSign())
1861 Flags |= 1 << bitc::ICMP_SAME_SIGN;
1862 } else if (const auto *ASC = dyn_cast<AddrSpaceCastInst>(Val: V)) {
1863 if (ASC->hasNonNull())
1864 Flags |= 1 << bitc::ASCI_NON_NULL;
1865 }
1866
1867 return Flags;
1868}
1869
1870void ModuleBitcodeWriter::writeValueAsMetadata(
1871 const ValueAsMetadata *MD, SmallVectorImpl<uint64_t> &Record) {
1872 // Mimic an MDNode with a value as one operand.
1873 Value *V = MD->getValue();
1874 Record.push_back(Elt: VE.getTypeID(T: V->getType()));
1875 Record.push_back(Elt: VE.getValueID(V));
1876 Stream.EmitRecord(Code: bitc::METADATA_VALUE, Vals: Record, Abbrev: 0);
1877 Record.clear();
1878}
1879
1880void ModuleBitcodeWriter::writeMDTuple(const MDTuple *N,
1881 SmallVectorImpl<uint64_t> &Record,
1882 unsigned Abbrev) {
1883 for (const MDOperand &MDO : N->operands()) {
1884 Metadata *MD = MDO;
1885 assert(!(MD && isa<LocalAsMetadata>(MD)) &&
1886 "Unexpected function-local metadata");
1887 Record.push_back(Elt: VE.getMetadataOrNullID(MD));
1888 }
1889 Stream.EmitRecord(Code: N->isDistinct() ? bitc::METADATA_DISTINCT_NODE
1890 : bitc::METADATA_NODE,
1891 Vals: Record, Abbrev);
1892 Record.clear();
1893}
1894
1895unsigned ModuleBitcodeWriter::createDILocationAbbrev() {
1896 // Assume the column is usually under 128, and always output the inlined-at
1897 // location (it's never more expensive than building an array size 1).
1898 auto Abbv = std::make_shared<BitCodeAbbrev>();
1899 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_LOCATION));
1900 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isDistinct
1901 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // line
1902 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // column
1903 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // scope
1904 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // inlinedAt
1905 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isImplicitCode
1906 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // atomGroup
1907 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // atomRank
1908 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
1909}
1910
1911void ModuleBitcodeWriter::writeDILocation(const DILocation *N,
1912 SmallVectorImpl<uint64_t> &Record,
1913 unsigned &Abbrev) {
1914 if (!Abbrev)
1915 Abbrev = createDILocationAbbrev();
1916
1917 Record.push_back(Elt: N->isDistinct());
1918 Record.push_back(Elt: N->getLine());
1919 Record.push_back(Elt: N->getColumn());
1920 Record.push_back(Elt: VE.getMetadataID(MD: N->getScope()));
1921 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getInlinedAt()));
1922 Record.push_back(Elt: N->isImplicitCode());
1923 Record.push_back(Elt: N->getAtomGroup());
1924 Record.push_back(Elt: N->getAtomRank());
1925 Stream.EmitRecord(Code: bitc::METADATA_LOCATION, Vals: Record, Abbrev);
1926 Record.clear();
1927}
1928
1929unsigned ModuleBitcodeWriter::createGenericDINodeAbbrev() {
1930 // Assume the column is usually under 128, and always output the inlined-at
1931 // location (it's never more expensive than building an array size 1).
1932 auto Abbv = std::make_shared<BitCodeAbbrev>();
1933 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_GENERIC_DEBUG));
1934 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1935 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1936 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1937 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1938 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1939 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1940 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
1941}
1942
1943void ModuleBitcodeWriter::writeGenericDINode(const GenericDINode *N,
1944 SmallVectorImpl<uint64_t> &Record,
1945 unsigned &Abbrev) {
1946 if (!Abbrev)
1947 Abbrev = createGenericDINodeAbbrev();
1948
1949 Record.push_back(Elt: N->isDistinct());
1950 Record.push_back(Elt: N->getTag());
1951 Record.push_back(Elt: 0); // Per-tag version field; unused for now.
1952
1953 for (auto &I : N->operands())
1954 Record.push_back(Elt: VE.getMetadataOrNullID(MD: I));
1955
1956 Stream.EmitRecord(Code: bitc::METADATA_GENERIC_DEBUG, Vals: Record, Abbrev);
1957 Record.clear();
1958}
1959
1960void ModuleBitcodeWriter::writeDISubrange(const DISubrange *N,
1961 SmallVectorImpl<uint64_t> &Record,
1962 unsigned Abbrev) {
1963 const uint64_t Version = 2 << 1;
1964 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
1965 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawCountNode()));
1966 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
1967 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
1968 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
1969
1970 Stream.EmitRecord(Code: bitc::METADATA_SUBRANGE, Vals: Record, Abbrev);
1971 Record.clear();
1972}
1973
1974void ModuleBitcodeWriter::writeDIGenericSubrange(
1975 const DIGenericSubrange *N, SmallVectorImpl<uint64_t> &Record,
1976 unsigned Abbrev) {
1977 Record.push_back(Elt: (uint64_t)N->isDistinct());
1978 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawCountNode()));
1979 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
1980 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
1981 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
1982
1983 Stream.EmitRecord(Code: bitc::METADATA_GENERIC_SUBRANGE, Vals: Record, Abbrev);
1984 Record.clear();
1985}
1986
1987void ModuleBitcodeWriter::writeDIEnumerator(const DIEnumerator *N,
1988 SmallVectorImpl<uint64_t> &Record,
1989 unsigned Abbrev) {
1990 const uint64_t IsBigInt = 1 << 2;
1991 Record.push_back(Elt: IsBigInt | (N->isUnsigned() << 1) | N->isDistinct());
1992 Record.push_back(Elt: N->getValue().getBitWidth());
1993 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1994 emitWideAPInt(Vals&: Record, A: N->getValue());
1995
1996 Stream.EmitRecord(Code: bitc::METADATA_ENUMERATOR, Vals: Record, Abbrev);
1997 Record.clear();
1998}
1999
2000void ModuleBitcodeWriter::writeDIBasicType(const DIBasicType *N,
2001 SmallVectorImpl<uint64_t> &Record,
2002 unsigned Abbrev) {
2003 const unsigned SizeIsMetadata = 0x2;
2004 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2005 Record.push_back(Elt: N->getTag());
2006 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2007 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2008 Record.push_back(Elt: N->getAlignInBits());
2009 Record.push_back(Elt: N->getEncoding());
2010 Record.push_back(Elt: N->getFlags());
2011 Record.push_back(Elt: N->getNumExtraInhabitants());
2012 Record.push_back(Elt: N->getDataSizeInBits());
2013 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2014 Record.push_back(Elt: N->getLine());
2015 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2016
2017 Stream.EmitRecord(Code: bitc::METADATA_BASIC_TYPE, Vals: Record, Abbrev);
2018 Record.clear();
2019}
2020
2021void ModuleBitcodeWriter::writeDIFixedPointType(
2022 const DIFixedPointType *N, SmallVectorImpl<uint64_t> &Record,
2023 unsigned Abbrev) {
2024 const unsigned SizeIsMetadata = 0x2;
2025 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2026 Record.push_back(Elt: N->getTag());
2027 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2028 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2029 Record.push_back(Elt: N->getAlignInBits());
2030 Record.push_back(Elt: N->getEncoding());
2031 Record.push_back(Elt: N->getFlags());
2032 Record.push_back(Elt: N->getKind());
2033 Record.push_back(Elt: N->getFactorRaw());
2034
2035 auto WriteWideInt = [&](const APInt &Value) {
2036 // Write an encoded word that holds the number of active words and
2037 // the number of bits.
2038 uint64_t NumWords = Value.getActiveWords();
2039 uint64_t Encoded = (NumWords << 32) | Value.getBitWidth();
2040 Record.push_back(Elt: Encoded);
2041 emitWideAPInt(Vals&: Record, A: Value);
2042 };
2043
2044 WriteWideInt(N->getNumeratorRaw());
2045 WriteWideInt(N->getDenominatorRaw());
2046
2047 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2048 Record.push_back(Elt: N->getLine());
2049 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2050
2051 Stream.EmitRecord(Code: bitc::METADATA_FIXED_POINT_TYPE, Vals: Record, Abbrev);
2052 Record.clear();
2053}
2054
2055void ModuleBitcodeWriter::writeDIStringType(const DIStringType *N,
2056 SmallVectorImpl<uint64_t> &Record,
2057 unsigned Abbrev) {
2058 const unsigned SizeIsMetadata = 0x2;
2059 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2060 Record.push_back(Elt: N->getTag());
2061 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2062 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLength()));
2063 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLengthExp()));
2064 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLocationExp()));
2065 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2066 Record.push_back(Elt: N->getAlignInBits());
2067 Record.push_back(Elt: N->getEncoding());
2068
2069 Stream.EmitRecord(Code: bitc::METADATA_STRING_TYPE, Vals: Record, Abbrev);
2070 Record.clear();
2071}
2072
2073void ModuleBitcodeWriter::writeDIDerivedType(const DIDerivedType *N,
2074 SmallVectorImpl<uint64_t> &Record,
2075 unsigned Abbrev) {
2076 const unsigned SizeIsMetadata = 0x2;
2077 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2078 Record.push_back(Elt: N->getTag());
2079 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2080 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2081 Record.push_back(Elt: N->getLine());
2082 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2083 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2084 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2085 Record.push_back(Elt: N->getAlignInBits());
2086 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawOffsetInBits()));
2087 Record.push_back(Elt: N->getFlags());
2088 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getExtraData()));
2089
2090 // DWARF address space is encoded as N->getDWARFAddressSpace() + 1. 0 means
2091 // that there is no DWARF address space associated with DIDerivedType.
2092 if (const auto &DWARFAddressSpace = N->getDWARFAddressSpace())
2093 Record.push_back(Elt: *DWARFAddressSpace + 1);
2094 else
2095 Record.push_back(Elt: 0);
2096
2097 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2098
2099 if (auto PtrAuthData = N->getPtrAuthData())
2100 Record.push_back(Elt: PtrAuthData->RawData);
2101 else
2102 Record.push_back(Elt: 0);
2103
2104 Stream.EmitRecord(Code: bitc::METADATA_DERIVED_TYPE, Vals: Record, Abbrev);
2105 Record.clear();
2106}
2107
2108void ModuleBitcodeWriter::writeDISubrangeType(const DISubrangeType *N,
2109 SmallVectorImpl<uint64_t> &Record,
2110 unsigned Abbrev) {
2111 const unsigned SizeIsMetadata = 0x2;
2112 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2113 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2114 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2115 Record.push_back(Elt: N->getLine());
2116 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2117 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2118 Record.push_back(Elt: N->getAlignInBits());
2119 Record.push_back(Elt: N->getFlags());
2120 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2121 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
2122 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
2123 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
2124 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawBias()));
2125
2126 Stream.EmitRecord(Code: bitc::METADATA_SUBRANGE_TYPE, Vals: Record, Abbrev);
2127 Record.clear();
2128}
2129
2130void ModuleBitcodeWriter::writeDICompositeType(
2131 const DICompositeType *N, SmallVectorImpl<uint64_t> &Record,
2132 unsigned Abbrev) {
2133 const unsigned IsNotUsedInOldTypeRef = 0x2;
2134 const unsigned SizeIsMetadata = 0x4;
2135 Record.push_back(Elt: SizeIsMetadata | IsNotUsedInOldTypeRef |
2136 (unsigned)N->isDistinct());
2137 Record.push_back(Elt: N->getTag());
2138 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2139 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2140 Record.push_back(Elt: N->getLine());
2141 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2142 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2143 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2144 Record.push_back(Elt: N->getAlignInBits());
2145 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawOffsetInBits()));
2146 Record.push_back(Elt: N->getFlags());
2147 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2148 Record.push_back(Elt: N->getRuntimeLang());
2149 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getVTableHolder()));
2150 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams().get()));
2151 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawIdentifier()));
2152 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDiscriminator()));
2153 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawDataLocation()));
2154 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawAssociated()));
2155 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawAllocated()));
2156 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawRank()));
2157 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2158 Record.push_back(Elt: N->getNumExtraInhabitants());
2159 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSpecification()));
2160 Record.push_back(
2161 Elt: N->getEnumKind().value_or(u: dwarf::DW_APPLE_ENUM_KIND_invalid));
2162 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawBitStride()));
2163
2164 Stream.EmitRecord(Code: bitc::METADATA_COMPOSITE_TYPE, Vals: Record, Abbrev);
2165 Record.clear();
2166}
2167
2168void ModuleBitcodeWriter::writeDISubroutineType(
2169 const DISubroutineType *N, SmallVectorImpl<uint64_t> &Record,
2170 unsigned Abbrev) {
2171 const unsigned HasNoOldTypeRefs = 0x2;
2172 Record.push_back(Elt: HasNoOldTypeRefs | (unsigned)N->isDistinct());
2173 Record.push_back(Elt: N->getFlags());
2174 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTypeArray().get()));
2175 Record.push_back(Elt: N->getCC());
2176
2177 Stream.EmitRecord(Code: bitc::METADATA_SUBROUTINE_TYPE, Vals: Record, Abbrev);
2178 Record.clear();
2179}
2180
2181void ModuleBitcodeWriter::writeDIFile(const DIFile *N,
2182 SmallVectorImpl<uint64_t> &Record,
2183 unsigned Abbrev) {
2184 Record.push_back(Elt: N->isDistinct());
2185 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFilename()));
2186 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawDirectory()));
2187 if (N->getRawChecksum()) {
2188 Record.push_back(Elt: N->getRawChecksum()->Kind);
2189 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawChecksum()->Value));
2190 } else {
2191 // Maintain backwards compatibility with the old internal representation of
2192 // CSK_None in ChecksumKind by writing nulls here when Checksum is None.
2193 Record.push_back(Elt: 0);
2194 Record.push_back(Elt: VE.getMetadataOrNullID(MD: nullptr));
2195 }
2196 auto Source = N->getRawSource();
2197 if (Source)
2198 Record.push_back(Elt: VE.getMetadataOrNullID(MD: Source));
2199
2200 Stream.EmitRecord(Code: bitc::METADATA_FILE, Vals: Record, Abbrev);
2201 Record.clear();
2202}
2203
2204void ModuleBitcodeWriter::writeDICompileUnit(const DICompileUnit *N,
2205 SmallVectorImpl<uint64_t> &Record,
2206 unsigned Abbrev) {
2207 assert(N->isDistinct() && "Expected distinct compile units");
2208 Record.push_back(/* IsDistinct */ Elt: true);
2209
2210 auto Lang = N->getSourceLanguage();
2211 Record.push_back(Elt: Lang.getName());
2212 // Set bit so the MetadataLoader can distniguish between versioned and
2213 // unversioned names.
2214 if (Lang.hasVersionedName())
2215 Record.back() ^= (uint64_t(1) << 63);
2216
2217 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2218 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawProducer()));
2219 Record.push_back(Elt: N->isOptimized());
2220 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFlags()));
2221 Record.push_back(Elt: N->getRuntimeVersion());
2222 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSplitDebugFilename()));
2223 Record.push_back(Elt: N->getEmissionKind());
2224 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getEnumTypes().get()));
2225 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRetainedTypes().get()));
2226 Record.push_back(/* subprograms */ Elt: 0);
2227 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getGlobalVariables().get()));
2228 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getImportedEntities().get()));
2229 Record.push_back(Elt: N->getDWOId());
2230 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getMacros().get()));
2231 Record.push_back(Elt: N->getSplitDebugInlining());
2232 Record.push_back(Elt: N->getDebugInfoForProfiling());
2233 Record.push_back(Elt: (unsigned)N->getNameTableKind());
2234 Record.push_back(Elt: N->getRangesBaseAddress());
2235 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSysRoot()));
2236 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSDK()));
2237 Record.push_back(Elt: Lang.hasVersionedName() ? Lang.getVersion() : 0);
2238 Record.push_back(Elt: Lang.getDialect());
2239
2240 Stream.EmitRecord(Code: bitc::METADATA_COMPILE_UNIT, Vals: Record, Abbrev);
2241 Record.clear();
2242}
2243
2244void ModuleBitcodeWriter::writeDISubprogram(const DISubprogram *N,
2245 SmallVectorImpl<uint64_t> &Record,
2246 unsigned Abbrev) {
2247 const uint64_t HasUnitFlag = 1 << 1;
2248 const uint64_t HasSPFlagsFlag = 1 << 2;
2249 Record.push_back(Elt: uint64_t(N->isDistinct()) | HasUnitFlag | HasSPFlagsFlag);
2250 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2251 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2252 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLinkageName()));
2253 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2254 Record.push_back(Elt: N->getLine());
2255 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2256 Record.push_back(Elt: N->getScopeLine());
2257 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getContainingType()));
2258 Record.push_back(Elt: N->getSPFlags());
2259 Record.push_back(Elt: N->getVirtualIndex());
2260 Record.push_back(Elt: N->getFlags());
2261 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUnit()));
2262 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams().get()));
2263 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDeclaration()));
2264 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRetainedNodes().get()));
2265 Record.push_back(Elt: N->getThisAdjustment());
2266 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getThrownTypes().get()));
2267 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2268 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawTargetFuncName()));
2269 Record.push_back(Elt: N->getKeyInstructionsEnabled());
2270
2271 Stream.EmitRecord(Code: bitc::METADATA_SUBPROGRAM, Vals: Record, Abbrev);
2272 Record.clear();
2273}
2274
2275void ModuleBitcodeWriter::writeDILexicalBlock(const DILexicalBlock *N,
2276 SmallVectorImpl<uint64_t> &Record,
2277 unsigned Abbrev) {
2278 Record.push_back(Elt: N->isDistinct());
2279 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2280 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2281 Record.push_back(Elt: N->getLine());
2282 Record.push_back(Elt: N->getColumn());
2283
2284 Stream.EmitRecord(Code: bitc::METADATA_LEXICAL_BLOCK, Vals: Record, Abbrev);
2285 Record.clear();
2286}
2287
2288void ModuleBitcodeWriter::writeDILexicalBlockFile(
2289 const DILexicalBlockFile *N, SmallVectorImpl<uint64_t> &Record,
2290 unsigned Abbrev) {
2291 Record.push_back(Elt: N->isDistinct());
2292 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2293 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2294 Record.push_back(Elt: N->getDiscriminator());
2295
2296 Stream.EmitRecord(Code: bitc::METADATA_LEXICAL_BLOCK_FILE, Vals: Record, Abbrev);
2297 Record.clear();
2298}
2299
2300void ModuleBitcodeWriter::writeDICommonBlock(const DICommonBlock *N,
2301 SmallVectorImpl<uint64_t> &Record,
2302 unsigned Abbrev) {
2303 Record.push_back(Elt: N->isDistinct());
2304 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2305 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDecl()));
2306 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2307 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2308 Record.push_back(Elt: N->getLineNo());
2309
2310 Stream.EmitRecord(Code: bitc::METADATA_COMMON_BLOCK, Vals: Record, Abbrev);
2311 Record.clear();
2312}
2313
2314void ModuleBitcodeWriter::writeDINamespace(const DINamespace *N,
2315 SmallVectorImpl<uint64_t> &Record,
2316 unsigned Abbrev) {
2317 Record.push_back(Elt: N->isDistinct() | N->getExportSymbols() << 1);
2318 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2319 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2320
2321 Stream.EmitRecord(Code: bitc::METADATA_NAMESPACE, Vals: Record, Abbrev);
2322 Record.clear();
2323}
2324
2325void ModuleBitcodeWriter::writeDIMacro(const DIMacro *N,
2326 SmallVectorImpl<uint64_t> &Record,
2327 unsigned Abbrev) {
2328 Record.push_back(Elt: N->isDistinct());
2329 Record.push_back(Elt: N->getMacinfoType());
2330 Record.push_back(Elt: N->getLine());
2331 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2332 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawValue()));
2333
2334 Stream.EmitRecord(Code: bitc::METADATA_MACRO, Vals: Record, Abbrev);
2335 Record.clear();
2336}
2337
2338void ModuleBitcodeWriter::writeDIMacroFile(const DIMacroFile *N,
2339 SmallVectorImpl<uint64_t> &Record,
2340 unsigned Abbrev) {
2341 Record.push_back(Elt: N->isDistinct());
2342 Record.push_back(Elt: N->getMacinfoType());
2343 Record.push_back(Elt: N->getLine());
2344 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2345 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2346
2347 Stream.EmitRecord(Code: bitc::METADATA_MACRO_FILE, Vals: Record, Abbrev);
2348 Record.clear();
2349}
2350
2351void ModuleBitcodeWriter::writeDIArgList(const DIArgList *N,
2352 SmallVectorImpl<uint64_t> &Record) {
2353 Record.reserve(N: N->getArgs().size());
2354 for (ValueAsMetadata *MD : N->getArgs())
2355 Record.push_back(Elt: VE.getMetadataID(MD));
2356
2357 Stream.EmitRecord(Code: bitc::METADATA_ARG_LIST, Vals: Record);
2358 Record.clear();
2359}
2360
2361void ModuleBitcodeWriter::writeDIModule(const DIModule *N,
2362 SmallVectorImpl<uint64_t> &Record,
2363 unsigned Abbrev) {
2364 Record.push_back(Elt: N->isDistinct());
2365 for (auto &I : N->operands())
2366 Record.push_back(Elt: VE.getMetadataOrNullID(MD: I));
2367 Record.push_back(Elt: N->getLineNo());
2368 Record.push_back(Elt: N->getIsDecl());
2369
2370 Stream.EmitRecord(Code: bitc::METADATA_MODULE, Vals: Record, Abbrev);
2371 Record.clear();
2372}
2373
2374void ModuleBitcodeWriter::writeDIAssignID(const DIAssignID *N,
2375 SmallVectorImpl<uint64_t> &Record,
2376 unsigned Abbrev) {
2377 // There are no arguments for this metadata type.
2378 Record.push_back(Elt: N->isDistinct());
2379 Stream.EmitRecord(Code: bitc::METADATA_ASSIGN_ID, Vals: Record, Abbrev);
2380 Record.clear();
2381}
2382
2383void ModuleBitcodeWriter::writeDITemplateTypeParameter(
2384 const DITemplateTypeParameter *N, SmallVectorImpl<uint64_t> &Record,
2385 unsigned Abbrev) {
2386 Record.push_back(Elt: N->isDistinct());
2387 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2388 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2389 Record.push_back(Elt: N->isDefault());
2390
2391 Stream.EmitRecord(Code: bitc::METADATA_TEMPLATE_TYPE, Vals: Record, Abbrev);
2392 Record.clear();
2393}
2394
2395void ModuleBitcodeWriter::writeDITemplateValueParameter(
2396 const DITemplateValueParameter *N, SmallVectorImpl<uint64_t> &Record,
2397 unsigned Abbrev) {
2398 Record.push_back(Elt: N->isDistinct());
2399 Record.push_back(Elt: N->getTag());
2400 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2401 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2402 Record.push_back(Elt: N->isDefault());
2403 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getValue()));
2404
2405 Stream.EmitRecord(Code: bitc::METADATA_TEMPLATE_VALUE, Vals: Record, Abbrev);
2406 Record.clear();
2407}
2408
2409void ModuleBitcodeWriter::writeDIGlobalVariable(
2410 const DIGlobalVariable *N, SmallVectorImpl<uint64_t> &Record,
2411 unsigned Abbrev) {
2412 const uint64_t Version = 2 << 1;
2413 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
2414 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2415 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2416 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLinkageName()));
2417 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2418 Record.push_back(Elt: N->getLine());
2419 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2420 Record.push_back(Elt: N->isLocalToUnit());
2421 Record.push_back(Elt: N->isDefinition());
2422 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStaticDataMemberDeclaration()));
2423 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams()));
2424 Record.push_back(Elt: N->getAlignInBits());
2425 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2426
2427 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_VAR, Vals: Record, Abbrev);
2428 Record.clear();
2429}
2430
2431void ModuleBitcodeWriter::writeDILocalVariable(
2432 const DILocalVariable *N, SmallVectorImpl<uint64_t> &Record,
2433 unsigned Abbrev) {
2434 // In order to support all possible bitcode formats in BitcodeReader we need
2435 // to distinguish the following cases:
2436 // 1) Record has no artificial tag (Record[1]),
2437 // has no obsolete inlinedAt field (Record[9]).
2438 // In this case Record size will be 8, HasAlignment flag is false.
2439 // 2) Record has artificial tag (Record[1]),
2440 // has no obsolete inlignedAt field (Record[9]).
2441 // In this case Record size will be 9, HasAlignment flag is false.
2442 // 3) Record has both artificial tag (Record[1]) and
2443 // obsolete inlignedAt field (Record[9]).
2444 // In this case Record size will be 10, HasAlignment flag is false.
2445 // 4) Record has neither artificial tag, nor inlignedAt field, but
2446 // HasAlignment flag is true and Record[8] contains alignment value.
2447 const uint64_t HasAlignmentFlag = 1 << 1;
2448 Record.push_back(Elt: (uint64_t)N->isDistinct() | HasAlignmentFlag);
2449 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2450 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2451 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2452 Record.push_back(Elt: N->getLine());
2453 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2454 Record.push_back(Elt: N->getArg());
2455 Record.push_back(Elt: N->getFlags());
2456 Record.push_back(Elt: N->getAlignInBits());
2457 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2458
2459 Stream.EmitRecord(Code: bitc::METADATA_LOCAL_VAR, Vals: Record, Abbrev);
2460 Record.clear();
2461}
2462
2463void ModuleBitcodeWriter::writeDILabel(
2464 const DILabel *N, SmallVectorImpl<uint64_t> &Record,
2465 unsigned Abbrev) {
2466 uint64_t IsArtificialFlag = uint64_t(N->isArtificial()) << 1;
2467 Record.push_back(Elt: (uint64_t)N->isDistinct() | IsArtificialFlag);
2468 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2469 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2470 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2471 Record.push_back(Elt: N->getLine());
2472 Record.push_back(Elt: N->getColumn());
2473 Record.push_back(Elt: N->getCoroSuspendIdx().has_value()
2474 ? (uint64_t)N->getCoroSuspendIdx().value()
2475 : std::numeric_limits<uint64_t>::max());
2476
2477 Stream.EmitRecord(Code: bitc::METADATA_LABEL, Vals: Record, Abbrev);
2478 Record.clear();
2479}
2480
2481void ModuleBitcodeWriter::writeDIExpression(const DIExpression *N,
2482 SmallVectorImpl<uint64_t> &Record,
2483 unsigned Abbrev) {
2484 Record.reserve(N: N->getElements().size() + 1);
2485 const uint64_t Version = 3 << 1;
2486 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
2487 Record.append(in_start: N->elements_begin(), in_end: N->elements_end());
2488
2489 Stream.EmitRecord(Code: bitc::METADATA_EXPRESSION, Vals: Record, Abbrev);
2490 Record.clear();
2491}
2492
2493void ModuleBitcodeWriter::writeDIGlobalVariableExpression(
2494 const DIGlobalVariableExpression *N, SmallVectorImpl<uint64_t> &Record,
2495 unsigned Abbrev) {
2496 Record.push_back(Elt: N->isDistinct());
2497 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getVariable()));
2498 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getExpression()));
2499
2500 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_VAR_EXPR, Vals: Record, Abbrev);
2501 Record.clear();
2502}
2503
2504void ModuleBitcodeWriter::writeDIObjCProperty(const DIObjCProperty *N,
2505 SmallVectorImpl<uint64_t> &Record,
2506 unsigned Abbrev) {
2507 Record.push_back(Elt: N->isDistinct());
2508 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2509 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2510 Record.push_back(Elt: N->getLine());
2511 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSetterName()));
2512 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawGetterName()));
2513 Record.push_back(Elt: N->getAttributes());
2514 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2515
2516 Stream.EmitRecord(Code: bitc::METADATA_OBJC_PROPERTY, Vals: Record, Abbrev);
2517 Record.clear();
2518}
2519
2520void ModuleBitcodeWriter::writeDIProperty(const DIProperty *N,
2521 SmallVectorImpl<uint64_t> &Record,
2522 unsigned Abbrev) {
2523 Record.push_back(Elt: N->isDistinct());
2524 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2525 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2526 Record.push_back(Elt: N->getLine());
2527 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2528 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBackingStorage()));
2529
2530 Stream.EmitRecord(Code: bitc::METADATA_PROPERTY, Vals: Record, Abbrev);
2531 Record.clear();
2532}
2533
2534void ModuleBitcodeWriter::writeDIImportedEntity(
2535 const DIImportedEntity *N, SmallVectorImpl<uint64_t> &Record,
2536 unsigned Abbrev) {
2537 Record.push_back(Elt: N->isDistinct());
2538 Record.push_back(Elt: N->getTag());
2539 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2540 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getEntity()));
2541 Record.push_back(Elt: N->getLine());
2542 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2543 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFile()));
2544 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2545
2546 Stream.EmitRecord(Code: bitc::METADATA_IMPORTED_ENTITY, Vals: Record, Abbrev);
2547 Record.clear();
2548}
2549
2550unsigned ModuleBitcodeWriter::createNamedMetadataAbbrev() {
2551 auto Abbv = std::make_shared<BitCodeAbbrev>();
2552 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_NAME));
2553 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2554 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2555 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
2556}
2557
2558void ModuleBitcodeWriter::writeNamedMetadata(
2559 SmallVectorImpl<uint64_t> &Record) {
2560 if (M.named_metadata_empty())
2561 return;
2562
2563 unsigned Abbrev = createNamedMetadataAbbrev();
2564 for (const NamedMDNode &NMD : M.named_metadata()) {
2565 // Write name.
2566 StringRef Str = NMD.getName();
2567 Record.append(in_start: Str.bytes_begin(), in_end: Str.bytes_end());
2568 Stream.EmitRecord(Code: bitc::METADATA_NAME, Vals: Record, Abbrev);
2569 Record.clear();
2570
2571 // Write named metadata operands.
2572 for (const MDNode *N : NMD.operands())
2573 Record.push_back(Elt: VE.getMetadataID(MD: N));
2574 Stream.EmitRecord(Code: bitc::METADATA_NAMED_NODE, Vals: Record, Abbrev: 0);
2575 Record.clear();
2576 }
2577}
2578
2579unsigned ModuleBitcodeWriter::createMetadataStringsAbbrev() {
2580 auto Abbv = std::make_shared<BitCodeAbbrev>();
2581 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_STRINGS));
2582 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // # of strings
2583 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // offset to chars
2584 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
2585 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
2586}
2587
2588/// Write out a record for MDString.
2589///
2590/// All the metadata strings in a metadata block are emitted in a single
2591/// record. The sizes and strings themselves are shoved into a blob.
2592void ModuleBitcodeWriter::writeMetadataStrings(
2593 ArrayRef<const Metadata *> Strings, SmallVectorImpl<uint64_t> &Record) {
2594 if (Strings.empty())
2595 return;
2596
2597 // Start the record with the number of strings.
2598 Record.push_back(Elt: bitc::METADATA_STRINGS);
2599 Record.push_back(Elt: Strings.size());
2600
2601 // Emit the sizes of the strings in the blob.
2602 SmallString<256> Blob;
2603 {
2604 BitstreamWriter W(Blob);
2605 for (const Metadata *MD : Strings)
2606 W.EmitVBR(Val: cast<MDString>(Val: MD)->getLength(), NumBits: 6);
2607 W.FlushToWord();
2608 }
2609
2610 // Add the offset to the strings to the record.
2611 Record.push_back(Elt: Blob.size());
2612
2613 // Add the strings to the blob.
2614 for (const Metadata *MD : Strings)
2615 Blob.append(RHS: cast<MDString>(Val: MD)->getString());
2616
2617 // Emit the final record.
2618 Stream.EmitRecordWithBlob(Abbrev: createMetadataStringsAbbrev(), Vals: Record, Blob);
2619 Record.clear();
2620}
2621
2622// Generates an enum to use as an index in the Abbrev array of Metadata record.
2623enum MetadataAbbrev : unsigned {
2624#define HANDLE_MDNODE_LEAF(CLASS) CLASS##AbbrevID,
2625#include "llvm/IR/Metadata.def"
2626 LastPlusOne
2627};
2628
2629void ModuleBitcodeWriter::writeMetadataRecords(
2630 ArrayRef<const Metadata *> MDs, SmallVectorImpl<uint64_t> &Record,
2631 std::vector<unsigned> *MDAbbrevs, std::vector<uint64_t> *IndexPos) {
2632 if (MDs.empty())
2633 return;
2634
2635 // Initialize MDNode abbreviations.
2636#define HANDLE_MDNODE_LEAF(CLASS) unsigned CLASS##Abbrev = 0;
2637#include "llvm/IR/Metadata.def"
2638
2639 for (const Metadata *MD : MDs) {
2640 if (IndexPos)
2641 IndexPos->push_back(x: Stream.GetCurrentBitNo());
2642 if (const MDNode *N = dyn_cast<MDNode>(Val: MD)) {
2643 assert(N->isResolved() && "Expected forward references to be resolved");
2644
2645 switch (N->getMetadataID()) {
2646 default:
2647 llvm_unreachable("Invalid MDNode subclass");
2648#define HANDLE_MDNODE_LEAF(CLASS) \
2649 case Metadata::CLASS##Kind: \
2650 if (MDAbbrevs) \
2651 write##CLASS(cast<CLASS>(N), Record, \
2652 (*MDAbbrevs)[MetadataAbbrev::CLASS##AbbrevID]); \
2653 else \
2654 write##CLASS(cast<CLASS>(N), Record, CLASS##Abbrev); \
2655 continue;
2656#include "llvm/IR/Metadata.def"
2657 }
2658 }
2659 if (auto *AL = dyn_cast<DIArgList>(Val: MD)) {
2660 writeDIArgList(N: AL, Record);
2661 continue;
2662 }
2663 writeValueAsMetadata(MD: cast<ValueAsMetadata>(Val: MD), Record);
2664 }
2665}
2666
2667void ModuleBitcodeWriter::writeModuleMetadata() {
2668 if (!VE.hasMDs() && M.named_metadata_empty())
2669 return;
2670
2671 Stream.EnterSubblock(BlockID: bitc::METADATA_BLOCK_ID, CodeLen: 4);
2672 SmallVector<uint64_t, 64> Record;
2673
2674 // Emit all abbrevs upfront, so that the reader can jump in the middle of the
2675 // block and load any metadata.
2676 std::vector<unsigned> MDAbbrevs;
2677
2678 MDAbbrevs.resize(new_size: MetadataAbbrev::LastPlusOne);
2679 MDAbbrevs[MetadataAbbrev::DILocationAbbrevID] = createDILocationAbbrev();
2680 MDAbbrevs[MetadataAbbrev::GenericDINodeAbbrevID] =
2681 createGenericDINodeAbbrev();
2682
2683 auto Abbv = std::make_shared<BitCodeAbbrev>();
2684 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_INDEX_OFFSET));
2685 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2686 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2687 unsigned OffsetAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2688
2689 Abbv = std::make_shared<BitCodeAbbrev>();
2690 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_INDEX));
2691 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2692 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
2693 unsigned IndexAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2694
2695 // Emit MDStrings together upfront.
2696 writeMetadataStrings(Strings: VE.getMDStrings(), Record);
2697
2698 // We only emit an index for the metadata record if we have more than a given
2699 // (naive) threshold of metadatas, otherwise it is not worth it.
2700 if (VE.getNonMDStrings().size() > IndexThreshold) {
2701 // Write a placeholder value in for the offset of the metadata index,
2702 // which is written after the records, so that it can include
2703 // the offset of each entry. The placeholder offset will be
2704 // updated after all records are emitted.
2705 uint64_t Vals[] = {0, 0};
2706 Stream.EmitRecord(Code: bitc::METADATA_INDEX_OFFSET, Vals, Abbrev: OffsetAbbrev);
2707 }
2708
2709 // Compute and save the bit offset to the current position, which will be
2710 // patched when we emit the index later. We can simply subtract the 64-bit
2711 // fixed size from the current bit number to get the location to backpatch.
2712 uint64_t IndexOffsetRecordBitPos = Stream.GetCurrentBitNo();
2713
2714 // This index will contain the bitpos for each individual record.
2715 std::vector<uint64_t> IndexPos;
2716 IndexPos.reserve(n: VE.getNonMDStrings().size());
2717
2718 // Write all the records
2719 writeMetadataRecords(MDs: VE.getNonMDStrings(), Record, MDAbbrevs: &MDAbbrevs, IndexPos: &IndexPos);
2720
2721 if (VE.getNonMDStrings().size() > IndexThreshold) {
2722 // Now that we have emitted all the records we will emit the index. But
2723 // first
2724 // backpatch the forward reference so that the reader can skip the records
2725 // efficiently.
2726 Stream.BackpatchWord64(BitNo: IndexOffsetRecordBitPos - 64,
2727 Val: Stream.GetCurrentBitNo() - IndexOffsetRecordBitPos);
2728
2729 // Delta encode the index.
2730 uint64_t PreviousValue = IndexOffsetRecordBitPos;
2731 for (auto &Elt : IndexPos) {
2732 auto EltDelta = Elt - PreviousValue;
2733 PreviousValue = Elt;
2734 Elt = EltDelta;
2735 }
2736 // Emit the index record.
2737 Stream.EmitRecord(Code: bitc::METADATA_INDEX, Vals: IndexPos, Abbrev: IndexAbbrev);
2738 IndexPos.clear();
2739 }
2740
2741 // Write the named metadata now.
2742 writeNamedMetadata(Record);
2743
2744 auto AddDeclAttachedMetadata = [&](const GlobalObject &GO) {
2745 SmallVector<uint64_t, 4> Record;
2746 Record.push_back(Elt: VE.getValueID(V: &GO));
2747 pushGlobalMetadataAttachment(Record, GO);
2748 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_DECL_ATTACHMENT, Vals: Record);
2749 };
2750 for (const Function &F : M)
2751 if (F.isDeclaration() && F.hasMetadata())
2752 AddDeclAttachedMetadata(F);
2753 for (const GlobalIFunc &GI : M.ifuncs())
2754 if (GI.hasMetadata())
2755 AddDeclAttachedMetadata(GI);
2756 // FIXME: Only store metadata for declarations here, and move data for global
2757 // variable definitions to a separate block (PR28134).
2758 for (const GlobalVariable &GV : M.globals())
2759 if (GV.hasMetadata())
2760 AddDeclAttachedMetadata(GV);
2761
2762 Stream.ExitBlock();
2763}
2764
2765void ModuleBitcodeWriter::writeFunctionMetadata(const Function &F) {
2766 if (!VE.hasMDs())
2767 return;
2768
2769 Stream.EnterSubblock(BlockID: bitc::METADATA_BLOCK_ID, CodeLen: 3);
2770 SmallVector<uint64_t, 64> Record;
2771 writeMetadataStrings(Strings: VE.getMDStrings(), Record);
2772 writeMetadataRecords(MDs: VE.getNonMDStrings(), Record);
2773 Stream.ExitBlock();
2774}
2775
2776void ModuleBitcodeWriter::pushGlobalMetadataAttachment(
2777 SmallVectorImpl<uint64_t> &Record, const GlobalObject &GO) {
2778 // [n x [id, mdnode]]
2779 SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2780 GO.getAllMetadata(MDs);
2781 for (const auto &I : MDs) {
2782 Record.push_back(Elt: I.first);
2783 Record.push_back(Elt: VE.getMetadataID(MD: I.second));
2784 }
2785}
2786
2787void ModuleBitcodeWriter::writeFunctionMetadataAttachment(const Function &F) {
2788 Stream.EnterSubblock(BlockID: bitc::METADATA_ATTACHMENT_ID, CodeLen: 3);
2789
2790 SmallVector<uint64_t, 64> Record;
2791
2792 if (F.hasMetadata()) {
2793 pushGlobalMetadataAttachment(Record, GO: F);
2794 Stream.EmitRecord(Code: bitc::METADATA_ATTACHMENT, Vals: Record, Abbrev: 0);
2795 Record.clear();
2796 }
2797
2798 // Write metadata attachments
2799 // METADATA_ATTACHMENT - [m x [value, [n x [id, mdnode]]]
2800 SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2801 for (const BasicBlock &BB : F)
2802 for (const Instruction &I : BB) {
2803 MDs.clear();
2804 I.getAllMetadataOtherThanDebugLoc(MDs);
2805
2806 // If no metadata, ignore instruction.
2807 if (MDs.empty()) continue;
2808
2809 Record.push_back(Elt: VE.getInstructionID(I: &I));
2810
2811 for (const auto &[ID, MD] : MDs) {
2812 Record.push_back(Elt: ID);
2813 Record.push_back(Elt: VE.getMetadataID(MD));
2814 }
2815 Stream.EmitRecord(Code: bitc::METADATA_ATTACHMENT, Vals: Record, Abbrev: 0);
2816 Record.clear();
2817 }
2818
2819 Stream.ExitBlock();
2820}
2821
2822void ModuleBitcodeWriter::writeModuleMetadataKinds() {
2823 SmallVector<uint64_t, 64> Record;
2824
2825 // Write metadata kinds
2826 // METADATA_KIND - [n x [id, name]]
2827 SmallVector<StringRef, 8> Names;
2828 M.getMDKindNames(Result&: Names);
2829
2830 if (Names.empty()) return;
2831
2832 Stream.EnterSubblock(BlockID: bitc::METADATA_KIND_BLOCK_ID, CodeLen: 3);
2833
2834 for (unsigned MDKindID = 0, e = Names.size(); MDKindID != e; ++MDKindID) {
2835 Record.push_back(Elt: MDKindID);
2836 StringRef KName = Names[MDKindID];
2837 Record.append(in_start: KName.begin(), in_end: KName.end());
2838
2839 Stream.EmitRecord(Code: bitc::METADATA_KIND, Vals: Record, Abbrev: 0);
2840 Record.clear();
2841 }
2842
2843 Stream.ExitBlock();
2844}
2845
2846void ModuleBitcodeWriter::writeOperandBundleTags() {
2847 // Write metadata kinds
2848 //
2849 // OPERAND_BUNDLE_TAGS_BLOCK_ID : N x OPERAND_BUNDLE_TAG
2850 //
2851 // OPERAND_BUNDLE_TAG - [strchr x N]
2852
2853 SmallVector<StringRef, 8> Tags;
2854 M.getOperandBundleTags(Result&: Tags);
2855
2856 if (Tags.empty())
2857 return;
2858
2859 Stream.EnterSubblock(BlockID: bitc::OPERAND_BUNDLE_TAGS_BLOCK_ID, CodeLen: 3);
2860
2861 SmallVector<uint64_t, 64> Record;
2862
2863 for (auto Tag : Tags) {
2864 Record.append(in_start: Tag.begin(), in_end: Tag.end());
2865
2866 Stream.EmitRecord(Code: bitc::OPERAND_BUNDLE_TAG, Vals: Record, Abbrev: 0);
2867 Record.clear();
2868 }
2869
2870 Stream.ExitBlock();
2871}
2872
2873void ModuleBitcodeWriter::writeSyncScopeNames() {
2874 SmallVector<StringRef, 8> SSNs;
2875 M.getContext().getSyncScopeNames(SSNs);
2876 if (SSNs.empty())
2877 return;
2878
2879 Stream.EnterSubblock(BlockID: bitc::SYNC_SCOPE_NAMES_BLOCK_ID, CodeLen: 2);
2880
2881 SmallVector<uint64_t, 64> Record;
2882 for (auto SSN : SSNs) {
2883 Record.append(in_start: SSN.begin(), in_end: SSN.end());
2884 Stream.EmitRecord(Code: bitc::SYNC_SCOPE_NAME, Vals: Record, Abbrev: 0);
2885 Record.clear();
2886 }
2887
2888 Stream.ExitBlock();
2889}
2890
2891void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal,
2892 bool isGlobal) {
2893 if (FirstVal == LastVal) return;
2894
2895 Stream.EnterSubblock(BlockID: bitc::CONSTANTS_BLOCK_ID, CodeLen: 4);
2896
2897 unsigned AggregateAbbrev = 0;
2898 unsigned String8Abbrev = 0;
2899 unsigned CString7Abbrev = 0;
2900 unsigned CString6Abbrev = 0;
2901 // If this is a constant pool for the module, emit module-specific abbrevs.
2902 if (isGlobal) {
2903 // Abbrev for CST_CODE_AGGREGATE.
2904 auto Abbv = std::make_shared<BitCodeAbbrev>();
2905 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_AGGREGATE));
2906 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2907 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(Value: LastVal+1)));
2908 AggregateAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2909
2910 // Abbrev for CST_CODE_STRING.
2911 Abbv = std::make_shared<BitCodeAbbrev>();
2912 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_STRING));
2913 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2914 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2915 String8Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2916 // Abbrev for CST_CODE_CSTRING.
2917 Abbv = std::make_shared<BitCodeAbbrev>();
2918 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2919 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2920 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
2921 CString7Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2922 // Abbrev for CST_CODE_CSTRING.
2923 Abbv = std::make_shared<BitCodeAbbrev>();
2924 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2925 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2926 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
2927 CString6Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2928 }
2929
2930 SmallVector<uint64_t, 64> Record;
2931
2932 const ValueEnumerator::ValueList &Vals = VE.getValues();
2933 Type *LastTy = nullptr;
2934 for (unsigned i = FirstVal; i != LastVal; ++i) {
2935 const Value *V = Vals[i].first;
2936 // If we need to switch types, do so now.
2937 if (V->getType() != LastTy) {
2938 LastTy = V->getType();
2939 Record.push_back(Elt: VE.getTypeID(T: LastTy));
2940 Stream.EmitRecord(Code: bitc::CST_CODE_SETTYPE, Vals: Record,
2941 Abbrev: CONSTANTS_SETTYPE_ABBREV);
2942 Record.clear();
2943 }
2944
2945 if (const InlineAsm *IA = dyn_cast<InlineAsm>(Val: V)) {
2946 Record.push_back(Elt: VE.getTypeID(T: IA->getFunctionType()));
2947 Record.push_back(
2948 Elt: unsigned(IA->hasSideEffects()) | unsigned(IA->isAlignStack()) << 1 |
2949 unsigned(IA->getDialect() & 1) << 2 | unsigned(IA->canThrow()) << 3);
2950
2951 // Add the asm string.
2952 StringRef AsmStr = IA->getAsmString();
2953 Record.push_back(Elt: AsmStr.size());
2954 Record.append(in_start: AsmStr.begin(), in_end: AsmStr.end());
2955
2956 // Add the constraint string.
2957 StringRef ConstraintStr = IA->getConstraintString();
2958 Record.push_back(Elt: ConstraintStr.size());
2959 Record.append(in_start: ConstraintStr.begin(), in_end: ConstraintStr.end());
2960 Stream.EmitRecord(Code: bitc::CST_CODE_INLINEASM, Vals: Record);
2961 Record.clear();
2962 continue;
2963 }
2964 const Constant *C = cast<Constant>(Val: V);
2965 unsigned Code = -1U;
2966 unsigned AbbrevToUse = 0;
2967 if (C->isNullValue()) {
2968 Code = bitc::CST_CODE_NULL;
2969 } else if (isa<PoisonValue>(Val: C)) {
2970 Code = bitc::CST_CODE_POISON;
2971 } else if (isa<UndefValue>(Val: C)) {
2972 Code = bitc::CST_CODE_UNDEF;
2973 } else if (const ConstantInt *IV = dyn_cast<ConstantInt>(Val: C)) {
2974 if (IV->getBitWidth() <= 64) {
2975 uint64_t V = IV->getSExtValue();
2976 emitSignedInt64(Vals&: Record, V);
2977 Code = bitc::CST_CODE_INTEGER;
2978 AbbrevToUse = CONSTANTS_INTEGER_ABBREV;
2979 } else { // Wide integers, > 64 bits in size.
2980 emitWideAPInt(Vals&: Record, A: IV->getValue());
2981 Code = bitc::CST_CODE_WIDE_INTEGER;
2982 }
2983 } else if (const ConstantByte *BV = dyn_cast<ConstantByte>(Val: C)) {
2984 if (BV->getBitWidth() <= 64) {
2985 uint64_t V = BV->getSExtValue();
2986 emitSignedInt64(Vals&: Record, V);
2987 Code = bitc::CST_CODE_BYTE;
2988 AbbrevToUse = CONSTANTS_BYTE_ABBREV;
2989 } else { // Wide bytes, > 64 bits in size.
2990 emitWideAPInt(Vals&: Record, A: BV->getValue());
2991 Code = bitc::CST_CODE_WIDE_BYTE;
2992 }
2993 } else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
2994 Code = bitc::CST_CODE_FLOAT;
2995 Type *Ty = CFP->getType()->getScalarType();
2996 if (Ty->isHalfTy() || Ty->isBFloatTy() || Ty->isFloatTy() ||
2997 Ty->isDoubleTy()) {
2998 Record.push_back(Elt: CFP->getValueAPF().bitcastToAPInt().getZExtValue());
2999 } else if (Ty->isX86_FP80Ty()) {
3000 // api needed to prevent premature destruction
3001 // bits are not in the same order as a normal i80 APInt, compensate.
3002 APInt api = CFP->getValueAPF().bitcastToAPInt();
3003 const uint64_t *p = api.getRawData();
3004 Record.push_back(Elt: (p[1] << 48) | (p[0] >> 16));
3005 Record.push_back(Elt: p[0] & 0xffffLL);
3006 } else if (Ty->isFP128Ty() || Ty->isPPC_FP128Ty()) {
3007 APInt api = CFP->getValueAPF().bitcastToAPInt();
3008 const uint64_t *p = api.getRawData();
3009 Record.push_back(Elt: p[0]);
3010 Record.push_back(Elt: p[1]);
3011 } else {
3012 assert(0 && "Unknown FP type!");
3013 }
3014 } else if (isa<ConstantDataSequential>(Val: C) &&
3015 cast<ConstantDataSequential>(Val: C)->isString()) {
3016 const ConstantDataSequential *Str = cast<ConstantDataSequential>(Val: C);
3017 // Emit constant strings specially.
3018 uint64_t NumElts = Str->getNumElements();
3019 // If this is a null-terminated string, use the denser CSTRING encoding.
3020 if (Str->isCString()) {
3021 Code = bitc::CST_CODE_CSTRING;
3022 --NumElts; // Don't encode the null, which isn't allowed by char6.
3023 } else {
3024 Code = bitc::CST_CODE_STRING;
3025 AbbrevToUse = String8Abbrev;
3026 }
3027 bool isCStr7 = Code == bitc::CST_CODE_CSTRING;
3028 bool isCStrChar6 = Code == bitc::CST_CODE_CSTRING;
3029 for (uint64_t i = 0; i != NumElts; ++i) {
3030 unsigned char V = Str->getElementAsInteger(i);
3031 Record.push_back(Elt: V);
3032 isCStr7 &= (V & 128) == 0;
3033 if (isCStrChar6)
3034 isCStrChar6 = BitCodeAbbrevOp::isChar6(C: V);
3035 }
3036
3037 if (isCStrChar6)
3038 AbbrevToUse = CString6Abbrev;
3039 else if (isCStr7)
3040 AbbrevToUse = CString7Abbrev;
3041 } else if (const ConstantDataSequential *CDS =
3042 dyn_cast<ConstantDataSequential>(Val: C)) {
3043 Code = bitc::CST_CODE_DATA;
3044 Type *EltTy = CDS->getElementType();
3045 if (isa<IntegerType>(Val: EltTy) || isa<ByteType>(Val: EltTy)) {
3046 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i)
3047 Record.push_back(Elt: CDS->getElementAsInteger(i));
3048 } else {
3049 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i)
3050 Record.push_back(
3051 Elt: CDS->getElementAsAPFloat(i).bitcastToAPInt().getLimitedValue());
3052 }
3053 } else if (isa<ConstantAggregate>(Val: C)) {
3054 Code = bitc::CST_CODE_AGGREGATE;
3055 for (const Value *Op : C->operands())
3056 Record.push_back(Elt: VE.getValueID(V: Op));
3057 AbbrevToUse = AggregateAbbrev;
3058 } else if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: C)) {
3059 switch (CE->getOpcode()) {
3060 default:
3061 if (Instruction::isCast(Opcode: CE->getOpcode())) {
3062 Code = bitc::CST_CODE_CE_CAST;
3063 Record.push_back(Elt: getEncodedCastOpcode(Opcode: CE->getOpcode()));
3064 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
3065 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3066 AbbrevToUse = CONSTANTS_CE_CAST_Abbrev;
3067 } else {
3068 assert(CE->getNumOperands() == 2 && "Unknown constant expr!");
3069 Code = bitc::CST_CODE_CE_BINOP;
3070 Record.push_back(Elt: getEncodedBinaryOpcode(Opcode: CE->getOpcode()));
3071 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3072 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3073 uint64_t Flags = getOptimizationFlags(V: CE);
3074 if (Flags != 0)
3075 Record.push_back(Elt: Flags);
3076 }
3077 break;
3078 case Instruction::FNeg: {
3079 assert(CE->getNumOperands() == 1 && "Unknown constant expr!");
3080 Code = bitc::CST_CODE_CE_UNOP;
3081 Record.push_back(Elt: getEncodedUnaryOpcode(Opcode: CE->getOpcode()));
3082 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3083 uint64_t Flags = getOptimizationFlags(V: CE);
3084 if (Flags != 0)
3085 Record.push_back(Elt: Flags);
3086 break;
3087 }
3088 case Instruction::GetElementPtr: {
3089 Code = bitc::CST_CODE_CE_GEP;
3090 const auto *GO = cast<GEPOperator>(Val: C);
3091 Record.push_back(Elt: VE.getTypeID(T: GO->getSourceElementType()));
3092 Record.push_back(Elt: getOptimizationFlags(V: GO));
3093 if (std::optional<ConstantRange> Range = GO->getInRange()) {
3094 Code = bitc::CST_CODE_CE_GEP_WITH_INRANGE;
3095 emitConstantRange(Record, CR: *Range, /*EmitBitWidth=*/true);
3096 }
3097 for (const Value *Op : CE->operands()) {
3098 Record.push_back(Elt: VE.getTypeID(T: Op->getType()));
3099 Record.push_back(Elt: VE.getValueID(V: Op));
3100 }
3101 break;
3102 }
3103 case Instruction::ExtractElement:
3104 Code = bitc::CST_CODE_CE_EXTRACTELT;
3105 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
3106 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3107 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 1)->getType()));
3108 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3109 break;
3110 case Instruction::InsertElement:
3111 Code = bitc::CST_CODE_CE_INSERTELT;
3112 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3113 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3114 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 2)->getType()));
3115 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 2)));
3116 break;
3117 case Instruction::ShuffleVector:
3118 // If the return type and argument types are the same, this is a
3119 // standard shufflevector instruction. If the types are different,
3120 // then the shuffle is widening or truncating the input vectors, and
3121 // the argument type must also be encoded.
3122 if (C->getType() == C->getOperand(i: 0)->getType()) {
3123 Code = bitc::CST_CODE_CE_SHUFFLEVEC;
3124 } else {
3125 Code = bitc::CST_CODE_CE_SHUFVEC_EX;
3126 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
3127 }
3128 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3129 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3130 Record.push_back(Elt: VE.getValueID(V: CE->getShuffleMaskForBitcode()));
3131 break;
3132 }
3133 } else if (const BlockAddress *BA = dyn_cast<BlockAddress>(Val: C)) {
3134 Code = bitc::CST_CODE_BLOCKADDRESS;
3135 Record.push_back(Elt: VE.getTypeID(T: BA->getFunction()->getType()));
3136 Record.push_back(Elt: VE.getValueID(V: BA->getFunction()));
3137 Record.push_back(Elt: VE.getGlobalBasicBlockID(BB: BA->getBasicBlock()));
3138 } else if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(Val: C)) {
3139 Code = bitc::CST_CODE_DSO_LOCAL_EQUIVALENT;
3140 Record.push_back(Elt: VE.getTypeID(T: Equiv->getGlobalValue()->getType()));
3141 Record.push_back(Elt: VE.getValueID(V: Equiv->getGlobalValue()));
3142 } else if (const auto *NC = dyn_cast<NoCFIValue>(Val: C)) {
3143 Code = bitc::CST_CODE_NO_CFI_VALUE;
3144 Record.push_back(Elt: VE.getTypeID(T: NC->getGlobalValue()->getType()));
3145 Record.push_back(Elt: VE.getValueID(V: NC->getGlobalValue()));
3146 } else if (const auto *CPA = dyn_cast<ConstantPtrAuth>(Val: C)) {
3147 Code = bitc::CST_CODE_PTRAUTH2;
3148 Record.push_back(Elt: VE.getValueID(V: CPA->getPointer()));
3149 Record.push_back(Elt: VE.getValueID(V: CPA->getKey()));
3150 Record.push_back(Elt: VE.getValueID(V: CPA->getDiscriminator()));
3151 Record.push_back(Elt: VE.getValueID(V: CPA->getAddrDiscriminator()));
3152 Record.push_back(Elt: VE.getValueID(V: CPA->getDeactivationSymbol()));
3153 } else {
3154#ifndef NDEBUG
3155 C->dump();
3156#endif
3157 llvm_unreachable("Unknown constant!");
3158 }
3159 Stream.EmitRecord(Code, Vals: Record, Abbrev: AbbrevToUse);
3160 Record.clear();
3161 }
3162
3163 Stream.ExitBlock();
3164}
3165
3166void ModuleBitcodeWriter::writeModuleConstants() {
3167 const ValueEnumerator::ValueList &Vals = VE.getValues();
3168
3169 // Find the first constant to emit, which is the first non-globalvalue value.
3170 // We know globalvalues have been emitted by WriteModuleInfo.
3171 for (unsigned i = 0, e = Vals.size(); i != e; ++i) {
3172 if (!isa<GlobalValue>(Val: Vals[i].first)) {
3173 writeConstants(FirstVal: i, LastVal: Vals.size(), isGlobal: true);
3174 return;
3175 }
3176 }
3177}
3178
3179/// pushValueAndType - The file has to encode both the value and type id for
3180/// many values, because we need to know what type to create for forward
3181/// references. However, most operands are not forward references, so this type
3182/// field is not needed.
3183///
3184/// This function adds V's value ID to Vals. If the value ID is higher than the
3185/// instruction ID, then it is a forward reference, and it also includes the
3186/// type ID. The value ID that is written is encoded relative to the InstID.
3187bool ModuleBitcodeWriter::pushValueAndType(const Value *V, unsigned InstID,
3188 SmallVectorImpl<unsigned> &Vals) {
3189 unsigned ValID = VE.getValueID(V);
3190 // Make encoding relative to the InstID.
3191 Vals.push_back(Elt: InstID - ValID);
3192 if (ValID >= InstID) {
3193 Vals.push_back(Elt: VE.getTypeID(T: V->getType()));
3194 return true;
3195 }
3196 return false;
3197}
3198
3199bool ModuleBitcodeWriter::pushValueOrMetadata(const Value *V, unsigned InstID,
3200 SmallVectorImpl<unsigned> &Vals) {
3201 bool IsMetadata = V->getType()->isMetadataTy();
3202 if (IsMetadata) {
3203 Vals.push_back(Elt: bitc::OB_METADATA);
3204 Metadata *MD = cast<MetadataAsValue>(Val: V)->getMetadata();
3205 unsigned ValID = VE.getMetadataID(MD);
3206 Vals.push_back(Elt: InstID - ValID);
3207 return false;
3208 }
3209 return pushValueAndType(V, InstID, Vals);
3210}
3211
3212void ModuleBitcodeWriter::writeOperandBundles(const CallBase &CS,
3213 unsigned InstID) {
3214 SmallVector<unsigned, 64> Record;
3215 LLVMContext &C = CS.getContext();
3216
3217 for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i) {
3218 const auto &Bundle = CS.getOperandBundleAt(Index: i);
3219 Record.push_back(Elt: C.getOperandBundleTagID(Tag: Bundle.getTagName()));
3220
3221 for (auto &Input : Bundle.Inputs)
3222 pushValueOrMetadata(V: Input, InstID, Vals&: Record);
3223
3224 Stream.EmitRecord(Code: bitc::FUNC_CODE_OPERAND_BUNDLE, Vals: Record);
3225 Record.clear();
3226 }
3227}
3228
3229/// pushValue - Like pushValueAndType, but where the type of the value is
3230/// omitted (perhaps it was already encoded in an earlier operand).
3231void ModuleBitcodeWriter::pushValue(const Value *V, unsigned InstID,
3232 SmallVectorImpl<unsigned> &Vals) {
3233 unsigned ValID = VE.getValueID(V);
3234 Vals.push_back(Elt: InstID - ValID);
3235}
3236
3237void ModuleBitcodeWriter::pushValueSigned(const Value *V, unsigned InstID,
3238 SmallVectorImpl<uint64_t> &Vals) {
3239 unsigned ValID = VE.getValueID(V);
3240 int64_t diff = ((int32_t)InstID - (int32_t)ValID);
3241 emitSignedInt64(Vals, V: diff);
3242}
3243
3244/// WriteInstruction - Emit an instruction to the specified stream.
3245void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
3246 unsigned InstID,
3247 SmallVectorImpl<unsigned> &Vals) {
3248 unsigned Code = 0;
3249 unsigned AbbrevToUse = 0;
3250 VE.setInstructionID(&I);
3251 switch (I.getOpcode()) {
3252 default:
3253 if (Instruction::isCast(Opcode: I.getOpcode())) {
3254 Code = bitc::FUNC_CODE_INST_CAST;
3255 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3256 AbbrevToUse = FUNCTION_INST_CAST_ABBREV;
3257 Vals.push_back(Elt: VE.getTypeID(T: I.getType()));
3258 Vals.push_back(Elt: getEncodedCastOpcode(Opcode: I.getOpcode()));
3259 uint64_t Flags = getOptimizationFlags(V: &I);
3260 if (Flags != 0) {
3261 if (AbbrevToUse == FUNCTION_INST_CAST_ABBREV)
3262 AbbrevToUse = FUNCTION_INST_CAST_FLAGS_ABBREV;
3263 Vals.push_back(Elt: Flags);
3264 }
3265 } else {
3266 assert(isa<BinaryOperator>(I) && "Unknown instruction!");
3267 Code = bitc::FUNC_CODE_INST_BINOP;
3268 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3269 AbbrevToUse = FUNCTION_INST_BINOP_ABBREV;
3270 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3271 Vals.push_back(Elt: getEncodedBinaryOpcode(Opcode: I.getOpcode()));
3272 uint64_t Flags = getOptimizationFlags(V: &I);
3273 if (Flags != 0) {
3274 if (AbbrevToUse == FUNCTION_INST_BINOP_ABBREV)
3275 AbbrevToUse = FUNCTION_INST_BINOP_FLAGS_ABBREV;
3276 Vals.push_back(Elt: Flags);
3277 }
3278 }
3279 break;
3280 case Instruction::FNeg: {
3281 Code = bitc::FUNC_CODE_INST_UNOP;
3282 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3283 AbbrevToUse = FUNCTION_INST_UNOP_ABBREV;
3284 Vals.push_back(Elt: getEncodedUnaryOpcode(Opcode: I.getOpcode()));
3285 uint64_t Flags = getOptimizationFlags(V: &I);
3286 if (Flags != 0) {
3287 if (AbbrevToUse == FUNCTION_INST_UNOP_ABBREV)
3288 AbbrevToUse = FUNCTION_INST_UNOP_FLAGS_ABBREV;
3289 Vals.push_back(Elt: Flags);
3290 }
3291 break;
3292 }
3293 case Instruction::GetElementPtr: {
3294 Code = bitc::FUNC_CODE_INST_GEP;
3295 AbbrevToUse = FUNCTION_INST_GEP_ABBREV;
3296 auto &GEPInst = cast<GetElementPtrInst>(Val: I);
3297 Vals.push_back(Elt: getOptimizationFlags(V: &I));
3298 Vals.push_back(Elt: VE.getTypeID(T: GEPInst.getSourceElementType()));
3299 for (const Value *Op : I.operands())
3300 pushValueAndType(V: Op, InstID, Vals);
3301 break;
3302 }
3303 case Instruction::ExtractValue: {
3304 Code = bitc::FUNC_CODE_INST_EXTRACTVAL;
3305 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3306 const ExtractValueInst *EVI = cast<ExtractValueInst>(Val: &I);
3307 Vals.append(in_start: EVI->idx_begin(), in_end: EVI->idx_end());
3308 break;
3309 }
3310 case Instruction::InsertValue: {
3311 Code = bitc::FUNC_CODE_INST_INSERTVAL;
3312 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3313 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3314 const InsertValueInst *IVI = cast<InsertValueInst>(Val: &I);
3315 Vals.append(in_start: IVI->idx_begin(), in_end: IVI->idx_end());
3316 break;
3317 }
3318 case Instruction::Select: {
3319 Code = bitc::FUNC_CODE_INST_VSELECT;
3320 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3321 pushValue(V: I.getOperand(i: 2), InstID, Vals);
3322 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3323 uint64_t Flags = getOptimizationFlags(V: &I);
3324 if (Flags != 0)
3325 Vals.push_back(Elt: Flags);
3326 break;
3327 }
3328 case Instruction::ExtractElement:
3329 Code = bitc::FUNC_CODE_INST_EXTRACTELT;
3330 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3331 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3332 break;
3333 case Instruction::InsertElement:
3334 Code = bitc::FUNC_CODE_INST_INSERTELT;
3335 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3336 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3337 pushValueAndType(V: I.getOperand(i: 2), InstID, Vals);
3338 break;
3339 case Instruction::ShuffleVector:
3340 Code = bitc::FUNC_CODE_INST_SHUFFLEVEC;
3341 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3342 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3343 pushValue(V: cast<ShuffleVectorInst>(Val: I).getShuffleMaskForBitcode(), InstID,
3344 Vals);
3345 break;
3346 case Instruction::ICmp:
3347 case Instruction::FCmp: {
3348 // compare returning Int1Ty or vector of Int1Ty
3349 Code = bitc::FUNC_CODE_INST_CMP2;
3350 AbbrevToUse = FUNCTION_INST_CMP_ABBREV;
3351 if (pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3352 AbbrevToUse = 0;
3353 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3354 Vals.push_back(Elt: cast<CmpInst>(Val: I).getPredicate());
3355 uint64_t Flags = getOptimizationFlags(V: &I);
3356 if (Flags != 0) {
3357 Vals.push_back(Elt: Flags);
3358 if (AbbrevToUse)
3359 AbbrevToUse = FUNCTION_INST_CMP_FLAGS_ABBREV;
3360 }
3361 break;
3362 }
3363
3364 case Instruction::Ret:
3365 {
3366 Code = bitc::FUNC_CODE_INST_RET;
3367 unsigned NumOperands = I.getNumOperands();
3368 if (NumOperands == 0)
3369 AbbrevToUse = FUNCTION_INST_RET_VOID_ABBREV;
3370 else if (NumOperands == 1) {
3371 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3372 AbbrevToUse = FUNCTION_INST_RET_VAL_ABBREV;
3373 } else {
3374 for (const Value *Op : I.operands())
3375 pushValueAndType(V: Op, InstID, Vals);
3376 }
3377 }
3378 break;
3379 case Instruction::UncondBr: {
3380 Code = bitc::FUNC_CODE_INST_BR;
3381 AbbrevToUse = FUNCTION_INST_BR_UNCOND_ABBREV;
3382 const UncondBrInst &II = cast<UncondBrInst>(Val: I);
3383 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 0)));
3384 } break;
3385 case Instruction::CondBr: {
3386 Code = bitc::FUNC_CODE_INST_BR;
3387 AbbrevToUse = FUNCTION_INST_BR_COND_ABBREV;
3388 const CondBrInst &II = cast<CondBrInst>(Val: I);
3389 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 0)));
3390 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 1)));
3391 pushValue(V: II.getCondition(), InstID, Vals);
3392 } break;
3393 case Instruction::Switch:
3394 {
3395 Code = bitc::FUNC_CODE_INST_SWITCH;
3396 const SwitchInst &SI = cast<SwitchInst>(Val: I);
3397 Vals.push_back(Elt: VE.getTypeID(T: SI.getCondition()->getType()));
3398 pushValue(V: SI.getCondition(), InstID, Vals);
3399 Vals.push_back(Elt: VE.getValueID(V: SI.getDefaultDest()));
3400 for (auto Case : SI.cases()) {
3401 Vals.push_back(Elt: VE.getValueID(V: Case.getCaseValue()));
3402 Vals.push_back(Elt: VE.getValueID(V: Case.getCaseSuccessor()));
3403 }
3404 }
3405 break;
3406 case Instruction::IndirectBr:
3407 Code = bitc::FUNC_CODE_INST_INDIRECTBR;
3408 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType()));
3409 // Encode the address operand as relative, but not the basic blocks.
3410 pushValue(V: I.getOperand(i: 0), InstID, Vals);
3411 for (const Value *Op : drop_begin(RangeOrContainer: I.operands()))
3412 Vals.push_back(Elt: VE.getValueID(V: Op));
3413 break;
3414
3415 case Instruction::Invoke: {
3416 const InvokeInst *II = cast<InvokeInst>(Val: &I);
3417 const Value *Callee = II->getCalledOperand();
3418 FunctionType *FTy = II->getFunctionType();
3419
3420 if (II->hasOperandBundles())
3421 writeOperandBundles(CS: *II, InstID);
3422
3423 Code = bitc::FUNC_CODE_INST_INVOKE;
3424
3425 Vals.push_back(Elt: VE.getAttributeListID(PAL: II->getAttributes()));
3426 Vals.push_back(Elt: II->getCallingConv() | 1 << 13);
3427 Vals.push_back(Elt: VE.getValueID(V: II->getNormalDest()));
3428 Vals.push_back(Elt: VE.getValueID(V: II->getUnwindDest()));
3429 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3430 pushValueAndType(V: Callee, InstID, Vals);
3431
3432 // Emit value #'s for the fixed parameters.
3433 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3434 pushValue(V: I.getOperand(i), InstID, Vals); // fixed param.
3435
3436 // Emit type/value pairs for varargs params.
3437 if (FTy->isVarArg()) {
3438 for (unsigned i = FTy->getNumParams(), e = II->arg_size(); i != e; ++i)
3439 pushValueAndType(V: I.getOperand(i), InstID, Vals); // vararg
3440 }
3441 break;
3442 }
3443 case Instruction::Resume:
3444 Code = bitc::FUNC_CODE_INST_RESUME;
3445 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3446 break;
3447 case Instruction::CleanupRet: {
3448 Code = bitc::FUNC_CODE_INST_CLEANUPRET;
3449 const auto &CRI = cast<CleanupReturnInst>(Val: I);
3450 pushValue(V: CRI.getCleanupPad(), InstID, Vals);
3451 if (CRI.hasUnwindDest())
3452 Vals.push_back(Elt: VE.getValueID(V: CRI.getUnwindDest()));
3453 break;
3454 }
3455 case Instruction::CatchRet: {
3456 Code = bitc::FUNC_CODE_INST_CATCHRET;
3457 const auto &CRI = cast<CatchReturnInst>(Val: I);
3458 pushValue(V: CRI.getCatchPad(), InstID, Vals);
3459 Vals.push_back(Elt: VE.getValueID(V: CRI.getSuccessor()));
3460 break;
3461 }
3462 case Instruction::CleanupPad:
3463 case Instruction::CatchPad: {
3464 const auto &FuncletPad = cast<FuncletPadInst>(Val: I);
3465 Code = isa<CatchPadInst>(Val: FuncletPad) ? bitc::FUNC_CODE_INST_CATCHPAD
3466 : bitc::FUNC_CODE_INST_CLEANUPPAD;
3467 pushValue(V: FuncletPad.getParentPad(), InstID, Vals);
3468
3469 unsigned NumArgOperands = FuncletPad.arg_size();
3470 Vals.push_back(Elt: NumArgOperands);
3471 for (unsigned Op = 0; Op != NumArgOperands; ++Op)
3472 pushValueAndType(V: FuncletPad.getArgOperand(i: Op), InstID, Vals);
3473 break;
3474 }
3475 case Instruction::CatchSwitch: {
3476 Code = bitc::FUNC_CODE_INST_CATCHSWITCH;
3477 const auto &CatchSwitch = cast<CatchSwitchInst>(Val: I);
3478
3479 pushValue(V: CatchSwitch.getParentPad(), InstID, Vals);
3480
3481 unsigned NumHandlers = CatchSwitch.getNumHandlers();
3482 Vals.push_back(Elt: NumHandlers);
3483 for (const BasicBlock *CatchPadBB : CatchSwitch.handlers())
3484 Vals.push_back(Elt: VE.getValueID(V: CatchPadBB));
3485
3486 if (CatchSwitch.hasUnwindDest())
3487 Vals.push_back(Elt: VE.getValueID(V: CatchSwitch.getUnwindDest()));
3488 break;
3489 }
3490 case Instruction::CallBr: {
3491 const CallBrInst *CBI = cast<CallBrInst>(Val: &I);
3492 const Value *Callee = CBI->getCalledOperand();
3493 FunctionType *FTy = CBI->getFunctionType();
3494
3495 if (CBI->hasOperandBundles())
3496 writeOperandBundles(CS: *CBI, InstID);
3497
3498 Code = bitc::FUNC_CODE_INST_CALLBR;
3499
3500 Vals.push_back(Elt: VE.getAttributeListID(PAL: CBI->getAttributes()));
3501
3502 Vals.push_back(Elt: CBI->getCallingConv() << bitc::CALL_CCONV |
3503 1 << bitc::CALL_EXPLICIT_TYPE);
3504
3505 Vals.push_back(Elt: VE.getValueID(V: CBI->getDefaultDest()));
3506 Vals.push_back(Elt: CBI->getNumIndirectDests());
3507 for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i)
3508 Vals.push_back(Elt: VE.getValueID(V: CBI->getIndirectDest(i)));
3509
3510 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3511 pushValueAndType(V: Callee, InstID, Vals);
3512
3513 // Emit value #'s for the fixed parameters.
3514 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3515 pushValue(V: I.getOperand(i), InstID, Vals); // fixed param.
3516
3517 // Emit type/value pairs for varargs params.
3518 if (FTy->isVarArg()) {
3519 for (unsigned i = FTy->getNumParams(), e = CBI->arg_size(); i != e; ++i)
3520 pushValueAndType(V: I.getOperand(i), InstID, Vals); // vararg
3521 }
3522 break;
3523 }
3524 case Instruction::Unreachable:
3525 Code = bitc::FUNC_CODE_INST_UNREACHABLE;
3526 AbbrevToUse = FUNCTION_INST_UNREACHABLE_ABBREV;
3527 break;
3528
3529 case Instruction::PHI: {
3530 const PHINode &PN = cast<PHINode>(Val: I);
3531 Code = bitc::FUNC_CODE_INST_PHI;
3532 // With the newer instruction encoding, forward references could give
3533 // negative valued IDs. This is most common for PHIs, so we use
3534 // signed VBRs.
3535 SmallVector<uint64_t, 128> Vals64;
3536 Vals64.push_back(Elt: VE.getTypeID(T: PN.getType()));
3537 for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) {
3538 pushValueSigned(V: PN.getIncomingValue(i), InstID, Vals&: Vals64);
3539 Vals64.push_back(Elt: VE.getValueID(V: PN.getIncomingBlock(i)));
3540 }
3541
3542 uint64_t Flags = getOptimizationFlags(V: &I);
3543 if (Flags != 0)
3544 Vals64.push_back(Elt: Flags);
3545
3546 // Emit a Vals64 vector and exit.
3547 Stream.EmitRecord(Code, Vals: Vals64, Abbrev: AbbrevToUse);
3548 Vals64.clear();
3549 return;
3550 }
3551
3552 case Instruction::LandingPad: {
3553 const LandingPadInst &LP = cast<LandingPadInst>(Val: I);
3554 Code = bitc::FUNC_CODE_INST_LANDINGPAD;
3555 Vals.push_back(Elt: VE.getTypeID(T: LP.getType()));
3556 Vals.push_back(Elt: LP.isCleanup());
3557 Vals.push_back(Elt: LP.getNumClauses());
3558 for (unsigned I = 0, E = LP.getNumClauses(); I != E; ++I) {
3559 if (LP.isCatch(Idx: I))
3560 Vals.push_back(Elt: LandingPadInst::Catch);
3561 else
3562 Vals.push_back(Elt: LandingPadInst::Filter);
3563 pushValueAndType(V: LP.getClause(Idx: I), InstID, Vals);
3564 }
3565 break;
3566 }
3567
3568 case Instruction::Alloca: {
3569 Code = bitc::FUNC_CODE_INST_ALLOCA;
3570 const AllocaInst &AI = cast<AllocaInst>(Val: I);
3571 Vals.push_back(Elt: VE.getTypeID(T: AI.getAllocatedType()));
3572 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType()));
3573 Vals.push_back(Elt: VE.getValueID(V: I.getOperand(i: 0))); // size.
3574 using APV = AllocaPackedValues;
3575 unsigned Record = 0;
3576 unsigned EncodedAlign = getEncodedAlign(Alignment: AI.getAlign());
3577 Bitfield::set<APV::AlignLower>(
3578 Packed&: Record, Value: EncodedAlign & ((1 << APV::AlignLower::Bits) - 1));
3579 Bitfield::set<APV::AlignUpper>(Packed&: Record,
3580 Value: EncodedAlign >> APV::AlignLower::Bits);
3581 Bitfield::set<APV::UsedWithInAlloca>(Packed&: Record, Value: AI.isUsedWithInAlloca());
3582 Bitfield::set<APV::ExplicitType>(Packed&: Record, Value: true);
3583 Bitfield::set<APV::SwiftError>(Packed&: Record, Value: AI.isSwiftError());
3584 Vals.push_back(Elt: Record);
3585
3586 unsigned AS = AI.getAddressSpace();
3587 if (AS != M.getDataLayout().getAllocaAddrSpace())
3588 Vals.push_back(Elt: AS);
3589 break;
3590 }
3591
3592 case Instruction::Load: {
3593 const auto &LI = cast<LoadInst>(Val: I);
3594 if (LI.isAtomic()) {
3595 Code = bitc::FUNC_CODE_INST_LOADATOMIC;
3596 pushValueAndType(V: LI.getOperand(i_nocapture: 0), InstID, Vals);
3597 } else {
3598 Code = bitc::FUNC_CODE_INST_LOAD;
3599 if (!pushValueAndType(V: LI.getOperand(i_nocapture: 0), InstID, Vals)) // ptr
3600 AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
3601 }
3602 Vals.push_back(Elt: VE.getTypeID(T: LI.getType()));
3603 Vals.push_back(Elt: getEncodedAlign(Alignment: LI.getAlign()));
3604 Vals.push_back(Elt: LI.isVolatile());
3605 if (LI.isAtomic()) {
3606 Vals.push_back(Elt: getEncodedOrdering(Ordering: LI.getOrdering()));
3607 Vals.push_back(Elt: getEncodedSyncScopeID(SSID: LI.getSyncScopeID()));
3608 if (LI.isElementwise())
3609 Vals.push_back(Elt: 1);
3610 }
3611 break;
3612 }
3613
3614 case Instruction::Store: {
3615 const auto &SI = cast<StoreInst>(Val: I);
3616 if (SI.isAtomic()) {
3617 Code = bitc::FUNC_CODE_INST_STOREATOMIC;
3618 } else {
3619 Code = bitc::FUNC_CODE_INST_STORE;
3620 AbbrevToUse = FUNCTION_INST_STORE_ABBREV;
3621 }
3622 if (pushValueAndType(V: I.getOperand(i: 1), InstID, Vals)) // ptrty + ptr
3623 AbbrevToUse = 0;
3624 if (pushValueAndType(V: I.getOperand(i: 0), InstID, Vals)) // valty + val
3625 AbbrevToUse = 0;
3626 Vals.push_back(Elt: getEncodedAlign(Alignment: SI.getAlign()));
3627 Vals.push_back(Elt: SI.isVolatile());
3628 if (SI.isAtomic()) {
3629 Vals.push_back(Elt: getEncodedOrdering(Ordering: SI.getOrdering()));
3630 Vals.push_back(Elt: getEncodedSyncScopeID(SSID: SI.getSyncScopeID()));
3631 if (SI.isElementwise())
3632 Vals.push_back(Elt: 1);
3633 }
3634 break;
3635 }
3636
3637 case Instruction::AtomicCmpXchg:
3638 Code = bitc::FUNC_CODE_INST_CMPXCHG;
3639 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals); // ptrty + ptr
3640 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals); // cmp.
3641 pushValue(V: I.getOperand(i: 2), InstID, Vals); // newval.
3642 Vals.push_back(Elt: cast<AtomicCmpXchgInst>(Val: I).isVolatile());
3643 Vals.push_back(
3644 Elt: getEncodedOrdering(Ordering: cast<AtomicCmpXchgInst>(Val: I).getSuccessOrdering()));
3645 Vals.push_back(
3646 Elt: getEncodedSyncScopeID(SSID: cast<AtomicCmpXchgInst>(Val: I).getSyncScopeID()));
3647 Vals.push_back(
3648 Elt: getEncodedOrdering(Ordering: cast<AtomicCmpXchgInst>(Val: I).getFailureOrdering()));
3649 Vals.push_back(Elt: cast<AtomicCmpXchgInst>(Val: I).isWeak());
3650 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<AtomicCmpXchgInst>(Val: I).getAlign()));
3651 break;
3652 case Instruction::AtomicRMW:
3653 Code = bitc::FUNC_CODE_INST_ATOMICRMW;
3654 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals); // ptrty + ptr
3655 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals); // valty + val
3656 Vals.push_back(Elt: getEncodedRMWOperation(I: cast<AtomicRMWInst>(Val: I)));
3657 Vals.push_back(Elt: cast<AtomicRMWInst>(Val: I).isVolatile());
3658 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<AtomicRMWInst>(Val: I).getOrdering()));
3659 Vals.push_back(
3660 Elt: getEncodedSyncScopeID(SSID: cast<AtomicRMWInst>(Val: I).getSyncScopeID()));
3661 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<AtomicRMWInst>(Val: I).getAlign()));
3662 break;
3663 case Instruction::Fence:
3664 Code = bitc::FUNC_CODE_INST_FENCE;
3665 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<FenceInst>(Val: I).getOrdering()));
3666 Vals.push_back(Elt: getEncodedSyncScopeID(SSID: cast<FenceInst>(Val: I).getSyncScopeID()));
3667 break;
3668 case Instruction::Call: {
3669 const CallInst &CI = cast<CallInst>(Val: I);
3670 FunctionType *FTy = CI.getFunctionType();
3671
3672 if (CI.hasOperandBundles())
3673 writeOperandBundles(CS: CI, InstID);
3674
3675 Code = bitc::FUNC_CODE_INST_CALL;
3676
3677 Vals.push_back(Elt: VE.getAttributeListID(PAL: CI.getAttributes()));
3678
3679 unsigned Flags = getOptimizationFlags(V: &I);
3680 Vals.push_back(Elt: CI.getCallingConv() << bitc::CALL_CCONV |
3681 unsigned(CI.isTailCall()) << bitc::CALL_TAIL |
3682 unsigned(CI.isMustTailCall()) << bitc::CALL_MUSTTAIL |
3683 1 << bitc::CALL_EXPLICIT_TYPE |
3684 unsigned(CI.isNoTailCall()) << bitc::CALL_NOTAIL |
3685 unsigned(Flags != 0) << bitc::CALL_FMF);
3686 if (Flags != 0)
3687 Vals.push_back(Elt: Flags);
3688
3689 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3690 pushValueAndType(V: CI.getCalledOperand(), InstID, Vals); // Callee
3691
3692 // Emit value #'s for the fixed parameters.
3693 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3694 pushValue(V: CI.getArgOperand(i), InstID, Vals); // fixed param.
3695
3696 // Emit type/value pairs for varargs params.
3697 if (FTy->isVarArg()) {
3698 for (unsigned i = FTy->getNumParams(), e = CI.arg_size(); i != e; ++i)
3699 pushValueAndType(V: CI.getArgOperand(i), InstID, Vals); // varargs
3700 }
3701 break;
3702 }
3703 case Instruction::VAArg:
3704 Code = bitc::FUNC_CODE_INST_VAARG;
3705 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType())); // valistty
3706 pushValue(V: I.getOperand(i: 0), InstID, Vals); // valist.
3707 Vals.push_back(Elt: VE.getTypeID(T: I.getType())); // restype.
3708 break;
3709 case Instruction::Freeze:
3710 Code = bitc::FUNC_CODE_INST_FREEZE;
3711 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3712 break;
3713 }
3714
3715 Stream.EmitRecord(Code, Vals, Abbrev: AbbrevToUse);
3716 Vals.clear();
3717}
3718
3719/// Write a GlobalValue VST to the module. The purpose of this data structure is
3720/// to allow clients to efficiently find the function body.
3721void ModuleBitcodeWriter::writeGlobalValueSymbolTable(
3722 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3723 // Get the offset of the VST we are writing, and backpatch it into
3724 // the VST forward declaration record.
3725 uint64_t VSTOffset = Stream.GetCurrentBitNo();
3726 // The BitcodeStartBit was the stream offset of the identification block.
3727 VSTOffset -= bitcodeStartBit();
3728 assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned");
3729 // Note that we add 1 here because the offset is relative to one word
3730 // before the start of the identification block, which was historically
3731 // always the start of the regular bitcode header.
3732 Stream.BackpatchWord(BitNo: VSTOffsetPlaceholder, Val: VSTOffset / 32 + 1);
3733
3734 Stream.EnterSubblock(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, CodeLen: 4);
3735
3736 auto Abbv = std::make_shared<BitCodeAbbrev>();
3737 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY));
3738 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
3739 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset
3740 unsigned FnEntryAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
3741
3742 for (const Function &F : M) {
3743 uint64_t Record[2];
3744
3745 if (F.isDeclaration())
3746 continue;
3747
3748 Record[0] = VE.getValueID(V: &F);
3749
3750 // Save the word offset of the function (from the start of the
3751 // actual bitcode written to the stream).
3752 uint64_t BitcodeIndex = FunctionToBitcodeIndex[&F] - bitcodeStartBit();
3753 assert((BitcodeIndex & 31) == 0 && "function block not 32-bit aligned");
3754 // Note that we add 1 here because the offset is relative to one word
3755 // before the start of the identification block, which was historically
3756 // always the start of the regular bitcode header.
3757 Record[1] = BitcodeIndex / 32 + 1;
3758
3759 Stream.EmitRecord(Code: bitc::VST_CODE_FNENTRY, Vals: Record, Abbrev: FnEntryAbbrev);
3760 }
3761
3762 Stream.ExitBlock();
3763}
3764
3765/// Emit names for arguments, instructions and basic blocks in a function.
3766void ModuleBitcodeWriter::writeFunctionLevelValueSymbolTable(
3767 const ValueSymbolTable &VST) {
3768 if (VST.empty())
3769 return;
3770
3771 Stream.EnterSubblock(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, CodeLen: 4);
3772
3773 // FIXME: Set up the abbrev, we know how many values there are!
3774 // FIXME: We know if the type names can use 7-bit ascii.
3775 SmallVector<uint64_t, 64> NameVals;
3776
3777 for (const ValueName &Name : VST) {
3778 // Figure out the encoding to use for the name.
3779 StringEncoding Bits = getStringEncoding(Str: Name.getKey());
3780
3781 unsigned AbbrevToUse = VST_ENTRY_8_ABBREV;
3782 NameVals.push_back(Elt: VE.getValueID(V: Name.getValue()));
3783
3784 // VST_CODE_ENTRY: [valueid, namechar x N]
3785 // VST_CODE_BBENTRY: [bbid, namechar x N]
3786 unsigned Code;
3787 if (isa<BasicBlock>(Val: Name.getValue())) {
3788 Code = bitc::VST_CODE_BBENTRY;
3789 if (Bits == SE_Char6)
3790 AbbrevToUse = VST_BBENTRY_6_ABBREV;
3791 } else {
3792 Code = bitc::VST_CODE_ENTRY;
3793 if (Bits == SE_Char6)
3794 AbbrevToUse = VST_ENTRY_6_ABBREV;
3795 else if (Bits == SE_Fixed7)
3796 AbbrevToUse = VST_ENTRY_7_ABBREV;
3797 }
3798
3799 for (const auto P : Name.getKey())
3800 NameVals.push_back(Elt: (unsigned char)P);
3801
3802 // Emit the finished record.
3803 Stream.EmitRecord(Code, Vals: NameVals, Abbrev: AbbrevToUse);
3804 NameVals.clear();
3805 }
3806
3807 Stream.ExitBlock();
3808}
3809
3810void ModuleBitcodeWriter::writeUseList(UseListOrder &&Order) {
3811 assert(Order.Shuffle.size() >= 2 && "Shuffle too small");
3812 unsigned Code;
3813 if (isa<BasicBlock>(Val: Order.V))
3814 Code = bitc::USELIST_CODE_BB;
3815 else
3816 Code = bitc::USELIST_CODE_DEFAULT;
3817
3818 SmallVector<uint64_t, 64> Record(Order.Shuffle.begin(), Order.Shuffle.end());
3819 Record.push_back(Elt: VE.getValueID(V: Order.V));
3820 Stream.EmitRecord(Code, Vals: Record);
3821}
3822
3823void ModuleBitcodeWriter::writeUseListBlock(const Function *F) {
3824 assert(VE.shouldPreserveUseListOrder() &&
3825 "Expected to be preserving use-list order");
3826
3827 auto hasMore = [&]() {
3828 return !VE.UseListOrders.empty() && VE.UseListOrders.back().F == F;
3829 };
3830 if (!hasMore())
3831 // Nothing to do.
3832 return;
3833
3834 Stream.EnterSubblock(BlockID: bitc::USELIST_BLOCK_ID, CodeLen: 3);
3835 while (hasMore()) {
3836 writeUseList(Order: std::move(VE.UseListOrders.back()));
3837 VE.UseListOrders.pop_back();
3838 }
3839 Stream.ExitBlock();
3840}
3841
3842/// Emit a function body to the module stream.
3843void ModuleBitcodeWriter::writeFunction(
3844 const Function &F,
3845 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3846 // Save the bitcode index of the start of this function block for recording
3847 // in the VST.
3848 FunctionToBitcodeIndex[&F] = Stream.GetCurrentBitNo();
3849
3850 Stream.EnterSubblock(BlockID: bitc::FUNCTION_BLOCK_ID, CodeLen: 5);
3851 VE.incorporateFunction(F);
3852
3853 SmallVector<unsigned, 64> Vals;
3854
3855 // Emit the number of basic blocks, so the reader can create them ahead of
3856 // time.
3857 Vals.push_back(Elt: VE.getBasicBlocks().size());
3858 Stream.EmitRecord(Code: bitc::FUNC_CODE_DECLAREBLOCKS, Vals);
3859 Vals.clear();
3860
3861 // If there are function-local constants, emit them now.
3862 unsigned CstStart, CstEnd;
3863 VE.getFunctionConstantRange(Start&: CstStart, End&: CstEnd);
3864 writeConstants(FirstVal: CstStart, LastVal: CstEnd, isGlobal: false);
3865
3866 // If there is function-local metadata, emit it now.
3867 writeFunctionMetadata(F);
3868
3869 // Keep a running idea of what the instruction ID is.
3870 unsigned InstID = CstEnd;
3871
3872 bool NeedsMetadataAttachment = F.hasMetadata();
3873
3874 DILocation *LastDL = nullptr;
3875 SmallSetVector<Function *, 4> BlockAddressUsers;
3876
3877 // Finally, emit all the instructions, in order.
3878 for (const BasicBlock &BB : F) {
3879 for (const Instruction &I : BB) {
3880 writeInstruction(I, InstID, Vals);
3881
3882 if (!I.getType()->isVoidTy())
3883 ++InstID;
3884
3885 // If the instruction has metadata, write a metadata attachment later.
3886 NeedsMetadataAttachment |= I.hasMetadataOtherThanDebugLoc();
3887
3888 // If the instruction has a debug location, emit it.
3889 if (DILocation *DL = I.getDebugLoc()) {
3890 if (DL == LastDL) {
3891 // Just repeat the same debug loc as last time.
3892 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_LOC_AGAIN, Vals);
3893 } else {
3894 Vals.push_back(Elt: DL->getLine());
3895 Vals.push_back(Elt: DL->getColumn());
3896 Vals.push_back(Elt: VE.getMetadataOrNullID(MD: DL->getScope()));
3897 Vals.push_back(Elt: VE.getMetadataOrNullID(MD: DL->getInlinedAt()));
3898 Vals.push_back(Elt: DL->isImplicitCode());
3899 Vals.push_back(Elt: DL->getAtomGroup());
3900 Vals.push_back(Elt: DL->getAtomRank());
3901 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_LOC, Vals,
3902 Abbrev: FUNCTION_DEBUG_LOC_ABBREV);
3903 Vals.clear();
3904 LastDL = DL;
3905 }
3906 }
3907
3908 // If the instruction has DbgRecords attached to it, emit them. Note that
3909 // they come after the instruction so that it's easy to attach them again
3910 // when reading the bitcode, even though conceptually the debug locations
3911 // start "before" the instruction.
3912 if (I.hasDbgRecords()) {
3913 /// Try to push the value only (unwrapped), otherwise push the
3914 /// metadata wrapped value. Returns true if the value was pushed
3915 /// without the ValueAsMetadata wrapper.
3916 auto PushValueOrMetadata = [&Vals, InstID,
3917 this](Metadata *RawLocation) {
3918 assert(RawLocation &&
3919 "RawLocation unexpectedly null in DbgVariableRecord");
3920 if (ValueAsMetadata *VAM = dyn_cast<ValueAsMetadata>(Val: RawLocation)) {
3921 SmallVector<unsigned, 2> ValAndType;
3922 // If the value is a fwd-ref the type is also pushed. We don't
3923 // want the type, so fwd-refs are kept wrapped (pushValueAndType
3924 // returns false if the value is pushed without type).
3925 if (!pushValueAndType(V: VAM->getValue(), InstID, Vals&: ValAndType)) {
3926 Vals.push_back(Elt: ValAndType[0]);
3927 return true;
3928 }
3929 }
3930 // The metadata is a DIArgList, or ValueAsMetadata wrapping a
3931 // fwd-ref. Push the metadata ID.
3932 Vals.push_back(Elt: VE.getMetadataID(MD: RawLocation));
3933 return false;
3934 };
3935
3936 // Write out non-instruction debug information attached to this
3937 // instruction. Write it after the instruction so that it's easy to
3938 // re-attach to the instruction reading the records in.
3939 for (DbgRecord &DR : I.getDbgMarker()->getDbgRecordRange()) {
3940 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
3941 Vals.push_back(Elt: VE.getMetadataID(MD: &*DLR->getDebugLoc()));
3942 Vals.push_back(Elt: VE.getMetadataID(MD: DLR->getLabel()));
3943 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_LABEL, Vals);
3944 Vals.clear();
3945 continue;
3946 }
3947
3948 // First 3 fields are common to all kinds:
3949 // DILocation, DILocalVariable, DIExpression
3950 // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE)
3951 // ..., LocationMetadata
3952 // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE - abbrev'd)
3953 // ..., Value
3954 // dbg_declare (FUNC_CODE_DEBUG_RECORD_DECLARE)
3955 // ..., LocationMetadata
3956 // dbg_assign (FUNC_CODE_DEBUG_RECORD_ASSIGN)
3957 // ..., LocationMetadata, DIAssignID, DIExpression, LocationMetadata
3958 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
3959 Vals.push_back(Elt: VE.getMetadataID(MD: &*DVR.getDebugLoc()));
3960 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getVariable()));
3961 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getExpression()));
3962 if (DVR.isDbgValue()) {
3963 if (PushValueOrMetadata(DVR.getRawLocation()))
3964 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE, Vals,
3965 Abbrev: FUNCTION_DEBUG_RECORD_VALUE_ABBREV);
3966 else
3967 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_VALUE, Vals);
3968 } else if (DVR.isDbgDeclare()) {
3969 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3970 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_DECLARE, Vals);
3971 } else if (DVR.isDbgDeclareValue()) {
3972 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3973 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_DECLARE_VALUE, Vals);
3974 } else {
3975 assert(DVR.isDbgAssign() && "Unexpected DbgRecord kind");
3976 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3977 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getAssignID()));
3978 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getAddressExpression()));
3979 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawAddress()));
3980 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_ASSIGN, Vals);
3981 }
3982 Vals.clear();
3983 }
3984 }
3985 }
3986
3987 if (BlockAddress *BA = BlockAddress::lookup(BB: &BB)) {
3988 SmallVector<Value *> Worklist{BA};
3989 SmallPtrSet<Value *, 8> Visited{BA};
3990 while (!Worklist.empty()) {
3991 Value *V = Worklist.pop_back_val();
3992 for (User *U : V->users()) {
3993 if (auto *I = dyn_cast<Instruction>(Val: U)) {
3994 Function *P = I->getFunction();
3995 if (P != &F)
3996 BlockAddressUsers.insert(X: P);
3997 } else if (isa<Constant>(Val: U) && !isa<GlobalValue>(Val: U) &&
3998 Visited.insert(Ptr: U).second)
3999 Worklist.push_back(Elt: U);
4000 }
4001 }
4002 }
4003 }
4004
4005 if (!BlockAddressUsers.empty()) {
4006 Vals.resize(N: BlockAddressUsers.size());
4007 for (auto I : llvm::enumerate(First&: BlockAddressUsers))
4008 Vals[I.index()] = VE.getValueID(V: I.value());
4009 Stream.EmitRecord(Code: bitc::FUNC_CODE_BLOCKADDR_USERS, Vals);
4010 Vals.clear();
4011 }
4012
4013 // Emit names for all the instructions etc.
4014 if (auto *Symtab = F.getValueSymbolTable())
4015 writeFunctionLevelValueSymbolTable(VST: *Symtab);
4016
4017 if (NeedsMetadataAttachment)
4018 writeFunctionMetadataAttachment(F);
4019 if (VE.shouldPreserveUseListOrder())
4020 writeUseListBlock(F: &F);
4021 VE.purgeFunction();
4022 Stream.ExitBlock();
4023}
4024
4025// Emit blockinfo, which defines the standard abbreviations etc.
4026void ModuleBitcodeWriter::writeBlockInfo() {
4027 // We only want to emit block info records for blocks that have multiple
4028 // instances: CONSTANTS_BLOCK, FUNCTION_BLOCK and VALUE_SYMTAB_BLOCK.
4029 // Other blocks can define their abbrevs inline.
4030 Stream.EnterBlockInfoBlock();
4031
4032 // Encode type indices using fixed size based on number of types.
4033 BitCodeAbbrevOp TypeAbbrevOp(BitCodeAbbrevOp::Fixed,
4034 VE.computeBitsRequiredForTypeIndices());
4035 // Encode value indices as 6-bit VBR.
4036 BitCodeAbbrevOp ValAbbrevOp(BitCodeAbbrevOp::VBR, 6);
4037
4038 { // 8-bit fixed-width VST_CODE_ENTRY/VST_CODE_BBENTRY strings.
4039 auto Abbv = std::make_shared<BitCodeAbbrev>();
4040 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3));
4041 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4042 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4043 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
4044 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
4045 VST_ENTRY_8_ABBREV)
4046 llvm_unreachable("Unexpected abbrev ordering!");
4047 }
4048
4049 { // 7-bit fixed width VST_CODE_ENTRY strings.
4050 auto Abbv = std::make_shared<BitCodeAbbrev>();
4051 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
4052 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4053 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4054 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
4055 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
4056 VST_ENTRY_7_ABBREV)
4057 llvm_unreachable("Unexpected abbrev ordering!");
4058 }
4059 { // 6-bit char6 VST_CODE_ENTRY strings.
4060 auto Abbv = std::make_shared<BitCodeAbbrev>();
4061 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
4062 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4063 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4064 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
4065 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
4066 VST_ENTRY_6_ABBREV)
4067 llvm_unreachable("Unexpected abbrev ordering!");
4068 }
4069 { // 6-bit char6 VST_CODE_BBENTRY strings.
4070 auto Abbv = std::make_shared<BitCodeAbbrev>();
4071 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_BBENTRY));
4072 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4073 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4074 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
4075 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
4076 VST_BBENTRY_6_ABBREV)
4077 llvm_unreachable("Unexpected abbrev ordering!");
4078 }
4079
4080 { // SETTYPE abbrev for CONSTANTS_BLOCK.
4081 auto Abbv = std::make_shared<BitCodeAbbrev>();
4082 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_SETTYPE));
4083 Abbv->Add(OpInfo: TypeAbbrevOp);
4084 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4085 CONSTANTS_SETTYPE_ABBREV)
4086 llvm_unreachable("Unexpected abbrev ordering!");
4087 }
4088
4089 { // INTEGER abbrev for CONSTANTS_BLOCK.
4090 auto Abbv = std::make_shared<BitCodeAbbrev>();
4091 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_INTEGER));
4092 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4093 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4094 CONSTANTS_INTEGER_ABBREV)
4095 llvm_unreachable("Unexpected abbrev ordering!");
4096 }
4097
4098 { // BYTE abbrev for CONSTANTS_BLOCK.
4099 auto Abbv = std::make_shared<BitCodeAbbrev>();
4100 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_BYTE));
4101 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4102 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4103 CONSTANTS_BYTE_ABBREV)
4104 llvm_unreachable("Unexpected abbrev ordering!");
4105 }
4106
4107 { // CE_CAST abbrev for CONSTANTS_BLOCK.
4108 auto Abbv = std::make_shared<BitCodeAbbrev>();
4109 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CE_CAST));
4110 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // cast opc
4111 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // typeid
4112 VE.computeBitsRequiredForTypeIndices()));
4113 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
4114
4115 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4116 CONSTANTS_CE_CAST_Abbrev)
4117 llvm_unreachable("Unexpected abbrev ordering!");
4118 }
4119 { // NULL abbrev for CONSTANTS_BLOCK.
4120 auto Abbv = std::make_shared<BitCodeAbbrev>();
4121 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_NULL));
4122 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4123 CONSTANTS_NULL_Abbrev)
4124 llvm_unreachable("Unexpected abbrev ordering!");
4125 }
4126
4127 // FIXME: This should only use space for first class types!
4128
4129 { // INST_LOAD abbrev for FUNCTION_BLOCK.
4130 auto Abbv = std::make_shared<BitCodeAbbrev>();
4131 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_LOAD));
4132 Abbv->Add(OpInfo: ValAbbrevOp); // Ptr
4133 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4134 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Align
4135 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
4136 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4137 FUNCTION_INST_LOAD_ABBREV)
4138 llvm_unreachable("Unexpected abbrev ordering!");
4139 }
4140 {
4141 auto Abbv = std::make_shared<BitCodeAbbrev>();
4142 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_STORE));
4143 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4144 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4145 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // align
4146 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
4147 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4148 FUNCTION_INST_STORE_ABBREV)
4149 llvm_unreachable("Unexpected abbrev ordering!");
4150 }
4151 { // INST_UNOP abbrev for FUNCTION_BLOCK.
4152 auto Abbv = std::make_shared<BitCodeAbbrev>();
4153 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
4154 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4155 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4156 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4157 FUNCTION_INST_UNOP_ABBREV)
4158 llvm_unreachable("Unexpected abbrev ordering!");
4159 }
4160 { // INST_UNOP_FLAGS abbrev for FUNCTION_BLOCK.
4161 auto Abbv = std::make_shared<BitCodeAbbrev>();
4162 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
4163 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4164 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4165 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4166 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4167 FUNCTION_INST_UNOP_FLAGS_ABBREV)
4168 llvm_unreachable("Unexpected abbrev ordering!");
4169 }
4170 { // INST_BINOP abbrev for FUNCTION_BLOCK.
4171 auto Abbv = std::make_shared<BitCodeAbbrev>();
4172 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
4173 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4174 Abbv->Add(OpInfo: ValAbbrevOp); // RHS
4175 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4176 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4177 FUNCTION_INST_BINOP_ABBREV)
4178 llvm_unreachable("Unexpected abbrev ordering!");
4179 }
4180 { // INST_BINOP_FLAGS abbrev for FUNCTION_BLOCK.
4181 auto Abbv = std::make_shared<BitCodeAbbrev>();
4182 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
4183 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4184 Abbv->Add(OpInfo: ValAbbrevOp); // RHS
4185 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4186 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4187 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4188 FUNCTION_INST_BINOP_FLAGS_ABBREV)
4189 llvm_unreachable("Unexpected abbrev ordering!");
4190 }
4191 { // INST_CAST abbrev for FUNCTION_BLOCK.
4192 auto Abbv = std::make_shared<BitCodeAbbrev>();
4193 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
4194 Abbv->Add(OpInfo: ValAbbrevOp); // OpVal
4195 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4196 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4197 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4198 FUNCTION_INST_CAST_ABBREV)
4199 llvm_unreachable("Unexpected abbrev ordering!");
4200 }
4201 { // INST_CAST_FLAGS abbrev for FUNCTION_BLOCK.
4202 auto Abbv = std::make_shared<BitCodeAbbrev>();
4203 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
4204 Abbv->Add(OpInfo: ValAbbrevOp); // OpVal
4205 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4206 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4207 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 9)); // flags
4208 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4209 FUNCTION_INST_CAST_FLAGS_ABBREV)
4210 llvm_unreachable("Unexpected abbrev ordering!");
4211 }
4212
4213 { // INST_RET abbrev for FUNCTION_BLOCK.
4214 auto Abbv = std::make_shared<BitCodeAbbrev>();
4215 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
4216 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4217 FUNCTION_INST_RET_VOID_ABBREV)
4218 llvm_unreachable("Unexpected abbrev ordering!");
4219 }
4220 { // INST_RET abbrev for FUNCTION_BLOCK.
4221 auto Abbv = std::make_shared<BitCodeAbbrev>();
4222 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
4223 Abbv->Add(OpInfo: ValAbbrevOp);
4224 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4225 FUNCTION_INST_RET_VAL_ABBREV)
4226 llvm_unreachable("Unexpected abbrev ordering!");
4227 }
4228 {
4229 auto Abbv = std::make_shared<BitCodeAbbrev>();
4230 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BR));
4231 // TODO: Use different abbrev for absolute value reference (succ0)?
4232 Abbv->Add(OpInfo: ValAbbrevOp); // succ0
4233 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4234 FUNCTION_INST_BR_UNCOND_ABBREV)
4235 llvm_unreachable("Unexpected abbrev ordering!");
4236 }
4237 {
4238 auto Abbv = std::make_shared<BitCodeAbbrev>();
4239 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BR));
4240 // TODO: Use different abbrev for absolute value references (succ0, succ1)?
4241 Abbv->Add(OpInfo: ValAbbrevOp); // succ0
4242 Abbv->Add(OpInfo: ValAbbrevOp); // succ1
4243 Abbv->Add(OpInfo: ValAbbrevOp); // cond
4244 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4245 FUNCTION_INST_BR_COND_ABBREV)
4246 llvm_unreachable("Unexpected abbrev ordering!");
4247 }
4248 { // INST_UNREACHABLE abbrev for FUNCTION_BLOCK.
4249 auto Abbv = std::make_shared<BitCodeAbbrev>();
4250 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNREACHABLE));
4251 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4252 FUNCTION_INST_UNREACHABLE_ABBREV)
4253 llvm_unreachable("Unexpected abbrev ordering!");
4254 }
4255 {
4256 auto Abbv = std::make_shared<BitCodeAbbrev>();
4257 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_GEP));
4258 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // flags
4259 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4260 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4261 Abbv->Add(OpInfo: ValAbbrevOp);
4262 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4263 FUNCTION_INST_GEP_ABBREV)
4264 llvm_unreachable("Unexpected abbrev ordering!");
4265 }
4266 {
4267 auto Abbv = std::make_shared<BitCodeAbbrev>();
4268 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CMP2));
4269 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4270 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4271 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // pred
4272 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4273 FUNCTION_INST_CMP_ABBREV)
4274 llvm_unreachable("Unexpected abbrev ordering!");
4275 }
4276 {
4277 auto Abbv = std::make_shared<BitCodeAbbrev>();
4278 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CMP2));
4279 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4280 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4281 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // pred
4282 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4283 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4284 FUNCTION_INST_CMP_FLAGS_ABBREV)
4285 llvm_unreachable("Unexpected abbrev ordering!");
4286 }
4287 {
4288 auto Abbv = std::make_shared<BitCodeAbbrev>();
4289 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE));
4290 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // dbgloc
4291 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // var
4292 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // expr
4293 Abbv->Add(OpInfo: ValAbbrevOp); // val
4294 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4295 FUNCTION_DEBUG_RECORD_VALUE_ABBREV)
4296 llvm_unreachable("Unexpected abbrev ordering! 1");
4297 }
4298 {
4299 auto Abbv = std::make_shared<BitCodeAbbrev>();
4300 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_LOC));
4301 // NOTE: No IsDistinct field for FUNC_CODE_DEBUG_LOC.
4302 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4303 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4304 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4305 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4306 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
4307 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Atom group.
4308 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 3)); // Atom rank.
4309 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4310 FUNCTION_DEBUG_LOC_ABBREV)
4311 llvm_unreachable("Unexpected abbrev ordering!");
4312 }
4313 Stream.ExitBlock();
4314}
4315
4316/// Write the module path strings, currently only used when generating
4317/// a combined index file.
4318void IndexBitcodeWriter::writeModStrings() {
4319 Stream.EnterSubblock(BlockID: bitc::MODULE_STRTAB_BLOCK_ID, CodeLen: 3);
4320
4321 // TODO: See which abbrev sizes we actually need to emit
4322
4323 // 8-bit fixed-width MST_ENTRY strings.
4324 auto Abbv = std::make_shared<BitCodeAbbrev>();
4325 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4326 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4327 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4328 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
4329 unsigned Abbrev8Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4330
4331 // 7-bit fixed width MST_ENTRY strings.
4332 Abbv = std::make_shared<BitCodeAbbrev>();
4333 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4334 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4335 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4336 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
4337 unsigned Abbrev7Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4338
4339 // 6-bit char6 MST_ENTRY strings.
4340 Abbv = std::make_shared<BitCodeAbbrev>();
4341 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4342 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4343 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4344 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
4345 unsigned Abbrev6Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4346
4347 // Module Hash, 160 bits SHA1. Optionally, emitted after each MST_CODE_ENTRY.
4348 Abbv = std::make_shared<BitCodeAbbrev>();
4349 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_HASH));
4350 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4351 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4352 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4353 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4354 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4355 unsigned AbbrevHash = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4356
4357 SmallVector<unsigned, 64> Vals;
4358 forEachModule(Callback: [&](const StringMapEntry<ModuleHash> &MPSE) {
4359 StringRef Key = MPSE.getKey();
4360 const auto &Hash = MPSE.getValue();
4361 StringEncoding Bits = getStringEncoding(Str: Key);
4362 unsigned AbbrevToUse = Abbrev8Bit;
4363 if (Bits == SE_Char6)
4364 AbbrevToUse = Abbrev6Bit;
4365 else if (Bits == SE_Fixed7)
4366 AbbrevToUse = Abbrev7Bit;
4367
4368 auto ModuleId = ModuleIdMap.size();
4369 ModuleIdMap[Key] = ModuleId;
4370 Vals.push_back(Elt: ModuleId);
4371 // Use bytes_begin/end() for unsigned char iteration.
4372 Vals.append(in_start: Key.bytes_begin(), in_end: Key.bytes_end());
4373
4374 // Emit the finished record.
4375 Stream.EmitRecord(Code: bitc::MST_CODE_ENTRY, Vals, Abbrev: AbbrevToUse);
4376
4377 // Emit an optional hash for the module now
4378 if (llvm::any_of(Range: Hash, P: [](uint32_t H) { return H; })) {
4379 Vals.assign(in_start: Hash.begin(), in_end: Hash.end());
4380 // Emit the hash record.
4381 Stream.EmitRecord(Code: bitc::MST_CODE_HASH, Vals, Abbrev: AbbrevHash);
4382 }
4383
4384 Vals.clear();
4385 });
4386 Stream.ExitBlock();
4387}
4388
4389/// Write the function type metadata related records that need to appear before
4390/// a function summary entry (whether per-module or combined).
4391template <typename Fn>
4392static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream,
4393 FunctionSummary *FS,
4394 Fn GetValueID) {
4395 if (!FS->type_tests().empty())
4396 Stream.EmitRecord(Code: bitc::FS_TYPE_TESTS, Vals: FS->type_tests());
4397
4398 SmallVector<uint64_t, 64> Record;
4399
4400 auto WriteVFuncIdVec = [&](uint64_t Ty,
4401 ArrayRef<FunctionSummary::VFuncId> VFs) {
4402 if (VFs.empty())
4403 return;
4404 Record.clear();
4405 for (auto &VF : VFs) {
4406 Record.push_back(Elt: VF.GUID);
4407 Record.push_back(Elt: VF.Offset);
4408 }
4409 Stream.EmitRecord(Code: Ty, Vals: Record);
4410 };
4411
4412 WriteVFuncIdVec(bitc::FS_TYPE_TEST_ASSUME_VCALLS,
4413 FS->type_test_assume_vcalls());
4414 WriteVFuncIdVec(bitc::FS_TYPE_CHECKED_LOAD_VCALLS,
4415 FS->type_checked_load_vcalls());
4416
4417 auto WriteConstVCallVec = [&](uint64_t Ty,
4418 ArrayRef<FunctionSummary::ConstVCall> VCs) {
4419 for (auto &VC : VCs) {
4420 Record.clear();
4421 Record.push_back(Elt: VC.VFunc.GUID);
4422 Record.push_back(Elt: VC.VFunc.Offset);
4423 llvm::append_range(C&: Record, R: VC.Args);
4424 Stream.EmitRecord(Code: Ty, Vals: Record);
4425 }
4426 };
4427
4428 WriteConstVCallVec(bitc::FS_TYPE_TEST_ASSUME_CONST_VCALL,
4429 FS->type_test_assume_const_vcalls());
4430 WriteConstVCallVec(bitc::FS_TYPE_CHECKED_LOAD_CONST_VCALL,
4431 FS->type_checked_load_const_vcalls());
4432
4433 auto WriteRange = [&](ConstantRange Range) {
4434 Range = Range.sextOrTrunc(BitWidth: FunctionSummary::ParamAccess::RangeWidth);
4435 assert(Range.getLower().getNumWords() == 1);
4436 assert(Range.getUpper().getNumWords() == 1);
4437 emitSignedInt64(Vals&: Record, V: *Range.getLower().getRawData());
4438 emitSignedInt64(Vals&: Record, V: *Range.getUpper().getRawData());
4439 };
4440
4441 if (!FS->paramAccesses().empty()) {
4442 Record.clear();
4443 for (auto &Arg : FS->paramAccesses()) {
4444 size_t UndoSize = Record.size();
4445 Record.push_back(Elt: Arg.ParamNo);
4446 WriteRange(Arg.Use);
4447 Record.push_back(Elt: Arg.Calls.size());
4448 for (auto &Call : Arg.Calls) {
4449 Record.push_back(Elt: Call.ParamNo);
4450 std::optional<unsigned> ValueID = GetValueID(Call.Callee);
4451 if (!ValueID) {
4452 // If ValueID is unknown we can't drop just this call, we must drop
4453 // entire parameter.
4454 Record.resize(N: UndoSize);
4455 break;
4456 }
4457 Record.push_back(Elt: *ValueID);
4458 WriteRange(Call.Offsets);
4459 }
4460 }
4461 if (!Record.empty())
4462 Stream.EmitRecord(Code: bitc::FS_PARAM_ACCESS, Vals: Record);
4463 }
4464}
4465
4466/// Collect type IDs from type tests used by function.
4467static void
4468getReferencedTypeIds(FunctionSummary *FS,
4469 std::set<GlobalValue::GUID> &ReferencedTypeIds) {
4470 if (!FS->type_tests().empty())
4471 for (auto &TT : FS->type_tests())
4472 ReferencedTypeIds.insert(x: TT);
4473
4474 auto GetReferencedTypesFromVFuncIdVec =
4475 [&](ArrayRef<FunctionSummary::VFuncId> VFs) {
4476 for (auto &VF : VFs)
4477 ReferencedTypeIds.insert(x: VF.GUID);
4478 };
4479
4480 GetReferencedTypesFromVFuncIdVec(FS->type_test_assume_vcalls());
4481 GetReferencedTypesFromVFuncIdVec(FS->type_checked_load_vcalls());
4482
4483 auto GetReferencedTypesFromConstVCallVec =
4484 [&](ArrayRef<FunctionSummary::ConstVCall> VCs) {
4485 for (auto &VC : VCs)
4486 ReferencedTypeIds.insert(x: VC.VFunc.GUID);
4487 };
4488
4489 GetReferencedTypesFromConstVCallVec(FS->type_test_assume_const_vcalls());
4490 GetReferencedTypesFromConstVCallVec(FS->type_checked_load_const_vcalls());
4491}
4492
4493static void writeWholeProgramDevirtResolutionByArg(
4494 SmallVector<uint64_t, 64> &NameVals, const std::vector<uint64_t> &args,
4495 const WholeProgramDevirtResolution::ByArg &ByArg) {
4496 NameVals.push_back(Elt: args.size());
4497 llvm::append_range(C&: NameVals, R: args);
4498
4499 NameVals.push_back(Elt: ByArg.TheKind);
4500 NameVals.push_back(Elt: ByArg.Info);
4501 NameVals.push_back(Elt: ByArg.Byte);
4502 NameVals.push_back(Elt: ByArg.Bit);
4503}
4504
4505static void writeWholeProgramDevirtResolution(
4506 SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
4507 uint64_t Id, const WholeProgramDevirtResolution &Wpd) {
4508 NameVals.push_back(Elt: Id);
4509
4510 NameVals.push_back(Elt: Wpd.TheKind);
4511 NameVals.push_back(Elt: StrtabBuilder.add(S: Wpd.SingleImplName));
4512 NameVals.push_back(Elt: Wpd.SingleImplName.size());
4513
4514 NameVals.push_back(Elt: Wpd.ResByArg.size());
4515 for (auto &A : Wpd.ResByArg)
4516 writeWholeProgramDevirtResolutionByArg(NameVals, args: A.first, ByArg: A.second);
4517}
4518
4519static void writeTypeIdSummaryRecord(SmallVector<uint64_t, 64> &NameVals,
4520 StringTableBuilder &StrtabBuilder,
4521 StringRef Id,
4522 const TypeIdSummary &Summary) {
4523 NameVals.push_back(Elt: StrtabBuilder.add(S: Id));
4524 NameVals.push_back(Elt: Id.size());
4525
4526 NameVals.push_back(Elt: Summary.TTRes.TheKind);
4527 NameVals.push_back(Elt: Summary.TTRes.SizeM1BitWidth);
4528 NameVals.push_back(Elt: Summary.TTRes.AlignLog2);
4529 NameVals.push_back(Elt: Summary.TTRes.SizeM1);
4530 NameVals.push_back(Elt: Summary.TTRes.BitMask);
4531 NameVals.push_back(Elt: Summary.TTRes.InlineBits);
4532
4533 for (auto &W : Summary.WPDRes)
4534 writeWholeProgramDevirtResolution(NameVals, StrtabBuilder, Id: W.first,
4535 Wpd: W.second);
4536}
4537
4538static void writeTypeIdCompatibleVtableSummaryRecord(
4539 SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
4540 StringRef Id, const TypeIdCompatibleVtableInfo &Summary,
4541 ValueEnumerator &VE) {
4542 NameVals.push_back(Elt: StrtabBuilder.add(S: Id));
4543 NameVals.push_back(Elt: Id.size());
4544
4545 for (auto &P : Summary) {
4546 NameVals.push_back(Elt: P.AddressPointOffset);
4547 NameVals.push_back(Elt: VE.getValueID(V: P.VTableVI.getValue()));
4548 }
4549}
4550
4551// Adds the allocation contexts to the CallStacks map. We simply use the
4552// size at the time the context was added as the CallStackId. This works because
4553// when we look up the call stacks later on we process the function summaries
4554// and their allocation records in the same exact order.
4555static void collectMemProfCallStacks(
4556 FunctionSummary *FS, std::function<LinearFrameId(unsigned)> GetStackIndex,
4557 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> &CallStacks) {
4558 // The interfaces in ProfileData/MemProf.h use a type alias for a stack frame
4559 // id offset into the index of the full stack frames. The ModuleSummaryIndex
4560 // currently uses unsigned. Make sure these stay in sync.
4561 static_assert(std::is_same_v<LinearFrameId, unsigned>);
4562 for (auto &AI : FS->allocs()) {
4563 for (auto &MIB : AI.MIBs) {
4564 SmallVector<unsigned> StackIdIndices;
4565 StackIdIndices.reserve(N: MIB.StackIdIndices.size());
4566 for (auto Id : MIB.StackIdIndices)
4567 StackIdIndices.push_back(Elt: GetStackIndex(Id));
4568 // The CallStackId is the size at the time this context was inserted.
4569 CallStacks.insert(KV: {CallStacks.size(), StackIdIndices});
4570 }
4571 }
4572}
4573
4574// Build the radix tree from the accumulated CallStacks, write out the resulting
4575// linearized radix tree array, and return the map of call stack positions into
4576// this array for use when writing the allocation records. The returned map is
4577// indexed by a CallStackId which in this case is implicitly determined by the
4578// order of function summaries and their allocation infos being written.
4579static DenseMap<CallStackId, LinearCallStackId> writeMemoryProfileRadixTree(
4580 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> &&CallStacks,
4581 BitstreamWriter &Stream, unsigned RadixAbbrev) {
4582 assert(!CallStacks.empty());
4583 DenseMap<unsigned, FrameStat> FrameHistogram =
4584 computeFrameHistogram<LinearFrameId>(MemProfCallStackData&: CallStacks);
4585 CallStackRadixTreeBuilder<LinearFrameId> Builder;
4586 // We don't need a MemProfFrameIndexes map as we have already converted the
4587 // full stack id hash to a linear offset into the StackIds array.
4588 Builder.build(MemProfCallStackData: std::move(CallStacks), /*MemProfFrameIndexes=*/nullptr,
4589 FrameHistogram);
4590 Stream.EmitRecord(Code: bitc::FS_CONTEXT_RADIX_TREE_ARRAY, Vals: Builder.getRadixArray(),
4591 Abbrev: RadixAbbrev);
4592 return Builder.takeCallStackPos();
4593}
4594
4595static void writeFunctionHeapProfileRecords(
4596 BitstreamWriter &Stream, FunctionSummary *FS, unsigned CallsiteAbbrev,
4597 unsigned AllocAbbrev, unsigned ContextIdAbbvId, bool PerModule,
4598 std::function<unsigned(const ValueInfo &VI)> GetValueID,
4599 std::function<unsigned(unsigned)> GetStackIndex,
4600 bool WriteContextSizeInfoIndex,
4601 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
4602 CallStackId &CallStackCount) {
4603 SmallVector<uint64_t> Record;
4604
4605 for (auto &CI : FS->callsites()) {
4606 Record.clear();
4607 // Per module callsite clones should always have a single entry of
4608 // value 0.
4609 assert(!PerModule || (CI.Clones.size() == 1 && CI.Clones[0] == 0));
4610 Record.push_back(Elt: GetValueID(CI.Callee));
4611 if (!PerModule) {
4612 Record.push_back(Elt: CI.StackIdIndices.size());
4613 Record.push_back(Elt: CI.Clones.size());
4614 }
4615 for (auto Id : CI.StackIdIndices)
4616 Record.push_back(Elt: GetStackIndex(Id));
4617 if (!PerModule)
4618 llvm::append_range(C&: Record, R: CI.Clones);
4619 Stream.EmitRecord(Code: PerModule ? bitc::FS_PERMODULE_CALLSITE_INFO
4620 : bitc::FS_COMBINED_CALLSITE_INFO,
4621 Vals: Record, Abbrev: CallsiteAbbrev);
4622 }
4623
4624 for (auto &AI : FS->allocs()) {
4625 Record.clear();
4626 // Per module alloc versions should always have a single entry of
4627 // value 0.
4628 assert(!PerModule || (AI.Versions.size() == 1 && AI.Versions[0] == 0));
4629 Record.push_back(Elt: AI.MIBs.size());
4630 if (!PerModule)
4631 Record.push_back(Elt: AI.Versions.size());
4632 for (auto &MIB : AI.MIBs) {
4633 Record.push_back(Elt: (uint8_t)MIB.AllocType);
4634 // The per-module summary always needs to include the alloc context, as we
4635 // use it during the thin link. For the combined index it is optional (see
4636 // comments where CombinedIndexMemProfContext is defined).
4637 if (PerModule || CombinedIndexMemProfContext) {
4638 // Record the index into the radix tree array for this context.
4639 assert(CallStackCount <= CallStackPos.size());
4640 Record.push_back(Elt: CallStackPos[CallStackCount++]);
4641 }
4642 }
4643 if (!PerModule)
4644 llvm::append_range(C&: Record, R: AI.Versions);
4645 assert(AI.ContextSizeInfos.empty() ||
4646 AI.ContextSizeInfos.size() == AI.MIBs.size());
4647 // Optionally emit the context size information if it exists.
4648 if (WriteContextSizeInfoIndex && !AI.ContextSizeInfos.empty()) {
4649 // The abbreviation id for the context ids record should have been created
4650 // if we are emitting the per-module index, which is where we write this
4651 // info.
4652 assert(ContextIdAbbvId);
4653 SmallVector<uint32_t> ContextIds;
4654 // At least one context id per ContextSizeInfos entry (MIB), broken into 2
4655 // halves.
4656 ContextIds.reserve(N: AI.ContextSizeInfos.size() * 2);
4657 for (auto &Infos : AI.ContextSizeInfos) {
4658 Record.push_back(Elt: Infos.size());
4659 for (auto [FullStackId, TotalSize] : Infos) {
4660 // The context ids are emitted separately as a fixed width array,
4661 // which is more efficient than a VBR given that these hashes are
4662 // typically close to 64-bits. The max fixed width entry is 32 bits so
4663 // it is split into 2.
4664 ContextIds.push_back(Elt: static_cast<uint32_t>(FullStackId >> 32));
4665 ContextIds.push_back(Elt: static_cast<uint32_t>(FullStackId));
4666 Record.push_back(Elt: TotalSize);
4667 }
4668 }
4669 // The context ids are expected by the reader to immediately precede the
4670 // associated alloc info record.
4671 Stream.EmitRecord(Code: bitc::FS_ALLOC_CONTEXT_IDS, Vals: ContextIds,
4672 Abbrev: ContextIdAbbvId);
4673 }
4674 Stream.EmitRecord(Code: PerModule
4675 ? bitc::FS_PERMODULE_ALLOC_INFO
4676 : (CombinedIndexMemProfContext
4677 ? bitc::FS_COMBINED_ALLOC_INFO
4678 : bitc::FS_COMBINED_ALLOC_INFO_NO_CONTEXT),
4679 Vals: Record, Abbrev: AllocAbbrev);
4680 }
4681}
4682
4683// Helper to emit a single function summary record.
4684void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
4685 SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
4686 unsigned ValueID, unsigned FSCallsProfileAbbrev, unsigned CallsiteAbbrev,
4687 unsigned AllocAbbrev, unsigned ContextIdAbbvId, const Function &F,
4688 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
4689 CallStackId &CallStackCount) {
4690 NameVals.push_back(Elt: ValueID);
4691
4692 FunctionSummary *FS = cast<FunctionSummary>(Val: Summary);
4693
4694 writeFunctionTypeMetadataRecords(
4695 Stream, FS, GetValueID: [&](const ValueInfo &VI) -> std::optional<unsigned> {
4696 return {VE.getValueID(V: VI.getValue())};
4697 });
4698
4699 auto SpecialRefCnts = FS->specialRefCounts();
4700 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: FS->flags()));
4701 NameVals.push_back(Elt: FS->instCount());
4702 NameVals.push_back(Elt: getEncodedFFlags(Flags: FS->fflags()));
4703 NameVals.push_back(Elt: FS->refs().size());
4704 NameVals.push_back(Elt: SpecialRefCnts.first); // rorefcnt
4705 NameVals.push_back(Elt: SpecialRefCnts.second); // worefcnt
4706
4707 for (auto &RI : FS->refs())
4708 NameVals.push_back(Elt: getValueId(VI: RI));
4709
4710 for (auto &ECI : FS->calls()) {
4711 NameVals.push_back(Elt: getValueId(VI: ECI.first));
4712 NameVals.push_back(Elt: getEncodedHotnessCallEdgeInfo(CI: ECI.second));
4713 }
4714
4715 // Emit the finished record.
4716 Stream.EmitRecord(Code: bitc::FS_PERMODULE_PROFILE, Vals: NameVals, Abbrev: FSCallsProfileAbbrev);
4717 NameVals.clear();
4718
4719 writeFunctionHeapProfileRecords(
4720 Stream, FS, CallsiteAbbrev, AllocAbbrev, ContextIdAbbvId,
4721 /*PerModule*/ true,
4722 /*GetValueId*/ GetValueID: [&](const ValueInfo &VI) { return getValueId(VI); },
4723 /*GetStackIndex*/ [&](unsigned I) { return I; },
4724 /*WriteContextSizeInfoIndex*/ true, CallStackPos, CallStackCount);
4725}
4726
4727// Collect the global value references in the given variable's initializer,
4728// and emit them in a summary record.
4729void ModuleBitcodeWriterBase::writeModuleLevelReferences(
4730 const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals,
4731 unsigned FSModRefsAbbrev, unsigned FSModVTableRefsAbbrev) {
4732 // Be a little lenient here, to accomodate older files without GUIDs
4733 // already computed and assigned as metadata.
4734 GlobalValue::GUID GUID = V.getGUIDOrFallback();
4735
4736 auto VI = Index->getValueInfo(GUID);
4737 if (!VI || VI.getSummaryList().empty()) {
4738 // Only declarations should not have a summary (a declaration might however
4739 // have a summary if the def was in module level asm).
4740 assert(V.isDeclaration());
4741 return;
4742 }
4743 auto *Summary = VI.getSummaryList()[0].get();
4744 NameVals.push_back(Elt: VE.getValueID(V: &V));
4745 GlobalVarSummary *VS = cast<GlobalVarSummary>(Val: Summary);
4746 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: VS->flags()));
4747 NameVals.push_back(Elt: getEncodedGVarFlags(Flags: VS->varflags()));
4748
4749 auto VTableFuncs = VS->vTableFuncs();
4750 if (!VTableFuncs.empty())
4751 NameVals.push_back(Elt: VS->refs().size());
4752
4753 unsigned SizeBeforeRefs = NameVals.size();
4754 for (auto &RI : VS->refs())
4755 NameVals.push_back(Elt: VE.getValueID(V: RI.getValue()));
4756 // Sort the refs for determinism output, the vector returned by FS->refs() has
4757 // been initialized from a DenseSet.
4758 llvm::sort(C: drop_begin(RangeOrContainer&: NameVals, N: SizeBeforeRefs));
4759
4760 if (VTableFuncs.empty())
4761 Stream.EmitRecord(Code: bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, Vals: NameVals,
4762 Abbrev: FSModRefsAbbrev);
4763 else {
4764 // VTableFuncs pairs should already be sorted by offset.
4765 for (auto &P : VTableFuncs) {
4766 NameVals.push_back(Elt: VE.getValueID(V: P.FuncVI.getValue()));
4767 NameVals.push_back(Elt: P.VTableOffset);
4768 }
4769
4770 Stream.EmitRecord(Code: bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS, Vals: NameVals,
4771 Abbrev: FSModVTableRefsAbbrev);
4772 }
4773 NameVals.clear();
4774}
4775
4776/// Emit the per-module summary section alongside the rest of
4777/// the module's bitcode.
4778void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
4779 // By default we compile with ThinLTO if the module has a summary, but the
4780 // client can request full LTO with a module flag.
4781 bool IsThinLTO = true;
4782 if (auto *MD =
4783 mdconst::extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: "ThinLTO")))
4784 IsThinLTO = MD->getZExtValue();
4785 Stream.EnterSubblock(BlockID: IsThinLTO ? bitc::GLOBALVAL_SUMMARY_BLOCK_ID
4786 : bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID,
4787 CodeLen: 4);
4788
4789 Stream.EmitRecord(
4790 Code: bitc::FS_VERSION,
4791 Vals: ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
4792
4793 // Write the index flags.
4794 uint64_t Flags = 0;
4795 // Bits 1-3 are set only in the combined index, skip them.
4796 if (Index->enableSplitLTOUnit())
4797 Flags |= 0x8;
4798 if (Index->hasUnifiedLTO())
4799 Flags |= 0x200;
4800
4801 Stream.EmitRecord(Code: bitc::FS_FLAGS, Vals: ArrayRef<uint64_t>{Flags});
4802
4803 if (Index->begin() == Index->end()) {
4804 Stream.ExitBlock();
4805 return;
4806 }
4807
4808 auto Abbv = std::make_shared<BitCodeAbbrev>();
4809 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_VALUE_GUID));
4810 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4811 // GUIDS often use up most of 64-bits, so encode as two Fixed 32.
4812 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4813 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4814 unsigned ValueGuidAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4815
4816 for (const auto &GVI : valueIds()) {
4817 Stream.EmitRecord(Code: bitc::FS_VALUE_GUID,
4818 Vals: ArrayRef<uint32_t>{GVI.second,
4819 static_cast<uint32_t>(GVI.first >> 32),
4820 static_cast<uint32_t>(GVI.first)},
4821 Abbrev: ValueGuidAbbrev);
4822 }
4823
4824 if (!Index->stackIds().empty()) {
4825 auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4826 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_STACK_IDS));
4827 // numids x stackid
4828 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4829 // The stack ids are hashes that are close to 64 bits in size, so emitting
4830 // as a pair of 32-bit fixed-width values is more efficient than a VBR.
4831 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4832 unsigned StackIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(StackIdAbbv));
4833 SmallVector<uint32_t> Vals;
4834 Vals.reserve(N: Index->stackIds().size() * 2);
4835 for (auto Id : Index->stackIds()) {
4836 Vals.push_back(Elt: static_cast<uint32_t>(Id >> 32));
4837 Vals.push_back(Elt: static_cast<uint32_t>(Id));
4838 }
4839 Stream.EmitRecord(Code: bitc::FS_STACK_IDS, Vals, Abbrev: StackIdAbbvId);
4840 }
4841
4842 unsigned ContextIdAbbvId = 0;
4843 if (metadataMayIncludeContextSizeInfo()) {
4844 // n x context id
4845 auto ContextIdAbbv = std::make_shared<BitCodeAbbrev>();
4846 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_ALLOC_CONTEXT_IDS));
4847 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4848 // The context ids are hashes that are close to 64 bits in size, so emitting
4849 // as a pair of 32-bit fixed-width values is more efficient than a VBR if we
4850 // are emitting them for all MIBs. Otherwise we use VBR to better compress 0
4851 // values that are expected to more frequently occur in an alloc's memprof
4852 // summary.
4853 if (metadataIncludesAllContextSizeInfo())
4854 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4855 else
4856 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4857 ContextIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(ContextIdAbbv));
4858 }
4859
4860 // Abbrev for FS_PERMODULE_PROFILE.
4861 Abbv = std::make_shared<BitCodeAbbrev>();
4862 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE));
4863 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4864 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // flags
4865 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
4866 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
4867 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4868 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
4869 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
4870 // numrefs x valueid, n x (valueid, hotness+tailcall flags)
4871 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4872 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4873 unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4874
4875 // Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS.
4876 Abbv = std::make_shared<BitCodeAbbrev>();
4877 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS));
4878 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4879 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4880 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
4881 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4882 unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4883
4884 // Abbrev for FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS.
4885 Abbv = std::make_shared<BitCodeAbbrev>();
4886 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS));
4887 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4888 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4889 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4890 // numrefs x valueid, n x (valueid , offset)
4891 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4892 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4893 unsigned FSModVTableRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4894
4895 // Abbrev for FS_ALIAS.
4896 Abbv = std::make_shared<BitCodeAbbrev>();
4897 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_ALIAS));
4898 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4899 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4900 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4901 unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4902
4903 // Abbrev for FS_TYPE_ID_METADATA
4904 Abbv = std::make_shared<BitCodeAbbrev>();
4905 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_TYPE_ID_METADATA));
4906 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid strtab index
4907 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid length
4908 // n x (valueid , offset)
4909 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4910 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4911 unsigned TypeIdCompatibleVtableAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4912
4913 Abbv = std::make_shared<BitCodeAbbrev>();
4914 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_CALLSITE_INFO));
4915 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4916 // n x stackidindex
4917 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4918 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4919 unsigned CallsiteAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4920
4921 Abbv = std::make_shared<BitCodeAbbrev>();
4922 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_ALLOC_INFO));
4923 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib
4924 // n x (alloc type, context radix tree index)
4925 // optional: nummib x (numcontext x total size)
4926 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4927 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4928 unsigned AllocAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4929
4930 Abbv = std::make_shared<BitCodeAbbrev>();
4931 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_CONTEXT_RADIX_TREE_ARRAY));
4932 // n x entry
4933 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4934 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4935 unsigned RadixAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4936
4937 // First walk through all the functions and collect the allocation contexts in
4938 // their associated summaries, for use in constructing a radix tree of
4939 // contexts. Note that we need to do this in the same order as the functions
4940 // are processed further below since the call stack positions in the resulting
4941 // radix tree array are identified based on this order.
4942 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> CallStacks;
4943 for (const Function &F : M) {
4944 // Summary emission does not support anonymous functions, they have to be
4945 // renamed using the anonymous function renaming pass.
4946 if (!F.hasName())
4947 report_fatal_error(reason: "Unexpected anonymous function when writing summary");
4948
4949 // Be a little lenient here, to accomodate older files without GUIDs
4950 // already computed and assigned as metadata.
4951 GlobalValue::GUID GUID = F.getGUIDOrFallback();
4952
4953 ValueInfo VI = Index->getValueInfo(GUID);
4954 if (!VI || VI.getSummaryList().empty()) {
4955 // Only declarations should not have a summary (a declaration might
4956 // however have a summary if the def was in module level asm).
4957 if (!F.isDeclaration())
4958 reportFatalUsageError(reason: "expected function definition " + F.getName() +
4959 " to have an associated value info.");
4960 continue;
4961 }
4962 auto *Summary = VI.getSummaryList()[0].get();
4963 FunctionSummary *FS = cast<FunctionSummary>(Val: Summary);
4964 collectMemProfCallStacks(
4965 FS, /*GetStackIndex*/ [](unsigned I) { return I; }, CallStacks);
4966 }
4967 // Finalize the radix tree, write it out, and get the map of positions in the
4968 // linearized tree array.
4969 DenseMap<CallStackId, LinearCallStackId> CallStackPos;
4970 if (!CallStacks.empty()) {
4971 CallStackPos =
4972 writeMemoryProfileRadixTree(CallStacks: std::move(CallStacks), Stream, RadixAbbrev);
4973 }
4974
4975 // Keep track of the current index into the CallStackPos map.
4976 CallStackId CallStackCount = 0;
4977
4978 SmallVector<uint64_t, 64> NameVals;
4979 // Iterate over the list of functions instead of the Index to
4980 // ensure the ordering is stable.
4981 for (const Function &F : M) {
4982 // Summary emission does not support anonymous functions, they have to
4983 // renamed using the anonymous function renaming pass.
4984 if (!F.hasName())
4985 report_fatal_error(reason: "Unexpected anonymous function when writing summary");
4986
4987 GlobalValue::GUID GUID = F.getGUIDOrFallback();
4988
4989 ValueInfo VI = Index->getValueInfo(GUID);
4990 if (!VI || VI.getSummaryList().empty()) {
4991 // Only declarations should not have a summary (a declaration might
4992 // however have a summary if the def was in module level asm).
4993 assert(F.isDeclaration());
4994 continue;
4995 }
4996 auto *Summary = VI.getSummaryList()[0].get();
4997 writePerModuleFunctionSummaryRecord(NameVals, Summary, ValueID: VE.getValueID(V: &F),
4998 FSCallsProfileAbbrev, CallsiteAbbrev,
4999 AllocAbbrev, ContextIdAbbvId, F,
5000 CallStackPos, CallStackCount);
5001 }
5002
5003 // Capture references from GlobalVariable initializers, which are outside
5004 // of a function scope.
5005 for (const GlobalVariable &G : M.globals())
5006 writeModuleLevelReferences(V: G, NameVals, FSModRefsAbbrev,
5007 FSModVTableRefsAbbrev);
5008
5009 for (const GlobalAlias &A : M.aliases()) {
5010 auto *Aliasee = A.getAliaseeObject();
5011 // Skip ifunc and nameless functions which don't have an entry in the
5012 // summary.
5013 if (!Aliasee->hasName() || isa<GlobalIFunc>(Val: Aliasee))
5014 continue;
5015 auto AliasId = VE.getValueID(V: &A);
5016 auto AliaseeId = VE.getValueID(V: Aliasee);
5017 NameVals.push_back(Elt: AliasId);
5018 auto *Summary = Index->getGlobalValueSummary(GV: A);
5019 AliasSummary *AS = cast<AliasSummary>(Val: Summary);
5020 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: AS->flags()));
5021 NameVals.push_back(Elt: AliaseeId);
5022 Stream.EmitRecord(Code: bitc::FS_ALIAS, Vals: NameVals, Abbrev: FSAliasAbbrev);
5023 NameVals.clear();
5024 }
5025
5026 for (auto &S : Index->typeIdCompatibleVtableMap()) {
5027 writeTypeIdCompatibleVtableSummaryRecord(NameVals, StrtabBuilder, Id: S.first,
5028 Summary: S.second, VE);
5029 Stream.EmitRecord(Code: bitc::FS_TYPE_ID_METADATA, Vals: NameVals,
5030 Abbrev: TypeIdCompatibleVtableAbbrev);
5031 NameVals.clear();
5032 }
5033
5034 if (Index->getBlockCount())
5035 Stream.EmitRecord(Code: bitc::FS_BLOCK_COUNT,
5036 Vals: ArrayRef<uint64_t>{Index->getBlockCount()});
5037
5038 Stream.ExitBlock();
5039}
5040
5041void ModuleBitcodeWriterBase::writeGUIDList() {
5042 const ValueEnumerator::ValueList &Vals = VE.getValues();
5043 const size_t Max = Vals.size();
5044
5045 std::vector<GlobalValue::GUID> GUIDs(Max, 0);
5046 for (const GlobalValue &GV : M.global_values()) {
5047 auto MaybeGUID = GV.getGUIDIfAssigned();
5048 if (!MaybeGUID)
5049 continue;
5050 auto GUID = *MaybeGUID;
5051
5052 const auto ValueID = VE.getValueID(V: &GV);
5053 GUIDs[ValueID] = GUID;
5054 }
5055
5056 auto Abbv = std::make_shared<BitCodeAbbrev>();
5057 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_GUIDLIST));
5058 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5059 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
5060 unsigned GUIDListAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5061
5062 SmallVector<uint32_t> RecordVals;
5063 RecordVals.reserve(N: Max * 2);
5064 for (auto GUID : GUIDs) {
5065 RecordVals.push_back(Elt: static_cast<uint32_t>(GUID >> 32));
5066 RecordVals.push_back(Elt: static_cast<uint32_t>(GUID));
5067 }
5068
5069 Stream.EmitRecord(Code: bitc::MODULE_CODE_GUIDLIST, Vals: RecordVals, Abbrev: GUIDListAbbrev);
5070}
5071
5072/// Emit the combined summary section into the combined index file.
5073void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
5074 Stream.EnterSubblock(BlockID: bitc::GLOBALVAL_SUMMARY_BLOCK_ID, CodeLen: 4);
5075 Stream.EmitRecord(
5076 Code: bitc::FS_VERSION,
5077 Vals: ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
5078
5079 // Write the index flags.
5080 Stream.EmitRecord(Code: bitc::FS_FLAGS, Vals: ArrayRef<uint64_t>{Index.getFlags()});
5081
5082 auto Abbv = std::make_shared<BitCodeAbbrev>();
5083 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_VALUE_GUID));
5084 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
5085 // GUIDS often use up most of 64-bits, so encode as two Fixed 32.
5086 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
5087 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
5088 unsigned ValueGuidAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5089
5090 for (const auto &GVI : valueIds()) {
5091 Stream.EmitRecord(Code: bitc::FS_VALUE_GUID,
5092 Vals: ArrayRef<uint32_t>{GVI.second,
5093 static_cast<uint32_t>(GVI.first >> 32),
5094 static_cast<uint32_t>(GVI.first)},
5095 Abbrev: ValueGuidAbbrev);
5096 }
5097
5098 // Write the stack ids used by this index, which will be a subset of those in
5099 // the full index in the case of distributed indexes.
5100 if (!StackIds.empty()) {
5101 auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
5102 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_STACK_IDS));
5103 // numids x stackid
5104 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5105 // The stack ids are hashes that are close to 64 bits in size, so emitting
5106 // as a pair of 32-bit fixed-width values is more efficient than a VBR.
5107 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
5108 unsigned StackIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(StackIdAbbv));
5109 SmallVector<uint32_t> Vals;
5110 Vals.reserve(N: StackIds.size() * 2);
5111 for (auto Id : StackIds) {
5112 Vals.push_back(Elt: static_cast<uint32_t>(Id >> 32));
5113 Vals.push_back(Elt: static_cast<uint32_t>(Id));
5114 }
5115 Stream.EmitRecord(Code: bitc::FS_STACK_IDS, Vals, Abbrev: StackIdAbbvId);
5116 }
5117
5118 // Abbrev for FS_COMBINED_PROFILE.
5119 Abbv = std::make_shared<BitCodeAbbrev>();
5120 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_PROFILE));
5121 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5122 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
5123 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
5124 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
5125 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
5126 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // entrycount
5127 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
5128 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
5129 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
5130 // numrefs x valueid, n x (valueid, hotness+tailcall flags)
5131 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5132 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5133 unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5134
5135 // Abbrev for FS_COMBINED_GLOBALVAR_INIT_REFS.
5136 Abbv = std::make_shared<BitCodeAbbrev>();
5137 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS));
5138 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5139 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
5140 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
5141 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
5142 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5143 unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5144
5145 // Abbrev for FS_COMBINED_ALIAS.
5146 Abbv = std::make_shared<BitCodeAbbrev>();
5147 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_ALIAS));
5148 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5149 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
5150 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
5151 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5152 unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5153
5154 Abbv = std::make_shared<BitCodeAbbrev>();
5155 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_CALLSITE_INFO));
5156 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5157 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numstackindices
5158 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
5159 // numstackindices x stackidindex, numver x version
5160 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5161 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5162 unsigned CallsiteAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5163
5164 Abbv = std::make_shared<BitCodeAbbrev>();
5165 Abbv->Add(OpInfo: BitCodeAbbrevOp(CombinedIndexMemProfContext
5166 ? bitc::FS_COMBINED_ALLOC_INFO
5167 : bitc::FS_COMBINED_ALLOC_INFO_NO_CONTEXT));
5168 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib
5169 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
5170 // nummib x (alloc type, context radix tree index),
5171 // numver x version
5172 // optional: nummib x total size
5173 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5174 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5175 unsigned AllocAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5176
5177 auto shouldImportValueAsDecl = [&](GlobalValueSummary *GVS) -> bool {
5178 if (DecSummaries == nullptr)
5179 return false;
5180 return DecSummaries->count(Ptr: GVS);
5181 };
5182
5183 // The aliases are emitted as a post-pass, and will point to the value
5184 // id of the aliasee. Save them in a vector for post-processing.
5185 SmallVector<AliasSummary *, 64> Aliases;
5186
5187 // Save the value id for each summary for alias emission.
5188 DenseMap<const GlobalValueSummary *, unsigned> SummaryToValueIdMap;
5189
5190 SmallVector<uint64_t, 64> NameVals;
5191
5192 // Set that will be populated during call to writeFunctionTypeMetadataRecords
5193 // with the type ids referenced by this index file.
5194 std::set<GlobalValue::GUID> ReferencedTypeIds;
5195
5196 // For local linkage, we also emit the original name separately
5197 // immediately after the record.
5198 auto MaybeEmitOriginalName = [&](GlobalValueSummary &S) {
5199 // We don't need to emit the original name if we are writing the index for
5200 // distributed backends (in which case ModuleToSummariesForIndex is
5201 // non-null). The original name is only needed during the thin link, since
5202 // for SamplePGO the indirect call targets for local functions have
5203 // have the original name annotated in profile.
5204 // Continue to emit it when writing out the entire combined index, which is
5205 // used in testing the thin link via llvm-lto.
5206 if (ModuleToSummariesForIndex || !GlobalValue::isLocalLinkage(Linkage: S.linkage()))
5207 return;
5208 NameVals.push_back(Elt: S.getOriginalName());
5209 Stream.EmitRecord(Code: bitc::FS_COMBINED_ORIGINAL_NAME, Vals: NameVals);
5210 NameVals.clear();
5211 };
5212
5213 DenseMap<CallStackId, LinearCallStackId> CallStackPos;
5214 if (CombinedIndexMemProfContext) {
5215 Abbv = std::make_shared<BitCodeAbbrev>();
5216 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_CONTEXT_RADIX_TREE_ARRAY));
5217 // n x entry
5218 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5219 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5220 unsigned RadixAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5221
5222 // First walk through all the functions and collect the allocation contexts
5223 // in their associated summaries, for use in constructing a radix tree of
5224 // contexts. Note that we need to do this in the same order as the functions
5225 // are processed further below since the call stack positions in the
5226 // resulting radix tree array are identified based on this order.
5227 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> CallStacks;
5228 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
5229 // Don't collect this when invoked for an aliasee, as it is not needed for
5230 // the alias summary. If the aliasee is to be imported, we will invoke
5231 // this separately with IsAliasee=false.
5232 if (IsAliasee)
5233 return;
5234 GlobalValueSummary *S = I.second;
5235 assert(S);
5236 auto *FS = dyn_cast<FunctionSummary>(Val: S);
5237 if (!FS)
5238 return;
5239 collectMemProfCallStacks(
5240 FS,
5241 /*GetStackIndex*/
5242 [&](unsigned I) {
5243 // Get the corresponding index into the list of StackIds actually
5244 // being written for this combined index (which may be a subset in
5245 // the case of distributed indexes).
5246 assert(StackIdIndicesToIndex.contains(I));
5247 return StackIdIndicesToIndex[I];
5248 },
5249 CallStacks);
5250 });
5251 // Finalize the radix tree, write it out, and get the map of positions in
5252 // the linearized tree array.
5253 if (!CallStacks.empty()) {
5254 CallStackPos = writeMemoryProfileRadixTree(CallStacks: std::move(CallStacks), Stream,
5255 RadixAbbrev);
5256 }
5257 }
5258
5259 // Keep track of the current index into the CallStackPos map. Not used if
5260 // CombinedIndexMemProfContext is false.
5261 CallStackId CallStackCount = 0;
5262
5263 DenseSet<GlobalValue::GUID> DefOrUseGUIDs;
5264 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
5265 GlobalValueSummary *S = I.second;
5266 assert(S);
5267 DefOrUseGUIDs.insert(V: I.first);
5268 for (const ValueInfo &VI : S->refs())
5269 DefOrUseGUIDs.insert(V: VI.getGUID());
5270
5271 auto ValueId = getValueId(ValGUID: I.first);
5272 assert(ValueId);
5273 SummaryToValueIdMap[S] = *ValueId;
5274
5275 // If this is invoked for an aliasee, we want to record the above
5276 // mapping, but then not emit a summary entry (if the aliasee is
5277 // to be imported, we will invoke this separately with IsAliasee=false).
5278 if (IsAliasee)
5279 return;
5280
5281 if (auto *AS = dyn_cast<AliasSummary>(Val: S)) {
5282 // Will process aliases as a post-pass because the reader wants all
5283 // global to be loaded first.
5284 Aliases.push_back(Elt: AS);
5285 return;
5286 }
5287
5288 if (auto *VS = dyn_cast<GlobalVarSummary>(Val: S)) {
5289 NameVals.push_back(Elt: *ValueId);
5290 assert(ModuleIdMap.count(VS->modulePath()));
5291 NameVals.push_back(Elt: ModuleIdMap[VS->modulePath()]);
5292 NameVals.push_back(
5293 Elt: getEncodedGVSummaryFlags(Flags: VS->flags(), ImportAsDecl: shouldImportValueAsDecl(VS)));
5294 NameVals.push_back(Elt: getEncodedGVarFlags(Flags: VS->varflags()));
5295 for (auto &RI : VS->refs()) {
5296 auto RefValueId = getValueId(ValGUID: RI.getGUID());
5297 if (!RefValueId)
5298 continue;
5299 NameVals.push_back(Elt: *RefValueId);
5300 }
5301
5302 // Emit the finished record.
5303 Stream.EmitRecord(Code: bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, Vals: NameVals,
5304 Abbrev: FSModRefsAbbrev);
5305 NameVals.clear();
5306 MaybeEmitOriginalName(*S);
5307 return;
5308 }
5309
5310 auto GetValueId = [&](const ValueInfo &VI) -> std::optional<unsigned> {
5311 if (!VI)
5312 return std::nullopt;
5313 return getValueId(ValGUID: VI.getGUID());
5314 };
5315
5316 auto *FS = cast<FunctionSummary>(Val: S);
5317 writeFunctionTypeMetadataRecords(Stream, FS, GetValueID: GetValueId);
5318 getReferencedTypeIds(FS, ReferencedTypeIds);
5319
5320 NameVals.push_back(Elt: *ValueId);
5321 assert(ModuleIdMap.count(FS->modulePath()));
5322 NameVals.push_back(Elt: ModuleIdMap[FS->modulePath()]);
5323 NameVals.push_back(
5324 Elt: getEncodedGVSummaryFlags(Flags: FS->flags(), ImportAsDecl: shouldImportValueAsDecl(FS)));
5325 NameVals.push_back(Elt: FS->instCount());
5326 NameVals.push_back(Elt: getEncodedFFlags(Flags: FS->fflags()));
5327 // TODO: Stop writing entry count and bump bitcode version.
5328 NameVals.push_back(Elt: 0 /* EntryCount */);
5329
5330 // Fill in below
5331 NameVals.push_back(Elt: 0); // numrefs
5332 NameVals.push_back(Elt: 0); // rorefcnt
5333 NameVals.push_back(Elt: 0); // worefcnt
5334
5335 unsigned Count = 0, RORefCnt = 0, WORefCnt = 0;
5336 for (auto &RI : FS->refs()) {
5337 auto RefValueId = getValueId(ValGUID: RI.getGUID());
5338 if (!RefValueId)
5339 continue;
5340 NameVals.push_back(Elt: *RefValueId);
5341 if (RI.isReadOnly())
5342 RORefCnt++;
5343 else if (RI.isWriteOnly())
5344 WORefCnt++;
5345 Count++;
5346 }
5347 NameVals[6] = Count;
5348 NameVals[7] = RORefCnt;
5349 NameVals[8] = WORefCnt;
5350
5351 for (auto &EI : FS->calls()) {
5352 // If this GUID doesn't have a value id, it doesn't have a function
5353 // summary and we don't need to record any calls to it.
5354 std::optional<unsigned> CallValueId = GetValueId(EI.first);
5355 if (!CallValueId)
5356 continue;
5357 NameVals.push_back(Elt: *CallValueId);
5358 NameVals.push_back(Elt: getEncodedHotnessCallEdgeInfo(CI: EI.second));
5359 }
5360
5361 // Emit the finished record.
5362 Stream.EmitRecord(Code: bitc::FS_COMBINED_PROFILE, Vals: NameVals,
5363 Abbrev: FSCallsProfileAbbrev);
5364 NameVals.clear();
5365
5366 writeFunctionHeapProfileRecords(
5367 Stream, FS, CallsiteAbbrev, AllocAbbrev, /*ContextIdAbbvId*/ 0,
5368 /*PerModule*/ false,
5369 /*GetValueId*/
5370 GetValueID: [&](const ValueInfo &VI) -> unsigned {
5371 std::optional<unsigned> ValueID = GetValueId(VI);
5372 // This can happen in shared index files for distributed ThinLTO if
5373 // the callee function summary is not included. Record 0 which we
5374 // will have to deal with conservatively when doing any kind of
5375 // validation in the ThinLTO backends.
5376 if (!ValueID)
5377 return 0;
5378 return *ValueID;
5379 },
5380 /*GetStackIndex*/
5381 [&](unsigned I) {
5382 // Get the corresponding index into the list of StackIds actually
5383 // being written for this combined index (which may be a subset in
5384 // the case of distributed indexes).
5385 assert(StackIdIndicesToIndex.contains(I));
5386 return StackIdIndicesToIndex[I];
5387 },
5388 /*WriteContextSizeInfoIndex*/ false, CallStackPos, CallStackCount);
5389
5390 MaybeEmitOriginalName(*S);
5391 });
5392
5393 for (auto *AS : Aliases) {
5394 auto AliasValueId = SummaryToValueIdMap[AS];
5395 assert(AliasValueId);
5396 NameVals.push_back(Elt: AliasValueId);
5397 assert(ModuleIdMap.count(AS->modulePath()));
5398 NameVals.push_back(Elt: ModuleIdMap[AS->modulePath()]);
5399 NameVals.push_back(
5400 Elt: getEncodedGVSummaryFlags(Flags: AS->flags(), ImportAsDecl: shouldImportValueAsDecl(AS)));
5401 // Set value id to 0 when an alias is imported but the aliasee summary is
5402 // not contained in the index.
5403 auto AliaseeValueId =
5404 AS->hasAliasee() ? SummaryToValueIdMap[&AS->getAliasee()] : 0;
5405 NameVals.push_back(Elt: AliaseeValueId);
5406
5407 // Emit the finished record.
5408 Stream.EmitRecord(Code: bitc::FS_COMBINED_ALIAS, Vals: NameVals, Abbrev: FSAliasAbbrev);
5409 NameVals.clear();
5410 MaybeEmitOriginalName(*AS);
5411
5412 if (AS->hasAliasee())
5413 if (auto *FS = dyn_cast<FunctionSummary>(Val: &AS->getAliasee()))
5414 getReferencedTypeIds(FS, ReferencedTypeIds);
5415 }
5416
5417 SmallVector<std::pair<StringRef, GlobalValue::GUID>, 4> Functions;
5418 auto EmitCfiFunctions = [&](const CfiFunctionIndex &CfiIndex,
5419 bitc::GlobalValueSummarySymtabCodes Code) {
5420 if (CfiIndex.empty())
5421 return;
5422 for (GlobalValue::GUID GUID : DefOrUseGUIDs) {
5423 auto Names = CfiIndex.getNamesForGUID(GUID);
5424 for (StringRef Name : Names)
5425 Functions.push_back(Elt: {Name, GUID});
5426 }
5427 if (Functions.empty())
5428 return;
5429 llvm::sort(C&: Functions);
5430 for (const auto &Record : Functions) {
5431 NameVals.push_back(Elt: Record.second);
5432 NameVals.push_back(Elt: StrtabBuilder.add(S: Record.first));
5433 NameVals.push_back(Elt: Record.first.size());
5434 }
5435 Stream.EmitRecord(Code, Vals: NameVals);
5436 NameVals.clear();
5437 Functions.clear();
5438 };
5439
5440 EmitCfiFunctions(Index.cfiFunctionDefs(), bitc::FS_CFI_FUNCTION_DEFS);
5441 EmitCfiFunctions(Index.cfiFunctionDecls(), bitc::FS_CFI_FUNCTION_DECLS);
5442
5443 // Walk the GUIDs that were referenced, and write the
5444 // corresponding type id records.
5445 for (auto &T : ReferencedTypeIds) {
5446 auto TidIter = Index.typeIds().equal_range(x: T);
5447 for (const auto &[GUID, TypeIdPair] : make_range(p: TidIter)) {
5448 writeTypeIdSummaryRecord(NameVals, StrtabBuilder, Id: TypeIdPair.first,
5449 Summary: TypeIdPair.second);
5450 Stream.EmitRecord(Code: bitc::FS_TYPE_ID, Vals: NameVals);
5451 NameVals.clear();
5452 }
5453 }
5454
5455 if (Index.getBlockCount())
5456 Stream.EmitRecord(Code: bitc::FS_BLOCK_COUNT,
5457 Vals: ArrayRef<uint64_t>{Index.getBlockCount()});
5458
5459 Stream.ExitBlock();
5460}
5461
5462/// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the
5463/// current llvm version, and a record for the epoch number.
5464static void writeIdentificationBlock(BitstreamWriter &Stream) {
5465 Stream.EnterSubblock(BlockID: bitc::IDENTIFICATION_BLOCK_ID, CodeLen: 5);
5466
5467 // Write the "user readable" string identifying the bitcode producer
5468 auto Abbv = std::make_shared<BitCodeAbbrev>();
5469 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_STRING));
5470 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5471 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
5472 auto StringAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5473 writeStringRecord(Stream, Code: bitc::IDENTIFICATION_CODE_STRING,
5474 Str: "LLVM" LLVM_VERSION_STRING, AbbrevToUse: StringAbbrev);
5475
5476 // Write the epoch version
5477 Abbv = std::make_shared<BitCodeAbbrev>();
5478 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_EPOCH));
5479 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
5480 auto EpochAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5481 constexpr std::array<unsigned, 1> Vals = {._M_elems: {bitc::BITCODE_CURRENT_EPOCH}};
5482 Stream.EmitRecord(Code: bitc::IDENTIFICATION_CODE_EPOCH, Vals, Abbrev: EpochAbbrev);
5483 Stream.ExitBlock();
5484}
5485
5486void ModuleBitcodeWriter::writeModuleHash(StringRef View) {
5487 // Emit the module's hash.
5488 // MODULE_CODE_HASH: [5*i32]
5489 if (GenerateHash) {
5490 uint32_t Vals[5];
5491 Hasher.update(Data: ArrayRef<uint8_t>(
5492 reinterpret_cast<const uint8_t *>(View.data()), View.size()));
5493 std::array<uint8_t, 20> Hash = Hasher.result();
5494 for (int Pos = 0; Pos < 20; Pos += 4) {
5495 Vals[Pos / 4] = support::endian::read32be(P: Hash.data() + Pos);
5496 }
5497
5498 // Emit the finished record.
5499 Stream.EmitRecord(Code: bitc::MODULE_CODE_HASH, Vals);
5500
5501 if (ModHash)
5502 // Save the written hash value.
5503 llvm::copy(Range&: Vals, Out: std::begin(cont&: *ModHash));
5504 }
5505}
5506
5507void ModuleBitcodeWriter::write() {
5508 writeIdentificationBlock(Stream);
5509
5510 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5511 // We will want to write the module hash at this point. Block any flushing so
5512 // we can have access to the whole underlying data later.
5513 Stream.markAndBlockFlushing();
5514
5515 writeModuleVersion();
5516
5517 // Emit blockinfo, which defines the standard abbreviations etc.
5518 writeBlockInfo();
5519
5520 // Emit information describing all of the types in the module.
5521 writeTypeTable();
5522
5523 // Emit information about attribute groups.
5524 writeAttributeGroupTable();
5525
5526 // Emit information about parameter attributes.
5527 writeAttributeTable();
5528
5529 writeComdats();
5530
5531 // Emit top-level description of module, including target triple, inline asm,
5532 // descriptors for global variables, and function prototype info.
5533 writeModuleInfo();
5534
5535 // Emit constants.
5536 writeModuleConstants();
5537
5538 // Emit metadata kind names.
5539 writeModuleMetadataKinds();
5540
5541 // Emit metadata.
5542 writeModuleMetadata();
5543
5544 // Emit module-level use-lists.
5545 if (VE.shouldPreserveUseListOrder())
5546 writeUseListBlock(F: nullptr);
5547
5548 writeOperandBundleTags();
5549 writeSyncScopeNames();
5550
5551 // Emit function bodies.
5552 DenseMap<const Function *, uint64_t> FunctionToBitcodeIndex;
5553 for (const Function &F : M)
5554 if (!F.isDeclaration())
5555 writeFunction(F, FunctionToBitcodeIndex);
5556
5557 // Need to write after the above call to WriteFunction which populates
5558 // the summary information in the index.
5559 if (Index)
5560 writePerModuleGlobalValueSummary();
5561
5562 writeGlobalValueSymbolTable(FunctionToBitcodeIndex);
5563
5564 writeModuleHash(View: Stream.getMarkedBufferAndResumeFlushing());
5565
5566 Stream.ExitBlock();
5567}
5568
5569static void writeInt32ToBuffer(uint32_t Value, SmallVectorImpl<char> &Buffer,
5570 uint32_t &Position) {
5571 support::endian::write32le(P: &Buffer[Position], V: Value);
5572 Position += 4;
5573}
5574
5575/// If generating a bc file on darwin, we have to emit a
5576/// header and trailer to make it compatible with the system archiver. To do
5577/// this we emit the following header, and then emit a trailer that pads the
5578/// file out to be a multiple of 16 bytes.
5579///
5580/// struct bc_header {
5581/// uint32_t Magic; // 0x0B17C0DE
5582/// uint32_t Version; // Version, currently always 0.
5583/// uint32_t BitcodeOffset; // Offset to traditional bitcode file.
5584/// uint32_t BitcodeSize; // Size of traditional bitcode file.
5585/// uint32_t CPUType; // CPU specifier.
5586/// ... potentially more later ...
5587/// };
5588static void emitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer,
5589 const Triple &TT) {
5590 unsigned CPUType = ~0U;
5591
5592 // Match x86_64-*, i[3-9]86-*, powerpc-*, powerpc64-*, arm-*, thumb-*,
5593 // armv[0-9]-*, thumbv[0-9]-*, armv5te-*, or armv6t2-*. The CPUType is a magic
5594 // number from /usr/include/mach/machine.h. It is ok to reproduce the
5595 // specific constants here because they are implicitly part of the Darwin ABI.
5596 enum {
5597 DARWIN_CPU_ARCH_ABI64 = 0x01000000,
5598 DARWIN_CPU_TYPE_X86 = 7,
5599 DARWIN_CPU_TYPE_ARM = 12,
5600 DARWIN_CPU_TYPE_POWERPC = 18
5601 };
5602
5603 Triple::ArchType Arch = TT.getArch();
5604 if (Arch == Triple::x86_64)
5605 CPUType = DARWIN_CPU_TYPE_X86 | DARWIN_CPU_ARCH_ABI64;
5606 else if (Arch == Triple::x86)
5607 CPUType = DARWIN_CPU_TYPE_X86;
5608 else if (Arch == Triple::ppc)
5609 CPUType = DARWIN_CPU_TYPE_POWERPC;
5610 else if (Arch == Triple::ppc64)
5611 CPUType = DARWIN_CPU_TYPE_POWERPC | DARWIN_CPU_ARCH_ABI64;
5612 else if (Arch == Triple::arm || Arch == Triple::thumb)
5613 CPUType = DARWIN_CPU_TYPE_ARM;
5614
5615 // Traditional Bitcode starts after header.
5616 assert(Buffer.size() >= BWH_HeaderSize &&
5617 "Expected header size to be reserved");
5618 unsigned BCOffset = BWH_HeaderSize;
5619 unsigned BCSize = Buffer.size() - BWH_HeaderSize;
5620
5621 // Write the magic and version.
5622 unsigned Position = 0;
5623 writeInt32ToBuffer(Value: 0x0B17C0DE, Buffer, Position);
5624 writeInt32ToBuffer(Value: 0, Buffer, Position); // Version.
5625 writeInt32ToBuffer(Value: BCOffset, Buffer, Position);
5626 writeInt32ToBuffer(Value: BCSize, Buffer, Position);
5627 writeInt32ToBuffer(Value: CPUType, Buffer, Position);
5628
5629 // If the file is not a multiple of 16 bytes, insert dummy padding.
5630 while (Buffer.size() & 15)
5631 Buffer.push_back(Elt: 0);
5632}
5633
5634/// Helper to write the header common to all bitcode files.
5635static void writeBitcodeHeader(BitstreamWriter &Stream) {
5636 // Emit the file header.
5637 Stream.Emit(Val: (unsigned)'B', NumBits: 8);
5638 Stream.Emit(Val: (unsigned)'C', NumBits: 8);
5639 Stream.Emit(Val: 0x0, NumBits: 4);
5640 Stream.Emit(Val: 0xC, NumBits: 4);
5641 Stream.Emit(Val: 0xE, NumBits: 4);
5642 Stream.Emit(Val: 0xD, NumBits: 4);
5643}
5644
5645BitcodeWriter::BitcodeWriter(SmallVectorImpl<char> &Buffer)
5646 : Stream(new BitstreamWriter(Buffer)) {
5647 writeBitcodeHeader(Stream&: *Stream);
5648}
5649
5650BitcodeWriter::BitcodeWriter(raw_ostream &FS)
5651 : Stream(new BitstreamWriter(FS, FlushThreshold)) {
5652 writeBitcodeHeader(Stream&: *Stream);
5653}
5654
5655BitcodeWriter::~BitcodeWriter() { assert(WroteStrtab); }
5656
5657void BitcodeWriter::writeBlob(unsigned Block, unsigned Record, StringRef Blob) {
5658 Stream->EnterSubblock(BlockID: Block, CodeLen: 3);
5659
5660 auto Abbv = std::make_shared<BitCodeAbbrev>();
5661 Abbv->Add(OpInfo: BitCodeAbbrevOp(Record));
5662 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
5663 auto AbbrevNo = Stream->EmitAbbrev(Abbv: std::move(Abbv));
5664
5665 Stream->EmitRecordWithBlob(Abbrev: AbbrevNo, Vals: ArrayRef<uint64_t>{Record}, Blob);
5666
5667 Stream->ExitBlock();
5668}
5669
5670void BitcodeWriter::writeSymtab() {
5671 assert(!WroteStrtab && !WroteSymtab);
5672
5673 // If any module has module-level inline asm, we will require a registered asm
5674 // parser for the target so that we can create an accurate symbol table for
5675 // the module.
5676 for (Module *M : Mods) {
5677 if (M->getModuleInlineAsm().empty())
5678 continue;
5679
5680 std::string Err;
5681 const Triple TT(M->getTargetTriple());
5682 const Target *T = TargetRegistry::lookupTarget(TheTriple: TT, Error&: Err);
5683 if (!T || !T->hasMCAsmParser())
5684 return;
5685 }
5686
5687 WroteSymtab = true;
5688 SmallVector<char, 0> Symtab;
5689 // The irsymtab::build function may be unable to create a symbol table if the
5690 // module is malformed (e.g. it contains an invalid alias). Writing a symbol
5691 // table is not required for correctness, but we still want to be able to
5692 // write malformed modules to bitcode files, so swallow the error.
5693 if (Error E = irsymtab::build(Mods, Symtab, StrtabBuilder, Alloc)) {
5694 consumeError(Err: std::move(E));
5695 return;
5696 }
5697
5698 writeBlob(Block: bitc::SYMTAB_BLOCK_ID, Record: bitc::SYMTAB_BLOB,
5699 Blob: {Symtab.data(), Symtab.size()});
5700}
5701
5702void BitcodeWriter::writeStrtab() {
5703 assert(!WroteStrtab);
5704
5705 std::vector<char> Strtab;
5706 StrtabBuilder.finalizeInOrder();
5707 Strtab.resize(new_size: StrtabBuilder.getSize());
5708 StrtabBuilder.write(Buf: (uint8_t *)Strtab.data());
5709
5710 writeBlob(Block: bitc::STRTAB_BLOCK_ID, Record: bitc::STRTAB_BLOB,
5711 Blob: {Strtab.data(), Strtab.size()});
5712
5713 WroteStrtab = true;
5714}
5715
5716void BitcodeWriter::copyStrtab(StringRef Strtab) {
5717 writeBlob(Block: bitc::STRTAB_BLOCK_ID, Record: bitc::STRTAB_BLOB, Blob: Strtab);
5718 WroteStrtab = true;
5719}
5720
5721void BitcodeWriter::writeModule(const Module &M,
5722 bool ShouldPreserveUseListOrder,
5723 const ModuleSummaryIndex *Index,
5724 bool GenerateHash, ModuleHash *ModHash) {
5725 assert(!WroteStrtab);
5726
5727 // The Mods vector is used by irsymtab::build, which requires non-const
5728 // Modules in case it needs to materialize metadata. But the bitcode writer
5729 // requires that the module is materialized, so we can cast to non-const here,
5730 // after checking that it is in fact materialized.
5731 assert(M.isMaterialized());
5732 Mods.push_back(x: const_cast<Module *>(&M));
5733
5734 ModuleBitcodeWriter ModuleWriter(M, StrtabBuilder, *Stream,
5735 ShouldPreserveUseListOrder, Index,
5736 GenerateHash, ModHash);
5737 ModuleWriter.write();
5738}
5739
5740void BitcodeWriter::writeIndex(
5741 const ModuleSummaryIndex *Index,
5742 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex,
5743 const GVSummaryPtrSet *DecSummaries) {
5744 IndexBitcodeWriter IndexWriter(*Stream, StrtabBuilder, *Index, DecSummaries,
5745 ModuleToSummariesForIndex);
5746 IndexWriter.write();
5747}
5748
5749/// Write the specified module to the specified output stream.
5750void llvm::WriteBitcodeToFile(const Module &M, raw_ostream &Out,
5751 bool ShouldPreserveUseListOrder,
5752 const ModuleSummaryIndex *Index,
5753 bool GenerateHash, ModuleHash *ModHash) {
5754 auto Write = [&](BitcodeWriter &Writer) {
5755 Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash,
5756 ModHash);
5757 Writer.writeSymtab();
5758 Writer.writeStrtab();
5759 };
5760 Triple TT(M.getTargetTriple());
5761 if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) {
5762 // If this is darwin or another generic macho target, reserve space for the
5763 // header. Note that the header is computed *after* the output is known, so
5764 // we currently explicitly use a buffer, write to it, and then subsequently
5765 // flush to Out.
5766 SmallVector<char, 0> Buffer;
5767 Buffer.reserve(N: 256 * 1024);
5768 Buffer.insert(I: Buffer.begin(), NumToInsert: BWH_HeaderSize, Elt: 0);
5769 BitcodeWriter Writer(Buffer);
5770 Write(Writer);
5771 emitDarwinBCHeaderAndTrailer(Buffer, TT);
5772 Out.write(Ptr: Buffer.data(), Size: Buffer.size());
5773 } else {
5774 BitcodeWriter Writer(Out);
5775 Write(Writer);
5776 }
5777}
5778
5779void IndexBitcodeWriter::write() {
5780 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5781
5782 writeModuleVersion();
5783
5784 // Write the module paths in the combined index.
5785 writeModStrings();
5786
5787 // Write the summary combined index records.
5788 writeCombinedGlobalValueSummary();
5789
5790 Stream.ExitBlock();
5791}
5792
5793// Write the specified module summary index to the given raw output stream,
5794// where it will be written in a new bitcode block. This is used when
5795// writing the combined index file for ThinLTO. When writing a subset of the
5796// index for a distributed backend, provide a \p ModuleToSummariesForIndex map.
5797void llvm::writeIndexToFile(
5798 const ModuleSummaryIndex &Index, raw_ostream &Out,
5799 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex,
5800 const GVSummaryPtrSet *DecSummaries) {
5801 SmallVector<char, 0> Buffer;
5802 Buffer.reserve(N: 256 * 1024);
5803
5804 BitcodeWriter Writer(Buffer);
5805 Writer.writeIndex(Index: &Index, ModuleToSummariesForIndex, DecSummaries);
5806 Writer.writeStrtab();
5807
5808 Out.write(Ptr: (char *)&Buffer.front(), Size: Buffer.size());
5809}
5810
5811namespace {
5812
5813/// Class to manage the bitcode writing for a thin link bitcode file.
5814class ThinLinkBitcodeWriter : public ModuleBitcodeWriterBase {
5815 /// ModHash is for use in ThinLTO incremental build, generated while writing
5816 /// the module bitcode file.
5817 const ModuleHash *ModHash;
5818
5819public:
5820 ThinLinkBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
5821 BitstreamWriter &Stream,
5822 const ModuleSummaryIndex &Index,
5823 const ModuleHash &ModHash)
5824 : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
5825 /*ShouldPreserveUseListOrder=*/false, &Index),
5826 ModHash(&ModHash) {}
5827
5828 void write();
5829
5830private:
5831 void writeSimplifiedModuleInfo();
5832};
5833
5834} // end anonymous namespace
5835
5836// This function writes a simpilified module info for thin link bitcode file.
5837// It only contains the source file name along with the name(the offset and
5838// size in strtab) and linkage for global values. For the global value info
5839// entry, in order to keep linkage at offset 5, there are three zeros used
5840// as padding.
5841void ThinLinkBitcodeWriter::writeSimplifiedModuleInfo() {
5842 SmallVector<unsigned, 64> Vals;
5843 // Emit the module's source file name.
5844 {
5845 StringEncoding Bits = getStringEncoding(Str: M.getSourceFileName());
5846 BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
5847 if (Bits == SE_Char6)
5848 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
5849 else if (Bits == SE_Fixed7)
5850 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
5851
5852 // MODULE_CODE_SOURCE_FILENAME: [namechar x N]
5853 auto Abbv = std::make_shared<BitCodeAbbrev>();
5854 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
5855 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5856 Abbv->Add(OpInfo: AbbrevOpToUse);
5857 unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5858
5859 for (const auto P : M.getSourceFileName())
5860 Vals.push_back(Elt: (unsigned char)P);
5861
5862 Stream.EmitRecord(Code: bitc::MODULE_CODE_SOURCE_FILENAME, Vals, Abbrev: FilenameAbbrev);
5863 Vals.clear();
5864 }
5865
5866 writeGUIDList();
5867
5868 // Emit the global variable information.
5869 for (const GlobalVariable &GV : M.globals()) {
5870 // GLOBALVAR: [strtab offset, strtab size, 0, 0, 0, linkage]
5871 Vals.push_back(Elt: StrtabBuilder.add(S: GV.getName()));
5872 Vals.push_back(Elt: GV.getName().size());
5873 Vals.push_back(Elt: 0);
5874 Vals.push_back(Elt: 0);
5875 Vals.push_back(Elt: 0);
5876 Vals.push_back(Elt: getEncodedLinkage(GV));
5877
5878 Stream.EmitRecord(Code: bitc::MODULE_CODE_GLOBALVAR, Vals);
5879 Vals.clear();
5880 }
5881
5882 // Emit the function proto information.
5883 for (const Function &F : M) {
5884 // FUNCTION: [strtab offset, strtab size, 0, 0, 0, linkage]
5885 Vals.push_back(Elt: StrtabBuilder.add(S: F.getName()));
5886 Vals.push_back(Elt: F.getName().size());
5887 Vals.push_back(Elt: 0);
5888 Vals.push_back(Elt: 0);
5889 Vals.push_back(Elt: 0);
5890 Vals.push_back(Elt: getEncodedLinkage(GV: F));
5891
5892 Stream.EmitRecord(Code: bitc::MODULE_CODE_FUNCTION, Vals);
5893 Vals.clear();
5894 }
5895
5896 // Emit the alias information.
5897 for (const GlobalAlias &A : M.aliases()) {
5898 // ALIAS: [strtab offset, strtab size, 0, 0, 0, linkage]
5899 Vals.push_back(Elt: StrtabBuilder.add(S: A.getName()));
5900 Vals.push_back(Elt: A.getName().size());
5901 Vals.push_back(Elt: 0);
5902 Vals.push_back(Elt: 0);
5903 Vals.push_back(Elt: 0);
5904 Vals.push_back(Elt: getEncodedLinkage(GV: A));
5905
5906 Stream.EmitRecord(Code: bitc::MODULE_CODE_ALIAS, Vals);
5907 Vals.clear();
5908 }
5909
5910 // Emit the ifunc information.
5911 for (const GlobalIFunc &I : M.ifuncs()) {
5912 // IFUNC: [strtab offset, strtab size, 0, 0, 0, linkage]
5913 Vals.push_back(Elt: StrtabBuilder.add(S: I.getName()));
5914 Vals.push_back(Elt: I.getName().size());
5915 Vals.push_back(Elt: 0);
5916 Vals.push_back(Elt: 0);
5917 Vals.push_back(Elt: 0);
5918 Vals.push_back(Elt: getEncodedLinkage(GV: I));
5919
5920 Stream.EmitRecord(Code: bitc::MODULE_CODE_IFUNC, Vals);
5921 Vals.clear();
5922 }
5923}
5924
5925void ThinLinkBitcodeWriter::write() {
5926 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5927
5928 writeModuleVersion();
5929
5930 writeSimplifiedModuleInfo();
5931
5932 writePerModuleGlobalValueSummary();
5933
5934 // Write module hash.
5935 Stream.EmitRecord(Code: bitc::MODULE_CODE_HASH, Vals: ArrayRef<uint32_t>(*ModHash));
5936
5937 Stream.ExitBlock();
5938}
5939
5940void BitcodeWriter::writeThinLinkBitcode(const Module &M,
5941 const ModuleSummaryIndex &Index,
5942 const ModuleHash &ModHash) {
5943 assert(!WroteStrtab);
5944
5945 // The Mods vector is used by irsymtab::build, which requires non-const
5946 // Modules in case it needs to materialize metadata. But the bitcode writer
5947 // requires that the module is materialized, so we can cast to non-const here,
5948 // after checking that it is in fact materialized.
5949 assert(M.isMaterialized());
5950 Mods.push_back(x: const_cast<Module *>(&M));
5951
5952 ThinLinkBitcodeWriter ThinLinkWriter(M, StrtabBuilder, *Stream, Index,
5953 ModHash);
5954 ThinLinkWriter.write();
5955}
5956
5957// Write the specified thin link bitcode file to the given raw output stream,
5958// where it will be written in a new bitcode block. This is used when
5959// writing the per-module index file for ThinLTO.
5960void llvm::writeThinLinkBitcodeToFile(const Module &M, raw_ostream &Out,
5961 const ModuleSummaryIndex &Index,
5962 const ModuleHash &ModHash) {
5963 SmallVector<char, 0> Buffer;
5964 Buffer.reserve(N: 256 * 1024);
5965
5966 BitcodeWriter Writer(Buffer);
5967 Writer.writeThinLinkBitcode(M, Index, ModHash);
5968 Writer.writeSymtab();
5969 Writer.writeStrtab();
5970
5971 Out.write(Ptr: (char *)&Buffer.front(), Size: Buffer.size());
5972}
5973
5974static const char *getSectionNameForBitcode(const Triple &T) {
5975 switch (T.getObjectFormat()) {
5976 case Triple::MachO:
5977 return "__LLVM,__bitcode";
5978 case Triple::COFF:
5979 case Triple::ELF:
5980 case Triple::Wasm:
5981 case Triple::UnknownObjectFormat:
5982 return ".llvmbc";
5983 case Triple::GOFF:
5984 llvm_unreachable("GOFF is not yet implemented");
5985 break;
5986 case Triple::SPIRV:
5987 if (T.getVendor() == Triple::AMD)
5988 return ".llvmbc";
5989 llvm_unreachable("SPIRV is not yet implemented");
5990 break;
5991 case Triple::XCOFF:
5992 llvm_unreachable("XCOFF is not yet implemented");
5993 break;
5994 case Triple::DXContainer:
5995 llvm_unreachable("DXContainer is not yet implemented");
5996 break;
5997 }
5998 llvm_unreachable("Unimplemented ObjectFormatType");
5999}
6000
6001static const char *getSectionNameForCommandline(const Triple &T) {
6002 switch (T.getObjectFormat()) {
6003 case Triple::MachO:
6004 return "__LLVM,__cmdline";
6005 case Triple::COFF:
6006 case Triple::ELF:
6007 case Triple::Wasm:
6008 case Triple::UnknownObjectFormat:
6009 return ".llvmcmd";
6010 case Triple::GOFF:
6011 llvm_unreachable("GOFF is not yet implemented");
6012 break;
6013 case Triple::SPIRV:
6014 if (T.getVendor() == Triple::AMD)
6015 return ".llvmcmd";
6016 llvm_unreachable("SPIRV is not yet implemented");
6017 break;
6018 case Triple::XCOFF:
6019 llvm_unreachable("XCOFF is not yet implemented");
6020 break;
6021 case Triple::DXContainer:
6022 llvm_unreachable("DXC is not yet implemented");
6023 break;
6024 }
6025 llvm_unreachable("Unimplemented ObjectFormatType");
6026}
6027
6028void llvm::embedBitcodeInModule(llvm::Module &M, llvm::MemoryBufferRef Buf,
6029 bool EmbedBitcode, bool EmbedCmdline,
6030 const std::vector<uint8_t> &CmdArgs) {
6031 // Save llvm.compiler.used and remove it.
6032 SmallVector<Constant *, 2> UsedArray;
6033 SmallVector<GlobalValue *, 4> UsedGlobals;
6034 GlobalVariable *Used = collectUsedGlobalVariables(M, Vec&: UsedGlobals, CompilerUsed: true);
6035 Type *UsedElementType = Used ? Used->getValueType()->getArrayElementType()
6036 : PointerType::getUnqual(C&: M.getContext());
6037 for (auto *GV : UsedGlobals) {
6038 if (GV->getName() != "llvm.embedded.module" &&
6039 GV->getName() != "llvm.cmdline")
6040 UsedArray.push_back(
6041 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
6042 }
6043 if (Used)
6044 Used->eraseFromParent();
6045
6046 // Embed the bitcode for the llvm module.
6047 std::string Data;
6048 ArrayRef<uint8_t> ModuleData;
6049 Triple T(M.getTargetTriple());
6050
6051 if (EmbedBitcode) {
6052 if (Buf.getBufferSize() == 0 ||
6053 !isBitcode(BufPtr: (const unsigned char *)Buf.getBufferStart(),
6054 BufEnd: (const unsigned char *)Buf.getBufferEnd())) {
6055 // If the input is LLVM Assembly, bitcode is produced by serializing
6056 // the module. Use-lists order need to be preserved in this case.
6057 llvm::raw_string_ostream OS(Data);
6058 llvm::WriteBitcodeToFile(M, Out&: OS, /* ShouldPreserveUseListOrder */ true);
6059 ModuleData =
6060 ArrayRef<uint8_t>((const uint8_t *)OS.str().data(), OS.str().size());
6061 } else
6062 // If the input is LLVM bitcode, write the input byte stream directly.
6063 ModuleData = ArrayRef<uint8_t>((const uint8_t *)Buf.getBufferStart(),
6064 Buf.getBufferSize());
6065 }
6066 llvm::Constant *ModuleConstant =
6067 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts: ModuleData);
6068 llvm::GlobalVariable *GV = new llvm::GlobalVariable(
6069 M, ModuleConstant->getType(), true, llvm::GlobalValue::PrivateLinkage,
6070 ModuleConstant);
6071 GV->setSection(getSectionNameForBitcode(T));
6072 // Set alignment to 1 to prevent padding between two contributions from input
6073 // sections after linking.
6074 GV->setAlignment(Align(1));
6075 UsedArray.push_back(
6076 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
6077 if (llvm::GlobalVariable *Old =
6078 M.getGlobalVariable(Name: "llvm.embedded.module", AllowInternal: true)) {
6079 assert(Old->hasZeroLiveUses() &&
6080 "llvm.embedded.module can only be used once in llvm.compiler.used");
6081 GV->takeName(V: Old);
6082 Old->eraseFromParent();
6083 } else {
6084 GV->setName("llvm.embedded.module");
6085 }
6086
6087 // Skip if only bitcode needs to be embedded.
6088 if (EmbedCmdline) {
6089 // Embed command-line options.
6090 ArrayRef<uint8_t> CmdData(const_cast<uint8_t *>(CmdArgs.data()),
6091 CmdArgs.size());
6092 llvm::Constant *CmdConstant =
6093 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts: CmdData);
6094 GV = new llvm::GlobalVariable(M, CmdConstant->getType(), true,
6095 llvm::GlobalValue::PrivateLinkage,
6096 CmdConstant);
6097 GV->setSection(getSectionNameForCommandline(T));
6098 GV->setAlignment(Align(1));
6099 UsedArray.push_back(
6100 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
6101 if (llvm::GlobalVariable *Old = M.getGlobalVariable(Name: "llvm.cmdline", AllowInternal: true)) {
6102 assert(Old->hasZeroLiveUses() &&
6103 "llvm.cmdline can only be used once in llvm.compiler.used");
6104 GV->takeName(V: Old);
6105 Old->eraseFromParent();
6106 } else {
6107 GV->setName("llvm.cmdline");
6108 }
6109 }
6110
6111 if (UsedArray.empty())
6112 return;
6113
6114 // Recreate llvm.compiler.used.
6115 ArrayType *ATy = ArrayType::get(ElementType: UsedElementType, NumElements: UsedArray.size());
6116 auto *NewUsed = new GlobalVariable(
6117 M, ATy, false, llvm::GlobalValue::AppendingLinkage,
6118 llvm::ConstantArray::get(T: ATy, V: UsedArray), "llvm.compiler.used");
6119 NewUsed->setSection("llvm.metadata");
6120}
6121