1//===- Bitcode/Writer/BitcodeWriter.cpp - Bitcode Writer ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Bitcode writer implementation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Bitcode/BitcodeWriter.h"
14#include "ValueEnumerator.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/DenseMap.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SetVector.h"
21#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/SmallString.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringMap.h"
25#include "llvm/ADT/StringRef.h"
26#include "llvm/Analysis/MemoryProfileInfo.h"
27#include "llvm/BinaryFormat/Dwarf.h"
28#include "llvm/Bitcode/BitcodeCommon.h"
29#include "llvm/Bitcode/BitcodeReader.h"
30#include "llvm/Bitcode/LLVMBitCodes.h"
31#include "llvm/Bitstream/BitCodes.h"
32#include "llvm/Bitstream/BitstreamWriter.h"
33#include "llvm/Config/llvm-config.h"
34#include "llvm/IR/Attributes.h"
35#include "llvm/IR/BasicBlock.h"
36#include "llvm/IR/Comdat.h"
37#include "llvm/IR/Constant.h"
38#include "llvm/IR/ConstantRangeList.h"
39#include "llvm/IR/Constants.h"
40#include "llvm/IR/DebugInfoMetadata.h"
41#include "llvm/IR/DebugLoc.h"
42#include "llvm/IR/DerivedTypes.h"
43#include "llvm/IR/Function.h"
44#include "llvm/IR/GlobalAlias.h"
45#include "llvm/IR/GlobalIFunc.h"
46#include "llvm/IR/GlobalObject.h"
47#include "llvm/IR/GlobalValue.h"
48#include "llvm/IR/GlobalVariable.h"
49#include "llvm/IR/InlineAsm.h"
50#include "llvm/IR/InstrTypes.h"
51#include "llvm/IR/Instruction.h"
52#include "llvm/IR/Instructions.h"
53#include "llvm/IR/LLVMContext.h"
54#include "llvm/IR/Metadata.h"
55#include "llvm/IR/Module.h"
56#include "llvm/IR/ModuleSummaryIndex.h"
57#include "llvm/IR/Operator.h"
58#include "llvm/IR/Type.h"
59#include "llvm/IR/UseListOrder.h"
60#include "llvm/IR/Value.h"
61#include "llvm/IR/ValueSymbolTable.h"
62#include "llvm/MC/StringTableBuilder.h"
63#include "llvm/MC/TargetRegistry.h"
64#include "llvm/Object/IRSymtab.h"
65#include "llvm/ProfileData/MemProf.h"
66#include "llvm/ProfileData/MemProfRadixTree.h"
67#include "llvm/Support/AtomicOrdering.h"
68#include "llvm/Support/Casting.h"
69#include "llvm/Support/CommandLine.h"
70#include "llvm/Support/Compiler.h"
71#include "llvm/Support/Endian.h"
72#include "llvm/Support/Error.h"
73#include "llvm/Support/ErrorHandling.h"
74#include "llvm/Support/MathExtras.h"
75#include "llvm/Support/SHA1.h"
76#include "llvm/Support/raw_ostream.h"
77#include "llvm/TargetParser/Triple.h"
78#include <algorithm>
79#include <cassert>
80#include <cstddef>
81#include <cstdint>
82#include <iterator>
83#include <map>
84#include <memory>
85#include <optional>
86#include <string>
87#include <utility>
88#include <vector>
89
90using namespace llvm;
91using namespace llvm::memprof;
92
93static cl::opt<unsigned>
94 IndexThreshold("bitcode-mdindex-threshold", cl::Hidden, cl::init(Val: 25),
95 cl::desc("Number of metadatas above which we emit an index "
96 "to enable lazy-loading"));
97static cl::opt<uint32_t> FlushThreshold(
98 "bitcode-flush-threshold", cl::Hidden, cl::init(Val: 512),
99 cl::desc("The threshold (unit M) for flushing LLVM bitcode."));
100
101// Since we only use the context information in the memprof summary records in
102// the LTO backends to do assertion checking, save time and space by only
103// serializing the context for non-NDEBUG builds.
104// TODO: Currently this controls writing context of the allocation info records,
105// which are larger and more expensive, but we should do this for the callsite
106// records as well.
107// FIXME: Convert to a const once this has undergone more sigificant testing.
108static cl::opt<bool>
109 CombinedIndexMemProfContext("combined-index-memprof-context", cl::Hidden,
110#ifdef NDEBUG
111 cl::init(Val: false),
112#else
113 cl::init(true),
114#endif
115 cl::desc(""));
116
117static cl::opt<bool> PreserveBitcodeUseListOrder(
118 "preserve-bc-uselistorder", cl::Hidden, cl::init(Val: true),
119 cl::desc("Preserve use-list order when writing LLVM bitcode."));
120
121namespace llvm {
122extern FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold;
123}
124
125namespace {
126
127/// These are manifest constants used by the bitcode writer. They do not need to
128/// be kept in sync with the reader, but need to be consistent within this file.
129enum {
130 // VALUE_SYMTAB_BLOCK abbrev id's.
131 VST_ENTRY_8_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
132 VST_ENTRY_7_ABBREV,
133 VST_ENTRY_6_ABBREV,
134 VST_BBENTRY_6_ABBREV,
135
136 // CONSTANTS_BLOCK abbrev id's.
137 CONSTANTS_SETTYPE_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
138 CONSTANTS_INTEGER_ABBREV,
139 CONSTANTS_BYTE_ABBREV,
140 CONSTANTS_CE_CAST_Abbrev,
141 CONSTANTS_NULL_Abbrev,
142
143 // FUNCTION_BLOCK abbrev id's.
144 FUNCTION_INST_LOAD_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
145 FUNCTION_INST_STORE_ABBREV,
146 FUNCTION_INST_UNOP_ABBREV,
147 FUNCTION_INST_UNOP_FLAGS_ABBREV,
148 FUNCTION_INST_BINOP_ABBREV,
149 FUNCTION_INST_BINOP_FLAGS_ABBREV,
150 FUNCTION_INST_CAST_ABBREV,
151 FUNCTION_INST_CAST_FLAGS_ABBREV,
152 FUNCTION_INST_RET_VOID_ABBREV,
153 FUNCTION_INST_RET_VAL_ABBREV,
154 FUNCTION_INST_BR_UNCOND_ABBREV,
155 FUNCTION_INST_BR_COND_ABBREV,
156 FUNCTION_INST_UNREACHABLE_ABBREV,
157 FUNCTION_INST_GEP_ABBREV,
158 FUNCTION_INST_CMP_ABBREV,
159 FUNCTION_INST_CMP_FLAGS_ABBREV,
160 FUNCTION_DEBUG_RECORD_VALUE_ABBREV,
161 FUNCTION_DEBUG_LOC_ABBREV,
162};
163
164/// Abstract class to manage the bitcode writing, subclassed for each bitcode
165/// file type.
166class BitcodeWriterBase {
167protected:
168 /// The stream created and owned by the client.
169 BitstreamWriter &Stream;
170
171 StringTableBuilder &StrtabBuilder;
172
173public:
174 /// Constructs a BitcodeWriterBase object that writes to the provided
175 /// \p Stream.
176 BitcodeWriterBase(BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder)
177 : Stream(Stream), StrtabBuilder(StrtabBuilder) {}
178
179protected:
180 void writeModuleVersion();
181};
182
183void BitcodeWriterBase::writeModuleVersion() {
184 // VERSION: [version#]
185 Stream.EmitRecord(Code: bitc::MODULE_CODE_VERSION, Vals: ArrayRef<uint64_t>{2});
186}
187
188/// Base class to manage the module bitcode writing, currently subclassed for
189/// ModuleBitcodeWriter and ThinLinkBitcodeWriter.
190class ModuleBitcodeWriterBase : public BitcodeWriterBase {
191protected:
192 /// The Module to write to bitcode.
193 const Module &M;
194
195 /// Enumerates ids for all values in the module.
196 ValueEnumerator VE;
197
198 /// Optional per-module index to write for ThinLTO.
199 const ModuleSummaryIndex *Index;
200
201 /// Map that holds the correspondence between GUIDs in the summary index,
202 /// that came from indirect call profiles, and a value id generated by this
203 /// class to use in the VST and summary block records.
204 std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
205
206 /// Tracks the last value id recorded in the GUIDToValueMap.
207 unsigned GlobalValueId;
208
209 /// Saves the offset of the VSTOffset record that must eventually be
210 /// backpatched with the offset of the actual VST.
211 uint64_t VSTOffsetPlaceholder = 0;
212
213public:
214 /// Constructs a ModuleBitcodeWriterBase object for the given Module,
215 /// writing to the provided \p Buffer.
216 ModuleBitcodeWriterBase(const Module &M, StringTableBuilder &StrtabBuilder,
217 BitstreamWriter &Stream,
218 bool ShouldPreserveUseListOrder,
219 const ModuleSummaryIndex *Index)
220 : BitcodeWriterBase(Stream, StrtabBuilder), M(M),
221 VE(M, PreserveBitcodeUseListOrder.getNumOccurrences()
222 ? PreserveBitcodeUseListOrder
223 : ShouldPreserveUseListOrder),
224 Index(Index) {
225 // Assign ValueIds to any callee values in the index that came from
226 // indirect call profiles and were recorded as a GUID not a Value*
227 // (which would have been assigned an ID by the ValueEnumerator).
228 // The starting ValueId is just after the number of values in the
229 // ValueEnumerator, so that they can be emitted in the VST.
230 GlobalValueId = VE.getValues().size();
231 if (!Index)
232 return;
233 // Sort by GUID for deterministic value ID assignment.
234 for (const auto &GUIDSummaryLists :
235 Index->sortedGlobalValueSummariesRange())
236 // Examine all summaries for this GUID.
237 for (auto &Summary : GUIDSummaryLists.second.getSummaryList())
238 if (auto *FS = dyn_cast<FunctionSummary>(Val: Summary.get())) {
239 // For each call in the function summary, see if the call
240 // is to a GUID (which means it is for an indirect call,
241 // otherwise we would have a Value for it). If so, synthesize
242 // a value id.
243 for (auto &CallEdge : FS->calls())
244 if (!CallEdge.first.haveGVs() || !CallEdge.first.getValue())
245 assignValueId(ValGUID: CallEdge.first.getGUID());
246
247 // For each referenced variables in the function summary, see if the
248 // variable is represented by a GUID (as opposed to a symbol to
249 // declarations or definitions in the module). If so, synthesize a
250 // value id.
251 for (auto &RefEdge : FS->refs())
252 if (!RefEdge.haveGVs() || !RefEdge.getValue())
253 assignValueId(ValGUID: RefEdge.getGUID());
254 }
255 }
256
257protected:
258 void writePerModuleGlobalValueSummary();
259 void writeGUIDList();
260
261private:
262 void writePerModuleFunctionSummaryRecord(
263 SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
264 unsigned ValueID, unsigned FSCallsProfileAbbrev, unsigned CallsiteAbbrev,
265 unsigned AllocAbbrev, unsigned ContextIdAbbvId, const Function &F,
266 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
267 CallStackId &CallStackCount);
268 void writeModuleLevelReferences(const GlobalVariable &V,
269 SmallVector<uint64_t, 64> &NameVals,
270 unsigned FSModRefsAbbrev,
271 unsigned FSModVTableRefsAbbrev);
272
273 void assignValueId(GlobalValue::GUID ValGUID) {
274 GUIDToValueIdMap[ValGUID] = ++GlobalValueId;
275 }
276
277 unsigned getValueId(GlobalValue::GUID ValGUID) {
278 const auto &VMI = GUIDToValueIdMap.find(x: ValGUID);
279 // Expect that any GUID value had a value Id assigned by an
280 // earlier call to assignValueId.
281 assert(VMI != GUIDToValueIdMap.end() &&
282 "GUID does not have assigned value Id");
283 return VMI->second;
284 }
285
286 // Helper to get the valueId for the type of value recorded in VI.
287 unsigned getValueId(ValueInfo VI) {
288 if (!VI.haveGVs() || !VI.getValue())
289 return getValueId(ValGUID: VI.getGUID());
290 return VE.getValueID(V: VI.getValue());
291 }
292
293 std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
294};
295
296/// Class to manage the bitcode writing for a module.
297class ModuleBitcodeWriter : public ModuleBitcodeWriterBase {
298 /// True if a module hash record should be written.
299 bool GenerateHash;
300
301 /// If non-null, when GenerateHash is true, the resulting hash is written
302 /// into ModHash.
303 ModuleHash *ModHash;
304
305 SHA1 Hasher;
306
307 /// The start bit of the identification block.
308 uint64_t BitcodeStartBit;
309
310public:
311 /// Constructs a ModuleBitcodeWriter object for the given Module,
312 /// writing to the provided \p Buffer.
313 ModuleBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
314 BitstreamWriter &Stream, bool ShouldPreserveUseListOrder,
315 const ModuleSummaryIndex *Index, bool GenerateHash,
316 ModuleHash *ModHash = nullptr)
317 : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
318 ShouldPreserveUseListOrder, Index),
319 GenerateHash(GenerateHash), ModHash(ModHash),
320 BitcodeStartBit(Stream.GetCurrentBitNo()) {}
321
322 /// Emit the current module to the bitstream.
323 void write();
324
325private:
326 uint64_t bitcodeStartBit() { return BitcodeStartBit; }
327
328 size_t addToStrtab(StringRef Str);
329
330 void writeAttributeGroupTable();
331 void writeAttributeTable();
332 void writeTypeTable();
333 void writeComdats();
334 void writeValueSymbolTableForwardDecl();
335 void writeModuleInfo();
336 void writeValueAsMetadata(const ValueAsMetadata *MD,
337 SmallVectorImpl<uint64_t> &Record);
338 void writeMDTuple(const MDTuple *N, SmallVectorImpl<uint64_t> &Record,
339 unsigned Abbrev);
340 unsigned createDILocationAbbrev();
341 void writeDILocation(const DILocation *N, SmallVectorImpl<uint64_t> &Record,
342 unsigned &Abbrev);
343 unsigned createGenericDINodeAbbrev();
344 void writeGenericDINode(const GenericDINode *N,
345 SmallVectorImpl<uint64_t> &Record, unsigned &Abbrev);
346 void writeDISubrange(const DISubrange *N, SmallVectorImpl<uint64_t> &Record,
347 unsigned Abbrev);
348 void writeDIGenericSubrange(const DIGenericSubrange *N,
349 SmallVectorImpl<uint64_t> &Record,
350 unsigned Abbrev);
351 void writeDIEnumerator(const DIEnumerator *N,
352 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
353 void writeDIBasicType(const DIBasicType *N, SmallVectorImpl<uint64_t> &Record,
354 unsigned Abbrev);
355 void writeDIFixedPointType(const DIFixedPointType *N,
356 SmallVectorImpl<uint64_t> &Record,
357 unsigned Abbrev);
358 void writeDIStringType(const DIStringType *N,
359 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
360 void writeDIDerivedType(const DIDerivedType *N,
361 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
362 void writeDISubrangeType(const DISubrangeType *N,
363 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
364 void writeDICompositeType(const DICompositeType *N,
365 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
366 void writeDISubroutineType(const DISubroutineType *N,
367 SmallVectorImpl<uint64_t> &Record,
368 unsigned Abbrev);
369 void writeDIFile(const DIFile *N, SmallVectorImpl<uint64_t> &Record,
370 unsigned Abbrev);
371 void writeDICompileUnit(const DICompileUnit *N,
372 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
373 void writeDISubprogram(const DISubprogram *N,
374 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
375 void writeDILexicalBlock(const DILexicalBlock *N,
376 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
377 void writeDILexicalBlockFile(const DILexicalBlockFile *N,
378 SmallVectorImpl<uint64_t> &Record,
379 unsigned Abbrev);
380 void writeDICommonBlock(const DICommonBlock *N,
381 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
382 void writeDINamespace(const DINamespace *N, SmallVectorImpl<uint64_t> &Record,
383 unsigned Abbrev);
384 void writeDIMacro(const DIMacro *N, SmallVectorImpl<uint64_t> &Record,
385 unsigned Abbrev);
386 void writeDIMacroFile(const DIMacroFile *N, SmallVectorImpl<uint64_t> &Record,
387 unsigned Abbrev);
388 void writeDIArgList(const DIArgList *N, SmallVectorImpl<uint64_t> &Record);
389 void writeDIModule(const DIModule *N, SmallVectorImpl<uint64_t> &Record,
390 unsigned Abbrev);
391 void writeDIAssignID(const DIAssignID *N, SmallVectorImpl<uint64_t> &Record,
392 unsigned Abbrev);
393 void writeDITemplateTypeParameter(const DITemplateTypeParameter *N,
394 SmallVectorImpl<uint64_t> &Record,
395 unsigned Abbrev);
396 void writeDITemplateValueParameter(const DITemplateValueParameter *N,
397 SmallVectorImpl<uint64_t> &Record,
398 unsigned Abbrev);
399 void writeDIGlobalVariable(const DIGlobalVariable *N,
400 SmallVectorImpl<uint64_t> &Record,
401 unsigned Abbrev);
402 void writeDILocalVariable(const DILocalVariable *N,
403 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
404 void writeDILabel(const DILabel *N,
405 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
406 void writeDIExpression(const DIExpression *N,
407 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
408 void writeDIGlobalVariableExpression(const DIGlobalVariableExpression *N,
409 SmallVectorImpl<uint64_t> &Record,
410 unsigned Abbrev);
411 void writeDIObjCProperty(const DIObjCProperty *N,
412 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
413 void writeDIImportedEntity(const DIImportedEntity *N,
414 SmallVectorImpl<uint64_t> &Record,
415 unsigned Abbrev);
416 unsigned createNamedMetadataAbbrev();
417 void writeNamedMetadata(SmallVectorImpl<uint64_t> &Record);
418 unsigned createMetadataStringsAbbrev();
419 void writeMetadataStrings(ArrayRef<const Metadata *> Strings,
420 SmallVectorImpl<uint64_t> &Record);
421 void writeMetadataRecords(ArrayRef<const Metadata *> MDs,
422 SmallVectorImpl<uint64_t> &Record,
423 std::vector<unsigned> *MDAbbrevs = nullptr,
424 std::vector<uint64_t> *IndexPos = nullptr);
425 void writeModuleMetadata();
426 void writeFunctionMetadata(const Function &F);
427 void writeFunctionMetadataAttachment(const Function &F);
428 void pushGlobalMetadataAttachment(SmallVectorImpl<uint64_t> &Record,
429 const GlobalObject &GO);
430 void writeModuleMetadataKinds();
431 void writeOperandBundleTags();
432 void writeSyncScopeNames();
433 void writeConstants(unsigned FirstVal, unsigned LastVal, bool isGlobal);
434 void writeModuleConstants();
435 bool pushValueAndType(const Value *V, unsigned InstID,
436 SmallVectorImpl<unsigned> &Vals);
437 bool pushValueOrMetadata(const Value *V, unsigned InstID,
438 SmallVectorImpl<unsigned> &Vals);
439 void writeOperandBundles(const CallBase &CB, unsigned InstID);
440 void pushValue(const Value *V, unsigned InstID,
441 SmallVectorImpl<unsigned> &Vals);
442 void pushValueSigned(const Value *V, unsigned InstID,
443 SmallVectorImpl<uint64_t> &Vals);
444 void writeInstruction(const Instruction &I, unsigned InstID,
445 SmallVectorImpl<unsigned> &Vals);
446 void writeFunctionLevelValueSymbolTable(const ValueSymbolTable &VST);
447 void writeGlobalValueSymbolTable(
448 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
449 void writeUseList(UseListOrder &&Order);
450 void writeUseListBlock(const Function *F);
451 void
452 writeFunction(const Function &F,
453 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
454 void writeBlockInfo();
455 void writeModuleHash(StringRef View);
456
457 unsigned getEncodedSyncScopeID(SyncScope::ID SSID) {
458 return unsigned(SSID);
459 }
460
461 unsigned getEncodedAlign(MaybeAlign Alignment) { return encode(A: Alignment); }
462};
463
464/// Class to manage the bitcode writing for a combined index.
465class IndexBitcodeWriter : public BitcodeWriterBase {
466 /// The combined index to write to bitcode.
467 const ModuleSummaryIndex &Index;
468
469 /// When writing combined summaries, provides the set of global value
470 /// summaries for which the value (function, function alias, etc) should be
471 /// imported as a declaration.
472 const GVSummaryPtrSet *DecSummaries = nullptr;
473
474 /// When writing a subset of the index for distributed backends, client
475 /// provides a map of modules to the corresponding GUIDs/summaries to write.
476 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex;
477
478 /// Map that holds the correspondence between the GUID used in the combined
479 /// index and a value id generated by this class to use in references.
480 std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
481
482 // The stack ids used by this index, which will be a subset of those in
483 // the full index in the case of distributed indexes.
484 std::vector<uint64_t> StackIds;
485
486 // Keep a map of the stack id indices used by records being written for this
487 // index to the index of the corresponding stack id in the above StackIds
488 // vector. Ensures we write each referenced stack id once.
489 DenseMap<unsigned, unsigned> StackIdIndicesToIndex;
490
491 /// Tracks the last value id recorded in the GUIDToValueMap.
492 unsigned GlobalValueId = 0;
493
494 /// Tracks the assignment of module paths in the module path string table to
495 /// an id assigned for use in summary references to the module path.
496 DenseMap<StringRef, uint64_t> ModuleIdMap;
497
498public:
499 /// Constructs a IndexBitcodeWriter object for the given combined index,
500 /// writing to the provided \p Buffer. When writing a subset of the index
501 /// for a distributed backend, provide a \p ModuleToSummariesForIndex map.
502 /// If provided, \p DecSummaries specifies the set of summaries for which
503 /// the corresponding functions or aliased functions should be imported as a
504 /// declaration (but not definition) for each module.
505 IndexBitcodeWriter(
506 BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder,
507 const ModuleSummaryIndex &Index,
508 const GVSummaryPtrSet *DecSummaries = nullptr,
509 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex = nullptr)
510 : BitcodeWriterBase(Stream, StrtabBuilder), Index(Index),
511 DecSummaries(DecSummaries),
512 ModuleToSummariesForIndex(ModuleToSummariesForIndex) {
513
514 // See if the StackIdIndex was already added to the StackId map and
515 // vector. If not, record it.
516 auto RecordStackIdReference = [&](unsigned StackIdIndex) {
517 // If the StackIdIndex is not yet in the map, the below insert ensures
518 // that it will point to the new StackIds vector entry we push to just
519 // below.
520 auto Inserted =
521 StackIdIndicesToIndex.insert(KV: {StackIdIndex, StackIds.size()});
522 if (Inserted.second)
523 StackIds.push_back(x: Index.getStackIdAtIndex(Index: StackIdIndex));
524 };
525
526 // Assign unique value ids to all summaries to be written, for use
527 // in writing out the call graph edges. Save the mapping from GUID
528 // to the new global value id to use when writing those edges, which
529 // are currently saved in the index in terms of GUID.
530 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
531 GUIDToValueIdMap[I.first] = ++GlobalValueId;
532 // If this is invoked for an aliasee, we want to record the above mapping,
533 // but not the information needed for its summary entry (if the aliasee is
534 // to be imported, we will invoke this separately with IsAliasee=false).
535 if (IsAliasee)
536 return;
537 auto *FS = dyn_cast<FunctionSummary>(Val: I.second);
538 if (!FS)
539 return;
540 // Record all stack id indices actually used in the summary entries being
541 // written, so that we can compact them in the case of distributed ThinLTO
542 // indexes.
543 for (auto &CI : FS->callsites()) {
544 // If the stack id list is empty, this callsite info was synthesized for
545 // a missing tail call frame. Ensure that the callee's GUID gets a value
546 // id. Normally we only generate these for defined summaries, which in
547 // the case of distributed ThinLTO is only the functions already defined
548 // in the module or that we want to import. We don't bother to include
549 // all the callee symbols as they aren't normally needed in the backend.
550 // However, for the synthesized callsite infos we do need the callee
551 // GUID in the backend so that we can correlate the identified callee
552 // with this callsite info (which for non-tail calls is done by the
553 // ordering of the callsite infos and verified via stack ids).
554 if (CI.StackIdIndices.empty()) {
555 GUIDToValueIdMap[CI.Callee.getGUID()] = ++GlobalValueId;
556 continue;
557 }
558 for (auto Idx : CI.StackIdIndices)
559 RecordStackIdReference(Idx);
560 }
561 if (CombinedIndexMemProfContext) {
562 for (auto &AI : FS->allocs())
563 for (auto &MIB : AI.MIBs)
564 for (auto Idx : MIB.StackIdIndices)
565 RecordStackIdReference(Idx);
566 }
567 });
568 }
569
570 /// The below iterator returns the GUID and associated summary.
571 using GVInfo = std::pair<GlobalValue::GUID, GlobalValueSummary *>;
572
573 /// Calls the callback for each value GUID and summary to be written to
574 /// bitcode. This hides the details of whether they are being pulled from the
575 /// entire index or just those in a provided ModuleToSummariesForIndex map.
576 template<typename Functor>
577 void forEachSummary(Functor Callback) {
578 if (ModuleToSummariesForIndex) {
579 for (auto &M : *ModuleToSummariesForIndex)
580 for (auto &Summary : M.second) {
581 Callback(Summary, false);
582 // Ensure aliasee is handled, e.g. for assigning a valueId,
583 // even if we are not importing the aliasee directly (the
584 // imported alias will contain a copy of aliasee).
585 if (auto *AS = dyn_cast<AliasSummary>(Val: Summary.getSecond()))
586 Callback({AS->getAliaseeGUID(), &AS->getAliasee()}, true);
587 }
588 } else {
589 // Sort by GUID for deterministic output.
590 for (const auto &Summaries : Index.sortedGlobalValueSummariesRange())
591 for (auto &Summary : Summaries.second.getSummaryList())
592 Callback({Summaries.first, Summary.get()}, false);
593 }
594 }
595
596 /// Calls the callback for each entry in the modulePaths StringMap that
597 /// should be written to the module path string table. This hides the details
598 /// of whether they are being pulled from the entire index or just those in a
599 /// provided ModuleToSummariesForIndex map.
600 template <typename Functor> void forEachModule(Functor Callback) {
601 if (ModuleToSummariesForIndex) {
602 for (const auto &M : *ModuleToSummariesForIndex) {
603 const auto &MPI = Index.modulePaths().find(Key: M.first);
604 if (MPI == Index.modulePaths().end()) {
605 // This should only happen if the bitcode file was empty, in which
606 // case we shouldn't be importing (the ModuleToSummariesForIndex
607 // would only include the module we are writing and index for).
608 assert(ModuleToSummariesForIndex->size() == 1);
609 continue;
610 }
611 Callback(*MPI);
612 }
613 } else {
614 // Since StringMap iteration order isn't guaranteed, order by path string
615 // first.
616 // FIXME: Make this a vector of StringMapEntry instead to avoid the later
617 // map lookup.
618 std::vector<StringRef> ModulePaths;
619 for (auto &[ModPath, _] : Index.modulePaths())
620 ModulePaths.push_back(x: ModPath);
621 llvm::sort(C&: ModulePaths);
622 for (auto &ModPath : ModulePaths)
623 Callback(*Index.modulePaths().find(Key: ModPath));
624 }
625 }
626
627 /// Main entry point for writing a combined index to bitcode.
628 void write();
629
630private:
631 void writeModStrings();
632 void writeCombinedGlobalValueSummary();
633
634 std::optional<unsigned> getValueId(GlobalValue::GUID ValGUID) {
635 auto VMI = GUIDToValueIdMap.find(x: ValGUID);
636 if (VMI == GUIDToValueIdMap.end())
637 return std::nullopt;
638 return VMI->second;
639 }
640
641 std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
642};
643
644} // end anonymous namespace
645
646static unsigned getEncodedCastOpcode(unsigned Opcode) {
647 switch (Opcode) {
648 default: llvm_unreachable("Unknown cast instruction!");
649 case Instruction::Trunc : return bitc::CAST_TRUNC;
650 case Instruction::ZExt : return bitc::CAST_ZEXT;
651 case Instruction::SExt : return bitc::CAST_SEXT;
652 case Instruction::FPToUI : return bitc::CAST_FPTOUI;
653 case Instruction::FPToSI : return bitc::CAST_FPTOSI;
654 case Instruction::UIToFP : return bitc::CAST_UITOFP;
655 case Instruction::SIToFP : return bitc::CAST_SITOFP;
656 case Instruction::FPTrunc : return bitc::CAST_FPTRUNC;
657 case Instruction::FPExt : return bitc::CAST_FPEXT;
658 case Instruction::PtrToAddr: return bitc::CAST_PTRTOADDR;
659 case Instruction::PtrToInt: return bitc::CAST_PTRTOINT;
660 case Instruction::IntToPtr: return bitc::CAST_INTTOPTR;
661 case Instruction::BitCast : return bitc::CAST_BITCAST;
662 case Instruction::AddrSpaceCast: return bitc::CAST_ADDRSPACECAST;
663 }
664}
665
666static unsigned getEncodedUnaryOpcode(unsigned Opcode) {
667 switch (Opcode) {
668 default: llvm_unreachable("Unknown binary instruction!");
669 case Instruction::FNeg: return bitc::UNOP_FNEG;
670 }
671}
672
673static unsigned getEncodedBinaryOpcode(unsigned Opcode) {
674 switch (Opcode) {
675 default: llvm_unreachable("Unknown binary instruction!");
676 case Instruction::Add:
677 case Instruction::FAdd: return bitc::BINOP_ADD;
678 case Instruction::Sub:
679 case Instruction::FSub: return bitc::BINOP_SUB;
680 case Instruction::Mul:
681 case Instruction::FMul: return bitc::BINOP_MUL;
682 case Instruction::UDiv: return bitc::BINOP_UDIV;
683 case Instruction::FDiv:
684 case Instruction::SDiv: return bitc::BINOP_SDIV;
685 case Instruction::URem: return bitc::BINOP_UREM;
686 case Instruction::FRem:
687 case Instruction::SRem: return bitc::BINOP_SREM;
688 case Instruction::Shl: return bitc::BINOP_SHL;
689 case Instruction::LShr: return bitc::BINOP_LSHR;
690 case Instruction::AShr: return bitc::BINOP_ASHR;
691 case Instruction::And: return bitc::BINOP_AND;
692 case Instruction::Or: return bitc::BINOP_OR;
693 case Instruction::Xor: return bitc::BINOP_XOR;
694 }
695}
696
697static unsigned getEncodedRMWOperation(const AtomicRMWInst &I) {
698 unsigned Encoding = 0;
699 switch (I.getOperation()) {
700 default: llvm_unreachable("Unknown RMW operation!");
701 case AtomicRMWInst::Xchg:
702 Encoding = bitc::RMW_XCHG;
703 break;
704 case AtomicRMWInst::Add:
705 Encoding = bitc::RMW_ADD;
706 break;
707 case AtomicRMWInst::Sub:
708 Encoding = bitc::RMW_SUB;
709 break;
710 case AtomicRMWInst::And:
711 Encoding = bitc::RMW_AND;
712 break;
713 case AtomicRMWInst::Nand:
714 Encoding = bitc::RMW_NAND;
715 break;
716 case AtomicRMWInst::Or:
717 Encoding = bitc::RMW_OR;
718 break;
719 case AtomicRMWInst::Xor:
720 Encoding = bitc::RMW_XOR;
721 break;
722 case AtomicRMWInst::Max:
723 Encoding = bitc::RMW_MAX;
724 break;
725 case AtomicRMWInst::Min:
726 Encoding = bitc::RMW_MIN;
727 break;
728 case AtomicRMWInst::UMax:
729 Encoding = bitc::RMW_UMAX;
730 break;
731 case AtomicRMWInst::UMin:
732 Encoding = bitc::RMW_UMIN;
733 break;
734 case AtomicRMWInst::FAdd:
735 Encoding = bitc::RMW_FADD;
736 break;
737 case AtomicRMWInst::FSub:
738 Encoding = bitc::RMW_FSUB;
739 break;
740 case AtomicRMWInst::FMax:
741 Encoding = bitc::RMW_FMAX;
742 break;
743 case AtomicRMWInst::FMin:
744 Encoding = bitc::RMW_FMIN;
745 break;
746 case AtomicRMWInst::FMaximum:
747 Encoding = bitc::RMW_FMAXIMUM;
748 break;
749 case AtomicRMWInst::FMinimum:
750 Encoding = bitc::RMW_FMINIMUM;
751 break;
752 case AtomicRMWInst::FMaximumNum:
753 Encoding = bitc::RMW_FMAXIMUMNUM;
754 break;
755 case AtomicRMWInst::FMinimumNum:
756 Encoding = bitc::RMW_FMINIMUMNUM;
757 break;
758 case AtomicRMWInst::UIncWrap:
759 Encoding = bitc::RMW_UINC_WRAP;
760 break;
761 case AtomicRMWInst::UDecWrap:
762 Encoding = bitc::RMW_UDEC_WRAP;
763 break;
764 case AtomicRMWInst::USubCond:
765 Encoding = bitc::RMW_USUB_COND;
766 break;
767 case AtomicRMWInst::USubSat:
768 Encoding = bitc::RMW_USUB_SAT;
769 break;
770 }
771
772 if (I.isElementwise())
773 Encoding |= bitc::RMW_ELEMENTWISE_FLAG;
774 return Encoding;
775}
776
777static unsigned getEncodedOrdering(AtomicOrdering Ordering) {
778 switch (Ordering) {
779 case AtomicOrdering::NotAtomic: return bitc::ORDERING_NOTATOMIC;
780 case AtomicOrdering::Unordered: return bitc::ORDERING_UNORDERED;
781 case AtomicOrdering::Monotonic: return bitc::ORDERING_MONOTONIC;
782 case AtomicOrdering::Acquire: return bitc::ORDERING_ACQUIRE;
783 case AtomicOrdering::Release: return bitc::ORDERING_RELEASE;
784 case AtomicOrdering::AcquireRelease: return bitc::ORDERING_ACQREL;
785 case AtomicOrdering::SequentiallyConsistent: return bitc::ORDERING_SEQCST;
786 }
787 llvm_unreachable("Invalid ordering");
788}
789
790static void writeStringRecord(BitstreamWriter &Stream, unsigned Code,
791 StringRef Str, unsigned AbbrevToUse) {
792 SmallVector<unsigned, 64> Vals;
793
794 // Code: [strchar x N]
795 for (char C : Str) {
796 if (AbbrevToUse && !BitCodeAbbrevOp::isChar6(C))
797 AbbrevToUse = 0;
798 Vals.push_back(Elt: C);
799 }
800
801 // Emit the finished record.
802 Stream.EmitRecord(Code, Vals, Abbrev: AbbrevToUse);
803}
804
805static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
806 switch (Kind) {
807 case Attribute::Alignment:
808 return bitc::ATTR_KIND_ALIGNMENT;
809 case Attribute::AllocAlign:
810 return bitc::ATTR_KIND_ALLOC_ALIGN;
811 case Attribute::AllocSize:
812 return bitc::ATTR_KIND_ALLOC_SIZE;
813 case Attribute::AlwaysInline:
814 return bitc::ATTR_KIND_ALWAYS_INLINE;
815 case Attribute::Builtin:
816 return bitc::ATTR_KIND_BUILTIN;
817 case Attribute::ByVal:
818 return bitc::ATTR_KIND_BY_VAL;
819 case Attribute::Convergent:
820 return bitc::ATTR_KIND_CONVERGENT;
821 case Attribute::InAlloca:
822 return bitc::ATTR_KIND_IN_ALLOCA;
823 case Attribute::Cold:
824 return bitc::ATTR_KIND_COLD;
825 case Attribute::DisableSanitizerInstrumentation:
826 return bitc::ATTR_KIND_DISABLE_SANITIZER_INSTRUMENTATION;
827 case Attribute::FnRetThunkExtern:
828 return bitc::ATTR_KIND_FNRETTHUNK_EXTERN;
829 case Attribute::Flatten:
830 return bitc::ATTR_KIND_FLATTEN;
831 case Attribute::Hot:
832 return bitc::ATTR_KIND_HOT;
833 case Attribute::ElementType:
834 return bitc::ATTR_KIND_ELEMENTTYPE;
835 case Attribute::HybridPatchable:
836 return bitc::ATTR_KIND_HYBRID_PATCHABLE;
837 case Attribute::InlineHint:
838 return bitc::ATTR_KIND_INLINE_HINT;
839 case Attribute::InReg:
840 return bitc::ATTR_KIND_IN_REG;
841 case Attribute::JumpTable:
842 return bitc::ATTR_KIND_JUMP_TABLE;
843 case Attribute::MinSize:
844 return bitc::ATTR_KIND_MIN_SIZE;
845 case Attribute::AllocatedPointer:
846 return bitc::ATTR_KIND_ALLOCATED_POINTER;
847 case Attribute::AllocKind:
848 return bitc::ATTR_KIND_ALLOC_KIND;
849 case Attribute::Memory:
850 return bitc::ATTR_KIND_MEMORY;
851 case Attribute::NoFPClass:
852 return bitc::ATTR_KIND_NOFPCLASS;
853 case Attribute::Naked:
854 return bitc::ATTR_KIND_NAKED;
855 case Attribute::Nest:
856 return bitc::ATTR_KIND_NEST;
857 case Attribute::NoAlias:
858 return bitc::ATTR_KIND_NO_ALIAS;
859 case Attribute::NoBuiltin:
860 return bitc::ATTR_KIND_NO_BUILTIN;
861 case Attribute::NoCallback:
862 return bitc::ATTR_KIND_NO_CALLBACK;
863 case Attribute::NoDivergenceSource:
864 return bitc::ATTR_KIND_NO_DIVERGENCE_SOURCE;
865 case Attribute::NoDuplicate:
866 return bitc::ATTR_KIND_NO_DUPLICATE;
867 case Attribute::NoFree:
868 return bitc::ATTR_KIND_NOFREE;
869 case Attribute::NoImplicitFloat:
870 return bitc::ATTR_KIND_NO_IMPLICIT_FLOAT;
871 case Attribute::NoInline:
872 return bitc::ATTR_KIND_NO_INLINE;
873 case Attribute::NoRecurse:
874 return bitc::ATTR_KIND_NO_RECURSE;
875 case Attribute::NoMerge:
876 return bitc::ATTR_KIND_NO_MERGE;
877 case Attribute::NonLazyBind:
878 return bitc::ATTR_KIND_NON_LAZY_BIND;
879 case Attribute::NonNull:
880 return bitc::ATTR_KIND_NON_NULL;
881 case Attribute::Dereferenceable:
882 return bitc::ATTR_KIND_DEREFERENCEABLE;
883 case Attribute::DereferenceableOrNull:
884 return bitc::ATTR_KIND_DEREFERENCEABLE_OR_NULL;
885 case Attribute::NoRedZone:
886 return bitc::ATTR_KIND_NO_RED_ZONE;
887 case Attribute::NoReturn:
888 return bitc::ATTR_KIND_NO_RETURN;
889 case Attribute::NoSync:
890 return bitc::ATTR_KIND_NOSYNC;
891 case Attribute::NoCfCheck:
892 return bitc::ATTR_KIND_NOCF_CHECK;
893 case Attribute::NoProfile:
894 return bitc::ATTR_KIND_NO_PROFILE;
895 case Attribute::SkipProfile:
896 return bitc::ATTR_KIND_SKIP_PROFILE;
897 case Attribute::NoUnwind:
898 return bitc::ATTR_KIND_NO_UNWIND;
899 case Attribute::NoSanitizeBounds:
900 return bitc::ATTR_KIND_NO_SANITIZE_BOUNDS;
901 case Attribute::NoSanitizeCoverage:
902 return bitc::ATTR_KIND_NO_SANITIZE_COVERAGE;
903 case Attribute::NullPointerIsValid:
904 return bitc::ATTR_KIND_NULL_POINTER_IS_VALID;
905 case Attribute::OptimizeForDebugging:
906 return bitc::ATTR_KIND_OPTIMIZE_FOR_DEBUGGING;
907 case Attribute::OptForFuzzing:
908 return bitc::ATTR_KIND_OPT_FOR_FUZZING;
909 case Attribute::OptimizeForSize:
910 return bitc::ATTR_KIND_OPTIMIZE_FOR_SIZE;
911 case Attribute::OptimizeNone:
912 return bitc::ATTR_KIND_OPTIMIZE_NONE;
913 case Attribute::ReadNone:
914 return bitc::ATTR_KIND_READ_NONE;
915 case Attribute::ReadOnly:
916 return bitc::ATTR_KIND_READ_ONLY;
917 case Attribute::Returned:
918 return bitc::ATTR_KIND_RETURNED;
919 case Attribute::ReturnsTwice:
920 return bitc::ATTR_KIND_RETURNS_TWICE;
921 case Attribute::SExt:
922 return bitc::ATTR_KIND_S_EXT;
923 case Attribute::Speculatable:
924 return bitc::ATTR_KIND_SPECULATABLE;
925 case Attribute::StackAlignment:
926 return bitc::ATTR_KIND_STACK_ALIGNMENT;
927 case Attribute::StackProtect:
928 return bitc::ATTR_KIND_STACK_PROTECT;
929 case Attribute::StackProtectReq:
930 return bitc::ATTR_KIND_STACK_PROTECT_REQ;
931 case Attribute::StackProtectStrong:
932 return bitc::ATTR_KIND_STACK_PROTECT_STRONG;
933 case Attribute::SafeStack:
934 return bitc::ATTR_KIND_SAFESTACK;
935 case Attribute::ShadowCallStack:
936 return bitc::ATTR_KIND_SHADOWCALLSTACK;
937 case Attribute::StrictFP:
938 return bitc::ATTR_KIND_STRICT_FP;
939 case Attribute::StructRet:
940 return bitc::ATTR_KIND_STRUCT_RET;
941 case Attribute::SanitizeAddress:
942 return bitc::ATTR_KIND_SANITIZE_ADDRESS;
943 case Attribute::SanitizeAllocToken:
944 return bitc::ATTR_KIND_SANITIZE_ALLOC_TOKEN;
945 case Attribute::SanitizeHWAddress:
946 return bitc::ATTR_KIND_SANITIZE_HWADDRESS;
947 case Attribute::SanitizeThread:
948 return bitc::ATTR_KIND_SANITIZE_THREAD;
949 case Attribute::SanitizeType:
950 return bitc::ATTR_KIND_SANITIZE_TYPE;
951 case Attribute::SanitizeMemory:
952 return bitc::ATTR_KIND_SANITIZE_MEMORY;
953 case Attribute::SanitizeNumericalStability:
954 return bitc::ATTR_KIND_SANITIZE_NUMERICAL_STABILITY;
955 case Attribute::SanitizeRealtime:
956 return bitc::ATTR_KIND_SANITIZE_REALTIME;
957 case Attribute::SanitizeRealtimeBlocking:
958 return bitc::ATTR_KIND_SANITIZE_REALTIME_BLOCKING;
959 case Attribute::SpeculativeLoadHardening:
960 return bitc::ATTR_KIND_SPECULATIVE_LOAD_HARDENING;
961 case Attribute::SwiftError:
962 return bitc::ATTR_KIND_SWIFT_ERROR;
963 case Attribute::SwiftSelf:
964 return bitc::ATTR_KIND_SWIFT_SELF;
965 case Attribute::SwiftAsync:
966 return bitc::ATTR_KIND_SWIFT_ASYNC;
967 case Attribute::UWTable:
968 return bitc::ATTR_KIND_UW_TABLE;
969 case Attribute::VScaleRange:
970 return bitc::ATTR_KIND_VSCALE_RANGE;
971 case Attribute::WillReturn:
972 return bitc::ATTR_KIND_WILLRETURN;
973 case Attribute::WriteOnly:
974 return bitc::ATTR_KIND_WRITEONLY;
975 case Attribute::ZExt:
976 return bitc::ATTR_KIND_Z_EXT;
977 case Attribute::ImmArg:
978 return bitc::ATTR_KIND_IMMARG;
979 case Attribute::SanitizeMemTag:
980 return bitc::ATTR_KIND_SANITIZE_MEMTAG;
981 case Attribute::Preallocated:
982 return bitc::ATTR_KIND_PREALLOCATED;
983 case Attribute::NoUndef:
984 return bitc::ATTR_KIND_NOUNDEF;
985 case Attribute::ByRef:
986 return bitc::ATTR_KIND_BYREF;
987 case Attribute::MustProgress:
988 return bitc::ATTR_KIND_MUSTPROGRESS;
989 case Attribute::PresplitCoroutine:
990 return bitc::ATTR_KIND_PRESPLIT_COROUTINE;
991 case Attribute::Writable:
992 return bitc::ATTR_KIND_WRITABLE;
993 case Attribute::CoroDestroyOnlyWhenComplete:
994 return bitc::ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE;
995 case Attribute::CoroElideSafe:
996 return bitc::ATTR_KIND_CORO_ELIDE_SAFE;
997 case Attribute::DeadOnUnwind:
998 return bitc::ATTR_KIND_DEAD_ON_UNWIND;
999 case Attribute::Range:
1000 return bitc::ATTR_KIND_RANGE;
1001 case Attribute::Initializes:
1002 return bitc::ATTR_KIND_INITIALIZES;
1003 case Attribute::NoExt:
1004 return bitc::ATTR_KIND_NO_EXT;
1005 case Attribute::Captures:
1006 return bitc::ATTR_KIND_CAPTURES;
1007 case Attribute::DeadOnReturn:
1008 return bitc::ATTR_KIND_DEAD_ON_RETURN;
1009 case Attribute::NoCreateUndefOrPoison:
1010 return bitc::ATTR_KIND_NO_CREATE_UNDEF_OR_POISON;
1011 case Attribute::DenormalFPEnv:
1012 return bitc::ATTR_KIND_DENORMAL_FPENV;
1013 case Attribute::NoOutline:
1014 return bitc::ATTR_KIND_NOOUTLINE;
1015 case Attribute::NoIPA:
1016 return bitc::ATTR_KIND_NOIPA;
1017 case Attribute::EndAttrKinds:
1018 llvm_unreachable("Can not encode end-attribute kinds marker.");
1019 case Attribute::None:
1020 llvm_unreachable("Can not encode none-attribute.");
1021 case Attribute::EmptyKey:
1022 case Attribute::TombstoneKey:
1023 llvm_unreachable("Trying to encode EmptyKey/TombstoneKey");
1024 }
1025
1026 llvm_unreachable("Trying to encode unknown attribute");
1027}
1028
1029static void emitSignedInt64(SmallVectorImpl<uint64_t> &Vals, uint64_t V) {
1030 if ((int64_t)V >= 0)
1031 Vals.push_back(Elt: V << 1);
1032 else
1033 Vals.push_back(Elt: (-V << 1) | 1);
1034}
1035
1036static void emitWideAPInt(SmallVectorImpl<uint64_t> &Vals, const APInt &A) {
1037 // We have an arbitrary precision integer value to write whose
1038 // bit width is > 64. However, in canonical unsigned integer
1039 // format it is likely that the high bits are going to be zero.
1040 // So, we only write the number of active words.
1041 unsigned NumWords = A.getActiveWords();
1042 const uint64_t *RawData = A.getRawData();
1043 for (unsigned i = 0; i < NumWords; i++)
1044 emitSignedInt64(Vals, V: RawData[i]);
1045}
1046
1047static void emitConstantRange(SmallVectorImpl<uint64_t> &Record,
1048 const ConstantRange &CR, bool EmitBitWidth) {
1049 unsigned BitWidth = CR.getBitWidth();
1050 if (EmitBitWidth)
1051 Record.push_back(Elt: BitWidth);
1052 if (BitWidth > 64) {
1053 Record.push_back(Elt: CR.getLower().getActiveWords() |
1054 (uint64_t(CR.getUpper().getActiveWords()) << 32));
1055 emitWideAPInt(Vals&: Record, A: CR.getLower());
1056 emitWideAPInt(Vals&: Record, A: CR.getUpper());
1057 } else {
1058 emitSignedInt64(Vals&: Record, V: CR.getLower().getSExtValue());
1059 emitSignedInt64(Vals&: Record, V: CR.getUpper().getSExtValue());
1060 }
1061}
1062
1063void ModuleBitcodeWriter::writeAttributeGroupTable() {
1064 const std::vector<ValueEnumerator::IndexAndAttrSet> &AttrGrps =
1065 VE.getAttributeGroups();
1066 if (AttrGrps.empty()) return;
1067
1068 Stream.EnterSubblock(BlockID: bitc::PARAMATTR_GROUP_BLOCK_ID, CodeLen: 3);
1069
1070 SmallVector<uint64_t, 64> Record;
1071 for (ValueEnumerator::IndexAndAttrSet Pair : AttrGrps) {
1072 unsigned AttrListIndex = Pair.first;
1073 AttributeSet AS = Pair.second;
1074 Record.push_back(Elt: VE.getAttributeGroupID(Group: Pair));
1075 Record.push_back(Elt: AttrListIndex);
1076
1077 for (Attribute Attr : AS) {
1078 if (Attr.isEnumAttribute()) {
1079 Record.push_back(Elt: 0);
1080 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1081 } else if (Attr.isIntAttribute()) {
1082 Record.push_back(Elt: 1);
1083 Attribute::AttrKind Kind = Attr.getKindAsEnum();
1084 Record.push_back(Elt: getAttrKindEncoding(Kind));
1085 if (Kind == Attribute::Memory) {
1086 // Version field for upgrading old memory effects.
1087 const uint64_t Version = 2;
1088 Record.push_back(Elt: (Version << 56) | Attr.getValueAsInt());
1089 } else {
1090 Record.push_back(Elt: Attr.getValueAsInt());
1091 }
1092 } else if (Attr.isStringAttribute()) {
1093 StringRef Kind = Attr.getKindAsString();
1094 StringRef Val = Attr.getValueAsString();
1095
1096 Record.push_back(Elt: Val.empty() ? 3 : 4);
1097 Record.append(in_start: Kind.begin(), in_end: Kind.end());
1098 Record.push_back(Elt: 0);
1099 if (!Val.empty()) {
1100 Record.append(in_start: Val.begin(), in_end: Val.end());
1101 Record.push_back(Elt: 0);
1102 }
1103 } else if (Attr.isTypeAttribute()) {
1104 Type *Ty = Attr.getValueAsType();
1105 Record.push_back(Elt: Ty ? 6 : 5);
1106 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1107 if (Ty)
1108 Record.push_back(Elt: VE.getTypeID(T: Attr.getValueAsType()));
1109 } else if (Attr.isConstantRangeAttribute()) {
1110 Record.push_back(Elt: 7);
1111 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1112 emitConstantRange(Record, CR: Attr.getValueAsConstantRange(),
1113 /*EmitBitWidth=*/true);
1114 } else {
1115 assert(Attr.isConstantRangeListAttribute());
1116 Record.push_back(Elt: 8);
1117 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1118 ArrayRef<ConstantRange> Val = Attr.getValueAsConstantRangeList();
1119 Record.push_back(Elt: Val.size());
1120 Record.push_back(Elt: Val[0].getBitWidth());
1121 for (auto &CR : Val)
1122 emitConstantRange(Record, CR, /*EmitBitWidth=*/false);
1123 }
1124 }
1125
1126 Stream.EmitRecord(Code: bitc::PARAMATTR_GRP_CODE_ENTRY, Vals: Record);
1127 Record.clear();
1128 }
1129
1130 Stream.ExitBlock();
1131}
1132
1133void ModuleBitcodeWriter::writeAttributeTable() {
1134 const std::vector<AttributeList> &Attrs = VE.getAttributeLists();
1135 if (Attrs.empty()) return;
1136
1137 Stream.EnterSubblock(BlockID: bitc::PARAMATTR_BLOCK_ID, CodeLen: 3);
1138
1139 SmallVector<uint64_t, 64> Record;
1140 for (const AttributeList &AL : Attrs) {
1141 for (unsigned i : AL.indexes()) {
1142 AttributeSet AS = AL.getAttributes(Index: i);
1143 if (AS.hasAttributes())
1144 Record.push_back(Elt: VE.getAttributeGroupID(Group: {i, AS}));
1145 }
1146
1147 Stream.EmitRecord(Code: bitc::PARAMATTR_CODE_ENTRY, Vals: Record);
1148 Record.clear();
1149 }
1150
1151 Stream.ExitBlock();
1152}
1153
1154/// WriteTypeTable - Write out the type table for a module.
1155void ModuleBitcodeWriter::writeTypeTable() {
1156 const ValueEnumerator::TypeList &TypeList = VE.getTypes();
1157
1158 Stream.EnterSubblock(BlockID: bitc::TYPE_BLOCK_ID_NEW, CodeLen: 4 /*count from # abbrevs */);
1159 SmallVector<uint64_t, 64> TypeVals;
1160
1161 uint64_t NumBits = VE.computeBitsRequiredForTypeIndices();
1162
1163 // Abbrev for TYPE_CODE_OPAQUE_POINTER.
1164 auto Abbv = std::make_shared<BitCodeAbbrev>();
1165 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_OPAQUE_POINTER));
1166 Abbv->Add(OpInfo: BitCodeAbbrevOp(0)); // Addrspace = 0
1167 unsigned OpaquePtrAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1168
1169 // Abbrev for TYPE_CODE_FUNCTION.
1170 Abbv = std::make_shared<BitCodeAbbrev>();
1171 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_FUNCTION));
1172 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isvararg
1173 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1174 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1175 unsigned FunctionAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1176
1177 // Abbrev for TYPE_CODE_STRUCT_ANON.
1178 Abbv = std::make_shared<BitCodeAbbrev>();
1179 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_ANON));
1180 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked
1181 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1182 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1183 unsigned StructAnonAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1184
1185 // Abbrev for TYPE_CODE_STRUCT_NAME.
1186 Abbv = std::make_shared<BitCodeAbbrev>();
1187 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAME));
1188 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1189 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
1190 unsigned StructNameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1191
1192 // Abbrev for TYPE_CODE_STRUCT_NAMED.
1193 Abbv = std::make_shared<BitCodeAbbrev>();
1194 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAMED));
1195 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked
1196 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1197 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1198 unsigned StructNamedAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1199
1200 // Abbrev for TYPE_CODE_ARRAY.
1201 Abbv = std::make_shared<BitCodeAbbrev>();
1202 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_ARRAY));
1203 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // size
1204 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1205 unsigned ArrayAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1206
1207 // Emit an entry count so the reader can reserve space.
1208 TypeVals.push_back(Elt: TypeList.size());
1209 Stream.EmitRecord(Code: bitc::TYPE_CODE_NUMENTRY, Vals: TypeVals);
1210 TypeVals.clear();
1211
1212 // Loop over all of the types, emitting each in turn.
1213 for (Type *T : TypeList) {
1214 int AbbrevToUse = 0;
1215 unsigned Code = 0;
1216
1217 switch (T->getTypeID()) {
1218 case Type::VoidTyID: Code = bitc::TYPE_CODE_VOID; break;
1219 case Type::HalfTyID: Code = bitc::TYPE_CODE_HALF; break;
1220 case Type::BFloatTyID: Code = bitc::TYPE_CODE_BFLOAT; break;
1221 case Type::FloatTyID: Code = bitc::TYPE_CODE_FLOAT; break;
1222 case Type::DoubleTyID: Code = bitc::TYPE_CODE_DOUBLE; break;
1223 case Type::X86_FP80TyID: Code = bitc::TYPE_CODE_X86_FP80; break;
1224 case Type::FP128TyID: Code = bitc::TYPE_CODE_FP128; break;
1225 case Type::PPC_FP128TyID: Code = bitc::TYPE_CODE_PPC_FP128; break;
1226 case Type::LabelTyID: Code = bitc::TYPE_CODE_LABEL; break;
1227 case Type::MetadataTyID:
1228 Code = bitc::TYPE_CODE_METADATA;
1229 break;
1230 case Type::X86_AMXTyID: Code = bitc::TYPE_CODE_X86_AMX; break;
1231 case Type::TokenTyID: Code = bitc::TYPE_CODE_TOKEN; break;
1232 case Type::ByteTyID:
1233 // BYTE: [width]
1234 Code = bitc::TYPE_CODE_BYTE;
1235 TypeVals.push_back(Elt: T->getByteBitWidth());
1236 break;
1237 case Type::IntegerTyID:
1238 // INTEGER: [width]
1239 Code = bitc::TYPE_CODE_INTEGER;
1240 TypeVals.push_back(Elt: cast<IntegerType>(Val: T)->getBitWidth());
1241 break;
1242 case Type::PointerTyID: {
1243 PointerType *PTy = cast<PointerType>(Val: T);
1244 unsigned AddressSpace = PTy->getAddressSpace();
1245 // OPAQUE_POINTER: [address space]
1246 Code = bitc::TYPE_CODE_OPAQUE_POINTER;
1247 TypeVals.push_back(Elt: AddressSpace);
1248 if (AddressSpace == 0)
1249 AbbrevToUse = OpaquePtrAbbrev;
1250 break;
1251 }
1252 case Type::FunctionTyID: {
1253 FunctionType *FT = cast<FunctionType>(Val: T);
1254 // FUNCTION: [isvararg, retty, paramty x N]
1255 Code = bitc::TYPE_CODE_FUNCTION;
1256 TypeVals.push_back(Elt: FT->isVarArg());
1257 TypeVals.push_back(Elt: VE.getTypeID(T: FT->getReturnType()));
1258 for (unsigned i = 0, e = FT->getNumParams(); i != e; ++i)
1259 TypeVals.push_back(Elt: VE.getTypeID(T: FT->getParamType(i)));
1260 AbbrevToUse = FunctionAbbrev;
1261 break;
1262 }
1263 case Type::StructTyID: {
1264 StructType *ST = cast<StructType>(Val: T);
1265 // STRUCT: [ispacked, eltty x N]
1266 TypeVals.push_back(Elt: ST->isPacked());
1267 // Output all of the element types.
1268 for (Type *ET : ST->elements())
1269 TypeVals.push_back(Elt: VE.getTypeID(T: ET));
1270
1271 if (ST->isLiteral()) {
1272 Code = bitc::TYPE_CODE_STRUCT_ANON;
1273 AbbrevToUse = StructAnonAbbrev;
1274 } else {
1275 if (ST->isOpaque()) {
1276 Code = bitc::TYPE_CODE_OPAQUE;
1277 } else {
1278 Code = bitc::TYPE_CODE_STRUCT_NAMED;
1279 AbbrevToUse = StructNamedAbbrev;
1280 }
1281
1282 // Emit the name if it is present.
1283 if (!ST->getName().empty())
1284 writeStringRecord(Stream, Code: bitc::TYPE_CODE_STRUCT_NAME, Str: ST->getName(),
1285 AbbrevToUse: StructNameAbbrev);
1286 }
1287 break;
1288 }
1289 case Type::ArrayTyID: {
1290 ArrayType *AT = cast<ArrayType>(Val: T);
1291 // ARRAY: [numelts, eltty]
1292 Code = bitc::TYPE_CODE_ARRAY;
1293 TypeVals.push_back(Elt: AT->getNumElements());
1294 TypeVals.push_back(Elt: VE.getTypeID(T: AT->getElementType()));
1295 AbbrevToUse = ArrayAbbrev;
1296 break;
1297 }
1298 case Type::FixedVectorTyID:
1299 case Type::ScalableVectorTyID: {
1300 VectorType *VT = cast<VectorType>(Val: T);
1301 // VECTOR [numelts, eltty] or
1302 // [numelts, eltty, scalable]
1303 Code = bitc::TYPE_CODE_VECTOR;
1304 TypeVals.push_back(Elt: VT->getElementCount().getKnownMinValue());
1305 TypeVals.push_back(Elt: VE.getTypeID(T: VT->getElementType()));
1306 if (isa<ScalableVectorType>(Val: VT))
1307 TypeVals.push_back(Elt: true);
1308 break;
1309 }
1310 case Type::TargetExtTyID: {
1311 TargetExtType *TET = cast<TargetExtType>(Val: T);
1312 Code = bitc::TYPE_CODE_TARGET_TYPE;
1313 writeStringRecord(Stream, Code: bitc::TYPE_CODE_STRUCT_NAME, Str: TET->getName(),
1314 AbbrevToUse: StructNameAbbrev);
1315 TypeVals.push_back(Elt: TET->getNumTypeParameters());
1316 for (Type *InnerTy : TET->type_params())
1317 TypeVals.push_back(Elt: VE.getTypeID(T: InnerTy));
1318 llvm::append_range(C&: TypeVals, R: TET->int_params());
1319 break;
1320 }
1321 case Type::TypedPointerTyID:
1322 llvm_unreachable("Typed pointers cannot be added to IR modules");
1323 }
1324
1325 // Emit the finished record.
1326 Stream.EmitRecord(Code, Vals: TypeVals, Abbrev: AbbrevToUse);
1327 TypeVals.clear();
1328 }
1329
1330 Stream.ExitBlock();
1331}
1332
1333static unsigned getEncodedLinkage(const GlobalValue::LinkageTypes Linkage) {
1334 switch (Linkage) {
1335 case GlobalValue::ExternalLinkage:
1336 return 0;
1337 case GlobalValue::WeakAnyLinkage:
1338 return 16;
1339 case GlobalValue::AppendingLinkage:
1340 return 2;
1341 case GlobalValue::InternalLinkage:
1342 return 3;
1343 case GlobalValue::LinkOnceAnyLinkage:
1344 return 18;
1345 case GlobalValue::ExternalWeakLinkage:
1346 return 7;
1347 case GlobalValue::CommonLinkage:
1348 return 8;
1349 case GlobalValue::PrivateLinkage:
1350 return 9;
1351 case GlobalValue::WeakODRLinkage:
1352 return 17;
1353 case GlobalValue::LinkOnceODRLinkage:
1354 return 19;
1355 case GlobalValue::AvailableExternallyLinkage:
1356 return 12;
1357 }
1358 llvm_unreachable("Invalid linkage");
1359}
1360
1361static unsigned getEncodedLinkage(const GlobalValue &GV) {
1362 return getEncodedLinkage(Linkage: GV.getLinkage());
1363}
1364
1365static uint64_t getEncodedFFlags(FunctionSummary::FFlags Flags) {
1366 uint64_t RawFlags = 0;
1367 RawFlags |= Flags.ReadNone;
1368 RawFlags |= (Flags.ReadOnly << 1);
1369 RawFlags |= (Flags.NoRecurse << 2);
1370 RawFlags |= (Flags.ReturnDoesNotAlias << 3);
1371 RawFlags |= (Flags.NoInline << 4);
1372 RawFlags |= (Flags.AlwaysInline << 5);
1373 RawFlags |= (Flags.NoUnwind << 6);
1374 RawFlags |= (Flags.MayThrow << 7);
1375 RawFlags |= (Flags.HasUnknownCall << 8);
1376 RawFlags |= (Flags.MustBeUnreachable << 9);
1377 return RawFlags;
1378}
1379
1380// Decode the flags for GlobalValue in the summary. See getDecodedGVSummaryFlags
1381// in BitcodeReader.cpp.
1382static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags,
1383 bool ImportAsDecl = false) {
1384 uint64_t RawFlags = 0;
1385
1386 RawFlags |= Flags.NotEligibleToImport; // bool
1387 RawFlags |= (Flags.Live << 1);
1388 RawFlags |= (Flags.DSOLocal << 2);
1389 RawFlags |= (Flags.CanAutoHide << 3);
1390
1391 // Linkage don't need to be remapped at that time for the summary. Any future
1392 // change to the getEncodedLinkage() function will need to be taken into
1393 // account here as well.
1394 RawFlags = (RawFlags << 4) | Flags.Linkage; // 4 bits
1395
1396 RawFlags |= (Flags.Visibility << 8); // 2 bits
1397
1398 unsigned ImportType = Flags.ImportType | ImportAsDecl;
1399 RawFlags |= (ImportType << 10); // 1 bit
1400
1401 RawFlags |= (Flags.NoRenameOnPromotion << 11); // 1 bit
1402
1403 return RawFlags;
1404}
1405
1406static uint64_t getEncodedGVarFlags(GlobalVarSummary::GVarFlags Flags) {
1407 uint64_t RawFlags = Flags.MaybeReadOnly | (Flags.MaybeWriteOnly << 1) |
1408 (Flags.Constant << 2) | Flags.VCallVisibility << 3;
1409 return RawFlags;
1410}
1411
1412static uint64_t getEncodedHotnessCallEdgeInfo(const CalleeInfo &CI) {
1413 uint64_t RawFlags = 0;
1414
1415 RawFlags |= CI.Hotness; // 3 bits
1416 RawFlags |= (CI.HasTailCall << 3); // 1 bit
1417
1418 return RawFlags;
1419}
1420
1421static unsigned getEncodedVisibility(const GlobalValue &GV) {
1422 switch (GV.getVisibility()) {
1423 case GlobalValue::DefaultVisibility: return 0;
1424 case GlobalValue::HiddenVisibility: return 1;
1425 case GlobalValue::ProtectedVisibility: return 2;
1426 }
1427 llvm_unreachable("Invalid visibility");
1428}
1429
1430static unsigned getEncodedDLLStorageClass(const GlobalValue &GV) {
1431 switch (GV.getDLLStorageClass()) {
1432 case GlobalValue::DefaultStorageClass: return 0;
1433 case GlobalValue::DLLImportStorageClass: return 1;
1434 case GlobalValue::DLLExportStorageClass: return 2;
1435 }
1436 llvm_unreachable("Invalid DLL storage class");
1437}
1438
1439static unsigned getEncodedThreadLocalMode(const GlobalValue &GV) {
1440 switch (GV.getThreadLocalMode()) {
1441 case GlobalVariable::NotThreadLocal: return 0;
1442 case GlobalVariable::GeneralDynamicTLSModel: return 1;
1443 case GlobalVariable::LocalDynamicTLSModel: return 2;
1444 case GlobalVariable::InitialExecTLSModel: return 3;
1445 case GlobalVariable::LocalExecTLSModel: return 4;
1446 }
1447 llvm_unreachable("Invalid TLS model");
1448}
1449
1450static unsigned getEncodedComdatSelectionKind(const Comdat &C) {
1451 switch (C.getSelectionKind()) {
1452 case Comdat::Any:
1453 return bitc::COMDAT_SELECTION_KIND_ANY;
1454 case Comdat::ExactMatch:
1455 return bitc::COMDAT_SELECTION_KIND_EXACT_MATCH;
1456 case Comdat::Largest:
1457 return bitc::COMDAT_SELECTION_KIND_LARGEST;
1458 case Comdat::NoDeduplicate:
1459 return bitc::COMDAT_SELECTION_KIND_NO_DUPLICATES;
1460 case Comdat::SameSize:
1461 return bitc::COMDAT_SELECTION_KIND_SAME_SIZE;
1462 }
1463 llvm_unreachable("Invalid selection kind");
1464}
1465
1466static unsigned getEncodedUnnamedAddr(const GlobalValue &GV) {
1467 switch (GV.getUnnamedAddr()) {
1468 case GlobalValue::UnnamedAddr::None: return 0;
1469 case GlobalValue::UnnamedAddr::Local: return 2;
1470 case GlobalValue::UnnamedAddr::Global: return 1;
1471 }
1472 llvm_unreachable("Invalid unnamed_addr");
1473}
1474
1475size_t ModuleBitcodeWriter::addToStrtab(StringRef Str) {
1476 if (GenerateHash)
1477 Hasher.update(Str);
1478 return StrtabBuilder.add(S: Str);
1479}
1480
1481void ModuleBitcodeWriter::writeComdats() {
1482 SmallVector<unsigned, 64> Vals;
1483 for (const Comdat *C : VE.getComdats()) {
1484 // COMDAT: [strtab offset, strtab size, selection_kind]
1485 Vals.push_back(Elt: addToStrtab(Str: C->getName()));
1486 Vals.push_back(Elt: C->getName().size());
1487 Vals.push_back(Elt: getEncodedComdatSelectionKind(C: *C));
1488 Stream.EmitRecord(Code: bitc::MODULE_CODE_COMDAT, Vals, /*AbbrevToUse=*/Abbrev: 0);
1489 Vals.clear();
1490 }
1491}
1492
1493/// Write a record that will eventually hold the word offset of the
1494/// module-level VST. For now the offset is 0, which will be backpatched
1495/// after the real VST is written. Saves the bit offset to backpatch.
1496void ModuleBitcodeWriter::writeValueSymbolTableForwardDecl() {
1497 // Write a placeholder value in for the offset of the real VST,
1498 // which is written after the function blocks so that it can include
1499 // the offset of each function. The placeholder offset will be
1500 // updated when the real VST is written.
1501 auto Abbv = std::make_shared<BitCodeAbbrev>();
1502 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_VSTOFFSET));
1503 // Blocks are 32-bit aligned, so we can use a 32-bit word offset to
1504 // hold the real VST offset. Must use fixed instead of VBR as we don't
1505 // know how many VBR chunks to reserve ahead of time.
1506 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
1507 unsigned VSTOffsetAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1508
1509 // Emit the placeholder
1510 uint64_t Vals[] = {bitc::MODULE_CODE_VSTOFFSET, 0};
1511 Stream.EmitRecordWithAbbrev(Abbrev: VSTOffsetAbbrev, Vals);
1512
1513 // Compute and save the bit offset to the placeholder, which will be
1514 // patched when the real VST is written. We can simply subtract the 32-bit
1515 // fixed size from the current bit number to get the location to backpatch.
1516 VSTOffsetPlaceholder = Stream.GetCurrentBitNo() - 32;
1517}
1518
1519enum StringEncoding { SE_Char6, SE_Fixed7, SE_Fixed8 };
1520
1521/// Determine the encoding to use for the given string name and length.
1522static StringEncoding getStringEncoding(StringRef Str) {
1523 bool isChar6 = true;
1524 for (char C : Str) {
1525 if (isChar6)
1526 isChar6 = BitCodeAbbrevOp::isChar6(C);
1527 if ((unsigned char)C & 128)
1528 // don't bother scanning the rest.
1529 return SE_Fixed8;
1530 }
1531 if (isChar6)
1532 return SE_Char6;
1533 return SE_Fixed7;
1534}
1535
1536static_assert(sizeof(GlobalValue::SanitizerMetadata) <= sizeof(unsigned),
1537 "Sanitizer Metadata is too large for naive serialization.");
1538static unsigned
1539serializeSanitizerMetadata(const GlobalValue::SanitizerMetadata &Meta) {
1540 return Meta.NoAddress | (Meta.NoHWAddress << 1) |
1541 (Meta.Memtag << 2) | (Meta.IsDynInit << 3);
1542}
1543
1544/// Emit top-level description of module, including target triple, inline asm,
1545/// descriptors for global variables, and function prototype info.
1546/// Returns the bit offset to backpatch with the location of the real VST.
1547void ModuleBitcodeWriter::writeModuleInfo() {
1548 // Emit various pieces of data attached to a module.
1549 if (!M.getTargetTriple().empty())
1550 writeStringRecord(Stream, Code: bitc::MODULE_CODE_TRIPLE,
1551 Str: M.getTargetTriple().str(), AbbrevToUse: 0 /*TODO*/);
1552 const std::string &DL = M.getDataLayoutStr();
1553 if (!DL.empty())
1554 writeStringRecord(Stream, Code: bitc::MODULE_CODE_DATALAYOUT, Str: DL, AbbrevToUse: 0 /*TODO*/);
1555
1556 for (const Module::GlobalAsmFragment &Frag : M.getModuleInlineAsm()) {
1557 SmallVector<std::pair<StringRef, StringRef>> Props =
1558 Frag.Props.getAsStrings();
1559 for (auto [Key, Value] : Props) {
1560 SmallVector<unsigned, 64> Record;
1561 Record.append(in_start: Key.begin(), in_end: Key.end());
1562 Record.push_back(Elt: 0);
1563 Record.append(in_start: Value.begin(), in_end: Value.end());
1564 Stream.EmitRecord(Code: bitc::MODULE_CODE_ASM_PROPERTY, Vals: Record);
1565 }
1566 writeStringRecord(Stream, Code: bitc::MODULE_CODE_ASM, Str: Frag.Asm, AbbrevToUse: 0 /*TODO*/);
1567 }
1568
1569 // Emit information about sections and GC, computing how many there are. Also
1570 // compute the maximum alignment value.
1571 std::map<std::string, unsigned> SectionMap;
1572 std::map<std::string, unsigned> GCMap;
1573 MaybeAlign MaxGVarAlignment;
1574 unsigned MaxGlobalType = 0;
1575 for (const GlobalVariable &GV : M.globals()) {
1576 if (MaybeAlign A = GV.getAlign())
1577 MaxGVarAlignment = !MaxGVarAlignment ? *A : std::max(a: *MaxGVarAlignment, b: *A);
1578 MaxGlobalType = std::max(a: MaxGlobalType, b: VE.getTypeID(T: GV.getValueType()));
1579 if (GV.hasSection()) {
1580 // Give section names unique ID's.
1581 unsigned &Entry = SectionMap[std::string(GV.getSection())];
1582 if (!Entry) {
1583 writeStringRecord(Stream, Code: bitc::MODULE_CODE_SECTIONNAME, Str: GV.getSection(),
1584 AbbrevToUse: 0 /*TODO*/);
1585 Entry = SectionMap.size();
1586 }
1587 }
1588 }
1589 for (const Function &F : M) {
1590 if (F.hasSection()) {
1591 // Give section names unique ID's.
1592 unsigned &Entry = SectionMap[std::string(F.getSection())];
1593 if (!Entry) {
1594 writeStringRecord(Stream, Code: bitc::MODULE_CODE_SECTIONNAME, Str: F.getSection(),
1595 AbbrevToUse: 0 /*TODO*/);
1596 Entry = SectionMap.size();
1597 }
1598 }
1599 if (F.hasGC()) {
1600 // Same for GC names.
1601 unsigned &Entry = GCMap[F.getGC()];
1602 if (!Entry) {
1603 writeStringRecord(Stream, Code: bitc::MODULE_CODE_GCNAME, Str: F.getGC(),
1604 AbbrevToUse: 0 /*TODO*/);
1605 Entry = GCMap.size();
1606 }
1607 }
1608 }
1609
1610 // Emit abbrev for globals, now that we know # sections and max alignment.
1611 unsigned SimpleGVarAbbrev = 0;
1612 if (!M.global_empty()) {
1613 // Add an abbrev for common globals with no visibility or thread localness.
1614 auto Abbv = std::make_shared<BitCodeAbbrev>();
1615 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_GLOBALVAR));
1616 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1617 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1618 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1619 Log2_32_Ceil(Value: MaxGlobalType+1)));
1620 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // AddrSpace << 2
1621 //| explicitType << 1
1622 //| constant
1623 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Initializer.
1624 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // Linkage.
1625 if (!MaxGVarAlignment) // Alignment.
1626 Abbv->Add(OpInfo: BitCodeAbbrevOp(0));
1627 else {
1628 unsigned MaxEncAlignment = getEncodedAlign(Alignment: MaxGVarAlignment);
1629 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1630 Log2_32_Ceil(Value: MaxEncAlignment+1)));
1631 }
1632 if (SectionMap.empty()) // Section.
1633 Abbv->Add(OpInfo: BitCodeAbbrevOp(0));
1634 else
1635 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1636 Log2_32_Ceil(Value: SectionMap.size()+1)));
1637 // Don't bother emitting vis + thread local.
1638 SimpleGVarAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1639 }
1640
1641 SmallVector<unsigned, 64> Vals;
1642 // Emit the module's source file name.
1643 {
1644 StringEncoding Bits = getStringEncoding(Str: M.getSourceFileName());
1645 BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
1646 if (Bits == SE_Char6)
1647 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
1648 else if (Bits == SE_Fixed7)
1649 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
1650
1651 // MODULE_CODE_SOURCE_FILENAME: [namechar x N]
1652 auto Abbv = std::make_shared<BitCodeAbbrev>();
1653 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
1654 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1655 Abbv->Add(OpInfo: AbbrevOpToUse);
1656 unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1657
1658 for (const auto P : M.getSourceFileName())
1659 Vals.push_back(Elt: (unsigned char)P);
1660
1661 // Emit the finished record.
1662 Stream.EmitRecord(Code: bitc::MODULE_CODE_SOURCE_FILENAME, Vals, Abbrev: FilenameAbbrev);
1663 Vals.clear();
1664 }
1665
1666 writeGUIDList();
1667
1668 // Emit the global variable information.
1669 for (const GlobalVariable &GV : M.globals()) {
1670 unsigned AbbrevToUse = 0;
1671
1672 // GLOBALVAR: [strtab offset, strtab size, type, isconst, initid,
1673 // linkage, alignment, section, visibility, threadlocal,
1674 // unnamed_addr, externally_initialized, dllstorageclass,
1675 // comdat, attributes, DSO_Local, GlobalSanitizer, code_model]
1676 Vals.push_back(Elt: addToStrtab(Str: GV.getName()));
1677 Vals.push_back(Elt: GV.getName().size());
1678 Vals.push_back(Elt: VE.getTypeID(T: GV.getValueType()));
1679 Vals.push_back(Elt: GV.getType()->getAddressSpace() << 2 | 2 | GV.isConstant());
1680 Vals.push_back(Elt: GV.isDeclaration() ? 0 :
1681 (VE.getValueID(V: GV.getInitializer()) + 1));
1682 Vals.push_back(Elt: getEncodedLinkage(GV));
1683 Vals.push_back(Elt: getEncodedAlign(Alignment: GV.getAlign()));
1684 Vals.push_back(Elt: GV.hasSection() ? SectionMap[std::string(GV.getSection())]
1685 : 0);
1686 if (GV.isThreadLocal() ||
1687 GV.getVisibility() != GlobalValue::DefaultVisibility ||
1688 GV.getUnnamedAddr() != GlobalValue::UnnamedAddr::None ||
1689 GV.isExternallyInitialized() ||
1690 GV.getDLLStorageClass() != GlobalValue::DefaultStorageClass ||
1691 GV.hasComdat() || GV.hasAttributes() || GV.isDSOLocal() ||
1692 GV.hasPartition() || GV.hasSanitizerMetadata() || GV.getCodeModel()) {
1693 Vals.push_back(Elt: getEncodedVisibility(GV));
1694 Vals.push_back(Elt: getEncodedThreadLocalMode(GV));
1695 Vals.push_back(Elt: getEncodedUnnamedAddr(GV));
1696 Vals.push_back(Elt: GV.isExternallyInitialized());
1697 Vals.push_back(Elt: getEncodedDLLStorageClass(GV));
1698 Vals.push_back(Elt: GV.hasComdat() ? VE.getComdatID(C: GV.getComdat()) : 0);
1699
1700 auto AL = GV.getAttributesAsList(index: AttributeList::FunctionIndex);
1701 Vals.push_back(Elt: VE.getAttributeListID(PAL: AL));
1702
1703 Vals.push_back(Elt: GV.isDSOLocal());
1704 Vals.push_back(Elt: addToStrtab(Str: GV.getPartition()));
1705 Vals.push_back(Elt: GV.getPartition().size());
1706
1707 Vals.push_back(Elt: (GV.hasSanitizerMetadata() ? serializeSanitizerMetadata(
1708 Meta: GV.getSanitizerMetadata())
1709 : 0));
1710 Vals.push_back(Elt: GV.getCodeModelRaw());
1711 } else {
1712 AbbrevToUse = SimpleGVarAbbrev;
1713 }
1714
1715 Stream.EmitRecord(Code: bitc::MODULE_CODE_GLOBALVAR, Vals, Abbrev: AbbrevToUse);
1716 Vals.clear();
1717 }
1718
1719 // Emit the function proto information.
1720 for (const Function &F : M) {
1721 // FUNCTION: [strtab offset, strtab size, type, callingconv, isproto,
1722 // linkage, paramattrs, alignment, section, visibility, gc,
1723 // unnamed_addr, prologuedata, dllstorageclass, comdat,
1724 // prefixdata, personalityfn, DSO_Local, addrspace,
1725 // partition_strtab, partition_size, prefalign]
1726 Vals.push_back(Elt: addToStrtab(Str: F.getName()));
1727 Vals.push_back(Elt: F.getName().size());
1728 Vals.push_back(Elt: VE.getTypeID(T: F.getFunctionType()));
1729 Vals.push_back(Elt: F.getCallingConv());
1730 Vals.push_back(Elt: F.isDeclaration());
1731 Vals.push_back(Elt: getEncodedLinkage(GV: F));
1732 Vals.push_back(Elt: VE.getAttributeListID(PAL: F.getAttributes()));
1733 Vals.push_back(Elt: getEncodedAlign(Alignment: F.getAlign()));
1734 Vals.push_back(Elt: F.hasSection() ? SectionMap[std::string(F.getSection())]
1735 : 0);
1736 Vals.push_back(Elt: getEncodedVisibility(GV: F));
1737 Vals.push_back(Elt: F.hasGC() ? GCMap[F.getGC()] : 0);
1738 Vals.push_back(Elt: getEncodedUnnamedAddr(GV: F));
1739 Vals.push_back(Elt: F.hasPrologueData() ? (VE.getValueID(V: F.getPrologueData()) + 1)
1740 : 0);
1741 Vals.push_back(Elt: getEncodedDLLStorageClass(GV: F));
1742 Vals.push_back(Elt: F.hasComdat() ? VE.getComdatID(C: F.getComdat()) : 0);
1743 Vals.push_back(Elt: F.hasPrefixData() ? (VE.getValueID(V: F.getPrefixData()) + 1)
1744 : 0);
1745 Vals.push_back(
1746 Elt: F.hasPersonalityFn() ? (VE.getValueID(V: F.getPersonalityFn()) + 1) : 0);
1747
1748 Vals.push_back(Elt: F.isDSOLocal());
1749 Vals.push_back(Elt: F.getAddressSpace());
1750 Vals.push_back(Elt: addToStrtab(Str: F.getPartition()));
1751 Vals.push_back(Elt: F.getPartition().size());
1752 Vals.push_back(Elt: getEncodedAlign(Alignment: F.getPreferredAlignment()));
1753
1754 unsigned AbbrevToUse = 0;
1755 Stream.EmitRecord(Code: bitc::MODULE_CODE_FUNCTION, Vals, Abbrev: AbbrevToUse);
1756 Vals.clear();
1757 }
1758
1759 // Emit the alias information.
1760 for (const GlobalAlias &A : M.aliases()) {
1761 // ALIAS: [strtab offset, strtab size, alias type, aliasee val#, linkage,
1762 // visibility, dllstorageclass, threadlocal, unnamed_addr,
1763 // DSO_Local]
1764 Vals.push_back(Elt: addToStrtab(Str: A.getName()));
1765 Vals.push_back(Elt: A.getName().size());
1766 Vals.push_back(Elt: VE.getTypeID(T: A.getValueType()));
1767 Vals.push_back(Elt: A.getType()->getAddressSpace());
1768 Vals.push_back(Elt: VE.getValueID(V: A.getAliasee()));
1769 Vals.push_back(Elt: getEncodedLinkage(GV: A));
1770 Vals.push_back(Elt: getEncodedVisibility(GV: A));
1771 Vals.push_back(Elt: getEncodedDLLStorageClass(GV: A));
1772 Vals.push_back(Elt: getEncodedThreadLocalMode(GV: A));
1773 Vals.push_back(Elt: getEncodedUnnamedAddr(GV: A));
1774 Vals.push_back(Elt: A.isDSOLocal());
1775 Vals.push_back(Elt: addToStrtab(Str: A.getPartition()));
1776 Vals.push_back(Elt: A.getPartition().size());
1777
1778 unsigned AbbrevToUse = 0;
1779 Stream.EmitRecord(Code: bitc::MODULE_CODE_ALIAS, Vals, Abbrev: AbbrevToUse);
1780 Vals.clear();
1781 }
1782
1783 // Emit the ifunc information.
1784 for (const GlobalIFunc &I : M.ifuncs()) {
1785 // IFUNC: [strtab offset, strtab size, ifunc type, address space, resolver
1786 // val#, linkage, visibility, DSO_Local]
1787 Vals.push_back(Elt: addToStrtab(Str: I.getName()));
1788 Vals.push_back(Elt: I.getName().size());
1789 Vals.push_back(Elt: VE.getTypeID(T: I.getValueType()));
1790 Vals.push_back(Elt: I.getType()->getAddressSpace());
1791 Vals.push_back(Elt: VE.getValueID(V: I.getResolver()));
1792 Vals.push_back(Elt: getEncodedLinkage(GV: I));
1793 Vals.push_back(Elt: getEncodedVisibility(GV: I));
1794 Vals.push_back(Elt: I.isDSOLocal());
1795 Vals.push_back(Elt: addToStrtab(Str: I.getPartition()));
1796 Vals.push_back(Elt: I.getPartition().size());
1797 Stream.EmitRecord(Code: bitc::MODULE_CODE_IFUNC, Vals);
1798 Vals.clear();
1799 }
1800
1801 writeValueSymbolTableForwardDecl();
1802}
1803
1804static uint64_t getOptimizationFlags(const Value *V) {
1805 uint64_t Flags = 0;
1806
1807 if (const auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V)) {
1808 if (OBO->hasNoSignedWrap())
1809 Flags |= 1 << bitc::OBO_NO_SIGNED_WRAP;
1810 if (OBO->hasNoUnsignedWrap())
1811 Flags |= 1 << bitc::OBO_NO_UNSIGNED_WRAP;
1812 } else if (const auto *PEO = dyn_cast<PossiblyExactOperator>(Val: V)) {
1813 if (PEO->isExact())
1814 Flags |= 1 << bitc::PEO_EXACT;
1815 } else if (const auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: V)) {
1816 if (PDI->isDisjoint())
1817 Flags |= 1 << bitc::PDI_DISJOINT;
1818 } else if (const auto *FPMO = dyn_cast<FPMathOperator>(Val: V)) {
1819 if (FPMO->hasAllowReassoc())
1820 Flags |= bitc::AllowReassoc;
1821 if (FPMO->hasNoNaNs())
1822 Flags |= bitc::NoNaNs;
1823 if (FPMO->hasNoInfs())
1824 Flags |= bitc::NoInfs;
1825 if (FPMO->hasNoSignedZeros())
1826 Flags |= bitc::NoSignedZeros;
1827 if (FPMO->hasAllowReciprocal())
1828 Flags |= bitc::AllowReciprocal;
1829 if (FPMO->hasAllowContract())
1830 Flags |= bitc::AllowContract;
1831 if (FPMO->hasApproxFunc())
1832 Flags |= bitc::ApproxFunc;
1833
1834 // Handle uitofp.
1835 if (const auto *NNI = dyn_cast<PossiblyNonNegInst>(Val: V)) {
1836 Flags <<= 1;
1837 if (NNI->hasNonNeg())
1838 Flags |= 1 << bitc::PNNI_NON_NEG;
1839 }
1840 } else if (const auto *NNI = dyn_cast<PossiblyNonNegInst>(Val: V)) {
1841 if (NNI->hasNonNeg())
1842 Flags |= 1 << bitc::PNNI_NON_NEG;
1843 } else if (const auto *TI = dyn_cast<TruncInst>(Val: V)) {
1844 if (TI->hasNoSignedWrap())
1845 Flags |= 1 << bitc::TIO_NO_SIGNED_WRAP;
1846 if (TI->hasNoUnsignedWrap())
1847 Flags |= 1 << bitc::TIO_NO_UNSIGNED_WRAP;
1848 } else if (const auto *GEP = dyn_cast<GEPOperator>(Val: V)) {
1849 if (GEP->isInBounds())
1850 Flags |= 1 << bitc::GEP_INBOUNDS;
1851 if (GEP->hasNoUnsignedSignedWrap())
1852 Flags |= 1 << bitc::GEP_NUSW;
1853 if (GEP->hasNoUnsignedWrap())
1854 Flags |= 1 << bitc::GEP_NUW;
1855 } else if (const auto *ICmp = dyn_cast<ICmpInst>(Val: V)) {
1856 if (ICmp->hasSameSign())
1857 Flags |= 1 << bitc::ICMP_SAME_SIGN;
1858 }
1859
1860 return Flags;
1861}
1862
1863void ModuleBitcodeWriter::writeValueAsMetadata(
1864 const ValueAsMetadata *MD, SmallVectorImpl<uint64_t> &Record) {
1865 // Mimic an MDNode with a value as one operand.
1866 Value *V = MD->getValue();
1867 Record.push_back(Elt: VE.getTypeID(T: V->getType()));
1868 Record.push_back(Elt: VE.getValueID(V));
1869 Stream.EmitRecord(Code: bitc::METADATA_VALUE, Vals: Record, Abbrev: 0);
1870 Record.clear();
1871}
1872
1873void ModuleBitcodeWriter::writeMDTuple(const MDTuple *N,
1874 SmallVectorImpl<uint64_t> &Record,
1875 unsigned Abbrev) {
1876 for (const MDOperand &MDO : N->operands()) {
1877 Metadata *MD = MDO;
1878 assert(!(MD && isa<LocalAsMetadata>(MD)) &&
1879 "Unexpected function-local metadata");
1880 Record.push_back(Elt: VE.getMetadataOrNullID(MD));
1881 }
1882 Stream.EmitRecord(Code: N->isDistinct() ? bitc::METADATA_DISTINCT_NODE
1883 : bitc::METADATA_NODE,
1884 Vals: Record, Abbrev);
1885 Record.clear();
1886}
1887
1888unsigned ModuleBitcodeWriter::createDILocationAbbrev() {
1889 // Assume the column is usually under 128, and always output the inlined-at
1890 // location (it's never more expensive than building an array size 1).
1891 auto Abbv = std::make_shared<BitCodeAbbrev>();
1892 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_LOCATION));
1893 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isDistinct
1894 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // line
1895 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // column
1896 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // scope
1897 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // inlinedAt
1898 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isImplicitCode
1899 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // atomGroup
1900 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // atomRank
1901 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
1902}
1903
1904void ModuleBitcodeWriter::writeDILocation(const DILocation *N,
1905 SmallVectorImpl<uint64_t> &Record,
1906 unsigned &Abbrev) {
1907 if (!Abbrev)
1908 Abbrev = createDILocationAbbrev();
1909
1910 Record.push_back(Elt: N->isDistinct());
1911 Record.push_back(Elt: N->getLine());
1912 Record.push_back(Elt: N->getColumn());
1913 Record.push_back(Elt: VE.getMetadataID(MD: N->getScope()));
1914 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getInlinedAt()));
1915 Record.push_back(Elt: N->isImplicitCode());
1916 Record.push_back(Elt: N->getAtomGroup());
1917 Record.push_back(Elt: N->getAtomRank());
1918 Stream.EmitRecord(Code: bitc::METADATA_LOCATION, Vals: Record, Abbrev);
1919 Record.clear();
1920}
1921
1922unsigned ModuleBitcodeWriter::createGenericDINodeAbbrev() {
1923 // Assume the column is usually under 128, and always output the inlined-at
1924 // location (it's never more expensive than building an array size 1).
1925 auto Abbv = std::make_shared<BitCodeAbbrev>();
1926 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_GENERIC_DEBUG));
1927 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1928 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1929 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1930 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1931 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1932 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1933 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
1934}
1935
1936void ModuleBitcodeWriter::writeGenericDINode(const GenericDINode *N,
1937 SmallVectorImpl<uint64_t> &Record,
1938 unsigned &Abbrev) {
1939 if (!Abbrev)
1940 Abbrev = createGenericDINodeAbbrev();
1941
1942 Record.push_back(Elt: N->isDistinct());
1943 Record.push_back(Elt: N->getTag());
1944 Record.push_back(Elt: 0); // Per-tag version field; unused for now.
1945
1946 for (auto &I : N->operands())
1947 Record.push_back(Elt: VE.getMetadataOrNullID(MD: I));
1948
1949 Stream.EmitRecord(Code: bitc::METADATA_GENERIC_DEBUG, Vals: Record, Abbrev);
1950 Record.clear();
1951}
1952
1953void ModuleBitcodeWriter::writeDISubrange(const DISubrange *N,
1954 SmallVectorImpl<uint64_t> &Record,
1955 unsigned Abbrev) {
1956 const uint64_t Version = 2 << 1;
1957 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
1958 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawCountNode()));
1959 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
1960 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
1961 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
1962
1963 Stream.EmitRecord(Code: bitc::METADATA_SUBRANGE, Vals: Record, Abbrev);
1964 Record.clear();
1965}
1966
1967void ModuleBitcodeWriter::writeDIGenericSubrange(
1968 const DIGenericSubrange *N, SmallVectorImpl<uint64_t> &Record,
1969 unsigned Abbrev) {
1970 Record.push_back(Elt: (uint64_t)N->isDistinct());
1971 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawCountNode()));
1972 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
1973 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
1974 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
1975
1976 Stream.EmitRecord(Code: bitc::METADATA_GENERIC_SUBRANGE, Vals: Record, Abbrev);
1977 Record.clear();
1978}
1979
1980void ModuleBitcodeWriter::writeDIEnumerator(const DIEnumerator *N,
1981 SmallVectorImpl<uint64_t> &Record,
1982 unsigned Abbrev) {
1983 const uint64_t IsBigInt = 1 << 2;
1984 Record.push_back(Elt: IsBigInt | (N->isUnsigned() << 1) | N->isDistinct());
1985 Record.push_back(Elt: N->getValue().getBitWidth());
1986 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1987 emitWideAPInt(Vals&: Record, A: N->getValue());
1988
1989 Stream.EmitRecord(Code: bitc::METADATA_ENUMERATOR, Vals: Record, Abbrev);
1990 Record.clear();
1991}
1992
1993void ModuleBitcodeWriter::writeDIBasicType(const DIBasicType *N,
1994 SmallVectorImpl<uint64_t> &Record,
1995 unsigned Abbrev) {
1996 const unsigned SizeIsMetadata = 0x2;
1997 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
1998 Record.push_back(Elt: N->getTag());
1999 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2000 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2001 Record.push_back(Elt: N->getAlignInBits());
2002 Record.push_back(Elt: N->getEncoding());
2003 Record.push_back(Elt: N->getFlags());
2004 Record.push_back(Elt: N->getNumExtraInhabitants());
2005 Record.push_back(Elt: N->getDataSizeInBits());
2006 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2007 Record.push_back(Elt: N->getLine());
2008 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2009
2010 Stream.EmitRecord(Code: bitc::METADATA_BASIC_TYPE, Vals: Record, Abbrev);
2011 Record.clear();
2012}
2013
2014void ModuleBitcodeWriter::writeDIFixedPointType(
2015 const DIFixedPointType *N, SmallVectorImpl<uint64_t> &Record,
2016 unsigned Abbrev) {
2017 const unsigned SizeIsMetadata = 0x2;
2018 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2019 Record.push_back(Elt: N->getTag());
2020 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2021 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2022 Record.push_back(Elt: N->getAlignInBits());
2023 Record.push_back(Elt: N->getEncoding());
2024 Record.push_back(Elt: N->getFlags());
2025 Record.push_back(Elt: N->getKind());
2026 Record.push_back(Elt: N->getFactorRaw());
2027
2028 auto WriteWideInt = [&](const APInt &Value) {
2029 // Write an encoded word that holds the number of active words and
2030 // the number of bits.
2031 uint64_t NumWords = Value.getActiveWords();
2032 uint64_t Encoded = (NumWords << 32) | Value.getBitWidth();
2033 Record.push_back(Elt: Encoded);
2034 emitWideAPInt(Vals&: Record, A: Value);
2035 };
2036
2037 WriteWideInt(N->getNumeratorRaw());
2038 WriteWideInt(N->getDenominatorRaw());
2039
2040 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2041 Record.push_back(Elt: N->getLine());
2042 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2043
2044 Stream.EmitRecord(Code: bitc::METADATA_FIXED_POINT_TYPE, Vals: Record, Abbrev);
2045 Record.clear();
2046}
2047
2048void ModuleBitcodeWriter::writeDIStringType(const DIStringType *N,
2049 SmallVectorImpl<uint64_t> &Record,
2050 unsigned Abbrev) {
2051 const unsigned SizeIsMetadata = 0x2;
2052 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2053 Record.push_back(Elt: N->getTag());
2054 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2055 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLength()));
2056 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLengthExp()));
2057 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLocationExp()));
2058 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2059 Record.push_back(Elt: N->getAlignInBits());
2060 Record.push_back(Elt: N->getEncoding());
2061
2062 Stream.EmitRecord(Code: bitc::METADATA_STRING_TYPE, Vals: Record, Abbrev);
2063 Record.clear();
2064}
2065
2066void ModuleBitcodeWriter::writeDIDerivedType(const DIDerivedType *N,
2067 SmallVectorImpl<uint64_t> &Record,
2068 unsigned Abbrev) {
2069 const unsigned SizeIsMetadata = 0x2;
2070 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2071 Record.push_back(Elt: N->getTag());
2072 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2073 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2074 Record.push_back(Elt: N->getLine());
2075 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2076 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2077 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2078 Record.push_back(Elt: N->getAlignInBits());
2079 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawOffsetInBits()));
2080 Record.push_back(Elt: N->getFlags());
2081 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getExtraData()));
2082
2083 // DWARF address space is encoded as N->getDWARFAddressSpace() + 1. 0 means
2084 // that there is no DWARF address space associated with DIDerivedType.
2085 if (const auto &DWARFAddressSpace = N->getDWARFAddressSpace())
2086 Record.push_back(Elt: *DWARFAddressSpace + 1);
2087 else
2088 Record.push_back(Elt: 0);
2089
2090 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2091
2092 if (auto PtrAuthData = N->getPtrAuthData())
2093 Record.push_back(Elt: PtrAuthData->RawData);
2094 else
2095 Record.push_back(Elt: 0);
2096
2097 Stream.EmitRecord(Code: bitc::METADATA_DERIVED_TYPE, Vals: Record, Abbrev);
2098 Record.clear();
2099}
2100
2101void ModuleBitcodeWriter::writeDISubrangeType(const DISubrangeType *N,
2102 SmallVectorImpl<uint64_t> &Record,
2103 unsigned Abbrev) {
2104 const unsigned SizeIsMetadata = 0x2;
2105 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2106 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2107 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2108 Record.push_back(Elt: N->getLine());
2109 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2110 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2111 Record.push_back(Elt: N->getAlignInBits());
2112 Record.push_back(Elt: N->getFlags());
2113 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2114 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
2115 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
2116 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
2117 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawBias()));
2118
2119 Stream.EmitRecord(Code: bitc::METADATA_SUBRANGE_TYPE, Vals: Record, Abbrev);
2120 Record.clear();
2121}
2122
2123void ModuleBitcodeWriter::writeDICompositeType(
2124 const DICompositeType *N, SmallVectorImpl<uint64_t> &Record,
2125 unsigned Abbrev) {
2126 const unsigned IsNotUsedInOldTypeRef = 0x2;
2127 const unsigned SizeIsMetadata = 0x4;
2128 Record.push_back(Elt: SizeIsMetadata | IsNotUsedInOldTypeRef |
2129 (unsigned)N->isDistinct());
2130 Record.push_back(Elt: N->getTag());
2131 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2132 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2133 Record.push_back(Elt: N->getLine());
2134 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2135 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2136 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2137 Record.push_back(Elt: N->getAlignInBits());
2138 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawOffsetInBits()));
2139 Record.push_back(Elt: N->getFlags());
2140 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2141 Record.push_back(Elt: N->getRuntimeLang());
2142 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getVTableHolder()));
2143 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams().get()));
2144 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawIdentifier()));
2145 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDiscriminator()));
2146 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawDataLocation()));
2147 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawAssociated()));
2148 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawAllocated()));
2149 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawRank()));
2150 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2151 Record.push_back(Elt: N->getNumExtraInhabitants());
2152 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSpecification()));
2153 Record.push_back(
2154 Elt: N->getEnumKind().value_or(u: dwarf::DW_APPLE_ENUM_KIND_invalid));
2155 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawBitStride()));
2156
2157 Stream.EmitRecord(Code: bitc::METADATA_COMPOSITE_TYPE, Vals: Record, Abbrev);
2158 Record.clear();
2159}
2160
2161void ModuleBitcodeWriter::writeDISubroutineType(
2162 const DISubroutineType *N, SmallVectorImpl<uint64_t> &Record,
2163 unsigned Abbrev) {
2164 const unsigned HasNoOldTypeRefs = 0x2;
2165 Record.push_back(Elt: HasNoOldTypeRefs | (unsigned)N->isDistinct());
2166 Record.push_back(Elt: N->getFlags());
2167 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTypeArray().get()));
2168 Record.push_back(Elt: N->getCC());
2169
2170 Stream.EmitRecord(Code: bitc::METADATA_SUBROUTINE_TYPE, Vals: Record, Abbrev);
2171 Record.clear();
2172}
2173
2174void ModuleBitcodeWriter::writeDIFile(const DIFile *N,
2175 SmallVectorImpl<uint64_t> &Record,
2176 unsigned Abbrev) {
2177 Record.push_back(Elt: N->isDistinct());
2178 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFilename()));
2179 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawDirectory()));
2180 if (N->getRawChecksum()) {
2181 Record.push_back(Elt: N->getRawChecksum()->Kind);
2182 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawChecksum()->Value));
2183 } else {
2184 // Maintain backwards compatibility with the old internal representation of
2185 // CSK_None in ChecksumKind by writing nulls here when Checksum is None.
2186 Record.push_back(Elt: 0);
2187 Record.push_back(Elt: VE.getMetadataOrNullID(MD: nullptr));
2188 }
2189 auto Source = N->getRawSource();
2190 if (Source)
2191 Record.push_back(Elt: VE.getMetadataOrNullID(MD: Source));
2192
2193 Stream.EmitRecord(Code: bitc::METADATA_FILE, Vals: Record, Abbrev);
2194 Record.clear();
2195}
2196
2197void ModuleBitcodeWriter::writeDICompileUnit(const DICompileUnit *N,
2198 SmallVectorImpl<uint64_t> &Record,
2199 unsigned Abbrev) {
2200 assert(N->isDistinct() && "Expected distinct compile units");
2201 Record.push_back(/* IsDistinct */ Elt: true);
2202
2203 auto Lang = N->getSourceLanguage();
2204 Record.push_back(Elt: Lang.getName());
2205 // Set bit so the MetadataLoader can distniguish between versioned and
2206 // unversioned names.
2207 if (Lang.hasVersionedName())
2208 Record.back() ^= (uint64_t(1) << 63);
2209
2210 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2211 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawProducer()));
2212 Record.push_back(Elt: N->isOptimized());
2213 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFlags()));
2214 Record.push_back(Elt: N->getRuntimeVersion());
2215 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSplitDebugFilename()));
2216 Record.push_back(Elt: N->getEmissionKind());
2217 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getEnumTypes().get()));
2218 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRetainedTypes().get()));
2219 Record.push_back(/* subprograms */ Elt: 0);
2220 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getGlobalVariables().get()));
2221 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getImportedEntities().get()));
2222 Record.push_back(Elt: N->getDWOId());
2223 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getMacros().get()));
2224 Record.push_back(Elt: N->getSplitDebugInlining());
2225 Record.push_back(Elt: N->getDebugInfoForProfiling());
2226 Record.push_back(Elt: (unsigned)N->getNameTableKind());
2227 Record.push_back(Elt: N->getRangesBaseAddress());
2228 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSysRoot()));
2229 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSDK()));
2230 Record.push_back(Elt: Lang.hasVersionedName() ? Lang.getVersion() : 0);
2231 Record.push_back(Elt: Lang.getDialect());
2232
2233 Stream.EmitRecord(Code: bitc::METADATA_COMPILE_UNIT, Vals: Record, Abbrev);
2234 Record.clear();
2235}
2236
2237void ModuleBitcodeWriter::writeDISubprogram(const DISubprogram *N,
2238 SmallVectorImpl<uint64_t> &Record,
2239 unsigned Abbrev) {
2240 const uint64_t HasUnitFlag = 1 << 1;
2241 const uint64_t HasSPFlagsFlag = 1 << 2;
2242 Record.push_back(Elt: uint64_t(N->isDistinct()) | HasUnitFlag | HasSPFlagsFlag);
2243 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2244 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2245 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLinkageName()));
2246 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2247 Record.push_back(Elt: N->getLine());
2248 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2249 Record.push_back(Elt: N->getScopeLine());
2250 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getContainingType()));
2251 Record.push_back(Elt: N->getSPFlags());
2252 Record.push_back(Elt: N->getVirtualIndex());
2253 Record.push_back(Elt: N->getFlags());
2254 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUnit()));
2255 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams().get()));
2256 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDeclaration()));
2257 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRetainedNodes().get()));
2258 Record.push_back(Elt: N->getThisAdjustment());
2259 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getThrownTypes().get()));
2260 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2261 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawTargetFuncName()));
2262 Record.push_back(Elt: N->getKeyInstructionsEnabled());
2263
2264 Stream.EmitRecord(Code: bitc::METADATA_SUBPROGRAM, Vals: Record, Abbrev);
2265 Record.clear();
2266}
2267
2268void ModuleBitcodeWriter::writeDILexicalBlock(const DILexicalBlock *N,
2269 SmallVectorImpl<uint64_t> &Record,
2270 unsigned Abbrev) {
2271 Record.push_back(Elt: N->isDistinct());
2272 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2273 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2274 Record.push_back(Elt: N->getLine());
2275 Record.push_back(Elt: N->getColumn());
2276
2277 Stream.EmitRecord(Code: bitc::METADATA_LEXICAL_BLOCK, Vals: Record, Abbrev);
2278 Record.clear();
2279}
2280
2281void ModuleBitcodeWriter::writeDILexicalBlockFile(
2282 const DILexicalBlockFile *N, SmallVectorImpl<uint64_t> &Record,
2283 unsigned Abbrev) {
2284 Record.push_back(Elt: N->isDistinct());
2285 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2286 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2287 Record.push_back(Elt: N->getDiscriminator());
2288
2289 Stream.EmitRecord(Code: bitc::METADATA_LEXICAL_BLOCK_FILE, Vals: Record, Abbrev);
2290 Record.clear();
2291}
2292
2293void ModuleBitcodeWriter::writeDICommonBlock(const DICommonBlock *N,
2294 SmallVectorImpl<uint64_t> &Record,
2295 unsigned Abbrev) {
2296 Record.push_back(Elt: N->isDistinct());
2297 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2298 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDecl()));
2299 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2300 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2301 Record.push_back(Elt: N->getLineNo());
2302
2303 Stream.EmitRecord(Code: bitc::METADATA_COMMON_BLOCK, Vals: Record, Abbrev);
2304 Record.clear();
2305}
2306
2307void ModuleBitcodeWriter::writeDINamespace(const DINamespace *N,
2308 SmallVectorImpl<uint64_t> &Record,
2309 unsigned Abbrev) {
2310 Record.push_back(Elt: N->isDistinct() | N->getExportSymbols() << 1);
2311 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2312 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2313
2314 Stream.EmitRecord(Code: bitc::METADATA_NAMESPACE, Vals: Record, Abbrev);
2315 Record.clear();
2316}
2317
2318void ModuleBitcodeWriter::writeDIMacro(const DIMacro *N,
2319 SmallVectorImpl<uint64_t> &Record,
2320 unsigned Abbrev) {
2321 Record.push_back(Elt: N->isDistinct());
2322 Record.push_back(Elt: N->getMacinfoType());
2323 Record.push_back(Elt: N->getLine());
2324 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2325 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawValue()));
2326
2327 Stream.EmitRecord(Code: bitc::METADATA_MACRO, Vals: Record, Abbrev);
2328 Record.clear();
2329}
2330
2331void ModuleBitcodeWriter::writeDIMacroFile(const DIMacroFile *N,
2332 SmallVectorImpl<uint64_t> &Record,
2333 unsigned Abbrev) {
2334 Record.push_back(Elt: N->isDistinct());
2335 Record.push_back(Elt: N->getMacinfoType());
2336 Record.push_back(Elt: N->getLine());
2337 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2338 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2339
2340 Stream.EmitRecord(Code: bitc::METADATA_MACRO_FILE, Vals: Record, Abbrev);
2341 Record.clear();
2342}
2343
2344void ModuleBitcodeWriter::writeDIArgList(const DIArgList *N,
2345 SmallVectorImpl<uint64_t> &Record) {
2346 Record.reserve(N: N->getArgs().size());
2347 for (ValueAsMetadata *MD : N->getArgs())
2348 Record.push_back(Elt: VE.getMetadataID(MD));
2349
2350 Stream.EmitRecord(Code: bitc::METADATA_ARG_LIST, Vals: Record);
2351 Record.clear();
2352}
2353
2354void ModuleBitcodeWriter::writeDIModule(const DIModule *N,
2355 SmallVectorImpl<uint64_t> &Record,
2356 unsigned Abbrev) {
2357 Record.push_back(Elt: N->isDistinct());
2358 for (auto &I : N->operands())
2359 Record.push_back(Elt: VE.getMetadataOrNullID(MD: I));
2360 Record.push_back(Elt: N->getLineNo());
2361 Record.push_back(Elt: N->getIsDecl());
2362
2363 Stream.EmitRecord(Code: bitc::METADATA_MODULE, Vals: Record, Abbrev);
2364 Record.clear();
2365}
2366
2367void ModuleBitcodeWriter::writeDIAssignID(const DIAssignID *N,
2368 SmallVectorImpl<uint64_t> &Record,
2369 unsigned Abbrev) {
2370 // There are no arguments for this metadata type.
2371 Record.push_back(Elt: N->isDistinct());
2372 Stream.EmitRecord(Code: bitc::METADATA_ASSIGN_ID, Vals: Record, Abbrev);
2373 Record.clear();
2374}
2375
2376void ModuleBitcodeWriter::writeDITemplateTypeParameter(
2377 const DITemplateTypeParameter *N, SmallVectorImpl<uint64_t> &Record,
2378 unsigned Abbrev) {
2379 Record.push_back(Elt: N->isDistinct());
2380 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2381 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2382 Record.push_back(Elt: N->isDefault());
2383
2384 Stream.EmitRecord(Code: bitc::METADATA_TEMPLATE_TYPE, Vals: Record, Abbrev);
2385 Record.clear();
2386}
2387
2388void ModuleBitcodeWriter::writeDITemplateValueParameter(
2389 const DITemplateValueParameter *N, SmallVectorImpl<uint64_t> &Record,
2390 unsigned Abbrev) {
2391 Record.push_back(Elt: N->isDistinct());
2392 Record.push_back(Elt: N->getTag());
2393 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2394 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2395 Record.push_back(Elt: N->isDefault());
2396 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getValue()));
2397
2398 Stream.EmitRecord(Code: bitc::METADATA_TEMPLATE_VALUE, Vals: Record, Abbrev);
2399 Record.clear();
2400}
2401
2402void ModuleBitcodeWriter::writeDIGlobalVariable(
2403 const DIGlobalVariable *N, SmallVectorImpl<uint64_t> &Record,
2404 unsigned Abbrev) {
2405 const uint64_t Version = 2 << 1;
2406 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
2407 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2408 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2409 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLinkageName()));
2410 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2411 Record.push_back(Elt: N->getLine());
2412 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2413 Record.push_back(Elt: N->isLocalToUnit());
2414 Record.push_back(Elt: N->isDefinition());
2415 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStaticDataMemberDeclaration()));
2416 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams()));
2417 Record.push_back(Elt: N->getAlignInBits());
2418 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2419
2420 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_VAR, Vals: Record, Abbrev);
2421 Record.clear();
2422}
2423
2424void ModuleBitcodeWriter::writeDILocalVariable(
2425 const DILocalVariable *N, SmallVectorImpl<uint64_t> &Record,
2426 unsigned Abbrev) {
2427 // In order to support all possible bitcode formats in BitcodeReader we need
2428 // to distinguish the following cases:
2429 // 1) Record has no artificial tag (Record[1]),
2430 // has no obsolete inlinedAt field (Record[9]).
2431 // In this case Record size will be 8, HasAlignment flag is false.
2432 // 2) Record has artificial tag (Record[1]),
2433 // has no obsolete inlignedAt field (Record[9]).
2434 // In this case Record size will be 9, HasAlignment flag is false.
2435 // 3) Record has both artificial tag (Record[1]) and
2436 // obsolete inlignedAt field (Record[9]).
2437 // In this case Record size will be 10, HasAlignment flag is false.
2438 // 4) Record has neither artificial tag, nor inlignedAt field, but
2439 // HasAlignment flag is true and Record[8] contains alignment value.
2440 const uint64_t HasAlignmentFlag = 1 << 1;
2441 Record.push_back(Elt: (uint64_t)N->isDistinct() | HasAlignmentFlag);
2442 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2443 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2444 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2445 Record.push_back(Elt: N->getLine());
2446 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2447 Record.push_back(Elt: N->getArg());
2448 Record.push_back(Elt: N->getFlags());
2449 Record.push_back(Elt: N->getAlignInBits());
2450 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2451
2452 Stream.EmitRecord(Code: bitc::METADATA_LOCAL_VAR, Vals: Record, Abbrev);
2453 Record.clear();
2454}
2455
2456void ModuleBitcodeWriter::writeDILabel(
2457 const DILabel *N, SmallVectorImpl<uint64_t> &Record,
2458 unsigned Abbrev) {
2459 uint64_t IsArtificialFlag = uint64_t(N->isArtificial()) << 1;
2460 Record.push_back(Elt: (uint64_t)N->isDistinct() | IsArtificialFlag);
2461 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2462 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2463 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2464 Record.push_back(Elt: N->getLine());
2465 Record.push_back(Elt: N->getColumn());
2466 Record.push_back(Elt: N->getCoroSuspendIdx().has_value()
2467 ? (uint64_t)N->getCoroSuspendIdx().value()
2468 : std::numeric_limits<uint64_t>::max());
2469
2470 Stream.EmitRecord(Code: bitc::METADATA_LABEL, Vals: Record, Abbrev);
2471 Record.clear();
2472}
2473
2474void ModuleBitcodeWriter::writeDIExpression(const DIExpression *N,
2475 SmallVectorImpl<uint64_t> &Record,
2476 unsigned Abbrev) {
2477 Record.reserve(N: N->getElements().size() + 1);
2478 const uint64_t Version = 3 << 1;
2479 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
2480 Record.append(in_start: N->elements_begin(), in_end: N->elements_end());
2481
2482 Stream.EmitRecord(Code: bitc::METADATA_EXPRESSION, Vals: Record, Abbrev);
2483 Record.clear();
2484}
2485
2486void ModuleBitcodeWriter::writeDIGlobalVariableExpression(
2487 const DIGlobalVariableExpression *N, SmallVectorImpl<uint64_t> &Record,
2488 unsigned Abbrev) {
2489 Record.push_back(Elt: N->isDistinct());
2490 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getVariable()));
2491 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getExpression()));
2492
2493 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_VAR_EXPR, Vals: Record, Abbrev);
2494 Record.clear();
2495}
2496
2497void ModuleBitcodeWriter::writeDIObjCProperty(const DIObjCProperty *N,
2498 SmallVectorImpl<uint64_t> &Record,
2499 unsigned Abbrev) {
2500 Record.push_back(Elt: N->isDistinct());
2501 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2502 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2503 Record.push_back(Elt: N->getLine());
2504 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSetterName()));
2505 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawGetterName()));
2506 Record.push_back(Elt: N->getAttributes());
2507 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2508
2509 Stream.EmitRecord(Code: bitc::METADATA_OBJC_PROPERTY, Vals: Record, Abbrev);
2510 Record.clear();
2511}
2512
2513void ModuleBitcodeWriter::writeDIImportedEntity(
2514 const DIImportedEntity *N, SmallVectorImpl<uint64_t> &Record,
2515 unsigned Abbrev) {
2516 Record.push_back(Elt: N->isDistinct());
2517 Record.push_back(Elt: N->getTag());
2518 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2519 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getEntity()));
2520 Record.push_back(Elt: N->getLine());
2521 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2522 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFile()));
2523 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2524
2525 Stream.EmitRecord(Code: bitc::METADATA_IMPORTED_ENTITY, Vals: Record, Abbrev);
2526 Record.clear();
2527}
2528
2529unsigned ModuleBitcodeWriter::createNamedMetadataAbbrev() {
2530 auto Abbv = std::make_shared<BitCodeAbbrev>();
2531 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_NAME));
2532 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2533 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2534 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
2535}
2536
2537void ModuleBitcodeWriter::writeNamedMetadata(
2538 SmallVectorImpl<uint64_t> &Record) {
2539 if (M.named_metadata_empty())
2540 return;
2541
2542 unsigned Abbrev = createNamedMetadataAbbrev();
2543 for (const NamedMDNode &NMD : M.named_metadata()) {
2544 // Write name.
2545 StringRef Str = NMD.getName();
2546 Record.append(in_start: Str.bytes_begin(), in_end: Str.bytes_end());
2547 Stream.EmitRecord(Code: bitc::METADATA_NAME, Vals: Record, Abbrev);
2548 Record.clear();
2549
2550 // Write named metadata operands.
2551 for (const MDNode *N : NMD.operands())
2552 Record.push_back(Elt: VE.getMetadataID(MD: N));
2553 Stream.EmitRecord(Code: bitc::METADATA_NAMED_NODE, Vals: Record, Abbrev: 0);
2554 Record.clear();
2555 }
2556}
2557
2558unsigned ModuleBitcodeWriter::createMetadataStringsAbbrev() {
2559 auto Abbv = std::make_shared<BitCodeAbbrev>();
2560 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_STRINGS));
2561 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // # of strings
2562 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // offset to chars
2563 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
2564 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
2565}
2566
2567/// Write out a record for MDString.
2568///
2569/// All the metadata strings in a metadata block are emitted in a single
2570/// record. The sizes and strings themselves are shoved into a blob.
2571void ModuleBitcodeWriter::writeMetadataStrings(
2572 ArrayRef<const Metadata *> Strings, SmallVectorImpl<uint64_t> &Record) {
2573 if (Strings.empty())
2574 return;
2575
2576 // Start the record with the number of strings.
2577 Record.push_back(Elt: bitc::METADATA_STRINGS);
2578 Record.push_back(Elt: Strings.size());
2579
2580 // Emit the sizes of the strings in the blob.
2581 SmallString<256> Blob;
2582 {
2583 BitstreamWriter W(Blob);
2584 for (const Metadata *MD : Strings)
2585 W.EmitVBR(Val: cast<MDString>(Val: MD)->getLength(), NumBits: 6);
2586 W.FlushToWord();
2587 }
2588
2589 // Add the offset to the strings to the record.
2590 Record.push_back(Elt: Blob.size());
2591
2592 // Add the strings to the blob.
2593 for (const Metadata *MD : Strings)
2594 Blob.append(RHS: cast<MDString>(Val: MD)->getString());
2595
2596 // Emit the final record.
2597 Stream.EmitRecordWithBlob(Abbrev: createMetadataStringsAbbrev(), Vals: Record, Blob);
2598 Record.clear();
2599}
2600
2601// Generates an enum to use as an index in the Abbrev array of Metadata record.
2602enum MetadataAbbrev : unsigned {
2603#define HANDLE_MDNODE_LEAF(CLASS) CLASS##AbbrevID,
2604#include "llvm/IR/Metadata.def"
2605 LastPlusOne
2606};
2607
2608void ModuleBitcodeWriter::writeMetadataRecords(
2609 ArrayRef<const Metadata *> MDs, SmallVectorImpl<uint64_t> &Record,
2610 std::vector<unsigned> *MDAbbrevs, std::vector<uint64_t> *IndexPos) {
2611 if (MDs.empty())
2612 return;
2613
2614 // Initialize MDNode abbreviations.
2615#define HANDLE_MDNODE_LEAF(CLASS) unsigned CLASS##Abbrev = 0;
2616#include "llvm/IR/Metadata.def"
2617
2618 for (const Metadata *MD : MDs) {
2619 if (IndexPos)
2620 IndexPos->push_back(x: Stream.GetCurrentBitNo());
2621 if (const MDNode *N = dyn_cast<MDNode>(Val: MD)) {
2622 assert(N->isResolved() && "Expected forward references to be resolved");
2623
2624 switch (N->getMetadataID()) {
2625 default:
2626 llvm_unreachable("Invalid MDNode subclass");
2627#define HANDLE_MDNODE_LEAF(CLASS) \
2628 case Metadata::CLASS##Kind: \
2629 if (MDAbbrevs) \
2630 write##CLASS(cast<CLASS>(N), Record, \
2631 (*MDAbbrevs)[MetadataAbbrev::CLASS##AbbrevID]); \
2632 else \
2633 write##CLASS(cast<CLASS>(N), Record, CLASS##Abbrev); \
2634 continue;
2635#include "llvm/IR/Metadata.def"
2636 }
2637 }
2638 if (auto *AL = dyn_cast<DIArgList>(Val: MD)) {
2639 writeDIArgList(N: AL, Record);
2640 continue;
2641 }
2642 writeValueAsMetadata(MD: cast<ValueAsMetadata>(Val: MD), Record);
2643 }
2644}
2645
2646void ModuleBitcodeWriter::writeModuleMetadata() {
2647 if (!VE.hasMDs() && M.named_metadata_empty())
2648 return;
2649
2650 Stream.EnterSubblock(BlockID: bitc::METADATA_BLOCK_ID, CodeLen: 4);
2651 SmallVector<uint64_t, 64> Record;
2652
2653 // Emit all abbrevs upfront, so that the reader can jump in the middle of the
2654 // block and load any metadata.
2655 std::vector<unsigned> MDAbbrevs;
2656
2657 MDAbbrevs.resize(new_size: MetadataAbbrev::LastPlusOne);
2658 MDAbbrevs[MetadataAbbrev::DILocationAbbrevID] = createDILocationAbbrev();
2659 MDAbbrevs[MetadataAbbrev::GenericDINodeAbbrevID] =
2660 createGenericDINodeAbbrev();
2661
2662 auto Abbv = std::make_shared<BitCodeAbbrev>();
2663 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_INDEX_OFFSET));
2664 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2665 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2666 unsigned OffsetAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2667
2668 Abbv = std::make_shared<BitCodeAbbrev>();
2669 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_INDEX));
2670 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2671 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
2672 unsigned IndexAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2673
2674 // Emit MDStrings together upfront.
2675 writeMetadataStrings(Strings: VE.getMDStrings(), Record);
2676
2677 // We only emit an index for the metadata record if we have more than a given
2678 // (naive) threshold of metadatas, otherwise it is not worth it.
2679 if (VE.getNonMDStrings().size() > IndexThreshold) {
2680 // Write a placeholder value in for the offset of the metadata index,
2681 // which is written after the records, so that it can include
2682 // the offset of each entry. The placeholder offset will be
2683 // updated after all records are emitted.
2684 uint64_t Vals[] = {0, 0};
2685 Stream.EmitRecord(Code: bitc::METADATA_INDEX_OFFSET, Vals, Abbrev: OffsetAbbrev);
2686 }
2687
2688 // Compute and save the bit offset to the current position, which will be
2689 // patched when we emit the index later. We can simply subtract the 64-bit
2690 // fixed size from the current bit number to get the location to backpatch.
2691 uint64_t IndexOffsetRecordBitPos = Stream.GetCurrentBitNo();
2692
2693 // This index will contain the bitpos for each individual record.
2694 std::vector<uint64_t> IndexPos;
2695 IndexPos.reserve(n: VE.getNonMDStrings().size());
2696
2697 // Write all the records
2698 writeMetadataRecords(MDs: VE.getNonMDStrings(), Record, MDAbbrevs: &MDAbbrevs, IndexPos: &IndexPos);
2699
2700 if (VE.getNonMDStrings().size() > IndexThreshold) {
2701 // Now that we have emitted all the records we will emit the index. But
2702 // first
2703 // backpatch the forward reference so that the reader can skip the records
2704 // efficiently.
2705 Stream.BackpatchWord64(BitNo: IndexOffsetRecordBitPos - 64,
2706 Val: Stream.GetCurrentBitNo() - IndexOffsetRecordBitPos);
2707
2708 // Delta encode the index.
2709 uint64_t PreviousValue = IndexOffsetRecordBitPos;
2710 for (auto &Elt : IndexPos) {
2711 auto EltDelta = Elt - PreviousValue;
2712 PreviousValue = Elt;
2713 Elt = EltDelta;
2714 }
2715 // Emit the index record.
2716 Stream.EmitRecord(Code: bitc::METADATA_INDEX, Vals: IndexPos, Abbrev: IndexAbbrev);
2717 IndexPos.clear();
2718 }
2719
2720 // Write the named metadata now.
2721 writeNamedMetadata(Record);
2722
2723 auto AddDeclAttachedMetadata = [&](const GlobalObject &GO) {
2724 SmallVector<uint64_t, 4> Record;
2725 Record.push_back(Elt: VE.getValueID(V: &GO));
2726 pushGlobalMetadataAttachment(Record, GO);
2727 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_DECL_ATTACHMENT, Vals: Record);
2728 };
2729 for (const Function &F : M)
2730 if (F.isDeclaration() && F.hasMetadata())
2731 AddDeclAttachedMetadata(F);
2732 for (const GlobalIFunc &GI : M.ifuncs())
2733 if (GI.hasMetadata())
2734 AddDeclAttachedMetadata(GI);
2735 // FIXME: Only store metadata for declarations here, and move data for global
2736 // variable definitions to a separate block (PR28134).
2737 for (const GlobalVariable &GV : M.globals())
2738 if (GV.hasMetadata())
2739 AddDeclAttachedMetadata(GV);
2740
2741 Stream.ExitBlock();
2742}
2743
2744void ModuleBitcodeWriter::writeFunctionMetadata(const Function &F) {
2745 if (!VE.hasMDs())
2746 return;
2747
2748 Stream.EnterSubblock(BlockID: bitc::METADATA_BLOCK_ID, CodeLen: 3);
2749 SmallVector<uint64_t, 64> Record;
2750 writeMetadataStrings(Strings: VE.getMDStrings(), Record);
2751 writeMetadataRecords(MDs: VE.getNonMDStrings(), Record);
2752 Stream.ExitBlock();
2753}
2754
2755void ModuleBitcodeWriter::pushGlobalMetadataAttachment(
2756 SmallVectorImpl<uint64_t> &Record, const GlobalObject &GO) {
2757 // [n x [id, mdnode]]
2758 SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2759 GO.getAllMetadata(MDs);
2760 for (const auto &I : MDs) {
2761 Record.push_back(Elt: I.first);
2762 Record.push_back(Elt: VE.getMetadataID(MD: I.second));
2763 }
2764}
2765
2766void ModuleBitcodeWriter::writeFunctionMetadataAttachment(const Function &F) {
2767 Stream.EnterSubblock(BlockID: bitc::METADATA_ATTACHMENT_ID, CodeLen: 3);
2768
2769 SmallVector<uint64_t, 64> Record;
2770
2771 if (F.hasMetadata()) {
2772 pushGlobalMetadataAttachment(Record, GO: F);
2773 Stream.EmitRecord(Code: bitc::METADATA_ATTACHMENT, Vals: Record, Abbrev: 0);
2774 Record.clear();
2775 }
2776
2777 // Write metadata attachments
2778 // METADATA_ATTACHMENT - [m x [value, [n x [id, mdnode]]]
2779 SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2780 for (const BasicBlock &BB : F)
2781 for (const Instruction &I : BB) {
2782 MDs.clear();
2783 I.getAllMetadataOtherThanDebugLoc(MDs);
2784
2785 // If no metadata, ignore instruction.
2786 if (MDs.empty()) continue;
2787
2788 Record.push_back(Elt: VE.getInstructionID(I: &I));
2789
2790 for (const auto &[ID, MD] : MDs) {
2791 Record.push_back(Elt: ID);
2792 Record.push_back(Elt: VE.getMetadataID(MD));
2793 }
2794 Stream.EmitRecord(Code: bitc::METADATA_ATTACHMENT, Vals: Record, Abbrev: 0);
2795 Record.clear();
2796 }
2797
2798 Stream.ExitBlock();
2799}
2800
2801void ModuleBitcodeWriter::writeModuleMetadataKinds() {
2802 SmallVector<uint64_t, 64> Record;
2803
2804 // Write metadata kinds
2805 // METADATA_KIND - [n x [id, name]]
2806 SmallVector<StringRef, 8> Names;
2807 M.getMDKindNames(Result&: Names);
2808
2809 if (Names.empty()) return;
2810
2811 Stream.EnterSubblock(BlockID: bitc::METADATA_KIND_BLOCK_ID, CodeLen: 3);
2812
2813 for (unsigned MDKindID = 0, e = Names.size(); MDKindID != e; ++MDKindID) {
2814 Record.push_back(Elt: MDKindID);
2815 StringRef KName = Names[MDKindID];
2816 Record.append(in_start: KName.begin(), in_end: KName.end());
2817
2818 Stream.EmitRecord(Code: bitc::METADATA_KIND, Vals: Record, Abbrev: 0);
2819 Record.clear();
2820 }
2821
2822 Stream.ExitBlock();
2823}
2824
2825void ModuleBitcodeWriter::writeOperandBundleTags() {
2826 // Write metadata kinds
2827 //
2828 // OPERAND_BUNDLE_TAGS_BLOCK_ID : N x OPERAND_BUNDLE_TAG
2829 //
2830 // OPERAND_BUNDLE_TAG - [strchr x N]
2831
2832 SmallVector<StringRef, 8> Tags;
2833 M.getOperandBundleTags(Result&: Tags);
2834
2835 if (Tags.empty())
2836 return;
2837
2838 Stream.EnterSubblock(BlockID: bitc::OPERAND_BUNDLE_TAGS_BLOCK_ID, CodeLen: 3);
2839
2840 SmallVector<uint64_t, 64> Record;
2841
2842 for (auto Tag : Tags) {
2843 Record.append(in_start: Tag.begin(), in_end: Tag.end());
2844
2845 Stream.EmitRecord(Code: bitc::OPERAND_BUNDLE_TAG, Vals: Record, Abbrev: 0);
2846 Record.clear();
2847 }
2848
2849 Stream.ExitBlock();
2850}
2851
2852void ModuleBitcodeWriter::writeSyncScopeNames() {
2853 SmallVector<StringRef, 8> SSNs;
2854 M.getContext().getSyncScopeNames(SSNs);
2855 if (SSNs.empty())
2856 return;
2857
2858 Stream.EnterSubblock(BlockID: bitc::SYNC_SCOPE_NAMES_BLOCK_ID, CodeLen: 2);
2859
2860 SmallVector<uint64_t, 64> Record;
2861 for (auto SSN : SSNs) {
2862 Record.append(in_start: SSN.begin(), in_end: SSN.end());
2863 Stream.EmitRecord(Code: bitc::SYNC_SCOPE_NAME, Vals: Record, Abbrev: 0);
2864 Record.clear();
2865 }
2866
2867 Stream.ExitBlock();
2868}
2869
2870void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal,
2871 bool isGlobal) {
2872 if (FirstVal == LastVal) return;
2873
2874 Stream.EnterSubblock(BlockID: bitc::CONSTANTS_BLOCK_ID, CodeLen: 4);
2875
2876 unsigned AggregateAbbrev = 0;
2877 unsigned String8Abbrev = 0;
2878 unsigned CString7Abbrev = 0;
2879 unsigned CString6Abbrev = 0;
2880 // If this is a constant pool for the module, emit module-specific abbrevs.
2881 if (isGlobal) {
2882 // Abbrev for CST_CODE_AGGREGATE.
2883 auto Abbv = std::make_shared<BitCodeAbbrev>();
2884 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_AGGREGATE));
2885 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2886 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(Value: LastVal+1)));
2887 AggregateAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2888
2889 // Abbrev for CST_CODE_STRING.
2890 Abbv = std::make_shared<BitCodeAbbrev>();
2891 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_STRING));
2892 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2893 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2894 String8Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2895 // Abbrev for CST_CODE_CSTRING.
2896 Abbv = std::make_shared<BitCodeAbbrev>();
2897 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2898 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2899 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
2900 CString7Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2901 // Abbrev for CST_CODE_CSTRING.
2902 Abbv = std::make_shared<BitCodeAbbrev>();
2903 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2904 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2905 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
2906 CString6Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2907 }
2908
2909 SmallVector<uint64_t, 64> Record;
2910
2911 const ValueEnumerator::ValueList &Vals = VE.getValues();
2912 Type *LastTy = nullptr;
2913 for (unsigned i = FirstVal; i != LastVal; ++i) {
2914 const Value *V = Vals[i].first;
2915 // If we need to switch types, do so now.
2916 if (V->getType() != LastTy) {
2917 LastTy = V->getType();
2918 Record.push_back(Elt: VE.getTypeID(T: LastTy));
2919 Stream.EmitRecord(Code: bitc::CST_CODE_SETTYPE, Vals: Record,
2920 Abbrev: CONSTANTS_SETTYPE_ABBREV);
2921 Record.clear();
2922 }
2923
2924 if (const InlineAsm *IA = dyn_cast<InlineAsm>(Val: V)) {
2925 Record.push_back(Elt: VE.getTypeID(T: IA->getFunctionType()));
2926 Record.push_back(
2927 Elt: unsigned(IA->hasSideEffects()) | unsigned(IA->isAlignStack()) << 1 |
2928 unsigned(IA->getDialect() & 1) << 2 | unsigned(IA->canThrow()) << 3);
2929
2930 // Add the asm string.
2931 StringRef AsmStr = IA->getAsmString();
2932 Record.push_back(Elt: AsmStr.size());
2933 Record.append(in_start: AsmStr.begin(), in_end: AsmStr.end());
2934
2935 // Add the constraint string.
2936 StringRef ConstraintStr = IA->getConstraintString();
2937 Record.push_back(Elt: ConstraintStr.size());
2938 Record.append(in_start: ConstraintStr.begin(), in_end: ConstraintStr.end());
2939 Stream.EmitRecord(Code: bitc::CST_CODE_INLINEASM, Vals: Record);
2940 Record.clear();
2941 continue;
2942 }
2943 const Constant *C = cast<Constant>(Val: V);
2944 unsigned Code = -1U;
2945 unsigned AbbrevToUse = 0;
2946 if (C->isNullValue()) {
2947 Code = bitc::CST_CODE_NULL;
2948 } else if (isa<PoisonValue>(Val: C)) {
2949 Code = bitc::CST_CODE_POISON;
2950 } else if (isa<UndefValue>(Val: C)) {
2951 Code = bitc::CST_CODE_UNDEF;
2952 } else if (const ConstantInt *IV = dyn_cast<ConstantInt>(Val: C)) {
2953 if (IV->getBitWidth() <= 64) {
2954 uint64_t V = IV->getSExtValue();
2955 emitSignedInt64(Vals&: Record, V);
2956 Code = bitc::CST_CODE_INTEGER;
2957 AbbrevToUse = CONSTANTS_INTEGER_ABBREV;
2958 } else { // Wide integers, > 64 bits in size.
2959 emitWideAPInt(Vals&: Record, A: IV->getValue());
2960 Code = bitc::CST_CODE_WIDE_INTEGER;
2961 }
2962 } else if (const ConstantByte *BV = dyn_cast<ConstantByte>(Val: C)) {
2963 if (BV->getBitWidth() <= 64) {
2964 uint64_t V = BV->getSExtValue();
2965 emitSignedInt64(Vals&: Record, V);
2966 Code = bitc::CST_CODE_BYTE;
2967 AbbrevToUse = CONSTANTS_BYTE_ABBREV;
2968 } else { // Wide bytes, > 64 bits in size.
2969 emitWideAPInt(Vals&: Record, A: BV->getValue());
2970 Code = bitc::CST_CODE_WIDE_BYTE;
2971 }
2972 } else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
2973 Code = bitc::CST_CODE_FLOAT;
2974 Type *Ty = CFP->getType()->getScalarType();
2975 if (Ty->isHalfTy() || Ty->isBFloatTy() || Ty->isFloatTy() ||
2976 Ty->isDoubleTy()) {
2977 Record.push_back(Elt: CFP->getValueAPF().bitcastToAPInt().getZExtValue());
2978 } else if (Ty->isX86_FP80Ty()) {
2979 // api needed to prevent premature destruction
2980 // bits are not in the same order as a normal i80 APInt, compensate.
2981 APInt api = CFP->getValueAPF().bitcastToAPInt();
2982 const uint64_t *p = api.getRawData();
2983 Record.push_back(Elt: (p[1] << 48) | (p[0] >> 16));
2984 Record.push_back(Elt: p[0] & 0xffffLL);
2985 } else if (Ty->isFP128Ty() || Ty->isPPC_FP128Ty()) {
2986 APInt api = CFP->getValueAPF().bitcastToAPInt();
2987 const uint64_t *p = api.getRawData();
2988 Record.push_back(Elt: p[0]);
2989 Record.push_back(Elt: p[1]);
2990 } else {
2991 assert(0 && "Unknown FP type!");
2992 }
2993 } else if (isa<ConstantDataSequential>(Val: C) &&
2994 cast<ConstantDataSequential>(Val: C)->isString()) {
2995 const ConstantDataSequential *Str = cast<ConstantDataSequential>(Val: C);
2996 // Emit constant strings specially.
2997 uint64_t NumElts = Str->getNumElements();
2998 // If this is a null-terminated string, use the denser CSTRING encoding.
2999 if (Str->isCString()) {
3000 Code = bitc::CST_CODE_CSTRING;
3001 --NumElts; // Don't encode the null, which isn't allowed by char6.
3002 } else {
3003 Code = bitc::CST_CODE_STRING;
3004 AbbrevToUse = String8Abbrev;
3005 }
3006 bool isCStr7 = Code == bitc::CST_CODE_CSTRING;
3007 bool isCStrChar6 = Code == bitc::CST_CODE_CSTRING;
3008 for (uint64_t i = 0; i != NumElts; ++i) {
3009 unsigned char V = Str->getElementAsInteger(i);
3010 Record.push_back(Elt: V);
3011 isCStr7 &= (V & 128) == 0;
3012 if (isCStrChar6)
3013 isCStrChar6 = BitCodeAbbrevOp::isChar6(C: V);
3014 }
3015
3016 if (isCStrChar6)
3017 AbbrevToUse = CString6Abbrev;
3018 else if (isCStr7)
3019 AbbrevToUse = CString7Abbrev;
3020 } else if (const ConstantDataSequential *CDS =
3021 dyn_cast<ConstantDataSequential>(Val: C)) {
3022 Code = bitc::CST_CODE_DATA;
3023 Type *EltTy = CDS->getElementType();
3024 if (isa<IntegerType>(Val: EltTy) || isa<ByteType>(Val: EltTy)) {
3025 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i)
3026 Record.push_back(Elt: CDS->getElementAsInteger(i));
3027 } else {
3028 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i)
3029 Record.push_back(
3030 Elt: CDS->getElementAsAPFloat(i).bitcastToAPInt().getLimitedValue());
3031 }
3032 } else if (isa<ConstantAggregate>(Val: C)) {
3033 Code = bitc::CST_CODE_AGGREGATE;
3034 for (const Value *Op : C->operands())
3035 Record.push_back(Elt: VE.getValueID(V: Op));
3036 AbbrevToUse = AggregateAbbrev;
3037 } else if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: C)) {
3038 switch (CE->getOpcode()) {
3039 default:
3040 if (Instruction::isCast(Opcode: CE->getOpcode())) {
3041 Code = bitc::CST_CODE_CE_CAST;
3042 Record.push_back(Elt: getEncodedCastOpcode(Opcode: CE->getOpcode()));
3043 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
3044 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3045 AbbrevToUse = CONSTANTS_CE_CAST_Abbrev;
3046 } else {
3047 assert(CE->getNumOperands() == 2 && "Unknown constant expr!");
3048 Code = bitc::CST_CODE_CE_BINOP;
3049 Record.push_back(Elt: getEncodedBinaryOpcode(Opcode: CE->getOpcode()));
3050 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3051 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3052 uint64_t Flags = getOptimizationFlags(V: CE);
3053 if (Flags != 0)
3054 Record.push_back(Elt: Flags);
3055 }
3056 break;
3057 case Instruction::FNeg: {
3058 assert(CE->getNumOperands() == 1 && "Unknown constant expr!");
3059 Code = bitc::CST_CODE_CE_UNOP;
3060 Record.push_back(Elt: getEncodedUnaryOpcode(Opcode: CE->getOpcode()));
3061 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3062 uint64_t Flags = getOptimizationFlags(V: CE);
3063 if (Flags != 0)
3064 Record.push_back(Elt: Flags);
3065 break;
3066 }
3067 case Instruction::GetElementPtr: {
3068 Code = bitc::CST_CODE_CE_GEP;
3069 const auto *GO = cast<GEPOperator>(Val: C);
3070 Record.push_back(Elt: VE.getTypeID(T: GO->getSourceElementType()));
3071 Record.push_back(Elt: getOptimizationFlags(V: GO));
3072 if (std::optional<ConstantRange> Range = GO->getInRange()) {
3073 Code = bitc::CST_CODE_CE_GEP_WITH_INRANGE;
3074 emitConstantRange(Record, CR: *Range, /*EmitBitWidth=*/true);
3075 }
3076 for (const Value *Op : CE->operands()) {
3077 Record.push_back(Elt: VE.getTypeID(T: Op->getType()));
3078 Record.push_back(Elt: VE.getValueID(V: Op));
3079 }
3080 break;
3081 }
3082 case Instruction::ExtractElement:
3083 Code = bitc::CST_CODE_CE_EXTRACTELT;
3084 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
3085 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3086 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 1)->getType()));
3087 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3088 break;
3089 case Instruction::InsertElement:
3090 Code = bitc::CST_CODE_CE_INSERTELT;
3091 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3092 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3093 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 2)->getType()));
3094 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 2)));
3095 break;
3096 case Instruction::ShuffleVector:
3097 // If the return type and argument types are the same, this is a
3098 // standard shufflevector instruction. If the types are different,
3099 // then the shuffle is widening or truncating the input vectors, and
3100 // the argument type must also be encoded.
3101 if (C->getType() == C->getOperand(i: 0)->getType()) {
3102 Code = bitc::CST_CODE_CE_SHUFFLEVEC;
3103 } else {
3104 Code = bitc::CST_CODE_CE_SHUFVEC_EX;
3105 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
3106 }
3107 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3108 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3109 Record.push_back(Elt: VE.getValueID(V: CE->getShuffleMaskForBitcode()));
3110 break;
3111 }
3112 } else if (const BlockAddress *BA = dyn_cast<BlockAddress>(Val: C)) {
3113 Code = bitc::CST_CODE_BLOCKADDRESS;
3114 Record.push_back(Elt: VE.getTypeID(T: BA->getFunction()->getType()));
3115 Record.push_back(Elt: VE.getValueID(V: BA->getFunction()));
3116 Record.push_back(Elt: VE.getGlobalBasicBlockID(BB: BA->getBasicBlock()));
3117 } else if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(Val: C)) {
3118 Code = bitc::CST_CODE_DSO_LOCAL_EQUIVALENT;
3119 Record.push_back(Elt: VE.getTypeID(T: Equiv->getGlobalValue()->getType()));
3120 Record.push_back(Elt: VE.getValueID(V: Equiv->getGlobalValue()));
3121 } else if (const auto *NC = dyn_cast<NoCFIValue>(Val: C)) {
3122 Code = bitc::CST_CODE_NO_CFI_VALUE;
3123 Record.push_back(Elt: VE.getTypeID(T: NC->getGlobalValue()->getType()));
3124 Record.push_back(Elt: VE.getValueID(V: NC->getGlobalValue()));
3125 } else if (const auto *CPA = dyn_cast<ConstantPtrAuth>(Val: C)) {
3126 Code = bitc::CST_CODE_PTRAUTH2;
3127 Record.push_back(Elt: VE.getValueID(V: CPA->getPointer()));
3128 Record.push_back(Elt: VE.getValueID(V: CPA->getKey()));
3129 Record.push_back(Elt: VE.getValueID(V: CPA->getDiscriminator()));
3130 Record.push_back(Elt: VE.getValueID(V: CPA->getAddrDiscriminator()));
3131 Record.push_back(Elt: VE.getValueID(V: CPA->getDeactivationSymbol()));
3132 } else {
3133#ifndef NDEBUG
3134 C->dump();
3135#endif
3136 llvm_unreachable("Unknown constant!");
3137 }
3138 Stream.EmitRecord(Code, Vals: Record, Abbrev: AbbrevToUse);
3139 Record.clear();
3140 }
3141
3142 Stream.ExitBlock();
3143}
3144
3145void ModuleBitcodeWriter::writeModuleConstants() {
3146 const ValueEnumerator::ValueList &Vals = VE.getValues();
3147
3148 // Find the first constant to emit, which is the first non-globalvalue value.
3149 // We know globalvalues have been emitted by WriteModuleInfo.
3150 for (unsigned i = 0, e = Vals.size(); i != e; ++i) {
3151 if (!isa<GlobalValue>(Val: Vals[i].first)) {
3152 writeConstants(FirstVal: i, LastVal: Vals.size(), isGlobal: true);
3153 return;
3154 }
3155 }
3156}
3157
3158/// pushValueAndType - The file has to encode both the value and type id for
3159/// many values, because we need to know what type to create for forward
3160/// references. However, most operands are not forward references, so this type
3161/// field is not needed.
3162///
3163/// This function adds V's value ID to Vals. If the value ID is higher than the
3164/// instruction ID, then it is a forward reference, and it also includes the
3165/// type ID. The value ID that is written is encoded relative to the InstID.
3166bool ModuleBitcodeWriter::pushValueAndType(const Value *V, unsigned InstID,
3167 SmallVectorImpl<unsigned> &Vals) {
3168 unsigned ValID = VE.getValueID(V);
3169 // Make encoding relative to the InstID.
3170 Vals.push_back(Elt: InstID - ValID);
3171 if (ValID >= InstID) {
3172 Vals.push_back(Elt: VE.getTypeID(T: V->getType()));
3173 return true;
3174 }
3175 return false;
3176}
3177
3178bool ModuleBitcodeWriter::pushValueOrMetadata(const Value *V, unsigned InstID,
3179 SmallVectorImpl<unsigned> &Vals) {
3180 bool IsMetadata = V->getType()->isMetadataTy();
3181 if (IsMetadata) {
3182 Vals.push_back(Elt: bitc::OB_METADATA);
3183 Metadata *MD = cast<MetadataAsValue>(Val: V)->getMetadata();
3184 unsigned ValID = VE.getMetadataID(MD);
3185 Vals.push_back(Elt: InstID - ValID);
3186 return false;
3187 }
3188 return pushValueAndType(V, InstID, Vals);
3189}
3190
3191void ModuleBitcodeWriter::writeOperandBundles(const CallBase &CS,
3192 unsigned InstID) {
3193 SmallVector<unsigned, 64> Record;
3194 LLVMContext &C = CS.getContext();
3195
3196 for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i) {
3197 const auto &Bundle = CS.getOperandBundleAt(Index: i);
3198 Record.push_back(Elt: C.getOperandBundleTagID(Tag: Bundle.getTagName()));
3199
3200 for (auto &Input : Bundle.Inputs)
3201 pushValueOrMetadata(V: Input, InstID, Vals&: Record);
3202
3203 Stream.EmitRecord(Code: bitc::FUNC_CODE_OPERAND_BUNDLE, Vals: Record);
3204 Record.clear();
3205 }
3206}
3207
3208/// pushValue - Like pushValueAndType, but where the type of the value is
3209/// omitted (perhaps it was already encoded in an earlier operand).
3210void ModuleBitcodeWriter::pushValue(const Value *V, unsigned InstID,
3211 SmallVectorImpl<unsigned> &Vals) {
3212 unsigned ValID = VE.getValueID(V);
3213 Vals.push_back(Elt: InstID - ValID);
3214}
3215
3216void ModuleBitcodeWriter::pushValueSigned(const Value *V, unsigned InstID,
3217 SmallVectorImpl<uint64_t> &Vals) {
3218 unsigned ValID = VE.getValueID(V);
3219 int64_t diff = ((int32_t)InstID - (int32_t)ValID);
3220 emitSignedInt64(Vals, V: diff);
3221}
3222
3223/// WriteInstruction - Emit an instruction to the specified stream.
3224void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
3225 unsigned InstID,
3226 SmallVectorImpl<unsigned> &Vals) {
3227 unsigned Code = 0;
3228 unsigned AbbrevToUse = 0;
3229 VE.setInstructionID(&I);
3230 switch (I.getOpcode()) {
3231 default:
3232 if (Instruction::isCast(Opcode: I.getOpcode())) {
3233 Code = bitc::FUNC_CODE_INST_CAST;
3234 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3235 AbbrevToUse = FUNCTION_INST_CAST_ABBREV;
3236 Vals.push_back(Elt: VE.getTypeID(T: I.getType()));
3237 Vals.push_back(Elt: getEncodedCastOpcode(Opcode: I.getOpcode()));
3238 uint64_t Flags = getOptimizationFlags(V: &I);
3239 if (Flags != 0) {
3240 if (AbbrevToUse == FUNCTION_INST_CAST_ABBREV)
3241 AbbrevToUse = FUNCTION_INST_CAST_FLAGS_ABBREV;
3242 Vals.push_back(Elt: Flags);
3243 }
3244 } else {
3245 assert(isa<BinaryOperator>(I) && "Unknown instruction!");
3246 Code = bitc::FUNC_CODE_INST_BINOP;
3247 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3248 AbbrevToUse = FUNCTION_INST_BINOP_ABBREV;
3249 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3250 Vals.push_back(Elt: getEncodedBinaryOpcode(Opcode: I.getOpcode()));
3251 uint64_t Flags = getOptimizationFlags(V: &I);
3252 if (Flags != 0) {
3253 if (AbbrevToUse == FUNCTION_INST_BINOP_ABBREV)
3254 AbbrevToUse = FUNCTION_INST_BINOP_FLAGS_ABBREV;
3255 Vals.push_back(Elt: Flags);
3256 }
3257 }
3258 break;
3259 case Instruction::FNeg: {
3260 Code = bitc::FUNC_CODE_INST_UNOP;
3261 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3262 AbbrevToUse = FUNCTION_INST_UNOP_ABBREV;
3263 Vals.push_back(Elt: getEncodedUnaryOpcode(Opcode: I.getOpcode()));
3264 uint64_t Flags = getOptimizationFlags(V: &I);
3265 if (Flags != 0) {
3266 if (AbbrevToUse == FUNCTION_INST_UNOP_ABBREV)
3267 AbbrevToUse = FUNCTION_INST_UNOP_FLAGS_ABBREV;
3268 Vals.push_back(Elt: Flags);
3269 }
3270 break;
3271 }
3272 case Instruction::GetElementPtr: {
3273 Code = bitc::FUNC_CODE_INST_GEP;
3274 AbbrevToUse = FUNCTION_INST_GEP_ABBREV;
3275 auto &GEPInst = cast<GetElementPtrInst>(Val: I);
3276 Vals.push_back(Elt: getOptimizationFlags(V: &I));
3277 Vals.push_back(Elt: VE.getTypeID(T: GEPInst.getSourceElementType()));
3278 for (const Value *Op : I.operands())
3279 pushValueAndType(V: Op, InstID, Vals);
3280 break;
3281 }
3282 case Instruction::ExtractValue: {
3283 Code = bitc::FUNC_CODE_INST_EXTRACTVAL;
3284 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3285 const ExtractValueInst *EVI = cast<ExtractValueInst>(Val: &I);
3286 Vals.append(in_start: EVI->idx_begin(), in_end: EVI->idx_end());
3287 break;
3288 }
3289 case Instruction::InsertValue: {
3290 Code = bitc::FUNC_CODE_INST_INSERTVAL;
3291 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3292 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3293 const InsertValueInst *IVI = cast<InsertValueInst>(Val: &I);
3294 Vals.append(in_start: IVI->idx_begin(), in_end: IVI->idx_end());
3295 break;
3296 }
3297 case Instruction::Select: {
3298 Code = bitc::FUNC_CODE_INST_VSELECT;
3299 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3300 pushValue(V: I.getOperand(i: 2), InstID, Vals);
3301 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3302 uint64_t Flags = getOptimizationFlags(V: &I);
3303 if (Flags != 0)
3304 Vals.push_back(Elt: Flags);
3305 break;
3306 }
3307 case Instruction::ExtractElement:
3308 Code = bitc::FUNC_CODE_INST_EXTRACTELT;
3309 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3310 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3311 break;
3312 case Instruction::InsertElement:
3313 Code = bitc::FUNC_CODE_INST_INSERTELT;
3314 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3315 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3316 pushValueAndType(V: I.getOperand(i: 2), InstID, Vals);
3317 break;
3318 case Instruction::ShuffleVector:
3319 Code = bitc::FUNC_CODE_INST_SHUFFLEVEC;
3320 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3321 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3322 pushValue(V: cast<ShuffleVectorInst>(Val: I).getShuffleMaskForBitcode(), InstID,
3323 Vals);
3324 break;
3325 case Instruction::ICmp:
3326 case Instruction::FCmp: {
3327 // compare returning Int1Ty or vector of Int1Ty
3328 Code = bitc::FUNC_CODE_INST_CMP2;
3329 AbbrevToUse = FUNCTION_INST_CMP_ABBREV;
3330 if (pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3331 AbbrevToUse = 0;
3332 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3333 Vals.push_back(Elt: cast<CmpInst>(Val: I).getPredicate());
3334 uint64_t Flags = getOptimizationFlags(V: &I);
3335 if (Flags != 0) {
3336 Vals.push_back(Elt: Flags);
3337 if (AbbrevToUse)
3338 AbbrevToUse = FUNCTION_INST_CMP_FLAGS_ABBREV;
3339 }
3340 break;
3341 }
3342
3343 case Instruction::Ret:
3344 {
3345 Code = bitc::FUNC_CODE_INST_RET;
3346 unsigned NumOperands = I.getNumOperands();
3347 if (NumOperands == 0)
3348 AbbrevToUse = FUNCTION_INST_RET_VOID_ABBREV;
3349 else if (NumOperands == 1) {
3350 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3351 AbbrevToUse = FUNCTION_INST_RET_VAL_ABBREV;
3352 } else {
3353 for (const Value *Op : I.operands())
3354 pushValueAndType(V: Op, InstID, Vals);
3355 }
3356 }
3357 break;
3358 case Instruction::UncondBr: {
3359 Code = bitc::FUNC_CODE_INST_BR;
3360 AbbrevToUse = FUNCTION_INST_BR_UNCOND_ABBREV;
3361 const UncondBrInst &II = cast<UncondBrInst>(Val: I);
3362 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 0)));
3363 } break;
3364 case Instruction::CondBr: {
3365 Code = bitc::FUNC_CODE_INST_BR;
3366 AbbrevToUse = FUNCTION_INST_BR_COND_ABBREV;
3367 const CondBrInst &II = cast<CondBrInst>(Val: I);
3368 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 0)));
3369 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 1)));
3370 pushValue(V: II.getCondition(), InstID, Vals);
3371 } break;
3372 case Instruction::Switch:
3373 {
3374 Code = bitc::FUNC_CODE_INST_SWITCH;
3375 const SwitchInst &SI = cast<SwitchInst>(Val: I);
3376 Vals.push_back(Elt: VE.getTypeID(T: SI.getCondition()->getType()));
3377 pushValue(V: SI.getCondition(), InstID, Vals);
3378 Vals.push_back(Elt: VE.getValueID(V: SI.getDefaultDest()));
3379 for (auto Case : SI.cases()) {
3380 Vals.push_back(Elt: VE.getValueID(V: Case.getCaseValue()));
3381 Vals.push_back(Elt: VE.getValueID(V: Case.getCaseSuccessor()));
3382 }
3383 }
3384 break;
3385 case Instruction::IndirectBr:
3386 Code = bitc::FUNC_CODE_INST_INDIRECTBR;
3387 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType()));
3388 // Encode the address operand as relative, but not the basic blocks.
3389 pushValue(V: I.getOperand(i: 0), InstID, Vals);
3390 for (const Value *Op : drop_begin(RangeOrContainer: I.operands()))
3391 Vals.push_back(Elt: VE.getValueID(V: Op));
3392 break;
3393
3394 case Instruction::Invoke: {
3395 const InvokeInst *II = cast<InvokeInst>(Val: &I);
3396 const Value *Callee = II->getCalledOperand();
3397 FunctionType *FTy = II->getFunctionType();
3398
3399 if (II->hasOperandBundles())
3400 writeOperandBundles(CS: *II, InstID);
3401
3402 Code = bitc::FUNC_CODE_INST_INVOKE;
3403
3404 Vals.push_back(Elt: VE.getAttributeListID(PAL: II->getAttributes()));
3405 Vals.push_back(Elt: II->getCallingConv() | 1 << 13);
3406 Vals.push_back(Elt: VE.getValueID(V: II->getNormalDest()));
3407 Vals.push_back(Elt: VE.getValueID(V: II->getUnwindDest()));
3408 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3409 pushValueAndType(V: Callee, InstID, Vals);
3410
3411 // Emit value #'s for the fixed parameters.
3412 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3413 pushValue(V: I.getOperand(i), InstID, Vals); // fixed param.
3414
3415 // Emit type/value pairs for varargs params.
3416 if (FTy->isVarArg()) {
3417 for (unsigned i = FTy->getNumParams(), e = II->arg_size(); i != e; ++i)
3418 pushValueAndType(V: I.getOperand(i), InstID, Vals); // vararg
3419 }
3420 break;
3421 }
3422 case Instruction::Resume:
3423 Code = bitc::FUNC_CODE_INST_RESUME;
3424 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3425 break;
3426 case Instruction::CleanupRet: {
3427 Code = bitc::FUNC_CODE_INST_CLEANUPRET;
3428 const auto &CRI = cast<CleanupReturnInst>(Val: I);
3429 pushValue(V: CRI.getCleanupPad(), InstID, Vals);
3430 if (CRI.hasUnwindDest())
3431 Vals.push_back(Elt: VE.getValueID(V: CRI.getUnwindDest()));
3432 break;
3433 }
3434 case Instruction::CatchRet: {
3435 Code = bitc::FUNC_CODE_INST_CATCHRET;
3436 const auto &CRI = cast<CatchReturnInst>(Val: I);
3437 pushValue(V: CRI.getCatchPad(), InstID, Vals);
3438 Vals.push_back(Elt: VE.getValueID(V: CRI.getSuccessor()));
3439 break;
3440 }
3441 case Instruction::CleanupPad:
3442 case Instruction::CatchPad: {
3443 const auto &FuncletPad = cast<FuncletPadInst>(Val: I);
3444 Code = isa<CatchPadInst>(Val: FuncletPad) ? bitc::FUNC_CODE_INST_CATCHPAD
3445 : bitc::FUNC_CODE_INST_CLEANUPPAD;
3446 pushValue(V: FuncletPad.getParentPad(), InstID, Vals);
3447
3448 unsigned NumArgOperands = FuncletPad.arg_size();
3449 Vals.push_back(Elt: NumArgOperands);
3450 for (unsigned Op = 0; Op != NumArgOperands; ++Op)
3451 pushValueAndType(V: FuncletPad.getArgOperand(i: Op), InstID, Vals);
3452 break;
3453 }
3454 case Instruction::CatchSwitch: {
3455 Code = bitc::FUNC_CODE_INST_CATCHSWITCH;
3456 const auto &CatchSwitch = cast<CatchSwitchInst>(Val: I);
3457
3458 pushValue(V: CatchSwitch.getParentPad(), InstID, Vals);
3459
3460 unsigned NumHandlers = CatchSwitch.getNumHandlers();
3461 Vals.push_back(Elt: NumHandlers);
3462 for (const BasicBlock *CatchPadBB : CatchSwitch.handlers())
3463 Vals.push_back(Elt: VE.getValueID(V: CatchPadBB));
3464
3465 if (CatchSwitch.hasUnwindDest())
3466 Vals.push_back(Elt: VE.getValueID(V: CatchSwitch.getUnwindDest()));
3467 break;
3468 }
3469 case Instruction::CallBr: {
3470 const CallBrInst *CBI = cast<CallBrInst>(Val: &I);
3471 const Value *Callee = CBI->getCalledOperand();
3472 FunctionType *FTy = CBI->getFunctionType();
3473
3474 if (CBI->hasOperandBundles())
3475 writeOperandBundles(CS: *CBI, InstID);
3476
3477 Code = bitc::FUNC_CODE_INST_CALLBR;
3478
3479 Vals.push_back(Elt: VE.getAttributeListID(PAL: CBI->getAttributes()));
3480
3481 Vals.push_back(Elt: CBI->getCallingConv() << bitc::CALL_CCONV |
3482 1 << bitc::CALL_EXPLICIT_TYPE);
3483
3484 Vals.push_back(Elt: VE.getValueID(V: CBI->getDefaultDest()));
3485 Vals.push_back(Elt: CBI->getNumIndirectDests());
3486 for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i)
3487 Vals.push_back(Elt: VE.getValueID(V: CBI->getIndirectDest(i)));
3488
3489 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3490 pushValueAndType(V: Callee, InstID, Vals);
3491
3492 // Emit value #'s for the fixed parameters.
3493 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3494 pushValue(V: I.getOperand(i), InstID, Vals); // fixed param.
3495
3496 // Emit type/value pairs for varargs params.
3497 if (FTy->isVarArg()) {
3498 for (unsigned i = FTy->getNumParams(), e = CBI->arg_size(); i != e; ++i)
3499 pushValueAndType(V: I.getOperand(i), InstID, Vals); // vararg
3500 }
3501 break;
3502 }
3503 case Instruction::Unreachable:
3504 Code = bitc::FUNC_CODE_INST_UNREACHABLE;
3505 AbbrevToUse = FUNCTION_INST_UNREACHABLE_ABBREV;
3506 break;
3507
3508 case Instruction::PHI: {
3509 const PHINode &PN = cast<PHINode>(Val: I);
3510 Code = bitc::FUNC_CODE_INST_PHI;
3511 // With the newer instruction encoding, forward references could give
3512 // negative valued IDs. This is most common for PHIs, so we use
3513 // signed VBRs.
3514 SmallVector<uint64_t, 128> Vals64;
3515 Vals64.push_back(Elt: VE.getTypeID(T: PN.getType()));
3516 for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) {
3517 pushValueSigned(V: PN.getIncomingValue(i), InstID, Vals&: Vals64);
3518 Vals64.push_back(Elt: VE.getValueID(V: PN.getIncomingBlock(i)));
3519 }
3520
3521 uint64_t Flags = getOptimizationFlags(V: &I);
3522 if (Flags != 0)
3523 Vals64.push_back(Elt: Flags);
3524
3525 // Emit a Vals64 vector and exit.
3526 Stream.EmitRecord(Code, Vals: Vals64, Abbrev: AbbrevToUse);
3527 Vals64.clear();
3528 return;
3529 }
3530
3531 case Instruction::LandingPad: {
3532 const LandingPadInst &LP = cast<LandingPadInst>(Val: I);
3533 Code = bitc::FUNC_CODE_INST_LANDINGPAD;
3534 Vals.push_back(Elt: VE.getTypeID(T: LP.getType()));
3535 Vals.push_back(Elt: LP.isCleanup());
3536 Vals.push_back(Elt: LP.getNumClauses());
3537 for (unsigned I = 0, E = LP.getNumClauses(); I != E; ++I) {
3538 if (LP.isCatch(Idx: I))
3539 Vals.push_back(Elt: LandingPadInst::Catch);
3540 else
3541 Vals.push_back(Elt: LandingPadInst::Filter);
3542 pushValueAndType(V: LP.getClause(Idx: I), InstID, Vals);
3543 }
3544 break;
3545 }
3546
3547 case Instruction::Alloca: {
3548 Code = bitc::FUNC_CODE_INST_ALLOCA;
3549 const AllocaInst &AI = cast<AllocaInst>(Val: I);
3550 Vals.push_back(Elt: VE.getTypeID(T: AI.getAllocatedType()));
3551 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType()));
3552 Vals.push_back(Elt: VE.getValueID(V: I.getOperand(i: 0))); // size.
3553 using APV = AllocaPackedValues;
3554 unsigned Record = 0;
3555 unsigned EncodedAlign = getEncodedAlign(Alignment: AI.getAlign());
3556 Bitfield::set<APV::AlignLower>(
3557 Packed&: Record, Value: EncodedAlign & ((1 << APV::AlignLower::Bits) - 1));
3558 Bitfield::set<APV::AlignUpper>(Packed&: Record,
3559 Value: EncodedAlign >> APV::AlignLower::Bits);
3560 Bitfield::set<APV::UsedWithInAlloca>(Packed&: Record, Value: AI.isUsedWithInAlloca());
3561 Bitfield::set<APV::ExplicitType>(Packed&: Record, Value: true);
3562 Bitfield::set<APV::SwiftError>(Packed&: Record, Value: AI.isSwiftError());
3563 Vals.push_back(Elt: Record);
3564
3565 unsigned AS = AI.getAddressSpace();
3566 if (AS != M.getDataLayout().getAllocaAddrSpace())
3567 Vals.push_back(Elt: AS);
3568 break;
3569 }
3570
3571 case Instruction::Load: {
3572 const auto &LI = cast<LoadInst>(Val: I);
3573 if (LI.isAtomic()) {
3574 Code = bitc::FUNC_CODE_INST_LOADATOMIC;
3575 pushValueAndType(V: LI.getOperand(i_nocapture: 0), InstID, Vals);
3576 } else {
3577 Code = bitc::FUNC_CODE_INST_LOAD;
3578 if (!pushValueAndType(V: LI.getOperand(i_nocapture: 0), InstID, Vals)) // ptr
3579 AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
3580 }
3581 Vals.push_back(Elt: VE.getTypeID(T: LI.getType()));
3582 Vals.push_back(Elt: getEncodedAlign(Alignment: LI.getAlign()));
3583 Vals.push_back(Elt: LI.isVolatile());
3584 if (LI.isAtomic()) {
3585 Vals.push_back(Elt: getEncodedOrdering(Ordering: LI.getOrdering()));
3586 Vals.push_back(Elt: getEncodedSyncScopeID(SSID: LI.getSyncScopeID()));
3587 if (LI.isElementwise())
3588 Vals.push_back(Elt: 1);
3589 }
3590 break;
3591 }
3592
3593 case Instruction::Store:
3594 if (cast<StoreInst>(Val: I).isAtomic()) {
3595 Code = bitc::FUNC_CODE_INST_STOREATOMIC;
3596 } else {
3597 Code = bitc::FUNC_CODE_INST_STORE;
3598 AbbrevToUse = FUNCTION_INST_STORE_ABBREV;
3599 }
3600 if (pushValueAndType(V: I.getOperand(i: 1), InstID, Vals)) // ptrty + ptr
3601 AbbrevToUse = 0;
3602 if (pushValueAndType(V: I.getOperand(i: 0), InstID, Vals)) // valty + val
3603 AbbrevToUse = 0;
3604 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<StoreInst>(Val: I).getAlign()));
3605 Vals.push_back(Elt: cast<StoreInst>(Val: I).isVolatile());
3606 if (cast<StoreInst>(Val: I).isAtomic()) {
3607 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<StoreInst>(Val: I).getOrdering()));
3608 Vals.push_back(
3609 Elt: getEncodedSyncScopeID(SSID: cast<StoreInst>(Val: I).getSyncScopeID()));
3610 }
3611 break;
3612 case Instruction::AtomicCmpXchg:
3613 Code = bitc::FUNC_CODE_INST_CMPXCHG;
3614 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals); // ptrty + ptr
3615 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals); // cmp.
3616 pushValue(V: I.getOperand(i: 2), InstID, Vals); // newval.
3617 Vals.push_back(Elt: cast<AtomicCmpXchgInst>(Val: I).isVolatile());
3618 Vals.push_back(
3619 Elt: getEncodedOrdering(Ordering: cast<AtomicCmpXchgInst>(Val: I).getSuccessOrdering()));
3620 Vals.push_back(
3621 Elt: getEncodedSyncScopeID(SSID: cast<AtomicCmpXchgInst>(Val: I).getSyncScopeID()));
3622 Vals.push_back(
3623 Elt: getEncodedOrdering(Ordering: cast<AtomicCmpXchgInst>(Val: I).getFailureOrdering()));
3624 Vals.push_back(Elt: cast<AtomicCmpXchgInst>(Val: I).isWeak());
3625 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<AtomicCmpXchgInst>(Val: I).getAlign()));
3626 break;
3627 case Instruction::AtomicRMW:
3628 Code = bitc::FUNC_CODE_INST_ATOMICRMW;
3629 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals); // ptrty + ptr
3630 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals); // valty + val
3631 Vals.push_back(Elt: getEncodedRMWOperation(I: cast<AtomicRMWInst>(Val: I)));
3632 Vals.push_back(Elt: cast<AtomicRMWInst>(Val: I).isVolatile());
3633 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<AtomicRMWInst>(Val: I).getOrdering()));
3634 Vals.push_back(
3635 Elt: getEncodedSyncScopeID(SSID: cast<AtomicRMWInst>(Val: I).getSyncScopeID()));
3636 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<AtomicRMWInst>(Val: I).getAlign()));
3637 break;
3638 case Instruction::Fence:
3639 Code = bitc::FUNC_CODE_INST_FENCE;
3640 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<FenceInst>(Val: I).getOrdering()));
3641 Vals.push_back(Elt: getEncodedSyncScopeID(SSID: cast<FenceInst>(Val: I).getSyncScopeID()));
3642 break;
3643 case Instruction::Call: {
3644 const CallInst &CI = cast<CallInst>(Val: I);
3645 FunctionType *FTy = CI.getFunctionType();
3646
3647 if (CI.hasOperandBundles())
3648 writeOperandBundles(CS: CI, InstID);
3649
3650 Code = bitc::FUNC_CODE_INST_CALL;
3651
3652 Vals.push_back(Elt: VE.getAttributeListID(PAL: CI.getAttributes()));
3653
3654 unsigned Flags = getOptimizationFlags(V: &I);
3655 Vals.push_back(Elt: CI.getCallingConv() << bitc::CALL_CCONV |
3656 unsigned(CI.isTailCall()) << bitc::CALL_TAIL |
3657 unsigned(CI.isMustTailCall()) << bitc::CALL_MUSTTAIL |
3658 1 << bitc::CALL_EXPLICIT_TYPE |
3659 unsigned(CI.isNoTailCall()) << bitc::CALL_NOTAIL |
3660 unsigned(Flags != 0) << bitc::CALL_FMF);
3661 if (Flags != 0)
3662 Vals.push_back(Elt: Flags);
3663
3664 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3665 pushValueAndType(V: CI.getCalledOperand(), InstID, Vals); // Callee
3666
3667 // Emit value #'s for the fixed parameters.
3668 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3669 pushValue(V: CI.getArgOperand(i), InstID, Vals); // fixed param.
3670
3671 // Emit type/value pairs for varargs params.
3672 if (FTy->isVarArg()) {
3673 for (unsigned i = FTy->getNumParams(), e = CI.arg_size(); i != e; ++i)
3674 pushValueAndType(V: CI.getArgOperand(i), InstID, Vals); // varargs
3675 }
3676 break;
3677 }
3678 case Instruction::VAArg:
3679 Code = bitc::FUNC_CODE_INST_VAARG;
3680 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType())); // valistty
3681 pushValue(V: I.getOperand(i: 0), InstID, Vals); // valist.
3682 Vals.push_back(Elt: VE.getTypeID(T: I.getType())); // restype.
3683 break;
3684 case Instruction::Freeze:
3685 Code = bitc::FUNC_CODE_INST_FREEZE;
3686 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3687 break;
3688 }
3689
3690 Stream.EmitRecord(Code, Vals, Abbrev: AbbrevToUse);
3691 Vals.clear();
3692}
3693
3694/// Write a GlobalValue VST to the module. The purpose of this data structure is
3695/// to allow clients to efficiently find the function body.
3696void ModuleBitcodeWriter::writeGlobalValueSymbolTable(
3697 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3698 // Get the offset of the VST we are writing, and backpatch it into
3699 // the VST forward declaration record.
3700 uint64_t VSTOffset = Stream.GetCurrentBitNo();
3701 // The BitcodeStartBit was the stream offset of the identification block.
3702 VSTOffset -= bitcodeStartBit();
3703 assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned");
3704 // Note that we add 1 here because the offset is relative to one word
3705 // before the start of the identification block, which was historically
3706 // always the start of the regular bitcode header.
3707 Stream.BackpatchWord(BitNo: VSTOffsetPlaceholder, Val: VSTOffset / 32 + 1);
3708
3709 Stream.EnterSubblock(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, CodeLen: 4);
3710
3711 auto Abbv = std::make_shared<BitCodeAbbrev>();
3712 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY));
3713 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
3714 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset
3715 unsigned FnEntryAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
3716
3717 for (const Function &F : M) {
3718 uint64_t Record[2];
3719
3720 if (F.isDeclaration())
3721 continue;
3722
3723 Record[0] = VE.getValueID(V: &F);
3724
3725 // Save the word offset of the function (from the start of the
3726 // actual bitcode written to the stream).
3727 uint64_t BitcodeIndex = FunctionToBitcodeIndex[&F] - bitcodeStartBit();
3728 assert((BitcodeIndex & 31) == 0 && "function block not 32-bit aligned");
3729 // Note that we add 1 here because the offset is relative to one word
3730 // before the start of the identification block, which was historically
3731 // always the start of the regular bitcode header.
3732 Record[1] = BitcodeIndex / 32 + 1;
3733
3734 Stream.EmitRecord(Code: bitc::VST_CODE_FNENTRY, Vals: Record, Abbrev: FnEntryAbbrev);
3735 }
3736
3737 Stream.ExitBlock();
3738}
3739
3740/// Emit names for arguments, instructions and basic blocks in a function.
3741void ModuleBitcodeWriter::writeFunctionLevelValueSymbolTable(
3742 const ValueSymbolTable &VST) {
3743 if (VST.empty())
3744 return;
3745
3746 Stream.EnterSubblock(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, CodeLen: 4);
3747
3748 // FIXME: Set up the abbrev, we know how many values there are!
3749 // FIXME: We know if the type names can use 7-bit ascii.
3750 SmallVector<uint64_t, 64> NameVals;
3751
3752 for (const ValueName &Name : VST) {
3753 // Figure out the encoding to use for the name.
3754 StringEncoding Bits = getStringEncoding(Str: Name.getKey());
3755
3756 unsigned AbbrevToUse = VST_ENTRY_8_ABBREV;
3757 NameVals.push_back(Elt: VE.getValueID(V: Name.getValue()));
3758
3759 // VST_CODE_ENTRY: [valueid, namechar x N]
3760 // VST_CODE_BBENTRY: [bbid, namechar x N]
3761 unsigned Code;
3762 if (isa<BasicBlock>(Val: Name.getValue())) {
3763 Code = bitc::VST_CODE_BBENTRY;
3764 if (Bits == SE_Char6)
3765 AbbrevToUse = VST_BBENTRY_6_ABBREV;
3766 } else {
3767 Code = bitc::VST_CODE_ENTRY;
3768 if (Bits == SE_Char6)
3769 AbbrevToUse = VST_ENTRY_6_ABBREV;
3770 else if (Bits == SE_Fixed7)
3771 AbbrevToUse = VST_ENTRY_7_ABBREV;
3772 }
3773
3774 for (const auto P : Name.getKey())
3775 NameVals.push_back(Elt: (unsigned char)P);
3776
3777 // Emit the finished record.
3778 Stream.EmitRecord(Code, Vals: NameVals, Abbrev: AbbrevToUse);
3779 NameVals.clear();
3780 }
3781
3782 Stream.ExitBlock();
3783}
3784
3785void ModuleBitcodeWriter::writeUseList(UseListOrder &&Order) {
3786 assert(Order.Shuffle.size() >= 2 && "Shuffle too small");
3787 unsigned Code;
3788 if (isa<BasicBlock>(Val: Order.V))
3789 Code = bitc::USELIST_CODE_BB;
3790 else
3791 Code = bitc::USELIST_CODE_DEFAULT;
3792
3793 SmallVector<uint64_t, 64> Record(Order.Shuffle.begin(), Order.Shuffle.end());
3794 Record.push_back(Elt: VE.getValueID(V: Order.V));
3795 Stream.EmitRecord(Code, Vals: Record);
3796}
3797
3798void ModuleBitcodeWriter::writeUseListBlock(const Function *F) {
3799 assert(VE.shouldPreserveUseListOrder() &&
3800 "Expected to be preserving use-list order");
3801
3802 auto hasMore = [&]() {
3803 return !VE.UseListOrders.empty() && VE.UseListOrders.back().F == F;
3804 };
3805 if (!hasMore())
3806 // Nothing to do.
3807 return;
3808
3809 Stream.EnterSubblock(BlockID: bitc::USELIST_BLOCK_ID, CodeLen: 3);
3810 while (hasMore()) {
3811 writeUseList(Order: std::move(VE.UseListOrders.back()));
3812 VE.UseListOrders.pop_back();
3813 }
3814 Stream.ExitBlock();
3815}
3816
3817/// Emit a function body to the module stream.
3818void ModuleBitcodeWriter::writeFunction(
3819 const Function &F,
3820 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3821 // Save the bitcode index of the start of this function block for recording
3822 // in the VST.
3823 FunctionToBitcodeIndex[&F] = Stream.GetCurrentBitNo();
3824
3825 Stream.EnterSubblock(BlockID: bitc::FUNCTION_BLOCK_ID, CodeLen: 5);
3826 VE.incorporateFunction(F);
3827
3828 SmallVector<unsigned, 64> Vals;
3829
3830 // Emit the number of basic blocks, so the reader can create them ahead of
3831 // time.
3832 Vals.push_back(Elt: VE.getBasicBlocks().size());
3833 Stream.EmitRecord(Code: bitc::FUNC_CODE_DECLAREBLOCKS, Vals);
3834 Vals.clear();
3835
3836 // If there are function-local constants, emit them now.
3837 unsigned CstStart, CstEnd;
3838 VE.getFunctionConstantRange(Start&: CstStart, End&: CstEnd);
3839 writeConstants(FirstVal: CstStart, LastVal: CstEnd, isGlobal: false);
3840
3841 // If there is function-local metadata, emit it now.
3842 writeFunctionMetadata(F);
3843
3844 // Keep a running idea of what the instruction ID is.
3845 unsigned InstID = CstEnd;
3846
3847 bool NeedsMetadataAttachment = F.hasMetadata();
3848
3849 DILocation *LastDL = nullptr;
3850 SmallSetVector<Function *, 4> BlockAddressUsers;
3851
3852 // Finally, emit all the instructions, in order.
3853 for (const BasicBlock &BB : F) {
3854 for (const Instruction &I : BB) {
3855 writeInstruction(I, InstID, Vals);
3856
3857 if (!I.getType()->isVoidTy())
3858 ++InstID;
3859
3860 // If the instruction has metadata, write a metadata attachment later.
3861 NeedsMetadataAttachment |= I.hasMetadataOtherThanDebugLoc();
3862
3863 // If the instruction has a debug location, emit it.
3864 if (DILocation *DL = I.getDebugLoc()) {
3865 if (DL == LastDL) {
3866 // Just repeat the same debug loc as last time.
3867 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_LOC_AGAIN, Vals);
3868 } else {
3869 Vals.push_back(Elt: DL->getLine());
3870 Vals.push_back(Elt: DL->getColumn());
3871 Vals.push_back(Elt: VE.getMetadataOrNullID(MD: DL->getScope()));
3872 Vals.push_back(Elt: VE.getMetadataOrNullID(MD: DL->getInlinedAt()));
3873 Vals.push_back(Elt: DL->isImplicitCode());
3874 Vals.push_back(Elt: DL->getAtomGroup());
3875 Vals.push_back(Elt: DL->getAtomRank());
3876 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_LOC, Vals,
3877 Abbrev: FUNCTION_DEBUG_LOC_ABBREV);
3878 Vals.clear();
3879 LastDL = DL;
3880 }
3881 }
3882
3883 // If the instruction has DbgRecords attached to it, emit them. Note that
3884 // they come after the instruction so that it's easy to attach them again
3885 // when reading the bitcode, even though conceptually the debug locations
3886 // start "before" the instruction.
3887 if (I.hasDbgRecords()) {
3888 /// Try to push the value only (unwrapped), otherwise push the
3889 /// metadata wrapped value. Returns true if the value was pushed
3890 /// without the ValueAsMetadata wrapper.
3891 auto PushValueOrMetadata = [&Vals, InstID,
3892 this](Metadata *RawLocation) {
3893 assert(RawLocation &&
3894 "RawLocation unexpectedly null in DbgVariableRecord");
3895 if (ValueAsMetadata *VAM = dyn_cast<ValueAsMetadata>(Val: RawLocation)) {
3896 SmallVector<unsigned, 2> ValAndType;
3897 // If the value is a fwd-ref the type is also pushed. We don't
3898 // want the type, so fwd-refs are kept wrapped (pushValueAndType
3899 // returns false if the value is pushed without type).
3900 if (!pushValueAndType(V: VAM->getValue(), InstID, Vals&: ValAndType)) {
3901 Vals.push_back(Elt: ValAndType[0]);
3902 return true;
3903 }
3904 }
3905 // The metadata is a DIArgList, or ValueAsMetadata wrapping a
3906 // fwd-ref. Push the metadata ID.
3907 Vals.push_back(Elt: VE.getMetadataID(MD: RawLocation));
3908 return false;
3909 };
3910
3911 // Write out non-instruction debug information attached to this
3912 // instruction. Write it after the instruction so that it's easy to
3913 // re-attach to the instruction reading the records in.
3914 for (DbgRecord &DR : I.DebugMarker->getDbgRecordRange()) {
3915 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
3916 Vals.push_back(Elt: VE.getMetadataID(MD: &*DLR->getDebugLoc()));
3917 Vals.push_back(Elt: VE.getMetadataID(MD: DLR->getLabel()));
3918 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_LABEL, Vals);
3919 Vals.clear();
3920 continue;
3921 }
3922
3923 // First 3 fields are common to all kinds:
3924 // DILocation, DILocalVariable, DIExpression
3925 // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE)
3926 // ..., LocationMetadata
3927 // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE - abbrev'd)
3928 // ..., Value
3929 // dbg_declare (FUNC_CODE_DEBUG_RECORD_DECLARE)
3930 // ..., LocationMetadata
3931 // dbg_assign (FUNC_CODE_DEBUG_RECORD_ASSIGN)
3932 // ..., LocationMetadata, DIAssignID, DIExpression, LocationMetadata
3933 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
3934 Vals.push_back(Elt: VE.getMetadataID(MD: &*DVR.getDebugLoc()));
3935 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getVariable()));
3936 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getExpression()));
3937 if (DVR.isDbgValue()) {
3938 if (PushValueOrMetadata(DVR.getRawLocation()))
3939 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE, Vals,
3940 Abbrev: FUNCTION_DEBUG_RECORD_VALUE_ABBREV);
3941 else
3942 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_VALUE, Vals);
3943 } else if (DVR.isDbgDeclare()) {
3944 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3945 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_DECLARE, Vals);
3946 } else if (DVR.isDbgDeclareValue()) {
3947 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3948 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_DECLARE_VALUE, Vals);
3949 } else {
3950 assert(DVR.isDbgAssign() && "Unexpected DbgRecord kind");
3951 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3952 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getAssignID()));
3953 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getAddressExpression()));
3954 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawAddress()));
3955 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_ASSIGN, Vals);
3956 }
3957 Vals.clear();
3958 }
3959 }
3960 }
3961
3962 if (BlockAddress *BA = BlockAddress::lookup(BB: &BB)) {
3963 SmallVector<Value *> Worklist{BA};
3964 SmallPtrSet<Value *, 8> Visited{BA};
3965 while (!Worklist.empty()) {
3966 Value *V = Worklist.pop_back_val();
3967 for (User *U : V->users()) {
3968 if (auto *I = dyn_cast<Instruction>(Val: U)) {
3969 Function *P = I->getFunction();
3970 if (P != &F)
3971 BlockAddressUsers.insert(X: P);
3972 } else if (isa<Constant>(Val: U) && !isa<GlobalValue>(Val: U) &&
3973 Visited.insert(Ptr: U).second)
3974 Worklist.push_back(Elt: U);
3975 }
3976 }
3977 }
3978 }
3979
3980 if (!BlockAddressUsers.empty()) {
3981 Vals.resize(N: BlockAddressUsers.size());
3982 for (auto I : llvm::enumerate(First&: BlockAddressUsers))
3983 Vals[I.index()] = VE.getValueID(V: I.value());
3984 Stream.EmitRecord(Code: bitc::FUNC_CODE_BLOCKADDR_USERS, Vals);
3985 Vals.clear();
3986 }
3987
3988 // Emit names for all the instructions etc.
3989 if (auto *Symtab = F.getValueSymbolTable())
3990 writeFunctionLevelValueSymbolTable(VST: *Symtab);
3991
3992 if (NeedsMetadataAttachment)
3993 writeFunctionMetadataAttachment(F);
3994 if (VE.shouldPreserveUseListOrder())
3995 writeUseListBlock(F: &F);
3996 VE.purgeFunction();
3997 Stream.ExitBlock();
3998}
3999
4000// Emit blockinfo, which defines the standard abbreviations etc.
4001void ModuleBitcodeWriter::writeBlockInfo() {
4002 // We only want to emit block info records for blocks that have multiple
4003 // instances: CONSTANTS_BLOCK, FUNCTION_BLOCK and VALUE_SYMTAB_BLOCK.
4004 // Other blocks can define their abbrevs inline.
4005 Stream.EnterBlockInfoBlock();
4006
4007 // Encode type indices using fixed size based on number of types.
4008 BitCodeAbbrevOp TypeAbbrevOp(BitCodeAbbrevOp::Fixed,
4009 VE.computeBitsRequiredForTypeIndices());
4010 // Encode value indices as 6-bit VBR.
4011 BitCodeAbbrevOp ValAbbrevOp(BitCodeAbbrevOp::VBR, 6);
4012
4013 { // 8-bit fixed-width VST_CODE_ENTRY/VST_CODE_BBENTRY strings.
4014 auto Abbv = std::make_shared<BitCodeAbbrev>();
4015 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3));
4016 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4017 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4018 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
4019 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
4020 VST_ENTRY_8_ABBREV)
4021 llvm_unreachable("Unexpected abbrev ordering!");
4022 }
4023
4024 { // 7-bit fixed width VST_CODE_ENTRY strings.
4025 auto Abbv = std::make_shared<BitCodeAbbrev>();
4026 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
4027 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4028 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4029 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
4030 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
4031 VST_ENTRY_7_ABBREV)
4032 llvm_unreachable("Unexpected abbrev ordering!");
4033 }
4034 { // 6-bit char6 VST_CODE_ENTRY strings.
4035 auto Abbv = std::make_shared<BitCodeAbbrev>();
4036 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
4037 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4038 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4039 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
4040 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
4041 VST_ENTRY_6_ABBREV)
4042 llvm_unreachable("Unexpected abbrev ordering!");
4043 }
4044 { // 6-bit char6 VST_CODE_BBENTRY strings.
4045 auto Abbv = std::make_shared<BitCodeAbbrev>();
4046 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_BBENTRY));
4047 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4048 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4049 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
4050 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
4051 VST_BBENTRY_6_ABBREV)
4052 llvm_unreachable("Unexpected abbrev ordering!");
4053 }
4054
4055 { // SETTYPE abbrev for CONSTANTS_BLOCK.
4056 auto Abbv = std::make_shared<BitCodeAbbrev>();
4057 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_SETTYPE));
4058 Abbv->Add(OpInfo: TypeAbbrevOp);
4059 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4060 CONSTANTS_SETTYPE_ABBREV)
4061 llvm_unreachable("Unexpected abbrev ordering!");
4062 }
4063
4064 { // INTEGER abbrev for CONSTANTS_BLOCK.
4065 auto Abbv = std::make_shared<BitCodeAbbrev>();
4066 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_INTEGER));
4067 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4068 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4069 CONSTANTS_INTEGER_ABBREV)
4070 llvm_unreachable("Unexpected abbrev ordering!");
4071 }
4072
4073 { // BYTE abbrev for CONSTANTS_BLOCK.
4074 auto Abbv = std::make_shared<BitCodeAbbrev>();
4075 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_BYTE));
4076 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4077 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4078 CONSTANTS_BYTE_ABBREV)
4079 llvm_unreachable("Unexpected abbrev ordering!");
4080 }
4081
4082 { // CE_CAST abbrev for CONSTANTS_BLOCK.
4083 auto Abbv = std::make_shared<BitCodeAbbrev>();
4084 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CE_CAST));
4085 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // cast opc
4086 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // typeid
4087 VE.computeBitsRequiredForTypeIndices()));
4088 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
4089
4090 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4091 CONSTANTS_CE_CAST_Abbrev)
4092 llvm_unreachable("Unexpected abbrev ordering!");
4093 }
4094 { // NULL abbrev for CONSTANTS_BLOCK.
4095 auto Abbv = std::make_shared<BitCodeAbbrev>();
4096 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_NULL));
4097 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4098 CONSTANTS_NULL_Abbrev)
4099 llvm_unreachable("Unexpected abbrev ordering!");
4100 }
4101
4102 // FIXME: This should only use space for first class types!
4103
4104 { // INST_LOAD abbrev for FUNCTION_BLOCK.
4105 auto Abbv = std::make_shared<BitCodeAbbrev>();
4106 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_LOAD));
4107 Abbv->Add(OpInfo: ValAbbrevOp); // Ptr
4108 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4109 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Align
4110 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
4111 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4112 FUNCTION_INST_LOAD_ABBREV)
4113 llvm_unreachable("Unexpected abbrev ordering!");
4114 }
4115 {
4116 auto Abbv = std::make_shared<BitCodeAbbrev>();
4117 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_STORE));
4118 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4119 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4120 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // align
4121 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
4122 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4123 FUNCTION_INST_STORE_ABBREV)
4124 llvm_unreachable("Unexpected abbrev ordering!");
4125 }
4126 { // INST_UNOP abbrev for FUNCTION_BLOCK.
4127 auto Abbv = std::make_shared<BitCodeAbbrev>();
4128 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
4129 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4130 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4131 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4132 FUNCTION_INST_UNOP_ABBREV)
4133 llvm_unreachable("Unexpected abbrev ordering!");
4134 }
4135 { // INST_UNOP_FLAGS abbrev for FUNCTION_BLOCK.
4136 auto Abbv = std::make_shared<BitCodeAbbrev>();
4137 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
4138 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4139 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4140 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4141 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4142 FUNCTION_INST_UNOP_FLAGS_ABBREV)
4143 llvm_unreachable("Unexpected abbrev ordering!");
4144 }
4145 { // INST_BINOP abbrev for FUNCTION_BLOCK.
4146 auto Abbv = std::make_shared<BitCodeAbbrev>();
4147 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
4148 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4149 Abbv->Add(OpInfo: ValAbbrevOp); // RHS
4150 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4151 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4152 FUNCTION_INST_BINOP_ABBREV)
4153 llvm_unreachable("Unexpected abbrev ordering!");
4154 }
4155 { // INST_BINOP_FLAGS abbrev for FUNCTION_BLOCK.
4156 auto Abbv = std::make_shared<BitCodeAbbrev>();
4157 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
4158 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4159 Abbv->Add(OpInfo: ValAbbrevOp); // RHS
4160 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4161 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4162 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4163 FUNCTION_INST_BINOP_FLAGS_ABBREV)
4164 llvm_unreachable("Unexpected abbrev ordering!");
4165 }
4166 { // INST_CAST abbrev for FUNCTION_BLOCK.
4167 auto Abbv = std::make_shared<BitCodeAbbrev>();
4168 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
4169 Abbv->Add(OpInfo: ValAbbrevOp); // OpVal
4170 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4171 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4172 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4173 FUNCTION_INST_CAST_ABBREV)
4174 llvm_unreachable("Unexpected abbrev ordering!");
4175 }
4176 { // INST_CAST_FLAGS abbrev for FUNCTION_BLOCK.
4177 auto Abbv = std::make_shared<BitCodeAbbrev>();
4178 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
4179 Abbv->Add(OpInfo: ValAbbrevOp); // OpVal
4180 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4181 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4182 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 9)); // flags
4183 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4184 FUNCTION_INST_CAST_FLAGS_ABBREV)
4185 llvm_unreachable("Unexpected abbrev ordering!");
4186 }
4187
4188 { // INST_RET abbrev for FUNCTION_BLOCK.
4189 auto Abbv = std::make_shared<BitCodeAbbrev>();
4190 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
4191 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4192 FUNCTION_INST_RET_VOID_ABBREV)
4193 llvm_unreachable("Unexpected abbrev ordering!");
4194 }
4195 { // INST_RET abbrev for FUNCTION_BLOCK.
4196 auto Abbv = std::make_shared<BitCodeAbbrev>();
4197 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
4198 Abbv->Add(OpInfo: ValAbbrevOp);
4199 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4200 FUNCTION_INST_RET_VAL_ABBREV)
4201 llvm_unreachable("Unexpected abbrev ordering!");
4202 }
4203 {
4204 auto Abbv = std::make_shared<BitCodeAbbrev>();
4205 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BR));
4206 // TODO: Use different abbrev for absolute value reference (succ0)?
4207 Abbv->Add(OpInfo: ValAbbrevOp); // succ0
4208 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4209 FUNCTION_INST_BR_UNCOND_ABBREV)
4210 llvm_unreachable("Unexpected abbrev ordering!");
4211 }
4212 {
4213 auto Abbv = std::make_shared<BitCodeAbbrev>();
4214 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BR));
4215 // TODO: Use different abbrev for absolute value references (succ0, succ1)?
4216 Abbv->Add(OpInfo: ValAbbrevOp); // succ0
4217 Abbv->Add(OpInfo: ValAbbrevOp); // succ1
4218 Abbv->Add(OpInfo: ValAbbrevOp); // cond
4219 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4220 FUNCTION_INST_BR_COND_ABBREV)
4221 llvm_unreachable("Unexpected abbrev ordering!");
4222 }
4223 { // INST_UNREACHABLE abbrev for FUNCTION_BLOCK.
4224 auto Abbv = std::make_shared<BitCodeAbbrev>();
4225 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNREACHABLE));
4226 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4227 FUNCTION_INST_UNREACHABLE_ABBREV)
4228 llvm_unreachable("Unexpected abbrev ordering!");
4229 }
4230 {
4231 auto Abbv = std::make_shared<BitCodeAbbrev>();
4232 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_GEP));
4233 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // flags
4234 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4235 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4236 Abbv->Add(OpInfo: ValAbbrevOp);
4237 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4238 FUNCTION_INST_GEP_ABBREV)
4239 llvm_unreachable("Unexpected abbrev ordering!");
4240 }
4241 {
4242 auto Abbv = std::make_shared<BitCodeAbbrev>();
4243 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CMP2));
4244 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4245 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4246 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // pred
4247 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4248 FUNCTION_INST_CMP_ABBREV)
4249 llvm_unreachable("Unexpected abbrev ordering!");
4250 }
4251 {
4252 auto Abbv = std::make_shared<BitCodeAbbrev>();
4253 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CMP2));
4254 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4255 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4256 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // pred
4257 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4258 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4259 FUNCTION_INST_CMP_FLAGS_ABBREV)
4260 llvm_unreachable("Unexpected abbrev ordering!");
4261 }
4262 {
4263 auto Abbv = std::make_shared<BitCodeAbbrev>();
4264 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE));
4265 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // dbgloc
4266 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // var
4267 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // expr
4268 Abbv->Add(OpInfo: ValAbbrevOp); // val
4269 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4270 FUNCTION_DEBUG_RECORD_VALUE_ABBREV)
4271 llvm_unreachable("Unexpected abbrev ordering! 1");
4272 }
4273 {
4274 auto Abbv = std::make_shared<BitCodeAbbrev>();
4275 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_LOC));
4276 // NOTE: No IsDistinct field for FUNC_CODE_DEBUG_LOC.
4277 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4278 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4279 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4280 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4281 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
4282 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Atom group.
4283 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 3)); // Atom rank.
4284 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4285 FUNCTION_DEBUG_LOC_ABBREV)
4286 llvm_unreachable("Unexpected abbrev ordering!");
4287 }
4288 Stream.ExitBlock();
4289}
4290
4291/// Write the module path strings, currently only used when generating
4292/// a combined index file.
4293void IndexBitcodeWriter::writeModStrings() {
4294 Stream.EnterSubblock(BlockID: bitc::MODULE_STRTAB_BLOCK_ID, CodeLen: 3);
4295
4296 // TODO: See which abbrev sizes we actually need to emit
4297
4298 // 8-bit fixed-width MST_ENTRY strings.
4299 auto Abbv = std::make_shared<BitCodeAbbrev>();
4300 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4301 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4302 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4303 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
4304 unsigned Abbrev8Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4305
4306 // 7-bit fixed width MST_ENTRY strings.
4307 Abbv = std::make_shared<BitCodeAbbrev>();
4308 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4309 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4310 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4311 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
4312 unsigned Abbrev7Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4313
4314 // 6-bit char6 MST_ENTRY strings.
4315 Abbv = std::make_shared<BitCodeAbbrev>();
4316 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4317 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4318 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4319 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
4320 unsigned Abbrev6Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4321
4322 // Module Hash, 160 bits SHA1. Optionally, emitted after each MST_CODE_ENTRY.
4323 Abbv = std::make_shared<BitCodeAbbrev>();
4324 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_HASH));
4325 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4326 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4327 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4328 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4329 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4330 unsigned AbbrevHash = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4331
4332 SmallVector<unsigned, 64> Vals;
4333 forEachModule(Callback: [&](const StringMapEntry<ModuleHash> &MPSE) {
4334 StringRef Key = MPSE.getKey();
4335 const auto &Hash = MPSE.getValue();
4336 StringEncoding Bits = getStringEncoding(Str: Key);
4337 unsigned AbbrevToUse = Abbrev8Bit;
4338 if (Bits == SE_Char6)
4339 AbbrevToUse = Abbrev6Bit;
4340 else if (Bits == SE_Fixed7)
4341 AbbrevToUse = Abbrev7Bit;
4342
4343 auto ModuleId = ModuleIdMap.size();
4344 ModuleIdMap[Key] = ModuleId;
4345 Vals.push_back(Elt: ModuleId);
4346 // Use bytes_begin/end() for unsigned char iteration.
4347 Vals.append(in_start: Key.bytes_begin(), in_end: Key.bytes_end());
4348
4349 // Emit the finished record.
4350 Stream.EmitRecord(Code: bitc::MST_CODE_ENTRY, Vals, Abbrev: AbbrevToUse);
4351
4352 // Emit an optional hash for the module now
4353 if (llvm::any_of(Range: Hash, P: [](uint32_t H) { return H; })) {
4354 Vals.assign(in_start: Hash.begin(), in_end: Hash.end());
4355 // Emit the hash record.
4356 Stream.EmitRecord(Code: bitc::MST_CODE_HASH, Vals, Abbrev: AbbrevHash);
4357 }
4358
4359 Vals.clear();
4360 });
4361 Stream.ExitBlock();
4362}
4363
4364/// Write the function type metadata related records that need to appear before
4365/// a function summary entry (whether per-module or combined).
4366template <typename Fn>
4367static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream,
4368 FunctionSummary *FS,
4369 Fn GetValueID) {
4370 if (!FS->type_tests().empty())
4371 Stream.EmitRecord(Code: bitc::FS_TYPE_TESTS, Vals: FS->type_tests());
4372
4373 SmallVector<uint64_t, 64> Record;
4374
4375 auto WriteVFuncIdVec = [&](uint64_t Ty,
4376 ArrayRef<FunctionSummary::VFuncId> VFs) {
4377 if (VFs.empty())
4378 return;
4379 Record.clear();
4380 for (auto &VF : VFs) {
4381 Record.push_back(Elt: VF.GUID);
4382 Record.push_back(Elt: VF.Offset);
4383 }
4384 Stream.EmitRecord(Code: Ty, Vals: Record);
4385 };
4386
4387 WriteVFuncIdVec(bitc::FS_TYPE_TEST_ASSUME_VCALLS,
4388 FS->type_test_assume_vcalls());
4389 WriteVFuncIdVec(bitc::FS_TYPE_CHECKED_LOAD_VCALLS,
4390 FS->type_checked_load_vcalls());
4391
4392 auto WriteConstVCallVec = [&](uint64_t Ty,
4393 ArrayRef<FunctionSummary::ConstVCall> VCs) {
4394 for (auto &VC : VCs) {
4395 Record.clear();
4396 Record.push_back(Elt: VC.VFunc.GUID);
4397 Record.push_back(Elt: VC.VFunc.Offset);
4398 llvm::append_range(C&: Record, R: VC.Args);
4399 Stream.EmitRecord(Code: Ty, Vals: Record);
4400 }
4401 };
4402
4403 WriteConstVCallVec(bitc::FS_TYPE_TEST_ASSUME_CONST_VCALL,
4404 FS->type_test_assume_const_vcalls());
4405 WriteConstVCallVec(bitc::FS_TYPE_CHECKED_LOAD_CONST_VCALL,
4406 FS->type_checked_load_const_vcalls());
4407
4408 auto WriteRange = [&](ConstantRange Range) {
4409 Range = Range.sextOrTrunc(BitWidth: FunctionSummary::ParamAccess::RangeWidth);
4410 assert(Range.getLower().getNumWords() == 1);
4411 assert(Range.getUpper().getNumWords() == 1);
4412 emitSignedInt64(Vals&: Record, V: *Range.getLower().getRawData());
4413 emitSignedInt64(Vals&: Record, V: *Range.getUpper().getRawData());
4414 };
4415
4416 if (!FS->paramAccesses().empty()) {
4417 Record.clear();
4418 for (auto &Arg : FS->paramAccesses()) {
4419 size_t UndoSize = Record.size();
4420 Record.push_back(Elt: Arg.ParamNo);
4421 WriteRange(Arg.Use);
4422 Record.push_back(Elt: Arg.Calls.size());
4423 for (auto &Call : Arg.Calls) {
4424 Record.push_back(Elt: Call.ParamNo);
4425 std::optional<unsigned> ValueID = GetValueID(Call.Callee);
4426 if (!ValueID) {
4427 // If ValueID is unknown we can't drop just this call, we must drop
4428 // entire parameter.
4429 Record.resize(N: UndoSize);
4430 break;
4431 }
4432 Record.push_back(Elt: *ValueID);
4433 WriteRange(Call.Offsets);
4434 }
4435 }
4436 if (!Record.empty())
4437 Stream.EmitRecord(Code: bitc::FS_PARAM_ACCESS, Vals: Record);
4438 }
4439}
4440
4441/// Collect type IDs from type tests used by function.
4442static void
4443getReferencedTypeIds(FunctionSummary *FS,
4444 std::set<GlobalValue::GUID> &ReferencedTypeIds) {
4445 if (!FS->type_tests().empty())
4446 for (auto &TT : FS->type_tests())
4447 ReferencedTypeIds.insert(x: TT);
4448
4449 auto GetReferencedTypesFromVFuncIdVec =
4450 [&](ArrayRef<FunctionSummary::VFuncId> VFs) {
4451 for (auto &VF : VFs)
4452 ReferencedTypeIds.insert(x: VF.GUID);
4453 };
4454
4455 GetReferencedTypesFromVFuncIdVec(FS->type_test_assume_vcalls());
4456 GetReferencedTypesFromVFuncIdVec(FS->type_checked_load_vcalls());
4457
4458 auto GetReferencedTypesFromConstVCallVec =
4459 [&](ArrayRef<FunctionSummary::ConstVCall> VCs) {
4460 for (auto &VC : VCs)
4461 ReferencedTypeIds.insert(x: VC.VFunc.GUID);
4462 };
4463
4464 GetReferencedTypesFromConstVCallVec(FS->type_test_assume_const_vcalls());
4465 GetReferencedTypesFromConstVCallVec(FS->type_checked_load_const_vcalls());
4466}
4467
4468static void writeWholeProgramDevirtResolutionByArg(
4469 SmallVector<uint64_t, 64> &NameVals, const std::vector<uint64_t> &args,
4470 const WholeProgramDevirtResolution::ByArg &ByArg) {
4471 NameVals.push_back(Elt: args.size());
4472 llvm::append_range(C&: NameVals, R: args);
4473
4474 NameVals.push_back(Elt: ByArg.TheKind);
4475 NameVals.push_back(Elt: ByArg.Info);
4476 NameVals.push_back(Elt: ByArg.Byte);
4477 NameVals.push_back(Elt: ByArg.Bit);
4478}
4479
4480static void writeWholeProgramDevirtResolution(
4481 SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
4482 uint64_t Id, const WholeProgramDevirtResolution &Wpd) {
4483 NameVals.push_back(Elt: Id);
4484
4485 NameVals.push_back(Elt: Wpd.TheKind);
4486 NameVals.push_back(Elt: StrtabBuilder.add(S: Wpd.SingleImplName));
4487 NameVals.push_back(Elt: Wpd.SingleImplName.size());
4488
4489 NameVals.push_back(Elt: Wpd.ResByArg.size());
4490 for (auto &A : Wpd.ResByArg)
4491 writeWholeProgramDevirtResolutionByArg(NameVals, args: A.first, ByArg: A.second);
4492}
4493
4494static void writeTypeIdSummaryRecord(SmallVector<uint64_t, 64> &NameVals,
4495 StringTableBuilder &StrtabBuilder,
4496 StringRef Id,
4497 const TypeIdSummary &Summary) {
4498 NameVals.push_back(Elt: StrtabBuilder.add(S: Id));
4499 NameVals.push_back(Elt: Id.size());
4500
4501 NameVals.push_back(Elt: Summary.TTRes.TheKind);
4502 NameVals.push_back(Elt: Summary.TTRes.SizeM1BitWidth);
4503 NameVals.push_back(Elt: Summary.TTRes.AlignLog2);
4504 NameVals.push_back(Elt: Summary.TTRes.SizeM1);
4505 NameVals.push_back(Elt: Summary.TTRes.BitMask);
4506 NameVals.push_back(Elt: Summary.TTRes.InlineBits);
4507
4508 for (auto &W : Summary.WPDRes)
4509 writeWholeProgramDevirtResolution(NameVals, StrtabBuilder, Id: W.first,
4510 Wpd: W.second);
4511}
4512
4513static void writeTypeIdCompatibleVtableSummaryRecord(
4514 SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
4515 StringRef Id, const TypeIdCompatibleVtableInfo &Summary,
4516 ValueEnumerator &VE) {
4517 NameVals.push_back(Elt: StrtabBuilder.add(S: Id));
4518 NameVals.push_back(Elt: Id.size());
4519
4520 for (auto &P : Summary) {
4521 NameVals.push_back(Elt: P.AddressPointOffset);
4522 NameVals.push_back(Elt: VE.getValueID(V: P.VTableVI.getValue()));
4523 }
4524}
4525
4526// Adds the allocation contexts to the CallStacks map. We simply use the
4527// size at the time the context was added as the CallStackId. This works because
4528// when we look up the call stacks later on we process the function summaries
4529// and their allocation records in the same exact order.
4530static void collectMemProfCallStacks(
4531 FunctionSummary *FS, std::function<LinearFrameId(unsigned)> GetStackIndex,
4532 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> &CallStacks) {
4533 // The interfaces in ProfileData/MemProf.h use a type alias for a stack frame
4534 // id offset into the index of the full stack frames. The ModuleSummaryIndex
4535 // currently uses unsigned. Make sure these stay in sync.
4536 static_assert(std::is_same_v<LinearFrameId, unsigned>);
4537 for (auto &AI : FS->allocs()) {
4538 for (auto &MIB : AI.MIBs) {
4539 SmallVector<unsigned> StackIdIndices;
4540 StackIdIndices.reserve(N: MIB.StackIdIndices.size());
4541 for (auto Id : MIB.StackIdIndices)
4542 StackIdIndices.push_back(Elt: GetStackIndex(Id));
4543 // The CallStackId is the size at the time this context was inserted.
4544 CallStacks.insert(KV: {CallStacks.size(), StackIdIndices});
4545 }
4546 }
4547}
4548
4549// Build the radix tree from the accumulated CallStacks, write out the resulting
4550// linearized radix tree array, and return the map of call stack positions into
4551// this array for use when writing the allocation records. The returned map is
4552// indexed by a CallStackId which in this case is implicitly determined by the
4553// order of function summaries and their allocation infos being written.
4554static DenseMap<CallStackId, LinearCallStackId> writeMemoryProfileRadixTree(
4555 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> &&CallStacks,
4556 BitstreamWriter &Stream, unsigned RadixAbbrev) {
4557 assert(!CallStacks.empty());
4558 DenseMap<unsigned, FrameStat> FrameHistogram =
4559 computeFrameHistogram<LinearFrameId>(MemProfCallStackData&: CallStacks);
4560 CallStackRadixTreeBuilder<LinearFrameId> Builder;
4561 // We don't need a MemProfFrameIndexes map as we have already converted the
4562 // full stack id hash to a linear offset into the StackIds array.
4563 Builder.build(MemProfCallStackData: std::move(CallStacks), /*MemProfFrameIndexes=*/nullptr,
4564 FrameHistogram);
4565 Stream.EmitRecord(Code: bitc::FS_CONTEXT_RADIX_TREE_ARRAY, Vals: Builder.getRadixArray(),
4566 Abbrev: RadixAbbrev);
4567 return Builder.takeCallStackPos();
4568}
4569
4570static void writeFunctionHeapProfileRecords(
4571 BitstreamWriter &Stream, FunctionSummary *FS, unsigned CallsiteAbbrev,
4572 unsigned AllocAbbrev, unsigned ContextIdAbbvId, bool PerModule,
4573 std::function<unsigned(const ValueInfo &VI)> GetValueID,
4574 std::function<unsigned(unsigned)> GetStackIndex,
4575 bool WriteContextSizeInfoIndex,
4576 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
4577 CallStackId &CallStackCount) {
4578 SmallVector<uint64_t> Record;
4579
4580 for (auto &CI : FS->callsites()) {
4581 Record.clear();
4582 // Per module callsite clones should always have a single entry of
4583 // value 0.
4584 assert(!PerModule || (CI.Clones.size() == 1 && CI.Clones[0] == 0));
4585 Record.push_back(Elt: GetValueID(CI.Callee));
4586 if (!PerModule) {
4587 Record.push_back(Elt: CI.StackIdIndices.size());
4588 Record.push_back(Elt: CI.Clones.size());
4589 }
4590 for (auto Id : CI.StackIdIndices)
4591 Record.push_back(Elt: GetStackIndex(Id));
4592 if (!PerModule)
4593 llvm::append_range(C&: Record, R: CI.Clones);
4594 Stream.EmitRecord(Code: PerModule ? bitc::FS_PERMODULE_CALLSITE_INFO
4595 : bitc::FS_COMBINED_CALLSITE_INFO,
4596 Vals: Record, Abbrev: CallsiteAbbrev);
4597 }
4598
4599 for (auto &AI : FS->allocs()) {
4600 Record.clear();
4601 // Per module alloc versions should always have a single entry of
4602 // value 0.
4603 assert(!PerModule || (AI.Versions.size() == 1 && AI.Versions[0] == 0));
4604 Record.push_back(Elt: AI.MIBs.size());
4605 if (!PerModule)
4606 Record.push_back(Elt: AI.Versions.size());
4607 for (auto &MIB : AI.MIBs) {
4608 Record.push_back(Elt: (uint8_t)MIB.AllocType);
4609 // The per-module summary always needs to include the alloc context, as we
4610 // use it during the thin link. For the combined index it is optional (see
4611 // comments where CombinedIndexMemProfContext is defined).
4612 if (PerModule || CombinedIndexMemProfContext) {
4613 // Record the index into the radix tree array for this context.
4614 assert(CallStackCount <= CallStackPos.size());
4615 Record.push_back(Elt: CallStackPos[CallStackCount++]);
4616 }
4617 }
4618 if (!PerModule)
4619 llvm::append_range(C&: Record, R: AI.Versions);
4620 assert(AI.ContextSizeInfos.empty() ||
4621 AI.ContextSizeInfos.size() == AI.MIBs.size());
4622 // Optionally emit the context size information if it exists.
4623 if (WriteContextSizeInfoIndex && !AI.ContextSizeInfos.empty()) {
4624 // The abbreviation id for the context ids record should have been created
4625 // if we are emitting the per-module index, which is where we write this
4626 // info.
4627 assert(ContextIdAbbvId);
4628 SmallVector<uint32_t> ContextIds;
4629 // At least one context id per ContextSizeInfos entry (MIB), broken into 2
4630 // halves.
4631 ContextIds.reserve(N: AI.ContextSizeInfos.size() * 2);
4632 for (auto &Infos : AI.ContextSizeInfos) {
4633 Record.push_back(Elt: Infos.size());
4634 for (auto [FullStackId, TotalSize] : Infos) {
4635 // The context ids are emitted separately as a fixed width array,
4636 // which is more efficient than a VBR given that these hashes are
4637 // typically close to 64-bits. The max fixed width entry is 32 bits so
4638 // it is split into 2.
4639 ContextIds.push_back(Elt: static_cast<uint32_t>(FullStackId >> 32));
4640 ContextIds.push_back(Elt: static_cast<uint32_t>(FullStackId));
4641 Record.push_back(Elt: TotalSize);
4642 }
4643 }
4644 // The context ids are expected by the reader to immediately precede the
4645 // associated alloc info record.
4646 Stream.EmitRecord(Code: bitc::FS_ALLOC_CONTEXT_IDS, Vals: ContextIds,
4647 Abbrev: ContextIdAbbvId);
4648 }
4649 Stream.EmitRecord(Code: PerModule
4650 ? bitc::FS_PERMODULE_ALLOC_INFO
4651 : (CombinedIndexMemProfContext
4652 ? bitc::FS_COMBINED_ALLOC_INFO
4653 : bitc::FS_COMBINED_ALLOC_INFO_NO_CONTEXT),
4654 Vals: Record, Abbrev: AllocAbbrev);
4655 }
4656}
4657
4658// Helper to emit a single function summary record.
4659void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
4660 SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
4661 unsigned ValueID, unsigned FSCallsProfileAbbrev, unsigned CallsiteAbbrev,
4662 unsigned AllocAbbrev, unsigned ContextIdAbbvId, const Function &F,
4663 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
4664 CallStackId &CallStackCount) {
4665 NameVals.push_back(Elt: ValueID);
4666
4667 FunctionSummary *FS = cast<FunctionSummary>(Val: Summary);
4668
4669 writeFunctionTypeMetadataRecords(
4670 Stream, FS, GetValueID: [&](const ValueInfo &VI) -> std::optional<unsigned> {
4671 return {VE.getValueID(V: VI.getValue())};
4672 });
4673
4674 auto SpecialRefCnts = FS->specialRefCounts();
4675 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: FS->flags()));
4676 NameVals.push_back(Elt: FS->instCount());
4677 NameVals.push_back(Elt: getEncodedFFlags(Flags: FS->fflags()));
4678 NameVals.push_back(Elt: FS->refs().size());
4679 NameVals.push_back(Elt: SpecialRefCnts.first); // rorefcnt
4680 NameVals.push_back(Elt: SpecialRefCnts.second); // worefcnt
4681
4682 for (auto &RI : FS->refs())
4683 NameVals.push_back(Elt: getValueId(VI: RI));
4684
4685 for (auto &ECI : FS->calls()) {
4686 NameVals.push_back(Elt: getValueId(VI: ECI.first));
4687 NameVals.push_back(Elt: getEncodedHotnessCallEdgeInfo(CI: ECI.second));
4688 }
4689
4690 // Emit the finished record.
4691 Stream.EmitRecord(Code: bitc::FS_PERMODULE_PROFILE, Vals: NameVals, Abbrev: FSCallsProfileAbbrev);
4692 NameVals.clear();
4693
4694 writeFunctionHeapProfileRecords(
4695 Stream, FS, CallsiteAbbrev, AllocAbbrev, ContextIdAbbvId,
4696 /*PerModule*/ true,
4697 /*GetValueId*/ GetValueID: [&](const ValueInfo &VI) { return getValueId(VI); },
4698 /*GetStackIndex*/ [&](unsigned I) { return I; },
4699 /*WriteContextSizeInfoIndex*/ true, CallStackPos, CallStackCount);
4700}
4701
4702// Collect the global value references in the given variable's initializer,
4703// and emit them in a summary record.
4704void ModuleBitcodeWriterBase::writeModuleLevelReferences(
4705 const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals,
4706 unsigned FSModRefsAbbrev, unsigned FSModVTableRefsAbbrev) {
4707 // Be a little lenient here, to accomodate older files without GUIDs
4708 // already computed and assigned as metadata.
4709 GlobalValue::GUID GUID = V.getGUIDOrFallback();
4710
4711 auto VI = Index->getValueInfo(GUID);
4712 if (!VI || VI.getSummaryList().empty()) {
4713 // Only declarations should not have a summary (a declaration might however
4714 // have a summary if the def was in module level asm).
4715 assert(V.isDeclaration());
4716 return;
4717 }
4718 auto *Summary = VI.getSummaryList()[0].get();
4719 NameVals.push_back(Elt: VE.getValueID(V: &V));
4720 GlobalVarSummary *VS = cast<GlobalVarSummary>(Val: Summary);
4721 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: VS->flags()));
4722 NameVals.push_back(Elt: getEncodedGVarFlags(Flags: VS->varflags()));
4723
4724 auto VTableFuncs = VS->vTableFuncs();
4725 if (!VTableFuncs.empty())
4726 NameVals.push_back(Elt: VS->refs().size());
4727
4728 unsigned SizeBeforeRefs = NameVals.size();
4729 for (auto &RI : VS->refs())
4730 NameVals.push_back(Elt: VE.getValueID(V: RI.getValue()));
4731 // Sort the refs for determinism output, the vector returned by FS->refs() has
4732 // been initialized from a DenseSet.
4733 llvm::sort(C: drop_begin(RangeOrContainer&: NameVals, N: SizeBeforeRefs));
4734
4735 if (VTableFuncs.empty())
4736 Stream.EmitRecord(Code: bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, Vals: NameVals,
4737 Abbrev: FSModRefsAbbrev);
4738 else {
4739 // VTableFuncs pairs should already be sorted by offset.
4740 for (auto &P : VTableFuncs) {
4741 NameVals.push_back(Elt: VE.getValueID(V: P.FuncVI.getValue()));
4742 NameVals.push_back(Elt: P.VTableOffset);
4743 }
4744
4745 Stream.EmitRecord(Code: bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS, Vals: NameVals,
4746 Abbrev: FSModVTableRefsAbbrev);
4747 }
4748 NameVals.clear();
4749}
4750
4751/// Emit the per-module summary section alongside the rest of
4752/// the module's bitcode.
4753void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
4754 // By default we compile with ThinLTO if the module has a summary, but the
4755 // client can request full LTO with a module flag.
4756 bool IsThinLTO = true;
4757 if (auto *MD =
4758 mdconst::extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: "ThinLTO")))
4759 IsThinLTO = MD->getZExtValue();
4760 Stream.EnterSubblock(BlockID: IsThinLTO ? bitc::GLOBALVAL_SUMMARY_BLOCK_ID
4761 : bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID,
4762 CodeLen: 4);
4763
4764 Stream.EmitRecord(
4765 Code: bitc::FS_VERSION,
4766 Vals: ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
4767
4768 // Write the index flags.
4769 uint64_t Flags = 0;
4770 // Bits 1-3 are set only in the combined index, skip them.
4771 if (Index->enableSplitLTOUnit())
4772 Flags |= 0x8;
4773 if (Index->hasUnifiedLTO())
4774 Flags |= 0x200;
4775
4776 Stream.EmitRecord(Code: bitc::FS_FLAGS, Vals: ArrayRef<uint64_t>{Flags});
4777
4778 if (Index->begin() == Index->end()) {
4779 Stream.ExitBlock();
4780 return;
4781 }
4782
4783 auto Abbv = std::make_shared<BitCodeAbbrev>();
4784 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_VALUE_GUID));
4785 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4786 // GUIDS often use up most of 64-bits, so encode as two Fixed 32.
4787 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4788 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4789 unsigned ValueGuidAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4790
4791 for (const auto &GVI : valueIds()) {
4792 Stream.EmitRecord(Code: bitc::FS_VALUE_GUID,
4793 Vals: ArrayRef<uint32_t>{GVI.second,
4794 static_cast<uint32_t>(GVI.first >> 32),
4795 static_cast<uint32_t>(GVI.first)},
4796 Abbrev: ValueGuidAbbrev);
4797 }
4798
4799 if (!Index->stackIds().empty()) {
4800 auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4801 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_STACK_IDS));
4802 // numids x stackid
4803 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4804 // The stack ids are hashes that are close to 64 bits in size, so emitting
4805 // as a pair of 32-bit fixed-width values is more efficient than a VBR.
4806 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4807 unsigned StackIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(StackIdAbbv));
4808 SmallVector<uint32_t> Vals;
4809 Vals.reserve(N: Index->stackIds().size() * 2);
4810 for (auto Id : Index->stackIds()) {
4811 Vals.push_back(Elt: static_cast<uint32_t>(Id >> 32));
4812 Vals.push_back(Elt: static_cast<uint32_t>(Id));
4813 }
4814 Stream.EmitRecord(Code: bitc::FS_STACK_IDS, Vals, Abbrev: StackIdAbbvId);
4815 }
4816
4817 unsigned ContextIdAbbvId = 0;
4818 if (metadataMayIncludeContextSizeInfo()) {
4819 // n x context id
4820 auto ContextIdAbbv = std::make_shared<BitCodeAbbrev>();
4821 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_ALLOC_CONTEXT_IDS));
4822 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4823 // The context ids are hashes that are close to 64 bits in size, so emitting
4824 // as a pair of 32-bit fixed-width values is more efficient than a VBR if we
4825 // are emitting them for all MIBs. Otherwise we use VBR to better compress 0
4826 // values that are expected to more frequently occur in an alloc's memprof
4827 // summary.
4828 if (metadataIncludesAllContextSizeInfo())
4829 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4830 else
4831 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4832 ContextIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(ContextIdAbbv));
4833 }
4834
4835 // Abbrev for FS_PERMODULE_PROFILE.
4836 Abbv = std::make_shared<BitCodeAbbrev>();
4837 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE));
4838 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4839 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // flags
4840 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
4841 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
4842 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4843 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
4844 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
4845 // numrefs x valueid, n x (valueid, hotness+tailcall flags)
4846 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4847 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4848 unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4849
4850 // Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS.
4851 Abbv = std::make_shared<BitCodeAbbrev>();
4852 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS));
4853 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4854 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4855 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
4856 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4857 unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4858
4859 // Abbrev for FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS.
4860 Abbv = std::make_shared<BitCodeAbbrev>();
4861 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS));
4862 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4863 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4864 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4865 // numrefs x valueid, n x (valueid , offset)
4866 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4867 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4868 unsigned FSModVTableRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4869
4870 // Abbrev for FS_ALIAS.
4871 Abbv = std::make_shared<BitCodeAbbrev>();
4872 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_ALIAS));
4873 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4874 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4875 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4876 unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4877
4878 // Abbrev for FS_TYPE_ID_METADATA
4879 Abbv = std::make_shared<BitCodeAbbrev>();
4880 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_TYPE_ID_METADATA));
4881 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid strtab index
4882 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid length
4883 // n x (valueid , offset)
4884 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4885 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4886 unsigned TypeIdCompatibleVtableAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4887
4888 Abbv = std::make_shared<BitCodeAbbrev>();
4889 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_CALLSITE_INFO));
4890 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4891 // n x stackidindex
4892 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4893 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4894 unsigned CallsiteAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4895
4896 Abbv = std::make_shared<BitCodeAbbrev>();
4897 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_ALLOC_INFO));
4898 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib
4899 // n x (alloc type, context radix tree index)
4900 // optional: nummib x (numcontext x total size)
4901 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4902 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4903 unsigned AllocAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4904
4905 Abbv = std::make_shared<BitCodeAbbrev>();
4906 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_CONTEXT_RADIX_TREE_ARRAY));
4907 // n x entry
4908 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4909 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4910 unsigned RadixAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4911
4912 // First walk through all the functions and collect the allocation contexts in
4913 // their associated summaries, for use in constructing a radix tree of
4914 // contexts. Note that we need to do this in the same order as the functions
4915 // are processed further below since the call stack positions in the resulting
4916 // radix tree array are identified based on this order.
4917 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> CallStacks;
4918 for (const Function &F : M) {
4919 // Summary emission does not support anonymous functions, they have to be
4920 // renamed using the anonymous function renaming pass.
4921 if (!F.hasName())
4922 report_fatal_error(reason: "Unexpected anonymous function when writing summary");
4923
4924 // Be a little lenient here, to accomodate older files without GUIDs
4925 // already computed and assigned as metadata.
4926 GlobalValue::GUID GUID = F.getGUIDOrFallback();
4927
4928 ValueInfo VI = Index->getValueInfo(GUID);
4929 if (!VI || VI.getSummaryList().empty()) {
4930 // Only declarations should not have a summary (a declaration might
4931 // however have a summary if the def was in module level asm).
4932 if (!F.isDeclaration())
4933 reportFatalUsageError(reason: "expected function definition " + F.getName() +
4934 " to have an associated value info.");
4935 continue;
4936 }
4937 auto *Summary = VI.getSummaryList()[0].get();
4938 FunctionSummary *FS = cast<FunctionSummary>(Val: Summary);
4939 collectMemProfCallStacks(
4940 FS, /*GetStackIndex*/ [](unsigned I) { return I; }, CallStacks);
4941 }
4942 // Finalize the radix tree, write it out, and get the map of positions in the
4943 // linearized tree array.
4944 DenseMap<CallStackId, LinearCallStackId> CallStackPos;
4945 if (!CallStacks.empty()) {
4946 CallStackPos =
4947 writeMemoryProfileRadixTree(CallStacks: std::move(CallStacks), Stream, RadixAbbrev);
4948 }
4949
4950 // Keep track of the current index into the CallStackPos map.
4951 CallStackId CallStackCount = 0;
4952
4953 SmallVector<uint64_t, 64> NameVals;
4954 // Iterate over the list of functions instead of the Index to
4955 // ensure the ordering is stable.
4956 for (const Function &F : M) {
4957 // Summary emission does not support anonymous functions, they have to
4958 // renamed using the anonymous function renaming pass.
4959 if (!F.hasName())
4960 report_fatal_error(reason: "Unexpected anonymous function when writing summary");
4961
4962 GlobalValue::GUID GUID = F.getGUIDOrFallback();
4963
4964 ValueInfo VI = Index->getValueInfo(GUID);
4965 if (!VI || VI.getSummaryList().empty()) {
4966 // Only declarations should not have a summary (a declaration might
4967 // however have a summary if the def was in module level asm).
4968 assert(F.isDeclaration());
4969 continue;
4970 }
4971 auto *Summary = VI.getSummaryList()[0].get();
4972 writePerModuleFunctionSummaryRecord(NameVals, Summary, ValueID: VE.getValueID(V: &F),
4973 FSCallsProfileAbbrev, CallsiteAbbrev,
4974 AllocAbbrev, ContextIdAbbvId, F,
4975 CallStackPos, CallStackCount);
4976 }
4977
4978 // Capture references from GlobalVariable initializers, which are outside
4979 // of a function scope.
4980 for (const GlobalVariable &G : M.globals())
4981 writeModuleLevelReferences(V: G, NameVals, FSModRefsAbbrev,
4982 FSModVTableRefsAbbrev);
4983
4984 for (const GlobalAlias &A : M.aliases()) {
4985 auto *Aliasee = A.getAliaseeObject();
4986 // Skip ifunc and nameless functions which don't have an entry in the
4987 // summary.
4988 if (!Aliasee->hasName() || isa<GlobalIFunc>(Val: Aliasee))
4989 continue;
4990 auto AliasId = VE.getValueID(V: &A);
4991 auto AliaseeId = VE.getValueID(V: Aliasee);
4992 NameVals.push_back(Elt: AliasId);
4993 auto *Summary = Index->getGlobalValueSummary(GV: A);
4994 AliasSummary *AS = cast<AliasSummary>(Val: Summary);
4995 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: AS->flags()));
4996 NameVals.push_back(Elt: AliaseeId);
4997 Stream.EmitRecord(Code: bitc::FS_ALIAS, Vals: NameVals, Abbrev: FSAliasAbbrev);
4998 NameVals.clear();
4999 }
5000
5001 for (auto &S : Index->typeIdCompatibleVtableMap()) {
5002 writeTypeIdCompatibleVtableSummaryRecord(NameVals, StrtabBuilder, Id: S.first,
5003 Summary: S.second, VE);
5004 Stream.EmitRecord(Code: bitc::FS_TYPE_ID_METADATA, Vals: NameVals,
5005 Abbrev: TypeIdCompatibleVtableAbbrev);
5006 NameVals.clear();
5007 }
5008
5009 if (Index->getBlockCount())
5010 Stream.EmitRecord(Code: bitc::FS_BLOCK_COUNT,
5011 Vals: ArrayRef<uint64_t>{Index->getBlockCount()});
5012
5013 Stream.ExitBlock();
5014}
5015
5016void ModuleBitcodeWriterBase::writeGUIDList() {
5017 const ValueEnumerator::ValueList &Vals = VE.getValues();
5018 const size_t Max = Vals.size();
5019
5020 std::vector<GlobalValue::GUID> GUIDs(Max, 0);
5021 for (const GlobalValue &GV : M.global_values()) {
5022 auto MaybeGUID = GV.getGUIDIfAssigned();
5023 if (!MaybeGUID)
5024 continue;
5025 auto GUID = *MaybeGUID;
5026
5027 const auto ValueID = VE.getValueID(V: &GV);
5028 GUIDs[ValueID] = GUID;
5029 }
5030
5031 auto Abbv = std::make_shared<BitCodeAbbrev>();
5032 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_GUIDLIST));
5033 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5034 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
5035 unsigned GUIDListAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5036
5037 SmallVector<uint32_t> RecordVals;
5038 RecordVals.reserve(N: Max * 2);
5039 for (auto GUID : GUIDs) {
5040 RecordVals.push_back(Elt: static_cast<uint32_t>(GUID >> 32));
5041 RecordVals.push_back(Elt: static_cast<uint32_t>(GUID));
5042 }
5043
5044 Stream.EmitRecord(Code: bitc::MODULE_CODE_GUIDLIST, Vals: RecordVals, Abbrev: GUIDListAbbrev);
5045}
5046
5047/// Emit the combined summary section into the combined index file.
5048void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
5049 Stream.EnterSubblock(BlockID: bitc::GLOBALVAL_SUMMARY_BLOCK_ID, CodeLen: 4);
5050 Stream.EmitRecord(
5051 Code: bitc::FS_VERSION,
5052 Vals: ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
5053
5054 // Write the index flags.
5055 Stream.EmitRecord(Code: bitc::FS_FLAGS, Vals: ArrayRef<uint64_t>{Index.getFlags()});
5056
5057 auto Abbv = std::make_shared<BitCodeAbbrev>();
5058 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_VALUE_GUID));
5059 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
5060 // GUIDS often use up most of 64-bits, so encode as two Fixed 32.
5061 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
5062 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
5063 unsigned ValueGuidAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5064
5065 for (const auto &GVI : valueIds()) {
5066 Stream.EmitRecord(Code: bitc::FS_VALUE_GUID,
5067 Vals: ArrayRef<uint32_t>{GVI.second,
5068 static_cast<uint32_t>(GVI.first >> 32),
5069 static_cast<uint32_t>(GVI.first)},
5070 Abbrev: ValueGuidAbbrev);
5071 }
5072
5073 // Write the stack ids used by this index, which will be a subset of those in
5074 // the full index in the case of distributed indexes.
5075 if (!StackIds.empty()) {
5076 auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
5077 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_STACK_IDS));
5078 // numids x stackid
5079 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5080 // The stack ids are hashes that are close to 64 bits in size, so emitting
5081 // as a pair of 32-bit fixed-width values is more efficient than a VBR.
5082 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
5083 unsigned StackIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(StackIdAbbv));
5084 SmallVector<uint32_t> Vals;
5085 Vals.reserve(N: StackIds.size() * 2);
5086 for (auto Id : StackIds) {
5087 Vals.push_back(Elt: static_cast<uint32_t>(Id >> 32));
5088 Vals.push_back(Elt: static_cast<uint32_t>(Id));
5089 }
5090 Stream.EmitRecord(Code: bitc::FS_STACK_IDS, Vals, Abbrev: StackIdAbbvId);
5091 }
5092
5093 // Abbrev for FS_COMBINED_PROFILE.
5094 Abbv = std::make_shared<BitCodeAbbrev>();
5095 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_PROFILE));
5096 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5097 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
5098 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
5099 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
5100 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
5101 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // entrycount
5102 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
5103 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
5104 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
5105 // numrefs x valueid, n x (valueid, hotness+tailcall flags)
5106 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5107 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5108 unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5109
5110 // Abbrev for FS_COMBINED_GLOBALVAR_INIT_REFS.
5111 Abbv = std::make_shared<BitCodeAbbrev>();
5112 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS));
5113 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5114 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
5115 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
5116 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
5117 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5118 unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5119
5120 // Abbrev for FS_COMBINED_ALIAS.
5121 Abbv = std::make_shared<BitCodeAbbrev>();
5122 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_ALIAS));
5123 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5124 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
5125 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
5126 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5127 unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5128
5129 Abbv = std::make_shared<BitCodeAbbrev>();
5130 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_CALLSITE_INFO));
5131 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5132 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numstackindices
5133 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
5134 // numstackindices x stackidindex, numver x version
5135 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5136 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5137 unsigned CallsiteAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5138
5139 Abbv = std::make_shared<BitCodeAbbrev>();
5140 Abbv->Add(OpInfo: BitCodeAbbrevOp(CombinedIndexMemProfContext
5141 ? bitc::FS_COMBINED_ALLOC_INFO
5142 : bitc::FS_COMBINED_ALLOC_INFO_NO_CONTEXT));
5143 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib
5144 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
5145 // nummib x (alloc type, context radix tree index),
5146 // numver x version
5147 // optional: nummib x total size
5148 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5149 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5150 unsigned AllocAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5151
5152 auto shouldImportValueAsDecl = [&](GlobalValueSummary *GVS) -> bool {
5153 if (DecSummaries == nullptr)
5154 return false;
5155 return DecSummaries->count(Ptr: GVS);
5156 };
5157
5158 // The aliases are emitted as a post-pass, and will point to the value
5159 // id of the aliasee. Save them in a vector for post-processing.
5160 SmallVector<AliasSummary *, 64> Aliases;
5161
5162 // Save the value id for each summary for alias emission.
5163 DenseMap<const GlobalValueSummary *, unsigned> SummaryToValueIdMap;
5164
5165 SmallVector<uint64_t, 64> NameVals;
5166
5167 // Set that will be populated during call to writeFunctionTypeMetadataRecords
5168 // with the type ids referenced by this index file.
5169 std::set<GlobalValue::GUID> ReferencedTypeIds;
5170
5171 // For local linkage, we also emit the original name separately
5172 // immediately after the record.
5173 auto MaybeEmitOriginalName = [&](GlobalValueSummary &S) {
5174 // We don't need to emit the original name if we are writing the index for
5175 // distributed backends (in which case ModuleToSummariesForIndex is
5176 // non-null). The original name is only needed during the thin link, since
5177 // for SamplePGO the indirect call targets for local functions have
5178 // have the original name annotated in profile.
5179 // Continue to emit it when writing out the entire combined index, which is
5180 // used in testing the thin link via llvm-lto.
5181 if (ModuleToSummariesForIndex || !GlobalValue::isLocalLinkage(Linkage: S.linkage()))
5182 return;
5183 NameVals.push_back(Elt: S.getOriginalName());
5184 Stream.EmitRecord(Code: bitc::FS_COMBINED_ORIGINAL_NAME, Vals: NameVals);
5185 NameVals.clear();
5186 };
5187
5188 DenseMap<CallStackId, LinearCallStackId> CallStackPos;
5189 if (CombinedIndexMemProfContext) {
5190 Abbv = std::make_shared<BitCodeAbbrev>();
5191 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_CONTEXT_RADIX_TREE_ARRAY));
5192 // n x entry
5193 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5194 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5195 unsigned RadixAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5196
5197 // First walk through all the functions and collect the allocation contexts
5198 // in their associated summaries, for use in constructing a radix tree of
5199 // contexts. Note that we need to do this in the same order as the functions
5200 // are processed further below since the call stack positions in the
5201 // resulting radix tree array are identified based on this order.
5202 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> CallStacks;
5203 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
5204 // Don't collect this when invoked for an aliasee, as it is not needed for
5205 // the alias summary. If the aliasee is to be imported, we will invoke
5206 // this separately with IsAliasee=false.
5207 if (IsAliasee)
5208 return;
5209 GlobalValueSummary *S = I.second;
5210 assert(S);
5211 auto *FS = dyn_cast<FunctionSummary>(Val: S);
5212 if (!FS)
5213 return;
5214 collectMemProfCallStacks(
5215 FS,
5216 /*GetStackIndex*/
5217 [&](unsigned I) {
5218 // Get the corresponding index into the list of StackIds actually
5219 // being written for this combined index (which may be a subset in
5220 // the case of distributed indexes).
5221 assert(StackIdIndicesToIndex.contains(I));
5222 return StackIdIndicesToIndex[I];
5223 },
5224 CallStacks);
5225 });
5226 // Finalize the radix tree, write it out, and get the map of positions in
5227 // the linearized tree array.
5228 if (!CallStacks.empty()) {
5229 CallStackPos = writeMemoryProfileRadixTree(CallStacks: std::move(CallStacks), Stream,
5230 RadixAbbrev);
5231 }
5232 }
5233
5234 // Keep track of the current index into the CallStackPos map. Not used if
5235 // CombinedIndexMemProfContext is false.
5236 CallStackId CallStackCount = 0;
5237
5238 DenseSet<GlobalValue::GUID> DefOrUseGUIDs;
5239 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
5240 GlobalValueSummary *S = I.second;
5241 assert(S);
5242 DefOrUseGUIDs.insert(V: I.first);
5243 for (const ValueInfo &VI : S->refs())
5244 DefOrUseGUIDs.insert(V: VI.getGUID());
5245
5246 auto ValueId = getValueId(ValGUID: I.first);
5247 assert(ValueId);
5248 SummaryToValueIdMap[S] = *ValueId;
5249
5250 // If this is invoked for an aliasee, we want to record the above
5251 // mapping, but then not emit a summary entry (if the aliasee is
5252 // to be imported, we will invoke this separately with IsAliasee=false).
5253 if (IsAliasee)
5254 return;
5255
5256 if (auto *AS = dyn_cast<AliasSummary>(Val: S)) {
5257 // Will process aliases as a post-pass because the reader wants all
5258 // global to be loaded first.
5259 Aliases.push_back(Elt: AS);
5260 return;
5261 }
5262
5263 if (auto *VS = dyn_cast<GlobalVarSummary>(Val: S)) {
5264 NameVals.push_back(Elt: *ValueId);
5265 assert(ModuleIdMap.count(VS->modulePath()));
5266 NameVals.push_back(Elt: ModuleIdMap[VS->modulePath()]);
5267 NameVals.push_back(
5268 Elt: getEncodedGVSummaryFlags(Flags: VS->flags(), ImportAsDecl: shouldImportValueAsDecl(VS)));
5269 NameVals.push_back(Elt: getEncodedGVarFlags(Flags: VS->varflags()));
5270 for (auto &RI : VS->refs()) {
5271 auto RefValueId = getValueId(ValGUID: RI.getGUID());
5272 if (!RefValueId)
5273 continue;
5274 NameVals.push_back(Elt: *RefValueId);
5275 }
5276
5277 // Emit the finished record.
5278 Stream.EmitRecord(Code: bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, Vals: NameVals,
5279 Abbrev: FSModRefsAbbrev);
5280 NameVals.clear();
5281 MaybeEmitOriginalName(*S);
5282 return;
5283 }
5284
5285 auto GetValueId = [&](const ValueInfo &VI) -> std::optional<unsigned> {
5286 if (!VI)
5287 return std::nullopt;
5288 return getValueId(ValGUID: VI.getGUID());
5289 };
5290
5291 auto *FS = cast<FunctionSummary>(Val: S);
5292 writeFunctionTypeMetadataRecords(Stream, FS, GetValueID: GetValueId);
5293 getReferencedTypeIds(FS, ReferencedTypeIds);
5294
5295 NameVals.push_back(Elt: *ValueId);
5296 assert(ModuleIdMap.count(FS->modulePath()));
5297 NameVals.push_back(Elt: ModuleIdMap[FS->modulePath()]);
5298 NameVals.push_back(
5299 Elt: getEncodedGVSummaryFlags(Flags: FS->flags(), ImportAsDecl: shouldImportValueAsDecl(FS)));
5300 NameVals.push_back(Elt: FS->instCount());
5301 NameVals.push_back(Elt: getEncodedFFlags(Flags: FS->fflags()));
5302 // TODO: Stop writing entry count and bump bitcode version.
5303 NameVals.push_back(Elt: 0 /* EntryCount */);
5304
5305 // Fill in below
5306 NameVals.push_back(Elt: 0); // numrefs
5307 NameVals.push_back(Elt: 0); // rorefcnt
5308 NameVals.push_back(Elt: 0); // worefcnt
5309
5310 unsigned Count = 0, RORefCnt = 0, WORefCnt = 0;
5311 for (auto &RI : FS->refs()) {
5312 auto RefValueId = getValueId(ValGUID: RI.getGUID());
5313 if (!RefValueId)
5314 continue;
5315 NameVals.push_back(Elt: *RefValueId);
5316 if (RI.isReadOnly())
5317 RORefCnt++;
5318 else if (RI.isWriteOnly())
5319 WORefCnt++;
5320 Count++;
5321 }
5322 NameVals[6] = Count;
5323 NameVals[7] = RORefCnt;
5324 NameVals[8] = WORefCnt;
5325
5326 for (auto &EI : FS->calls()) {
5327 // If this GUID doesn't have a value id, it doesn't have a function
5328 // summary and we don't need to record any calls to it.
5329 std::optional<unsigned> CallValueId = GetValueId(EI.first);
5330 if (!CallValueId)
5331 continue;
5332 NameVals.push_back(Elt: *CallValueId);
5333 NameVals.push_back(Elt: getEncodedHotnessCallEdgeInfo(CI: EI.second));
5334 }
5335
5336 // Emit the finished record.
5337 Stream.EmitRecord(Code: bitc::FS_COMBINED_PROFILE, Vals: NameVals,
5338 Abbrev: FSCallsProfileAbbrev);
5339 NameVals.clear();
5340
5341 writeFunctionHeapProfileRecords(
5342 Stream, FS, CallsiteAbbrev, AllocAbbrev, /*ContextIdAbbvId*/ 0,
5343 /*PerModule*/ false,
5344 /*GetValueId*/
5345 GetValueID: [&](const ValueInfo &VI) -> unsigned {
5346 std::optional<unsigned> ValueID = GetValueId(VI);
5347 // This can happen in shared index files for distributed ThinLTO if
5348 // the callee function summary is not included. Record 0 which we
5349 // will have to deal with conservatively when doing any kind of
5350 // validation in the ThinLTO backends.
5351 if (!ValueID)
5352 return 0;
5353 return *ValueID;
5354 },
5355 /*GetStackIndex*/
5356 [&](unsigned I) {
5357 // Get the corresponding index into the list of StackIds actually
5358 // being written for this combined index (which may be a subset in
5359 // the case of distributed indexes).
5360 assert(StackIdIndicesToIndex.contains(I));
5361 return StackIdIndicesToIndex[I];
5362 },
5363 /*WriteContextSizeInfoIndex*/ false, CallStackPos, CallStackCount);
5364
5365 MaybeEmitOriginalName(*S);
5366 });
5367
5368 for (auto *AS : Aliases) {
5369 auto AliasValueId = SummaryToValueIdMap[AS];
5370 assert(AliasValueId);
5371 NameVals.push_back(Elt: AliasValueId);
5372 assert(ModuleIdMap.count(AS->modulePath()));
5373 NameVals.push_back(Elt: ModuleIdMap[AS->modulePath()]);
5374 NameVals.push_back(
5375 Elt: getEncodedGVSummaryFlags(Flags: AS->flags(), ImportAsDecl: shouldImportValueAsDecl(AS)));
5376 // Set value id to 0 when an alias is imported but the aliasee summary is
5377 // not contained in the index.
5378 auto AliaseeValueId =
5379 AS->hasAliasee() ? SummaryToValueIdMap[&AS->getAliasee()] : 0;
5380 NameVals.push_back(Elt: AliaseeValueId);
5381
5382 // Emit the finished record.
5383 Stream.EmitRecord(Code: bitc::FS_COMBINED_ALIAS, Vals: NameVals, Abbrev: FSAliasAbbrev);
5384 NameVals.clear();
5385 MaybeEmitOriginalName(*AS);
5386
5387 if (AS->hasAliasee())
5388 if (auto *FS = dyn_cast<FunctionSummary>(Val: &AS->getAliasee()))
5389 getReferencedTypeIds(FS, ReferencedTypeIds);
5390 }
5391
5392 SmallVector<std::pair<StringRef, GlobalValue::GUID>, 4> Functions;
5393 auto EmitCfiFunctions = [&](const CfiFunctionIndex &CfiIndex,
5394 bitc::GlobalValueSummarySymtabCodes Code) {
5395 if (CfiIndex.empty())
5396 return;
5397 for (GlobalValue::GUID GUID : DefOrUseGUIDs) {
5398 auto Names = CfiIndex.getNamesForGUID(GUID);
5399 for (StringRef Name : Names)
5400 Functions.push_back(Elt: {Name, GUID});
5401 }
5402 if (Functions.empty())
5403 return;
5404 llvm::sort(C&: Functions);
5405 for (const auto &Record : Functions) {
5406 NameVals.push_back(Elt: Record.second);
5407 NameVals.push_back(Elt: StrtabBuilder.add(S: Record.first));
5408 NameVals.push_back(Elt: Record.first.size());
5409 }
5410 Stream.EmitRecord(Code, Vals: NameVals);
5411 NameVals.clear();
5412 Functions.clear();
5413 };
5414
5415 EmitCfiFunctions(Index.cfiFunctionDefs(), bitc::FS_CFI_FUNCTION_DEFS);
5416 EmitCfiFunctions(Index.cfiFunctionDecls(), bitc::FS_CFI_FUNCTION_DECLS);
5417
5418 // Walk the GUIDs that were referenced, and write the
5419 // corresponding type id records.
5420 for (auto &T : ReferencedTypeIds) {
5421 auto TidIter = Index.typeIds().equal_range(x: T);
5422 for (const auto &[GUID, TypeIdPair] : make_range(p: TidIter)) {
5423 writeTypeIdSummaryRecord(NameVals, StrtabBuilder, Id: TypeIdPair.first,
5424 Summary: TypeIdPair.second);
5425 Stream.EmitRecord(Code: bitc::FS_TYPE_ID, Vals: NameVals);
5426 NameVals.clear();
5427 }
5428 }
5429
5430 if (Index.getBlockCount())
5431 Stream.EmitRecord(Code: bitc::FS_BLOCK_COUNT,
5432 Vals: ArrayRef<uint64_t>{Index.getBlockCount()});
5433
5434 Stream.ExitBlock();
5435}
5436
5437/// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the
5438/// current llvm version, and a record for the epoch number.
5439static void writeIdentificationBlock(BitstreamWriter &Stream) {
5440 Stream.EnterSubblock(BlockID: bitc::IDENTIFICATION_BLOCK_ID, CodeLen: 5);
5441
5442 // Write the "user readable" string identifying the bitcode producer
5443 auto Abbv = std::make_shared<BitCodeAbbrev>();
5444 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_STRING));
5445 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5446 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
5447 auto StringAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5448 writeStringRecord(Stream, Code: bitc::IDENTIFICATION_CODE_STRING,
5449 Str: "LLVM" LLVM_VERSION_STRING, AbbrevToUse: StringAbbrev);
5450
5451 // Write the epoch version
5452 Abbv = std::make_shared<BitCodeAbbrev>();
5453 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_EPOCH));
5454 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
5455 auto EpochAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5456 constexpr std::array<unsigned, 1> Vals = {._M_elems: {bitc::BITCODE_CURRENT_EPOCH}};
5457 Stream.EmitRecord(Code: bitc::IDENTIFICATION_CODE_EPOCH, Vals, Abbrev: EpochAbbrev);
5458 Stream.ExitBlock();
5459}
5460
5461void ModuleBitcodeWriter::writeModuleHash(StringRef View) {
5462 // Emit the module's hash.
5463 // MODULE_CODE_HASH: [5*i32]
5464 if (GenerateHash) {
5465 uint32_t Vals[5];
5466 Hasher.update(Data: ArrayRef<uint8_t>(
5467 reinterpret_cast<const uint8_t *>(View.data()), View.size()));
5468 std::array<uint8_t, 20> Hash = Hasher.result();
5469 for (int Pos = 0; Pos < 20; Pos += 4) {
5470 Vals[Pos / 4] = support::endian::read32be(P: Hash.data() + Pos);
5471 }
5472
5473 // Emit the finished record.
5474 Stream.EmitRecord(Code: bitc::MODULE_CODE_HASH, Vals);
5475
5476 if (ModHash)
5477 // Save the written hash value.
5478 llvm::copy(Range&: Vals, Out: std::begin(cont&: *ModHash));
5479 }
5480}
5481
5482void ModuleBitcodeWriter::write() {
5483 writeIdentificationBlock(Stream);
5484
5485 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5486 // We will want to write the module hash at this point. Block any flushing so
5487 // we can have access to the whole underlying data later.
5488 Stream.markAndBlockFlushing();
5489
5490 writeModuleVersion();
5491
5492 // Emit blockinfo, which defines the standard abbreviations etc.
5493 writeBlockInfo();
5494
5495 // Emit information describing all of the types in the module.
5496 writeTypeTable();
5497
5498 // Emit information about attribute groups.
5499 writeAttributeGroupTable();
5500
5501 // Emit information about parameter attributes.
5502 writeAttributeTable();
5503
5504 writeComdats();
5505
5506 // Emit top-level description of module, including target triple, inline asm,
5507 // descriptors for global variables, and function prototype info.
5508 writeModuleInfo();
5509
5510 // Emit constants.
5511 writeModuleConstants();
5512
5513 // Emit metadata kind names.
5514 writeModuleMetadataKinds();
5515
5516 // Emit metadata.
5517 writeModuleMetadata();
5518
5519 // Emit module-level use-lists.
5520 if (VE.shouldPreserveUseListOrder())
5521 writeUseListBlock(F: nullptr);
5522
5523 writeOperandBundleTags();
5524 writeSyncScopeNames();
5525
5526 // Emit function bodies.
5527 DenseMap<const Function *, uint64_t> FunctionToBitcodeIndex;
5528 for (const Function &F : M)
5529 if (!F.isDeclaration())
5530 writeFunction(F, FunctionToBitcodeIndex);
5531
5532 // Need to write after the above call to WriteFunction which populates
5533 // the summary information in the index.
5534 if (Index)
5535 writePerModuleGlobalValueSummary();
5536
5537 writeGlobalValueSymbolTable(FunctionToBitcodeIndex);
5538
5539 writeModuleHash(View: Stream.getMarkedBufferAndResumeFlushing());
5540
5541 Stream.ExitBlock();
5542}
5543
5544static void writeInt32ToBuffer(uint32_t Value, SmallVectorImpl<char> &Buffer,
5545 uint32_t &Position) {
5546 support::endian::write32le(P: &Buffer[Position], V: Value);
5547 Position += 4;
5548}
5549
5550/// If generating a bc file on darwin, we have to emit a
5551/// header and trailer to make it compatible with the system archiver. To do
5552/// this we emit the following header, and then emit a trailer that pads the
5553/// file out to be a multiple of 16 bytes.
5554///
5555/// struct bc_header {
5556/// uint32_t Magic; // 0x0B17C0DE
5557/// uint32_t Version; // Version, currently always 0.
5558/// uint32_t BitcodeOffset; // Offset to traditional bitcode file.
5559/// uint32_t BitcodeSize; // Size of traditional bitcode file.
5560/// uint32_t CPUType; // CPU specifier.
5561/// ... potentially more later ...
5562/// };
5563static void emitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer,
5564 const Triple &TT) {
5565 unsigned CPUType = ~0U;
5566
5567 // Match x86_64-*, i[3-9]86-*, powerpc-*, powerpc64-*, arm-*, thumb-*,
5568 // armv[0-9]-*, thumbv[0-9]-*, armv5te-*, or armv6t2-*. The CPUType is a magic
5569 // number from /usr/include/mach/machine.h. It is ok to reproduce the
5570 // specific constants here because they are implicitly part of the Darwin ABI.
5571 enum {
5572 DARWIN_CPU_ARCH_ABI64 = 0x01000000,
5573 DARWIN_CPU_TYPE_X86 = 7,
5574 DARWIN_CPU_TYPE_ARM = 12,
5575 DARWIN_CPU_TYPE_POWERPC = 18
5576 };
5577
5578 Triple::ArchType Arch = TT.getArch();
5579 if (Arch == Triple::x86_64)
5580 CPUType = DARWIN_CPU_TYPE_X86 | DARWIN_CPU_ARCH_ABI64;
5581 else if (Arch == Triple::x86)
5582 CPUType = DARWIN_CPU_TYPE_X86;
5583 else if (Arch == Triple::ppc)
5584 CPUType = DARWIN_CPU_TYPE_POWERPC;
5585 else if (Arch == Triple::ppc64)
5586 CPUType = DARWIN_CPU_TYPE_POWERPC | DARWIN_CPU_ARCH_ABI64;
5587 else if (Arch == Triple::arm || Arch == Triple::thumb)
5588 CPUType = DARWIN_CPU_TYPE_ARM;
5589
5590 // Traditional Bitcode starts after header.
5591 assert(Buffer.size() >= BWH_HeaderSize &&
5592 "Expected header size to be reserved");
5593 unsigned BCOffset = BWH_HeaderSize;
5594 unsigned BCSize = Buffer.size() - BWH_HeaderSize;
5595
5596 // Write the magic and version.
5597 unsigned Position = 0;
5598 writeInt32ToBuffer(Value: 0x0B17C0DE, Buffer, Position);
5599 writeInt32ToBuffer(Value: 0, Buffer, Position); // Version.
5600 writeInt32ToBuffer(Value: BCOffset, Buffer, Position);
5601 writeInt32ToBuffer(Value: BCSize, Buffer, Position);
5602 writeInt32ToBuffer(Value: CPUType, Buffer, Position);
5603
5604 // If the file is not a multiple of 16 bytes, insert dummy padding.
5605 while (Buffer.size() & 15)
5606 Buffer.push_back(Elt: 0);
5607}
5608
5609/// Helper to write the header common to all bitcode files.
5610static void writeBitcodeHeader(BitstreamWriter &Stream) {
5611 // Emit the file header.
5612 Stream.Emit(Val: (unsigned)'B', NumBits: 8);
5613 Stream.Emit(Val: (unsigned)'C', NumBits: 8);
5614 Stream.Emit(Val: 0x0, NumBits: 4);
5615 Stream.Emit(Val: 0xC, NumBits: 4);
5616 Stream.Emit(Val: 0xE, NumBits: 4);
5617 Stream.Emit(Val: 0xD, NumBits: 4);
5618}
5619
5620BitcodeWriter::BitcodeWriter(SmallVectorImpl<char> &Buffer)
5621 : Stream(new BitstreamWriter(Buffer)) {
5622 writeBitcodeHeader(Stream&: *Stream);
5623}
5624
5625BitcodeWriter::BitcodeWriter(raw_ostream &FS)
5626 : Stream(new BitstreamWriter(FS, FlushThreshold)) {
5627 writeBitcodeHeader(Stream&: *Stream);
5628}
5629
5630BitcodeWriter::~BitcodeWriter() { assert(WroteStrtab); }
5631
5632void BitcodeWriter::writeBlob(unsigned Block, unsigned Record, StringRef Blob) {
5633 Stream->EnterSubblock(BlockID: Block, CodeLen: 3);
5634
5635 auto Abbv = std::make_shared<BitCodeAbbrev>();
5636 Abbv->Add(OpInfo: BitCodeAbbrevOp(Record));
5637 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
5638 auto AbbrevNo = Stream->EmitAbbrev(Abbv: std::move(Abbv));
5639
5640 Stream->EmitRecordWithBlob(Abbrev: AbbrevNo, Vals: ArrayRef<uint64_t>{Record}, Blob);
5641
5642 Stream->ExitBlock();
5643}
5644
5645void BitcodeWriter::writeSymtab() {
5646 assert(!WroteStrtab && !WroteSymtab);
5647
5648 // If any module has module-level inline asm, we will require a registered asm
5649 // parser for the target so that we can create an accurate symbol table for
5650 // the module.
5651 for (Module *M : Mods) {
5652 if (M->getModuleInlineAsm().empty())
5653 continue;
5654
5655 std::string Err;
5656 const Triple TT(M->getTargetTriple());
5657 const Target *T = TargetRegistry::lookupTarget(TheTriple: TT, Error&: Err);
5658 if (!T || !T->hasMCAsmParser())
5659 return;
5660 }
5661
5662 WroteSymtab = true;
5663 SmallVector<char, 0> Symtab;
5664 // The irsymtab::build function may be unable to create a symbol table if the
5665 // module is malformed (e.g. it contains an invalid alias). Writing a symbol
5666 // table is not required for correctness, but we still want to be able to
5667 // write malformed modules to bitcode files, so swallow the error.
5668 if (Error E = irsymtab::build(Mods, Symtab, StrtabBuilder, Alloc)) {
5669 consumeError(Err: std::move(E));
5670 return;
5671 }
5672
5673 writeBlob(Block: bitc::SYMTAB_BLOCK_ID, Record: bitc::SYMTAB_BLOB,
5674 Blob: {Symtab.data(), Symtab.size()});
5675}
5676
5677void BitcodeWriter::writeStrtab() {
5678 assert(!WroteStrtab);
5679
5680 std::vector<char> Strtab;
5681 StrtabBuilder.finalizeInOrder();
5682 Strtab.resize(new_size: StrtabBuilder.getSize());
5683 StrtabBuilder.write(Buf: (uint8_t *)Strtab.data());
5684
5685 writeBlob(Block: bitc::STRTAB_BLOCK_ID, Record: bitc::STRTAB_BLOB,
5686 Blob: {Strtab.data(), Strtab.size()});
5687
5688 WroteStrtab = true;
5689}
5690
5691void BitcodeWriter::copyStrtab(StringRef Strtab) {
5692 writeBlob(Block: bitc::STRTAB_BLOCK_ID, Record: bitc::STRTAB_BLOB, Blob: Strtab);
5693 WroteStrtab = true;
5694}
5695
5696void BitcodeWriter::writeModule(const Module &M,
5697 bool ShouldPreserveUseListOrder,
5698 const ModuleSummaryIndex *Index,
5699 bool GenerateHash, ModuleHash *ModHash) {
5700 assert(!WroteStrtab);
5701
5702 // The Mods vector is used by irsymtab::build, which requires non-const
5703 // Modules in case it needs to materialize metadata. But the bitcode writer
5704 // requires that the module is materialized, so we can cast to non-const here,
5705 // after checking that it is in fact materialized.
5706 assert(M.isMaterialized());
5707 Mods.push_back(x: const_cast<Module *>(&M));
5708
5709 ModuleBitcodeWriter ModuleWriter(M, StrtabBuilder, *Stream,
5710 ShouldPreserveUseListOrder, Index,
5711 GenerateHash, ModHash);
5712 ModuleWriter.write();
5713}
5714
5715void BitcodeWriter::writeIndex(
5716 const ModuleSummaryIndex *Index,
5717 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex,
5718 const GVSummaryPtrSet *DecSummaries) {
5719 IndexBitcodeWriter IndexWriter(*Stream, StrtabBuilder, *Index, DecSummaries,
5720 ModuleToSummariesForIndex);
5721 IndexWriter.write();
5722}
5723
5724/// Write the specified module to the specified output stream.
5725void llvm::WriteBitcodeToFile(const Module &M, raw_ostream &Out,
5726 bool ShouldPreserveUseListOrder,
5727 const ModuleSummaryIndex *Index,
5728 bool GenerateHash, ModuleHash *ModHash) {
5729 auto Write = [&](BitcodeWriter &Writer) {
5730 Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash,
5731 ModHash);
5732 Writer.writeSymtab();
5733 Writer.writeStrtab();
5734 };
5735 Triple TT(M.getTargetTriple());
5736 if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) {
5737 // If this is darwin or another generic macho target, reserve space for the
5738 // header. Note that the header is computed *after* the output is known, so
5739 // we currently explicitly use a buffer, write to it, and then subsequently
5740 // flush to Out.
5741 SmallVector<char, 0> Buffer;
5742 Buffer.reserve(N: 256 * 1024);
5743 Buffer.insert(I: Buffer.begin(), NumToInsert: BWH_HeaderSize, Elt: 0);
5744 BitcodeWriter Writer(Buffer);
5745 Write(Writer);
5746 emitDarwinBCHeaderAndTrailer(Buffer, TT);
5747 Out.write(Ptr: Buffer.data(), Size: Buffer.size());
5748 } else {
5749 BitcodeWriter Writer(Out);
5750 Write(Writer);
5751 }
5752}
5753
5754void IndexBitcodeWriter::write() {
5755 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5756
5757 writeModuleVersion();
5758
5759 // Write the module paths in the combined index.
5760 writeModStrings();
5761
5762 // Write the summary combined index records.
5763 writeCombinedGlobalValueSummary();
5764
5765 Stream.ExitBlock();
5766}
5767
5768// Write the specified module summary index to the given raw output stream,
5769// where it will be written in a new bitcode block. This is used when
5770// writing the combined index file for ThinLTO. When writing a subset of the
5771// index for a distributed backend, provide a \p ModuleToSummariesForIndex map.
5772void llvm::writeIndexToFile(
5773 const ModuleSummaryIndex &Index, raw_ostream &Out,
5774 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex,
5775 const GVSummaryPtrSet *DecSummaries) {
5776 SmallVector<char, 0> Buffer;
5777 Buffer.reserve(N: 256 * 1024);
5778
5779 BitcodeWriter Writer(Buffer);
5780 Writer.writeIndex(Index: &Index, ModuleToSummariesForIndex, DecSummaries);
5781 Writer.writeStrtab();
5782
5783 Out.write(Ptr: (char *)&Buffer.front(), Size: Buffer.size());
5784}
5785
5786namespace {
5787
5788/// Class to manage the bitcode writing for a thin link bitcode file.
5789class ThinLinkBitcodeWriter : public ModuleBitcodeWriterBase {
5790 /// ModHash is for use in ThinLTO incremental build, generated while writing
5791 /// the module bitcode file.
5792 const ModuleHash *ModHash;
5793
5794public:
5795 ThinLinkBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
5796 BitstreamWriter &Stream,
5797 const ModuleSummaryIndex &Index,
5798 const ModuleHash &ModHash)
5799 : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
5800 /*ShouldPreserveUseListOrder=*/false, &Index),
5801 ModHash(&ModHash) {}
5802
5803 void write();
5804
5805private:
5806 void writeSimplifiedModuleInfo();
5807};
5808
5809} // end anonymous namespace
5810
5811// This function writes a simpilified module info for thin link bitcode file.
5812// It only contains the source file name along with the name(the offset and
5813// size in strtab) and linkage for global values. For the global value info
5814// entry, in order to keep linkage at offset 5, there are three zeros used
5815// as padding.
5816void ThinLinkBitcodeWriter::writeSimplifiedModuleInfo() {
5817 SmallVector<unsigned, 64> Vals;
5818 // Emit the module's source file name.
5819 {
5820 StringEncoding Bits = getStringEncoding(Str: M.getSourceFileName());
5821 BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
5822 if (Bits == SE_Char6)
5823 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
5824 else if (Bits == SE_Fixed7)
5825 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
5826
5827 // MODULE_CODE_SOURCE_FILENAME: [namechar x N]
5828 auto Abbv = std::make_shared<BitCodeAbbrev>();
5829 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
5830 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5831 Abbv->Add(OpInfo: AbbrevOpToUse);
5832 unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5833
5834 for (const auto P : M.getSourceFileName())
5835 Vals.push_back(Elt: (unsigned char)P);
5836
5837 Stream.EmitRecord(Code: bitc::MODULE_CODE_SOURCE_FILENAME, Vals, Abbrev: FilenameAbbrev);
5838 Vals.clear();
5839 }
5840
5841 writeGUIDList();
5842
5843 // Emit the global variable information.
5844 for (const GlobalVariable &GV : M.globals()) {
5845 // GLOBALVAR: [strtab offset, strtab size, 0, 0, 0, linkage]
5846 Vals.push_back(Elt: StrtabBuilder.add(S: GV.getName()));
5847 Vals.push_back(Elt: GV.getName().size());
5848 Vals.push_back(Elt: 0);
5849 Vals.push_back(Elt: 0);
5850 Vals.push_back(Elt: 0);
5851 Vals.push_back(Elt: getEncodedLinkage(GV));
5852
5853 Stream.EmitRecord(Code: bitc::MODULE_CODE_GLOBALVAR, Vals);
5854 Vals.clear();
5855 }
5856
5857 // Emit the function proto information.
5858 for (const Function &F : M) {
5859 // FUNCTION: [strtab offset, strtab size, 0, 0, 0, linkage]
5860 Vals.push_back(Elt: StrtabBuilder.add(S: F.getName()));
5861 Vals.push_back(Elt: F.getName().size());
5862 Vals.push_back(Elt: 0);
5863 Vals.push_back(Elt: 0);
5864 Vals.push_back(Elt: 0);
5865 Vals.push_back(Elt: getEncodedLinkage(GV: F));
5866
5867 Stream.EmitRecord(Code: bitc::MODULE_CODE_FUNCTION, Vals);
5868 Vals.clear();
5869 }
5870
5871 // Emit the alias information.
5872 for (const GlobalAlias &A : M.aliases()) {
5873 // ALIAS: [strtab offset, strtab size, 0, 0, 0, linkage]
5874 Vals.push_back(Elt: StrtabBuilder.add(S: A.getName()));
5875 Vals.push_back(Elt: A.getName().size());
5876 Vals.push_back(Elt: 0);
5877 Vals.push_back(Elt: 0);
5878 Vals.push_back(Elt: 0);
5879 Vals.push_back(Elt: getEncodedLinkage(GV: A));
5880
5881 Stream.EmitRecord(Code: bitc::MODULE_CODE_ALIAS, Vals);
5882 Vals.clear();
5883 }
5884
5885 // Emit the ifunc information.
5886 for (const GlobalIFunc &I : M.ifuncs()) {
5887 // IFUNC: [strtab offset, strtab size, 0, 0, 0, linkage]
5888 Vals.push_back(Elt: StrtabBuilder.add(S: I.getName()));
5889 Vals.push_back(Elt: I.getName().size());
5890 Vals.push_back(Elt: 0);
5891 Vals.push_back(Elt: 0);
5892 Vals.push_back(Elt: 0);
5893 Vals.push_back(Elt: getEncodedLinkage(GV: I));
5894
5895 Stream.EmitRecord(Code: bitc::MODULE_CODE_IFUNC, Vals);
5896 Vals.clear();
5897 }
5898}
5899
5900void ThinLinkBitcodeWriter::write() {
5901 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5902
5903 writeModuleVersion();
5904
5905 writeSimplifiedModuleInfo();
5906
5907 writePerModuleGlobalValueSummary();
5908
5909 // Write module hash.
5910 Stream.EmitRecord(Code: bitc::MODULE_CODE_HASH, Vals: ArrayRef<uint32_t>(*ModHash));
5911
5912 Stream.ExitBlock();
5913}
5914
5915void BitcodeWriter::writeThinLinkBitcode(const Module &M,
5916 const ModuleSummaryIndex &Index,
5917 const ModuleHash &ModHash) {
5918 assert(!WroteStrtab);
5919
5920 // The Mods vector is used by irsymtab::build, which requires non-const
5921 // Modules in case it needs to materialize metadata. But the bitcode writer
5922 // requires that the module is materialized, so we can cast to non-const here,
5923 // after checking that it is in fact materialized.
5924 assert(M.isMaterialized());
5925 Mods.push_back(x: const_cast<Module *>(&M));
5926
5927 ThinLinkBitcodeWriter ThinLinkWriter(M, StrtabBuilder, *Stream, Index,
5928 ModHash);
5929 ThinLinkWriter.write();
5930}
5931
5932// Write the specified thin link bitcode file to the given raw output stream,
5933// where it will be written in a new bitcode block. This is used when
5934// writing the per-module index file for ThinLTO.
5935void llvm::writeThinLinkBitcodeToFile(const Module &M, raw_ostream &Out,
5936 const ModuleSummaryIndex &Index,
5937 const ModuleHash &ModHash) {
5938 SmallVector<char, 0> Buffer;
5939 Buffer.reserve(N: 256 * 1024);
5940
5941 BitcodeWriter Writer(Buffer);
5942 Writer.writeThinLinkBitcode(M, Index, ModHash);
5943 Writer.writeSymtab();
5944 Writer.writeStrtab();
5945
5946 Out.write(Ptr: (char *)&Buffer.front(), Size: Buffer.size());
5947}
5948
5949static const char *getSectionNameForBitcode(const Triple &T) {
5950 switch (T.getObjectFormat()) {
5951 case Triple::MachO:
5952 return "__LLVM,__bitcode";
5953 case Triple::COFF:
5954 case Triple::ELF:
5955 case Triple::Wasm:
5956 case Triple::UnknownObjectFormat:
5957 return ".llvmbc";
5958 case Triple::GOFF:
5959 llvm_unreachable("GOFF is not yet implemented");
5960 break;
5961 case Triple::SPIRV:
5962 if (T.getVendor() == Triple::AMD)
5963 return ".llvmbc";
5964 llvm_unreachable("SPIRV is not yet implemented");
5965 break;
5966 case Triple::XCOFF:
5967 llvm_unreachable("XCOFF is not yet implemented");
5968 break;
5969 case Triple::DXContainer:
5970 llvm_unreachable("DXContainer is not yet implemented");
5971 break;
5972 }
5973 llvm_unreachable("Unimplemented ObjectFormatType");
5974}
5975
5976static const char *getSectionNameForCommandline(const Triple &T) {
5977 switch (T.getObjectFormat()) {
5978 case Triple::MachO:
5979 return "__LLVM,__cmdline";
5980 case Triple::COFF:
5981 case Triple::ELF:
5982 case Triple::Wasm:
5983 case Triple::UnknownObjectFormat:
5984 return ".llvmcmd";
5985 case Triple::GOFF:
5986 llvm_unreachable("GOFF is not yet implemented");
5987 break;
5988 case Triple::SPIRV:
5989 if (T.getVendor() == Triple::AMD)
5990 return ".llvmcmd";
5991 llvm_unreachable("SPIRV is not yet implemented");
5992 break;
5993 case Triple::XCOFF:
5994 llvm_unreachable("XCOFF is not yet implemented");
5995 break;
5996 case Triple::DXContainer:
5997 llvm_unreachable("DXC is not yet implemented");
5998 break;
5999 }
6000 llvm_unreachable("Unimplemented ObjectFormatType");
6001}
6002
6003void llvm::embedBitcodeInModule(llvm::Module &M, llvm::MemoryBufferRef Buf,
6004 bool EmbedBitcode, bool EmbedCmdline,
6005 const std::vector<uint8_t> &CmdArgs) {
6006 // Save llvm.compiler.used and remove it.
6007 SmallVector<Constant *, 2> UsedArray;
6008 SmallVector<GlobalValue *, 4> UsedGlobals;
6009 GlobalVariable *Used = collectUsedGlobalVariables(M, Vec&: UsedGlobals, CompilerUsed: true);
6010 Type *UsedElementType = Used ? Used->getValueType()->getArrayElementType()
6011 : PointerType::getUnqual(C&: M.getContext());
6012 for (auto *GV : UsedGlobals) {
6013 if (GV->getName() != "llvm.embedded.module" &&
6014 GV->getName() != "llvm.cmdline")
6015 UsedArray.push_back(
6016 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
6017 }
6018 if (Used)
6019 Used->eraseFromParent();
6020
6021 // Embed the bitcode for the llvm module.
6022 std::string Data;
6023 ArrayRef<uint8_t> ModuleData;
6024 Triple T(M.getTargetTriple());
6025
6026 if (EmbedBitcode) {
6027 if (Buf.getBufferSize() == 0 ||
6028 !isBitcode(BufPtr: (const unsigned char *)Buf.getBufferStart(),
6029 BufEnd: (const unsigned char *)Buf.getBufferEnd())) {
6030 // If the input is LLVM Assembly, bitcode is produced by serializing
6031 // the module. Use-lists order need to be preserved in this case.
6032 llvm::raw_string_ostream OS(Data);
6033 llvm::WriteBitcodeToFile(M, Out&: OS, /* ShouldPreserveUseListOrder */ true);
6034 ModuleData =
6035 ArrayRef<uint8_t>((const uint8_t *)OS.str().data(), OS.str().size());
6036 } else
6037 // If the input is LLVM bitcode, write the input byte stream directly.
6038 ModuleData = ArrayRef<uint8_t>((const uint8_t *)Buf.getBufferStart(),
6039 Buf.getBufferSize());
6040 }
6041 llvm::Constant *ModuleConstant =
6042 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts: ModuleData);
6043 llvm::GlobalVariable *GV = new llvm::GlobalVariable(
6044 M, ModuleConstant->getType(), true, llvm::GlobalValue::PrivateLinkage,
6045 ModuleConstant);
6046 GV->setSection(getSectionNameForBitcode(T));
6047 // Set alignment to 1 to prevent padding between two contributions from input
6048 // sections after linking.
6049 GV->setAlignment(Align(1));
6050 UsedArray.push_back(
6051 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
6052 if (llvm::GlobalVariable *Old =
6053 M.getGlobalVariable(Name: "llvm.embedded.module", AllowInternal: true)) {
6054 assert(Old->hasZeroLiveUses() &&
6055 "llvm.embedded.module can only be used once in llvm.compiler.used");
6056 GV->takeName(V: Old);
6057 Old->eraseFromParent();
6058 } else {
6059 GV->setName("llvm.embedded.module");
6060 }
6061
6062 // Skip if only bitcode needs to be embedded.
6063 if (EmbedCmdline) {
6064 // Embed command-line options.
6065 ArrayRef<uint8_t> CmdData(const_cast<uint8_t *>(CmdArgs.data()),
6066 CmdArgs.size());
6067 llvm::Constant *CmdConstant =
6068 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts: CmdData);
6069 GV = new llvm::GlobalVariable(M, CmdConstant->getType(), true,
6070 llvm::GlobalValue::PrivateLinkage,
6071 CmdConstant);
6072 GV->setSection(getSectionNameForCommandline(T));
6073 GV->setAlignment(Align(1));
6074 UsedArray.push_back(
6075 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
6076 if (llvm::GlobalVariable *Old = M.getGlobalVariable(Name: "llvm.cmdline", AllowInternal: true)) {
6077 assert(Old->hasZeroLiveUses() &&
6078 "llvm.cmdline can only be used once in llvm.compiler.used");
6079 GV->takeName(V: Old);
6080 Old->eraseFromParent();
6081 } else {
6082 GV->setName("llvm.cmdline");
6083 }
6084 }
6085
6086 if (UsedArray.empty())
6087 return;
6088
6089 // Recreate llvm.compiler.used.
6090 ArrayType *ATy = ArrayType::get(ElementType: UsedElementType, NumElements: UsedArray.size());
6091 auto *NewUsed = new GlobalVariable(
6092 M, ATy, false, llvm::GlobalValue::AppendingLinkage,
6093 llvm::ConstantArray::get(T: ATy, V: UsedArray), "llvm.compiler.used");
6094 NewUsed->setSection("llvm.metadata");
6095}
6096