1//===- Bitcode/Writer/BitcodeWriter.cpp - Bitcode Writer ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Bitcode writer implementation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Bitcode/BitcodeWriter.h"
14#include "ValueEnumerator.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/DenseMap.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SetVector.h"
21#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/SmallString.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringMap.h"
25#include "llvm/ADT/StringRef.h"
26#include "llvm/Analysis/MemoryProfileInfo.h"
27#include "llvm/BinaryFormat/Dwarf.h"
28#include "llvm/Bitcode/BitcodeCommon.h"
29#include "llvm/Bitcode/BitcodeReader.h"
30#include "llvm/Bitcode/LLVMBitCodes.h"
31#include "llvm/Bitstream/BitCodes.h"
32#include "llvm/Bitstream/BitstreamWriter.h"
33#include "llvm/Config/llvm-config.h"
34#include "llvm/IR/Attributes.h"
35#include "llvm/IR/BasicBlock.h"
36#include "llvm/IR/Comdat.h"
37#include "llvm/IR/Constant.h"
38#include "llvm/IR/ConstantRangeList.h"
39#include "llvm/IR/Constants.h"
40#include "llvm/IR/DebugInfoMetadata.h"
41#include "llvm/IR/DebugLoc.h"
42#include "llvm/IR/DerivedTypes.h"
43#include "llvm/IR/Function.h"
44#include "llvm/IR/GlobalAlias.h"
45#include "llvm/IR/GlobalIFunc.h"
46#include "llvm/IR/GlobalObject.h"
47#include "llvm/IR/GlobalValue.h"
48#include "llvm/IR/GlobalVariable.h"
49#include "llvm/IR/InlineAsm.h"
50#include "llvm/IR/InstrTypes.h"
51#include "llvm/IR/Instruction.h"
52#include "llvm/IR/Instructions.h"
53#include "llvm/IR/LLVMContext.h"
54#include "llvm/IR/Metadata.h"
55#include "llvm/IR/Module.h"
56#include "llvm/IR/ModuleSummaryIndex.h"
57#include "llvm/IR/Operator.h"
58#include "llvm/IR/Type.h"
59#include "llvm/IR/UseListOrder.h"
60#include "llvm/IR/Value.h"
61#include "llvm/IR/ValueSymbolTable.h"
62#include "llvm/MC/StringTableBuilder.h"
63#include "llvm/MC/TargetRegistry.h"
64#include "llvm/Object/IRSymtab.h"
65#include "llvm/ProfileData/MemProf.h"
66#include "llvm/ProfileData/MemProfRadixTree.h"
67#include "llvm/Support/AtomicOrdering.h"
68#include "llvm/Support/Casting.h"
69#include "llvm/Support/CommandLine.h"
70#include "llvm/Support/Compiler.h"
71#include "llvm/Support/Endian.h"
72#include "llvm/Support/Error.h"
73#include "llvm/Support/ErrorHandling.h"
74#include "llvm/Support/MathExtras.h"
75#include "llvm/Support/SHA1.h"
76#include "llvm/Support/raw_ostream.h"
77#include "llvm/TargetParser/Triple.h"
78#include <algorithm>
79#include <cassert>
80#include <cstddef>
81#include <cstdint>
82#include <iterator>
83#include <map>
84#include <memory>
85#include <optional>
86#include <string>
87#include <utility>
88#include <vector>
89
90using namespace llvm;
91using namespace llvm::memprof;
92
93static cl::opt<unsigned>
94 IndexThreshold("bitcode-mdindex-threshold", cl::Hidden, cl::init(Val: 25),
95 cl::desc("Number of metadatas above which we emit an index "
96 "to enable lazy-loading"));
97static cl::opt<uint32_t> FlushThreshold(
98 "bitcode-flush-threshold", cl::Hidden, cl::init(Val: 512),
99 cl::desc("The threshold (unit M) for flushing LLVM bitcode."));
100
101// Since we only use the context information in the memprof summary records in
102// the LTO backends to do assertion checking, save time and space by only
103// serializing the context for non-NDEBUG builds.
104// TODO: Currently this controls writing context of the allocation info records,
105// which are larger and more expensive, but we should do this for the callsite
106// records as well.
107// FIXME: Convert to a const once this has undergone more sigificant testing.
108static cl::opt<bool>
109 CombinedIndexMemProfContext("combined-index-memprof-context", cl::Hidden,
110#ifdef NDEBUG
111 cl::init(Val: false),
112#else
113 cl::init(true),
114#endif
115 cl::desc(""));
116
117static cl::opt<bool> PreserveBitcodeUseListOrder(
118 "preserve-bc-uselistorder", cl::Hidden, cl::init(Val: true),
119 cl::desc("Preserve use-list order when writing LLVM bitcode."));
120
121namespace llvm {
122extern FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold;
123}
124
125namespace {
126
127/// These are manifest constants used by the bitcode writer. They do not need to
128/// be kept in sync with the reader, but need to be consistent within this file.
129enum {
130 // VALUE_SYMTAB_BLOCK abbrev id's.
131 VST_ENTRY_8_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
132 VST_ENTRY_7_ABBREV,
133 VST_ENTRY_6_ABBREV,
134 VST_BBENTRY_6_ABBREV,
135
136 // CONSTANTS_BLOCK abbrev id's.
137 CONSTANTS_SETTYPE_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
138 CONSTANTS_INTEGER_ABBREV,
139 CONSTANTS_BYTE_ABBREV,
140 CONSTANTS_CE_CAST_Abbrev,
141 CONSTANTS_NULL_Abbrev,
142
143 // FUNCTION_BLOCK abbrev id's.
144 FUNCTION_INST_LOAD_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
145 FUNCTION_INST_STORE_ABBREV,
146 FUNCTION_INST_UNOP_ABBREV,
147 FUNCTION_INST_UNOP_FLAGS_ABBREV,
148 FUNCTION_INST_BINOP_ABBREV,
149 FUNCTION_INST_BINOP_FLAGS_ABBREV,
150 FUNCTION_INST_CAST_ABBREV,
151 FUNCTION_INST_CAST_FLAGS_ABBREV,
152 FUNCTION_INST_RET_VOID_ABBREV,
153 FUNCTION_INST_RET_VAL_ABBREV,
154 FUNCTION_INST_BR_UNCOND_ABBREV,
155 FUNCTION_INST_BR_COND_ABBREV,
156 FUNCTION_INST_UNREACHABLE_ABBREV,
157 FUNCTION_INST_GEP_ABBREV,
158 FUNCTION_INST_CMP_ABBREV,
159 FUNCTION_INST_CMP_FLAGS_ABBREV,
160 FUNCTION_DEBUG_RECORD_VALUE_ABBREV,
161 FUNCTION_DEBUG_LOC_ABBREV,
162};
163
164/// Abstract class to manage the bitcode writing, subclassed for each bitcode
165/// file type.
166class BitcodeWriterBase {
167protected:
168 /// The stream created and owned by the client.
169 BitstreamWriter &Stream;
170
171 StringTableBuilder &StrtabBuilder;
172
173public:
174 /// Constructs a BitcodeWriterBase object that writes to the provided
175 /// \p Stream.
176 BitcodeWriterBase(BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder)
177 : Stream(Stream), StrtabBuilder(StrtabBuilder) {}
178
179protected:
180 void writeModuleVersion();
181};
182
183void BitcodeWriterBase::writeModuleVersion() {
184 // VERSION: [version#]
185 Stream.EmitRecord(Code: bitc::MODULE_CODE_VERSION, Vals: ArrayRef<uint64_t>{2});
186}
187
188/// Base class to manage the module bitcode writing, currently subclassed for
189/// ModuleBitcodeWriter and ThinLinkBitcodeWriter.
190class ModuleBitcodeWriterBase : public BitcodeWriterBase {
191protected:
192 /// The Module to write to bitcode.
193 const Module &M;
194
195 /// Enumerates ids for all values in the module.
196 ValueEnumerator VE;
197
198 /// Optional per-module index to write for ThinLTO.
199 const ModuleSummaryIndex *Index;
200
201 /// Map that holds the correspondence between GUIDs in the summary index,
202 /// that came from indirect call profiles, and a value id generated by this
203 /// class to use in the VST and summary block records.
204 std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
205
206 /// Tracks the last value id recorded in the GUIDToValueMap.
207 unsigned GlobalValueId;
208
209 /// Saves the offset of the VSTOffset record that must eventually be
210 /// backpatched with the offset of the actual VST.
211 uint64_t VSTOffsetPlaceholder = 0;
212
213public:
214 /// Constructs a ModuleBitcodeWriterBase object for the given Module,
215 /// writing to the provided \p Buffer.
216 ModuleBitcodeWriterBase(const Module &M, StringTableBuilder &StrtabBuilder,
217 BitstreamWriter &Stream,
218 bool ShouldPreserveUseListOrder,
219 const ModuleSummaryIndex *Index)
220 : BitcodeWriterBase(Stream, StrtabBuilder), M(M),
221 VE(M, PreserveBitcodeUseListOrder.getNumOccurrences()
222 ? PreserveBitcodeUseListOrder
223 : ShouldPreserveUseListOrder),
224 Index(Index) {
225 // Assign ValueIds to any callee values in the index that came from
226 // indirect call profiles and were recorded as a GUID not a Value*
227 // (which would have been assigned an ID by the ValueEnumerator).
228 // The starting ValueId is just after the number of values in the
229 // ValueEnumerator, so that they can be emitted in the VST.
230 GlobalValueId = VE.getValues().size();
231 if (!Index)
232 return;
233 for (const auto &GUIDSummaryLists : *Index)
234 // Examine all summaries for this GUID.
235 for (auto &Summary : GUIDSummaryLists.second.getSummaryList())
236 if (auto FS = dyn_cast<FunctionSummary>(Val: Summary.get())) {
237 // For each call in the function summary, see if the call
238 // is to a GUID (which means it is for an indirect call,
239 // otherwise we would have a Value for it). If so, synthesize
240 // a value id.
241 for (auto &CallEdge : FS->calls())
242 if (!CallEdge.first.haveGVs() || !CallEdge.first.getValue())
243 assignValueId(ValGUID: CallEdge.first.getGUID());
244
245 // For each referenced variables in the function summary, see if the
246 // variable is represented by a GUID (as opposed to a symbol to
247 // declarations or definitions in the module). If so, synthesize a
248 // value id.
249 for (auto &RefEdge : FS->refs())
250 if (!RefEdge.haveGVs() || !RefEdge.getValue())
251 assignValueId(ValGUID: RefEdge.getGUID());
252 }
253 }
254
255protected:
256 void writePerModuleGlobalValueSummary();
257
258private:
259 void writePerModuleFunctionSummaryRecord(
260 SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
261 unsigned ValueID, unsigned FSCallsProfileAbbrev, unsigned CallsiteAbbrev,
262 unsigned AllocAbbrev, unsigned ContextIdAbbvId, const Function &F,
263 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
264 CallStackId &CallStackCount);
265 void writeModuleLevelReferences(const GlobalVariable &V,
266 SmallVector<uint64_t, 64> &NameVals,
267 unsigned FSModRefsAbbrev,
268 unsigned FSModVTableRefsAbbrev);
269
270 void assignValueId(GlobalValue::GUID ValGUID) {
271 GUIDToValueIdMap[ValGUID] = ++GlobalValueId;
272 }
273
274 unsigned getValueId(GlobalValue::GUID ValGUID) {
275 const auto &VMI = GUIDToValueIdMap.find(x: ValGUID);
276 // Expect that any GUID value had a value Id assigned by an
277 // earlier call to assignValueId.
278 assert(VMI != GUIDToValueIdMap.end() &&
279 "GUID does not have assigned value Id");
280 return VMI->second;
281 }
282
283 // Helper to get the valueId for the type of value recorded in VI.
284 unsigned getValueId(ValueInfo VI) {
285 if (!VI.haveGVs() || !VI.getValue())
286 return getValueId(ValGUID: VI.getGUID());
287 return VE.getValueID(V: VI.getValue());
288 }
289
290 std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
291};
292
293/// Class to manage the bitcode writing for a module.
294class ModuleBitcodeWriter : public ModuleBitcodeWriterBase {
295 /// True if a module hash record should be written.
296 bool GenerateHash;
297
298 /// If non-null, when GenerateHash is true, the resulting hash is written
299 /// into ModHash.
300 ModuleHash *ModHash;
301
302 SHA1 Hasher;
303
304 /// The start bit of the identification block.
305 uint64_t BitcodeStartBit;
306
307public:
308 /// Constructs a ModuleBitcodeWriter object for the given Module,
309 /// writing to the provided \p Buffer.
310 ModuleBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
311 BitstreamWriter &Stream, bool ShouldPreserveUseListOrder,
312 const ModuleSummaryIndex *Index, bool GenerateHash,
313 ModuleHash *ModHash = nullptr)
314 : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
315 ShouldPreserveUseListOrder, Index),
316 GenerateHash(GenerateHash), ModHash(ModHash),
317 BitcodeStartBit(Stream.GetCurrentBitNo()) {}
318
319 /// Emit the current module to the bitstream.
320 void write();
321
322private:
323 uint64_t bitcodeStartBit() { return BitcodeStartBit; }
324
325 size_t addToStrtab(StringRef Str);
326
327 void writeAttributeGroupTable();
328 void writeAttributeTable();
329 void writeTypeTable();
330 void writeComdats();
331 void writeValueSymbolTableForwardDecl();
332 void writeModuleInfo();
333 void writeValueAsMetadata(const ValueAsMetadata *MD,
334 SmallVectorImpl<uint64_t> &Record);
335 void writeMDTuple(const MDTuple *N, SmallVectorImpl<uint64_t> &Record,
336 unsigned Abbrev);
337 unsigned createDILocationAbbrev();
338 void writeDILocation(const DILocation *N, SmallVectorImpl<uint64_t> &Record,
339 unsigned &Abbrev);
340 unsigned createGenericDINodeAbbrev();
341 void writeGenericDINode(const GenericDINode *N,
342 SmallVectorImpl<uint64_t> &Record, unsigned &Abbrev);
343 void writeDISubrange(const DISubrange *N, SmallVectorImpl<uint64_t> &Record,
344 unsigned Abbrev);
345 void writeDIGenericSubrange(const DIGenericSubrange *N,
346 SmallVectorImpl<uint64_t> &Record,
347 unsigned Abbrev);
348 void writeDIEnumerator(const DIEnumerator *N,
349 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
350 void writeDIBasicType(const DIBasicType *N, SmallVectorImpl<uint64_t> &Record,
351 unsigned Abbrev);
352 void writeDIFixedPointType(const DIFixedPointType *N,
353 SmallVectorImpl<uint64_t> &Record,
354 unsigned Abbrev);
355 void writeDIStringType(const DIStringType *N,
356 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
357 void writeDIDerivedType(const DIDerivedType *N,
358 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
359 void writeDISubrangeType(const DISubrangeType *N,
360 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
361 void writeDICompositeType(const DICompositeType *N,
362 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
363 void writeDISubroutineType(const DISubroutineType *N,
364 SmallVectorImpl<uint64_t> &Record,
365 unsigned Abbrev);
366 void writeDIFile(const DIFile *N, SmallVectorImpl<uint64_t> &Record,
367 unsigned Abbrev);
368 void writeDICompileUnit(const DICompileUnit *N,
369 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
370 void writeDISubprogram(const DISubprogram *N,
371 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
372 void writeDILexicalBlock(const DILexicalBlock *N,
373 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
374 void writeDILexicalBlockFile(const DILexicalBlockFile *N,
375 SmallVectorImpl<uint64_t> &Record,
376 unsigned Abbrev);
377 void writeDICommonBlock(const DICommonBlock *N,
378 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
379 void writeDINamespace(const DINamespace *N, SmallVectorImpl<uint64_t> &Record,
380 unsigned Abbrev);
381 void writeDIMacro(const DIMacro *N, SmallVectorImpl<uint64_t> &Record,
382 unsigned Abbrev);
383 void writeDIMacroFile(const DIMacroFile *N, SmallVectorImpl<uint64_t> &Record,
384 unsigned Abbrev);
385 void writeDIArgList(const DIArgList *N, SmallVectorImpl<uint64_t> &Record);
386 void writeDIModule(const DIModule *N, SmallVectorImpl<uint64_t> &Record,
387 unsigned Abbrev);
388 void writeDIAssignID(const DIAssignID *N, SmallVectorImpl<uint64_t> &Record,
389 unsigned Abbrev);
390 void writeDITemplateTypeParameter(const DITemplateTypeParameter *N,
391 SmallVectorImpl<uint64_t> &Record,
392 unsigned Abbrev);
393 void writeDITemplateValueParameter(const DITemplateValueParameter *N,
394 SmallVectorImpl<uint64_t> &Record,
395 unsigned Abbrev);
396 void writeDIGlobalVariable(const DIGlobalVariable *N,
397 SmallVectorImpl<uint64_t> &Record,
398 unsigned Abbrev);
399 void writeDILocalVariable(const DILocalVariable *N,
400 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
401 void writeDILabel(const DILabel *N,
402 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
403 void writeDIExpression(const DIExpression *N,
404 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
405 void writeDIGlobalVariableExpression(const DIGlobalVariableExpression *N,
406 SmallVectorImpl<uint64_t> &Record,
407 unsigned Abbrev);
408 void writeDIObjCProperty(const DIObjCProperty *N,
409 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
410 void writeDIImportedEntity(const DIImportedEntity *N,
411 SmallVectorImpl<uint64_t> &Record,
412 unsigned Abbrev);
413 unsigned createNamedMetadataAbbrev();
414 void writeNamedMetadata(SmallVectorImpl<uint64_t> &Record);
415 unsigned createMetadataStringsAbbrev();
416 void writeMetadataStrings(ArrayRef<const Metadata *> Strings,
417 SmallVectorImpl<uint64_t> &Record);
418 void writeMetadataRecords(ArrayRef<const Metadata *> MDs,
419 SmallVectorImpl<uint64_t> &Record,
420 std::vector<unsigned> *MDAbbrevs = nullptr,
421 std::vector<uint64_t> *IndexPos = nullptr);
422 void writeModuleMetadata();
423 void writeFunctionMetadata(const Function &F);
424 void writeFunctionMetadataAttachment(const Function &F);
425 void pushGlobalMetadataAttachment(SmallVectorImpl<uint64_t> &Record,
426 const GlobalObject &GO);
427 void writeModuleMetadataKinds();
428 void writeOperandBundleTags();
429 void writeSyncScopeNames();
430 void writeConstants(unsigned FirstVal, unsigned LastVal, bool isGlobal);
431 void writeModuleConstants();
432 bool pushValueAndType(const Value *V, unsigned InstID,
433 SmallVectorImpl<unsigned> &Vals);
434 bool pushValueOrMetadata(const Value *V, unsigned InstID,
435 SmallVectorImpl<unsigned> &Vals);
436 void writeOperandBundles(const CallBase &CB, unsigned InstID);
437 void pushValue(const Value *V, unsigned InstID,
438 SmallVectorImpl<unsigned> &Vals);
439 void pushValueSigned(const Value *V, unsigned InstID,
440 SmallVectorImpl<uint64_t> &Vals);
441 void writeInstruction(const Instruction &I, unsigned InstID,
442 SmallVectorImpl<unsigned> &Vals);
443 void writeFunctionLevelValueSymbolTable(const ValueSymbolTable &VST);
444 void writeGlobalValueSymbolTable(
445 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
446 void writeUseList(UseListOrder &&Order);
447 void writeUseListBlock(const Function *F);
448 void
449 writeFunction(const Function &F,
450 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
451 void writeBlockInfo();
452 void writeModuleHash(StringRef View);
453
454 unsigned getEncodedSyncScopeID(SyncScope::ID SSID) {
455 return unsigned(SSID);
456 }
457
458 unsigned getEncodedAlign(MaybeAlign Alignment) { return encode(A: Alignment); }
459};
460
461/// Class to manage the bitcode writing for a combined index.
462class IndexBitcodeWriter : public BitcodeWriterBase {
463 /// The combined index to write to bitcode.
464 const ModuleSummaryIndex &Index;
465
466 /// When writing combined summaries, provides the set of global value
467 /// summaries for which the value (function, function alias, etc) should be
468 /// imported as a declaration.
469 const GVSummaryPtrSet *DecSummaries = nullptr;
470
471 /// When writing a subset of the index for distributed backends, client
472 /// provides a map of modules to the corresponding GUIDs/summaries to write.
473 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex;
474
475 /// Map that holds the correspondence between the GUID used in the combined
476 /// index and a value id generated by this class to use in references.
477 std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
478
479 // The stack ids used by this index, which will be a subset of those in
480 // the full index in the case of distributed indexes.
481 std::vector<uint64_t> StackIds;
482
483 // Keep a map of the stack id indices used by records being written for this
484 // index to the index of the corresponding stack id in the above StackIds
485 // vector. Ensures we write each referenced stack id once.
486 DenseMap<unsigned, unsigned> StackIdIndicesToIndex;
487
488 /// Tracks the last value id recorded in the GUIDToValueMap.
489 unsigned GlobalValueId = 0;
490
491 /// Tracks the assignment of module paths in the module path string table to
492 /// an id assigned for use in summary references to the module path.
493 DenseMap<StringRef, uint64_t> ModuleIdMap;
494
495public:
496 /// Constructs a IndexBitcodeWriter object for the given combined index,
497 /// writing to the provided \p Buffer. When writing a subset of the index
498 /// for a distributed backend, provide a \p ModuleToSummariesForIndex map.
499 /// If provided, \p DecSummaries specifies the set of summaries for which
500 /// the corresponding functions or aliased functions should be imported as a
501 /// declaration (but not definition) for each module.
502 IndexBitcodeWriter(
503 BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder,
504 const ModuleSummaryIndex &Index,
505 const GVSummaryPtrSet *DecSummaries = nullptr,
506 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex = nullptr)
507 : BitcodeWriterBase(Stream, StrtabBuilder), Index(Index),
508 DecSummaries(DecSummaries),
509 ModuleToSummariesForIndex(ModuleToSummariesForIndex) {
510
511 // See if the StackIdIndex was already added to the StackId map and
512 // vector. If not, record it.
513 auto RecordStackIdReference = [&](unsigned StackIdIndex) {
514 // If the StackIdIndex is not yet in the map, the below insert ensures
515 // that it will point to the new StackIds vector entry we push to just
516 // below.
517 auto Inserted =
518 StackIdIndicesToIndex.insert(KV: {StackIdIndex, StackIds.size()});
519 if (Inserted.second)
520 StackIds.push_back(x: Index.getStackIdAtIndex(Index: StackIdIndex));
521 };
522
523 // Assign unique value ids to all summaries to be written, for use
524 // in writing out the call graph edges. Save the mapping from GUID
525 // to the new global value id to use when writing those edges, which
526 // are currently saved in the index in terms of GUID.
527 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
528 GUIDToValueIdMap[I.first] = ++GlobalValueId;
529 // If this is invoked for an aliasee, we want to record the above mapping,
530 // but not the information needed for its summary entry (if the aliasee is
531 // to be imported, we will invoke this separately with IsAliasee=false).
532 if (IsAliasee)
533 return;
534 auto *FS = dyn_cast<FunctionSummary>(Val: I.second);
535 if (!FS)
536 return;
537 // Record all stack id indices actually used in the summary entries being
538 // written, so that we can compact them in the case of distributed ThinLTO
539 // indexes.
540 for (auto &CI : FS->callsites()) {
541 // If the stack id list is empty, this callsite info was synthesized for
542 // a missing tail call frame. Ensure that the callee's GUID gets a value
543 // id. Normally we only generate these for defined summaries, which in
544 // the case of distributed ThinLTO is only the functions already defined
545 // in the module or that we want to import. We don't bother to include
546 // all the callee symbols as they aren't normally needed in the backend.
547 // However, for the synthesized callsite infos we do need the callee
548 // GUID in the backend so that we can correlate the identified callee
549 // with this callsite info (which for non-tail calls is done by the
550 // ordering of the callsite infos and verified via stack ids).
551 if (CI.StackIdIndices.empty()) {
552 GUIDToValueIdMap[CI.Callee.getGUID()] = ++GlobalValueId;
553 continue;
554 }
555 for (auto Idx : CI.StackIdIndices)
556 RecordStackIdReference(Idx);
557 }
558 if (CombinedIndexMemProfContext) {
559 for (auto &AI : FS->allocs())
560 for (auto &MIB : AI.MIBs)
561 for (auto Idx : MIB.StackIdIndices)
562 RecordStackIdReference(Idx);
563 }
564 });
565 }
566
567 /// The below iterator returns the GUID and associated summary.
568 using GVInfo = std::pair<GlobalValue::GUID, GlobalValueSummary *>;
569
570 /// Calls the callback for each value GUID and summary to be written to
571 /// bitcode. This hides the details of whether they are being pulled from the
572 /// entire index or just those in a provided ModuleToSummariesForIndex map.
573 template<typename Functor>
574 void forEachSummary(Functor Callback) {
575 if (ModuleToSummariesForIndex) {
576 for (auto &M : *ModuleToSummariesForIndex)
577 for (auto &Summary : M.second) {
578 Callback(Summary, false);
579 // Ensure aliasee is handled, e.g. for assigning a valueId,
580 // even if we are not importing the aliasee directly (the
581 // imported alias will contain a copy of aliasee).
582 if (auto *AS = dyn_cast<AliasSummary>(Val: Summary.getSecond()))
583 Callback({AS->getAliaseeGUID(), &AS->getAliasee()}, true);
584 }
585 } else {
586 for (auto &Summaries : Index)
587 for (auto &Summary : Summaries.second.getSummaryList())
588 Callback({Summaries.first, Summary.get()}, false);
589 }
590 }
591
592 /// Calls the callback for each entry in the modulePaths StringMap that
593 /// should be written to the module path string table. This hides the details
594 /// of whether they are being pulled from the entire index or just those in a
595 /// provided ModuleToSummariesForIndex map.
596 template <typename Functor> void forEachModule(Functor Callback) {
597 if (ModuleToSummariesForIndex) {
598 for (const auto &M : *ModuleToSummariesForIndex) {
599 const auto &MPI = Index.modulePaths().find(Key: M.first);
600 if (MPI == Index.modulePaths().end()) {
601 // This should only happen if the bitcode file was empty, in which
602 // case we shouldn't be importing (the ModuleToSummariesForIndex
603 // would only include the module we are writing and index for).
604 assert(ModuleToSummariesForIndex->size() == 1);
605 continue;
606 }
607 Callback(*MPI);
608 }
609 } else {
610 // Since StringMap iteration order isn't guaranteed, order by path string
611 // first.
612 // FIXME: Make this a vector of StringMapEntry instead to avoid the later
613 // map lookup.
614 std::vector<StringRef> ModulePaths;
615 for (auto &[ModPath, _] : Index.modulePaths())
616 ModulePaths.push_back(x: ModPath);
617 llvm::sort(C&: ModulePaths);
618 for (auto &ModPath : ModulePaths)
619 Callback(*Index.modulePaths().find(Key: ModPath));
620 }
621 }
622
623 /// Main entry point for writing a combined index to bitcode.
624 void write();
625
626private:
627 void writeModStrings();
628 void writeCombinedGlobalValueSummary();
629
630 std::optional<unsigned> getValueId(GlobalValue::GUID ValGUID) {
631 auto VMI = GUIDToValueIdMap.find(x: ValGUID);
632 if (VMI == GUIDToValueIdMap.end())
633 return std::nullopt;
634 return VMI->second;
635 }
636
637 std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
638};
639
640} // end anonymous namespace
641
642static unsigned getEncodedCastOpcode(unsigned Opcode) {
643 switch (Opcode) {
644 default: llvm_unreachable("Unknown cast instruction!");
645 case Instruction::Trunc : return bitc::CAST_TRUNC;
646 case Instruction::ZExt : return bitc::CAST_ZEXT;
647 case Instruction::SExt : return bitc::CAST_SEXT;
648 case Instruction::FPToUI : return bitc::CAST_FPTOUI;
649 case Instruction::FPToSI : return bitc::CAST_FPTOSI;
650 case Instruction::UIToFP : return bitc::CAST_UITOFP;
651 case Instruction::SIToFP : return bitc::CAST_SITOFP;
652 case Instruction::FPTrunc : return bitc::CAST_FPTRUNC;
653 case Instruction::FPExt : return bitc::CAST_FPEXT;
654 case Instruction::PtrToAddr: return bitc::CAST_PTRTOADDR;
655 case Instruction::PtrToInt: return bitc::CAST_PTRTOINT;
656 case Instruction::IntToPtr: return bitc::CAST_INTTOPTR;
657 case Instruction::BitCast : return bitc::CAST_BITCAST;
658 case Instruction::AddrSpaceCast: return bitc::CAST_ADDRSPACECAST;
659 }
660}
661
662static unsigned getEncodedUnaryOpcode(unsigned Opcode) {
663 switch (Opcode) {
664 default: llvm_unreachable("Unknown binary instruction!");
665 case Instruction::FNeg: return bitc::UNOP_FNEG;
666 }
667}
668
669static unsigned getEncodedBinaryOpcode(unsigned Opcode) {
670 switch (Opcode) {
671 default: llvm_unreachable("Unknown binary instruction!");
672 case Instruction::Add:
673 case Instruction::FAdd: return bitc::BINOP_ADD;
674 case Instruction::Sub:
675 case Instruction::FSub: return bitc::BINOP_SUB;
676 case Instruction::Mul:
677 case Instruction::FMul: return bitc::BINOP_MUL;
678 case Instruction::UDiv: return bitc::BINOP_UDIV;
679 case Instruction::FDiv:
680 case Instruction::SDiv: return bitc::BINOP_SDIV;
681 case Instruction::URem: return bitc::BINOP_UREM;
682 case Instruction::FRem:
683 case Instruction::SRem: return bitc::BINOP_SREM;
684 case Instruction::Shl: return bitc::BINOP_SHL;
685 case Instruction::LShr: return bitc::BINOP_LSHR;
686 case Instruction::AShr: return bitc::BINOP_ASHR;
687 case Instruction::And: return bitc::BINOP_AND;
688 case Instruction::Or: return bitc::BINOP_OR;
689 case Instruction::Xor: return bitc::BINOP_XOR;
690 }
691}
692
693static unsigned getEncodedRMWOperation(AtomicRMWInst::BinOp Op) {
694 switch (Op) {
695 default: llvm_unreachable("Unknown RMW operation!");
696 case AtomicRMWInst::Xchg: return bitc::RMW_XCHG;
697 case AtomicRMWInst::Add: return bitc::RMW_ADD;
698 case AtomicRMWInst::Sub: return bitc::RMW_SUB;
699 case AtomicRMWInst::And: return bitc::RMW_AND;
700 case AtomicRMWInst::Nand: return bitc::RMW_NAND;
701 case AtomicRMWInst::Or: return bitc::RMW_OR;
702 case AtomicRMWInst::Xor: return bitc::RMW_XOR;
703 case AtomicRMWInst::Max: return bitc::RMW_MAX;
704 case AtomicRMWInst::Min: return bitc::RMW_MIN;
705 case AtomicRMWInst::UMax: return bitc::RMW_UMAX;
706 case AtomicRMWInst::UMin: return bitc::RMW_UMIN;
707 case AtomicRMWInst::FAdd: return bitc::RMW_FADD;
708 case AtomicRMWInst::FSub: return bitc::RMW_FSUB;
709 case AtomicRMWInst::FMax: return bitc::RMW_FMAX;
710 case AtomicRMWInst::FMin: return bitc::RMW_FMIN;
711 case AtomicRMWInst::FMaximum:
712 return bitc::RMW_FMAXIMUM;
713 case AtomicRMWInst::FMinimum:
714 return bitc::RMW_FMINIMUM;
715 case AtomicRMWInst::UIncWrap:
716 return bitc::RMW_UINC_WRAP;
717 case AtomicRMWInst::UDecWrap:
718 return bitc::RMW_UDEC_WRAP;
719 case AtomicRMWInst::USubCond:
720 return bitc::RMW_USUB_COND;
721 case AtomicRMWInst::USubSat:
722 return bitc::RMW_USUB_SAT;
723 }
724}
725
726static unsigned getEncodedOrdering(AtomicOrdering Ordering) {
727 switch (Ordering) {
728 case AtomicOrdering::NotAtomic: return bitc::ORDERING_NOTATOMIC;
729 case AtomicOrdering::Unordered: return bitc::ORDERING_UNORDERED;
730 case AtomicOrdering::Monotonic: return bitc::ORDERING_MONOTONIC;
731 case AtomicOrdering::Acquire: return bitc::ORDERING_ACQUIRE;
732 case AtomicOrdering::Release: return bitc::ORDERING_RELEASE;
733 case AtomicOrdering::AcquireRelease: return bitc::ORDERING_ACQREL;
734 case AtomicOrdering::SequentiallyConsistent: return bitc::ORDERING_SEQCST;
735 }
736 llvm_unreachable("Invalid ordering");
737}
738
739static void writeStringRecord(BitstreamWriter &Stream, unsigned Code,
740 StringRef Str, unsigned AbbrevToUse) {
741 SmallVector<unsigned, 64> Vals;
742
743 // Code: [strchar x N]
744 for (char C : Str) {
745 if (AbbrevToUse && !BitCodeAbbrevOp::isChar6(C))
746 AbbrevToUse = 0;
747 Vals.push_back(Elt: C);
748 }
749
750 // Emit the finished record.
751 Stream.EmitRecord(Code, Vals, Abbrev: AbbrevToUse);
752}
753
754static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
755 switch (Kind) {
756 case Attribute::Alignment:
757 return bitc::ATTR_KIND_ALIGNMENT;
758 case Attribute::AllocAlign:
759 return bitc::ATTR_KIND_ALLOC_ALIGN;
760 case Attribute::AllocSize:
761 return bitc::ATTR_KIND_ALLOC_SIZE;
762 case Attribute::AlwaysInline:
763 return bitc::ATTR_KIND_ALWAYS_INLINE;
764 case Attribute::Builtin:
765 return bitc::ATTR_KIND_BUILTIN;
766 case Attribute::ByVal:
767 return bitc::ATTR_KIND_BY_VAL;
768 case Attribute::Convergent:
769 return bitc::ATTR_KIND_CONVERGENT;
770 case Attribute::InAlloca:
771 return bitc::ATTR_KIND_IN_ALLOCA;
772 case Attribute::Cold:
773 return bitc::ATTR_KIND_COLD;
774 case Attribute::DisableSanitizerInstrumentation:
775 return bitc::ATTR_KIND_DISABLE_SANITIZER_INSTRUMENTATION;
776 case Attribute::FnRetThunkExtern:
777 return bitc::ATTR_KIND_FNRETTHUNK_EXTERN;
778 case Attribute::Hot:
779 return bitc::ATTR_KIND_HOT;
780 case Attribute::ElementType:
781 return bitc::ATTR_KIND_ELEMENTTYPE;
782 case Attribute::HybridPatchable:
783 return bitc::ATTR_KIND_HYBRID_PATCHABLE;
784 case Attribute::InlineHint:
785 return bitc::ATTR_KIND_INLINE_HINT;
786 case Attribute::InReg:
787 return bitc::ATTR_KIND_IN_REG;
788 case Attribute::JumpTable:
789 return bitc::ATTR_KIND_JUMP_TABLE;
790 case Attribute::MinSize:
791 return bitc::ATTR_KIND_MIN_SIZE;
792 case Attribute::AllocatedPointer:
793 return bitc::ATTR_KIND_ALLOCATED_POINTER;
794 case Attribute::AllocKind:
795 return bitc::ATTR_KIND_ALLOC_KIND;
796 case Attribute::Memory:
797 return bitc::ATTR_KIND_MEMORY;
798 case Attribute::NoFPClass:
799 return bitc::ATTR_KIND_NOFPCLASS;
800 case Attribute::Naked:
801 return bitc::ATTR_KIND_NAKED;
802 case Attribute::Nest:
803 return bitc::ATTR_KIND_NEST;
804 case Attribute::NoAlias:
805 return bitc::ATTR_KIND_NO_ALIAS;
806 case Attribute::NoBuiltin:
807 return bitc::ATTR_KIND_NO_BUILTIN;
808 case Attribute::NoCallback:
809 return bitc::ATTR_KIND_NO_CALLBACK;
810 case Attribute::NoDivergenceSource:
811 return bitc::ATTR_KIND_NO_DIVERGENCE_SOURCE;
812 case Attribute::NoDuplicate:
813 return bitc::ATTR_KIND_NO_DUPLICATE;
814 case Attribute::NoFree:
815 return bitc::ATTR_KIND_NOFREE;
816 case Attribute::NoImplicitFloat:
817 return bitc::ATTR_KIND_NO_IMPLICIT_FLOAT;
818 case Attribute::NoInline:
819 return bitc::ATTR_KIND_NO_INLINE;
820 case Attribute::NoRecurse:
821 return bitc::ATTR_KIND_NO_RECURSE;
822 case Attribute::NoMerge:
823 return bitc::ATTR_KIND_NO_MERGE;
824 case Attribute::NonLazyBind:
825 return bitc::ATTR_KIND_NON_LAZY_BIND;
826 case Attribute::NonNull:
827 return bitc::ATTR_KIND_NON_NULL;
828 case Attribute::Dereferenceable:
829 return bitc::ATTR_KIND_DEREFERENCEABLE;
830 case Attribute::DereferenceableOrNull:
831 return bitc::ATTR_KIND_DEREFERENCEABLE_OR_NULL;
832 case Attribute::NoRedZone:
833 return bitc::ATTR_KIND_NO_RED_ZONE;
834 case Attribute::NoReturn:
835 return bitc::ATTR_KIND_NO_RETURN;
836 case Attribute::NoSync:
837 return bitc::ATTR_KIND_NOSYNC;
838 case Attribute::NoCfCheck:
839 return bitc::ATTR_KIND_NOCF_CHECK;
840 case Attribute::NoProfile:
841 return bitc::ATTR_KIND_NO_PROFILE;
842 case Attribute::SkipProfile:
843 return bitc::ATTR_KIND_SKIP_PROFILE;
844 case Attribute::NoUnwind:
845 return bitc::ATTR_KIND_NO_UNWIND;
846 case Attribute::NoSanitizeBounds:
847 return bitc::ATTR_KIND_NO_SANITIZE_BOUNDS;
848 case Attribute::NoSanitizeCoverage:
849 return bitc::ATTR_KIND_NO_SANITIZE_COVERAGE;
850 case Attribute::NullPointerIsValid:
851 return bitc::ATTR_KIND_NULL_POINTER_IS_VALID;
852 case Attribute::OptimizeForDebugging:
853 return bitc::ATTR_KIND_OPTIMIZE_FOR_DEBUGGING;
854 case Attribute::OptForFuzzing:
855 return bitc::ATTR_KIND_OPT_FOR_FUZZING;
856 case Attribute::OptimizeForSize:
857 return bitc::ATTR_KIND_OPTIMIZE_FOR_SIZE;
858 case Attribute::OptimizeNone:
859 return bitc::ATTR_KIND_OPTIMIZE_NONE;
860 case Attribute::ReadNone:
861 return bitc::ATTR_KIND_READ_NONE;
862 case Attribute::ReadOnly:
863 return bitc::ATTR_KIND_READ_ONLY;
864 case Attribute::Returned:
865 return bitc::ATTR_KIND_RETURNED;
866 case Attribute::ReturnsTwice:
867 return bitc::ATTR_KIND_RETURNS_TWICE;
868 case Attribute::SExt:
869 return bitc::ATTR_KIND_S_EXT;
870 case Attribute::Speculatable:
871 return bitc::ATTR_KIND_SPECULATABLE;
872 case Attribute::StackAlignment:
873 return bitc::ATTR_KIND_STACK_ALIGNMENT;
874 case Attribute::StackProtect:
875 return bitc::ATTR_KIND_STACK_PROTECT;
876 case Attribute::StackProtectReq:
877 return bitc::ATTR_KIND_STACK_PROTECT_REQ;
878 case Attribute::StackProtectStrong:
879 return bitc::ATTR_KIND_STACK_PROTECT_STRONG;
880 case Attribute::SafeStack:
881 return bitc::ATTR_KIND_SAFESTACK;
882 case Attribute::ShadowCallStack:
883 return bitc::ATTR_KIND_SHADOWCALLSTACK;
884 case Attribute::StrictFP:
885 return bitc::ATTR_KIND_STRICT_FP;
886 case Attribute::StructRet:
887 return bitc::ATTR_KIND_STRUCT_RET;
888 case Attribute::SanitizeAddress:
889 return bitc::ATTR_KIND_SANITIZE_ADDRESS;
890 case Attribute::SanitizeAllocToken:
891 return bitc::ATTR_KIND_SANITIZE_ALLOC_TOKEN;
892 case Attribute::SanitizeHWAddress:
893 return bitc::ATTR_KIND_SANITIZE_HWADDRESS;
894 case Attribute::SanitizeThread:
895 return bitc::ATTR_KIND_SANITIZE_THREAD;
896 case Attribute::SanitizeType:
897 return bitc::ATTR_KIND_SANITIZE_TYPE;
898 case Attribute::SanitizeMemory:
899 return bitc::ATTR_KIND_SANITIZE_MEMORY;
900 case Attribute::SanitizeNumericalStability:
901 return bitc::ATTR_KIND_SANITIZE_NUMERICAL_STABILITY;
902 case Attribute::SanitizeRealtime:
903 return bitc::ATTR_KIND_SANITIZE_REALTIME;
904 case Attribute::SanitizeRealtimeBlocking:
905 return bitc::ATTR_KIND_SANITIZE_REALTIME_BLOCKING;
906 case Attribute::SpeculativeLoadHardening:
907 return bitc::ATTR_KIND_SPECULATIVE_LOAD_HARDENING;
908 case Attribute::SwiftError:
909 return bitc::ATTR_KIND_SWIFT_ERROR;
910 case Attribute::SwiftSelf:
911 return bitc::ATTR_KIND_SWIFT_SELF;
912 case Attribute::SwiftAsync:
913 return bitc::ATTR_KIND_SWIFT_ASYNC;
914 case Attribute::UWTable:
915 return bitc::ATTR_KIND_UW_TABLE;
916 case Attribute::VScaleRange:
917 return bitc::ATTR_KIND_VSCALE_RANGE;
918 case Attribute::WillReturn:
919 return bitc::ATTR_KIND_WILLRETURN;
920 case Attribute::WriteOnly:
921 return bitc::ATTR_KIND_WRITEONLY;
922 case Attribute::ZExt:
923 return bitc::ATTR_KIND_Z_EXT;
924 case Attribute::ImmArg:
925 return bitc::ATTR_KIND_IMMARG;
926 case Attribute::SanitizeMemTag:
927 return bitc::ATTR_KIND_SANITIZE_MEMTAG;
928 case Attribute::Preallocated:
929 return bitc::ATTR_KIND_PREALLOCATED;
930 case Attribute::NoUndef:
931 return bitc::ATTR_KIND_NOUNDEF;
932 case Attribute::ByRef:
933 return bitc::ATTR_KIND_BYREF;
934 case Attribute::MustProgress:
935 return bitc::ATTR_KIND_MUSTPROGRESS;
936 case Attribute::PresplitCoroutine:
937 return bitc::ATTR_KIND_PRESPLIT_COROUTINE;
938 case Attribute::Writable:
939 return bitc::ATTR_KIND_WRITABLE;
940 case Attribute::CoroDestroyOnlyWhenComplete:
941 return bitc::ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE;
942 case Attribute::CoroElideSafe:
943 return bitc::ATTR_KIND_CORO_ELIDE_SAFE;
944 case Attribute::DeadOnUnwind:
945 return bitc::ATTR_KIND_DEAD_ON_UNWIND;
946 case Attribute::Range:
947 return bitc::ATTR_KIND_RANGE;
948 case Attribute::Initializes:
949 return bitc::ATTR_KIND_INITIALIZES;
950 case Attribute::NoExt:
951 return bitc::ATTR_KIND_NO_EXT;
952 case Attribute::Captures:
953 return bitc::ATTR_KIND_CAPTURES;
954 case Attribute::DeadOnReturn:
955 return bitc::ATTR_KIND_DEAD_ON_RETURN;
956 case Attribute::NoCreateUndefOrPoison:
957 return bitc::ATTR_KIND_NO_CREATE_UNDEF_OR_POISON;
958 case Attribute::DenormalFPEnv:
959 return bitc::ATTR_KIND_DENORMAL_FPENV;
960 case Attribute::NoOutline:
961 return bitc::ATTR_KIND_NOOUTLINE;
962 case Attribute::EndAttrKinds:
963 llvm_unreachable("Can not encode end-attribute kinds marker.");
964 case Attribute::None:
965 llvm_unreachable("Can not encode none-attribute.");
966 case Attribute::EmptyKey:
967 case Attribute::TombstoneKey:
968 llvm_unreachable("Trying to encode EmptyKey/TombstoneKey");
969 }
970
971 llvm_unreachable("Trying to encode unknown attribute");
972}
973
974static void emitSignedInt64(SmallVectorImpl<uint64_t> &Vals, uint64_t V) {
975 if ((int64_t)V >= 0)
976 Vals.push_back(Elt: V << 1);
977 else
978 Vals.push_back(Elt: (-V << 1) | 1);
979}
980
981static void emitWideAPInt(SmallVectorImpl<uint64_t> &Vals, const APInt &A) {
982 // We have an arbitrary precision integer value to write whose
983 // bit width is > 64. However, in canonical unsigned integer
984 // format it is likely that the high bits are going to be zero.
985 // So, we only write the number of active words.
986 unsigned NumWords = A.getActiveWords();
987 const uint64_t *RawData = A.getRawData();
988 for (unsigned i = 0; i < NumWords; i++)
989 emitSignedInt64(Vals, V: RawData[i]);
990}
991
992static void emitConstantRange(SmallVectorImpl<uint64_t> &Record,
993 const ConstantRange &CR, bool EmitBitWidth) {
994 unsigned BitWidth = CR.getBitWidth();
995 if (EmitBitWidth)
996 Record.push_back(Elt: BitWidth);
997 if (BitWidth > 64) {
998 Record.push_back(Elt: CR.getLower().getActiveWords() |
999 (uint64_t(CR.getUpper().getActiveWords()) << 32));
1000 emitWideAPInt(Vals&: Record, A: CR.getLower());
1001 emitWideAPInt(Vals&: Record, A: CR.getUpper());
1002 } else {
1003 emitSignedInt64(Vals&: Record, V: CR.getLower().getSExtValue());
1004 emitSignedInt64(Vals&: Record, V: CR.getUpper().getSExtValue());
1005 }
1006}
1007
1008void ModuleBitcodeWriter::writeAttributeGroupTable() {
1009 const std::vector<ValueEnumerator::IndexAndAttrSet> &AttrGrps =
1010 VE.getAttributeGroups();
1011 if (AttrGrps.empty()) return;
1012
1013 Stream.EnterSubblock(BlockID: bitc::PARAMATTR_GROUP_BLOCK_ID, CodeLen: 3);
1014
1015 SmallVector<uint64_t, 64> Record;
1016 for (ValueEnumerator::IndexAndAttrSet Pair : AttrGrps) {
1017 unsigned AttrListIndex = Pair.first;
1018 AttributeSet AS = Pair.second;
1019 Record.push_back(Elt: VE.getAttributeGroupID(Group: Pair));
1020 Record.push_back(Elt: AttrListIndex);
1021
1022 for (Attribute Attr : AS) {
1023 if (Attr.isEnumAttribute()) {
1024 Record.push_back(Elt: 0);
1025 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1026 } else if (Attr.isIntAttribute()) {
1027 Record.push_back(Elt: 1);
1028 Attribute::AttrKind Kind = Attr.getKindAsEnum();
1029 Record.push_back(Elt: getAttrKindEncoding(Kind));
1030 if (Kind == Attribute::Memory) {
1031 // Version field for upgrading old memory effects.
1032 const uint64_t Version = 1;
1033 Record.push_back(Elt: (Version << 56) | Attr.getValueAsInt());
1034 } else {
1035 Record.push_back(Elt: Attr.getValueAsInt());
1036 }
1037 } else if (Attr.isStringAttribute()) {
1038 StringRef Kind = Attr.getKindAsString();
1039 StringRef Val = Attr.getValueAsString();
1040
1041 Record.push_back(Elt: Val.empty() ? 3 : 4);
1042 Record.append(in_start: Kind.begin(), in_end: Kind.end());
1043 Record.push_back(Elt: 0);
1044 if (!Val.empty()) {
1045 Record.append(in_start: Val.begin(), in_end: Val.end());
1046 Record.push_back(Elt: 0);
1047 }
1048 } else if (Attr.isTypeAttribute()) {
1049 Type *Ty = Attr.getValueAsType();
1050 Record.push_back(Elt: Ty ? 6 : 5);
1051 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1052 if (Ty)
1053 Record.push_back(Elt: VE.getTypeID(T: Attr.getValueAsType()));
1054 } else if (Attr.isConstantRangeAttribute()) {
1055 Record.push_back(Elt: 7);
1056 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1057 emitConstantRange(Record, CR: Attr.getValueAsConstantRange(),
1058 /*EmitBitWidth=*/true);
1059 } else {
1060 assert(Attr.isConstantRangeListAttribute());
1061 Record.push_back(Elt: 8);
1062 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1063 ArrayRef<ConstantRange> Val = Attr.getValueAsConstantRangeList();
1064 Record.push_back(Elt: Val.size());
1065 Record.push_back(Elt: Val[0].getBitWidth());
1066 for (auto &CR : Val)
1067 emitConstantRange(Record, CR, /*EmitBitWidth=*/false);
1068 }
1069 }
1070
1071 Stream.EmitRecord(Code: bitc::PARAMATTR_GRP_CODE_ENTRY, Vals: Record);
1072 Record.clear();
1073 }
1074
1075 Stream.ExitBlock();
1076}
1077
1078void ModuleBitcodeWriter::writeAttributeTable() {
1079 const std::vector<AttributeList> &Attrs = VE.getAttributeLists();
1080 if (Attrs.empty()) return;
1081
1082 Stream.EnterSubblock(BlockID: bitc::PARAMATTR_BLOCK_ID, CodeLen: 3);
1083
1084 SmallVector<uint64_t, 64> Record;
1085 for (const AttributeList &AL : Attrs) {
1086 for (unsigned i : AL.indexes()) {
1087 AttributeSet AS = AL.getAttributes(Index: i);
1088 if (AS.hasAttributes())
1089 Record.push_back(Elt: VE.getAttributeGroupID(Group: {i, AS}));
1090 }
1091
1092 Stream.EmitRecord(Code: bitc::PARAMATTR_CODE_ENTRY, Vals: Record);
1093 Record.clear();
1094 }
1095
1096 Stream.ExitBlock();
1097}
1098
1099/// WriteTypeTable - Write out the type table for a module.
1100void ModuleBitcodeWriter::writeTypeTable() {
1101 const ValueEnumerator::TypeList &TypeList = VE.getTypes();
1102
1103 Stream.EnterSubblock(BlockID: bitc::TYPE_BLOCK_ID_NEW, CodeLen: 4 /*count from # abbrevs */);
1104 SmallVector<uint64_t, 64> TypeVals;
1105
1106 uint64_t NumBits = VE.computeBitsRequiredForTypeIndices();
1107
1108 // Abbrev for TYPE_CODE_OPAQUE_POINTER.
1109 auto Abbv = std::make_shared<BitCodeAbbrev>();
1110 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_OPAQUE_POINTER));
1111 Abbv->Add(OpInfo: BitCodeAbbrevOp(0)); // Addrspace = 0
1112 unsigned OpaquePtrAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1113
1114 // Abbrev for TYPE_CODE_FUNCTION.
1115 Abbv = std::make_shared<BitCodeAbbrev>();
1116 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_FUNCTION));
1117 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isvararg
1118 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1119 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1120 unsigned FunctionAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1121
1122 // Abbrev for TYPE_CODE_STRUCT_ANON.
1123 Abbv = std::make_shared<BitCodeAbbrev>();
1124 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_ANON));
1125 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked
1126 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1127 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1128 unsigned StructAnonAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1129
1130 // Abbrev for TYPE_CODE_STRUCT_NAME.
1131 Abbv = std::make_shared<BitCodeAbbrev>();
1132 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAME));
1133 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1134 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
1135 unsigned StructNameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1136
1137 // Abbrev for TYPE_CODE_STRUCT_NAMED.
1138 Abbv = std::make_shared<BitCodeAbbrev>();
1139 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAMED));
1140 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked
1141 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1142 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1143 unsigned StructNamedAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1144
1145 // Abbrev for TYPE_CODE_ARRAY.
1146 Abbv = std::make_shared<BitCodeAbbrev>();
1147 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_ARRAY));
1148 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // size
1149 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1150 unsigned ArrayAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1151
1152 // Emit an entry count so the reader can reserve space.
1153 TypeVals.push_back(Elt: TypeList.size());
1154 Stream.EmitRecord(Code: bitc::TYPE_CODE_NUMENTRY, Vals: TypeVals);
1155 TypeVals.clear();
1156
1157 // Loop over all of the types, emitting each in turn.
1158 for (Type *T : TypeList) {
1159 int AbbrevToUse = 0;
1160 unsigned Code = 0;
1161
1162 switch (T->getTypeID()) {
1163 case Type::VoidTyID: Code = bitc::TYPE_CODE_VOID; break;
1164 case Type::HalfTyID: Code = bitc::TYPE_CODE_HALF; break;
1165 case Type::BFloatTyID: Code = bitc::TYPE_CODE_BFLOAT; break;
1166 case Type::FloatTyID: Code = bitc::TYPE_CODE_FLOAT; break;
1167 case Type::DoubleTyID: Code = bitc::TYPE_CODE_DOUBLE; break;
1168 case Type::X86_FP80TyID: Code = bitc::TYPE_CODE_X86_FP80; break;
1169 case Type::FP128TyID: Code = bitc::TYPE_CODE_FP128; break;
1170 case Type::PPC_FP128TyID: Code = bitc::TYPE_CODE_PPC_FP128; break;
1171 case Type::LabelTyID: Code = bitc::TYPE_CODE_LABEL; break;
1172 case Type::MetadataTyID:
1173 Code = bitc::TYPE_CODE_METADATA;
1174 break;
1175 case Type::X86_AMXTyID: Code = bitc::TYPE_CODE_X86_AMX; break;
1176 case Type::TokenTyID: Code = bitc::TYPE_CODE_TOKEN; break;
1177 case Type::ByteTyID:
1178 // BYTE: [width]
1179 Code = bitc::TYPE_CODE_BYTE;
1180 TypeVals.push_back(Elt: T->getByteBitWidth());
1181 break;
1182 case Type::IntegerTyID:
1183 // INTEGER: [width]
1184 Code = bitc::TYPE_CODE_INTEGER;
1185 TypeVals.push_back(Elt: cast<IntegerType>(Val: T)->getBitWidth());
1186 break;
1187 case Type::PointerTyID: {
1188 PointerType *PTy = cast<PointerType>(Val: T);
1189 unsigned AddressSpace = PTy->getAddressSpace();
1190 // OPAQUE_POINTER: [address space]
1191 Code = bitc::TYPE_CODE_OPAQUE_POINTER;
1192 TypeVals.push_back(Elt: AddressSpace);
1193 if (AddressSpace == 0)
1194 AbbrevToUse = OpaquePtrAbbrev;
1195 break;
1196 }
1197 case Type::FunctionTyID: {
1198 FunctionType *FT = cast<FunctionType>(Val: T);
1199 // FUNCTION: [isvararg, retty, paramty x N]
1200 Code = bitc::TYPE_CODE_FUNCTION;
1201 TypeVals.push_back(Elt: FT->isVarArg());
1202 TypeVals.push_back(Elt: VE.getTypeID(T: FT->getReturnType()));
1203 for (unsigned i = 0, e = FT->getNumParams(); i != e; ++i)
1204 TypeVals.push_back(Elt: VE.getTypeID(T: FT->getParamType(i)));
1205 AbbrevToUse = FunctionAbbrev;
1206 break;
1207 }
1208 case Type::StructTyID: {
1209 StructType *ST = cast<StructType>(Val: T);
1210 // STRUCT: [ispacked, eltty x N]
1211 TypeVals.push_back(Elt: ST->isPacked());
1212 // Output all of the element types.
1213 for (Type *ET : ST->elements())
1214 TypeVals.push_back(Elt: VE.getTypeID(T: ET));
1215
1216 if (ST->isLiteral()) {
1217 Code = bitc::TYPE_CODE_STRUCT_ANON;
1218 AbbrevToUse = StructAnonAbbrev;
1219 } else {
1220 if (ST->isOpaque()) {
1221 Code = bitc::TYPE_CODE_OPAQUE;
1222 } else {
1223 Code = bitc::TYPE_CODE_STRUCT_NAMED;
1224 AbbrevToUse = StructNamedAbbrev;
1225 }
1226
1227 // Emit the name if it is present.
1228 if (!ST->getName().empty())
1229 writeStringRecord(Stream, Code: bitc::TYPE_CODE_STRUCT_NAME, Str: ST->getName(),
1230 AbbrevToUse: StructNameAbbrev);
1231 }
1232 break;
1233 }
1234 case Type::ArrayTyID: {
1235 ArrayType *AT = cast<ArrayType>(Val: T);
1236 // ARRAY: [numelts, eltty]
1237 Code = bitc::TYPE_CODE_ARRAY;
1238 TypeVals.push_back(Elt: AT->getNumElements());
1239 TypeVals.push_back(Elt: VE.getTypeID(T: AT->getElementType()));
1240 AbbrevToUse = ArrayAbbrev;
1241 break;
1242 }
1243 case Type::FixedVectorTyID:
1244 case Type::ScalableVectorTyID: {
1245 VectorType *VT = cast<VectorType>(Val: T);
1246 // VECTOR [numelts, eltty] or
1247 // [numelts, eltty, scalable]
1248 Code = bitc::TYPE_CODE_VECTOR;
1249 TypeVals.push_back(Elt: VT->getElementCount().getKnownMinValue());
1250 TypeVals.push_back(Elt: VE.getTypeID(T: VT->getElementType()));
1251 if (isa<ScalableVectorType>(Val: VT))
1252 TypeVals.push_back(Elt: true);
1253 break;
1254 }
1255 case Type::TargetExtTyID: {
1256 TargetExtType *TET = cast<TargetExtType>(Val: T);
1257 Code = bitc::TYPE_CODE_TARGET_TYPE;
1258 writeStringRecord(Stream, Code: bitc::TYPE_CODE_STRUCT_NAME, Str: TET->getName(),
1259 AbbrevToUse: StructNameAbbrev);
1260 TypeVals.push_back(Elt: TET->getNumTypeParameters());
1261 for (Type *InnerTy : TET->type_params())
1262 TypeVals.push_back(Elt: VE.getTypeID(T: InnerTy));
1263 llvm::append_range(C&: TypeVals, R: TET->int_params());
1264 break;
1265 }
1266 case Type::TypedPointerTyID:
1267 llvm_unreachable("Typed pointers cannot be added to IR modules");
1268 }
1269
1270 // Emit the finished record.
1271 Stream.EmitRecord(Code, Vals: TypeVals, Abbrev: AbbrevToUse);
1272 TypeVals.clear();
1273 }
1274
1275 Stream.ExitBlock();
1276}
1277
1278static unsigned getEncodedLinkage(const GlobalValue::LinkageTypes Linkage) {
1279 switch (Linkage) {
1280 case GlobalValue::ExternalLinkage:
1281 return 0;
1282 case GlobalValue::WeakAnyLinkage:
1283 return 16;
1284 case GlobalValue::AppendingLinkage:
1285 return 2;
1286 case GlobalValue::InternalLinkage:
1287 return 3;
1288 case GlobalValue::LinkOnceAnyLinkage:
1289 return 18;
1290 case GlobalValue::ExternalWeakLinkage:
1291 return 7;
1292 case GlobalValue::CommonLinkage:
1293 return 8;
1294 case GlobalValue::PrivateLinkage:
1295 return 9;
1296 case GlobalValue::WeakODRLinkage:
1297 return 17;
1298 case GlobalValue::LinkOnceODRLinkage:
1299 return 19;
1300 case GlobalValue::AvailableExternallyLinkage:
1301 return 12;
1302 }
1303 llvm_unreachable("Invalid linkage");
1304}
1305
1306static unsigned getEncodedLinkage(const GlobalValue &GV) {
1307 return getEncodedLinkage(Linkage: GV.getLinkage());
1308}
1309
1310static uint64_t getEncodedFFlags(FunctionSummary::FFlags Flags) {
1311 uint64_t RawFlags = 0;
1312 RawFlags |= Flags.ReadNone;
1313 RawFlags |= (Flags.ReadOnly << 1);
1314 RawFlags |= (Flags.NoRecurse << 2);
1315 RawFlags |= (Flags.ReturnDoesNotAlias << 3);
1316 RawFlags |= (Flags.NoInline << 4);
1317 RawFlags |= (Flags.AlwaysInline << 5);
1318 RawFlags |= (Flags.NoUnwind << 6);
1319 RawFlags |= (Flags.MayThrow << 7);
1320 RawFlags |= (Flags.HasUnknownCall << 8);
1321 RawFlags |= (Flags.MustBeUnreachable << 9);
1322 return RawFlags;
1323}
1324
1325// Decode the flags for GlobalValue in the summary. See getDecodedGVSummaryFlags
1326// in BitcodeReader.cpp.
1327static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags,
1328 bool ImportAsDecl = false) {
1329 uint64_t RawFlags = 0;
1330
1331 RawFlags |= Flags.NotEligibleToImport; // bool
1332 RawFlags |= (Flags.Live << 1);
1333 RawFlags |= (Flags.DSOLocal << 2);
1334 RawFlags |= (Flags.CanAutoHide << 3);
1335
1336 // Linkage don't need to be remapped at that time for the summary. Any future
1337 // change to the getEncodedLinkage() function will need to be taken into
1338 // account here as well.
1339 RawFlags = (RawFlags << 4) | Flags.Linkage; // 4 bits
1340
1341 RawFlags |= (Flags.Visibility << 8); // 2 bits
1342
1343 unsigned ImportType = Flags.ImportType | ImportAsDecl;
1344 RawFlags |= (ImportType << 10); // 1 bit
1345
1346 RawFlags |= (Flags.NoRenameOnPromotion << 11); // 1 bit
1347
1348 return RawFlags;
1349}
1350
1351static uint64_t getEncodedGVarFlags(GlobalVarSummary::GVarFlags Flags) {
1352 uint64_t RawFlags = Flags.MaybeReadOnly | (Flags.MaybeWriteOnly << 1) |
1353 (Flags.Constant << 2) | Flags.VCallVisibility << 3;
1354 return RawFlags;
1355}
1356
1357static uint64_t getEncodedHotnessCallEdgeInfo(const CalleeInfo &CI) {
1358 uint64_t RawFlags = 0;
1359
1360 RawFlags |= CI.Hotness; // 3 bits
1361 RawFlags |= (CI.HasTailCall << 3); // 1 bit
1362
1363 return RawFlags;
1364}
1365
1366static unsigned getEncodedVisibility(const GlobalValue &GV) {
1367 switch (GV.getVisibility()) {
1368 case GlobalValue::DefaultVisibility: return 0;
1369 case GlobalValue::HiddenVisibility: return 1;
1370 case GlobalValue::ProtectedVisibility: return 2;
1371 }
1372 llvm_unreachable("Invalid visibility");
1373}
1374
1375static unsigned getEncodedDLLStorageClass(const GlobalValue &GV) {
1376 switch (GV.getDLLStorageClass()) {
1377 case GlobalValue::DefaultStorageClass: return 0;
1378 case GlobalValue::DLLImportStorageClass: return 1;
1379 case GlobalValue::DLLExportStorageClass: return 2;
1380 }
1381 llvm_unreachable("Invalid DLL storage class");
1382}
1383
1384static unsigned getEncodedThreadLocalMode(const GlobalValue &GV) {
1385 switch (GV.getThreadLocalMode()) {
1386 case GlobalVariable::NotThreadLocal: return 0;
1387 case GlobalVariable::GeneralDynamicTLSModel: return 1;
1388 case GlobalVariable::LocalDynamicTLSModel: return 2;
1389 case GlobalVariable::InitialExecTLSModel: return 3;
1390 case GlobalVariable::LocalExecTLSModel: return 4;
1391 }
1392 llvm_unreachable("Invalid TLS model");
1393}
1394
1395static unsigned getEncodedComdatSelectionKind(const Comdat &C) {
1396 switch (C.getSelectionKind()) {
1397 case Comdat::Any:
1398 return bitc::COMDAT_SELECTION_KIND_ANY;
1399 case Comdat::ExactMatch:
1400 return bitc::COMDAT_SELECTION_KIND_EXACT_MATCH;
1401 case Comdat::Largest:
1402 return bitc::COMDAT_SELECTION_KIND_LARGEST;
1403 case Comdat::NoDeduplicate:
1404 return bitc::COMDAT_SELECTION_KIND_NO_DUPLICATES;
1405 case Comdat::SameSize:
1406 return bitc::COMDAT_SELECTION_KIND_SAME_SIZE;
1407 }
1408 llvm_unreachable("Invalid selection kind");
1409}
1410
1411static unsigned getEncodedUnnamedAddr(const GlobalValue &GV) {
1412 switch (GV.getUnnamedAddr()) {
1413 case GlobalValue::UnnamedAddr::None: return 0;
1414 case GlobalValue::UnnamedAddr::Local: return 2;
1415 case GlobalValue::UnnamedAddr::Global: return 1;
1416 }
1417 llvm_unreachable("Invalid unnamed_addr");
1418}
1419
1420size_t ModuleBitcodeWriter::addToStrtab(StringRef Str) {
1421 if (GenerateHash)
1422 Hasher.update(Str);
1423 return StrtabBuilder.add(S: Str);
1424}
1425
1426void ModuleBitcodeWriter::writeComdats() {
1427 SmallVector<unsigned, 64> Vals;
1428 for (const Comdat *C : VE.getComdats()) {
1429 // COMDAT: [strtab offset, strtab size, selection_kind]
1430 Vals.push_back(Elt: addToStrtab(Str: C->getName()));
1431 Vals.push_back(Elt: C->getName().size());
1432 Vals.push_back(Elt: getEncodedComdatSelectionKind(C: *C));
1433 Stream.EmitRecord(Code: bitc::MODULE_CODE_COMDAT, Vals, /*AbbrevToUse=*/Abbrev: 0);
1434 Vals.clear();
1435 }
1436}
1437
1438/// Write a record that will eventually hold the word offset of the
1439/// module-level VST. For now the offset is 0, which will be backpatched
1440/// after the real VST is written. Saves the bit offset to backpatch.
1441void ModuleBitcodeWriter::writeValueSymbolTableForwardDecl() {
1442 // Write a placeholder value in for the offset of the real VST,
1443 // which is written after the function blocks so that it can include
1444 // the offset of each function. The placeholder offset will be
1445 // updated when the real VST is written.
1446 auto Abbv = std::make_shared<BitCodeAbbrev>();
1447 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_VSTOFFSET));
1448 // Blocks are 32-bit aligned, so we can use a 32-bit word offset to
1449 // hold the real VST offset. Must use fixed instead of VBR as we don't
1450 // know how many VBR chunks to reserve ahead of time.
1451 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
1452 unsigned VSTOffsetAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1453
1454 // Emit the placeholder
1455 uint64_t Vals[] = {bitc::MODULE_CODE_VSTOFFSET, 0};
1456 Stream.EmitRecordWithAbbrev(Abbrev: VSTOffsetAbbrev, Vals);
1457
1458 // Compute and save the bit offset to the placeholder, which will be
1459 // patched when the real VST is written. We can simply subtract the 32-bit
1460 // fixed size from the current bit number to get the location to backpatch.
1461 VSTOffsetPlaceholder = Stream.GetCurrentBitNo() - 32;
1462}
1463
1464enum StringEncoding { SE_Char6, SE_Fixed7, SE_Fixed8 };
1465
1466/// Determine the encoding to use for the given string name and length.
1467static StringEncoding getStringEncoding(StringRef Str) {
1468 bool isChar6 = true;
1469 for (char C : Str) {
1470 if (isChar6)
1471 isChar6 = BitCodeAbbrevOp::isChar6(C);
1472 if ((unsigned char)C & 128)
1473 // don't bother scanning the rest.
1474 return SE_Fixed8;
1475 }
1476 if (isChar6)
1477 return SE_Char6;
1478 return SE_Fixed7;
1479}
1480
1481static_assert(sizeof(GlobalValue::SanitizerMetadata) <= sizeof(unsigned),
1482 "Sanitizer Metadata is too large for naive serialization.");
1483static unsigned
1484serializeSanitizerMetadata(const GlobalValue::SanitizerMetadata &Meta) {
1485 return Meta.NoAddress | (Meta.NoHWAddress << 1) |
1486 (Meta.Memtag << 2) | (Meta.IsDynInit << 3);
1487}
1488
1489/// Emit top-level description of module, including target triple, inline asm,
1490/// descriptors for global variables, and function prototype info.
1491/// Returns the bit offset to backpatch with the location of the real VST.
1492void ModuleBitcodeWriter::writeModuleInfo() {
1493 // Emit various pieces of data attached to a module.
1494 if (!M.getTargetTriple().empty())
1495 writeStringRecord(Stream, Code: bitc::MODULE_CODE_TRIPLE,
1496 Str: M.getTargetTriple().str(), AbbrevToUse: 0 /*TODO*/);
1497 const std::string &DL = M.getDataLayoutStr();
1498 if (!DL.empty())
1499 writeStringRecord(Stream, Code: bitc::MODULE_CODE_DATALAYOUT, Str: DL, AbbrevToUse: 0 /*TODO*/);
1500 if (!M.getModuleInlineAsm().empty())
1501 writeStringRecord(Stream, Code: bitc::MODULE_CODE_ASM, Str: M.getModuleInlineAsm(),
1502 AbbrevToUse: 0 /*TODO*/);
1503
1504 // Emit information about sections and GC, computing how many there are. Also
1505 // compute the maximum alignment value.
1506 std::map<std::string, unsigned> SectionMap;
1507 std::map<std::string, unsigned> GCMap;
1508 MaybeAlign MaxGVarAlignment;
1509 unsigned MaxGlobalType = 0;
1510 for (const GlobalVariable &GV : M.globals()) {
1511 if (MaybeAlign A = GV.getAlign())
1512 MaxGVarAlignment = !MaxGVarAlignment ? *A : std::max(a: *MaxGVarAlignment, b: *A);
1513 MaxGlobalType = std::max(a: MaxGlobalType, b: VE.getTypeID(T: GV.getValueType()));
1514 if (GV.hasSection()) {
1515 // Give section names unique ID's.
1516 unsigned &Entry = SectionMap[std::string(GV.getSection())];
1517 if (!Entry) {
1518 writeStringRecord(Stream, Code: bitc::MODULE_CODE_SECTIONNAME, Str: GV.getSection(),
1519 AbbrevToUse: 0 /*TODO*/);
1520 Entry = SectionMap.size();
1521 }
1522 }
1523 }
1524 for (const Function &F : M) {
1525 if (F.hasSection()) {
1526 // Give section names unique ID's.
1527 unsigned &Entry = SectionMap[std::string(F.getSection())];
1528 if (!Entry) {
1529 writeStringRecord(Stream, Code: bitc::MODULE_CODE_SECTIONNAME, Str: F.getSection(),
1530 AbbrevToUse: 0 /*TODO*/);
1531 Entry = SectionMap.size();
1532 }
1533 }
1534 if (F.hasGC()) {
1535 // Same for GC names.
1536 unsigned &Entry = GCMap[F.getGC()];
1537 if (!Entry) {
1538 writeStringRecord(Stream, Code: bitc::MODULE_CODE_GCNAME, Str: F.getGC(),
1539 AbbrevToUse: 0 /*TODO*/);
1540 Entry = GCMap.size();
1541 }
1542 }
1543 }
1544
1545 // Emit abbrev for globals, now that we know # sections and max alignment.
1546 unsigned SimpleGVarAbbrev = 0;
1547 if (!M.global_empty()) {
1548 // Add an abbrev for common globals with no visibility or thread localness.
1549 auto Abbv = std::make_shared<BitCodeAbbrev>();
1550 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_GLOBALVAR));
1551 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1552 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1553 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1554 Log2_32_Ceil(Value: MaxGlobalType+1)));
1555 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // AddrSpace << 2
1556 //| explicitType << 1
1557 //| constant
1558 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Initializer.
1559 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // Linkage.
1560 if (!MaxGVarAlignment) // Alignment.
1561 Abbv->Add(OpInfo: BitCodeAbbrevOp(0));
1562 else {
1563 unsigned MaxEncAlignment = getEncodedAlign(Alignment: MaxGVarAlignment);
1564 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1565 Log2_32_Ceil(Value: MaxEncAlignment+1)));
1566 }
1567 if (SectionMap.empty()) // Section.
1568 Abbv->Add(OpInfo: BitCodeAbbrevOp(0));
1569 else
1570 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1571 Log2_32_Ceil(Value: SectionMap.size()+1)));
1572 // Don't bother emitting vis + thread local.
1573 SimpleGVarAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1574 }
1575
1576 SmallVector<unsigned, 64> Vals;
1577 // Emit the module's source file name.
1578 {
1579 StringEncoding Bits = getStringEncoding(Str: M.getSourceFileName());
1580 BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
1581 if (Bits == SE_Char6)
1582 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
1583 else if (Bits == SE_Fixed7)
1584 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
1585
1586 // MODULE_CODE_SOURCE_FILENAME: [namechar x N]
1587 auto Abbv = std::make_shared<BitCodeAbbrev>();
1588 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
1589 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1590 Abbv->Add(OpInfo: AbbrevOpToUse);
1591 unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1592
1593 for (const auto P : M.getSourceFileName())
1594 Vals.push_back(Elt: (unsigned char)P);
1595
1596 // Emit the finished record.
1597 Stream.EmitRecord(Code: bitc::MODULE_CODE_SOURCE_FILENAME, Vals, Abbrev: FilenameAbbrev);
1598 Vals.clear();
1599 }
1600
1601 // Emit the global variable information.
1602 for (const GlobalVariable &GV : M.globals()) {
1603 unsigned AbbrevToUse = 0;
1604
1605 // GLOBALVAR: [strtab offset, strtab size, type, isconst, initid,
1606 // linkage, alignment, section, visibility, threadlocal,
1607 // unnamed_addr, externally_initialized, dllstorageclass,
1608 // comdat, attributes, DSO_Local, GlobalSanitizer, code_model]
1609 Vals.push_back(Elt: addToStrtab(Str: GV.getName()));
1610 Vals.push_back(Elt: GV.getName().size());
1611 Vals.push_back(Elt: VE.getTypeID(T: GV.getValueType()));
1612 Vals.push_back(Elt: GV.getType()->getAddressSpace() << 2 | 2 | GV.isConstant());
1613 Vals.push_back(Elt: GV.isDeclaration() ? 0 :
1614 (VE.getValueID(V: GV.getInitializer()) + 1));
1615 Vals.push_back(Elt: getEncodedLinkage(GV));
1616 Vals.push_back(Elt: getEncodedAlign(Alignment: GV.getAlign()));
1617 Vals.push_back(Elt: GV.hasSection() ? SectionMap[std::string(GV.getSection())]
1618 : 0);
1619 if (GV.isThreadLocal() ||
1620 GV.getVisibility() != GlobalValue::DefaultVisibility ||
1621 GV.getUnnamedAddr() != GlobalValue::UnnamedAddr::None ||
1622 GV.isExternallyInitialized() ||
1623 GV.getDLLStorageClass() != GlobalValue::DefaultStorageClass ||
1624 GV.hasComdat() || GV.hasAttributes() || GV.isDSOLocal() ||
1625 GV.hasPartition() || GV.hasSanitizerMetadata() || GV.getCodeModel()) {
1626 Vals.push_back(Elt: getEncodedVisibility(GV));
1627 Vals.push_back(Elt: getEncodedThreadLocalMode(GV));
1628 Vals.push_back(Elt: getEncodedUnnamedAddr(GV));
1629 Vals.push_back(Elt: GV.isExternallyInitialized());
1630 Vals.push_back(Elt: getEncodedDLLStorageClass(GV));
1631 Vals.push_back(Elt: GV.hasComdat() ? VE.getComdatID(C: GV.getComdat()) : 0);
1632
1633 auto AL = GV.getAttributesAsList(index: AttributeList::FunctionIndex);
1634 Vals.push_back(Elt: VE.getAttributeListID(PAL: AL));
1635
1636 Vals.push_back(Elt: GV.isDSOLocal());
1637 Vals.push_back(Elt: addToStrtab(Str: GV.getPartition()));
1638 Vals.push_back(Elt: GV.getPartition().size());
1639
1640 Vals.push_back(Elt: (GV.hasSanitizerMetadata() ? serializeSanitizerMetadata(
1641 Meta: GV.getSanitizerMetadata())
1642 : 0));
1643 Vals.push_back(Elt: GV.getCodeModelRaw());
1644 } else {
1645 AbbrevToUse = SimpleGVarAbbrev;
1646 }
1647
1648 Stream.EmitRecord(Code: bitc::MODULE_CODE_GLOBALVAR, Vals, Abbrev: AbbrevToUse);
1649 Vals.clear();
1650 }
1651
1652 // Emit the function proto information.
1653 for (const Function &F : M) {
1654 // FUNCTION: [strtab offset, strtab size, type, callingconv, isproto,
1655 // linkage, paramattrs, alignment, section, visibility, gc,
1656 // unnamed_addr, prologuedata, dllstorageclass, comdat,
1657 // prefixdata, personalityfn, DSO_Local, addrspace,
1658 // partition_strtab, partition_size, prefalign]
1659 Vals.push_back(Elt: addToStrtab(Str: F.getName()));
1660 Vals.push_back(Elt: F.getName().size());
1661 Vals.push_back(Elt: VE.getTypeID(T: F.getFunctionType()));
1662 Vals.push_back(Elt: F.getCallingConv());
1663 Vals.push_back(Elt: F.isDeclaration());
1664 Vals.push_back(Elt: getEncodedLinkage(GV: F));
1665 Vals.push_back(Elt: VE.getAttributeListID(PAL: F.getAttributes()));
1666 Vals.push_back(Elt: getEncodedAlign(Alignment: F.getAlign()));
1667 Vals.push_back(Elt: F.hasSection() ? SectionMap[std::string(F.getSection())]
1668 : 0);
1669 Vals.push_back(Elt: getEncodedVisibility(GV: F));
1670 Vals.push_back(Elt: F.hasGC() ? GCMap[F.getGC()] : 0);
1671 Vals.push_back(Elt: getEncodedUnnamedAddr(GV: F));
1672 Vals.push_back(Elt: F.hasPrologueData() ? (VE.getValueID(V: F.getPrologueData()) + 1)
1673 : 0);
1674 Vals.push_back(Elt: getEncodedDLLStorageClass(GV: F));
1675 Vals.push_back(Elt: F.hasComdat() ? VE.getComdatID(C: F.getComdat()) : 0);
1676 Vals.push_back(Elt: F.hasPrefixData() ? (VE.getValueID(V: F.getPrefixData()) + 1)
1677 : 0);
1678 Vals.push_back(
1679 Elt: F.hasPersonalityFn() ? (VE.getValueID(V: F.getPersonalityFn()) + 1) : 0);
1680
1681 Vals.push_back(Elt: F.isDSOLocal());
1682 Vals.push_back(Elt: F.getAddressSpace());
1683 Vals.push_back(Elt: addToStrtab(Str: F.getPartition()));
1684 Vals.push_back(Elt: F.getPartition().size());
1685 Vals.push_back(Elt: getEncodedAlign(Alignment: F.getPreferredAlignment()));
1686
1687 unsigned AbbrevToUse = 0;
1688 Stream.EmitRecord(Code: bitc::MODULE_CODE_FUNCTION, Vals, Abbrev: AbbrevToUse);
1689 Vals.clear();
1690 }
1691
1692 // Emit the alias information.
1693 for (const GlobalAlias &A : M.aliases()) {
1694 // ALIAS: [strtab offset, strtab size, alias type, aliasee val#, linkage,
1695 // visibility, dllstorageclass, threadlocal, unnamed_addr,
1696 // DSO_Local]
1697 Vals.push_back(Elt: addToStrtab(Str: A.getName()));
1698 Vals.push_back(Elt: A.getName().size());
1699 Vals.push_back(Elt: VE.getTypeID(T: A.getValueType()));
1700 Vals.push_back(Elt: A.getType()->getAddressSpace());
1701 Vals.push_back(Elt: VE.getValueID(V: A.getAliasee()));
1702 Vals.push_back(Elt: getEncodedLinkage(GV: A));
1703 Vals.push_back(Elt: getEncodedVisibility(GV: A));
1704 Vals.push_back(Elt: getEncodedDLLStorageClass(GV: A));
1705 Vals.push_back(Elt: getEncodedThreadLocalMode(GV: A));
1706 Vals.push_back(Elt: getEncodedUnnamedAddr(GV: A));
1707 Vals.push_back(Elt: A.isDSOLocal());
1708 Vals.push_back(Elt: addToStrtab(Str: A.getPartition()));
1709 Vals.push_back(Elt: A.getPartition().size());
1710
1711 unsigned AbbrevToUse = 0;
1712 Stream.EmitRecord(Code: bitc::MODULE_CODE_ALIAS, Vals, Abbrev: AbbrevToUse);
1713 Vals.clear();
1714 }
1715
1716 // Emit the ifunc information.
1717 for (const GlobalIFunc &I : M.ifuncs()) {
1718 // IFUNC: [strtab offset, strtab size, ifunc type, address space, resolver
1719 // val#, linkage, visibility, DSO_Local]
1720 Vals.push_back(Elt: addToStrtab(Str: I.getName()));
1721 Vals.push_back(Elt: I.getName().size());
1722 Vals.push_back(Elt: VE.getTypeID(T: I.getValueType()));
1723 Vals.push_back(Elt: I.getType()->getAddressSpace());
1724 Vals.push_back(Elt: VE.getValueID(V: I.getResolver()));
1725 Vals.push_back(Elt: getEncodedLinkage(GV: I));
1726 Vals.push_back(Elt: getEncodedVisibility(GV: I));
1727 Vals.push_back(Elt: I.isDSOLocal());
1728 Vals.push_back(Elt: addToStrtab(Str: I.getPartition()));
1729 Vals.push_back(Elt: I.getPartition().size());
1730 Stream.EmitRecord(Code: bitc::MODULE_CODE_IFUNC, Vals);
1731 Vals.clear();
1732 }
1733
1734 writeValueSymbolTableForwardDecl();
1735}
1736
1737static uint64_t getOptimizationFlags(const Value *V) {
1738 uint64_t Flags = 0;
1739
1740 if (const auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V)) {
1741 if (OBO->hasNoSignedWrap())
1742 Flags |= 1 << bitc::OBO_NO_SIGNED_WRAP;
1743 if (OBO->hasNoUnsignedWrap())
1744 Flags |= 1 << bitc::OBO_NO_UNSIGNED_WRAP;
1745 } else if (const auto *PEO = dyn_cast<PossiblyExactOperator>(Val: V)) {
1746 if (PEO->isExact())
1747 Flags |= 1 << bitc::PEO_EXACT;
1748 } else if (const auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: V)) {
1749 if (PDI->isDisjoint())
1750 Flags |= 1 << bitc::PDI_DISJOINT;
1751 } else if (const auto *FPMO = dyn_cast<FPMathOperator>(Val: V)) {
1752 if (FPMO->hasAllowReassoc())
1753 Flags |= bitc::AllowReassoc;
1754 if (FPMO->hasNoNaNs())
1755 Flags |= bitc::NoNaNs;
1756 if (FPMO->hasNoInfs())
1757 Flags |= bitc::NoInfs;
1758 if (FPMO->hasNoSignedZeros())
1759 Flags |= bitc::NoSignedZeros;
1760 if (FPMO->hasAllowReciprocal())
1761 Flags |= bitc::AllowReciprocal;
1762 if (FPMO->hasAllowContract())
1763 Flags |= bitc::AllowContract;
1764 if (FPMO->hasApproxFunc())
1765 Flags |= bitc::ApproxFunc;
1766 } else if (const auto *NNI = dyn_cast<PossiblyNonNegInst>(Val: V)) {
1767 if (NNI->hasNonNeg())
1768 Flags |= 1 << bitc::PNNI_NON_NEG;
1769 } else if (const auto *TI = dyn_cast<TruncInst>(Val: V)) {
1770 if (TI->hasNoSignedWrap())
1771 Flags |= 1 << bitc::TIO_NO_SIGNED_WRAP;
1772 if (TI->hasNoUnsignedWrap())
1773 Flags |= 1 << bitc::TIO_NO_UNSIGNED_WRAP;
1774 } else if (const auto *GEP = dyn_cast<GEPOperator>(Val: V)) {
1775 if (GEP->isInBounds())
1776 Flags |= 1 << bitc::GEP_INBOUNDS;
1777 if (GEP->hasNoUnsignedSignedWrap())
1778 Flags |= 1 << bitc::GEP_NUSW;
1779 if (GEP->hasNoUnsignedWrap())
1780 Flags |= 1 << bitc::GEP_NUW;
1781 } else if (const auto *ICmp = dyn_cast<ICmpInst>(Val: V)) {
1782 if (ICmp->hasSameSign())
1783 Flags |= 1 << bitc::ICMP_SAME_SIGN;
1784 }
1785
1786 return Flags;
1787}
1788
1789void ModuleBitcodeWriter::writeValueAsMetadata(
1790 const ValueAsMetadata *MD, SmallVectorImpl<uint64_t> &Record) {
1791 // Mimic an MDNode with a value as one operand.
1792 Value *V = MD->getValue();
1793 Record.push_back(Elt: VE.getTypeID(T: V->getType()));
1794 Record.push_back(Elt: VE.getValueID(V));
1795 Stream.EmitRecord(Code: bitc::METADATA_VALUE, Vals: Record, Abbrev: 0);
1796 Record.clear();
1797}
1798
1799void ModuleBitcodeWriter::writeMDTuple(const MDTuple *N,
1800 SmallVectorImpl<uint64_t> &Record,
1801 unsigned Abbrev) {
1802 for (const MDOperand &MDO : N->operands()) {
1803 Metadata *MD = MDO;
1804 assert(!(MD && isa<LocalAsMetadata>(MD)) &&
1805 "Unexpected function-local metadata");
1806 Record.push_back(Elt: VE.getMetadataOrNullID(MD));
1807 }
1808 Stream.EmitRecord(Code: N->isDistinct() ? bitc::METADATA_DISTINCT_NODE
1809 : bitc::METADATA_NODE,
1810 Vals: Record, Abbrev);
1811 Record.clear();
1812}
1813
1814unsigned ModuleBitcodeWriter::createDILocationAbbrev() {
1815 // Assume the column is usually under 128, and always output the inlined-at
1816 // location (it's never more expensive than building an array size 1).
1817 auto Abbv = std::make_shared<BitCodeAbbrev>();
1818 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_LOCATION));
1819 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isDistinct
1820 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // line
1821 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // column
1822 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // scope
1823 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // inlinedAt
1824 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isImplicitCode
1825 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // atomGroup
1826 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // atomRank
1827 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
1828}
1829
1830void ModuleBitcodeWriter::writeDILocation(const DILocation *N,
1831 SmallVectorImpl<uint64_t> &Record,
1832 unsigned &Abbrev) {
1833 if (!Abbrev)
1834 Abbrev = createDILocationAbbrev();
1835
1836 Record.push_back(Elt: N->isDistinct());
1837 Record.push_back(Elt: N->getLine());
1838 Record.push_back(Elt: N->getColumn());
1839 Record.push_back(Elt: VE.getMetadataID(MD: N->getScope()));
1840 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getInlinedAt()));
1841 Record.push_back(Elt: N->isImplicitCode());
1842 Record.push_back(Elt: N->getAtomGroup());
1843 Record.push_back(Elt: N->getAtomRank());
1844 Stream.EmitRecord(Code: bitc::METADATA_LOCATION, Vals: Record, Abbrev);
1845 Record.clear();
1846}
1847
1848unsigned ModuleBitcodeWriter::createGenericDINodeAbbrev() {
1849 // Assume the column is usually under 128, and always output the inlined-at
1850 // location (it's never more expensive than building an array size 1).
1851 auto Abbv = std::make_shared<BitCodeAbbrev>();
1852 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_GENERIC_DEBUG));
1853 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1854 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1855 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1856 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1857 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1858 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1859 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
1860}
1861
1862void ModuleBitcodeWriter::writeGenericDINode(const GenericDINode *N,
1863 SmallVectorImpl<uint64_t> &Record,
1864 unsigned &Abbrev) {
1865 if (!Abbrev)
1866 Abbrev = createGenericDINodeAbbrev();
1867
1868 Record.push_back(Elt: N->isDistinct());
1869 Record.push_back(Elt: N->getTag());
1870 Record.push_back(Elt: 0); // Per-tag version field; unused for now.
1871
1872 for (auto &I : N->operands())
1873 Record.push_back(Elt: VE.getMetadataOrNullID(MD: I));
1874
1875 Stream.EmitRecord(Code: bitc::METADATA_GENERIC_DEBUG, Vals: Record, Abbrev);
1876 Record.clear();
1877}
1878
1879void ModuleBitcodeWriter::writeDISubrange(const DISubrange *N,
1880 SmallVectorImpl<uint64_t> &Record,
1881 unsigned Abbrev) {
1882 const uint64_t Version = 2 << 1;
1883 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
1884 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawCountNode()));
1885 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
1886 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
1887 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
1888
1889 Stream.EmitRecord(Code: bitc::METADATA_SUBRANGE, Vals: Record, Abbrev);
1890 Record.clear();
1891}
1892
1893void ModuleBitcodeWriter::writeDIGenericSubrange(
1894 const DIGenericSubrange *N, SmallVectorImpl<uint64_t> &Record,
1895 unsigned Abbrev) {
1896 Record.push_back(Elt: (uint64_t)N->isDistinct());
1897 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawCountNode()));
1898 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
1899 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
1900 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
1901
1902 Stream.EmitRecord(Code: bitc::METADATA_GENERIC_SUBRANGE, Vals: Record, Abbrev);
1903 Record.clear();
1904}
1905
1906void ModuleBitcodeWriter::writeDIEnumerator(const DIEnumerator *N,
1907 SmallVectorImpl<uint64_t> &Record,
1908 unsigned Abbrev) {
1909 const uint64_t IsBigInt = 1 << 2;
1910 Record.push_back(Elt: IsBigInt | (N->isUnsigned() << 1) | N->isDistinct());
1911 Record.push_back(Elt: N->getValue().getBitWidth());
1912 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1913 emitWideAPInt(Vals&: Record, A: N->getValue());
1914
1915 Stream.EmitRecord(Code: bitc::METADATA_ENUMERATOR, Vals: Record, Abbrev);
1916 Record.clear();
1917}
1918
1919void ModuleBitcodeWriter::writeDIBasicType(const DIBasicType *N,
1920 SmallVectorImpl<uint64_t> &Record,
1921 unsigned Abbrev) {
1922 const unsigned SizeIsMetadata = 0x2;
1923 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
1924 Record.push_back(Elt: N->getTag());
1925 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1926 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
1927 Record.push_back(Elt: N->getAlignInBits());
1928 Record.push_back(Elt: N->getEncoding());
1929 Record.push_back(Elt: N->getFlags());
1930 Record.push_back(Elt: N->getNumExtraInhabitants());
1931 Record.push_back(Elt: N->getDataSizeInBits());
1932
1933 Stream.EmitRecord(Code: bitc::METADATA_BASIC_TYPE, Vals: Record, Abbrev);
1934 Record.clear();
1935}
1936
1937void ModuleBitcodeWriter::writeDIFixedPointType(
1938 const DIFixedPointType *N, SmallVectorImpl<uint64_t> &Record,
1939 unsigned Abbrev) {
1940 const unsigned SizeIsMetadata = 0x2;
1941 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
1942 Record.push_back(Elt: N->getTag());
1943 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1944 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
1945 Record.push_back(Elt: N->getAlignInBits());
1946 Record.push_back(Elt: N->getEncoding());
1947 Record.push_back(Elt: N->getFlags());
1948 Record.push_back(Elt: N->getKind());
1949 Record.push_back(Elt: N->getFactorRaw());
1950
1951 auto WriteWideInt = [&](const APInt &Value) {
1952 // Write an encoded word that holds the number of active words and
1953 // the number of bits.
1954 uint64_t NumWords = Value.getActiveWords();
1955 uint64_t Encoded = (NumWords << 32) | Value.getBitWidth();
1956 Record.push_back(Elt: Encoded);
1957 emitWideAPInt(Vals&: Record, A: Value);
1958 };
1959
1960 WriteWideInt(N->getNumeratorRaw());
1961 WriteWideInt(N->getDenominatorRaw());
1962
1963 Stream.EmitRecord(Code: bitc::METADATA_FIXED_POINT_TYPE, Vals: Record, Abbrev);
1964 Record.clear();
1965}
1966
1967void ModuleBitcodeWriter::writeDIStringType(const DIStringType *N,
1968 SmallVectorImpl<uint64_t> &Record,
1969 unsigned Abbrev) {
1970 const unsigned SizeIsMetadata = 0x2;
1971 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
1972 Record.push_back(Elt: N->getTag());
1973 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1974 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLength()));
1975 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLengthExp()));
1976 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLocationExp()));
1977 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
1978 Record.push_back(Elt: N->getAlignInBits());
1979 Record.push_back(Elt: N->getEncoding());
1980
1981 Stream.EmitRecord(Code: bitc::METADATA_STRING_TYPE, Vals: Record, Abbrev);
1982 Record.clear();
1983}
1984
1985void ModuleBitcodeWriter::writeDIDerivedType(const DIDerivedType *N,
1986 SmallVectorImpl<uint64_t> &Record,
1987 unsigned Abbrev) {
1988 const unsigned SizeIsMetadata = 0x2;
1989 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
1990 Record.push_back(Elt: N->getTag());
1991 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1992 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
1993 Record.push_back(Elt: N->getLine());
1994 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
1995 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
1996 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
1997 Record.push_back(Elt: N->getAlignInBits());
1998 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawOffsetInBits()));
1999 Record.push_back(Elt: N->getFlags());
2000 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getExtraData()));
2001
2002 // DWARF address space is encoded as N->getDWARFAddressSpace() + 1. 0 means
2003 // that there is no DWARF address space associated with DIDerivedType.
2004 if (const auto &DWARFAddressSpace = N->getDWARFAddressSpace())
2005 Record.push_back(Elt: *DWARFAddressSpace + 1);
2006 else
2007 Record.push_back(Elt: 0);
2008
2009 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2010
2011 if (auto PtrAuthData = N->getPtrAuthData())
2012 Record.push_back(Elt: PtrAuthData->RawData);
2013 else
2014 Record.push_back(Elt: 0);
2015
2016 Stream.EmitRecord(Code: bitc::METADATA_DERIVED_TYPE, Vals: Record, Abbrev);
2017 Record.clear();
2018}
2019
2020void ModuleBitcodeWriter::writeDISubrangeType(const DISubrangeType *N,
2021 SmallVectorImpl<uint64_t> &Record,
2022 unsigned Abbrev) {
2023 const unsigned SizeIsMetadata = 0x2;
2024 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2025 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2026 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2027 Record.push_back(Elt: N->getLine());
2028 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2029 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2030 Record.push_back(Elt: N->getAlignInBits());
2031 Record.push_back(Elt: N->getFlags());
2032 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2033 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
2034 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
2035 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
2036 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawBias()));
2037
2038 Stream.EmitRecord(Code: bitc::METADATA_SUBRANGE_TYPE, Vals: Record, Abbrev);
2039 Record.clear();
2040}
2041
2042void ModuleBitcodeWriter::writeDICompositeType(
2043 const DICompositeType *N, SmallVectorImpl<uint64_t> &Record,
2044 unsigned Abbrev) {
2045 const unsigned IsNotUsedInOldTypeRef = 0x2;
2046 const unsigned SizeIsMetadata = 0x4;
2047 Record.push_back(Elt: SizeIsMetadata | IsNotUsedInOldTypeRef |
2048 (unsigned)N->isDistinct());
2049 Record.push_back(Elt: N->getTag());
2050 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2051 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2052 Record.push_back(Elt: N->getLine());
2053 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2054 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2055 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2056 Record.push_back(Elt: N->getAlignInBits());
2057 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawOffsetInBits()));
2058 Record.push_back(Elt: N->getFlags());
2059 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2060 Record.push_back(Elt: N->getRuntimeLang());
2061 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getVTableHolder()));
2062 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams().get()));
2063 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawIdentifier()));
2064 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDiscriminator()));
2065 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawDataLocation()));
2066 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawAssociated()));
2067 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawAllocated()));
2068 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawRank()));
2069 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2070 Record.push_back(Elt: N->getNumExtraInhabitants());
2071 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSpecification()));
2072 Record.push_back(
2073 Elt: N->getEnumKind().value_or(u: dwarf::DW_APPLE_ENUM_KIND_invalid));
2074 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawBitStride()));
2075
2076 Stream.EmitRecord(Code: bitc::METADATA_COMPOSITE_TYPE, Vals: Record, Abbrev);
2077 Record.clear();
2078}
2079
2080void ModuleBitcodeWriter::writeDISubroutineType(
2081 const DISubroutineType *N, SmallVectorImpl<uint64_t> &Record,
2082 unsigned Abbrev) {
2083 const unsigned HasNoOldTypeRefs = 0x2;
2084 Record.push_back(Elt: HasNoOldTypeRefs | (unsigned)N->isDistinct());
2085 Record.push_back(Elt: N->getFlags());
2086 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTypeArray().get()));
2087 Record.push_back(Elt: N->getCC());
2088
2089 Stream.EmitRecord(Code: bitc::METADATA_SUBROUTINE_TYPE, Vals: Record, Abbrev);
2090 Record.clear();
2091}
2092
2093void ModuleBitcodeWriter::writeDIFile(const DIFile *N,
2094 SmallVectorImpl<uint64_t> &Record,
2095 unsigned Abbrev) {
2096 Record.push_back(Elt: N->isDistinct());
2097 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFilename()));
2098 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawDirectory()));
2099 if (N->getRawChecksum()) {
2100 Record.push_back(Elt: N->getRawChecksum()->Kind);
2101 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawChecksum()->Value));
2102 } else {
2103 // Maintain backwards compatibility with the old internal representation of
2104 // CSK_None in ChecksumKind by writing nulls here when Checksum is None.
2105 Record.push_back(Elt: 0);
2106 Record.push_back(Elt: VE.getMetadataOrNullID(MD: nullptr));
2107 }
2108 auto Source = N->getRawSource();
2109 if (Source)
2110 Record.push_back(Elt: VE.getMetadataOrNullID(MD: Source));
2111
2112 Stream.EmitRecord(Code: bitc::METADATA_FILE, Vals: Record, Abbrev);
2113 Record.clear();
2114}
2115
2116void ModuleBitcodeWriter::writeDICompileUnit(const DICompileUnit *N,
2117 SmallVectorImpl<uint64_t> &Record,
2118 unsigned Abbrev) {
2119 assert(N->isDistinct() && "Expected distinct compile units");
2120 Record.push_back(/* IsDistinct */ Elt: true);
2121
2122 auto Lang = N->getSourceLanguage();
2123 Record.push_back(Elt: Lang.getName());
2124 // Set bit so the MetadataLoader can distniguish between versioned and
2125 // unversioned names.
2126 if (Lang.hasVersionedName())
2127 Record.back() ^= (uint64_t(1) << 63);
2128
2129 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2130 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawProducer()));
2131 Record.push_back(Elt: N->isOptimized());
2132 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFlags()));
2133 Record.push_back(Elt: N->getRuntimeVersion());
2134 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSplitDebugFilename()));
2135 Record.push_back(Elt: N->getEmissionKind());
2136 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getEnumTypes().get()));
2137 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRetainedTypes().get()));
2138 Record.push_back(/* subprograms */ Elt: 0);
2139 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getGlobalVariables().get()));
2140 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getImportedEntities().get()));
2141 Record.push_back(Elt: N->getDWOId());
2142 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getMacros().get()));
2143 Record.push_back(Elt: N->getSplitDebugInlining());
2144 Record.push_back(Elt: N->getDebugInfoForProfiling());
2145 Record.push_back(Elt: (unsigned)N->getNameTableKind());
2146 Record.push_back(Elt: N->getRangesBaseAddress());
2147 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSysRoot()));
2148 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSDK()));
2149 Record.push_back(Elt: Lang.hasVersionedName() ? Lang.getVersion() : 0);
2150
2151 Stream.EmitRecord(Code: bitc::METADATA_COMPILE_UNIT, Vals: Record, Abbrev);
2152 Record.clear();
2153}
2154
2155void ModuleBitcodeWriter::writeDISubprogram(const DISubprogram *N,
2156 SmallVectorImpl<uint64_t> &Record,
2157 unsigned Abbrev) {
2158 const uint64_t HasUnitFlag = 1 << 1;
2159 const uint64_t HasSPFlagsFlag = 1 << 2;
2160 Record.push_back(Elt: uint64_t(N->isDistinct()) | HasUnitFlag | HasSPFlagsFlag);
2161 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2162 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2163 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLinkageName()));
2164 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2165 Record.push_back(Elt: N->getLine());
2166 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2167 Record.push_back(Elt: N->getScopeLine());
2168 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getContainingType()));
2169 Record.push_back(Elt: N->getSPFlags());
2170 Record.push_back(Elt: N->getVirtualIndex());
2171 Record.push_back(Elt: N->getFlags());
2172 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUnit()));
2173 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams().get()));
2174 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDeclaration()));
2175 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRetainedNodes().get()));
2176 Record.push_back(Elt: N->getThisAdjustment());
2177 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getThrownTypes().get()));
2178 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2179 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawTargetFuncName()));
2180 Record.push_back(Elt: N->getKeyInstructionsEnabled());
2181
2182 Stream.EmitRecord(Code: bitc::METADATA_SUBPROGRAM, Vals: Record, Abbrev);
2183 Record.clear();
2184}
2185
2186void ModuleBitcodeWriter::writeDILexicalBlock(const DILexicalBlock *N,
2187 SmallVectorImpl<uint64_t> &Record,
2188 unsigned Abbrev) {
2189 Record.push_back(Elt: N->isDistinct());
2190 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2191 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2192 Record.push_back(Elt: N->getLine());
2193 Record.push_back(Elt: N->getColumn());
2194
2195 Stream.EmitRecord(Code: bitc::METADATA_LEXICAL_BLOCK, Vals: Record, Abbrev);
2196 Record.clear();
2197}
2198
2199void ModuleBitcodeWriter::writeDILexicalBlockFile(
2200 const DILexicalBlockFile *N, SmallVectorImpl<uint64_t> &Record,
2201 unsigned Abbrev) {
2202 Record.push_back(Elt: N->isDistinct());
2203 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2204 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2205 Record.push_back(Elt: N->getDiscriminator());
2206
2207 Stream.EmitRecord(Code: bitc::METADATA_LEXICAL_BLOCK_FILE, Vals: Record, Abbrev);
2208 Record.clear();
2209}
2210
2211void ModuleBitcodeWriter::writeDICommonBlock(const DICommonBlock *N,
2212 SmallVectorImpl<uint64_t> &Record,
2213 unsigned Abbrev) {
2214 Record.push_back(Elt: N->isDistinct());
2215 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2216 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDecl()));
2217 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2218 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2219 Record.push_back(Elt: N->getLineNo());
2220
2221 Stream.EmitRecord(Code: bitc::METADATA_COMMON_BLOCK, Vals: Record, Abbrev);
2222 Record.clear();
2223}
2224
2225void ModuleBitcodeWriter::writeDINamespace(const DINamespace *N,
2226 SmallVectorImpl<uint64_t> &Record,
2227 unsigned Abbrev) {
2228 Record.push_back(Elt: N->isDistinct() | N->getExportSymbols() << 1);
2229 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2230 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2231
2232 Stream.EmitRecord(Code: bitc::METADATA_NAMESPACE, Vals: Record, Abbrev);
2233 Record.clear();
2234}
2235
2236void ModuleBitcodeWriter::writeDIMacro(const DIMacro *N,
2237 SmallVectorImpl<uint64_t> &Record,
2238 unsigned Abbrev) {
2239 Record.push_back(Elt: N->isDistinct());
2240 Record.push_back(Elt: N->getMacinfoType());
2241 Record.push_back(Elt: N->getLine());
2242 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2243 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawValue()));
2244
2245 Stream.EmitRecord(Code: bitc::METADATA_MACRO, Vals: Record, Abbrev);
2246 Record.clear();
2247}
2248
2249void ModuleBitcodeWriter::writeDIMacroFile(const DIMacroFile *N,
2250 SmallVectorImpl<uint64_t> &Record,
2251 unsigned Abbrev) {
2252 Record.push_back(Elt: N->isDistinct());
2253 Record.push_back(Elt: N->getMacinfoType());
2254 Record.push_back(Elt: N->getLine());
2255 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2256 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2257
2258 Stream.EmitRecord(Code: bitc::METADATA_MACRO_FILE, Vals: Record, Abbrev);
2259 Record.clear();
2260}
2261
2262void ModuleBitcodeWriter::writeDIArgList(const DIArgList *N,
2263 SmallVectorImpl<uint64_t> &Record) {
2264 Record.reserve(N: N->getArgs().size());
2265 for (ValueAsMetadata *MD : N->getArgs())
2266 Record.push_back(Elt: VE.getMetadataID(MD));
2267
2268 Stream.EmitRecord(Code: bitc::METADATA_ARG_LIST, Vals: Record);
2269 Record.clear();
2270}
2271
2272void ModuleBitcodeWriter::writeDIModule(const DIModule *N,
2273 SmallVectorImpl<uint64_t> &Record,
2274 unsigned Abbrev) {
2275 Record.push_back(Elt: N->isDistinct());
2276 for (auto &I : N->operands())
2277 Record.push_back(Elt: VE.getMetadataOrNullID(MD: I));
2278 Record.push_back(Elt: N->getLineNo());
2279 Record.push_back(Elt: N->getIsDecl());
2280
2281 Stream.EmitRecord(Code: bitc::METADATA_MODULE, Vals: Record, Abbrev);
2282 Record.clear();
2283}
2284
2285void ModuleBitcodeWriter::writeDIAssignID(const DIAssignID *N,
2286 SmallVectorImpl<uint64_t> &Record,
2287 unsigned Abbrev) {
2288 // There are no arguments for this metadata type.
2289 Record.push_back(Elt: N->isDistinct());
2290 Stream.EmitRecord(Code: bitc::METADATA_ASSIGN_ID, Vals: Record, Abbrev);
2291 Record.clear();
2292}
2293
2294void ModuleBitcodeWriter::writeDITemplateTypeParameter(
2295 const DITemplateTypeParameter *N, SmallVectorImpl<uint64_t> &Record,
2296 unsigned Abbrev) {
2297 Record.push_back(Elt: N->isDistinct());
2298 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2299 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2300 Record.push_back(Elt: N->isDefault());
2301
2302 Stream.EmitRecord(Code: bitc::METADATA_TEMPLATE_TYPE, Vals: Record, Abbrev);
2303 Record.clear();
2304}
2305
2306void ModuleBitcodeWriter::writeDITemplateValueParameter(
2307 const DITemplateValueParameter *N, SmallVectorImpl<uint64_t> &Record,
2308 unsigned Abbrev) {
2309 Record.push_back(Elt: N->isDistinct());
2310 Record.push_back(Elt: N->getTag());
2311 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2312 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2313 Record.push_back(Elt: N->isDefault());
2314 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getValue()));
2315
2316 Stream.EmitRecord(Code: bitc::METADATA_TEMPLATE_VALUE, Vals: Record, Abbrev);
2317 Record.clear();
2318}
2319
2320void ModuleBitcodeWriter::writeDIGlobalVariable(
2321 const DIGlobalVariable *N, SmallVectorImpl<uint64_t> &Record,
2322 unsigned Abbrev) {
2323 const uint64_t Version = 2 << 1;
2324 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
2325 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2326 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2327 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLinkageName()));
2328 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2329 Record.push_back(Elt: N->getLine());
2330 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2331 Record.push_back(Elt: N->isLocalToUnit());
2332 Record.push_back(Elt: N->isDefinition());
2333 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStaticDataMemberDeclaration()));
2334 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams()));
2335 Record.push_back(Elt: N->getAlignInBits());
2336 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2337
2338 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_VAR, Vals: Record, Abbrev);
2339 Record.clear();
2340}
2341
2342void ModuleBitcodeWriter::writeDILocalVariable(
2343 const DILocalVariable *N, SmallVectorImpl<uint64_t> &Record,
2344 unsigned Abbrev) {
2345 // In order to support all possible bitcode formats in BitcodeReader we need
2346 // to distinguish the following cases:
2347 // 1) Record has no artificial tag (Record[1]),
2348 // has no obsolete inlinedAt field (Record[9]).
2349 // In this case Record size will be 8, HasAlignment flag is false.
2350 // 2) Record has artificial tag (Record[1]),
2351 // has no obsolete inlignedAt field (Record[9]).
2352 // In this case Record size will be 9, HasAlignment flag is false.
2353 // 3) Record has both artificial tag (Record[1]) and
2354 // obsolete inlignedAt field (Record[9]).
2355 // In this case Record size will be 10, HasAlignment flag is false.
2356 // 4) Record has neither artificial tag, nor inlignedAt field, but
2357 // HasAlignment flag is true and Record[8] contains alignment value.
2358 const uint64_t HasAlignmentFlag = 1 << 1;
2359 Record.push_back(Elt: (uint64_t)N->isDistinct() | HasAlignmentFlag);
2360 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2361 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2362 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2363 Record.push_back(Elt: N->getLine());
2364 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2365 Record.push_back(Elt: N->getArg());
2366 Record.push_back(Elt: N->getFlags());
2367 Record.push_back(Elt: N->getAlignInBits());
2368 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2369
2370 Stream.EmitRecord(Code: bitc::METADATA_LOCAL_VAR, Vals: Record, Abbrev);
2371 Record.clear();
2372}
2373
2374void ModuleBitcodeWriter::writeDILabel(
2375 const DILabel *N, SmallVectorImpl<uint64_t> &Record,
2376 unsigned Abbrev) {
2377 uint64_t IsArtificialFlag = uint64_t(N->isArtificial()) << 1;
2378 Record.push_back(Elt: (uint64_t)N->isDistinct() | IsArtificialFlag);
2379 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2380 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2381 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2382 Record.push_back(Elt: N->getLine());
2383 Record.push_back(Elt: N->getColumn());
2384 Record.push_back(Elt: N->getCoroSuspendIdx().has_value()
2385 ? (uint64_t)N->getCoroSuspendIdx().value()
2386 : std::numeric_limits<uint64_t>::max());
2387
2388 Stream.EmitRecord(Code: bitc::METADATA_LABEL, Vals: Record, Abbrev);
2389 Record.clear();
2390}
2391
2392void ModuleBitcodeWriter::writeDIExpression(const DIExpression *N,
2393 SmallVectorImpl<uint64_t> &Record,
2394 unsigned Abbrev) {
2395 Record.reserve(N: N->getElements().size() + 1);
2396 const uint64_t Version = 3 << 1;
2397 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
2398 Record.append(in_start: N->elements_begin(), in_end: N->elements_end());
2399
2400 Stream.EmitRecord(Code: bitc::METADATA_EXPRESSION, Vals: Record, Abbrev);
2401 Record.clear();
2402}
2403
2404void ModuleBitcodeWriter::writeDIGlobalVariableExpression(
2405 const DIGlobalVariableExpression *N, SmallVectorImpl<uint64_t> &Record,
2406 unsigned Abbrev) {
2407 Record.push_back(Elt: N->isDistinct());
2408 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getVariable()));
2409 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getExpression()));
2410
2411 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_VAR_EXPR, Vals: Record, Abbrev);
2412 Record.clear();
2413}
2414
2415void ModuleBitcodeWriter::writeDIObjCProperty(const DIObjCProperty *N,
2416 SmallVectorImpl<uint64_t> &Record,
2417 unsigned Abbrev) {
2418 Record.push_back(Elt: N->isDistinct());
2419 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2420 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2421 Record.push_back(Elt: N->getLine());
2422 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSetterName()));
2423 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawGetterName()));
2424 Record.push_back(Elt: N->getAttributes());
2425 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2426
2427 Stream.EmitRecord(Code: bitc::METADATA_OBJC_PROPERTY, Vals: Record, Abbrev);
2428 Record.clear();
2429}
2430
2431void ModuleBitcodeWriter::writeDIImportedEntity(
2432 const DIImportedEntity *N, SmallVectorImpl<uint64_t> &Record,
2433 unsigned Abbrev) {
2434 Record.push_back(Elt: N->isDistinct());
2435 Record.push_back(Elt: N->getTag());
2436 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2437 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getEntity()));
2438 Record.push_back(Elt: N->getLine());
2439 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2440 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFile()));
2441 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2442
2443 Stream.EmitRecord(Code: bitc::METADATA_IMPORTED_ENTITY, Vals: Record, Abbrev);
2444 Record.clear();
2445}
2446
2447unsigned ModuleBitcodeWriter::createNamedMetadataAbbrev() {
2448 auto Abbv = std::make_shared<BitCodeAbbrev>();
2449 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_NAME));
2450 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2451 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2452 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
2453}
2454
2455void ModuleBitcodeWriter::writeNamedMetadata(
2456 SmallVectorImpl<uint64_t> &Record) {
2457 if (M.named_metadata_empty())
2458 return;
2459
2460 unsigned Abbrev = createNamedMetadataAbbrev();
2461 for (const NamedMDNode &NMD : M.named_metadata()) {
2462 // Write name.
2463 StringRef Str = NMD.getName();
2464 Record.append(in_start: Str.bytes_begin(), in_end: Str.bytes_end());
2465 Stream.EmitRecord(Code: bitc::METADATA_NAME, Vals: Record, Abbrev);
2466 Record.clear();
2467
2468 // Write named metadata operands.
2469 for (const MDNode *N : NMD.operands())
2470 Record.push_back(Elt: VE.getMetadataID(MD: N));
2471 Stream.EmitRecord(Code: bitc::METADATA_NAMED_NODE, Vals: Record, Abbrev: 0);
2472 Record.clear();
2473 }
2474}
2475
2476unsigned ModuleBitcodeWriter::createMetadataStringsAbbrev() {
2477 auto Abbv = std::make_shared<BitCodeAbbrev>();
2478 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_STRINGS));
2479 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // # of strings
2480 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // offset to chars
2481 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
2482 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
2483}
2484
2485/// Write out a record for MDString.
2486///
2487/// All the metadata strings in a metadata block are emitted in a single
2488/// record. The sizes and strings themselves are shoved into a blob.
2489void ModuleBitcodeWriter::writeMetadataStrings(
2490 ArrayRef<const Metadata *> Strings, SmallVectorImpl<uint64_t> &Record) {
2491 if (Strings.empty())
2492 return;
2493
2494 // Start the record with the number of strings.
2495 Record.push_back(Elt: bitc::METADATA_STRINGS);
2496 Record.push_back(Elt: Strings.size());
2497
2498 // Emit the sizes of the strings in the blob.
2499 SmallString<256> Blob;
2500 {
2501 BitstreamWriter W(Blob);
2502 for (const Metadata *MD : Strings)
2503 W.EmitVBR(Val: cast<MDString>(Val: MD)->getLength(), NumBits: 6);
2504 W.FlushToWord();
2505 }
2506
2507 // Add the offset to the strings to the record.
2508 Record.push_back(Elt: Blob.size());
2509
2510 // Add the strings to the blob.
2511 for (const Metadata *MD : Strings)
2512 Blob.append(RHS: cast<MDString>(Val: MD)->getString());
2513
2514 // Emit the final record.
2515 Stream.EmitRecordWithBlob(Abbrev: createMetadataStringsAbbrev(), Vals: Record, Blob);
2516 Record.clear();
2517}
2518
2519// Generates an enum to use as an index in the Abbrev array of Metadata record.
2520enum MetadataAbbrev : unsigned {
2521#define HANDLE_MDNODE_LEAF(CLASS) CLASS##AbbrevID,
2522#include "llvm/IR/Metadata.def"
2523 LastPlusOne
2524};
2525
2526void ModuleBitcodeWriter::writeMetadataRecords(
2527 ArrayRef<const Metadata *> MDs, SmallVectorImpl<uint64_t> &Record,
2528 std::vector<unsigned> *MDAbbrevs, std::vector<uint64_t> *IndexPos) {
2529 if (MDs.empty())
2530 return;
2531
2532 // Initialize MDNode abbreviations.
2533#define HANDLE_MDNODE_LEAF(CLASS) unsigned CLASS##Abbrev = 0;
2534#include "llvm/IR/Metadata.def"
2535
2536 for (const Metadata *MD : MDs) {
2537 if (IndexPos)
2538 IndexPos->push_back(x: Stream.GetCurrentBitNo());
2539 if (const MDNode *N = dyn_cast<MDNode>(Val: MD)) {
2540 assert(N->isResolved() && "Expected forward references to be resolved");
2541
2542 switch (N->getMetadataID()) {
2543 default:
2544 llvm_unreachable("Invalid MDNode subclass");
2545#define HANDLE_MDNODE_LEAF(CLASS) \
2546 case Metadata::CLASS##Kind: \
2547 if (MDAbbrevs) \
2548 write##CLASS(cast<CLASS>(N), Record, \
2549 (*MDAbbrevs)[MetadataAbbrev::CLASS##AbbrevID]); \
2550 else \
2551 write##CLASS(cast<CLASS>(N), Record, CLASS##Abbrev); \
2552 continue;
2553#include "llvm/IR/Metadata.def"
2554 }
2555 }
2556 if (auto *AL = dyn_cast<DIArgList>(Val: MD)) {
2557 writeDIArgList(N: AL, Record);
2558 continue;
2559 }
2560 writeValueAsMetadata(MD: cast<ValueAsMetadata>(Val: MD), Record);
2561 }
2562}
2563
2564void ModuleBitcodeWriter::writeModuleMetadata() {
2565 if (!VE.hasMDs() && M.named_metadata_empty())
2566 return;
2567
2568 Stream.EnterSubblock(BlockID: bitc::METADATA_BLOCK_ID, CodeLen: 4);
2569 SmallVector<uint64_t, 64> Record;
2570
2571 // Emit all abbrevs upfront, so that the reader can jump in the middle of the
2572 // block and load any metadata.
2573 std::vector<unsigned> MDAbbrevs;
2574
2575 MDAbbrevs.resize(new_size: MetadataAbbrev::LastPlusOne);
2576 MDAbbrevs[MetadataAbbrev::DILocationAbbrevID] = createDILocationAbbrev();
2577 MDAbbrevs[MetadataAbbrev::GenericDINodeAbbrevID] =
2578 createGenericDINodeAbbrev();
2579
2580 auto Abbv = std::make_shared<BitCodeAbbrev>();
2581 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_INDEX_OFFSET));
2582 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2583 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2584 unsigned OffsetAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2585
2586 Abbv = std::make_shared<BitCodeAbbrev>();
2587 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_INDEX));
2588 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2589 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
2590 unsigned IndexAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2591
2592 // Emit MDStrings together upfront.
2593 writeMetadataStrings(Strings: VE.getMDStrings(), Record);
2594
2595 // We only emit an index for the metadata record if we have more than a given
2596 // (naive) threshold of metadatas, otherwise it is not worth it.
2597 if (VE.getNonMDStrings().size() > IndexThreshold) {
2598 // Write a placeholder value in for the offset of the metadata index,
2599 // which is written after the records, so that it can include
2600 // the offset of each entry. The placeholder offset will be
2601 // updated after all records are emitted.
2602 uint64_t Vals[] = {0, 0};
2603 Stream.EmitRecord(Code: bitc::METADATA_INDEX_OFFSET, Vals, Abbrev: OffsetAbbrev);
2604 }
2605
2606 // Compute and save the bit offset to the current position, which will be
2607 // patched when we emit the index later. We can simply subtract the 64-bit
2608 // fixed size from the current bit number to get the location to backpatch.
2609 uint64_t IndexOffsetRecordBitPos = Stream.GetCurrentBitNo();
2610
2611 // This index will contain the bitpos for each individual record.
2612 std::vector<uint64_t> IndexPos;
2613 IndexPos.reserve(n: VE.getNonMDStrings().size());
2614
2615 // Write all the records
2616 writeMetadataRecords(MDs: VE.getNonMDStrings(), Record, MDAbbrevs: &MDAbbrevs, IndexPos: &IndexPos);
2617
2618 if (VE.getNonMDStrings().size() > IndexThreshold) {
2619 // Now that we have emitted all the records we will emit the index. But
2620 // first
2621 // backpatch the forward reference so that the reader can skip the records
2622 // efficiently.
2623 Stream.BackpatchWord64(BitNo: IndexOffsetRecordBitPos - 64,
2624 Val: Stream.GetCurrentBitNo() - IndexOffsetRecordBitPos);
2625
2626 // Delta encode the index.
2627 uint64_t PreviousValue = IndexOffsetRecordBitPos;
2628 for (auto &Elt : IndexPos) {
2629 auto EltDelta = Elt - PreviousValue;
2630 PreviousValue = Elt;
2631 Elt = EltDelta;
2632 }
2633 // Emit the index record.
2634 Stream.EmitRecord(Code: bitc::METADATA_INDEX, Vals: IndexPos, Abbrev: IndexAbbrev);
2635 IndexPos.clear();
2636 }
2637
2638 // Write the named metadata now.
2639 writeNamedMetadata(Record);
2640
2641 auto AddDeclAttachedMetadata = [&](const GlobalObject &GO) {
2642 SmallVector<uint64_t, 4> Record;
2643 Record.push_back(Elt: VE.getValueID(V: &GO));
2644 pushGlobalMetadataAttachment(Record, GO);
2645 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_DECL_ATTACHMENT, Vals: Record);
2646 };
2647 for (const Function &F : M)
2648 if (F.isDeclaration() && F.hasMetadata())
2649 AddDeclAttachedMetadata(F);
2650 for (const GlobalIFunc &GI : M.ifuncs())
2651 if (GI.hasMetadata())
2652 AddDeclAttachedMetadata(GI);
2653 // FIXME: Only store metadata for declarations here, and move data for global
2654 // variable definitions to a separate block (PR28134).
2655 for (const GlobalVariable &GV : M.globals())
2656 if (GV.hasMetadata())
2657 AddDeclAttachedMetadata(GV);
2658
2659 Stream.ExitBlock();
2660}
2661
2662void ModuleBitcodeWriter::writeFunctionMetadata(const Function &F) {
2663 if (!VE.hasMDs())
2664 return;
2665
2666 Stream.EnterSubblock(BlockID: bitc::METADATA_BLOCK_ID, CodeLen: 3);
2667 SmallVector<uint64_t, 64> Record;
2668 writeMetadataStrings(Strings: VE.getMDStrings(), Record);
2669 writeMetadataRecords(MDs: VE.getNonMDStrings(), Record);
2670 Stream.ExitBlock();
2671}
2672
2673void ModuleBitcodeWriter::pushGlobalMetadataAttachment(
2674 SmallVectorImpl<uint64_t> &Record, const GlobalObject &GO) {
2675 // [n x [id, mdnode]]
2676 SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2677 GO.getAllMetadata(MDs);
2678 for (const auto &I : MDs) {
2679 Record.push_back(Elt: I.first);
2680 Record.push_back(Elt: VE.getMetadataID(MD: I.second));
2681 }
2682}
2683
2684void ModuleBitcodeWriter::writeFunctionMetadataAttachment(const Function &F) {
2685 Stream.EnterSubblock(BlockID: bitc::METADATA_ATTACHMENT_ID, CodeLen: 3);
2686
2687 SmallVector<uint64_t, 64> Record;
2688
2689 if (F.hasMetadata()) {
2690 pushGlobalMetadataAttachment(Record, GO: F);
2691 Stream.EmitRecord(Code: bitc::METADATA_ATTACHMENT, Vals: Record, Abbrev: 0);
2692 Record.clear();
2693 }
2694
2695 // Write metadata attachments
2696 // METADATA_ATTACHMENT - [m x [value, [n x [id, mdnode]]]
2697 SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2698 for (const BasicBlock &BB : F)
2699 for (const Instruction &I : BB) {
2700 MDs.clear();
2701 I.getAllMetadataOtherThanDebugLoc(MDs);
2702
2703 // If no metadata, ignore instruction.
2704 if (MDs.empty()) continue;
2705
2706 Record.push_back(Elt: VE.getInstructionID(I: &I));
2707
2708 for (const auto &[ID, MD] : MDs) {
2709 Record.push_back(Elt: ID);
2710 Record.push_back(Elt: VE.getMetadataID(MD));
2711 }
2712 Stream.EmitRecord(Code: bitc::METADATA_ATTACHMENT, Vals: Record, Abbrev: 0);
2713 Record.clear();
2714 }
2715
2716 Stream.ExitBlock();
2717}
2718
2719void ModuleBitcodeWriter::writeModuleMetadataKinds() {
2720 SmallVector<uint64_t, 64> Record;
2721
2722 // Write metadata kinds
2723 // METADATA_KIND - [n x [id, name]]
2724 SmallVector<StringRef, 8> Names;
2725 M.getMDKindNames(Result&: Names);
2726
2727 if (Names.empty()) return;
2728
2729 Stream.EnterSubblock(BlockID: bitc::METADATA_KIND_BLOCK_ID, CodeLen: 3);
2730
2731 for (unsigned MDKindID = 0, e = Names.size(); MDKindID != e; ++MDKindID) {
2732 Record.push_back(Elt: MDKindID);
2733 StringRef KName = Names[MDKindID];
2734 Record.append(in_start: KName.begin(), in_end: KName.end());
2735
2736 Stream.EmitRecord(Code: bitc::METADATA_KIND, Vals: Record, Abbrev: 0);
2737 Record.clear();
2738 }
2739
2740 Stream.ExitBlock();
2741}
2742
2743void ModuleBitcodeWriter::writeOperandBundleTags() {
2744 // Write metadata kinds
2745 //
2746 // OPERAND_BUNDLE_TAGS_BLOCK_ID : N x OPERAND_BUNDLE_TAG
2747 //
2748 // OPERAND_BUNDLE_TAG - [strchr x N]
2749
2750 SmallVector<StringRef, 8> Tags;
2751 M.getOperandBundleTags(Result&: Tags);
2752
2753 if (Tags.empty())
2754 return;
2755
2756 Stream.EnterSubblock(BlockID: bitc::OPERAND_BUNDLE_TAGS_BLOCK_ID, CodeLen: 3);
2757
2758 SmallVector<uint64_t, 64> Record;
2759
2760 for (auto Tag : Tags) {
2761 Record.append(in_start: Tag.begin(), in_end: Tag.end());
2762
2763 Stream.EmitRecord(Code: bitc::OPERAND_BUNDLE_TAG, Vals: Record, Abbrev: 0);
2764 Record.clear();
2765 }
2766
2767 Stream.ExitBlock();
2768}
2769
2770void ModuleBitcodeWriter::writeSyncScopeNames() {
2771 SmallVector<StringRef, 8> SSNs;
2772 M.getContext().getSyncScopeNames(SSNs);
2773 if (SSNs.empty())
2774 return;
2775
2776 Stream.EnterSubblock(BlockID: bitc::SYNC_SCOPE_NAMES_BLOCK_ID, CodeLen: 2);
2777
2778 SmallVector<uint64_t, 64> Record;
2779 for (auto SSN : SSNs) {
2780 Record.append(in_start: SSN.begin(), in_end: SSN.end());
2781 Stream.EmitRecord(Code: bitc::SYNC_SCOPE_NAME, Vals: Record, Abbrev: 0);
2782 Record.clear();
2783 }
2784
2785 Stream.ExitBlock();
2786}
2787
2788void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal,
2789 bool isGlobal) {
2790 if (FirstVal == LastVal) return;
2791
2792 Stream.EnterSubblock(BlockID: bitc::CONSTANTS_BLOCK_ID, CodeLen: 4);
2793
2794 unsigned AggregateAbbrev = 0;
2795 unsigned String8Abbrev = 0;
2796 unsigned CString7Abbrev = 0;
2797 unsigned CString6Abbrev = 0;
2798 // If this is a constant pool for the module, emit module-specific abbrevs.
2799 if (isGlobal) {
2800 // Abbrev for CST_CODE_AGGREGATE.
2801 auto Abbv = std::make_shared<BitCodeAbbrev>();
2802 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_AGGREGATE));
2803 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2804 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(Value: LastVal+1)));
2805 AggregateAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2806
2807 // Abbrev for CST_CODE_STRING.
2808 Abbv = std::make_shared<BitCodeAbbrev>();
2809 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_STRING));
2810 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2811 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2812 String8Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2813 // Abbrev for CST_CODE_CSTRING.
2814 Abbv = std::make_shared<BitCodeAbbrev>();
2815 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2816 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2817 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
2818 CString7Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2819 // Abbrev for CST_CODE_CSTRING.
2820 Abbv = std::make_shared<BitCodeAbbrev>();
2821 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2822 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2823 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
2824 CString6Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2825 }
2826
2827 SmallVector<uint64_t, 64> Record;
2828
2829 const ValueEnumerator::ValueList &Vals = VE.getValues();
2830 Type *LastTy = nullptr;
2831 for (unsigned i = FirstVal; i != LastVal; ++i) {
2832 const Value *V = Vals[i].first;
2833 // If we need to switch types, do so now.
2834 if (V->getType() != LastTy) {
2835 LastTy = V->getType();
2836 Record.push_back(Elt: VE.getTypeID(T: LastTy));
2837 Stream.EmitRecord(Code: bitc::CST_CODE_SETTYPE, Vals: Record,
2838 Abbrev: CONSTANTS_SETTYPE_ABBREV);
2839 Record.clear();
2840 }
2841
2842 if (const InlineAsm *IA = dyn_cast<InlineAsm>(Val: V)) {
2843 Record.push_back(Elt: VE.getTypeID(T: IA->getFunctionType()));
2844 Record.push_back(
2845 Elt: unsigned(IA->hasSideEffects()) | unsigned(IA->isAlignStack()) << 1 |
2846 unsigned(IA->getDialect() & 1) << 2 | unsigned(IA->canThrow()) << 3);
2847
2848 // Add the asm string.
2849 StringRef AsmStr = IA->getAsmString();
2850 Record.push_back(Elt: AsmStr.size());
2851 Record.append(in_start: AsmStr.begin(), in_end: AsmStr.end());
2852
2853 // Add the constraint string.
2854 StringRef ConstraintStr = IA->getConstraintString();
2855 Record.push_back(Elt: ConstraintStr.size());
2856 Record.append(in_start: ConstraintStr.begin(), in_end: ConstraintStr.end());
2857 Stream.EmitRecord(Code: bitc::CST_CODE_INLINEASM, Vals: Record);
2858 Record.clear();
2859 continue;
2860 }
2861 const Constant *C = cast<Constant>(Val: V);
2862 unsigned Code = -1U;
2863 unsigned AbbrevToUse = 0;
2864 if (C->isNullValue()) {
2865 Code = bitc::CST_CODE_NULL;
2866 } else if (isa<PoisonValue>(Val: C)) {
2867 Code = bitc::CST_CODE_POISON;
2868 } else if (isa<UndefValue>(Val: C)) {
2869 Code = bitc::CST_CODE_UNDEF;
2870 } else if (const ConstantInt *IV = dyn_cast<ConstantInt>(Val: C)) {
2871 if (IV->getBitWidth() <= 64) {
2872 uint64_t V = IV->getSExtValue();
2873 emitSignedInt64(Vals&: Record, V);
2874 Code = bitc::CST_CODE_INTEGER;
2875 AbbrevToUse = CONSTANTS_INTEGER_ABBREV;
2876 } else { // Wide integers, > 64 bits in size.
2877 emitWideAPInt(Vals&: Record, A: IV->getValue());
2878 Code = bitc::CST_CODE_WIDE_INTEGER;
2879 }
2880 } else if (const ConstantByte *BV = dyn_cast<ConstantByte>(Val: C)) {
2881 if (BV->getBitWidth() <= 64) {
2882 uint64_t V = BV->getSExtValue();
2883 emitSignedInt64(Vals&: Record, V);
2884 Code = bitc::CST_CODE_BYTE;
2885 AbbrevToUse = CONSTANTS_BYTE_ABBREV;
2886 } else { // Wide bytes, > 64 bits in size.
2887 emitWideAPInt(Vals&: Record, A: BV->getValue());
2888 Code = bitc::CST_CODE_WIDE_BYTE;
2889 }
2890 } else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
2891 Code = bitc::CST_CODE_FLOAT;
2892 Type *Ty = CFP->getType()->getScalarType();
2893 if (Ty->isHalfTy() || Ty->isBFloatTy() || Ty->isFloatTy() ||
2894 Ty->isDoubleTy()) {
2895 Record.push_back(Elt: CFP->getValueAPF().bitcastToAPInt().getZExtValue());
2896 } else if (Ty->isX86_FP80Ty()) {
2897 // api needed to prevent premature destruction
2898 // bits are not in the same order as a normal i80 APInt, compensate.
2899 APInt api = CFP->getValueAPF().bitcastToAPInt();
2900 const uint64_t *p = api.getRawData();
2901 Record.push_back(Elt: (p[1] << 48) | (p[0] >> 16));
2902 Record.push_back(Elt: p[0] & 0xffffLL);
2903 } else if (Ty->isFP128Ty() || Ty->isPPC_FP128Ty()) {
2904 APInt api = CFP->getValueAPF().bitcastToAPInt();
2905 const uint64_t *p = api.getRawData();
2906 Record.push_back(Elt: p[0]);
2907 Record.push_back(Elt: p[1]);
2908 } else {
2909 assert(0 && "Unknown FP type!");
2910 }
2911 } else if (isa<ConstantDataSequential>(Val: C) &&
2912 cast<ConstantDataSequential>(Val: C)->isString()) {
2913 const ConstantDataSequential *Str = cast<ConstantDataSequential>(Val: C);
2914 // Emit constant strings specially.
2915 uint64_t NumElts = Str->getNumElements();
2916 // If this is a null-terminated string, use the denser CSTRING encoding.
2917 if (Str->isCString()) {
2918 Code = bitc::CST_CODE_CSTRING;
2919 --NumElts; // Don't encode the null, which isn't allowed by char6.
2920 } else {
2921 Code = bitc::CST_CODE_STRING;
2922 AbbrevToUse = String8Abbrev;
2923 }
2924 bool isCStr7 = Code == bitc::CST_CODE_CSTRING;
2925 bool isCStrChar6 = Code == bitc::CST_CODE_CSTRING;
2926 for (uint64_t i = 0; i != NumElts; ++i) {
2927 unsigned char V = Str->getElementAsInteger(i);
2928 Record.push_back(Elt: V);
2929 isCStr7 &= (V & 128) == 0;
2930 if (isCStrChar6)
2931 isCStrChar6 = BitCodeAbbrevOp::isChar6(C: V);
2932 }
2933
2934 if (isCStrChar6)
2935 AbbrevToUse = CString6Abbrev;
2936 else if (isCStr7)
2937 AbbrevToUse = CString7Abbrev;
2938 } else if (const ConstantDataSequential *CDS =
2939 dyn_cast<ConstantDataSequential>(Val: C)) {
2940 Code = bitc::CST_CODE_DATA;
2941 Type *EltTy = CDS->getElementType();
2942 if (isa<IntegerType>(Val: EltTy) || isa<ByteType>(Val: EltTy)) {
2943 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i)
2944 Record.push_back(Elt: CDS->getElementAsInteger(i));
2945 } else {
2946 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i)
2947 Record.push_back(
2948 Elt: CDS->getElementAsAPFloat(i).bitcastToAPInt().getLimitedValue());
2949 }
2950 } else if (isa<ConstantAggregate>(Val: C)) {
2951 Code = bitc::CST_CODE_AGGREGATE;
2952 for (const Value *Op : C->operands())
2953 Record.push_back(Elt: VE.getValueID(V: Op));
2954 AbbrevToUse = AggregateAbbrev;
2955 } else if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: C)) {
2956 switch (CE->getOpcode()) {
2957 default:
2958 if (Instruction::isCast(Opcode: CE->getOpcode())) {
2959 Code = bitc::CST_CODE_CE_CAST;
2960 Record.push_back(Elt: getEncodedCastOpcode(Opcode: CE->getOpcode()));
2961 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
2962 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2963 AbbrevToUse = CONSTANTS_CE_CAST_Abbrev;
2964 } else {
2965 assert(CE->getNumOperands() == 2 && "Unknown constant expr!");
2966 Code = bitc::CST_CODE_CE_BINOP;
2967 Record.push_back(Elt: getEncodedBinaryOpcode(Opcode: CE->getOpcode()));
2968 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2969 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
2970 uint64_t Flags = getOptimizationFlags(V: CE);
2971 if (Flags != 0)
2972 Record.push_back(Elt: Flags);
2973 }
2974 break;
2975 case Instruction::FNeg: {
2976 assert(CE->getNumOperands() == 1 && "Unknown constant expr!");
2977 Code = bitc::CST_CODE_CE_UNOP;
2978 Record.push_back(Elt: getEncodedUnaryOpcode(Opcode: CE->getOpcode()));
2979 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2980 uint64_t Flags = getOptimizationFlags(V: CE);
2981 if (Flags != 0)
2982 Record.push_back(Elt: Flags);
2983 break;
2984 }
2985 case Instruction::GetElementPtr: {
2986 Code = bitc::CST_CODE_CE_GEP;
2987 const auto *GO = cast<GEPOperator>(Val: C);
2988 Record.push_back(Elt: VE.getTypeID(T: GO->getSourceElementType()));
2989 Record.push_back(Elt: getOptimizationFlags(V: GO));
2990 if (std::optional<ConstantRange> Range = GO->getInRange()) {
2991 Code = bitc::CST_CODE_CE_GEP_WITH_INRANGE;
2992 emitConstantRange(Record, CR: *Range, /*EmitBitWidth=*/true);
2993 }
2994 for (const Value *Op : CE->operands()) {
2995 Record.push_back(Elt: VE.getTypeID(T: Op->getType()));
2996 Record.push_back(Elt: VE.getValueID(V: Op));
2997 }
2998 break;
2999 }
3000 case Instruction::ExtractElement:
3001 Code = bitc::CST_CODE_CE_EXTRACTELT;
3002 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
3003 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3004 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 1)->getType()));
3005 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3006 break;
3007 case Instruction::InsertElement:
3008 Code = bitc::CST_CODE_CE_INSERTELT;
3009 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3010 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3011 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 2)->getType()));
3012 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 2)));
3013 break;
3014 case Instruction::ShuffleVector:
3015 // If the return type and argument types are the same, this is a
3016 // standard shufflevector instruction. If the types are different,
3017 // then the shuffle is widening or truncating the input vectors, and
3018 // the argument type must also be encoded.
3019 if (C->getType() == C->getOperand(i: 0)->getType()) {
3020 Code = bitc::CST_CODE_CE_SHUFFLEVEC;
3021 } else {
3022 Code = bitc::CST_CODE_CE_SHUFVEC_EX;
3023 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
3024 }
3025 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3026 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3027 Record.push_back(Elt: VE.getValueID(V: CE->getShuffleMaskForBitcode()));
3028 break;
3029 }
3030 } else if (const BlockAddress *BA = dyn_cast<BlockAddress>(Val: C)) {
3031 Code = bitc::CST_CODE_BLOCKADDRESS;
3032 Record.push_back(Elt: VE.getTypeID(T: BA->getFunction()->getType()));
3033 Record.push_back(Elt: VE.getValueID(V: BA->getFunction()));
3034 Record.push_back(Elt: VE.getGlobalBasicBlockID(BB: BA->getBasicBlock()));
3035 } else if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(Val: C)) {
3036 Code = bitc::CST_CODE_DSO_LOCAL_EQUIVALENT;
3037 Record.push_back(Elt: VE.getTypeID(T: Equiv->getGlobalValue()->getType()));
3038 Record.push_back(Elt: VE.getValueID(V: Equiv->getGlobalValue()));
3039 } else if (const auto *NC = dyn_cast<NoCFIValue>(Val: C)) {
3040 Code = bitc::CST_CODE_NO_CFI_VALUE;
3041 Record.push_back(Elt: VE.getTypeID(T: NC->getGlobalValue()->getType()));
3042 Record.push_back(Elt: VE.getValueID(V: NC->getGlobalValue()));
3043 } else if (const auto *CPA = dyn_cast<ConstantPtrAuth>(Val: C)) {
3044 Code = bitc::CST_CODE_PTRAUTH2;
3045 Record.push_back(Elt: VE.getValueID(V: CPA->getPointer()));
3046 Record.push_back(Elt: VE.getValueID(V: CPA->getKey()));
3047 Record.push_back(Elt: VE.getValueID(V: CPA->getDiscriminator()));
3048 Record.push_back(Elt: VE.getValueID(V: CPA->getAddrDiscriminator()));
3049 Record.push_back(Elt: VE.getValueID(V: CPA->getDeactivationSymbol()));
3050 } else {
3051#ifndef NDEBUG
3052 C->dump();
3053#endif
3054 llvm_unreachable("Unknown constant!");
3055 }
3056 Stream.EmitRecord(Code, Vals: Record, Abbrev: AbbrevToUse);
3057 Record.clear();
3058 }
3059
3060 Stream.ExitBlock();
3061}
3062
3063void ModuleBitcodeWriter::writeModuleConstants() {
3064 const ValueEnumerator::ValueList &Vals = VE.getValues();
3065
3066 // Find the first constant to emit, which is the first non-globalvalue value.
3067 // We know globalvalues have been emitted by WriteModuleInfo.
3068 for (unsigned i = 0, e = Vals.size(); i != e; ++i) {
3069 if (!isa<GlobalValue>(Val: Vals[i].first)) {
3070 writeConstants(FirstVal: i, LastVal: Vals.size(), isGlobal: true);
3071 return;
3072 }
3073 }
3074}
3075
3076/// pushValueAndType - The file has to encode both the value and type id for
3077/// many values, because we need to know what type to create for forward
3078/// references. However, most operands are not forward references, so this type
3079/// field is not needed.
3080///
3081/// This function adds V's value ID to Vals. If the value ID is higher than the
3082/// instruction ID, then it is a forward reference, and it also includes the
3083/// type ID. The value ID that is written is encoded relative to the InstID.
3084bool ModuleBitcodeWriter::pushValueAndType(const Value *V, unsigned InstID,
3085 SmallVectorImpl<unsigned> &Vals) {
3086 unsigned ValID = VE.getValueID(V);
3087 // Make encoding relative to the InstID.
3088 Vals.push_back(Elt: InstID - ValID);
3089 if (ValID >= InstID) {
3090 Vals.push_back(Elt: VE.getTypeID(T: V->getType()));
3091 return true;
3092 }
3093 return false;
3094}
3095
3096bool ModuleBitcodeWriter::pushValueOrMetadata(const Value *V, unsigned InstID,
3097 SmallVectorImpl<unsigned> &Vals) {
3098 bool IsMetadata = V->getType()->isMetadataTy();
3099 if (IsMetadata) {
3100 Vals.push_back(Elt: bitc::OB_METADATA);
3101 Metadata *MD = cast<MetadataAsValue>(Val: V)->getMetadata();
3102 unsigned ValID = VE.getMetadataID(MD);
3103 Vals.push_back(Elt: InstID - ValID);
3104 return false;
3105 }
3106 return pushValueAndType(V, InstID, Vals);
3107}
3108
3109void ModuleBitcodeWriter::writeOperandBundles(const CallBase &CS,
3110 unsigned InstID) {
3111 SmallVector<unsigned, 64> Record;
3112 LLVMContext &C = CS.getContext();
3113
3114 for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i) {
3115 const auto &Bundle = CS.getOperandBundleAt(Index: i);
3116 Record.push_back(Elt: C.getOperandBundleTagID(Tag: Bundle.getTagName()));
3117
3118 for (auto &Input : Bundle.Inputs)
3119 pushValueOrMetadata(V: Input, InstID, Vals&: Record);
3120
3121 Stream.EmitRecord(Code: bitc::FUNC_CODE_OPERAND_BUNDLE, Vals: Record);
3122 Record.clear();
3123 }
3124}
3125
3126/// pushValue - Like pushValueAndType, but where the type of the value is
3127/// omitted (perhaps it was already encoded in an earlier operand).
3128void ModuleBitcodeWriter::pushValue(const Value *V, unsigned InstID,
3129 SmallVectorImpl<unsigned> &Vals) {
3130 unsigned ValID = VE.getValueID(V);
3131 Vals.push_back(Elt: InstID - ValID);
3132}
3133
3134void ModuleBitcodeWriter::pushValueSigned(const Value *V, unsigned InstID,
3135 SmallVectorImpl<uint64_t> &Vals) {
3136 unsigned ValID = VE.getValueID(V);
3137 int64_t diff = ((int32_t)InstID - (int32_t)ValID);
3138 emitSignedInt64(Vals, V: diff);
3139}
3140
3141/// WriteInstruction - Emit an instruction to the specified stream.
3142void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
3143 unsigned InstID,
3144 SmallVectorImpl<unsigned> &Vals) {
3145 unsigned Code = 0;
3146 unsigned AbbrevToUse = 0;
3147 VE.setInstructionID(&I);
3148 switch (I.getOpcode()) {
3149 default:
3150 if (Instruction::isCast(Opcode: I.getOpcode())) {
3151 Code = bitc::FUNC_CODE_INST_CAST;
3152 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3153 AbbrevToUse = FUNCTION_INST_CAST_ABBREV;
3154 Vals.push_back(Elt: VE.getTypeID(T: I.getType()));
3155 Vals.push_back(Elt: getEncodedCastOpcode(Opcode: I.getOpcode()));
3156 uint64_t Flags = getOptimizationFlags(V: &I);
3157 if (Flags != 0) {
3158 if (AbbrevToUse == FUNCTION_INST_CAST_ABBREV)
3159 AbbrevToUse = FUNCTION_INST_CAST_FLAGS_ABBREV;
3160 Vals.push_back(Elt: Flags);
3161 }
3162 } else {
3163 assert(isa<BinaryOperator>(I) && "Unknown instruction!");
3164 Code = bitc::FUNC_CODE_INST_BINOP;
3165 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3166 AbbrevToUse = FUNCTION_INST_BINOP_ABBREV;
3167 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3168 Vals.push_back(Elt: getEncodedBinaryOpcode(Opcode: I.getOpcode()));
3169 uint64_t Flags = getOptimizationFlags(V: &I);
3170 if (Flags != 0) {
3171 if (AbbrevToUse == FUNCTION_INST_BINOP_ABBREV)
3172 AbbrevToUse = FUNCTION_INST_BINOP_FLAGS_ABBREV;
3173 Vals.push_back(Elt: Flags);
3174 }
3175 }
3176 break;
3177 case Instruction::FNeg: {
3178 Code = bitc::FUNC_CODE_INST_UNOP;
3179 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3180 AbbrevToUse = FUNCTION_INST_UNOP_ABBREV;
3181 Vals.push_back(Elt: getEncodedUnaryOpcode(Opcode: I.getOpcode()));
3182 uint64_t Flags = getOptimizationFlags(V: &I);
3183 if (Flags != 0) {
3184 if (AbbrevToUse == FUNCTION_INST_UNOP_ABBREV)
3185 AbbrevToUse = FUNCTION_INST_UNOP_FLAGS_ABBREV;
3186 Vals.push_back(Elt: Flags);
3187 }
3188 break;
3189 }
3190 case Instruction::GetElementPtr: {
3191 Code = bitc::FUNC_CODE_INST_GEP;
3192 AbbrevToUse = FUNCTION_INST_GEP_ABBREV;
3193 auto &GEPInst = cast<GetElementPtrInst>(Val: I);
3194 Vals.push_back(Elt: getOptimizationFlags(V: &I));
3195 Vals.push_back(Elt: VE.getTypeID(T: GEPInst.getSourceElementType()));
3196 for (const Value *Op : I.operands())
3197 pushValueAndType(V: Op, InstID, Vals);
3198 break;
3199 }
3200 case Instruction::ExtractValue: {
3201 Code = bitc::FUNC_CODE_INST_EXTRACTVAL;
3202 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3203 const ExtractValueInst *EVI = cast<ExtractValueInst>(Val: &I);
3204 Vals.append(in_start: EVI->idx_begin(), in_end: EVI->idx_end());
3205 break;
3206 }
3207 case Instruction::InsertValue: {
3208 Code = bitc::FUNC_CODE_INST_INSERTVAL;
3209 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3210 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3211 const InsertValueInst *IVI = cast<InsertValueInst>(Val: &I);
3212 Vals.append(in_start: IVI->idx_begin(), in_end: IVI->idx_end());
3213 break;
3214 }
3215 case Instruction::Select: {
3216 Code = bitc::FUNC_CODE_INST_VSELECT;
3217 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3218 pushValue(V: I.getOperand(i: 2), InstID, Vals);
3219 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3220 uint64_t Flags = getOptimizationFlags(V: &I);
3221 if (Flags != 0)
3222 Vals.push_back(Elt: Flags);
3223 break;
3224 }
3225 case Instruction::ExtractElement:
3226 Code = bitc::FUNC_CODE_INST_EXTRACTELT;
3227 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3228 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3229 break;
3230 case Instruction::InsertElement:
3231 Code = bitc::FUNC_CODE_INST_INSERTELT;
3232 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3233 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3234 pushValueAndType(V: I.getOperand(i: 2), InstID, Vals);
3235 break;
3236 case Instruction::ShuffleVector:
3237 Code = bitc::FUNC_CODE_INST_SHUFFLEVEC;
3238 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3239 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3240 pushValue(V: cast<ShuffleVectorInst>(Val: I).getShuffleMaskForBitcode(), InstID,
3241 Vals);
3242 break;
3243 case Instruction::ICmp:
3244 case Instruction::FCmp: {
3245 // compare returning Int1Ty or vector of Int1Ty
3246 Code = bitc::FUNC_CODE_INST_CMP2;
3247 AbbrevToUse = FUNCTION_INST_CMP_ABBREV;
3248 if (pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3249 AbbrevToUse = 0;
3250 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3251 Vals.push_back(Elt: cast<CmpInst>(Val: I).getPredicate());
3252 uint64_t Flags = getOptimizationFlags(V: &I);
3253 if (Flags != 0) {
3254 Vals.push_back(Elt: Flags);
3255 if (AbbrevToUse)
3256 AbbrevToUse = FUNCTION_INST_CMP_FLAGS_ABBREV;
3257 }
3258 break;
3259 }
3260
3261 case Instruction::Ret:
3262 {
3263 Code = bitc::FUNC_CODE_INST_RET;
3264 unsigned NumOperands = I.getNumOperands();
3265 if (NumOperands == 0)
3266 AbbrevToUse = FUNCTION_INST_RET_VOID_ABBREV;
3267 else if (NumOperands == 1) {
3268 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3269 AbbrevToUse = FUNCTION_INST_RET_VAL_ABBREV;
3270 } else {
3271 for (const Value *Op : I.operands())
3272 pushValueAndType(V: Op, InstID, Vals);
3273 }
3274 }
3275 break;
3276 case Instruction::UncondBr: {
3277 Code = bitc::FUNC_CODE_INST_BR;
3278 AbbrevToUse = FUNCTION_INST_BR_UNCOND_ABBREV;
3279 const UncondBrInst &II = cast<UncondBrInst>(Val: I);
3280 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 0)));
3281 } break;
3282 case Instruction::CondBr: {
3283 Code = bitc::FUNC_CODE_INST_BR;
3284 AbbrevToUse = FUNCTION_INST_BR_COND_ABBREV;
3285 const CondBrInst &II = cast<CondBrInst>(Val: I);
3286 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 0)));
3287 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 1)));
3288 pushValue(V: II.getCondition(), InstID, Vals);
3289 } break;
3290 case Instruction::Switch:
3291 {
3292 Code = bitc::FUNC_CODE_INST_SWITCH;
3293 const SwitchInst &SI = cast<SwitchInst>(Val: I);
3294 Vals.push_back(Elt: VE.getTypeID(T: SI.getCondition()->getType()));
3295 pushValue(V: SI.getCondition(), InstID, Vals);
3296 Vals.push_back(Elt: VE.getValueID(V: SI.getDefaultDest()));
3297 for (auto Case : SI.cases()) {
3298 Vals.push_back(Elt: VE.getValueID(V: Case.getCaseValue()));
3299 Vals.push_back(Elt: VE.getValueID(V: Case.getCaseSuccessor()));
3300 }
3301 }
3302 break;
3303 case Instruction::IndirectBr:
3304 Code = bitc::FUNC_CODE_INST_INDIRECTBR;
3305 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType()));
3306 // Encode the address operand as relative, but not the basic blocks.
3307 pushValue(V: I.getOperand(i: 0), InstID, Vals);
3308 for (const Value *Op : drop_begin(RangeOrContainer: I.operands()))
3309 Vals.push_back(Elt: VE.getValueID(V: Op));
3310 break;
3311
3312 case Instruction::Invoke: {
3313 const InvokeInst *II = cast<InvokeInst>(Val: &I);
3314 const Value *Callee = II->getCalledOperand();
3315 FunctionType *FTy = II->getFunctionType();
3316
3317 if (II->hasOperandBundles())
3318 writeOperandBundles(CS: *II, InstID);
3319
3320 Code = bitc::FUNC_CODE_INST_INVOKE;
3321
3322 Vals.push_back(Elt: VE.getAttributeListID(PAL: II->getAttributes()));
3323 Vals.push_back(Elt: II->getCallingConv() | 1 << 13);
3324 Vals.push_back(Elt: VE.getValueID(V: II->getNormalDest()));
3325 Vals.push_back(Elt: VE.getValueID(V: II->getUnwindDest()));
3326 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3327 pushValueAndType(V: Callee, InstID, Vals);
3328
3329 // Emit value #'s for the fixed parameters.
3330 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3331 pushValue(V: I.getOperand(i), InstID, Vals); // fixed param.
3332
3333 // Emit type/value pairs for varargs params.
3334 if (FTy->isVarArg()) {
3335 for (unsigned i = FTy->getNumParams(), e = II->arg_size(); i != e; ++i)
3336 pushValueAndType(V: I.getOperand(i), InstID, Vals); // vararg
3337 }
3338 break;
3339 }
3340 case Instruction::Resume:
3341 Code = bitc::FUNC_CODE_INST_RESUME;
3342 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3343 break;
3344 case Instruction::CleanupRet: {
3345 Code = bitc::FUNC_CODE_INST_CLEANUPRET;
3346 const auto &CRI = cast<CleanupReturnInst>(Val: I);
3347 pushValue(V: CRI.getCleanupPad(), InstID, Vals);
3348 if (CRI.hasUnwindDest())
3349 Vals.push_back(Elt: VE.getValueID(V: CRI.getUnwindDest()));
3350 break;
3351 }
3352 case Instruction::CatchRet: {
3353 Code = bitc::FUNC_CODE_INST_CATCHRET;
3354 const auto &CRI = cast<CatchReturnInst>(Val: I);
3355 pushValue(V: CRI.getCatchPad(), InstID, Vals);
3356 Vals.push_back(Elt: VE.getValueID(V: CRI.getSuccessor()));
3357 break;
3358 }
3359 case Instruction::CleanupPad:
3360 case Instruction::CatchPad: {
3361 const auto &FuncletPad = cast<FuncletPadInst>(Val: I);
3362 Code = isa<CatchPadInst>(Val: FuncletPad) ? bitc::FUNC_CODE_INST_CATCHPAD
3363 : bitc::FUNC_CODE_INST_CLEANUPPAD;
3364 pushValue(V: FuncletPad.getParentPad(), InstID, Vals);
3365
3366 unsigned NumArgOperands = FuncletPad.arg_size();
3367 Vals.push_back(Elt: NumArgOperands);
3368 for (unsigned Op = 0; Op != NumArgOperands; ++Op)
3369 pushValueAndType(V: FuncletPad.getArgOperand(i: Op), InstID, Vals);
3370 break;
3371 }
3372 case Instruction::CatchSwitch: {
3373 Code = bitc::FUNC_CODE_INST_CATCHSWITCH;
3374 const auto &CatchSwitch = cast<CatchSwitchInst>(Val: I);
3375
3376 pushValue(V: CatchSwitch.getParentPad(), InstID, Vals);
3377
3378 unsigned NumHandlers = CatchSwitch.getNumHandlers();
3379 Vals.push_back(Elt: NumHandlers);
3380 for (const BasicBlock *CatchPadBB : CatchSwitch.handlers())
3381 Vals.push_back(Elt: VE.getValueID(V: CatchPadBB));
3382
3383 if (CatchSwitch.hasUnwindDest())
3384 Vals.push_back(Elt: VE.getValueID(V: CatchSwitch.getUnwindDest()));
3385 break;
3386 }
3387 case Instruction::CallBr: {
3388 const CallBrInst *CBI = cast<CallBrInst>(Val: &I);
3389 const Value *Callee = CBI->getCalledOperand();
3390 FunctionType *FTy = CBI->getFunctionType();
3391
3392 if (CBI->hasOperandBundles())
3393 writeOperandBundles(CS: *CBI, InstID);
3394
3395 Code = bitc::FUNC_CODE_INST_CALLBR;
3396
3397 Vals.push_back(Elt: VE.getAttributeListID(PAL: CBI->getAttributes()));
3398
3399 Vals.push_back(Elt: CBI->getCallingConv() << bitc::CALL_CCONV |
3400 1 << bitc::CALL_EXPLICIT_TYPE);
3401
3402 Vals.push_back(Elt: VE.getValueID(V: CBI->getDefaultDest()));
3403 Vals.push_back(Elt: CBI->getNumIndirectDests());
3404 for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i)
3405 Vals.push_back(Elt: VE.getValueID(V: CBI->getIndirectDest(i)));
3406
3407 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3408 pushValueAndType(V: Callee, InstID, Vals);
3409
3410 // Emit value #'s for the fixed parameters.
3411 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3412 pushValue(V: I.getOperand(i), InstID, Vals); // fixed param.
3413
3414 // Emit type/value pairs for varargs params.
3415 if (FTy->isVarArg()) {
3416 for (unsigned i = FTy->getNumParams(), e = CBI->arg_size(); i != e; ++i)
3417 pushValueAndType(V: I.getOperand(i), InstID, Vals); // vararg
3418 }
3419 break;
3420 }
3421 case Instruction::Unreachable:
3422 Code = bitc::FUNC_CODE_INST_UNREACHABLE;
3423 AbbrevToUse = FUNCTION_INST_UNREACHABLE_ABBREV;
3424 break;
3425
3426 case Instruction::PHI: {
3427 const PHINode &PN = cast<PHINode>(Val: I);
3428 Code = bitc::FUNC_CODE_INST_PHI;
3429 // With the newer instruction encoding, forward references could give
3430 // negative valued IDs. This is most common for PHIs, so we use
3431 // signed VBRs.
3432 SmallVector<uint64_t, 128> Vals64;
3433 Vals64.push_back(Elt: VE.getTypeID(T: PN.getType()));
3434 for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) {
3435 pushValueSigned(V: PN.getIncomingValue(i), InstID, Vals&: Vals64);
3436 Vals64.push_back(Elt: VE.getValueID(V: PN.getIncomingBlock(i)));
3437 }
3438
3439 uint64_t Flags = getOptimizationFlags(V: &I);
3440 if (Flags != 0)
3441 Vals64.push_back(Elt: Flags);
3442
3443 // Emit a Vals64 vector and exit.
3444 Stream.EmitRecord(Code, Vals: Vals64, Abbrev: AbbrevToUse);
3445 Vals64.clear();
3446 return;
3447 }
3448
3449 case Instruction::LandingPad: {
3450 const LandingPadInst &LP = cast<LandingPadInst>(Val: I);
3451 Code = bitc::FUNC_CODE_INST_LANDINGPAD;
3452 Vals.push_back(Elt: VE.getTypeID(T: LP.getType()));
3453 Vals.push_back(Elt: LP.isCleanup());
3454 Vals.push_back(Elt: LP.getNumClauses());
3455 for (unsigned I = 0, E = LP.getNumClauses(); I != E; ++I) {
3456 if (LP.isCatch(Idx: I))
3457 Vals.push_back(Elt: LandingPadInst::Catch);
3458 else
3459 Vals.push_back(Elt: LandingPadInst::Filter);
3460 pushValueAndType(V: LP.getClause(Idx: I), InstID, Vals);
3461 }
3462 break;
3463 }
3464
3465 case Instruction::Alloca: {
3466 Code = bitc::FUNC_CODE_INST_ALLOCA;
3467 const AllocaInst &AI = cast<AllocaInst>(Val: I);
3468 Vals.push_back(Elt: VE.getTypeID(T: AI.getAllocatedType()));
3469 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType()));
3470 Vals.push_back(Elt: VE.getValueID(V: I.getOperand(i: 0))); // size.
3471 using APV = AllocaPackedValues;
3472 unsigned Record = 0;
3473 unsigned EncodedAlign = getEncodedAlign(Alignment: AI.getAlign());
3474 Bitfield::set<APV::AlignLower>(
3475 Packed&: Record, Value: EncodedAlign & ((1 << APV::AlignLower::Bits) - 1));
3476 Bitfield::set<APV::AlignUpper>(Packed&: Record,
3477 Value: EncodedAlign >> APV::AlignLower::Bits);
3478 Bitfield::set<APV::UsedWithInAlloca>(Packed&: Record, Value: AI.isUsedWithInAlloca());
3479 Bitfield::set<APV::ExplicitType>(Packed&: Record, Value: true);
3480 Bitfield::set<APV::SwiftError>(Packed&: Record, Value: AI.isSwiftError());
3481 Vals.push_back(Elt: Record);
3482
3483 unsigned AS = AI.getAddressSpace();
3484 if (AS != M.getDataLayout().getAllocaAddrSpace())
3485 Vals.push_back(Elt: AS);
3486 break;
3487 }
3488
3489 case Instruction::Load:
3490 if (cast<LoadInst>(Val: I).isAtomic()) {
3491 Code = bitc::FUNC_CODE_INST_LOADATOMIC;
3492 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3493 } else {
3494 Code = bitc::FUNC_CODE_INST_LOAD;
3495 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals)) // ptr
3496 AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
3497 }
3498 Vals.push_back(Elt: VE.getTypeID(T: I.getType()));
3499 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<LoadInst>(Val: I).getAlign()));
3500 Vals.push_back(Elt: cast<LoadInst>(Val: I).isVolatile());
3501 if (cast<LoadInst>(Val: I).isAtomic()) {
3502 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<LoadInst>(Val: I).getOrdering()));
3503 Vals.push_back(Elt: getEncodedSyncScopeID(SSID: cast<LoadInst>(Val: I).getSyncScopeID()));
3504 }
3505 break;
3506 case Instruction::Store:
3507 if (cast<StoreInst>(Val: I).isAtomic()) {
3508 Code = bitc::FUNC_CODE_INST_STOREATOMIC;
3509 } else {
3510 Code = bitc::FUNC_CODE_INST_STORE;
3511 AbbrevToUse = FUNCTION_INST_STORE_ABBREV;
3512 }
3513 if (pushValueAndType(V: I.getOperand(i: 1), InstID, Vals)) // ptrty + ptr
3514 AbbrevToUse = 0;
3515 if (pushValueAndType(V: I.getOperand(i: 0), InstID, Vals)) // valty + val
3516 AbbrevToUse = 0;
3517 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<StoreInst>(Val: I).getAlign()));
3518 Vals.push_back(Elt: cast<StoreInst>(Val: I).isVolatile());
3519 if (cast<StoreInst>(Val: I).isAtomic()) {
3520 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<StoreInst>(Val: I).getOrdering()));
3521 Vals.push_back(
3522 Elt: getEncodedSyncScopeID(SSID: cast<StoreInst>(Val: I).getSyncScopeID()));
3523 }
3524 break;
3525 case Instruction::AtomicCmpXchg:
3526 Code = bitc::FUNC_CODE_INST_CMPXCHG;
3527 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals); // ptrty + ptr
3528 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals); // cmp.
3529 pushValue(V: I.getOperand(i: 2), InstID, Vals); // newval.
3530 Vals.push_back(Elt: cast<AtomicCmpXchgInst>(Val: I).isVolatile());
3531 Vals.push_back(
3532 Elt: getEncodedOrdering(Ordering: cast<AtomicCmpXchgInst>(Val: I).getSuccessOrdering()));
3533 Vals.push_back(
3534 Elt: getEncodedSyncScopeID(SSID: cast<AtomicCmpXchgInst>(Val: I).getSyncScopeID()));
3535 Vals.push_back(
3536 Elt: getEncodedOrdering(Ordering: cast<AtomicCmpXchgInst>(Val: I).getFailureOrdering()));
3537 Vals.push_back(Elt: cast<AtomicCmpXchgInst>(Val: I).isWeak());
3538 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<AtomicCmpXchgInst>(Val: I).getAlign()));
3539 break;
3540 case Instruction::AtomicRMW:
3541 Code = bitc::FUNC_CODE_INST_ATOMICRMW;
3542 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals); // ptrty + ptr
3543 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals); // valty + val
3544 Vals.push_back(
3545 Elt: getEncodedRMWOperation(Op: cast<AtomicRMWInst>(Val: I).getOperation()));
3546 Vals.push_back(Elt: cast<AtomicRMWInst>(Val: I).isVolatile());
3547 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<AtomicRMWInst>(Val: I).getOrdering()));
3548 Vals.push_back(
3549 Elt: getEncodedSyncScopeID(SSID: cast<AtomicRMWInst>(Val: I).getSyncScopeID()));
3550 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<AtomicRMWInst>(Val: I).getAlign()));
3551 break;
3552 case Instruction::Fence:
3553 Code = bitc::FUNC_CODE_INST_FENCE;
3554 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<FenceInst>(Val: I).getOrdering()));
3555 Vals.push_back(Elt: getEncodedSyncScopeID(SSID: cast<FenceInst>(Val: I).getSyncScopeID()));
3556 break;
3557 case Instruction::Call: {
3558 const CallInst &CI = cast<CallInst>(Val: I);
3559 FunctionType *FTy = CI.getFunctionType();
3560
3561 if (CI.hasOperandBundles())
3562 writeOperandBundles(CS: CI, InstID);
3563
3564 Code = bitc::FUNC_CODE_INST_CALL;
3565
3566 Vals.push_back(Elt: VE.getAttributeListID(PAL: CI.getAttributes()));
3567
3568 unsigned Flags = getOptimizationFlags(V: &I);
3569 Vals.push_back(Elt: CI.getCallingConv() << bitc::CALL_CCONV |
3570 unsigned(CI.isTailCall()) << bitc::CALL_TAIL |
3571 unsigned(CI.isMustTailCall()) << bitc::CALL_MUSTTAIL |
3572 1 << bitc::CALL_EXPLICIT_TYPE |
3573 unsigned(CI.isNoTailCall()) << bitc::CALL_NOTAIL |
3574 unsigned(Flags != 0) << bitc::CALL_FMF);
3575 if (Flags != 0)
3576 Vals.push_back(Elt: Flags);
3577
3578 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3579 pushValueAndType(V: CI.getCalledOperand(), InstID, Vals); // Callee
3580
3581 // Emit value #'s for the fixed parameters.
3582 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3583 pushValue(V: CI.getArgOperand(i), InstID, Vals); // fixed param.
3584
3585 // Emit type/value pairs for varargs params.
3586 if (FTy->isVarArg()) {
3587 for (unsigned i = FTy->getNumParams(), e = CI.arg_size(); i != e; ++i)
3588 pushValueAndType(V: CI.getArgOperand(i), InstID, Vals); // varargs
3589 }
3590 break;
3591 }
3592 case Instruction::VAArg:
3593 Code = bitc::FUNC_CODE_INST_VAARG;
3594 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType())); // valistty
3595 pushValue(V: I.getOperand(i: 0), InstID, Vals); // valist.
3596 Vals.push_back(Elt: VE.getTypeID(T: I.getType())); // restype.
3597 break;
3598 case Instruction::Freeze:
3599 Code = bitc::FUNC_CODE_INST_FREEZE;
3600 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3601 break;
3602 }
3603
3604 Stream.EmitRecord(Code, Vals, Abbrev: AbbrevToUse);
3605 Vals.clear();
3606}
3607
3608/// Write a GlobalValue VST to the module. The purpose of this data structure is
3609/// to allow clients to efficiently find the function body.
3610void ModuleBitcodeWriter::writeGlobalValueSymbolTable(
3611 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3612 // Get the offset of the VST we are writing, and backpatch it into
3613 // the VST forward declaration record.
3614 uint64_t VSTOffset = Stream.GetCurrentBitNo();
3615 // The BitcodeStartBit was the stream offset of the identification block.
3616 VSTOffset -= bitcodeStartBit();
3617 assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned");
3618 // Note that we add 1 here because the offset is relative to one word
3619 // before the start of the identification block, which was historically
3620 // always the start of the regular bitcode header.
3621 Stream.BackpatchWord(BitNo: VSTOffsetPlaceholder, Val: VSTOffset / 32 + 1);
3622
3623 Stream.EnterSubblock(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, CodeLen: 4);
3624
3625 auto Abbv = std::make_shared<BitCodeAbbrev>();
3626 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY));
3627 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
3628 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset
3629 unsigned FnEntryAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
3630
3631 for (const Function &F : M) {
3632 uint64_t Record[2];
3633
3634 if (F.isDeclaration())
3635 continue;
3636
3637 Record[0] = VE.getValueID(V: &F);
3638
3639 // Save the word offset of the function (from the start of the
3640 // actual bitcode written to the stream).
3641 uint64_t BitcodeIndex = FunctionToBitcodeIndex[&F] - bitcodeStartBit();
3642 assert((BitcodeIndex & 31) == 0 && "function block not 32-bit aligned");
3643 // Note that we add 1 here because the offset is relative to one word
3644 // before the start of the identification block, which was historically
3645 // always the start of the regular bitcode header.
3646 Record[1] = BitcodeIndex / 32 + 1;
3647
3648 Stream.EmitRecord(Code: bitc::VST_CODE_FNENTRY, Vals: Record, Abbrev: FnEntryAbbrev);
3649 }
3650
3651 Stream.ExitBlock();
3652}
3653
3654/// Emit names for arguments, instructions and basic blocks in a function.
3655void ModuleBitcodeWriter::writeFunctionLevelValueSymbolTable(
3656 const ValueSymbolTable &VST) {
3657 if (VST.empty())
3658 return;
3659
3660 Stream.EnterSubblock(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, CodeLen: 4);
3661
3662 // FIXME: Set up the abbrev, we know how many values there are!
3663 // FIXME: We know if the type names can use 7-bit ascii.
3664 SmallVector<uint64_t, 64> NameVals;
3665
3666 for (const ValueName &Name : VST) {
3667 // Figure out the encoding to use for the name.
3668 StringEncoding Bits = getStringEncoding(Str: Name.getKey());
3669
3670 unsigned AbbrevToUse = VST_ENTRY_8_ABBREV;
3671 NameVals.push_back(Elt: VE.getValueID(V: Name.getValue()));
3672
3673 // VST_CODE_ENTRY: [valueid, namechar x N]
3674 // VST_CODE_BBENTRY: [bbid, namechar x N]
3675 unsigned Code;
3676 if (isa<BasicBlock>(Val: Name.getValue())) {
3677 Code = bitc::VST_CODE_BBENTRY;
3678 if (Bits == SE_Char6)
3679 AbbrevToUse = VST_BBENTRY_6_ABBREV;
3680 } else {
3681 Code = bitc::VST_CODE_ENTRY;
3682 if (Bits == SE_Char6)
3683 AbbrevToUse = VST_ENTRY_6_ABBREV;
3684 else if (Bits == SE_Fixed7)
3685 AbbrevToUse = VST_ENTRY_7_ABBREV;
3686 }
3687
3688 for (const auto P : Name.getKey())
3689 NameVals.push_back(Elt: (unsigned char)P);
3690
3691 // Emit the finished record.
3692 Stream.EmitRecord(Code, Vals: NameVals, Abbrev: AbbrevToUse);
3693 NameVals.clear();
3694 }
3695
3696 Stream.ExitBlock();
3697}
3698
3699void ModuleBitcodeWriter::writeUseList(UseListOrder &&Order) {
3700 assert(Order.Shuffle.size() >= 2 && "Shuffle too small");
3701 unsigned Code;
3702 if (isa<BasicBlock>(Val: Order.V))
3703 Code = bitc::USELIST_CODE_BB;
3704 else
3705 Code = bitc::USELIST_CODE_DEFAULT;
3706
3707 SmallVector<uint64_t, 64> Record(Order.Shuffle.begin(), Order.Shuffle.end());
3708 Record.push_back(Elt: VE.getValueID(V: Order.V));
3709 Stream.EmitRecord(Code, Vals: Record);
3710}
3711
3712void ModuleBitcodeWriter::writeUseListBlock(const Function *F) {
3713 assert(VE.shouldPreserveUseListOrder() &&
3714 "Expected to be preserving use-list order");
3715
3716 auto hasMore = [&]() {
3717 return !VE.UseListOrders.empty() && VE.UseListOrders.back().F == F;
3718 };
3719 if (!hasMore())
3720 // Nothing to do.
3721 return;
3722
3723 Stream.EnterSubblock(BlockID: bitc::USELIST_BLOCK_ID, CodeLen: 3);
3724 while (hasMore()) {
3725 writeUseList(Order: std::move(VE.UseListOrders.back()));
3726 VE.UseListOrders.pop_back();
3727 }
3728 Stream.ExitBlock();
3729}
3730
3731/// Emit a function body to the module stream.
3732void ModuleBitcodeWriter::writeFunction(
3733 const Function &F,
3734 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3735 // Save the bitcode index of the start of this function block for recording
3736 // in the VST.
3737 FunctionToBitcodeIndex[&F] = Stream.GetCurrentBitNo();
3738
3739 Stream.EnterSubblock(BlockID: bitc::FUNCTION_BLOCK_ID, CodeLen: 5);
3740 VE.incorporateFunction(F);
3741
3742 SmallVector<unsigned, 64> Vals;
3743
3744 // Emit the number of basic blocks, so the reader can create them ahead of
3745 // time.
3746 Vals.push_back(Elt: VE.getBasicBlocks().size());
3747 Stream.EmitRecord(Code: bitc::FUNC_CODE_DECLAREBLOCKS, Vals);
3748 Vals.clear();
3749
3750 // If there are function-local constants, emit them now.
3751 unsigned CstStart, CstEnd;
3752 VE.getFunctionConstantRange(Start&: CstStart, End&: CstEnd);
3753 writeConstants(FirstVal: CstStart, LastVal: CstEnd, isGlobal: false);
3754
3755 // If there is function-local metadata, emit it now.
3756 writeFunctionMetadata(F);
3757
3758 // Keep a running idea of what the instruction ID is.
3759 unsigned InstID = CstEnd;
3760
3761 bool NeedsMetadataAttachment = F.hasMetadata();
3762
3763 DILocation *LastDL = nullptr;
3764 SmallSetVector<Function *, 4> BlockAddressUsers;
3765
3766 // Finally, emit all the instructions, in order.
3767 for (const BasicBlock &BB : F) {
3768 for (const Instruction &I : BB) {
3769 writeInstruction(I, InstID, Vals);
3770
3771 if (!I.getType()->isVoidTy())
3772 ++InstID;
3773
3774 // If the instruction has metadata, write a metadata attachment later.
3775 NeedsMetadataAttachment |= I.hasMetadataOtherThanDebugLoc();
3776
3777 // If the instruction has a debug location, emit it.
3778 if (DILocation *DL = I.getDebugLoc()) {
3779 if (DL == LastDL) {
3780 // Just repeat the same debug loc as last time.
3781 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_LOC_AGAIN, Vals);
3782 } else {
3783 Vals.push_back(Elt: DL->getLine());
3784 Vals.push_back(Elt: DL->getColumn());
3785 Vals.push_back(Elt: VE.getMetadataOrNullID(MD: DL->getScope()));
3786 Vals.push_back(Elt: VE.getMetadataOrNullID(MD: DL->getInlinedAt()));
3787 Vals.push_back(Elt: DL->isImplicitCode());
3788 Vals.push_back(Elt: DL->getAtomGroup());
3789 Vals.push_back(Elt: DL->getAtomRank());
3790 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_LOC, Vals,
3791 Abbrev: FUNCTION_DEBUG_LOC_ABBREV);
3792 Vals.clear();
3793 LastDL = DL;
3794 }
3795 }
3796
3797 // If the instruction has DbgRecords attached to it, emit them. Note that
3798 // they come after the instruction so that it's easy to attach them again
3799 // when reading the bitcode, even though conceptually the debug locations
3800 // start "before" the instruction.
3801 if (I.hasDbgRecords()) {
3802 /// Try to push the value only (unwrapped), otherwise push the
3803 /// metadata wrapped value. Returns true if the value was pushed
3804 /// without the ValueAsMetadata wrapper.
3805 auto PushValueOrMetadata = [&Vals, InstID,
3806 this](Metadata *RawLocation) {
3807 assert(RawLocation &&
3808 "RawLocation unexpectedly null in DbgVariableRecord");
3809 if (ValueAsMetadata *VAM = dyn_cast<ValueAsMetadata>(Val: RawLocation)) {
3810 SmallVector<unsigned, 2> ValAndType;
3811 // If the value is a fwd-ref the type is also pushed. We don't
3812 // want the type, so fwd-refs are kept wrapped (pushValueAndType
3813 // returns false if the value is pushed without type).
3814 if (!pushValueAndType(V: VAM->getValue(), InstID, Vals&: ValAndType)) {
3815 Vals.push_back(Elt: ValAndType[0]);
3816 return true;
3817 }
3818 }
3819 // The metadata is a DIArgList, or ValueAsMetadata wrapping a
3820 // fwd-ref. Push the metadata ID.
3821 Vals.push_back(Elt: VE.getMetadataID(MD: RawLocation));
3822 return false;
3823 };
3824
3825 // Write out non-instruction debug information attached to this
3826 // instruction. Write it after the instruction so that it's easy to
3827 // re-attach to the instruction reading the records in.
3828 for (DbgRecord &DR : I.DebugMarker->getDbgRecordRange()) {
3829 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
3830 Vals.push_back(Elt: VE.getMetadataID(MD: &*DLR->getDebugLoc()));
3831 Vals.push_back(Elt: VE.getMetadataID(MD: DLR->getLabel()));
3832 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_LABEL, Vals);
3833 Vals.clear();
3834 continue;
3835 }
3836
3837 // First 3 fields are common to all kinds:
3838 // DILocation, DILocalVariable, DIExpression
3839 // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE)
3840 // ..., LocationMetadata
3841 // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE - abbrev'd)
3842 // ..., Value
3843 // dbg_declare (FUNC_CODE_DEBUG_RECORD_DECLARE)
3844 // ..., LocationMetadata
3845 // dbg_assign (FUNC_CODE_DEBUG_RECORD_ASSIGN)
3846 // ..., LocationMetadata, DIAssignID, DIExpression, LocationMetadata
3847 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
3848 Vals.push_back(Elt: VE.getMetadataID(MD: &*DVR.getDebugLoc()));
3849 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getVariable()));
3850 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getExpression()));
3851 if (DVR.isDbgValue()) {
3852 if (PushValueOrMetadata(DVR.getRawLocation()))
3853 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE, Vals,
3854 Abbrev: FUNCTION_DEBUG_RECORD_VALUE_ABBREV);
3855 else
3856 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_VALUE, Vals);
3857 } else if (DVR.isDbgDeclare()) {
3858 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3859 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_DECLARE, Vals);
3860 } else if (DVR.isDbgDeclareValue()) {
3861 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3862 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_DECLARE_VALUE, Vals);
3863 } else {
3864 assert(DVR.isDbgAssign() && "Unexpected DbgRecord kind");
3865 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3866 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getAssignID()));
3867 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getAddressExpression()));
3868 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawAddress()));
3869 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_ASSIGN, Vals);
3870 }
3871 Vals.clear();
3872 }
3873 }
3874 }
3875
3876 if (BlockAddress *BA = BlockAddress::lookup(BB: &BB)) {
3877 SmallVector<Value *> Worklist{BA};
3878 SmallPtrSet<Value *, 8> Visited{BA};
3879 while (!Worklist.empty()) {
3880 Value *V = Worklist.pop_back_val();
3881 for (User *U : V->users()) {
3882 if (auto *I = dyn_cast<Instruction>(Val: U)) {
3883 Function *P = I->getFunction();
3884 if (P != &F)
3885 BlockAddressUsers.insert(X: P);
3886 } else if (isa<Constant>(Val: U) && !isa<GlobalValue>(Val: U) &&
3887 Visited.insert(Ptr: U).second)
3888 Worklist.push_back(Elt: U);
3889 }
3890 }
3891 }
3892 }
3893
3894 if (!BlockAddressUsers.empty()) {
3895 Vals.resize(N: BlockAddressUsers.size());
3896 for (auto I : llvm::enumerate(First&: BlockAddressUsers))
3897 Vals[I.index()] = VE.getValueID(V: I.value());
3898 Stream.EmitRecord(Code: bitc::FUNC_CODE_BLOCKADDR_USERS, Vals);
3899 Vals.clear();
3900 }
3901
3902 // Emit names for all the instructions etc.
3903 if (auto *Symtab = F.getValueSymbolTable())
3904 writeFunctionLevelValueSymbolTable(VST: *Symtab);
3905
3906 if (NeedsMetadataAttachment)
3907 writeFunctionMetadataAttachment(F);
3908 if (VE.shouldPreserveUseListOrder())
3909 writeUseListBlock(F: &F);
3910 VE.purgeFunction();
3911 Stream.ExitBlock();
3912}
3913
3914// Emit blockinfo, which defines the standard abbreviations etc.
3915void ModuleBitcodeWriter::writeBlockInfo() {
3916 // We only want to emit block info records for blocks that have multiple
3917 // instances: CONSTANTS_BLOCK, FUNCTION_BLOCK and VALUE_SYMTAB_BLOCK.
3918 // Other blocks can define their abbrevs inline.
3919 Stream.EnterBlockInfoBlock();
3920
3921 // Encode type indices using fixed size based on number of types.
3922 BitCodeAbbrevOp TypeAbbrevOp(BitCodeAbbrevOp::Fixed,
3923 VE.computeBitsRequiredForTypeIndices());
3924 // Encode value indices as 6-bit VBR.
3925 BitCodeAbbrevOp ValAbbrevOp(BitCodeAbbrevOp::VBR, 6);
3926
3927 { // 8-bit fixed-width VST_CODE_ENTRY/VST_CODE_BBENTRY strings.
3928 auto Abbv = std::make_shared<BitCodeAbbrev>();
3929 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3));
3930 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3931 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3932 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
3933 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3934 VST_ENTRY_8_ABBREV)
3935 llvm_unreachable("Unexpected abbrev ordering!");
3936 }
3937
3938 { // 7-bit fixed width VST_CODE_ENTRY strings.
3939 auto Abbv = std::make_shared<BitCodeAbbrev>();
3940 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
3941 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3942 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3943 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
3944 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3945 VST_ENTRY_7_ABBREV)
3946 llvm_unreachable("Unexpected abbrev ordering!");
3947 }
3948 { // 6-bit char6 VST_CODE_ENTRY strings.
3949 auto Abbv = std::make_shared<BitCodeAbbrev>();
3950 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
3951 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3952 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3953 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
3954 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3955 VST_ENTRY_6_ABBREV)
3956 llvm_unreachable("Unexpected abbrev ordering!");
3957 }
3958 { // 6-bit char6 VST_CODE_BBENTRY strings.
3959 auto Abbv = std::make_shared<BitCodeAbbrev>();
3960 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_BBENTRY));
3961 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3962 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3963 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
3964 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3965 VST_BBENTRY_6_ABBREV)
3966 llvm_unreachable("Unexpected abbrev ordering!");
3967 }
3968
3969 { // SETTYPE abbrev for CONSTANTS_BLOCK.
3970 auto Abbv = std::make_shared<BitCodeAbbrev>();
3971 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_SETTYPE));
3972 Abbv->Add(OpInfo: TypeAbbrevOp);
3973 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3974 CONSTANTS_SETTYPE_ABBREV)
3975 llvm_unreachable("Unexpected abbrev ordering!");
3976 }
3977
3978 { // INTEGER abbrev for CONSTANTS_BLOCK.
3979 auto Abbv = std::make_shared<BitCodeAbbrev>();
3980 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_INTEGER));
3981 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3982 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3983 CONSTANTS_INTEGER_ABBREV)
3984 llvm_unreachable("Unexpected abbrev ordering!");
3985 }
3986
3987 { // BYTE abbrev for CONSTANTS_BLOCK.
3988 auto Abbv = std::make_shared<BitCodeAbbrev>();
3989 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_BYTE));
3990 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3991 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3992 CONSTANTS_BYTE_ABBREV)
3993 llvm_unreachable("Unexpected abbrev ordering!");
3994 }
3995
3996 { // CE_CAST abbrev for CONSTANTS_BLOCK.
3997 auto Abbv = std::make_shared<BitCodeAbbrev>();
3998 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CE_CAST));
3999 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // cast opc
4000 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // typeid
4001 VE.computeBitsRequiredForTypeIndices()));
4002 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
4003
4004 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4005 CONSTANTS_CE_CAST_Abbrev)
4006 llvm_unreachable("Unexpected abbrev ordering!");
4007 }
4008 { // NULL abbrev for CONSTANTS_BLOCK.
4009 auto Abbv = std::make_shared<BitCodeAbbrev>();
4010 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_NULL));
4011 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4012 CONSTANTS_NULL_Abbrev)
4013 llvm_unreachable("Unexpected abbrev ordering!");
4014 }
4015
4016 // FIXME: This should only use space for first class types!
4017
4018 { // INST_LOAD abbrev for FUNCTION_BLOCK.
4019 auto Abbv = std::make_shared<BitCodeAbbrev>();
4020 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_LOAD));
4021 Abbv->Add(OpInfo: ValAbbrevOp); // Ptr
4022 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4023 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Align
4024 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
4025 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4026 FUNCTION_INST_LOAD_ABBREV)
4027 llvm_unreachable("Unexpected abbrev ordering!");
4028 }
4029 {
4030 auto Abbv = std::make_shared<BitCodeAbbrev>();
4031 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_STORE));
4032 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4033 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4034 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // align
4035 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
4036 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4037 FUNCTION_INST_STORE_ABBREV)
4038 llvm_unreachable("Unexpected abbrev ordering!");
4039 }
4040 { // INST_UNOP abbrev for FUNCTION_BLOCK.
4041 auto Abbv = std::make_shared<BitCodeAbbrev>();
4042 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
4043 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4044 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4045 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4046 FUNCTION_INST_UNOP_ABBREV)
4047 llvm_unreachable("Unexpected abbrev ordering!");
4048 }
4049 { // INST_UNOP_FLAGS abbrev for FUNCTION_BLOCK.
4050 auto Abbv = std::make_shared<BitCodeAbbrev>();
4051 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
4052 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4053 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4054 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4055 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4056 FUNCTION_INST_UNOP_FLAGS_ABBREV)
4057 llvm_unreachable("Unexpected abbrev ordering!");
4058 }
4059 { // INST_BINOP abbrev for FUNCTION_BLOCK.
4060 auto Abbv = std::make_shared<BitCodeAbbrev>();
4061 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
4062 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4063 Abbv->Add(OpInfo: ValAbbrevOp); // RHS
4064 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4065 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4066 FUNCTION_INST_BINOP_ABBREV)
4067 llvm_unreachable("Unexpected abbrev ordering!");
4068 }
4069 { // INST_BINOP_FLAGS abbrev for FUNCTION_BLOCK.
4070 auto Abbv = std::make_shared<BitCodeAbbrev>();
4071 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
4072 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4073 Abbv->Add(OpInfo: ValAbbrevOp); // RHS
4074 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4075 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4076 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4077 FUNCTION_INST_BINOP_FLAGS_ABBREV)
4078 llvm_unreachable("Unexpected abbrev ordering!");
4079 }
4080 { // INST_CAST abbrev for FUNCTION_BLOCK.
4081 auto Abbv = std::make_shared<BitCodeAbbrev>();
4082 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
4083 Abbv->Add(OpInfo: ValAbbrevOp); // OpVal
4084 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4085 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4086 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4087 FUNCTION_INST_CAST_ABBREV)
4088 llvm_unreachable("Unexpected abbrev ordering!");
4089 }
4090 { // INST_CAST_FLAGS abbrev for FUNCTION_BLOCK.
4091 auto Abbv = std::make_shared<BitCodeAbbrev>();
4092 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
4093 Abbv->Add(OpInfo: ValAbbrevOp); // OpVal
4094 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4095 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4096 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4097 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4098 FUNCTION_INST_CAST_FLAGS_ABBREV)
4099 llvm_unreachable("Unexpected abbrev ordering!");
4100 }
4101
4102 { // INST_RET abbrev for FUNCTION_BLOCK.
4103 auto Abbv = std::make_shared<BitCodeAbbrev>();
4104 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
4105 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4106 FUNCTION_INST_RET_VOID_ABBREV)
4107 llvm_unreachable("Unexpected abbrev ordering!");
4108 }
4109 { // INST_RET abbrev for FUNCTION_BLOCK.
4110 auto Abbv = std::make_shared<BitCodeAbbrev>();
4111 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
4112 Abbv->Add(OpInfo: ValAbbrevOp);
4113 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4114 FUNCTION_INST_RET_VAL_ABBREV)
4115 llvm_unreachable("Unexpected abbrev ordering!");
4116 }
4117 {
4118 auto Abbv = std::make_shared<BitCodeAbbrev>();
4119 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BR));
4120 // TODO: Use different abbrev for absolute value reference (succ0)?
4121 Abbv->Add(OpInfo: ValAbbrevOp); // succ0
4122 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4123 FUNCTION_INST_BR_UNCOND_ABBREV)
4124 llvm_unreachable("Unexpected abbrev ordering!");
4125 }
4126 {
4127 auto Abbv = std::make_shared<BitCodeAbbrev>();
4128 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BR));
4129 // TODO: Use different abbrev for absolute value references (succ0, succ1)?
4130 Abbv->Add(OpInfo: ValAbbrevOp); // succ0
4131 Abbv->Add(OpInfo: ValAbbrevOp); // succ1
4132 Abbv->Add(OpInfo: ValAbbrevOp); // cond
4133 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4134 FUNCTION_INST_BR_COND_ABBREV)
4135 llvm_unreachable("Unexpected abbrev ordering!");
4136 }
4137 { // INST_UNREACHABLE abbrev for FUNCTION_BLOCK.
4138 auto Abbv = std::make_shared<BitCodeAbbrev>();
4139 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNREACHABLE));
4140 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4141 FUNCTION_INST_UNREACHABLE_ABBREV)
4142 llvm_unreachable("Unexpected abbrev ordering!");
4143 }
4144 {
4145 auto Abbv = std::make_shared<BitCodeAbbrev>();
4146 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_GEP));
4147 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // flags
4148 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4149 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4150 Abbv->Add(OpInfo: ValAbbrevOp);
4151 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4152 FUNCTION_INST_GEP_ABBREV)
4153 llvm_unreachable("Unexpected abbrev ordering!");
4154 }
4155 {
4156 auto Abbv = std::make_shared<BitCodeAbbrev>();
4157 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CMP2));
4158 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4159 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4160 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // pred
4161 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4162 FUNCTION_INST_CMP_ABBREV)
4163 llvm_unreachable("Unexpected abbrev ordering!");
4164 }
4165 {
4166 auto Abbv = std::make_shared<BitCodeAbbrev>();
4167 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CMP2));
4168 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4169 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4170 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // pred
4171 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4172 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4173 FUNCTION_INST_CMP_FLAGS_ABBREV)
4174 llvm_unreachable("Unexpected abbrev ordering!");
4175 }
4176 {
4177 auto Abbv = std::make_shared<BitCodeAbbrev>();
4178 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE));
4179 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // dbgloc
4180 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // var
4181 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // expr
4182 Abbv->Add(OpInfo: ValAbbrevOp); // val
4183 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4184 FUNCTION_DEBUG_RECORD_VALUE_ABBREV)
4185 llvm_unreachable("Unexpected abbrev ordering! 1");
4186 }
4187 {
4188 auto Abbv = std::make_shared<BitCodeAbbrev>();
4189 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_LOC));
4190 // NOTE: No IsDistinct field for FUNC_CODE_DEBUG_LOC.
4191 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4192 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4193 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4194 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4195 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
4196 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Atom group.
4197 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 3)); // Atom rank.
4198 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4199 FUNCTION_DEBUG_LOC_ABBREV)
4200 llvm_unreachable("Unexpected abbrev ordering!");
4201 }
4202 Stream.ExitBlock();
4203}
4204
4205/// Write the module path strings, currently only used when generating
4206/// a combined index file.
4207void IndexBitcodeWriter::writeModStrings() {
4208 Stream.EnterSubblock(BlockID: bitc::MODULE_STRTAB_BLOCK_ID, CodeLen: 3);
4209
4210 // TODO: See which abbrev sizes we actually need to emit
4211
4212 // 8-bit fixed-width MST_ENTRY strings.
4213 auto Abbv = std::make_shared<BitCodeAbbrev>();
4214 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4215 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4216 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4217 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
4218 unsigned Abbrev8Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4219
4220 // 7-bit fixed width MST_ENTRY strings.
4221 Abbv = std::make_shared<BitCodeAbbrev>();
4222 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4223 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4224 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4225 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
4226 unsigned Abbrev7Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4227
4228 // 6-bit char6 MST_ENTRY strings.
4229 Abbv = std::make_shared<BitCodeAbbrev>();
4230 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4231 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4232 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4233 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
4234 unsigned Abbrev6Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4235
4236 // Module Hash, 160 bits SHA1. Optionally, emitted after each MST_CODE_ENTRY.
4237 Abbv = std::make_shared<BitCodeAbbrev>();
4238 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_HASH));
4239 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4240 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4241 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4242 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4243 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4244 unsigned AbbrevHash = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4245
4246 SmallVector<unsigned, 64> Vals;
4247 forEachModule(Callback: [&](const StringMapEntry<ModuleHash> &MPSE) {
4248 StringRef Key = MPSE.getKey();
4249 const auto &Hash = MPSE.getValue();
4250 StringEncoding Bits = getStringEncoding(Str: Key);
4251 unsigned AbbrevToUse = Abbrev8Bit;
4252 if (Bits == SE_Char6)
4253 AbbrevToUse = Abbrev6Bit;
4254 else if (Bits == SE_Fixed7)
4255 AbbrevToUse = Abbrev7Bit;
4256
4257 auto ModuleId = ModuleIdMap.size();
4258 ModuleIdMap[Key] = ModuleId;
4259 Vals.push_back(Elt: ModuleId);
4260 Vals.append(in_start: Key.begin(), in_end: Key.end());
4261
4262 // Emit the finished record.
4263 Stream.EmitRecord(Code: bitc::MST_CODE_ENTRY, Vals, Abbrev: AbbrevToUse);
4264
4265 // Emit an optional hash for the module now
4266 if (llvm::any_of(Range: Hash, P: [](uint32_t H) { return H; })) {
4267 Vals.assign(in_start: Hash.begin(), in_end: Hash.end());
4268 // Emit the hash record.
4269 Stream.EmitRecord(Code: bitc::MST_CODE_HASH, Vals, Abbrev: AbbrevHash);
4270 }
4271
4272 Vals.clear();
4273 });
4274 Stream.ExitBlock();
4275}
4276
4277/// Write the function type metadata related records that need to appear before
4278/// a function summary entry (whether per-module or combined).
4279template <typename Fn>
4280static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream,
4281 FunctionSummary *FS,
4282 Fn GetValueID) {
4283 if (!FS->type_tests().empty())
4284 Stream.EmitRecord(Code: bitc::FS_TYPE_TESTS, Vals: FS->type_tests());
4285
4286 SmallVector<uint64_t, 64> Record;
4287
4288 auto WriteVFuncIdVec = [&](uint64_t Ty,
4289 ArrayRef<FunctionSummary::VFuncId> VFs) {
4290 if (VFs.empty())
4291 return;
4292 Record.clear();
4293 for (auto &VF : VFs) {
4294 Record.push_back(Elt: VF.GUID);
4295 Record.push_back(Elt: VF.Offset);
4296 }
4297 Stream.EmitRecord(Code: Ty, Vals: Record);
4298 };
4299
4300 WriteVFuncIdVec(bitc::FS_TYPE_TEST_ASSUME_VCALLS,
4301 FS->type_test_assume_vcalls());
4302 WriteVFuncIdVec(bitc::FS_TYPE_CHECKED_LOAD_VCALLS,
4303 FS->type_checked_load_vcalls());
4304
4305 auto WriteConstVCallVec = [&](uint64_t Ty,
4306 ArrayRef<FunctionSummary::ConstVCall> VCs) {
4307 for (auto &VC : VCs) {
4308 Record.clear();
4309 Record.push_back(Elt: VC.VFunc.GUID);
4310 Record.push_back(Elt: VC.VFunc.Offset);
4311 llvm::append_range(C&: Record, R: VC.Args);
4312 Stream.EmitRecord(Code: Ty, Vals: Record);
4313 }
4314 };
4315
4316 WriteConstVCallVec(bitc::FS_TYPE_TEST_ASSUME_CONST_VCALL,
4317 FS->type_test_assume_const_vcalls());
4318 WriteConstVCallVec(bitc::FS_TYPE_CHECKED_LOAD_CONST_VCALL,
4319 FS->type_checked_load_const_vcalls());
4320
4321 auto WriteRange = [&](ConstantRange Range) {
4322 Range = Range.sextOrTrunc(BitWidth: FunctionSummary::ParamAccess::RangeWidth);
4323 assert(Range.getLower().getNumWords() == 1);
4324 assert(Range.getUpper().getNumWords() == 1);
4325 emitSignedInt64(Vals&: Record, V: *Range.getLower().getRawData());
4326 emitSignedInt64(Vals&: Record, V: *Range.getUpper().getRawData());
4327 };
4328
4329 if (!FS->paramAccesses().empty()) {
4330 Record.clear();
4331 for (auto &Arg : FS->paramAccesses()) {
4332 size_t UndoSize = Record.size();
4333 Record.push_back(Elt: Arg.ParamNo);
4334 WriteRange(Arg.Use);
4335 Record.push_back(Elt: Arg.Calls.size());
4336 for (auto &Call : Arg.Calls) {
4337 Record.push_back(Elt: Call.ParamNo);
4338 std::optional<unsigned> ValueID = GetValueID(Call.Callee);
4339 if (!ValueID) {
4340 // If ValueID is unknown we can't drop just this call, we must drop
4341 // entire parameter.
4342 Record.resize(N: UndoSize);
4343 break;
4344 }
4345 Record.push_back(Elt: *ValueID);
4346 WriteRange(Call.Offsets);
4347 }
4348 }
4349 if (!Record.empty())
4350 Stream.EmitRecord(Code: bitc::FS_PARAM_ACCESS, Vals: Record);
4351 }
4352}
4353
4354/// Collect type IDs from type tests used by function.
4355static void
4356getReferencedTypeIds(FunctionSummary *FS,
4357 std::set<GlobalValue::GUID> &ReferencedTypeIds) {
4358 if (!FS->type_tests().empty())
4359 for (auto &TT : FS->type_tests())
4360 ReferencedTypeIds.insert(x: TT);
4361
4362 auto GetReferencedTypesFromVFuncIdVec =
4363 [&](ArrayRef<FunctionSummary::VFuncId> VFs) {
4364 for (auto &VF : VFs)
4365 ReferencedTypeIds.insert(x: VF.GUID);
4366 };
4367
4368 GetReferencedTypesFromVFuncIdVec(FS->type_test_assume_vcalls());
4369 GetReferencedTypesFromVFuncIdVec(FS->type_checked_load_vcalls());
4370
4371 auto GetReferencedTypesFromConstVCallVec =
4372 [&](ArrayRef<FunctionSummary::ConstVCall> VCs) {
4373 for (auto &VC : VCs)
4374 ReferencedTypeIds.insert(x: VC.VFunc.GUID);
4375 };
4376
4377 GetReferencedTypesFromConstVCallVec(FS->type_test_assume_const_vcalls());
4378 GetReferencedTypesFromConstVCallVec(FS->type_checked_load_const_vcalls());
4379}
4380
4381static void writeWholeProgramDevirtResolutionByArg(
4382 SmallVector<uint64_t, 64> &NameVals, const std::vector<uint64_t> &args,
4383 const WholeProgramDevirtResolution::ByArg &ByArg) {
4384 NameVals.push_back(Elt: args.size());
4385 llvm::append_range(C&: NameVals, R: args);
4386
4387 NameVals.push_back(Elt: ByArg.TheKind);
4388 NameVals.push_back(Elt: ByArg.Info);
4389 NameVals.push_back(Elt: ByArg.Byte);
4390 NameVals.push_back(Elt: ByArg.Bit);
4391}
4392
4393static void writeWholeProgramDevirtResolution(
4394 SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
4395 uint64_t Id, const WholeProgramDevirtResolution &Wpd) {
4396 NameVals.push_back(Elt: Id);
4397
4398 NameVals.push_back(Elt: Wpd.TheKind);
4399 NameVals.push_back(Elt: StrtabBuilder.add(S: Wpd.SingleImplName));
4400 NameVals.push_back(Elt: Wpd.SingleImplName.size());
4401
4402 NameVals.push_back(Elt: Wpd.ResByArg.size());
4403 for (auto &A : Wpd.ResByArg)
4404 writeWholeProgramDevirtResolutionByArg(NameVals, args: A.first, ByArg: A.second);
4405}
4406
4407static void writeTypeIdSummaryRecord(SmallVector<uint64_t, 64> &NameVals,
4408 StringTableBuilder &StrtabBuilder,
4409 StringRef Id,
4410 const TypeIdSummary &Summary) {
4411 NameVals.push_back(Elt: StrtabBuilder.add(S: Id));
4412 NameVals.push_back(Elt: Id.size());
4413
4414 NameVals.push_back(Elt: Summary.TTRes.TheKind);
4415 NameVals.push_back(Elt: Summary.TTRes.SizeM1BitWidth);
4416 NameVals.push_back(Elt: Summary.TTRes.AlignLog2);
4417 NameVals.push_back(Elt: Summary.TTRes.SizeM1);
4418 NameVals.push_back(Elt: Summary.TTRes.BitMask);
4419 NameVals.push_back(Elt: Summary.TTRes.InlineBits);
4420
4421 for (auto &W : Summary.WPDRes)
4422 writeWholeProgramDevirtResolution(NameVals, StrtabBuilder, Id: W.first,
4423 Wpd: W.second);
4424}
4425
4426static void writeTypeIdCompatibleVtableSummaryRecord(
4427 SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
4428 StringRef Id, const TypeIdCompatibleVtableInfo &Summary,
4429 ValueEnumerator &VE) {
4430 NameVals.push_back(Elt: StrtabBuilder.add(S: Id));
4431 NameVals.push_back(Elt: Id.size());
4432
4433 for (auto &P : Summary) {
4434 NameVals.push_back(Elt: P.AddressPointOffset);
4435 NameVals.push_back(Elt: VE.getValueID(V: P.VTableVI.getValue()));
4436 }
4437}
4438
4439// Adds the allocation contexts to the CallStacks map. We simply use the
4440// size at the time the context was added as the CallStackId. This works because
4441// when we look up the call stacks later on we process the function summaries
4442// and their allocation records in the same exact order.
4443static void collectMemProfCallStacks(
4444 FunctionSummary *FS, std::function<LinearFrameId(unsigned)> GetStackIndex,
4445 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> &CallStacks) {
4446 // The interfaces in ProfileData/MemProf.h use a type alias for a stack frame
4447 // id offset into the index of the full stack frames. The ModuleSummaryIndex
4448 // currently uses unsigned. Make sure these stay in sync.
4449 static_assert(std::is_same_v<LinearFrameId, unsigned>);
4450 for (auto &AI : FS->allocs()) {
4451 for (auto &MIB : AI.MIBs) {
4452 SmallVector<unsigned> StackIdIndices;
4453 StackIdIndices.reserve(N: MIB.StackIdIndices.size());
4454 for (auto Id : MIB.StackIdIndices)
4455 StackIdIndices.push_back(Elt: GetStackIndex(Id));
4456 // The CallStackId is the size at the time this context was inserted.
4457 CallStacks.insert(KV: {CallStacks.size(), StackIdIndices});
4458 }
4459 }
4460}
4461
4462// Build the radix tree from the accumulated CallStacks, write out the resulting
4463// linearized radix tree array, and return the map of call stack positions into
4464// this array for use when writing the allocation records. The returned map is
4465// indexed by a CallStackId which in this case is implicitly determined by the
4466// order of function summaries and their allocation infos being written.
4467static DenseMap<CallStackId, LinearCallStackId> writeMemoryProfileRadixTree(
4468 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> &&CallStacks,
4469 BitstreamWriter &Stream, unsigned RadixAbbrev) {
4470 assert(!CallStacks.empty());
4471 DenseMap<unsigned, FrameStat> FrameHistogram =
4472 computeFrameHistogram<LinearFrameId>(MemProfCallStackData&: CallStacks);
4473 CallStackRadixTreeBuilder<LinearFrameId> Builder;
4474 // We don't need a MemProfFrameIndexes map as we have already converted the
4475 // full stack id hash to a linear offset into the StackIds array.
4476 Builder.build(MemProfCallStackData: std::move(CallStacks), /*MemProfFrameIndexes=*/nullptr,
4477 FrameHistogram);
4478 Stream.EmitRecord(Code: bitc::FS_CONTEXT_RADIX_TREE_ARRAY, Vals: Builder.getRadixArray(),
4479 Abbrev: RadixAbbrev);
4480 return Builder.takeCallStackPos();
4481}
4482
4483static void writeFunctionHeapProfileRecords(
4484 BitstreamWriter &Stream, FunctionSummary *FS, unsigned CallsiteAbbrev,
4485 unsigned AllocAbbrev, unsigned ContextIdAbbvId, bool PerModule,
4486 std::function<unsigned(const ValueInfo &VI)> GetValueID,
4487 std::function<unsigned(unsigned)> GetStackIndex,
4488 bool WriteContextSizeInfoIndex,
4489 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
4490 CallStackId &CallStackCount) {
4491 SmallVector<uint64_t> Record;
4492
4493 for (auto &CI : FS->callsites()) {
4494 Record.clear();
4495 // Per module callsite clones should always have a single entry of
4496 // value 0.
4497 assert(!PerModule || (CI.Clones.size() == 1 && CI.Clones[0] == 0));
4498 Record.push_back(Elt: GetValueID(CI.Callee));
4499 if (!PerModule) {
4500 Record.push_back(Elt: CI.StackIdIndices.size());
4501 Record.push_back(Elt: CI.Clones.size());
4502 }
4503 for (auto Id : CI.StackIdIndices)
4504 Record.push_back(Elt: GetStackIndex(Id));
4505 if (!PerModule)
4506 llvm::append_range(C&: Record, R: CI.Clones);
4507 Stream.EmitRecord(Code: PerModule ? bitc::FS_PERMODULE_CALLSITE_INFO
4508 : bitc::FS_COMBINED_CALLSITE_INFO,
4509 Vals: Record, Abbrev: CallsiteAbbrev);
4510 }
4511
4512 for (auto &AI : FS->allocs()) {
4513 Record.clear();
4514 // Per module alloc versions should always have a single entry of
4515 // value 0.
4516 assert(!PerModule || (AI.Versions.size() == 1 && AI.Versions[0] == 0));
4517 Record.push_back(Elt: AI.MIBs.size());
4518 if (!PerModule)
4519 Record.push_back(Elt: AI.Versions.size());
4520 for (auto &MIB : AI.MIBs) {
4521 Record.push_back(Elt: (uint8_t)MIB.AllocType);
4522 // The per-module summary always needs to include the alloc context, as we
4523 // use it during the thin link. For the combined index it is optional (see
4524 // comments where CombinedIndexMemProfContext is defined).
4525 if (PerModule || CombinedIndexMemProfContext) {
4526 // Record the index into the radix tree array for this context.
4527 assert(CallStackCount <= CallStackPos.size());
4528 Record.push_back(Elt: CallStackPos[CallStackCount++]);
4529 }
4530 }
4531 if (!PerModule)
4532 llvm::append_range(C&: Record, R: AI.Versions);
4533 assert(AI.ContextSizeInfos.empty() ||
4534 AI.ContextSizeInfos.size() == AI.MIBs.size());
4535 // Optionally emit the context size information if it exists.
4536 if (WriteContextSizeInfoIndex && !AI.ContextSizeInfos.empty()) {
4537 // The abbreviation id for the context ids record should have been created
4538 // if we are emitting the per-module index, which is where we write this
4539 // info.
4540 assert(ContextIdAbbvId);
4541 SmallVector<uint32_t> ContextIds;
4542 // At least one context id per ContextSizeInfos entry (MIB), broken into 2
4543 // halves.
4544 ContextIds.reserve(N: AI.ContextSizeInfos.size() * 2);
4545 for (auto &Infos : AI.ContextSizeInfos) {
4546 Record.push_back(Elt: Infos.size());
4547 for (auto [FullStackId, TotalSize] : Infos) {
4548 // The context ids are emitted separately as a fixed width array,
4549 // which is more efficient than a VBR given that these hashes are
4550 // typically close to 64-bits. The max fixed width entry is 32 bits so
4551 // it is split into 2.
4552 ContextIds.push_back(Elt: static_cast<uint32_t>(FullStackId >> 32));
4553 ContextIds.push_back(Elt: static_cast<uint32_t>(FullStackId));
4554 Record.push_back(Elt: TotalSize);
4555 }
4556 }
4557 // The context ids are expected by the reader to immediately precede the
4558 // associated alloc info record.
4559 Stream.EmitRecord(Code: bitc::FS_ALLOC_CONTEXT_IDS, Vals: ContextIds,
4560 Abbrev: ContextIdAbbvId);
4561 }
4562 Stream.EmitRecord(Code: PerModule
4563 ? bitc::FS_PERMODULE_ALLOC_INFO
4564 : (CombinedIndexMemProfContext
4565 ? bitc::FS_COMBINED_ALLOC_INFO
4566 : bitc::FS_COMBINED_ALLOC_INFO_NO_CONTEXT),
4567 Vals: Record, Abbrev: AllocAbbrev);
4568 }
4569}
4570
4571// Helper to emit a single function summary record.
4572void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
4573 SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
4574 unsigned ValueID, unsigned FSCallsProfileAbbrev, unsigned CallsiteAbbrev,
4575 unsigned AllocAbbrev, unsigned ContextIdAbbvId, const Function &F,
4576 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
4577 CallStackId &CallStackCount) {
4578 NameVals.push_back(Elt: ValueID);
4579
4580 FunctionSummary *FS = cast<FunctionSummary>(Val: Summary);
4581
4582 writeFunctionTypeMetadataRecords(
4583 Stream, FS, GetValueID: [&](const ValueInfo &VI) -> std::optional<unsigned> {
4584 return {VE.getValueID(V: VI.getValue())};
4585 });
4586
4587 auto SpecialRefCnts = FS->specialRefCounts();
4588 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: FS->flags()));
4589 NameVals.push_back(Elt: FS->instCount());
4590 NameVals.push_back(Elt: getEncodedFFlags(Flags: FS->fflags()));
4591 NameVals.push_back(Elt: FS->refs().size());
4592 NameVals.push_back(Elt: SpecialRefCnts.first); // rorefcnt
4593 NameVals.push_back(Elt: SpecialRefCnts.second); // worefcnt
4594
4595 for (auto &RI : FS->refs())
4596 NameVals.push_back(Elt: getValueId(VI: RI));
4597
4598 for (auto &ECI : FS->calls()) {
4599 NameVals.push_back(Elt: getValueId(VI: ECI.first));
4600 NameVals.push_back(Elt: getEncodedHotnessCallEdgeInfo(CI: ECI.second));
4601 }
4602
4603 // Emit the finished record.
4604 Stream.EmitRecord(Code: bitc::FS_PERMODULE_PROFILE, Vals: NameVals, Abbrev: FSCallsProfileAbbrev);
4605 NameVals.clear();
4606
4607 writeFunctionHeapProfileRecords(
4608 Stream, FS, CallsiteAbbrev, AllocAbbrev, ContextIdAbbvId,
4609 /*PerModule*/ true,
4610 /*GetValueId*/ GetValueID: [&](const ValueInfo &VI) { return getValueId(VI); },
4611 /*GetStackIndex*/ [&](unsigned I) { return I; },
4612 /*WriteContextSizeInfoIndex*/ true, CallStackPos, CallStackCount);
4613}
4614
4615// Collect the global value references in the given variable's initializer,
4616// and emit them in a summary record.
4617void ModuleBitcodeWriterBase::writeModuleLevelReferences(
4618 const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals,
4619 unsigned FSModRefsAbbrev, unsigned FSModVTableRefsAbbrev) {
4620 auto VI = Index->getValueInfo(GUID: V.getGUID());
4621 if (!VI || VI.getSummaryList().empty()) {
4622 // Only declarations should not have a summary (a declaration might however
4623 // have a summary if the def was in module level asm).
4624 assert(V.isDeclaration());
4625 return;
4626 }
4627 auto *Summary = VI.getSummaryList()[0].get();
4628 NameVals.push_back(Elt: VE.getValueID(V: &V));
4629 GlobalVarSummary *VS = cast<GlobalVarSummary>(Val: Summary);
4630 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: VS->flags()));
4631 NameVals.push_back(Elt: getEncodedGVarFlags(Flags: VS->varflags()));
4632
4633 auto VTableFuncs = VS->vTableFuncs();
4634 if (!VTableFuncs.empty())
4635 NameVals.push_back(Elt: VS->refs().size());
4636
4637 unsigned SizeBeforeRefs = NameVals.size();
4638 for (auto &RI : VS->refs())
4639 NameVals.push_back(Elt: VE.getValueID(V: RI.getValue()));
4640 // Sort the refs for determinism output, the vector returned by FS->refs() has
4641 // been initialized from a DenseSet.
4642 llvm::sort(C: drop_begin(RangeOrContainer&: NameVals, N: SizeBeforeRefs));
4643
4644 if (VTableFuncs.empty())
4645 Stream.EmitRecord(Code: bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, Vals: NameVals,
4646 Abbrev: FSModRefsAbbrev);
4647 else {
4648 // VTableFuncs pairs should already be sorted by offset.
4649 for (auto &P : VTableFuncs) {
4650 NameVals.push_back(Elt: VE.getValueID(V: P.FuncVI.getValue()));
4651 NameVals.push_back(Elt: P.VTableOffset);
4652 }
4653
4654 Stream.EmitRecord(Code: bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS, Vals: NameVals,
4655 Abbrev: FSModVTableRefsAbbrev);
4656 }
4657 NameVals.clear();
4658}
4659
4660/// Emit the per-module summary section alongside the rest of
4661/// the module's bitcode.
4662void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
4663 // By default we compile with ThinLTO if the module has a summary, but the
4664 // client can request full LTO with a module flag.
4665 bool IsThinLTO = true;
4666 if (auto *MD =
4667 mdconst::extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: "ThinLTO")))
4668 IsThinLTO = MD->getZExtValue();
4669 Stream.EnterSubblock(BlockID: IsThinLTO ? bitc::GLOBALVAL_SUMMARY_BLOCK_ID
4670 : bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID,
4671 CodeLen: 4);
4672
4673 Stream.EmitRecord(
4674 Code: bitc::FS_VERSION,
4675 Vals: ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
4676
4677 // Write the index flags.
4678 uint64_t Flags = 0;
4679 // Bits 1-3 are set only in the combined index, skip them.
4680 if (Index->enableSplitLTOUnit())
4681 Flags |= 0x8;
4682 if (Index->hasUnifiedLTO())
4683 Flags |= 0x200;
4684
4685 Stream.EmitRecord(Code: bitc::FS_FLAGS, Vals: ArrayRef<uint64_t>{Flags});
4686
4687 if (Index->begin() == Index->end()) {
4688 Stream.ExitBlock();
4689 return;
4690 }
4691
4692 auto Abbv = std::make_shared<BitCodeAbbrev>();
4693 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_VALUE_GUID));
4694 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4695 // GUIDS often use up most of 64-bits, so encode as two Fixed 32.
4696 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4697 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4698 unsigned ValueGuidAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4699
4700 for (const auto &GVI : valueIds()) {
4701 Stream.EmitRecord(Code: bitc::FS_VALUE_GUID,
4702 Vals: ArrayRef<uint32_t>{GVI.second,
4703 static_cast<uint32_t>(GVI.first >> 32),
4704 static_cast<uint32_t>(GVI.first)},
4705 Abbrev: ValueGuidAbbrev);
4706 }
4707
4708 if (!Index->stackIds().empty()) {
4709 auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4710 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_STACK_IDS));
4711 // numids x stackid
4712 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4713 // The stack ids are hashes that are close to 64 bits in size, so emitting
4714 // as a pair of 32-bit fixed-width values is more efficient than a VBR.
4715 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4716 unsigned StackIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(StackIdAbbv));
4717 SmallVector<uint32_t> Vals;
4718 Vals.reserve(N: Index->stackIds().size() * 2);
4719 for (auto Id : Index->stackIds()) {
4720 Vals.push_back(Elt: static_cast<uint32_t>(Id >> 32));
4721 Vals.push_back(Elt: static_cast<uint32_t>(Id));
4722 }
4723 Stream.EmitRecord(Code: bitc::FS_STACK_IDS, Vals, Abbrev: StackIdAbbvId);
4724 }
4725
4726 unsigned ContextIdAbbvId = 0;
4727 if (metadataMayIncludeContextSizeInfo()) {
4728 // n x context id
4729 auto ContextIdAbbv = std::make_shared<BitCodeAbbrev>();
4730 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_ALLOC_CONTEXT_IDS));
4731 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4732 // The context ids are hashes that are close to 64 bits in size, so emitting
4733 // as a pair of 32-bit fixed-width values is more efficient than a VBR if we
4734 // are emitting them for all MIBs. Otherwise we use VBR to better compress 0
4735 // values that are expected to more frequently occur in an alloc's memprof
4736 // summary.
4737 if (metadataIncludesAllContextSizeInfo())
4738 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4739 else
4740 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4741 ContextIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(ContextIdAbbv));
4742 }
4743
4744 // Abbrev for FS_PERMODULE_PROFILE.
4745 Abbv = std::make_shared<BitCodeAbbrev>();
4746 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE));
4747 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4748 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // flags
4749 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
4750 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
4751 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4752 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
4753 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
4754 // numrefs x valueid, n x (valueid, hotness+tailcall flags)
4755 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4756 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4757 unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4758
4759 // Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS.
4760 Abbv = std::make_shared<BitCodeAbbrev>();
4761 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS));
4762 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4763 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4764 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
4765 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4766 unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4767
4768 // Abbrev for FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS.
4769 Abbv = std::make_shared<BitCodeAbbrev>();
4770 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS));
4771 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4772 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4773 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4774 // numrefs x valueid, n x (valueid , offset)
4775 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4776 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4777 unsigned FSModVTableRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4778
4779 // Abbrev for FS_ALIAS.
4780 Abbv = std::make_shared<BitCodeAbbrev>();
4781 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_ALIAS));
4782 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4783 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4784 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4785 unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4786
4787 // Abbrev for FS_TYPE_ID_METADATA
4788 Abbv = std::make_shared<BitCodeAbbrev>();
4789 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_TYPE_ID_METADATA));
4790 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid strtab index
4791 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid length
4792 // n x (valueid , offset)
4793 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4794 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4795 unsigned TypeIdCompatibleVtableAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4796
4797 Abbv = std::make_shared<BitCodeAbbrev>();
4798 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_CALLSITE_INFO));
4799 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4800 // n x stackidindex
4801 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4802 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4803 unsigned CallsiteAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4804
4805 Abbv = std::make_shared<BitCodeAbbrev>();
4806 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_ALLOC_INFO));
4807 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib
4808 // n x (alloc type, context radix tree index)
4809 // optional: nummib x (numcontext x total size)
4810 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4811 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4812 unsigned AllocAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4813
4814 Abbv = std::make_shared<BitCodeAbbrev>();
4815 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_CONTEXT_RADIX_TREE_ARRAY));
4816 // n x entry
4817 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4818 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4819 unsigned RadixAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4820
4821 // First walk through all the functions and collect the allocation contexts in
4822 // their associated summaries, for use in constructing a radix tree of
4823 // contexts. Note that we need to do this in the same order as the functions
4824 // are processed further below since the call stack positions in the resulting
4825 // radix tree array are identified based on this order.
4826 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> CallStacks;
4827 for (const Function &F : M) {
4828 // Summary emission does not support anonymous functions, they have to be
4829 // renamed using the anonymous function renaming pass.
4830 if (!F.hasName())
4831 report_fatal_error(reason: "Unexpected anonymous function when writing summary");
4832
4833 ValueInfo VI = Index->getValueInfo(GUID: F.getGUID());
4834 if (!VI || VI.getSummaryList().empty()) {
4835 // Only declarations should not have a summary (a declaration might
4836 // however have a summary if the def was in module level asm).
4837 assert(F.isDeclaration());
4838 continue;
4839 }
4840 auto *Summary = VI.getSummaryList()[0].get();
4841 FunctionSummary *FS = cast<FunctionSummary>(Val: Summary);
4842 collectMemProfCallStacks(
4843 FS, /*GetStackIndex*/ [](unsigned I) { return I; }, CallStacks);
4844 }
4845 // Finalize the radix tree, write it out, and get the map of positions in the
4846 // linearized tree array.
4847 DenseMap<CallStackId, LinearCallStackId> CallStackPos;
4848 if (!CallStacks.empty()) {
4849 CallStackPos =
4850 writeMemoryProfileRadixTree(CallStacks: std::move(CallStacks), Stream, RadixAbbrev);
4851 }
4852
4853 // Keep track of the current index into the CallStackPos map.
4854 CallStackId CallStackCount = 0;
4855
4856 SmallVector<uint64_t, 64> NameVals;
4857 // Iterate over the list of functions instead of the Index to
4858 // ensure the ordering is stable.
4859 for (const Function &F : M) {
4860 // Summary emission does not support anonymous functions, they have to
4861 // renamed using the anonymous function renaming pass.
4862 if (!F.hasName())
4863 report_fatal_error(reason: "Unexpected anonymous function when writing summary");
4864
4865 ValueInfo VI = Index->getValueInfo(GUID: F.getGUID());
4866 if (!VI || VI.getSummaryList().empty()) {
4867 // Only declarations should not have a summary (a declaration might
4868 // however have a summary if the def was in module level asm).
4869 assert(F.isDeclaration());
4870 continue;
4871 }
4872 auto *Summary = VI.getSummaryList()[0].get();
4873 writePerModuleFunctionSummaryRecord(NameVals, Summary, ValueID: VE.getValueID(V: &F),
4874 FSCallsProfileAbbrev, CallsiteAbbrev,
4875 AllocAbbrev, ContextIdAbbvId, F,
4876 CallStackPos, CallStackCount);
4877 }
4878
4879 // Capture references from GlobalVariable initializers, which are outside
4880 // of a function scope.
4881 for (const GlobalVariable &G : M.globals())
4882 writeModuleLevelReferences(V: G, NameVals, FSModRefsAbbrev,
4883 FSModVTableRefsAbbrev);
4884
4885 for (const GlobalAlias &A : M.aliases()) {
4886 auto *Aliasee = A.getAliaseeObject();
4887 // Skip ifunc and nameless functions which don't have an entry in the
4888 // summary.
4889 if (!Aliasee->hasName() || isa<GlobalIFunc>(Val: Aliasee))
4890 continue;
4891 auto AliasId = VE.getValueID(V: &A);
4892 auto AliaseeId = VE.getValueID(V: Aliasee);
4893 NameVals.push_back(Elt: AliasId);
4894 auto *Summary = Index->getGlobalValueSummary(GV: A);
4895 AliasSummary *AS = cast<AliasSummary>(Val: Summary);
4896 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: AS->flags()));
4897 NameVals.push_back(Elt: AliaseeId);
4898 Stream.EmitRecord(Code: bitc::FS_ALIAS, Vals: NameVals, Abbrev: FSAliasAbbrev);
4899 NameVals.clear();
4900 }
4901
4902 for (auto &S : Index->typeIdCompatibleVtableMap()) {
4903 writeTypeIdCompatibleVtableSummaryRecord(NameVals, StrtabBuilder, Id: S.first,
4904 Summary: S.second, VE);
4905 Stream.EmitRecord(Code: bitc::FS_TYPE_ID_METADATA, Vals: NameVals,
4906 Abbrev: TypeIdCompatibleVtableAbbrev);
4907 NameVals.clear();
4908 }
4909
4910 if (Index->getBlockCount())
4911 Stream.EmitRecord(Code: bitc::FS_BLOCK_COUNT,
4912 Vals: ArrayRef<uint64_t>{Index->getBlockCount()});
4913
4914 Stream.ExitBlock();
4915}
4916
4917/// Emit the combined summary section into the combined index file.
4918void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
4919 Stream.EnterSubblock(BlockID: bitc::GLOBALVAL_SUMMARY_BLOCK_ID, CodeLen: 4);
4920 Stream.EmitRecord(
4921 Code: bitc::FS_VERSION,
4922 Vals: ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
4923
4924 // Write the index flags.
4925 Stream.EmitRecord(Code: bitc::FS_FLAGS, Vals: ArrayRef<uint64_t>{Index.getFlags()});
4926
4927 auto Abbv = std::make_shared<BitCodeAbbrev>();
4928 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_VALUE_GUID));
4929 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4930 // GUIDS often use up most of 64-bits, so encode as two Fixed 32.
4931 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4932 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4933 unsigned ValueGuidAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4934
4935 for (const auto &GVI : valueIds()) {
4936 Stream.EmitRecord(Code: bitc::FS_VALUE_GUID,
4937 Vals: ArrayRef<uint32_t>{GVI.second,
4938 static_cast<uint32_t>(GVI.first >> 32),
4939 static_cast<uint32_t>(GVI.first)},
4940 Abbrev: ValueGuidAbbrev);
4941 }
4942
4943 // Write the stack ids used by this index, which will be a subset of those in
4944 // the full index in the case of distributed indexes.
4945 if (!StackIds.empty()) {
4946 auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4947 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_STACK_IDS));
4948 // numids x stackid
4949 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4950 // The stack ids are hashes that are close to 64 bits in size, so emitting
4951 // as a pair of 32-bit fixed-width values is more efficient than a VBR.
4952 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4953 unsigned StackIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(StackIdAbbv));
4954 SmallVector<uint32_t> Vals;
4955 Vals.reserve(N: StackIds.size() * 2);
4956 for (auto Id : StackIds) {
4957 Vals.push_back(Elt: static_cast<uint32_t>(Id >> 32));
4958 Vals.push_back(Elt: static_cast<uint32_t>(Id));
4959 }
4960 Stream.EmitRecord(Code: bitc::FS_STACK_IDS, Vals, Abbrev: StackIdAbbvId);
4961 }
4962
4963 // Abbrev for FS_COMBINED_PROFILE.
4964 Abbv = std::make_shared<BitCodeAbbrev>();
4965 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_PROFILE));
4966 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4967 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
4968 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4969 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
4970 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
4971 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // entrycount
4972 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4973 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
4974 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
4975 // numrefs x valueid, n x (valueid, hotness+tailcall flags)
4976 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4977 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4978 unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4979
4980 // Abbrev for FS_COMBINED_GLOBALVAR_INIT_REFS.
4981 Abbv = std::make_shared<BitCodeAbbrev>();
4982 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS));
4983 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4984 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
4985 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4986 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
4987 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4988 unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4989
4990 // Abbrev for FS_COMBINED_ALIAS.
4991 Abbv = std::make_shared<BitCodeAbbrev>();
4992 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_ALIAS));
4993 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4994 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
4995 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4996 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4997 unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4998
4999 Abbv = std::make_shared<BitCodeAbbrev>();
5000 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_CALLSITE_INFO));
5001 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5002 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numstackindices
5003 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
5004 // numstackindices x stackidindex, numver x version
5005 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5006 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5007 unsigned CallsiteAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5008
5009 Abbv = std::make_shared<BitCodeAbbrev>();
5010 Abbv->Add(OpInfo: BitCodeAbbrevOp(CombinedIndexMemProfContext
5011 ? bitc::FS_COMBINED_ALLOC_INFO
5012 : bitc::FS_COMBINED_ALLOC_INFO_NO_CONTEXT));
5013 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib
5014 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
5015 // nummib x (alloc type, context radix tree index),
5016 // numver x version
5017 // optional: nummib x total size
5018 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5019 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5020 unsigned AllocAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5021
5022 auto shouldImportValueAsDecl = [&](GlobalValueSummary *GVS) -> bool {
5023 if (DecSummaries == nullptr)
5024 return false;
5025 return DecSummaries->count(x: GVS);
5026 };
5027
5028 // The aliases are emitted as a post-pass, and will point to the value
5029 // id of the aliasee. Save them in a vector for post-processing.
5030 SmallVector<AliasSummary *, 64> Aliases;
5031
5032 // Save the value id for each summary for alias emission.
5033 DenseMap<const GlobalValueSummary *, unsigned> SummaryToValueIdMap;
5034
5035 SmallVector<uint64_t, 64> NameVals;
5036
5037 // Set that will be populated during call to writeFunctionTypeMetadataRecords
5038 // with the type ids referenced by this index file.
5039 std::set<GlobalValue::GUID> ReferencedTypeIds;
5040
5041 // For local linkage, we also emit the original name separately
5042 // immediately after the record.
5043 auto MaybeEmitOriginalName = [&](GlobalValueSummary &S) {
5044 // We don't need to emit the original name if we are writing the index for
5045 // distributed backends (in which case ModuleToSummariesForIndex is
5046 // non-null). The original name is only needed during the thin link, since
5047 // for SamplePGO the indirect call targets for local functions have
5048 // have the original name annotated in profile.
5049 // Continue to emit it when writing out the entire combined index, which is
5050 // used in testing the thin link via llvm-lto.
5051 if (ModuleToSummariesForIndex || !GlobalValue::isLocalLinkage(Linkage: S.linkage()))
5052 return;
5053 NameVals.push_back(Elt: S.getOriginalName());
5054 Stream.EmitRecord(Code: bitc::FS_COMBINED_ORIGINAL_NAME, Vals: NameVals);
5055 NameVals.clear();
5056 };
5057
5058 DenseMap<CallStackId, LinearCallStackId> CallStackPos;
5059 if (CombinedIndexMemProfContext) {
5060 Abbv = std::make_shared<BitCodeAbbrev>();
5061 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_CONTEXT_RADIX_TREE_ARRAY));
5062 // n x entry
5063 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5064 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5065 unsigned RadixAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5066
5067 // First walk through all the functions and collect the allocation contexts
5068 // in their associated summaries, for use in constructing a radix tree of
5069 // contexts. Note that we need to do this in the same order as the functions
5070 // are processed further below since the call stack positions in the
5071 // resulting radix tree array are identified based on this order.
5072 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> CallStacks;
5073 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
5074 // Don't collect this when invoked for an aliasee, as it is not needed for
5075 // the alias summary. If the aliasee is to be imported, we will invoke
5076 // this separately with IsAliasee=false.
5077 if (IsAliasee)
5078 return;
5079 GlobalValueSummary *S = I.second;
5080 assert(S);
5081 auto *FS = dyn_cast<FunctionSummary>(Val: S);
5082 if (!FS)
5083 return;
5084 collectMemProfCallStacks(
5085 FS,
5086 /*GetStackIndex*/
5087 [&](unsigned I) {
5088 // Get the corresponding index into the list of StackIds actually
5089 // being written for this combined index (which may be a subset in
5090 // the case of distributed indexes).
5091 assert(StackIdIndicesToIndex.contains(I));
5092 return StackIdIndicesToIndex[I];
5093 },
5094 CallStacks);
5095 });
5096 // Finalize the radix tree, write it out, and get the map of positions in
5097 // the linearized tree array.
5098 if (!CallStacks.empty()) {
5099 CallStackPos = writeMemoryProfileRadixTree(CallStacks: std::move(CallStacks), Stream,
5100 RadixAbbrev);
5101 }
5102 }
5103
5104 // Keep track of the current index into the CallStackPos map. Not used if
5105 // CombinedIndexMemProfContext is false.
5106 CallStackId CallStackCount = 0;
5107
5108 DenseSet<GlobalValue::GUID> DefOrUseGUIDs;
5109 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
5110 GlobalValueSummary *S = I.second;
5111 assert(S);
5112 DefOrUseGUIDs.insert(V: I.first);
5113 for (const ValueInfo &VI : S->refs())
5114 DefOrUseGUIDs.insert(V: VI.getGUID());
5115
5116 auto ValueId = getValueId(ValGUID: I.first);
5117 assert(ValueId);
5118 SummaryToValueIdMap[S] = *ValueId;
5119
5120 // If this is invoked for an aliasee, we want to record the above
5121 // mapping, but then not emit a summary entry (if the aliasee is
5122 // to be imported, we will invoke this separately with IsAliasee=false).
5123 if (IsAliasee)
5124 return;
5125
5126 if (auto *AS = dyn_cast<AliasSummary>(Val: S)) {
5127 // Will process aliases as a post-pass because the reader wants all
5128 // global to be loaded first.
5129 Aliases.push_back(Elt: AS);
5130 return;
5131 }
5132
5133 if (auto *VS = dyn_cast<GlobalVarSummary>(Val: S)) {
5134 NameVals.push_back(Elt: *ValueId);
5135 assert(ModuleIdMap.count(VS->modulePath()));
5136 NameVals.push_back(Elt: ModuleIdMap[VS->modulePath()]);
5137 NameVals.push_back(
5138 Elt: getEncodedGVSummaryFlags(Flags: VS->flags(), ImportAsDecl: shouldImportValueAsDecl(VS)));
5139 NameVals.push_back(Elt: getEncodedGVarFlags(Flags: VS->varflags()));
5140 for (auto &RI : VS->refs()) {
5141 auto RefValueId = getValueId(ValGUID: RI.getGUID());
5142 if (!RefValueId)
5143 continue;
5144 NameVals.push_back(Elt: *RefValueId);
5145 }
5146
5147 // Emit the finished record.
5148 Stream.EmitRecord(Code: bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, Vals: NameVals,
5149 Abbrev: FSModRefsAbbrev);
5150 NameVals.clear();
5151 MaybeEmitOriginalName(*S);
5152 return;
5153 }
5154
5155 auto GetValueId = [&](const ValueInfo &VI) -> std::optional<unsigned> {
5156 if (!VI)
5157 return std::nullopt;
5158 return getValueId(ValGUID: VI.getGUID());
5159 };
5160
5161 auto *FS = cast<FunctionSummary>(Val: S);
5162 writeFunctionTypeMetadataRecords(Stream, FS, GetValueID: GetValueId);
5163 getReferencedTypeIds(FS, ReferencedTypeIds);
5164
5165 NameVals.push_back(Elt: *ValueId);
5166 assert(ModuleIdMap.count(FS->modulePath()));
5167 NameVals.push_back(Elt: ModuleIdMap[FS->modulePath()]);
5168 NameVals.push_back(
5169 Elt: getEncodedGVSummaryFlags(Flags: FS->flags(), ImportAsDecl: shouldImportValueAsDecl(FS)));
5170 NameVals.push_back(Elt: FS->instCount());
5171 NameVals.push_back(Elt: getEncodedFFlags(Flags: FS->fflags()));
5172 // TODO: Stop writing entry count and bump bitcode version.
5173 NameVals.push_back(Elt: 0 /* EntryCount */);
5174
5175 // Fill in below
5176 NameVals.push_back(Elt: 0); // numrefs
5177 NameVals.push_back(Elt: 0); // rorefcnt
5178 NameVals.push_back(Elt: 0); // worefcnt
5179
5180 unsigned Count = 0, RORefCnt = 0, WORefCnt = 0;
5181 for (auto &RI : FS->refs()) {
5182 auto RefValueId = getValueId(ValGUID: RI.getGUID());
5183 if (!RefValueId)
5184 continue;
5185 NameVals.push_back(Elt: *RefValueId);
5186 if (RI.isReadOnly())
5187 RORefCnt++;
5188 else if (RI.isWriteOnly())
5189 WORefCnt++;
5190 Count++;
5191 }
5192 NameVals[6] = Count;
5193 NameVals[7] = RORefCnt;
5194 NameVals[8] = WORefCnt;
5195
5196 for (auto &EI : FS->calls()) {
5197 // If this GUID doesn't have a value id, it doesn't have a function
5198 // summary and we don't need to record any calls to it.
5199 std::optional<unsigned> CallValueId = GetValueId(EI.first);
5200 if (!CallValueId)
5201 continue;
5202 NameVals.push_back(Elt: *CallValueId);
5203 NameVals.push_back(Elt: getEncodedHotnessCallEdgeInfo(CI: EI.second));
5204 }
5205
5206 // Emit the finished record.
5207 Stream.EmitRecord(Code: bitc::FS_COMBINED_PROFILE, Vals: NameVals,
5208 Abbrev: FSCallsProfileAbbrev);
5209 NameVals.clear();
5210
5211 writeFunctionHeapProfileRecords(
5212 Stream, FS, CallsiteAbbrev, AllocAbbrev, /*ContextIdAbbvId*/ 0,
5213 /*PerModule*/ false,
5214 /*GetValueId*/
5215 GetValueID: [&](const ValueInfo &VI) -> unsigned {
5216 std::optional<unsigned> ValueID = GetValueId(VI);
5217 // This can happen in shared index files for distributed ThinLTO if
5218 // the callee function summary is not included. Record 0 which we
5219 // will have to deal with conservatively when doing any kind of
5220 // validation in the ThinLTO backends.
5221 if (!ValueID)
5222 return 0;
5223 return *ValueID;
5224 },
5225 /*GetStackIndex*/
5226 [&](unsigned I) {
5227 // Get the corresponding index into the list of StackIds actually
5228 // being written for this combined index (which may be a subset in
5229 // the case of distributed indexes).
5230 assert(StackIdIndicesToIndex.contains(I));
5231 return StackIdIndicesToIndex[I];
5232 },
5233 /*WriteContextSizeInfoIndex*/ false, CallStackPos, CallStackCount);
5234
5235 MaybeEmitOriginalName(*S);
5236 });
5237
5238 for (auto *AS : Aliases) {
5239 auto AliasValueId = SummaryToValueIdMap[AS];
5240 assert(AliasValueId);
5241 NameVals.push_back(Elt: AliasValueId);
5242 assert(ModuleIdMap.count(AS->modulePath()));
5243 NameVals.push_back(Elt: ModuleIdMap[AS->modulePath()]);
5244 NameVals.push_back(
5245 Elt: getEncodedGVSummaryFlags(Flags: AS->flags(), ImportAsDecl: shouldImportValueAsDecl(AS)));
5246 // Set value id to 0 when an alias is imported but the aliasee summary is
5247 // not contained in the index.
5248 auto AliaseeValueId =
5249 AS->hasAliasee() ? SummaryToValueIdMap[&AS->getAliasee()] : 0;
5250 NameVals.push_back(Elt: AliaseeValueId);
5251
5252 // Emit the finished record.
5253 Stream.EmitRecord(Code: bitc::FS_COMBINED_ALIAS, Vals: NameVals, Abbrev: FSAliasAbbrev);
5254 NameVals.clear();
5255 MaybeEmitOriginalName(*AS);
5256
5257 if (AS->hasAliasee())
5258 if (auto *FS = dyn_cast<FunctionSummary>(Val: &AS->getAliasee()))
5259 getReferencedTypeIds(FS, ReferencedTypeIds);
5260 }
5261
5262 SmallVector<StringRef, 4> Functions;
5263 auto EmitCfiFunctions = [&](const CfiFunctionIndex &CfiIndex,
5264 bitc::GlobalValueSummarySymtabCodes Code) {
5265 if (CfiIndex.empty())
5266 return;
5267 for (GlobalValue::GUID GUID : DefOrUseGUIDs) {
5268 auto Defs = CfiIndex.forGuid(GUID);
5269 llvm::append_range(C&: Functions, R&: Defs);
5270 }
5271 if (Functions.empty())
5272 return;
5273 llvm::sort(C&: Functions);
5274 for (const auto &S : Functions) {
5275 NameVals.push_back(Elt: StrtabBuilder.add(S));
5276 NameVals.push_back(Elt: S.size());
5277 }
5278 Stream.EmitRecord(Code, Vals: NameVals);
5279 NameVals.clear();
5280 Functions.clear();
5281 };
5282
5283 EmitCfiFunctions(Index.cfiFunctionDefs(), bitc::FS_CFI_FUNCTION_DEFS);
5284 EmitCfiFunctions(Index.cfiFunctionDecls(), bitc::FS_CFI_FUNCTION_DECLS);
5285
5286 // Walk the GUIDs that were referenced, and write the
5287 // corresponding type id records.
5288 for (auto &T : ReferencedTypeIds) {
5289 auto TidIter = Index.typeIds().equal_range(x: T);
5290 for (const auto &[GUID, TypeIdPair] : make_range(p: TidIter)) {
5291 writeTypeIdSummaryRecord(NameVals, StrtabBuilder, Id: TypeIdPair.first,
5292 Summary: TypeIdPair.second);
5293 Stream.EmitRecord(Code: bitc::FS_TYPE_ID, Vals: NameVals);
5294 NameVals.clear();
5295 }
5296 }
5297
5298 if (Index.getBlockCount())
5299 Stream.EmitRecord(Code: bitc::FS_BLOCK_COUNT,
5300 Vals: ArrayRef<uint64_t>{Index.getBlockCount()});
5301
5302 Stream.ExitBlock();
5303}
5304
5305/// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the
5306/// current llvm version, and a record for the epoch number.
5307static void writeIdentificationBlock(BitstreamWriter &Stream) {
5308 Stream.EnterSubblock(BlockID: bitc::IDENTIFICATION_BLOCK_ID, CodeLen: 5);
5309
5310 // Write the "user readable" string identifying the bitcode producer
5311 auto Abbv = std::make_shared<BitCodeAbbrev>();
5312 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_STRING));
5313 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5314 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
5315 auto StringAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5316 writeStringRecord(Stream, Code: bitc::IDENTIFICATION_CODE_STRING,
5317 Str: "LLVM" LLVM_VERSION_STRING, AbbrevToUse: StringAbbrev);
5318
5319 // Write the epoch version
5320 Abbv = std::make_shared<BitCodeAbbrev>();
5321 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_EPOCH));
5322 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
5323 auto EpochAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5324 constexpr std::array<unsigned, 1> Vals = {._M_elems: {bitc::BITCODE_CURRENT_EPOCH}};
5325 Stream.EmitRecord(Code: bitc::IDENTIFICATION_CODE_EPOCH, Vals, Abbrev: EpochAbbrev);
5326 Stream.ExitBlock();
5327}
5328
5329void ModuleBitcodeWriter::writeModuleHash(StringRef View) {
5330 // Emit the module's hash.
5331 // MODULE_CODE_HASH: [5*i32]
5332 if (GenerateHash) {
5333 uint32_t Vals[5];
5334 Hasher.update(Data: ArrayRef<uint8_t>(
5335 reinterpret_cast<const uint8_t *>(View.data()), View.size()));
5336 std::array<uint8_t, 20> Hash = Hasher.result();
5337 for (int Pos = 0; Pos < 20; Pos += 4) {
5338 Vals[Pos / 4] = support::endian::read32be(P: Hash.data() + Pos);
5339 }
5340
5341 // Emit the finished record.
5342 Stream.EmitRecord(Code: bitc::MODULE_CODE_HASH, Vals);
5343
5344 if (ModHash)
5345 // Save the written hash value.
5346 llvm::copy(Range&: Vals, Out: std::begin(cont&: *ModHash));
5347 }
5348}
5349
5350void ModuleBitcodeWriter::write() {
5351 writeIdentificationBlock(Stream);
5352
5353 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5354 // We will want to write the module hash at this point. Block any flushing so
5355 // we can have access to the whole underlying data later.
5356 Stream.markAndBlockFlushing();
5357
5358 writeModuleVersion();
5359
5360 // Emit blockinfo, which defines the standard abbreviations etc.
5361 writeBlockInfo();
5362
5363 // Emit information describing all of the types in the module.
5364 writeTypeTable();
5365
5366 // Emit information about attribute groups.
5367 writeAttributeGroupTable();
5368
5369 // Emit information about parameter attributes.
5370 writeAttributeTable();
5371
5372 writeComdats();
5373
5374 // Emit top-level description of module, including target triple, inline asm,
5375 // descriptors for global variables, and function prototype info.
5376 writeModuleInfo();
5377
5378 // Emit constants.
5379 writeModuleConstants();
5380
5381 // Emit metadata kind names.
5382 writeModuleMetadataKinds();
5383
5384 // Emit metadata.
5385 writeModuleMetadata();
5386
5387 // Emit module-level use-lists.
5388 if (VE.shouldPreserveUseListOrder())
5389 writeUseListBlock(F: nullptr);
5390
5391 writeOperandBundleTags();
5392 writeSyncScopeNames();
5393
5394 // Emit function bodies.
5395 DenseMap<const Function *, uint64_t> FunctionToBitcodeIndex;
5396 for (const Function &F : M)
5397 if (!F.isDeclaration())
5398 writeFunction(F, FunctionToBitcodeIndex);
5399
5400 // Need to write after the above call to WriteFunction which populates
5401 // the summary information in the index.
5402 if (Index)
5403 writePerModuleGlobalValueSummary();
5404
5405 writeGlobalValueSymbolTable(FunctionToBitcodeIndex);
5406
5407 writeModuleHash(View: Stream.getMarkedBufferAndResumeFlushing());
5408
5409 Stream.ExitBlock();
5410}
5411
5412static void writeInt32ToBuffer(uint32_t Value, SmallVectorImpl<char> &Buffer,
5413 uint32_t &Position) {
5414 support::endian::write32le(P: &Buffer[Position], V: Value);
5415 Position += 4;
5416}
5417
5418/// If generating a bc file on darwin, we have to emit a
5419/// header and trailer to make it compatible with the system archiver. To do
5420/// this we emit the following header, and then emit a trailer that pads the
5421/// file out to be a multiple of 16 bytes.
5422///
5423/// struct bc_header {
5424/// uint32_t Magic; // 0x0B17C0DE
5425/// uint32_t Version; // Version, currently always 0.
5426/// uint32_t BitcodeOffset; // Offset to traditional bitcode file.
5427/// uint32_t BitcodeSize; // Size of traditional bitcode file.
5428/// uint32_t CPUType; // CPU specifier.
5429/// ... potentially more later ...
5430/// };
5431static void emitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer,
5432 const Triple &TT) {
5433 unsigned CPUType = ~0U;
5434
5435 // Match x86_64-*, i[3-9]86-*, powerpc-*, powerpc64-*, arm-*, thumb-*,
5436 // armv[0-9]-*, thumbv[0-9]-*, armv5te-*, or armv6t2-*. The CPUType is a magic
5437 // number from /usr/include/mach/machine.h. It is ok to reproduce the
5438 // specific constants here because they are implicitly part of the Darwin ABI.
5439 enum {
5440 DARWIN_CPU_ARCH_ABI64 = 0x01000000,
5441 DARWIN_CPU_TYPE_X86 = 7,
5442 DARWIN_CPU_TYPE_ARM = 12,
5443 DARWIN_CPU_TYPE_POWERPC = 18
5444 };
5445
5446 Triple::ArchType Arch = TT.getArch();
5447 if (Arch == Triple::x86_64)
5448 CPUType = DARWIN_CPU_TYPE_X86 | DARWIN_CPU_ARCH_ABI64;
5449 else if (Arch == Triple::x86)
5450 CPUType = DARWIN_CPU_TYPE_X86;
5451 else if (Arch == Triple::ppc)
5452 CPUType = DARWIN_CPU_TYPE_POWERPC;
5453 else if (Arch == Triple::ppc64)
5454 CPUType = DARWIN_CPU_TYPE_POWERPC | DARWIN_CPU_ARCH_ABI64;
5455 else if (Arch == Triple::arm || Arch == Triple::thumb)
5456 CPUType = DARWIN_CPU_TYPE_ARM;
5457
5458 // Traditional Bitcode starts after header.
5459 assert(Buffer.size() >= BWH_HeaderSize &&
5460 "Expected header size to be reserved");
5461 unsigned BCOffset = BWH_HeaderSize;
5462 unsigned BCSize = Buffer.size() - BWH_HeaderSize;
5463
5464 // Write the magic and version.
5465 unsigned Position = 0;
5466 writeInt32ToBuffer(Value: 0x0B17C0DE, Buffer, Position);
5467 writeInt32ToBuffer(Value: 0, Buffer, Position); // Version.
5468 writeInt32ToBuffer(Value: BCOffset, Buffer, Position);
5469 writeInt32ToBuffer(Value: BCSize, Buffer, Position);
5470 writeInt32ToBuffer(Value: CPUType, Buffer, Position);
5471
5472 // If the file is not a multiple of 16 bytes, insert dummy padding.
5473 while (Buffer.size() & 15)
5474 Buffer.push_back(Elt: 0);
5475}
5476
5477/// Helper to write the header common to all bitcode files.
5478static void writeBitcodeHeader(BitstreamWriter &Stream) {
5479 // Emit the file header.
5480 Stream.Emit(Val: (unsigned)'B', NumBits: 8);
5481 Stream.Emit(Val: (unsigned)'C', NumBits: 8);
5482 Stream.Emit(Val: 0x0, NumBits: 4);
5483 Stream.Emit(Val: 0xC, NumBits: 4);
5484 Stream.Emit(Val: 0xE, NumBits: 4);
5485 Stream.Emit(Val: 0xD, NumBits: 4);
5486}
5487
5488BitcodeWriter::BitcodeWriter(SmallVectorImpl<char> &Buffer)
5489 : Stream(new BitstreamWriter(Buffer)) {
5490 writeBitcodeHeader(Stream&: *Stream);
5491}
5492
5493BitcodeWriter::BitcodeWriter(raw_ostream &FS)
5494 : Stream(new BitstreamWriter(FS, FlushThreshold)) {
5495 writeBitcodeHeader(Stream&: *Stream);
5496}
5497
5498BitcodeWriter::~BitcodeWriter() { assert(WroteStrtab); }
5499
5500void BitcodeWriter::writeBlob(unsigned Block, unsigned Record, StringRef Blob) {
5501 Stream->EnterSubblock(BlockID: Block, CodeLen: 3);
5502
5503 auto Abbv = std::make_shared<BitCodeAbbrev>();
5504 Abbv->Add(OpInfo: BitCodeAbbrevOp(Record));
5505 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
5506 auto AbbrevNo = Stream->EmitAbbrev(Abbv: std::move(Abbv));
5507
5508 Stream->EmitRecordWithBlob(Abbrev: AbbrevNo, Vals: ArrayRef<uint64_t>{Record}, Blob);
5509
5510 Stream->ExitBlock();
5511}
5512
5513void BitcodeWriter::writeSymtab() {
5514 assert(!WroteStrtab && !WroteSymtab);
5515
5516 // If any module has module-level inline asm, we will require a registered asm
5517 // parser for the target so that we can create an accurate symbol table for
5518 // the module.
5519 for (Module *M : Mods) {
5520 if (M->getModuleInlineAsm().empty())
5521 continue;
5522
5523 std::string Err;
5524 const Triple TT(M->getTargetTriple());
5525 const Target *T = TargetRegistry::lookupTarget(TheTriple: TT, Error&: Err);
5526 if (!T || !T->hasMCAsmParser())
5527 return;
5528 }
5529
5530 WroteSymtab = true;
5531 SmallVector<char, 0> Symtab;
5532 // The irsymtab::build function may be unable to create a symbol table if the
5533 // module is malformed (e.g. it contains an invalid alias). Writing a symbol
5534 // table is not required for correctness, but we still want to be able to
5535 // write malformed modules to bitcode files, so swallow the error.
5536 if (Error E = irsymtab::build(Mods, Symtab, StrtabBuilder, Alloc)) {
5537 consumeError(Err: std::move(E));
5538 return;
5539 }
5540
5541 writeBlob(Block: bitc::SYMTAB_BLOCK_ID, Record: bitc::SYMTAB_BLOB,
5542 Blob: {Symtab.data(), Symtab.size()});
5543}
5544
5545void BitcodeWriter::writeStrtab() {
5546 assert(!WroteStrtab);
5547
5548 std::vector<char> Strtab;
5549 StrtabBuilder.finalizeInOrder();
5550 Strtab.resize(new_size: StrtabBuilder.getSize());
5551 StrtabBuilder.write(Buf: (uint8_t *)Strtab.data());
5552
5553 writeBlob(Block: bitc::STRTAB_BLOCK_ID, Record: bitc::STRTAB_BLOB,
5554 Blob: {Strtab.data(), Strtab.size()});
5555
5556 WroteStrtab = true;
5557}
5558
5559void BitcodeWriter::copyStrtab(StringRef Strtab) {
5560 writeBlob(Block: bitc::STRTAB_BLOCK_ID, Record: bitc::STRTAB_BLOB, Blob: Strtab);
5561 WroteStrtab = true;
5562}
5563
5564void BitcodeWriter::writeModule(const Module &M,
5565 bool ShouldPreserveUseListOrder,
5566 const ModuleSummaryIndex *Index,
5567 bool GenerateHash, ModuleHash *ModHash) {
5568 assert(!WroteStrtab);
5569
5570 // The Mods vector is used by irsymtab::build, which requires non-const
5571 // Modules in case it needs to materialize metadata. But the bitcode writer
5572 // requires that the module is materialized, so we can cast to non-const here,
5573 // after checking that it is in fact materialized.
5574 assert(M.isMaterialized());
5575 Mods.push_back(x: const_cast<Module *>(&M));
5576
5577 ModuleBitcodeWriter ModuleWriter(M, StrtabBuilder, *Stream,
5578 ShouldPreserveUseListOrder, Index,
5579 GenerateHash, ModHash);
5580 ModuleWriter.write();
5581}
5582
5583void BitcodeWriter::writeIndex(
5584 const ModuleSummaryIndex *Index,
5585 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex,
5586 const GVSummaryPtrSet *DecSummaries) {
5587 IndexBitcodeWriter IndexWriter(*Stream, StrtabBuilder, *Index, DecSummaries,
5588 ModuleToSummariesForIndex);
5589 IndexWriter.write();
5590}
5591
5592/// Write the specified module to the specified output stream.
5593void llvm::WriteBitcodeToFile(const Module &M, raw_ostream &Out,
5594 bool ShouldPreserveUseListOrder,
5595 const ModuleSummaryIndex *Index,
5596 bool GenerateHash, ModuleHash *ModHash) {
5597 auto Write = [&](BitcodeWriter &Writer) {
5598 Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash,
5599 ModHash);
5600 Writer.writeSymtab();
5601 Writer.writeStrtab();
5602 };
5603 Triple TT(M.getTargetTriple());
5604 if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) {
5605 // If this is darwin or another generic macho target, reserve space for the
5606 // header. Note that the header is computed *after* the output is known, so
5607 // we currently explicitly use a buffer, write to it, and then subsequently
5608 // flush to Out.
5609 SmallVector<char, 0> Buffer;
5610 Buffer.reserve(N: 256 * 1024);
5611 Buffer.insert(I: Buffer.begin(), NumToInsert: BWH_HeaderSize, Elt: 0);
5612 BitcodeWriter Writer(Buffer);
5613 Write(Writer);
5614 emitDarwinBCHeaderAndTrailer(Buffer, TT);
5615 Out.write(Ptr: Buffer.data(), Size: Buffer.size());
5616 } else {
5617 BitcodeWriter Writer(Out);
5618 Write(Writer);
5619 }
5620}
5621
5622void IndexBitcodeWriter::write() {
5623 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5624
5625 writeModuleVersion();
5626
5627 // Write the module paths in the combined index.
5628 writeModStrings();
5629
5630 // Write the summary combined index records.
5631 writeCombinedGlobalValueSummary();
5632
5633 Stream.ExitBlock();
5634}
5635
5636// Write the specified module summary index to the given raw output stream,
5637// where it will be written in a new bitcode block. This is used when
5638// writing the combined index file for ThinLTO. When writing a subset of the
5639// index for a distributed backend, provide a \p ModuleToSummariesForIndex map.
5640void llvm::writeIndexToFile(
5641 const ModuleSummaryIndex &Index, raw_ostream &Out,
5642 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex,
5643 const GVSummaryPtrSet *DecSummaries) {
5644 SmallVector<char, 0> Buffer;
5645 Buffer.reserve(N: 256 * 1024);
5646
5647 BitcodeWriter Writer(Buffer);
5648 Writer.writeIndex(Index: &Index, ModuleToSummariesForIndex, DecSummaries);
5649 Writer.writeStrtab();
5650
5651 Out.write(Ptr: (char *)&Buffer.front(), Size: Buffer.size());
5652}
5653
5654namespace {
5655
5656/// Class to manage the bitcode writing for a thin link bitcode file.
5657class ThinLinkBitcodeWriter : public ModuleBitcodeWriterBase {
5658 /// ModHash is for use in ThinLTO incremental build, generated while writing
5659 /// the module bitcode file.
5660 const ModuleHash *ModHash;
5661
5662public:
5663 ThinLinkBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
5664 BitstreamWriter &Stream,
5665 const ModuleSummaryIndex &Index,
5666 const ModuleHash &ModHash)
5667 : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
5668 /*ShouldPreserveUseListOrder=*/false, &Index),
5669 ModHash(&ModHash) {}
5670
5671 void write();
5672
5673private:
5674 void writeSimplifiedModuleInfo();
5675};
5676
5677} // end anonymous namespace
5678
5679// This function writes a simpilified module info for thin link bitcode file.
5680// It only contains the source file name along with the name(the offset and
5681// size in strtab) and linkage for global values. For the global value info
5682// entry, in order to keep linkage at offset 5, there are three zeros used
5683// as padding.
5684void ThinLinkBitcodeWriter::writeSimplifiedModuleInfo() {
5685 SmallVector<unsigned, 64> Vals;
5686 // Emit the module's source file name.
5687 {
5688 StringEncoding Bits = getStringEncoding(Str: M.getSourceFileName());
5689 BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
5690 if (Bits == SE_Char6)
5691 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
5692 else if (Bits == SE_Fixed7)
5693 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
5694
5695 // MODULE_CODE_SOURCE_FILENAME: [namechar x N]
5696 auto Abbv = std::make_shared<BitCodeAbbrev>();
5697 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
5698 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5699 Abbv->Add(OpInfo: AbbrevOpToUse);
5700 unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5701
5702 for (const auto P : M.getSourceFileName())
5703 Vals.push_back(Elt: (unsigned char)P);
5704
5705 Stream.EmitRecord(Code: bitc::MODULE_CODE_SOURCE_FILENAME, Vals, Abbrev: FilenameAbbrev);
5706 Vals.clear();
5707 }
5708
5709 // Emit the global variable information.
5710 for (const GlobalVariable &GV : M.globals()) {
5711 // GLOBALVAR: [strtab offset, strtab size, 0, 0, 0, linkage]
5712 Vals.push_back(Elt: StrtabBuilder.add(S: GV.getName()));
5713 Vals.push_back(Elt: GV.getName().size());
5714 Vals.push_back(Elt: 0);
5715 Vals.push_back(Elt: 0);
5716 Vals.push_back(Elt: 0);
5717 Vals.push_back(Elt: getEncodedLinkage(GV));
5718
5719 Stream.EmitRecord(Code: bitc::MODULE_CODE_GLOBALVAR, Vals);
5720 Vals.clear();
5721 }
5722
5723 // Emit the function proto information.
5724 for (const Function &F : M) {
5725 // FUNCTION: [strtab offset, strtab size, 0, 0, 0, linkage]
5726 Vals.push_back(Elt: StrtabBuilder.add(S: F.getName()));
5727 Vals.push_back(Elt: F.getName().size());
5728 Vals.push_back(Elt: 0);
5729 Vals.push_back(Elt: 0);
5730 Vals.push_back(Elt: 0);
5731 Vals.push_back(Elt: getEncodedLinkage(GV: F));
5732
5733 Stream.EmitRecord(Code: bitc::MODULE_CODE_FUNCTION, Vals);
5734 Vals.clear();
5735 }
5736
5737 // Emit the alias information.
5738 for (const GlobalAlias &A : M.aliases()) {
5739 // ALIAS: [strtab offset, strtab size, 0, 0, 0, linkage]
5740 Vals.push_back(Elt: StrtabBuilder.add(S: A.getName()));
5741 Vals.push_back(Elt: A.getName().size());
5742 Vals.push_back(Elt: 0);
5743 Vals.push_back(Elt: 0);
5744 Vals.push_back(Elt: 0);
5745 Vals.push_back(Elt: getEncodedLinkage(GV: A));
5746
5747 Stream.EmitRecord(Code: bitc::MODULE_CODE_ALIAS, Vals);
5748 Vals.clear();
5749 }
5750
5751 // Emit the ifunc information.
5752 for (const GlobalIFunc &I : M.ifuncs()) {
5753 // IFUNC: [strtab offset, strtab size, 0, 0, 0, linkage]
5754 Vals.push_back(Elt: StrtabBuilder.add(S: I.getName()));
5755 Vals.push_back(Elt: I.getName().size());
5756 Vals.push_back(Elt: 0);
5757 Vals.push_back(Elt: 0);
5758 Vals.push_back(Elt: 0);
5759 Vals.push_back(Elt: getEncodedLinkage(GV: I));
5760
5761 Stream.EmitRecord(Code: bitc::MODULE_CODE_IFUNC, Vals);
5762 Vals.clear();
5763 }
5764}
5765
5766void ThinLinkBitcodeWriter::write() {
5767 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5768
5769 writeModuleVersion();
5770
5771 writeSimplifiedModuleInfo();
5772
5773 writePerModuleGlobalValueSummary();
5774
5775 // Write module hash.
5776 Stream.EmitRecord(Code: bitc::MODULE_CODE_HASH, Vals: ArrayRef<uint32_t>(*ModHash));
5777
5778 Stream.ExitBlock();
5779}
5780
5781void BitcodeWriter::writeThinLinkBitcode(const Module &M,
5782 const ModuleSummaryIndex &Index,
5783 const ModuleHash &ModHash) {
5784 assert(!WroteStrtab);
5785
5786 // The Mods vector is used by irsymtab::build, which requires non-const
5787 // Modules in case it needs to materialize metadata. But the bitcode writer
5788 // requires that the module is materialized, so we can cast to non-const here,
5789 // after checking that it is in fact materialized.
5790 assert(M.isMaterialized());
5791 Mods.push_back(x: const_cast<Module *>(&M));
5792
5793 ThinLinkBitcodeWriter ThinLinkWriter(M, StrtabBuilder, *Stream, Index,
5794 ModHash);
5795 ThinLinkWriter.write();
5796}
5797
5798// Write the specified thin link bitcode file to the given raw output stream,
5799// where it will be written in a new bitcode block. This is used when
5800// writing the per-module index file for ThinLTO.
5801void llvm::writeThinLinkBitcodeToFile(const Module &M, raw_ostream &Out,
5802 const ModuleSummaryIndex &Index,
5803 const ModuleHash &ModHash) {
5804 SmallVector<char, 0> Buffer;
5805 Buffer.reserve(N: 256 * 1024);
5806
5807 BitcodeWriter Writer(Buffer);
5808 Writer.writeThinLinkBitcode(M, Index, ModHash);
5809 Writer.writeSymtab();
5810 Writer.writeStrtab();
5811
5812 Out.write(Ptr: (char *)&Buffer.front(), Size: Buffer.size());
5813}
5814
5815static const char *getSectionNameForBitcode(const Triple &T) {
5816 switch (T.getObjectFormat()) {
5817 case Triple::MachO:
5818 return "__LLVM,__bitcode";
5819 case Triple::COFF:
5820 case Triple::ELF:
5821 case Triple::Wasm:
5822 case Triple::UnknownObjectFormat:
5823 return ".llvmbc";
5824 case Triple::GOFF:
5825 llvm_unreachable("GOFF is not yet implemented");
5826 break;
5827 case Triple::SPIRV:
5828 if (T.getVendor() == Triple::AMD)
5829 return ".llvmbc";
5830 llvm_unreachable("SPIRV is not yet implemented");
5831 break;
5832 case Triple::XCOFF:
5833 llvm_unreachable("XCOFF is not yet implemented");
5834 break;
5835 case Triple::DXContainer:
5836 llvm_unreachable("DXContainer is not yet implemented");
5837 break;
5838 }
5839 llvm_unreachable("Unimplemented ObjectFormatType");
5840}
5841
5842static const char *getSectionNameForCommandline(const Triple &T) {
5843 switch (T.getObjectFormat()) {
5844 case Triple::MachO:
5845 return "__LLVM,__cmdline";
5846 case Triple::COFF:
5847 case Triple::ELF:
5848 case Triple::Wasm:
5849 case Triple::UnknownObjectFormat:
5850 return ".llvmcmd";
5851 case Triple::GOFF:
5852 llvm_unreachable("GOFF is not yet implemented");
5853 break;
5854 case Triple::SPIRV:
5855 if (T.getVendor() == Triple::AMD)
5856 return ".llvmcmd";
5857 llvm_unreachable("SPIRV is not yet implemented");
5858 break;
5859 case Triple::XCOFF:
5860 llvm_unreachable("XCOFF is not yet implemented");
5861 break;
5862 case Triple::DXContainer:
5863 llvm_unreachable("DXC is not yet implemented");
5864 break;
5865 }
5866 llvm_unreachable("Unimplemented ObjectFormatType");
5867}
5868
5869void llvm::embedBitcodeInModule(llvm::Module &M, llvm::MemoryBufferRef Buf,
5870 bool EmbedBitcode, bool EmbedCmdline,
5871 const std::vector<uint8_t> &CmdArgs) {
5872 // Save llvm.compiler.used and remove it.
5873 SmallVector<Constant *, 2> UsedArray;
5874 SmallVector<GlobalValue *, 4> UsedGlobals;
5875 GlobalVariable *Used = collectUsedGlobalVariables(M, Vec&: UsedGlobals, CompilerUsed: true);
5876 Type *UsedElementType = Used ? Used->getValueType()->getArrayElementType()
5877 : PointerType::getUnqual(C&: M.getContext());
5878 for (auto *GV : UsedGlobals) {
5879 if (GV->getName() != "llvm.embedded.module" &&
5880 GV->getName() != "llvm.cmdline")
5881 UsedArray.push_back(
5882 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
5883 }
5884 if (Used)
5885 Used->eraseFromParent();
5886
5887 // Embed the bitcode for the llvm module.
5888 std::string Data;
5889 ArrayRef<uint8_t> ModuleData;
5890 Triple T(M.getTargetTriple());
5891
5892 if (EmbedBitcode) {
5893 if (Buf.getBufferSize() == 0 ||
5894 !isBitcode(BufPtr: (const unsigned char *)Buf.getBufferStart(),
5895 BufEnd: (const unsigned char *)Buf.getBufferEnd())) {
5896 // If the input is LLVM Assembly, bitcode is produced by serializing
5897 // the module. Use-lists order need to be preserved in this case.
5898 llvm::raw_string_ostream OS(Data);
5899 llvm::WriteBitcodeToFile(M, Out&: OS, /* ShouldPreserveUseListOrder */ true);
5900 ModuleData =
5901 ArrayRef<uint8_t>((const uint8_t *)OS.str().data(), OS.str().size());
5902 } else
5903 // If the input is LLVM bitcode, write the input byte stream directly.
5904 ModuleData = ArrayRef<uint8_t>((const uint8_t *)Buf.getBufferStart(),
5905 Buf.getBufferSize());
5906 }
5907 llvm::Constant *ModuleConstant =
5908 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts: ModuleData);
5909 llvm::GlobalVariable *GV = new llvm::GlobalVariable(
5910 M, ModuleConstant->getType(), true, llvm::GlobalValue::PrivateLinkage,
5911 ModuleConstant);
5912 GV->setSection(getSectionNameForBitcode(T));
5913 // Set alignment to 1 to prevent padding between two contributions from input
5914 // sections after linking.
5915 GV->setAlignment(Align(1));
5916 UsedArray.push_back(
5917 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
5918 if (llvm::GlobalVariable *Old =
5919 M.getGlobalVariable(Name: "llvm.embedded.module", AllowInternal: true)) {
5920 assert(Old->hasZeroLiveUses() &&
5921 "llvm.embedded.module can only be used once in llvm.compiler.used");
5922 GV->takeName(V: Old);
5923 Old->eraseFromParent();
5924 } else {
5925 GV->setName("llvm.embedded.module");
5926 }
5927
5928 // Skip if only bitcode needs to be embedded.
5929 if (EmbedCmdline) {
5930 // Embed command-line options.
5931 ArrayRef<uint8_t> CmdData(const_cast<uint8_t *>(CmdArgs.data()),
5932 CmdArgs.size());
5933 llvm::Constant *CmdConstant =
5934 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts: CmdData);
5935 GV = new llvm::GlobalVariable(M, CmdConstant->getType(), true,
5936 llvm::GlobalValue::PrivateLinkage,
5937 CmdConstant);
5938 GV->setSection(getSectionNameForCommandline(T));
5939 GV->setAlignment(Align(1));
5940 UsedArray.push_back(
5941 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
5942 if (llvm::GlobalVariable *Old = M.getGlobalVariable(Name: "llvm.cmdline", AllowInternal: true)) {
5943 assert(Old->hasZeroLiveUses() &&
5944 "llvm.cmdline can only be used once in llvm.compiler.used");
5945 GV->takeName(V: Old);
5946 Old->eraseFromParent();
5947 } else {
5948 GV->setName("llvm.cmdline");
5949 }
5950 }
5951
5952 if (UsedArray.empty())
5953 return;
5954
5955 // Recreate llvm.compiler.used.
5956 ArrayType *ATy = ArrayType::get(ElementType: UsedElementType, NumElements: UsedArray.size());
5957 auto *NewUsed = new GlobalVariable(
5958 M, ATy, false, llvm::GlobalValue::AppendingLinkage,
5959 llvm::ConstantArray::get(T: ATy, V: UsedArray), "llvm.compiler.used");
5960 NewUsed->setSection("llvm.metadata");
5961}
5962