1//===- Bitcode/Writer/BitcodeWriter.cpp - Bitcode Writer ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Bitcode writer implementation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Bitcode/BitcodeWriter.h"
14#include "ValueEnumerator.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/DenseMap.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SetVector.h"
21#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/SmallString.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringMap.h"
25#include "llvm/ADT/StringRef.h"
26#include "llvm/Analysis/MemoryProfileInfo.h"
27#include "llvm/BinaryFormat/Dwarf.h"
28#include "llvm/Bitcode/BitcodeCommon.h"
29#include "llvm/Bitcode/BitcodeReader.h"
30#include "llvm/Bitcode/LLVMBitCodes.h"
31#include "llvm/Bitstream/BitCodes.h"
32#include "llvm/Bitstream/BitstreamWriter.h"
33#include "llvm/Config/llvm-config.h"
34#include "llvm/IR/Attributes.h"
35#include "llvm/IR/BasicBlock.h"
36#include "llvm/IR/Comdat.h"
37#include "llvm/IR/Constant.h"
38#include "llvm/IR/ConstantRangeList.h"
39#include "llvm/IR/Constants.h"
40#include "llvm/IR/DebugInfoMetadata.h"
41#include "llvm/IR/DebugLoc.h"
42#include "llvm/IR/DerivedTypes.h"
43#include "llvm/IR/Function.h"
44#include "llvm/IR/GlobalAlias.h"
45#include "llvm/IR/GlobalIFunc.h"
46#include "llvm/IR/GlobalObject.h"
47#include "llvm/IR/GlobalValue.h"
48#include "llvm/IR/GlobalVariable.h"
49#include "llvm/IR/InlineAsm.h"
50#include "llvm/IR/InstrTypes.h"
51#include "llvm/IR/Instruction.h"
52#include "llvm/IR/Instructions.h"
53#include "llvm/IR/LLVMContext.h"
54#include "llvm/IR/Metadata.h"
55#include "llvm/IR/Module.h"
56#include "llvm/IR/ModuleSummaryIndex.h"
57#include "llvm/IR/Operator.h"
58#include "llvm/IR/Type.h"
59#include "llvm/IR/UseListOrder.h"
60#include "llvm/IR/Value.h"
61#include "llvm/IR/ValueSymbolTable.h"
62#include "llvm/MC/StringTableBuilder.h"
63#include "llvm/MC/TargetRegistry.h"
64#include "llvm/Object/IRSymtab.h"
65#include "llvm/ProfileData/MemProf.h"
66#include "llvm/ProfileData/MemProfRadixTree.h"
67#include "llvm/Support/AtomicOrdering.h"
68#include "llvm/Support/Casting.h"
69#include "llvm/Support/CommandLine.h"
70#include "llvm/Support/Compiler.h"
71#include "llvm/Support/Endian.h"
72#include "llvm/Support/Error.h"
73#include "llvm/Support/ErrorHandling.h"
74#include "llvm/Support/MathExtras.h"
75#include "llvm/Support/SHA1.h"
76#include "llvm/Support/raw_ostream.h"
77#include "llvm/TargetParser/Triple.h"
78#include <algorithm>
79#include <cassert>
80#include <cstddef>
81#include <cstdint>
82#include <iterator>
83#include <map>
84#include <memory>
85#include <optional>
86#include <string>
87#include <utility>
88#include <vector>
89
90using namespace llvm;
91using namespace llvm::memprof;
92
93static cl::opt<unsigned>
94 IndexThreshold("bitcode-mdindex-threshold", cl::Hidden, cl::init(Val: 25),
95 cl::desc("Number of metadatas above which we emit an index "
96 "to enable lazy-loading"));
97static cl::opt<uint32_t> FlushThreshold(
98 "bitcode-flush-threshold", cl::Hidden, cl::init(Val: 512),
99 cl::desc("The threshold (unit M) for flushing LLVM bitcode."));
100
101// Since we only use the context information in the memprof summary records in
102// the LTO backends to do assertion checking, save time and space by only
103// serializing the context for non-NDEBUG builds.
104// TODO: Currently this controls writing context of the allocation info records,
105// which are larger and more expensive, but we should do this for the callsite
106// records as well.
107// FIXME: Convert to a const once this has undergone more sigificant testing.
108static cl::opt<bool>
109 CombinedIndexMemProfContext("combined-index-memprof-context", cl::Hidden,
110#ifdef NDEBUG
111 cl::init(Val: false),
112#else
113 cl::init(true),
114#endif
115 cl::desc(""));
116
117static cl::opt<bool> PreserveBitcodeUseListOrder(
118 "preserve-bc-uselistorder", cl::Hidden, cl::init(Val: true),
119 cl::desc("Preserve use-list order when writing LLVM bitcode."));
120
121namespace llvm {
122extern FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold;
123}
124
125namespace {
126
127/// These are manifest constants used by the bitcode writer. They do not need to
128/// be kept in sync with the reader, but need to be consistent within this file.
129enum {
130 // VALUE_SYMTAB_BLOCK abbrev id's.
131 VST_ENTRY_8_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
132 VST_ENTRY_7_ABBREV,
133 VST_ENTRY_6_ABBREV,
134 VST_BBENTRY_6_ABBREV,
135
136 // CONSTANTS_BLOCK abbrev id's.
137 CONSTANTS_SETTYPE_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
138 CONSTANTS_INTEGER_ABBREV,
139 CONSTANTS_BYTE_ABBREV,
140 CONSTANTS_CE_CAST_Abbrev,
141 CONSTANTS_NULL_Abbrev,
142
143 // FUNCTION_BLOCK abbrev id's.
144 FUNCTION_INST_LOAD_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
145 FUNCTION_INST_STORE_ABBREV,
146 FUNCTION_INST_UNOP_ABBREV,
147 FUNCTION_INST_UNOP_FLAGS_ABBREV,
148 FUNCTION_INST_BINOP_ABBREV,
149 FUNCTION_INST_BINOP_FLAGS_ABBREV,
150 FUNCTION_INST_CAST_ABBREV,
151 FUNCTION_INST_CAST_FLAGS_ABBREV,
152 FUNCTION_INST_RET_VOID_ABBREV,
153 FUNCTION_INST_RET_VAL_ABBREV,
154 FUNCTION_INST_BR_UNCOND_ABBREV,
155 FUNCTION_INST_BR_COND_ABBREV,
156 FUNCTION_INST_UNREACHABLE_ABBREV,
157 FUNCTION_INST_GEP_ABBREV,
158 FUNCTION_INST_CMP_ABBREV,
159 FUNCTION_INST_CMP_FLAGS_ABBREV,
160 FUNCTION_DEBUG_RECORD_VALUE_ABBREV,
161 FUNCTION_DEBUG_LOC_ABBREV,
162};
163
164/// Abstract class to manage the bitcode writing, subclassed for each bitcode
165/// file type.
166class BitcodeWriterBase {
167protected:
168 /// The stream created and owned by the client.
169 BitstreamWriter &Stream;
170
171 StringTableBuilder &StrtabBuilder;
172
173public:
174 /// Constructs a BitcodeWriterBase object that writes to the provided
175 /// \p Stream.
176 BitcodeWriterBase(BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder)
177 : Stream(Stream), StrtabBuilder(StrtabBuilder) {}
178
179protected:
180 void writeModuleVersion();
181};
182
183void BitcodeWriterBase::writeModuleVersion() {
184 // VERSION: [version#]
185 Stream.EmitRecord(Code: bitc::MODULE_CODE_VERSION, Vals: ArrayRef<uint64_t>{2});
186}
187
188/// Base class to manage the module bitcode writing, currently subclassed for
189/// ModuleBitcodeWriter and ThinLinkBitcodeWriter.
190class ModuleBitcodeWriterBase : public BitcodeWriterBase {
191protected:
192 /// The Module to write to bitcode.
193 const Module &M;
194
195 /// Enumerates ids for all values in the module.
196 ValueEnumerator VE;
197
198 /// Optional per-module index to write for ThinLTO.
199 const ModuleSummaryIndex *Index;
200
201 /// Map that holds the correspondence between GUIDs in the summary index,
202 /// that came from indirect call profiles, and a value id generated by this
203 /// class to use in the VST and summary block records.
204 std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
205
206 /// Tracks the last value id recorded in the GUIDToValueMap.
207 unsigned GlobalValueId;
208
209 /// Saves the offset of the VSTOffset record that must eventually be
210 /// backpatched with the offset of the actual VST.
211 uint64_t VSTOffsetPlaceholder = 0;
212
213public:
214 /// Constructs a ModuleBitcodeWriterBase object for the given Module,
215 /// writing to the provided \p Buffer.
216 ModuleBitcodeWriterBase(const Module &M, StringTableBuilder &StrtabBuilder,
217 BitstreamWriter &Stream,
218 bool ShouldPreserveUseListOrder,
219 const ModuleSummaryIndex *Index)
220 : BitcodeWriterBase(Stream, StrtabBuilder), M(M),
221 VE(M, PreserveBitcodeUseListOrder.getNumOccurrences()
222 ? PreserveBitcodeUseListOrder
223 : ShouldPreserveUseListOrder),
224 Index(Index) {
225 // Assign ValueIds to any callee values in the index that came from
226 // indirect call profiles and were recorded as a GUID not a Value*
227 // (which would have been assigned an ID by the ValueEnumerator).
228 // The starting ValueId is just after the number of values in the
229 // ValueEnumerator, so that they can be emitted in the VST.
230 GlobalValueId = VE.getValues().size();
231 if (!Index)
232 return;
233 for (const auto &GUIDSummaryLists : *Index)
234 // Examine all summaries for this GUID.
235 for (auto &Summary : GUIDSummaryLists.second.getSummaryList())
236 if (auto FS = dyn_cast<FunctionSummary>(Val: Summary.get())) {
237 // For each call in the function summary, see if the call
238 // is to a GUID (which means it is for an indirect call,
239 // otherwise we would have a Value for it). If so, synthesize
240 // a value id.
241 for (auto &CallEdge : FS->calls())
242 if (!CallEdge.first.haveGVs() || !CallEdge.first.getValue())
243 assignValueId(ValGUID: CallEdge.first.getGUID());
244
245 // For each referenced variables in the function summary, see if the
246 // variable is represented by a GUID (as opposed to a symbol to
247 // declarations or definitions in the module). If so, synthesize a
248 // value id.
249 for (auto &RefEdge : FS->refs())
250 if (!RefEdge.haveGVs() || !RefEdge.getValue())
251 assignValueId(ValGUID: RefEdge.getGUID());
252 }
253 }
254
255protected:
256 void writePerModuleGlobalValueSummary();
257
258private:
259 void writePerModuleFunctionSummaryRecord(
260 SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
261 unsigned ValueID, unsigned FSCallsProfileAbbrev, unsigned CallsiteAbbrev,
262 unsigned AllocAbbrev, unsigned ContextIdAbbvId, const Function &F,
263 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
264 CallStackId &CallStackCount);
265 void writeModuleLevelReferences(const GlobalVariable &V,
266 SmallVector<uint64_t, 64> &NameVals,
267 unsigned FSModRefsAbbrev,
268 unsigned FSModVTableRefsAbbrev);
269
270 void assignValueId(GlobalValue::GUID ValGUID) {
271 GUIDToValueIdMap[ValGUID] = ++GlobalValueId;
272 }
273
274 unsigned getValueId(GlobalValue::GUID ValGUID) {
275 const auto &VMI = GUIDToValueIdMap.find(x: ValGUID);
276 // Expect that any GUID value had a value Id assigned by an
277 // earlier call to assignValueId.
278 assert(VMI != GUIDToValueIdMap.end() &&
279 "GUID does not have assigned value Id");
280 return VMI->second;
281 }
282
283 // Helper to get the valueId for the type of value recorded in VI.
284 unsigned getValueId(ValueInfo VI) {
285 if (!VI.haveGVs() || !VI.getValue())
286 return getValueId(ValGUID: VI.getGUID());
287 return VE.getValueID(V: VI.getValue());
288 }
289
290 std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
291};
292
293/// Class to manage the bitcode writing for a module.
294class ModuleBitcodeWriter : public ModuleBitcodeWriterBase {
295 /// True if a module hash record should be written.
296 bool GenerateHash;
297
298 /// If non-null, when GenerateHash is true, the resulting hash is written
299 /// into ModHash.
300 ModuleHash *ModHash;
301
302 SHA1 Hasher;
303
304 /// The start bit of the identification block.
305 uint64_t BitcodeStartBit;
306
307public:
308 /// Constructs a ModuleBitcodeWriter object for the given Module,
309 /// writing to the provided \p Buffer.
310 ModuleBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
311 BitstreamWriter &Stream, bool ShouldPreserveUseListOrder,
312 const ModuleSummaryIndex *Index, bool GenerateHash,
313 ModuleHash *ModHash = nullptr)
314 : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
315 ShouldPreserveUseListOrder, Index),
316 GenerateHash(GenerateHash), ModHash(ModHash),
317 BitcodeStartBit(Stream.GetCurrentBitNo()) {}
318
319 /// Emit the current module to the bitstream.
320 void write();
321
322private:
323 uint64_t bitcodeStartBit() { return BitcodeStartBit; }
324
325 size_t addToStrtab(StringRef Str);
326
327 void writeAttributeGroupTable();
328 void writeAttributeTable();
329 void writeTypeTable();
330 void writeComdats();
331 void writeValueSymbolTableForwardDecl();
332 void writeModuleInfo();
333 void writeValueAsMetadata(const ValueAsMetadata *MD,
334 SmallVectorImpl<uint64_t> &Record);
335 void writeMDTuple(const MDTuple *N, SmallVectorImpl<uint64_t> &Record,
336 unsigned Abbrev);
337 unsigned createDILocationAbbrev();
338 void writeDILocation(const DILocation *N, SmallVectorImpl<uint64_t> &Record,
339 unsigned &Abbrev);
340 unsigned createGenericDINodeAbbrev();
341 void writeGenericDINode(const GenericDINode *N,
342 SmallVectorImpl<uint64_t> &Record, unsigned &Abbrev);
343 void writeDISubrange(const DISubrange *N, SmallVectorImpl<uint64_t> &Record,
344 unsigned Abbrev);
345 void writeDIGenericSubrange(const DIGenericSubrange *N,
346 SmallVectorImpl<uint64_t> &Record,
347 unsigned Abbrev);
348 void writeDIEnumerator(const DIEnumerator *N,
349 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
350 void writeDIBasicType(const DIBasicType *N, SmallVectorImpl<uint64_t> &Record,
351 unsigned Abbrev);
352 void writeDIFixedPointType(const DIFixedPointType *N,
353 SmallVectorImpl<uint64_t> &Record,
354 unsigned Abbrev);
355 void writeDIStringType(const DIStringType *N,
356 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
357 void writeDIDerivedType(const DIDerivedType *N,
358 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
359 void writeDISubrangeType(const DISubrangeType *N,
360 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
361 void writeDICompositeType(const DICompositeType *N,
362 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
363 void writeDISubroutineType(const DISubroutineType *N,
364 SmallVectorImpl<uint64_t> &Record,
365 unsigned Abbrev);
366 void writeDIFile(const DIFile *N, SmallVectorImpl<uint64_t> &Record,
367 unsigned Abbrev);
368 void writeDICompileUnit(const DICompileUnit *N,
369 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
370 void writeDISubprogram(const DISubprogram *N,
371 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
372 void writeDILexicalBlock(const DILexicalBlock *N,
373 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
374 void writeDILexicalBlockFile(const DILexicalBlockFile *N,
375 SmallVectorImpl<uint64_t> &Record,
376 unsigned Abbrev);
377 void writeDICommonBlock(const DICommonBlock *N,
378 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
379 void writeDINamespace(const DINamespace *N, SmallVectorImpl<uint64_t> &Record,
380 unsigned Abbrev);
381 void writeDIMacro(const DIMacro *N, SmallVectorImpl<uint64_t> &Record,
382 unsigned Abbrev);
383 void writeDIMacroFile(const DIMacroFile *N, SmallVectorImpl<uint64_t> &Record,
384 unsigned Abbrev);
385 void writeDIArgList(const DIArgList *N, SmallVectorImpl<uint64_t> &Record);
386 void writeDIModule(const DIModule *N, SmallVectorImpl<uint64_t> &Record,
387 unsigned Abbrev);
388 void writeDIAssignID(const DIAssignID *N, SmallVectorImpl<uint64_t> &Record,
389 unsigned Abbrev);
390 void writeDITemplateTypeParameter(const DITemplateTypeParameter *N,
391 SmallVectorImpl<uint64_t> &Record,
392 unsigned Abbrev);
393 void writeDITemplateValueParameter(const DITemplateValueParameter *N,
394 SmallVectorImpl<uint64_t> &Record,
395 unsigned Abbrev);
396 void writeDIGlobalVariable(const DIGlobalVariable *N,
397 SmallVectorImpl<uint64_t> &Record,
398 unsigned Abbrev);
399 void writeDILocalVariable(const DILocalVariable *N,
400 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
401 void writeDILabel(const DILabel *N,
402 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
403 void writeDIExpression(const DIExpression *N,
404 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
405 void writeDIGlobalVariableExpression(const DIGlobalVariableExpression *N,
406 SmallVectorImpl<uint64_t> &Record,
407 unsigned Abbrev);
408 void writeDIObjCProperty(const DIObjCProperty *N,
409 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
410 void writeDIImportedEntity(const DIImportedEntity *N,
411 SmallVectorImpl<uint64_t> &Record,
412 unsigned Abbrev);
413 unsigned createNamedMetadataAbbrev();
414 void writeNamedMetadata(SmallVectorImpl<uint64_t> &Record);
415 unsigned createMetadataStringsAbbrev();
416 void writeMetadataStrings(ArrayRef<const Metadata *> Strings,
417 SmallVectorImpl<uint64_t> &Record);
418 void writeMetadataRecords(ArrayRef<const Metadata *> MDs,
419 SmallVectorImpl<uint64_t> &Record,
420 std::vector<unsigned> *MDAbbrevs = nullptr,
421 std::vector<uint64_t> *IndexPos = nullptr);
422 void writeModuleMetadata();
423 void writeFunctionMetadata(const Function &F);
424 void writeFunctionMetadataAttachment(const Function &F);
425 void pushGlobalMetadataAttachment(SmallVectorImpl<uint64_t> &Record,
426 const GlobalObject &GO);
427 void writeModuleMetadataKinds();
428 void writeOperandBundleTags();
429 void writeSyncScopeNames();
430 void writeConstants(unsigned FirstVal, unsigned LastVal, bool isGlobal);
431 void writeModuleConstants();
432 bool pushValueAndType(const Value *V, unsigned InstID,
433 SmallVectorImpl<unsigned> &Vals);
434 bool pushValueOrMetadata(const Value *V, unsigned InstID,
435 SmallVectorImpl<unsigned> &Vals);
436 void writeOperandBundles(const CallBase &CB, unsigned InstID);
437 void pushValue(const Value *V, unsigned InstID,
438 SmallVectorImpl<unsigned> &Vals);
439 void pushValueSigned(const Value *V, unsigned InstID,
440 SmallVectorImpl<uint64_t> &Vals);
441 void writeInstruction(const Instruction &I, unsigned InstID,
442 SmallVectorImpl<unsigned> &Vals);
443 void writeFunctionLevelValueSymbolTable(const ValueSymbolTable &VST);
444 void writeGlobalValueSymbolTable(
445 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
446 void writeUseList(UseListOrder &&Order);
447 void writeUseListBlock(const Function *F);
448 void
449 writeFunction(const Function &F,
450 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
451 void writeBlockInfo();
452 void writeModuleHash(StringRef View);
453
454 unsigned getEncodedSyncScopeID(SyncScope::ID SSID) {
455 return unsigned(SSID);
456 }
457
458 unsigned getEncodedAlign(MaybeAlign Alignment) { return encode(A: Alignment); }
459};
460
461/// Class to manage the bitcode writing for a combined index.
462class IndexBitcodeWriter : public BitcodeWriterBase {
463 /// The combined index to write to bitcode.
464 const ModuleSummaryIndex &Index;
465
466 /// When writing combined summaries, provides the set of global value
467 /// summaries for which the value (function, function alias, etc) should be
468 /// imported as a declaration.
469 const GVSummaryPtrSet *DecSummaries = nullptr;
470
471 /// When writing a subset of the index for distributed backends, client
472 /// provides a map of modules to the corresponding GUIDs/summaries to write.
473 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex;
474
475 /// Map that holds the correspondence between the GUID used in the combined
476 /// index and a value id generated by this class to use in references.
477 std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
478
479 // The stack ids used by this index, which will be a subset of those in
480 // the full index in the case of distributed indexes.
481 std::vector<uint64_t> StackIds;
482
483 // Keep a map of the stack id indices used by records being written for this
484 // index to the index of the corresponding stack id in the above StackIds
485 // vector. Ensures we write each referenced stack id once.
486 DenseMap<unsigned, unsigned> StackIdIndicesToIndex;
487
488 /// Tracks the last value id recorded in the GUIDToValueMap.
489 unsigned GlobalValueId = 0;
490
491 /// Tracks the assignment of module paths in the module path string table to
492 /// an id assigned for use in summary references to the module path.
493 DenseMap<StringRef, uint64_t> ModuleIdMap;
494
495public:
496 /// Constructs a IndexBitcodeWriter object for the given combined index,
497 /// writing to the provided \p Buffer. When writing a subset of the index
498 /// for a distributed backend, provide a \p ModuleToSummariesForIndex map.
499 /// If provided, \p DecSummaries specifies the set of summaries for which
500 /// the corresponding functions or aliased functions should be imported as a
501 /// declaration (but not definition) for each module.
502 IndexBitcodeWriter(
503 BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder,
504 const ModuleSummaryIndex &Index,
505 const GVSummaryPtrSet *DecSummaries = nullptr,
506 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex = nullptr)
507 : BitcodeWriterBase(Stream, StrtabBuilder), Index(Index),
508 DecSummaries(DecSummaries),
509 ModuleToSummariesForIndex(ModuleToSummariesForIndex) {
510
511 // See if the StackIdIndex was already added to the StackId map and
512 // vector. If not, record it.
513 auto RecordStackIdReference = [&](unsigned StackIdIndex) {
514 // If the StackIdIndex is not yet in the map, the below insert ensures
515 // that it will point to the new StackIds vector entry we push to just
516 // below.
517 auto Inserted =
518 StackIdIndicesToIndex.insert(KV: {StackIdIndex, StackIds.size()});
519 if (Inserted.second)
520 StackIds.push_back(x: Index.getStackIdAtIndex(Index: StackIdIndex));
521 };
522
523 // Assign unique value ids to all summaries to be written, for use
524 // in writing out the call graph edges. Save the mapping from GUID
525 // to the new global value id to use when writing those edges, which
526 // are currently saved in the index in terms of GUID.
527 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
528 GUIDToValueIdMap[I.first] = ++GlobalValueId;
529 // If this is invoked for an aliasee, we want to record the above mapping,
530 // but not the information needed for its summary entry (if the aliasee is
531 // to be imported, we will invoke this separately with IsAliasee=false).
532 if (IsAliasee)
533 return;
534 auto *FS = dyn_cast<FunctionSummary>(Val: I.second);
535 if (!FS)
536 return;
537 // Record all stack id indices actually used in the summary entries being
538 // written, so that we can compact them in the case of distributed ThinLTO
539 // indexes.
540 for (auto &CI : FS->callsites()) {
541 // If the stack id list is empty, this callsite info was synthesized for
542 // a missing tail call frame. Ensure that the callee's GUID gets a value
543 // id. Normally we only generate these for defined summaries, which in
544 // the case of distributed ThinLTO is only the functions already defined
545 // in the module or that we want to import. We don't bother to include
546 // all the callee symbols as they aren't normally needed in the backend.
547 // However, for the synthesized callsite infos we do need the callee
548 // GUID in the backend so that we can correlate the identified callee
549 // with this callsite info (which for non-tail calls is done by the
550 // ordering of the callsite infos and verified via stack ids).
551 if (CI.StackIdIndices.empty()) {
552 GUIDToValueIdMap[CI.Callee.getGUID()] = ++GlobalValueId;
553 continue;
554 }
555 for (auto Idx : CI.StackIdIndices)
556 RecordStackIdReference(Idx);
557 }
558 if (CombinedIndexMemProfContext) {
559 for (auto &AI : FS->allocs())
560 for (auto &MIB : AI.MIBs)
561 for (auto Idx : MIB.StackIdIndices)
562 RecordStackIdReference(Idx);
563 }
564 });
565 }
566
567 /// The below iterator returns the GUID and associated summary.
568 using GVInfo = std::pair<GlobalValue::GUID, GlobalValueSummary *>;
569
570 /// Calls the callback for each value GUID and summary to be written to
571 /// bitcode. This hides the details of whether they are being pulled from the
572 /// entire index or just those in a provided ModuleToSummariesForIndex map.
573 template<typename Functor>
574 void forEachSummary(Functor Callback) {
575 if (ModuleToSummariesForIndex) {
576 for (auto &M : *ModuleToSummariesForIndex)
577 for (auto &Summary : M.second) {
578 Callback(Summary, false);
579 // Ensure aliasee is handled, e.g. for assigning a valueId,
580 // even if we are not importing the aliasee directly (the
581 // imported alias will contain a copy of aliasee).
582 if (auto *AS = dyn_cast<AliasSummary>(Val: Summary.getSecond()))
583 Callback({AS->getAliaseeGUID(), &AS->getAliasee()}, true);
584 }
585 } else {
586 for (auto &Summaries : Index)
587 for (auto &Summary : Summaries.second.getSummaryList())
588 Callback({Summaries.first, Summary.get()}, false);
589 }
590 }
591
592 /// Calls the callback for each entry in the modulePaths StringMap that
593 /// should be written to the module path string table. This hides the details
594 /// of whether they are being pulled from the entire index or just those in a
595 /// provided ModuleToSummariesForIndex map.
596 template <typename Functor> void forEachModule(Functor Callback) {
597 if (ModuleToSummariesForIndex) {
598 for (const auto &M : *ModuleToSummariesForIndex) {
599 const auto &MPI = Index.modulePaths().find(Key: M.first);
600 if (MPI == Index.modulePaths().end()) {
601 // This should only happen if the bitcode file was empty, in which
602 // case we shouldn't be importing (the ModuleToSummariesForIndex
603 // would only include the module we are writing and index for).
604 assert(ModuleToSummariesForIndex->size() == 1);
605 continue;
606 }
607 Callback(*MPI);
608 }
609 } else {
610 // Since StringMap iteration order isn't guaranteed, order by path string
611 // first.
612 // FIXME: Make this a vector of StringMapEntry instead to avoid the later
613 // map lookup.
614 std::vector<StringRef> ModulePaths;
615 for (auto &[ModPath, _] : Index.modulePaths())
616 ModulePaths.push_back(x: ModPath);
617 llvm::sort(C&: ModulePaths);
618 for (auto &ModPath : ModulePaths)
619 Callback(*Index.modulePaths().find(Key: ModPath));
620 }
621 }
622
623 /// Main entry point for writing a combined index to bitcode.
624 void write();
625
626private:
627 void writeModStrings();
628 void writeCombinedGlobalValueSummary();
629
630 std::optional<unsigned> getValueId(GlobalValue::GUID ValGUID) {
631 auto VMI = GUIDToValueIdMap.find(x: ValGUID);
632 if (VMI == GUIDToValueIdMap.end())
633 return std::nullopt;
634 return VMI->second;
635 }
636
637 std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
638};
639
640} // end anonymous namespace
641
642static unsigned getEncodedCastOpcode(unsigned Opcode) {
643 switch (Opcode) {
644 default: llvm_unreachable("Unknown cast instruction!");
645 case Instruction::Trunc : return bitc::CAST_TRUNC;
646 case Instruction::ZExt : return bitc::CAST_ZEXT;
647 case Instruction::SExt : return bitc::CAST_SEXT;
648 case Instruction::FPToUI : return bitc::CAST_FPTOUI;
649 case Instruction::FPToSI : return bitc::CAST_FPTOSI;
650 case Instruction::UIToFP : return bitc::CAST_UITOFP;
651 case Instruction::SIToFP : return bitc::CAST_SITOFP;
652 case Instruction::FPTrunc : return bitc::CAST_FPTRUNC;
653 case Instruction::FPExt : return bitc::CAST_FPEXT;
654 case Instruction::PtrToAddr: return bitc::CAST_PTRTOADDR;
655 case Instruction::PtrToInt: return bitc::CAST_PTRTOINT;
656 case Instruction::IntToPtr: return bitc::CAST_INTTOPTR;
657 case Instruction::BitCast : return bitc::CAST_BITCAST;
658 case Instruction::AddrSpaceCast: return bitc::CAST_ADDRSPACECAST;
659 }
660}
661
662static unsigned getEncodedUnaryOpcode(unsigned Opcode) {
663 switch (Opcode) {
664 default: llvm_unreachable("Unknown binary instruction!");
665 case Instruction::FNeg: return bitc::UNOP_FNEG;
666 }
667}
668
669static unsigned getEncodedBinaryOpcode(unsigned Opcode) {
670 switch (Opcode) {
671 default: llvm_unreachable("Unknown binary instruction!");
672 case Instruction::Add:
673 case Instruction::FAdd: return bitc::BINOP_ADD;
674 case Instruction::Sub:
675 case Instruction::FSub: return bitc::BINOP_SUB;
676 case Instruction::Mul:
677 case Instruction::FMul: return bitc::BINOP_MUL;
678 case Instruction::UDiv: return bitc::BINOP_UDIV;
679 case Instruction::FDiv:
680 case Instruction::SDiv: return bitc::BINOP_SDIV;
681 case Instruction::URem: return bitc::BINOP_UREM;
682 case Instruction::FRem:
683 case Instruction::SRem: return bitc::BINOP_SREM;
684 case Instruction::Shl: return bitc::BINOP_SHL;
685 case Instruction::LShr: return bitc::BINOP_LSHR;
686 case Instruction::AShr: return bitc::BINOP_ASHR;
687 case Instruction::And: return bitc::BINOP_AND;
688 case Instruction::Or: return bitc::BINOP_OR;
689 case Instruction::Xor: return bitc::BINOP_XOR;
690 }
691}
692
693static unsigned getEncodedRMWOperation(const AtomicRMWInst &I) {
694 unsigned Encoding = 0;
695 switch (I.getOperation()) {
696 default: llvm_unreachable("Unknown RMW operation!");
697 case AtomicRMWInst::Xchg:
698 Encoding = bitc::RMW_XCHG;
699 break;
700 case AtomicRMWInst::Add:
701 Encoding = bitc::RMW_ADD;
702 break;
703 case AtomicRMWInst::Sub:
704 Encoding = bitc::RMW_SUB;
705 break;
706 case AtomicRMWInst::And:
707 Encoding = bitc::RMW_AND;
708 break;
709 case AtomicRMWInst::Nand:
710 Encoding = bitc::RMW_NAND;
711 break;
712 case AtomicRMWInst::Or:
713 Encoding = bitc::RMW_OR;
714 break;
715 case AtomicRMWInst::Xor:
716 Encoding = bitc::RMW_XOR;
717 break;
718 case AtomicRMWInst::Max:
719 Encoding = bitc::RMW_MAX;
720 break;
721 case AtomicRMWInst::Min:
722 Encoding = bitc::RMW_MIN;
723 break;
724 case AtomicRMWInst::UMax:
725 Encoding = bitc::RMW_UMAX;
726 break;
727 case AtomicRMWInst::UMin:
728 Encoding = bitc::RMW_UMIN;
729 break;
730 case AtomicRMWInst::FAdd:
731 Encoding = bitc::RMW_FADD;
732 break;
733 case AtomicRMWInst::FSub:
734 Encoding = bitc::RMW_FSUB;
735 break;
736 case AtomicRMWInst::FMax:
737 Encoding = bitc::RMW_FMAX;
738 break;
739 case AtomicRMWInst::FMin:
740 Encoding = bitc::RMW_FMIN;
741 break;
742 case AtomicRMWInst::FMaximum:
743 Encoding = bitc::RMW_FMAXIMUM;
744 break;
745 case AtomicRMWInst::FMinimum:
746 Encoding = bitc::RMW_FMINIMUM;
747 break;
748 case AtomicRMWInst::FMaximumNum:
749 Encoding = bitc::RMW_FMAXIMUMNUM;
750 break;
751 case AtomicRMWInst::FMinimumNum:
752 Encoding = bitc::RMW_FMINIMUMNUM;
753 break;
754 case AtomicRMWInst::UIncWrap:
755 Encoding = bitc::RMW_UINC_WRAP;
756 break;
757 case AtomicRMWInst::UDecWrap:
758 Encoding = bitc::RMW_UDEC_WRAP;
759 break;
760 case AtomicRMWInst::USubCond:
761 Encoding = bitc::RMW_USUB_COND;
762 break;
763 case AtomicRMWInst::USubSat:
764 Encoding = bitc::RMW_USUB_SAT;
765 break;
766 }
767
768 if (I.isElementwise())
769 Encoding |= bitc::RMW_ELEMENTWISE_FLAG;
770 return Encoding;
771}
772
773static unsigned getEncodedOrdering(AtomicOrdering Ordering) {
774 switch (Ordering) {
775 case AtomicOrdering::NotAtomic: return bitc::ORDERING_NOTATOMIC;
776 case AtomicOrdering::Unordered: return bitc::ORDERING_UNORDERED;
777 case AtomicOrdering::Monotonic: return bitc::ORDERING_MONOTONIC;
778 case AtomicOrdering::Acquire: return bitc::ORDERING_ACQUIRE;
779 case AtomicOrdering::Release: return bitc::ORDERING_RELEASE;
780 case AtomicOrdering::AcquireRelease: return bitc::ORDERING_ACQREL;
781 case AtomicOrdering::SequentiallyConsistent: return bitc::ORDERING_SEQCST;
782 }
783 llvm_unreachable("Invalid ordering");
784}
785
786static void writeStringRecord(BitstreamWriter &Stream, unsigned Code,
787 StringRef Str, unsigned AbbrevToUse) {
788 SmallVector<unsigned, 64> Vals;
789
790 // Code: [strchar x N]
791 for (char C : Str) {
792 if (AbbrevToUse && !BitCodeAbbrevOp::isChar6(C))
793 AbbrevToUse = 0;
794 Vals.push_back(Elt: C);
795 }
796
797 // Emit the finished record.
798 Stream.EmitRecord(Code, Vals, Abbrev: AbbrevToUse);
799}
800
801static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
802 switch (Kind) {
803 case Attribute::Alignment:
804 return bitc::ATTR_KIND_ALIGNMENT;
805 case Attribute::AllocAlign:
806 return bitc::ATTR_KIND_ALLOC_ALIGN;
807 case Attribute::AllocSize:
808 return bitc::ATTR_KIND_ALLOC_SIZE;
809 case Attribute::AlwaysInline:
810 return bitc::ATTR_KIND_ALWAYS_INLINE;
811 case Attribute::Builtin:
812 return bitc::ATTR_KIND_BUILTIN;
813 case Attribute::ByVal:
814 return bitc::ATTR_KIND_BY_VAL;
815 case Attribute::Convergent:
816 return bitc::ATTR_KIND_CONVERGENT;
817 case Attribute::InAlloca:
818 return bitc::ATTR_KIND_IN_ALLOCA;
819 case Attribute::Cold:
820 return bitc::ATTR_KIND_COLD;
821 case Attribute::DisableSanitizerInstrumentation:
822 return bitc::ATTR_KIND_DISABLE_SANITIZER_INSTRUMENTATION;
823 case Attribute::FnRetThunkExtern:
824 return bitc::ATTR_KIND_FNRETTHUNK_EXTERN;
825 case Attribute::Flatten:
826 return bitc::ATTR_KIND_FLATTEN;
827 case Attribute::Hot:
828 return bitc::ATTR_KIND_HOT;
829 case Attribute::ElementType:
830 return bitc::ATTR_KIND_ELEMENTTYPE;
831 case Attribute::HybridPatchable:
832 return bitc::ATTR_KIND_HYBRID_PATCHABLE;
833 case Attribute::InlineHint:
834 return bitc::ATTR_KIND_INLINE_HINT;
835 case Attribute::InReg:
836 return bitc::ATTR_KIND_IN_REG;
837 case Attribute::JumpTable:
838 return bitc::ATTR_KIND_JUMP_TABLE;
839 case Attribute::MinSize:
840 return bitc::ATTR_KIND_MIN_SIZE;
841 case Attribute::AllocatedPointer:
842 return bitc::ATTR_KIND_ALLOCATED_POINTER;
843 case Attribute::AllocKind:
844 return bitc::ATTR_KIND_ALLOC_KIND;
845 case Attribute::Memory:
846 return bitc::ATTR_KIND_MEMORY;
847 case Attribute::NoFPClass:
848 return bitc::ATTR_KIND_NOFPCLASS;
849 case Attribute::Naked:
850 return bitc::ATTR_KIND_NAKED;
851 case Attribute::Nest:
852 return bitc::ATTR_KIND_NEST;
853 case Attribute::NoAlias:
854 return bitc::ATTR_KIND_NO_ALIAS;
855 case Attribute::NoBuiltin:
856 return bitc::ATTR_KIND_NO_BUILTIN;
857 case Attribute::NoCallback:
858 return bitc::ATTR_KIND_NO_CALLBACK;
859 case Attribute::NoDivergenceSource:
860 return bitc::ATTR_KIND_NO_DIVERGENCE_SOURCE;
861 case Attribute::NoDuplicate:
862 return bitc::ATTR_KIND_NO_DUPLICATE;
863 case Attribute::NoFree:
864 return bitc::ATTR_KIND_NOFREE;
865 case Attribute::NoImplicitFloat:
866 return bitc::ATTR_KIND_NO_IMPLICIT_FLOAT;
867 case Attribute::NoInline:
868 return bitc::ATTR_KIND_NO_INLINE;
869 case Attribute::NoRecurse:
870 return bitc::ATTR_KIND_NO_RECURSE;
871 case Attribute::NoMerge:
872 return bitc::ATTR_KIND_NO_MERGE;
873 case Attribute::NonLazyBind:
874 return bitc::ATTR_KIND_NON_LAZY_BIND;
875 case Attribute::NonNull:
876 return bitc::ATTR_KIND_NON_NULL;
877 case Attribute::Dereferenceable:
878 return bitc::ATTR_KIND_DEREFERENCEABLE;
879 case Attribute::DereferenceableOrNull:
880 return bitc::ATTR_KIND_DEREFERENCEABLE_OR_NULL;
881 case Attribute::NoRedZone:
882 return bitc::ATTR_KIND_NO_RED_ZONE;
883 case Attribute::NoReturn:
884 return bitc::ATTR_KIND_NO_RETURN;
885 case Attribute::NoSync:
886 return bitc::ATTR_KIND_NOSYNC;
887 case Attribute::NoCfCheck:
888 return bitc::ATTR_KIND_NOCF_CHECK;
889 case Attribute::NoProfile:
890 return bitc::ATTR_KIND_NO_PROFILE;
891 case Attribute::SkipProfile:
892 return bitc::ATTR_KIND_SKIP_PROFILE;
893 case Attribute::NoUnwind:
894 return bitc::ATTR_KIND_NO_UNWIND;
895 case Attribute::NoSanitizeBounds:
896 return bitc::ATTR_KIND_NO_SANITIZE_BOUNDS;
897 case Attribute::NoSanitizeCoverage:
898 return bitc::ATTR_KIND_NO_SANITIZE_COVERAGE;
899 case Attribute::NullPointerIsValid:
900 return bitc::ATTR_KIND_NULL_POINTER_IS_VALID;
901 case Attribute::OptimizeForDebugging:
902 return bitc::ATTR_KIND_OPTIMIZE_FOR_DEBUGGING;
903 case Attribute::OptForFuzzing:
904 return bitc::ATTR_KIND_OPT_FOR_FUZZING;
905 case Attribute::OptimizeForSize:
906 return bitc::ATTR_KIND_OPTIMIZE_FOR_SIZE;
907 case Attribute::OptimizeNone:
908 return bitc::ATTR_KIND_OPTIMIZE_NONE;
909 case Attribute::ReadNone:
910 return bitc::ATTR_KIND_READ_NONE;
911 case Attribute::ReadOnly:
912 return bitc::ATTR_KIND_READ_ONLY;
913 case Attribute::Returned:
914 return bitc::ATTR_KIND_RETURNED;
915 case Attribute::ReturnsTwice:
916 return bitc::ATTR_KIND_RETURNS_TWICE;
917 case Attribute::SExt:
918 return bitc::ATTR_KIND_S_EXT;
919 case Attribute::Speculatable:
920 return bitc::ATTR_KIND_SPECULATABLE;
921 case Attribute::StackAlignment:
922 return bitc::ATTR_KIND_STACK_ALIGNMENT;
923 case Attribute::StackProtect:
924 return bitc::ATTR_KIND_STACK_PROTECT;
925 case Attribute::StackProtectReq:
926 return bitc::ATTR_KIND_STACK_PROTECT_REQ;
927 case Attribute::StackProtectStrong:
928 return bitc::ATTR_KIND_STACK_PROTECT_STRONG;
929 case Attribute::SafeStack:
930 return bitc::ATTR_KIND_SAFESTACK;
931 case Attribute::ShadowCallStack:
932 return bitc::ATTR_KIND_SHADOWCALLSTACK;
933 case Attribute::StrictFP:
934 return bitc::ATTR_KIND_STRICT_FP;
935 case Attribute::StructRet:
936 return bitc::ATTR_KIND_STRUCT_RET;
937 case Attribute::SanitizeAddress:
938 return bitc::ATTR_KIND_SANITIZE_ADDRESS;
939 case Attribute::SanitizeAllocToken:
940 return bitc::ATTR_KIND_SANITIZE_ALLOC_TOKEN;
941 case Attribute::SanitizeHWAddress:
942 return bitc::ATTR_KIND_SANITIZE_HWADDRESS;
943 case Attribute::SanitizeThread:
944 return bitc::ATTR_KIND_SANITIZE_THREAD;
945 case Attribute::SanitizeType:
946 return bitc::ATTR_KIND_SANITIZE_TYPE;
947 case Attribute::SanitizeMemory:
948 return bitc::ATTR_KIND_SANITIZE_MEMORY;
949 case Attribute::SanitizeNumericalStability:
950 return bitc::ATTR_KIND_SANITIZE_NUMERICAL_STABILITY;
951 case Attribute::SanitizeRealtime:
952 return bitc::ATTR_KIND_SANITIZE_REALTIME;
953 case Attribute::SanitizeRealtimeBlocking:
954 return bitc::ATTR_KIND_SANITIZE_REALTIME_BLOCKING;
955 case Attribute::SpeculativeLoadHardening:
956 return bitc::ATTR_KIND_SPECULATIVE_LOAD_HARDENING;
957 case Attribute::SwiftError:
958 return bitc::ATTR_KIND_SWIFT_ERROR;
959 case Attribute::SwiftSelf:
960 return bitc::ATTR_KIND_SWIFT_SELF;
961 case Attribute::SwiftAsync:
962 return bitc::ATTR_KIND_SWIFT_ASYNC;
963 case Attribute::UWTable:
964 return bitc::ATTR_KIND_UW_TABLE;
965 case Attribute::VScaleRange:
966 return bitc::ATTR_KIND_VSCALE_RANGE;
967 case Attribute::WillReturn:
968 return bitc::ATTR_KIND_WILLRETURN;
969 case Attribute::WriteOnly:
970 return bitc::ATTR_KIND_WRITEONLY;
971 case Attribute::ZExt:
972 return bitc::ATTR_KIND_Z_EXT;
973 case Attribute::ImmArg:
974 return bitc::ATTR_KIND_IMMARG;
975 case Attribute::SanitizeMemTag:
976 return bitc::ATTR_KIND_SANITIZE_MEMTAG;
977 case Attribute::Preallocated:
978 return bitc::ATTR_KIND_PREALLOCATED;
979 case Attribute::NoUndef:
980 return bitc::ATTR_KIND_NOUNDEF;
981 case Attribute::ByRef:
982 return bitc::ATTR_KIND_BYREF;
983 case Attribute::MustProgress:
984 return bitc::ATTR_KIND_MUSTPROGRESS;
985 case Attribute::PresplitCoroutine:
986 return bitc::ATTR_KIND_PRESPLIT_COROUTINE;
987 case Attribute::Writable:
988 return bitc::ATTR_KIND_WRITABLE;
989 case Attribute::CoroDestroyOnlyWhenComplete:
990 return bitc::ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE;
991 case Attribute::CoroElideSafe:
992 return bitc::ATTR_KIND_CORO_ELIDE_SAFE;
993 case Attribute::DeadOnUnwind:
994 return bitc::ATTR_KIND_DEAD_ON_UNWIND;
995 case Attribute::Range:
996 return bitc::ATTR_KIND_RANGE;
997 case Attribute::Initializes:
998 return bitc::ATTR_KIND_INITIALIZES;
999 case Attribute::NoExt:
1000 return bitc::ATTR_KIND_NO_EXT;
1001 case Attribute::Captures:
1002 return bitc::ATTR_KIND_CAPTURES;
1003 case Attribute::DeadOnReturn:
1004 return bitc::ATTR_KIND_DEAD_ON_RETURN;
1005 case Attribute::NoCreateUndefOrPoison:
1006 return bitc::ATTR_KIND_NO_CREATE_UNDEF_OR_POISON;
1007 case Attribute::DenormalFPEnv:
1008 return bitc::ATTR_KIND_DENORMAL_FPENV;
1009 case Attribute::NoOutline:
1010 return bitc::ATTR_KIND_NOOUTLINE;
1011 case Attribute::NoIPA:
1012 return bitc::ATTR_KIND_NOIPA;
1013 case Attribute::EndAttrKinds:
1014 llvm_unreachable("Can not encode end-attribute kinds marker.");
1015 case Attribute::None:
1016 llvm_unreachable("Can not encode none-attribute.");
1017 case Attribute::EmptyKey:
1018 case Attribute::TombstoneKey:
1019 llvm_unreachable("Trying to encode EmptyKey/TombstoneKey");
1020 }
1021
1022 llvm_unreachable("Trying to encode unknown attribute");
1023}
1024
1025static void emitSignedInt64(SmallVectorImpl<uint64_t> &Vals, uint64_t V) {
1026 if ((int64_t)V >= 0)
1027 Vals.push_back(Elt: V << 1);
1028 else
1029 Vals.push_back(Elt: (-V << 1) | 1);
1030}
1031
1032static void emitWideAPInt(SmallVectorImpl<uint64_t> &Vals, const APInt &A) {
1033 // We have an arbitrary precision integer value to write whose
1034 // bit width is > 64. However, in canonical unsigned integer
1035 // format it is likely that the high bits are going to be zero.
1036 // So, we only write the number of active words.
1037 unsigned NumWords = A.getActiveWords();
1038 const uint64_t *RawData = A.getRawData();
1039 for (unsigned i = 0; i < NumWords; i++)
1040 emitSignedInt64(Vals, V: RawData[i]);
1041}
1042
1043static void emitConstantRange(SmallVectorImpl<uint64_t> &Record,
1044 const ConstantRange &CR, bool EmitBitWidth) {
1045 unsigned BitWidth = CR.getBitWidth();
1046 if (EmitBitWidth)
1047 Record.push_back(Elt: BitWidth);
1048 if (BitWidth > 64) {
1049 Record.push_back(Elt: CR.getLower().getActiveWords() |
1050 (uint64_t(CR.getUpper().getActiveWords()) << 32));
1051 emitWideAPInt(Vals&: Record, A: CR.getLower());
1052 emitWideAPInt(Vals&: Record, A: CR.getUpper());
1053 } else {
1054 emitSignedInt64(Vals&: Record, V: CR.getLower().getSExtValue());
1055 emitSignedInt64(Vals&: Record, V: CR.getUpper().getSExtValue());
1056 }
1057}
1058
1059void ModuleBitcodeWriter::writeAttributeGroupTable() {
1060 const std::vector<ValueEnumerator::IndexAndAttrSet> &AttrGrps =
1061 VE.getAttributeGroups();
1062 if (AttrGrps.empty()) return;
1063
1064 Stream.EnterSubblock(BlockID: bitc::PARAMATTR_GROUP_BLOCK_ID, CodeLen: 3);
1065
1066 SmallVector<uint64_t, 64> Record;
1067 for (ValueEnumerator::IndexAndAttrSet Pair : AttrGrps) {
1068 unsigned AttrListIndex = Pair.first;
1069 AttributeSet AS = Pair.second;
1070 Record.push_back(Elt: VE.getAttributeGroupID(Group: Pair));
1071 Record.push_back(Elt: AttrListIndex);
1072
1073 for (Attribute Attr : AS) {
1074 if (Attr.isEnumAttribute()) {
1075 Record.push_back(Elt: 0);
1076 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1077 } else if (Attr.isIntAttribute()) {
1078 Record.push_back(Elt: 1);
1079 Attribute::AttrKind Kind = Attr.getKindAsEnum();
1080 Record.push_back(Elt: getAttrKindEncoding(Kind));
1081 if (Kind == Attribute::Memory) {
1082 // Version field for upgrading old memory effects.
1083 const uint64_t Version = 1;
1084 Record.push_back(Elt: (Version << 56) | Attr.getValueAsInt());
1085 } else {
1086 Record.push_back(Elt: Attr.getValueAsInt());
1087 }
1088 } else if (Attr.isStringAttribute()) {
1089 StringRef Kind = Attr.getKindAsString();
1090 StringRef Val = Attr.getValueAsString();
1091
1092 Record.push_back(Elt: Val.empty() ? 3 : 4);
1093 Record.append(in_start: Kind.begin(), in_end: Kind.end());
1094 Record.push_back(Elt: 0);
1095 if (!Val.empty()) {
1096 Record.append(in_start: Val.begin(), in_end: Val.end());
1097 Record.push_back(Elt: 0);
1098 }
1099 } else if (Attr.isTypeAttribute()) {
1100 Type *Ty = Attr.getValueAsType();
1101 Record.push_back(Elt: Ty ? 6 : 5);
1102 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1103 if (Ty)
1104 Record.push_back(Elt: VE.getTypeID(T: Attr.getValueAsType()));
1105 } else if (Attr.isConstantRangeAttribute()) {
1106 Record.push_back(Elt: 7);
1107 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1108 emitConstantRange(Record, CR: Attr.getValueAsConstantRange(),
1109 /*EmitBitWidth=*/true);
1110 } else {
1111 assert(Attr.isConstantRangeListAttribute());
1112 Record.push_back(Elt: 8);
1113 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1114 ArrayRef<ConstantRange> Val = Attr.getValueAsConstantRangeList();
1115 Record.push_back(Elt: Val.size());
1116 Record.push_back(Elt: Val[0].getBitWidth());
1117 for (auto &CR : Val)
1118 emitConstantRange(Record, CR, /*EmitBitWidth=*/false);
1119 }
1120 }
1121
1122 Stream.EmitRecord(Code: bitc::PARAMATTR_GRP_CODE_ENTRY, Vals: Record);
1123 Record.clear();
1124 }
1125
1126 Stream.ExitBlock();
1127}
1128
1129void ModuleBitcodeWriter::writeAttributeTable() {
1130 const std::vector<AttributeList> &Attrs = VE.getAttributeLists();
1131 if (Attrs.empty()) return;
1132
1133 Stream.EnterSubblock(BlockID: bitc::PARAMATTR_BLOCK_ID, CodeLen: 3);
1134
1135 SmallVector<uint64_t, 64> Record;
1136 for (const AttributeList &AL : Attrs) {
1137 for (unsigned i : AL.indexes()) {
1138 AttributeSet AS = AL.getAttributes(Index: i);
1139 if (AS.hasAttributes())
1140 Record.push_back(Elt: VE.getAttributeGroupID(Group: {i, AS}));
1141 }
1142
1143 Stream.EmitRecord(Code: bitc::PARAMATTR_CODE_ENTRY, Vals: Record);
1144 Record.clear();
1145 }
1146
1147 Stream.ExitBlock();
1148}
1149
1150/// WriteTypeTable - Write out the type table for a module.
1151void ModuleBitcodeWriter::writeTypeTable() {
1152 const ValueEnumerator::TypeList &TypeList = VE.getTypes();
1153
1154 Stream.EnterSubblock(BlockID: bitc::TYPE_BLOCK_ID_NEW, CodeLen: 4 /*count from # abbrevs */);
1155 SmallVector<uint64_t, 64> TypeVals;
1156
1157 uint64_t NumBits = VE.computeBitsRequiredForTypeIndices();
1158
1159 // Abbrev for TYPE_CODE_OPAQUE_POINTER.
1160 auto Abbv = std::make_shared<BitCodeAbbrev>();
1161 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_OPAQUE_POINTER));
1162 Abbv->Add(OpInfo: BitCodeAbbrevOp(0)); // Addrspace = 0
1163 unsigned OpaquePtrAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1164
1165 // Abbrev for TYPE_CODE_FUNCTION.
1166 Abbv = std::make_shared<BitCodeAbbrev>();
1167 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_FUNCTION));
1168 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isvararg
1169 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1170 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1171 unsigned FunctionAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1172
1173 // Abbrev for TYPE_CODE_STRUCT_ANON.
1174 Abbv = std::make_shared<BitCodeAbbrev>();
1175 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_ANON));
1176 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked
1177 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1178 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1179 unsigned StructAnonAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1180
1181 // Abbrev for TYPE_CODE_STRUCT_NAME.
1182 Abbv = std::make_shared<BitCodeAbbrev>();
1183 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAME));
1184 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1185 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
1186 unsigned StructNameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1187
1188 // Abbrev for TYPE_CODE_STRUCT_NAMED.
1189 Abbv = std::make_shared<BitCodeAbbrev>();
1190 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAMED));
1191 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked
1192 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1193 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1194 unsigned StructNamedAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1195
1196 // Abbrev for TYPE_CODE_ARRAY.
1197 Abbv = std::make_shared<BitCodeAbbrev>();
1198 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_ARRAY));
1199 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // size
1200 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1201 unsigned ArrayAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1202
1203 // Emit an entry count so the reader can reserve space.
1204 TypeVals.push_back(Elt: TypeList.size());
1205 Stream.EmitRecord(Code: bitc::TYPE_CODE_NUMENTRY, Vals: TypeVals);
1206 TypeVals.clear();
1207
1208 // Loop over all of the types, emitting each in turn.
1209 for (Type *T : TypeList) {
1210 int AbbrevToUse = 0;
1211 unsigned Code = 0;
1212
1213 switch (T->getTypeID()) {
1214 case Type::VoidTyID: Code = bitc::TYPE_CODE_VOID; break;
1215 case Type::HalfTyID: Code = bitc::TYPE_CODE_HALF; break;
1216 case Type::BFloatTyID: Code = bitc::TYPE_CODE_BFLOAT; break;
1217 case Type::FloatTyID: Code = bitc::TYPE_CODE_FLOAT; break;
1218 case Type::DoubleTyID: Code = bitc::TYPE_CODE_DOUBLE; break;
1219 case Type::X86_FP80TyID: Code = bitc::TYPE_CODE_X86_FP80; break;
1220 case Type::FP128TyID: Code = bitc::TYPE_CODE_FP128; break;
1221 case Type::PPC_FP128TyID: Code = bitc::TYPE_CODE_PPC_FP128; break;
1222 case Type::LabelTyID: Code = bitc::TYPE_CODE_LABEL; break;
1223 case Type::MetadataTyID:
1224 Code = bitc::TYPE_CODE_METADATA;
1225 break;
1226 case Type::X86_AMXTyID: Code = bitc::TYPE_CODE_X86_AMX; break;
1227 case Type::TokenTyID: Code = bitc::TYPE_CODE_TOKEN; break;
1228 case Type::ByteTyID:
1229 // BYTE: [width]
1230 Code = bitc::TYPE_CODE_BYTE;
1231 TypeVals.push_back(Elt: T->getByteBitWidth());
1232 break;
1233 case Type::IntegerTyID:
1234 // INTEGER: [width]
1235 Code = bitc::TYPE_CODE_INTEGER;
1236 TypeVals.push_back(Elt: cast<IntegerType>(Val: T)->getBitWidth());
1237 break;
1238 case Type::PointerTyID: {
1239 PointerType *PTy = cast<PointerType>(Val: T);
1240 unsigned AddressSpace = PTy->getAddressSpace();
1241 // OPAQUE_POINTER: [address space]
1242 Code = bitc::TYPE_CODE_OPAQUE_POINTER;
1243 TypeVals.push_back(Elt: AddressSpace);
1244 if (AddressSpace == 0)
1245 AbbrevToUse = OpaquePtrAbbrev;
1246 break;
1247 }
1248 case Type::FunctionTyID: {
1249 FunctionType *FT = cast<FunctionType>(Val: T);
1250 // FUNCTION: [isvararg, retty, paramty x N]
1251 Code = bitc::TYPE_CODE_FUNCTION;
1252 TypeVals.push_back(Elt: FT->isVarArg());
1253 TypeVals.push_back(Elt: VE.getTypeID(T: FT->getReturnType()));
1254 for (unsigned i = 0, e = FT->getNumParams(); i != e; ++i)
1255 TypeVals.push_back(Elt: VE.getTypeID(T: FT->getParamType(i)));
1256 AbbrevToUse = FunctionAbbrev;
1257 break;
1258 }
1259 case Type::StructTyID: {
1260 StructType *ST = cast<StructType>(Val: T);
1261 // STRUCT: [ispacked, eltty x N]
1262 TypeVals.push_back(Elt: ST->isPacked());
1263 // Output all of the element types.
1264 for (Type *ET : ST->elements())
1265 TypeVals.push_back(Elt: VE.getTypeID(T: ET));
1266
1267 if (ST->isLiteral()) {
1268 Code = bitc::TYPE_CODE_STRUCT_ANON;
1269 AbbrevToUse = StructAnonAbbrev;
1270 } else {
1271 if (ST->isOpaque()) {
1272 Code = bitc::TYPE_CODE_OPAQUE;
1273 } else {
1274 Code = bitc::TYPE_CODE_STRUCT_NAMED;
1275 AbbrevToUse = StructNamedAbbrev;
1276 }
1277
1278 // Emit the name if it is present.
1279 if (!ST->getName().empty())
1280 writeStringRecord(Stream, Code: bitc::TYPE_CODE_STRUCT_NAME, Str: ST->getName(),
1281 AbbrevToUse: StructNameAbbrev);
1282 }
1283 break;
1284 }
1285 case Type::ArrayTyID: {
1286 ArrayType *AT = cast<ArrayType>(Val: T);
1287 // ARRAY: [numelts, eltty]
1288 Code = bitc::TYPE_CODE_ARRAY;
1289 TypeVals.push_back(Elt: AT->getNumElements());
1290 TypeVals.push_back(Elt: VE.getTypeID(T: AT->getElementType()));
1291 AbbrevToUse = ArrayAbbrev;
1292 break;
1293 }
1294 case Type::FixedVectorTyID:
1295 case Type::ScalableVectorTyID: {
1296 VectorType *VT = cast<VectorType>(Val: T);
1297 // VECTOR [numelts, eltty] or
1298 // [numelts, eltty, scalable]
1299 Code = bitc::TYPE_CODE_VECTOR;
1300 TypeVals.push_back(Elt: VT->getElementCount().getKnownMinValue());
1301 TypeVals.push_back(Elt: VE.getTypeID(T: VT->getElementType()));
1302 if (isa<ScalableVectorType>(Val: VT))
1303 TypeVals.push_back(Elt: true);
1304 break;
1305 }
1306 case Type::TargetExtTyID: {
1307 TargetExtType *TET = cast<TargetExtType>(Val: T);
1308 Code = bitc::TYPE_CODE_TARGET_TYPE;
1309 writeStringRecord(Stream, Code: bitc::TYPE_CODE_STRUCT_NAME, Str: TET->getName(),
1310 AbbrevToUse: StructNameAbbrev);
1311 TypeVals.push_back(Elt: TET->getNumTypeParameters());
1312 for (Type *InnerTy : TET->type_params())
1313 TypeVals.push_back(Elt: VE.getTypeID(T: InnerTy));
1314 llvm::append_range(C&: TypeVals, R: TET->int_params());
1315 break;
1316 }
1317 case Type::TypedPointerTyID:
1318 llvm_unreachable("Typed pointers cannot be added to IR modules");
1319 }
1320
1321 // Emit the finished record.
1322 Stream.EmitRecord(Code, Vals: TypeVals, Abbrev: AbbrevToUse);
1323 TypeVals.clear();
1324 }
1325
1326 Stream.ExitBlock();
1327}
1328
1329static unsigned getEncodedLinkage(const GlobalValue::LinkageTypes Linkage) {
1330 switch (Linkage) {
1331 case GlobalValue::ExternalLinkage:
1332 return 0;
1333 case GlobalValue::WeakAnyLinkage:
1334 return 16;
1335 case GlobalValue::AppendingLinkage:
1336 return 2;
1337 case GlobalValue::InternalLinkage:
1338 return 3;
1339 case GlobalValue::LinkOnceAnyLinkage:
1340 return 18;
1341 case GlobalValue::ExternalWeakLinkage:
1342 return 7;
1343 case GlobalValue::CommonLinkage:
1344 return 8;
1345 case GlobalValue::PrivateLinkage:
1346 return 9;
1347 case GlobalValue::WeakODRLinkage:
1348 return 17;
1349 case GlobalValue::LinkOnceODRLinkage:
1350 return 19;
1351 case GlobalValue::AvailableExternallyLinkage:
1352 return 12;
1353 }
1354 llvm_unreachable("Invalid linkage");
1355}
1356
1357static unsigned getEncodedLinkage(const GlobalValue &GV) {
1358 return getEncodedLinkage(Linkage: GV.getLinkage());
1359}
1360
1361static uint64_t getEncodedFFlags(FunctionSummary::FFlags Flags) {
1362 uint64_t RawFlags = 0;
1363 RawFlags |= Flags.ReadNone;
1364 RawFlags |= (Flags.ReadOnly << 1);
1365 RawFlags |= (Flags.NoRecurse << 2);
1366 RawFlags |= (Flags.ReturnDoesNotAlias << 3);
1367 RawFlags |= (Flags.NoInline << 4);
1368 RawFlags |= (Flags.AlwaysInline << 5);
1369 RawFlags |= (Flags.NoUnwind << 6);
1370 RawFlags |= (Flags.MayThrow << 7);
1371 RawFlags |= (Flags.HasUnknownCall << 8);
1372 RawFlags |= (Flags.MustBeUnreachable << 9);
1373 return RawFlags;
1374}
1375
1376// Decode the flags for GlobalValue in the summary. See getDecodedGVSummaryFlags
1377// in BitcodeReader.cpp.
1378static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags,
1379 bool ImportAsDecl = false) {
1380 uint64_t RawFlags = 0;
1381
1382 RawFlags |= Flags.NotEligibleToImport; // bool
1383 RawFlags |= (Flags.Live << 1);
1384 RawFlags |= (Flags.DSOLocal << 2);
1385 RawFlags |= (Flags.CanAutoHide << 3);
1386
1387 // Linkage don't need to be remapped at that time for the summary. Any future
1388 // change to the getEncodedLinkage() function will need to be taken into
1389 // account here as well.
1390 RawFlags = (RawFlags << 4) | Flags.Linkage; // 4 bits
1391
1392 RawFlags |= (Flags.Visibility << 8); // 2 bits
1393
1394 unsigned ImportType = Flags.ImportType | ImportAsDecl;
1395 RawFlags |= (ImportType << 10); // 1 bit
1396
1397 RawFlags |= (Flags.NoRenameOnPromotion << 11); // 1 bit
1398
1399 return RawFlags;
1400}
1401
1402static uint64_t getEncodedGVarFlags(GlobalVarSummary::GVarFlags Flags) {
1403 uint64_t RawFlags = Flags.MaybeReadOnly | (Flags.MaybeWriteOnly << 1) |
1404 (Flags.Constant << 2) | Flags.VCallVisibility << 3;
1405 return RawFlags;
1406}
1407
1408static uint64_t getEncodedHotnessCallEdgeInfo(const CalleeInfo &CI) {
1409 uint64_t RawFlags = 0;
1410
1411 RawFlags |= CI.Hotness; // 3 bits
1412 RawFlags |= (CI.HasTailCall << 3); // 1 bit
1413
1414 return RawFlags;
1415}
1416
1417static unsigned getEncodedVisibility(const GlobalValue &GV) {
1418 switch (GV.getVisibility()) {
1419 case GlobalValue::DefaultVisibility: return 0;
1420 case GlobalValue::HiddenVisibility: return 1;
1421 case GlobalValue::ProtectedVisibility: return 2;
1422 }
1423 llvm_unreachable("Invalid visibility");
1424}
1425
1426static unsigned getEncodedDLLStorageClass(const GlobalValue &GV) {
1427 switch (GV.getDLLStorageClass()) {
1428 case GlobalValue::DefaultStorageClass: return 0;
1429 case GlobalValue::DLLImportStorageClass: return 1;
1430 case GlobalValue::DLLExportStorageClass: return 2;
1431 }
1432 llvm_unreachable("Invalid DLL storage class");
1433}
1434
1435static unsigned getEncodedThreadLocalMode(const GlobalValue &GV) {
1436 switch (GV.getThreadLocalMode()) {
1437 case GlobalVariable::NotThreadLocal: return 0;
1438 case GlobalVariable::GeneralDynamicTLSModel: return 1;
1439 case GlobalVariable::LocalDynamicTLSModel: return 2;
1440 case GlobalVariable::InitialExecTLSModel: return 3;
1441 case GlobalVariable::LocalExecTLSModel: return 4;
1442 }
1443 llvm_unreachable("Invalid TLS model");
1444}
1445
1446static unsigned getEncodedComdatSelectionKind(const Comdat &C) {
1447 switch (C.getSelectionKind()) {
1448 case Comdat::Any:
1449 return bitc::COMDAT_SELECTION_KIND_ANY;
1450 case Comdat::ExactMatch:
1451 return bitc::COMDAT_SELECTION_KIND_EXACT_MATCH;
1452 case Comdat::Largest:
1453 return bitc::COMDAT_SELECTION_KIND_LARGEST;
1454 case Comdat::NoDeduplicate:
1455 return bitc::COMDAT_SELECTION_KIND_NO_DUPLICATES;
1456 case Comdat::SameSize:
1457 return bitc::COMDAT_SELECTION_KIND_SAME_SIZE;
1458 }
1459 llvm_unreachable("Invalid selection kind");
1460}
1461
1462static unsigned getEncodedUnnamedAddr(const GlobalValue &GV) {
1463 switch (GV.getUnnamedAddr()) {
1464 case GlobalValue::UnnamedAddr::None: return 0;
1465 case GlobalValue::UnnamedAddr::Local: return 2;
1466 case GlobalValue::UnnamedAddr::Global: return 1;
1467 }
1468 llvm_unreachable("Invalid unnamed_addr");
1469}
1470
1471size_t ModuleBitcodeWriter::addToStrtab(StringRef Str) {
1472 if (GenerateHash)
1473 Hasher.update(Str);
1474 return StrtabBuilder.add(S: Str);
1475}
1476
1477void ModuleBitcodeWriter::writeComdats() {
1478 SmallVector<unsigned, 64> Vals;
1479 for (const Comdat *C : VE.getComdats()) {
1480 // COMDAT: [strtab offset, strtab size, selection_kind]
1481 Vals.push_back(Elt: addToStrtab(Str: C->getName()));
1482 Vals.push_back(Elt: C->getName().size());
1483 Vals.push_back(Elt: getEncodedComdatSelectionKind(C: *C));
1484 Stream.EmitRecord(Code: bitc::MODULE_CODE_COMDAT, Vals, /*AbbrevToUse=*/Abbrev: 0);
1485 Vals.clear();
1486 }
1487}
1488
1489/// Write a record that will eventually hold the word offset of the
1490/// module-level VST. For now the offset is 0, which will be backpatched
1491/// after the real VST is written. Saves the bit offset to backpatch.
1492void ModuleBitcodeWriter::writeValueSymbolTableForwardDecl() {
1493 // Write a placeholder value in for the offset of the real VST,
1494 // which is written after the function blocks so that it can include
1495 // the offset of each function. The placeholder offset will be
1496 // updated when the real VST is written.
1497 auto Abbv = std::make_shared<BitCodeAbbrev>();
1498 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_VSTOFFSET));
1499 // Blocks are 32-bit aligned, so we can use a 32-bit word offset to
1500 // hold the real VST offset. Must use fixed instead of VBR as we don't
1501 // know how many VBR chunks to reserve ahead of time.
1502 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
1503 unsigned VSTOffsetAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1504
1505 // Emit the placeholder
1506 uint64_t Vals[] = {bitc::MODULE_CODE_VSTOFFSET, 0};
1507 Stream.EmitRecordWithAbbrev(Abbrev: VSTOffsetAbbrev, Vals);
1508
1509 // Compute and save the bit offset to the placeholder, which will be
1510 // patched when the real VST is written. We can simply subtract the 32-bit
1511 // fixed size from the current bit number to get the location to backpatch.
1512 VSTOffsetPlaceholder = Stream.GetCurrentBitNo() - 32;
1513}
1514
1515enum StringEncoding { SE_Char6, SE_Fixed7, SE_Fixed8 };
1516
1517/// Determine the encoding to use for the given string name and length.
1518static StringEncoding getStringEncoding(StringRef Str) {
1519 bool isChar6 = true;
1520 for (char C : Str) {
1521 if (isChar6)
1522 isChar6 = BitCodeAbbrevOp::isChar6(C);
1523 if ((unsigned char)C & 128)
1524 // don't bother scanning the rest.
1525 return SE_Fixed8;
1526 }
1527 if (isChar6)
1528 return SE_Char6;
1529 return SE_Fixed7;
1530}
1531
1532static_assert(sizeof(GlobalValue::SanitizerMetadata) <= sizeof(unsigned),
1533 "Sanitizer Metadata is too large for naive serialization.");
1534static unsigned
1535serializeSanitizerMetadata(const GlobalValue::SanitizerMetadata &Meta) {
1536 return Meta.NoAddress | (Meta.NoHWAddress << 1) |
1537 (Meta.Memtag << 2) | (Meta.IsDynInit << 3);
1538}
1539
1540/// Emit top-level description of module, including target triple, inline asm,
1541/// descriptors for global variables, and function prototype info.
1542/// Returns the bit offset to backpatch with the location of the real VST.
1543void ModuleBitcodeWriter::writeModuleInfo() {
1544 // Emit various pieces of data attached to a module.
1545 if (!M.getTargetTriple().empty())
1546 writeStringRecord(Stream, Code: bitc::MODULE_CODE_TRIPLE,
1547 Str: M.getTargetTriple().str(), AbbrevToUse: 0 /*TODO*/);
1548 const std::string &DL = M.getDataLayoutStr();
1549 if (!DL.empty())
1550 writeStringRecord(Stream, Code: bitc::MODULE_CODE_DATALAYOUT, Str: DL, AbbrevToUse: 0 /*TODO*/);
1551 if (!M.getModuleInlineAsm().empty())
1552 writeStringRecord(Stream, Code: bitc::MODULE_CODE_ASM, Str: M.getModuleInlineAsm(),
1553 AbbrevToUse: 0 /*TODO*/);
1554
1555 // Emit information about sections and GC, computing how many there are. Also
1556 // compute the maximum alignment value.
1557 std::map<std::string, unsigned> SectionMap;
1558 std::map<std::string, unsigned> GCMap;
1559 MaybeAlign MaxGVarAlignment;
1560 unsigned MaxGlobalType = 0;
1561 for (const GlobalVariable &GV : M.globals()) {
1562 if (MaybeAlign A = GV.getAlign())
1563 MaxGVarAlignment = !MaxGVarAlignment ? *A : std::max(a: *MaxGVarAlignment, b: *A);
1564 MaxGlobalType = std::max(a: MaxGlobalType, b: VE.getTypeID(T: GV.getValueType()));
1565 if (GV.hasSection()) {
1566 // Give section names unique ID's.
1567 unsigned &Entry = SectionMap[std::string(GV.getSection())];
1568 if (!Entry) {
1569 writeStringRecord(Stream, Code: bitc::MODULE_CODE_SECTIONNAME, Str: GV.getSection(),
1570 AbbrevToUse: 0 /*TODO*/);
1571 Entry = SectionMap.size();
1572 }
1573 }
1574 }
1575 for (const Function &F : M) {
1576 if (F.hasSection()) {
1577 // Give section names unique ID's.
1578 unsigned &Entry = SectionMap[std::string(F.getSection())];
1579 if (!Entry) {
1580 writeStringRecord(Stream, Code: bitc::MODULE_CODE_SECTIONNAME, Str: F.getSection(),
1581 AbbrevToUse: 0 /*TODO*/);
1582 Entry = SectionMap.size();
1583 }
1584 }
1585 if (F.hasGC()) {
1586 // Same for GC names.
1587 unsigned &Entry = GCMap[F.getGC()];
1588 if (!Entry) {
1589 writeStringRecord(Stream, Code: bitc::MODULE_CODE_GCNAME, Str: F.getGC(),
1590 AbbrevToUse: 0 /*TODO*/);
1591 Entry = GCMap.size();
1592 }
1593 }
1594 }
1595
1596 // Emit abbrev for globals, now that we know # sections and max alignment.
1597 unsigned SimpleGVarAbbrev = 0;
1598 if (!M.global_empty()) {
1599 // Add an abbrev for common globals with no visibility or thread localness.
1600 auto Abbv = std::make_shared<BitCodeAbbrev>();
1601 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_GLOBALVAR));
1602 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1603 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1604 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1605 Log2_32_Ceil(Value: MaxGlobalType+1)));
1606 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // AddrSpace << 2
1607 //| explicitType << 1
1608 //| constant
1609 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Initializer.
1610 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // Linkage.
1611 if (!MaxGVarAlignment) // Alignment.
1612 Abbv->Add(OpInfo: BitCodeAbbrevOp(0));
1613 else {
1614 unsigned MaxEncAlignment = getEncodedAlign(Alignment: MaxGVarAlignment);
1615 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1616 Log2_32_Ceil(Value: MaxEncAlignment+1)));
1617 }
1618 if (SectionMap.empty()) // Section.
1619 Abbv->Add(OpInfo: BitCodeAbbrevOp(0));
1620 else
1621 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1622 Log2_32_Ceil(Value: SectionMap.size()+1)));
1623 // Don't bother emitting vis + thread local.
1624 SimpleGVarAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1625 }
1626
1627 SmallVector<unsigned, 64> Vals;
1628 // Emit the module's source file name.
1629 {
1630 StringEncoding Bits = getStringEncoding(Str: M.getSourceFileName());
1631 BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
1632 if (Bits == SE_Char6)
1633 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
1634 else if (Bits == SE_Fixed7)
1635 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
1636
1637 // MODULE_CODE_SOURCE_FILENAME: [namechar x N]
1638 auto Abbv = std::make_shared<BitCodeAbbrev>();
1639 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
1640 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1641 Abbv->Add(OpInfo: AbbrevOpToUse);
1642 unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1643
1644 for (const auto P : M.getSourceFileName())
1645 Vals.push_back(Elt: (unsigned char)P);
1646
1647 // Emit the finished record.
1648 Stream.EmitRecord(Code: bitc::MODULE_CODE_SOURCE_FILENAME, Vals, Abbrev: FilenameAbbrev);
1649 Vals.clear();
1650 }
1651
1652 // Emit the global variable information.
1653 for (const GlobalVariable &GV : M.globals()) {
1654 unsigned AbbrevToUse = 0;
1655
1656 // GLOBALVAR: [strtab offset, strtab size, type, isconst, initid,
1657 // linkage, alignment, section, visibility, threadlocal,
1658 // unnamed_addr, externally_initialized, dllstorageclass,
1659 // comdat, attributes, DSO_Local, GlobalSanitizer, code_model]
1660 Vals.push_back(Elt: addToStrtab(Str: GV.getName()));
1661 Vals.push_back(Elt: GV.getName().size());
1662 Vals.push_back(Elt: VE.getTypeID(T: GV.getValueType()));
1663 Vals.push_back(Elt: GV.getType()->getAddressSpace() << 2 | 2 | GV.isConstant());
1664 Vals.push_back(Elt: GV.isDeclaration() ? 0 :
1665 (VE.getValueID(V: GV.getInitializer()) + 1));
1666 Vals.push_back(Elt: getEncodedLinkage(GV));
1667 Vals.push_back(Elt: getEncodedAlign(Alignment: GV.getAlign()));
1668 Vals.push_back(Elt: GV.hasSection() ? SectionMap[std::string(GV.getSection())]
1669 : 0);
1670 if (GV.isThreadLocal() ||
1671 GV.getVisibility() != GlobalValue::DefaultVisibility ||
1672 GV.getUnnamedAddr() != GlobalValue::UnnamedAddr::None ||
1673 GV.isExternallyInitialized() ||
1674 GV.getDLLStorageClass() != GlobalValue::DefaultStorageClass ||
1675 GV.hasComdat() || GV.hasAttributes() || GV.isDSOLocal() ||
1676 GV.hasPartition() || GV.hasSanitizerMetadata() || GV.getCodeModel()) {
1677 Vals.push_back(Elt: getEncodedVisibility(GV));
1678 Vals.push_back(Elt: getEncodedThreadLocalMode(GV));
1679 Vals.push_back(Elt: getEncodedUnnamedAddr(GV));
1680 Vals.push_back(Elt: GV.isExternallyInitialized());
1681 Vals.push_back(Elt: getEncodedDLLStorageClass(GV));
1682 Vals.push_back(Elt: GV.hasComdat() ? VE.getComdatID(C: GV.getComdat()) : 0);
1683
1684 auto AL = GV.getAttributesAsList(index: AttributeList::FunctionIndex);
1685 Vals.push_back(Elt: VE.getAttributeListID(PAL: AL));
1686
1687 Vals.push_back(Elt: GV.isDSOLocal());
1688 Vals.push_back(Elt: addToStrtab(Str: GV.getPartition()));
1689 Vals.push_back(Elt: GV.getPartition().size());
1690
1691 Vals.push_back(Elt: (GV.hasSanitizerMetadata() ? serializeSanitizerMetadata(
1692 Meta: GV.getSanitizerMetadata())
1693 : 0));
1694 Vals.push_back(Elt: GV.getCodeModelRaw());
1695 } else {
1696 AbbrevToUse = SimpleGVarAbbrev;
1697 }
1698
1699 Stream.EmitRecord(Code: bitc::MODULE_CODE_GLOBALVAR, Vals, Abbrev: AbbrevToUse);
1700 Vals.clear();
1701 }
1702
1703 // Emit the function proto information.
1704 for (const Function &F : M) {
1705 // FUNCTION: [strtab offset, strtab size, type, callingconv, isproto,
1706 // linkage, paramattrs, alignment, section, visibility, gc,
1707 // unnamed_addr, prologuedata, dllstorageclass, comdat,
1708 // prefixdata, personalityfn, DSO_Local, addrspace,
1709 // partition_strtab, partition_size, prefalign]
1710 Vals.push_back(Elt: addToStrtab(Str: F.getName()));
1711 Vals.push_back(Elt: F.getName().size());
1712 Vals.push_back(Elt: VE.getTypeID(T: F.getFunctionType()));
1713 Vals.push_back(Elt: F.getCallingConv());
1714 Vals.push_back(Elt: F.isDeclaration());
1715 Vals.push_back(Elt: getEncodedLinkage(GV: F));
1716 Vals.push_back(Elt: VE.getAttributeListID(PAL: F.getAttributes()));
1717 Vals.push_back(Elt: getEncodedAlign(Alignment: F.getAlign()));
1718 Vals.push_back(Elt: F.hasSection() ? SectionMap[std::string(F.getSection())]
1719 : 0);
1720 Vals.push_back(Elt: getEncodedVisibility(GV: F));
1721 Vals.push_back(Elt: F.hasGC() ? GCMap[F.getGC()] : 0);
1722 Vals.push_back(Elt: getEncodedUnnamedAddr(GV: F));
1723 Vals.push_back(Elt: F.hasPrologueData() ? (VE.getValueID(V: F.getPrologueData()) + 1)
1724 : 0);
1725 Vals.push_back(Elt: getEncodedDLLStorageClass(GV: F));
1726 Vals.push_back(Elt: F.hasComdat() ? VE.getComdatID(C: F.getComdat()) : 0);
1727 Vals.push_back(Elt: F.hasPrefixData() ? (VE.getValueID(V: F.getPrefixData()) + 1)
1728 : 0);
1729 Vals.push_back(
1730 Elt: F.hasPersonalityFn() ? (VE.getValueID(V: F.getPersonalityFn()) + 1) : 0);
1731
1732 Vals.push_back(Elt: F.isDSOLocal());
1733 Vals.push_back(Elt: F.getAddressSpace());
1734 Vals.push_back(Elt: addToStrtab(Str: F.getPartition()));
1735 Vals.push_back(Elt: F.getPartition().size());
1736 Vals.push_back(Elt: getEncodedAlign(Alignment: F.getPreferredAlignment()));
1737
1738 unsigned AbbrevToUse = 0;
1739 Stream.EmitRecord(Code: bitc::MODULE_CODE_FUNCTION, Vals, Abbrev: AbbrevToUse);
1740 Vals.clear();
1741 }
1742
1743 // Emit the alias information.
1744 for (const GlobalAlias &A : M.aliases()) {
1745 // ALIAS: [strtab offset, strtab size, alias type, aliasee val#, linkage,
1746 // visibility, dllstorageclass, threadlocal, unnamed_addr,
1747 // DSO_Local]
1748 Vals.push_back(Elt: addToStrtab(Str: A.getName()));
1749 Vals.push_back(Elt: A.getName().size());
1750 Vals.push_back(Elt: VE.getTypeID(T: A.getValueType()));
1751 Vals.push_back(Elt: A.getType()->getAddressSpace());
1752 Vals.push_back(Elt: VE.getValueID(V: A.getAliasee()));
1753 Vals.push_back(Elt: getEncodedLinkage(GV: A));
1754 Vals.push_back(Elt: getEncodedVisibility(GV: A));
1755 Vals.push_back(Elt: getEncodedDLLStorageClass(GV: A));
1756 Vals.push_back(Elt: getEncodedThreadLocalMode(GV: A));
1757 Vals.push_back(Elt: getEncodedUnnamedAddr(GV: A));
1758 Vals.push_back(Elt: A.isDSOLocal());
1759 Vals.push_back(Elt: addToStrtab(Str: A.getPartition()));
1760 Vals.push_back(Elt: A.getPartition().size());
1761
1762 unsigned AbbrevToUse = 0;
1763 Stream.EmitRecord(Code: bitc::MODULE_CODE_ALIAS, Vals, Abbrev: AbbrevToUse);
1764 Vals.clear();
1765 }
1766
1767 // Emit the ifunc information.
1768 for (const GlobalIFunc &I : M.ifuncs()) {
1769 // IFUNC: [strtab offset, strtab size, ifunc type, address space, resolver
1770 // val#, linkage, visibility, DSO_Local]
1771 Vals.push_back(Elt: addToStrtab(Str: I.getName()));
1772 Vals.push_back(Elt: I.getName().size());
1773 Vals.push_back(Elt: VE.getTypeID(T: I.getValueType()));
1774 Vals.push_back(Elt: I.getType()->getAddressSpace());
1775 Vals.push_back(Elt: VE.getValueID(V: I.getResolver()));
1776 Vals.push_back(Elt: getEncodedLinkage(GV: I));
1777 Vals.push_back(Elt: getEncodedVisibility(GV: I));
1778 Vals.push_back(Elt: I.isDSOLocal());
1779 Vals.push_back(Elt: addToStrtab(Str: I.getPartition()));
1780 Vals.push_back(Elt: I.getPartition().size());
1781 Stream.EmitRecord(Code: bitc::MODULE_CODE_IFUNC, Vals);
1782 Vals.clear();
1783 }
1784
1785 writeValueSymbolTableForwardDecl();
1786}
1787
1788static uint64_t getOptimizationFlags(const Value *V) {
1789 uint64_t Flags = 0;
1790
1791 if (const auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V)) {
1792 if (OBO->hasNoSignedWrap())
1793 Flags |= 1 << bitc::OBO_NO_SIGNED_WRAP;
1794 if (OBO->hasNoUnsignedWrap())
1795 Flags |= 1 << bitc::OBO_NO_UNSIGNED_WRAP;
1796 } else if (const auto *PEO = dyn_cast<PossiblyExactOperator>(Val: V)) {
1797 if (PEO->isExact())
1798 Flags |= 1 << bitc::PEO_EXACT;
1799 } else if (const auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: V)) {
1800 if (PDI->isDisjoint())
1801 Flags |= 1 << bitc::PDI_DISJOINT;
1802 } else if (const auto *FPMO = dyn_cast<FPMathOperator>(Val: V)) {
1803 if (FPMO->hasAllowReassoc())
1804 Flags |= bitc::AllowReassoc;
1805 if (FPMO->hasNoNaNs())
1806 Flags |= bitc::NoNaNs;
1807 if (FPMO->hasNoInfs())
1808 Flags |= bitc::NoInfs;
1809 if (FPMO->hasNoSignedZeros())
1810 Flags |= bitc::NoSignedZeros;
1811 if (FPMO->hasAllowReciprocal())
1812 Flags |= bitc::AllowReciprocal;
1813 if (FPMO->hasAllowContract())
1814 Flags |= bitc::AllowContract;
1815 if (FPMO->hasApproxFunc())
1816 Flags |= bitc::ApproxFunc;
1817
1818 // Handle uitofp.
1819 if (const auto *NNI = dyn_cast<PossiblyNonNegInst>(Val: V)) {
1820 Flags <<= 1;
1821 if (NNI->hasNonNeg())
1822 Flags |= 1 << bitc::PNNI_NON_NEG;
1823 }
1824 } else if (const auto *NNI = dyn_cast<PossiblyNonNegInst>(Val: V)) {
1825 if (NNI->hasNonNeg())
1826 Flags |= 1 << bitc::PNNI_NON_NEG;
1827 } else if (const auto *TI = dyn_cast<TruncInst>(Val: V)) {
1828 if (TI->hasNoSignedWrap())
1829 Flags |= 1 << bitc::TIO_NO_SIGNED_WRAP;
1830 if (TI->hasNoUnsignedWrap())
1831 Flags |= 1 << bitc::TIO_NO_UNSIGNED_WRAP;
1832 } else if (const auto *GEP = dyn_cast<GEPOperator>(Val: V)) {
1833 if (GEP->isInBounds())
1834 Flags |= 1 << bitc::GEP_INBOUNDS;
1835 if (GEP->hasNoUnsignedSignedWrap())
1836 Flags |= 1 << bitc::GEP_NUSW;
1837 if (GEP->hasNoUnsignedWrap())
1838 Flags |= 1 << bitc::GEP_NUW;
1839 } else if (const auto *ICmp = dyn_cast<ICmpInst>(Val: V)) {
1840 if (ICmp->hasSameSign())
1841 Flags |= 1 << bitc::ICMP_SAME_SIGN;
1842 }
1843
1844 return Flags;
1845}
1846
1847void ModuleBitcodeWriter::writeValueAsMetadata(
1848 const ValueAsMetadata *MD, SmallVectorImpl<uint64_t> &Record) {
1849 // Mimic an MDNode with a value as one operand.
1850 Value *V = MD->getValue();
1851 Record.push_back(Elt: VE.getTypeID(T: V->getType()));
1852 Record.push_back(Elt: VE.getValueID(V));
1853 Stream.EmitRecord(Code: bitc::METADATA_VALUE, Vals: Record, Abbrev: 0);
1854 Record.clear();
1855}
1856
1857void ModuleBitcodeWriter::writeMDTuple(const MDTuple *N,
1858 SmallVectorImpl<uint64_t> &Record,
1859 unsigned Abbrev) {
1860 for (const MDOperand &MDO : N->operands()) {
1861 Metadata *MD = MDO;
1862 assert(!(MD && isa<LocalAsMetadata>(MD)) &&
1863 "Unexpected function-local metadata");
1864 Record.push_back(Elt: VE.getMetadataOrNullID(MD));
1865 }
1866 Stream.EmitRecord(Code: N->isDistinct() ? bitc::METADATA_DISTINCT_NODE
1867 : bitc::METADATA_NODE,
1868 Vals: Record, Abbrev);
1869 Record.clear();
1870}
1871
1872unsigned ModuleBitcodeWriter::createDILocationAbbrev() {
1873 // Assume the column is usually under 128, and always output the inlined-at
1874 // location (it's never more expensive than building an array size 1).
1875 auto Abbv = std::make_shared<BitCodeAbbrev>();
1876 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_LOCATION));
1877 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isDistinct
1878 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // line
1879 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // column
1880 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // scope
1881 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // inlinedAt
1882 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isImplicitCode
1883 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // atomGroup
1884 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // atomRank
1885 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
1886}
1887
1888void ModuleBitcodeWriter::writeDILocation(const DILocation *N,
1889 SmallVectorImpl<uint64_t> &Record,
1890 unsigned &Abbrev) {
1891 if (!Abbrev)
1892 Abbrev = createDILocationAbbrev();
1893
1894 Record.push_back(Elt: N->isDistinct());
1895 Record.push_back(Elt: N->getLine());
1896 Record.push_back(Elt: N->getColumn());
1897 Record.push_back(Elt: VE.getMetadataID(MD: N->getScope()));
1898 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getInlinedAt()));
1899 Record.push_back(Elt: N->isImplicitCode());
1900 Record.push_back(Elt: N->getAtomGroup());
1901 Record.push_back(Elt: N->getAtomRank());
1902 Stream.EmitRecord(Code: bitc::METADATA_LOCATION, Vals: Record, Abbrev);
1903 Record.clear();
1904}
1905
1906unsigned ModuleBitcodeWriter::createGenericDINodeAbbrev() {
1907 // Assume the column is usually under 128, and always output the inlined-at
1908 // location (it's never more expensive than building an array size 1).
1909 auto Abbv = std::make_shared<BitCodeAbbrev>();
1910 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_GENERIC_DEBUG));
1911 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1912 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1913 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1914 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1915 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1916 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1917 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
1918}
1919
1920void ModuleBitcodeWriter::writeGenericDINode(const GenericDINode *N,
1921 SmallVectorImpl<uint64_t> &Record,
1922 unsigned &Abbrev) {
1923 if (!Abbrev)
1924 Abbrev = createGenericDINodeAbbrev();
1925
1926 Record.push_back(Elt: N->isDistinct());
1927 Record.push_back(Elt: N->getTag());
1928 Record.push_back(Elt: 0); // Per-tag version field; unused for now.
1929
1930 for (auto &I : N->operands())
1931 Record.push_back(Elt: VE.getMetadataOrNullID(MD: I));
1932
1933 Stream.EmitRecord(Code: bitc::METADATA_GENERIC_DEBUG, Vals: Record, Abbrev);
1934 Record.clear();
1935}
1936
1937void ModuleBitcodeWriter::writeDISubrange(const DISubrange *N,
1938 SmallVectorImpl<uint64_t> &Record,
1939 unsigned Abbrev) {
1940 const uint64_t Version = 2 << 1;
1941 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
1942 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawCountNode()));
1943 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
1944 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
1945 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
1946
1947 Stream.EmitRecord(Code: bitc::METADATA_SUBRANGE, Vals: Record, Abbrev);
1948 Record.clear();
1949}
1950
1951void ModuleBitcodeWriter::writeDIGenericSubrange(
1952 const DIGenericSubrange *N, SmallVectorImpl<uint64_t> &Record,
1953 unsigned Abbrev) {
1954 Record.push_back(Elt: (uint64_t)N->isDistinct());
1955 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawCountNode()));
1956 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
1957 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
1958 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
1959
1960 Stream.EmitRecord(Code: bitc::METADATA_GENERIC_SUBRANGE, Vals: Record, Abbrev);
1961 Record.clear();
1962}
1963
1964void ModuleBitcodeWriter::writeDIEnumerator(const DIEnumerator *N,
1965 SmallVectorImpl<uint64_t> &Record,
1966 unsigned Abbrev) {
1967 const uint64_t IsBigInt = 1 << 2;
1968 Record.push_back(Elt: IsBigInt | (N->isUnsigned() << 1) | N->isDistinct());
1969 Record.push_back(Elt: N->getValue().getBitWidth());
1970 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1971 emitWideAPInt(Vals&: Record, A: N->getValue());
1972
1973 Stream.EmitRecord(Code: bitc::METADATA_ENUMERATOR, Vals: Record, Abbrev);
1974 Record.clear();
1975}
1976
1977void ModuleBitcodeWriter::writeDIBasicType(const DIBasicType *N,
1978 SmallVectorImpl<uint64_t> &Record,
1979 unsigned Abbrev) {
1980 const unsigned SizeIsMetadata = 0x2;
1981 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
1982 Record.push_back(Elt: N->getTag());
1983 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1984 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
1985 Record.push_back(Elt: N->getAlignInBits());
1986 Record.push_back(Elt: N->getEncoding());
1987 Record.push_back(Elt: N->getFlags());
1988 Record.push_back(Elt: N->getNumExtraInhabitants());
1989 Record.push_back(Elt: N->getDataSizeInBits());
1990 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
1991 Record.push_back(Elt: N->getLine());
1992 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
1993
1994 Stream.EmitRecord(Code: bitc::METADATA_BASIC_TYPE, Vals: Record, Abbrev);
1995 Record.clear();
1996}
1997
1998void ModuleBitcodeWriter::writeDIFixedPointType(
1999 const DIFixedPointType *N, SmallVectorImpl<uint64_t> &Record,
2000 unsigned Abbrev) {
2001 const unsigned SizeIsMetadata = 0x2;
2002 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2003 Record.push_back(Elt: N->getTag());
2004 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2005 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2006 Record.push_back(Elt: N->getAlignInBits());
2007 Record.push_back(Elt: N->getEncoding());
2008 Record.push_back(Elt: N->getFlags());
2009 Record.push_back(Elt: N->getKind());
2010 Record.push_back(Elt: N->getFactorRaw());
2011
2012 auto WriteWideInt = [&](const APInt &Value) {
2013 // Write an encoded word that holds the number of active words and
2014 // the number of bits.
2015 uint64_t NumWords = Value.getActiveWords();
2016 uint64_t Encoded = (NumWords << 32) | Value.getBitWidth();
2017 Record.push_back(Elt: Encoded);
2018 emitWideAPInt(Vals&: Record, A: Value);
2019 };
2020
2021 WriteWideInt(N->getNumeratorRaw());
2022 WriteWideInt(N->getDenominatorRaw());
2023
2024 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2025 Record.push_back(Elt: N->getLine());
2026 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2027
2028 Stream.EmitRecord(Code: bitc::METADATA_FIXED_POINT_TYPE, Vals: Record, Abbrev);
2029 Record.clear();
2030}
2031
2032void ModuleBitcodeWriter::writeDIStringType(const DIStringType *N,
2033 SmallVectorImpl<uint64_t> &Record,
2034 unsigned Abbrev) {
2035 const unsigned SizeIsMetadata = 0x2;
2036 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2037 Record.push_back(Elt: N->getTag());
2038 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2039 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLength()));
2040 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLengthExp()));
2041 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLocationExp()));
2042 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2043 Record.push_back(Elt: N->getAlignInBits());
2044 Record.push_back(Elt: N->getEncoding());
2045
2046 Stream.EmitRecord(Code: bitc::METADATA_STRING_TYPE, Vals: Record, Abbrev);
2047 Record.clear();
2048}
2049
2050void ModuleBitcodeWriter::writeDIDerivedType(const DIDerivedType *N,
2051 SmallVectorImpl<uint64_t> &Record,
2052 unsigned Abbrev) {
2053 const unsigned SizeIsMetadata = 0x2;
2054 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2055 Record.push_back(Elt: N->getTag());
2056 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2057 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2058 Record.push_back(Elt: N->getLine());
2059 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2060 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2061 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2062 Record.push_back(Elt: N->getAlignInBits());
2063 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawOffsetInBits()));
2064 Record.push_back(Elt: N->getFlags());
2065 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getExtraData()));
2066
2067 // DWARF address space is encoded as N->getDWARFAddressSpace() + 1. 0 means
2068 // that there is no DWARF address space associated with DIDerivedType.
2069 if (const auto &DWARFAddressSpace = N->getDWARFAddressSpace())
2070 Record.push_back(Elt: *DWARFAddressSpace + 1);
2071 else
2072 Record.push_back(Elt: 0);
2073
2074 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2075
2076 if (auto PtrAuthData = N->getPtrAuthData())
2077 Record.push_back(Elt: PtrAuthData->RawData);
2078 else
2079 Record.push_back(Elt: 0);
2080
2081 Stream.EmitRecord(Code: bitc::METADATA_DERIVED_TYPE, Vals: Record, Abbrev);
2082 Record.clear();
2083}
2084
2085void ModuleBitcodeWriter::writeDISubrangeType(const DISubrangeType *N,
2086 SmallVectorImpl<uint64_t> &Record,
2087 unsigned Abbrev) {
2088 const unsigned SizeIsMetadata = 0x2;
2089 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2090 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2091 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2092 Record.push_back(Elt: N->getLine());
2093 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2094 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2095 Record.push_back(Elt: N->getAlignInBits());
2096 Record.push_back(Elt: N->getFlags());
2097 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2098 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
2099 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
2100 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
2101 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawBias()));
2102
2103 Stream.EmitRecord(Code: bitc::METADATA_SUBRANGE_TYPE, Vals: Record, Abbrev);
2104 Record.clear();
2105}
2106
2107void ModuleBitcodeWriter::writeDICompositeType(
2108 const DICompositeType *N, SmallVectorImpl<uint64_t> &Record,
2109 unsigned Abbrev) {
2110 const unsigned IsNotUsedInOldTypeRef = 0x2;
2111 const unsigned SizeIsMetadata = 0x4;
2112 Record.push_back(Elt: SizeIsMetadata | IsNotUsedInOldTypeRef |
2113 (unsigned)N->isDistinct());
2114 Record.push_back(Elt: N->getTag());
2115 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2116 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2117 Record.push_back(Elt: N->getLine());
2118 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2119 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2120 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2121 Record.push_back(Elt: N->getAlignInBits());
2122 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawOffsetInBits()));
2123 Record.push_back(Elt: N->getFlags());
2124 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2125 Record.push_back(Elt: N->getRuntimeLang());
2126 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getVTableHolder()));
2127 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams().get()));
2128 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawIdentifier()));
2129 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDiscriminator()));
2130 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawDataLocation()));
2131 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawAssociated()));
2132 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawAllocated()));
2133 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawRank()));
2134 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2135 Record.push_back(Elt: N->getNumExtraInhabitants());
2136 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSpecification()));
2137 Record.push_back(
2138 Elt: N->getEnumKind().value_or(u: dwarf::DW_APPLE_ENUM_KIND_invalid));
2139 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawBitStride()));
2140
2141 Stream.EmitRecord(Code: bitc::METADATA_COMPOSITE_TYPE, Vals: Record, Abbrev);
2142 Record.clear();
2143}
2144
2145void ModuleBitcodeWriter::writeDISubroutineType(
2146 const DISubroutineType *N, SmallVectorImpl<uint64_t> &Record,
2147 unsigned Abbrev) {
2148 const unsigned HasNoOldTypeRefs = 0x2;
2149 Record.push_back(Elt: HasNoOldTypeRefs | (unsigned)N->isDistinct());
2150 Record.push_back(Elt: N->getFlags());
2151 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTypeArray().get()));
2152 Record.push_back(Elt: N->getCC());
2153
2154 Stream.EmitRecord(Code: bitc::METADATA_SUBROUTINE_TYPE, Vals: Record, Abbrev);
2155 Record.clear();
2156}
2157
2158void ModuleBitcodeWriter::writeDIFile(const DIFile *N,
2159 SmallVectorImpl<uint64_t> &Record,
2160 unsigned Abbrev) {
2161 Record.push_back(Elt: N->isDistinct());
2162 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFilename()));
2163 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawDirectory()));
2164 if (N->getRawChecksum()) {
2165 Record.push_back(Elt: N->getRawChecksum()->Kind);
2166 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawChecksum()->Value));
2167 } else {
2168 // Maintain backwards compatibility with the old internal representation of
2169 // CSK_None in ChecksumKind by writing nulls here when Checksum is None.
2170 Record.push_back(Elt: 0);
2171 Record.push_back(Elt: VE.getMetadataOrNullID(MD: nullptr));
2172 }
2173 auto Source = N->getRawSource();
2174 if (Source)
2175 Record.push_back(Elt: VE.getMetadataOrNullID(MD: Source));
2176
2177 Stream.EmitRecord(Code: bitc::METADATA_FILE, Vals: Record, Abbrev);
2178 Record.clear();
2179}
2180
2181void ModuleBitcodeWriter::writeDICompileUnit(const DICompileUnit *N,
2182 SmallVectorImpl<uint64_t> &Record,
2183 unsigned Abbrev) {
2184 assert(N->isDistinct() && "Expected distinct compile units");
2185 Record.push_back(/* IsDistinct */ Elt: true);
2186
2187 auto Lang = N->getSourceLanguage();
2188 Record.push_back(Elt: Lang.getName());
2189 // Set bit so the MetadataLoader can distniguish between versioned and
2190 // unversioned names.
2191 if (Lang.hasVersionedName())
2192 Record.back() ^= (uint64_t(1) << 63);
2193
2194 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2195 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawProducer()));
2196 Record.push_back(Elt: N->isOptimized());
2197 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFlags()));
2198 Record.push_back(Elt: N->getRuntimeVersion());
2199 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSplitDebugFilename()));
2200 Record.push_back(Elt: N->getEmissionKind());
2201 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getEnumTypes().get()));
2202 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRetainedTypes().get()));
2203 Record.push_back(/* subprograms */ Elt: 0);
2204 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getGlobalVariables().get()));
2205 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getImportedEntities().get()));
2206 Record.push_back(Elt: N->getDWOId());
2207 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getMacros().get()));
2208 Record.push_back(Elt: N->getSplitDebugInlining());
2209 Record.push_back(Elt: N->getDebugInfoForProfiling());
2210 Record.push_back(Elt: (unsigned)N->getNameTableKind());
2211 Record.push_back(Elt: N->getRangesBaseAddress());
2212 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSysRoot()));
2213 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSDK()));
2214 Record.push_back(Elt: Lang.hasVersionedName() ? Lang.getVersion() : 0);
2215 Record.push_back(Elt: Lang.getDialect());
2216
2217 Stream.EmitRecord(Code: bitc::METADATA_COMPILE_UNIT, Vals: Record, Abbrev);
2218 Record.clear();
2219}
2220
2221void ModuleBitcodeWriter::writeDISubprogram(const DISubprogram *N,
2222 SmallVectorImpl<uint64_t> &Record,
2223 unsigned Abbrev) {
2224 const uint64_t HasUnitFlag = 1 << 1;
2225 const uint64_t HasSPFlagsFlag = 1 << 2;
2226 Record.push_back(Elt: uint64_t(N->isDistinct()) | HasUnitFlag | HasSPFlagsFlag);
2227 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2228 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2229 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLinkageName()));
2230 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2231 Record.push_back(Elt: N->getLine());
2232 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2233 Record.push_back(Elt: N->getScopeLine());
2234 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getContainingType()));
2235 Record.push_back(Elt: N->getSPFlags());
2236 Record.push_back(Elt: N->getVirtualIndex());
2237 Record.push_back(Elt: N->getFlags());
2238 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUnit()));
2239 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams().get()));
2240 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDeclaration()));
2241 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRetainedNodes().get()));
2242 Record.push_back(Elt: N->getThisAdjustment());
2243 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getThrownTypes().get()));
2244 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2245 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawTargetFuncName()));
2246 Record.push_back(Elt: N->getKeyInstructionsEnabled());
2247
2248 Stream.EmitRecord(Code: bitc::METADATA_SUBPROGRAM, Vals: Record, Abbrev);
2249 Record.clear();
2250}
2251
2252void ModuleBitcodeWriter::writeDILexicalBlock(const DILexicalBlock *N,
2253 SmallVectorImpl<uint64_t> &Record,
2254 unsigned Abbrev) {
2255 Record.push_back(Elt: N->isDistinct());
2256 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2257 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2258 Record.push_back(Elt: N->getLine());
2259 Record.push_back(Elt: N->getColumn());
2260
2261 Stream.EmitRecord(Code: bitc::METADATA_LEXICAL_BLOCK, Vals: Record, Abbrev);
2262 Record.clear();
2263}
2264
2265void ModuleBitcodeWriter::writeDILexicalBlockFile(
2266 const DILexicalBlockFile *N, SmallVectorImpl<uint64_t> &Record,
2267 unsigned Abbrev) {
2268 Record.push_back(Elt: N->isDistinct());
2269 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2270 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2271 Record.push_back(Elt: N->getDiscriminator());
2272
2273 Stream.EmitRecord(Code: bitc::METADATA_LEXICAL_BLOCK_FILE, Vals: Record, Abbrev);
2274 Record.clear();
2275}
2276
2277void ModuleBitcodeWriter::writeDICommonBlock(const DICommonBlock *N,
2278 SmallVectorImpl<uint64_t> &Record,
2279 unsigned Abbrev) {
2280 Record.push_back(Elt: N->isDistinct());
2281 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2282 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDecl()));
2283 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2284 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2285 Record.push_back(Elt: N->getLineNo());
2286
2287 Stream.EmitRecord(Code: bitc::METADATA_COMMON_BLOCK, Vals: Record, Abbrev);
2288 Record.clear();
2289}
2290
2291void ModuleBitcodeWriter::writeDINamespace(const DINamespace *N,
2292 SmallVectorImpl<uint64_t> &Record,
2293 unsigned Abbrev) {
2294 Record.push_back(Elt: N->isDistinct() | N->getExportSymbols() << 1);
2295 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2296 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2297
2298 Stream.EmitRecord(Code: bitc::METADATA_NAMESPACE, Vals: Record, Abbrev);
2299 Record.clear();
2300}
2301
2302void ModuleBitcodeWriter::writeDIMacro(const DIMacro *N,
2303 SmallVectorImpl<uint64_t> &Record,
2304 unsigned Abbrev) {
2305 Record.push_back(Elt: N->isDistinct());
2306 Record.push_back(Elt: N->getMacinfoType());
2307 Record.push_back(Elt: N->getLine());
2308 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2309 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawValue()));
2310
2311 Stream.EmitRecord(Code: bitc::METADATA_MACRO, Vals: Record, Abbrev);
2312 Record.clear();
2313}
2314
2315void ModuleBitcodeWriter::writeDIMacroFile(const DIMacroFile *N,
2316 SmallVectorImpl<uint64_t> &Record,
2317 unsigned Abbrev) {
2318 Record.push_back(Elt: N->isDistinct());
2319 Record.push_back(Elt: N->getMacinfoType());
2320 Record.push_back(Elt: N->getLine());
2321 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2322 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2323
2324 Stream.EmitRecord(Code: bitc::METADATA_MACRO_FILE, Vals: Record, Abbrev);
2325 Record.clear();
2326}
2327
2328void ModuleBitcodeWriter::writeDIArgList(const DIArgList *N,
2329 SmallVectorImpl<uint64_t> &Record) {
2330 Record.reserve(N: N->getArgs().size());
2331 for (ValueAsMetadata *MD : N->getArgs())
2332 Record.push_back(Elt: VE.getMetadataID(MD));
2333
2334 Stream.EmitRecord(Code: bitc::METADATA_ARG_LIST, Vals: Record);
2335 Record.clear();
2336}
2337
2338void ModuleBitcodeWriter::writeDIModule(const DIModule *N,
2339 SmallVectorImpl<uint64_t> &Record,
2340 unsigned Abbrev) {
2341 Record.push_back(Elt: N->isDistinct());
2342 for (auto &I : N->operands())
2343 Record.push_back(Elt: VE.getMetadataOrNullID(MD: I));
2344 Record.push_back(Elt: N->getLineNo());
2345 Record.push_back(Elt: N->getIsDecl());
2346
2347 Stream.EmitRecord(Code: bitc::METADATA_MODULE, Vals: Record, Abbrev);
2348 Record.clear();
2349}
2350
2351void ModuleBitcodeWriter::writeDIAssignID(const DIAssignID *N,
2352 SmallVectorImpl<uint64_t> &Record,
2353 unsigned Abbrev) {
2354 // There are no arguments for this metadata type.
2355 Record.push_back(Elt: N->isDistinct());
2356 Stream.EmitRecord(Code: bitc::METADATA_ASSIGN_ID, Vals: Record, Abbrev);
2357 Record.clear();
2358}
2359
2360void ModuleBitcodeWriter::writeDITemplateTypeParameter(
2361 const DITemplateTypeParameter *N, SmallVectorImpl<uint64_t> &Record,
2362 unsigned Abbrev) {
2363 Record.push_back(Elt: N->isDistinct());
2364 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2365 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2366 Record.push_back(Elt: N->isDefault());
2367
2368 Stream.EmitRecord(Code: bitc::METADATA_TEMPLATE_TYPE, Vals: Record, Abbrev);
2369 Record.clear();
2370}
2371
2372void ModuleBitcodeWriter::writeDITemplateValueParameter(
2373 const DITemplateValueParameter *N, SmallVectorImpl<uint64_t> &Record,
2374 unsigned Abbrev) {
2375 Record.push_back(Elt: N->isDistinct());
2376 Record.push_back(Elt: N->getTag());
2377 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2378 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2379 Record.push_back(Elt: N->isDefault());
2380 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getValue()));
2381
2382 Stream.EmitRecord(Code: bitc::METADATA_TEMPLATE_VALUE, Vals: Record, Abbrev);
2383 Record.clear();
2384}
2385
2386void ModuleBitcodeWriter::writeDIGlobalVariable(
2387 const DIGlobalVariable *N, SmallVectorImpl<uint64_t> &Record,
2388 unsigned Abbrev) {
2389 const uint64_t Version = 2 << 1;
2390 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
2391 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2392 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2393 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLinkageName()));
2394 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2395 Record.push_back(Elt: N->getLine());
2396 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2397 Record.push_back(Elt: N->isLocalToUnit());
2398 Record.push_back(Elt: N->isDefinition());
2399 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStaticDataMemberDeclaration()));
2400 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams()));
2401 Record.push_back(Elt: N->getAlignInBits());
2402 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2403
2404 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_VAR, Vals: Record, Abbrev);
2405 Record.clear();
2406}
2407
2408void ModuleBitcodeWriter::writeDILocalVariable(
2409 const DILocalVariable *N, SmallVectorImpl<uint64_t> &Record,
2410 unsigned Abbrev) {
2411 // In order to support all possible bitcode formats in BitcodeReader we need
2412 // to distinguish the following cases:
2413 // 1) Record has no artificial tag (Record[1]),
2414 // has no obsolete inlinedAt field (Record[9]).
2415 // In this case Record size will be 8, HasAlignment flag is false.
2416 // 2) Record has artificial tag (Record[1]),
2417 // has no obsolete inlignedAt field (Record[9]).
2418 // In this case Record size will be 9, HasAlignment flag is false.
2419 // 3) Record has both artificial tag (Record[1]) and
2420 // obsolete inlignedAt field (Record[9]).
2421 // In this case Record size will be 10, HasAlignment flag is false.
2422 // 4) Record has neither artificial tag, nor inlignedAt field, but
2423 // HasAlignment flag is true and Record[8] contains alignment value.
2424 const uint64_t HasAlignmentFlag = 1 << 1;
2425 Record.push_back(Elt: (uint64_t)N->isDistinct() | HasAlignmentFlag);
2426 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2427 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2428 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2429 Record.push_back(Elt: N->getLine());
2430 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2431 Record.push_back(Elt: N->getArg());
2432 Record.push_back(Elt: N->getFlags());
2433 Record.push_back(Elt: N->getAlignInBits());
2434 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2435
2436 Stream.EmitRecord(Code: bitc::METADATA_LOCAL_VAR, Vals: Record, Abbrev);
2437 Record.clear();
2438}
2439
2440void ModuleBitcodeWriter::writeDILabel(
2441 const DILabel *N, SmallVectorImpl<uint64_t> &Record,
2442 unsigned Abbrev) {
2443 uint64_t IsArtificialFlag = uint64_t(N->isArtificial()) << 1;
2444 Record.push_back(Elt: (uint64_t)N->isDistinct() | IsArtificialFlag);
2445 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2446 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2447 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2448 Record.push_back(Elt: N->getLine());
2449 Record.push_back(Elt: N->getColumn());
2450 Record.push_back(Elt: N->getCoroSuspendIdx().has_value()
2451 ? (uint64_t)N->getCoroSuspendIdx().value()
2452 : std::numeric_limits<uint64_t>::max());
2453
2454 Stream.EmitRecord(Code: bitc::METADATA_LABEL, Vals: Record, Abbrev);
2455 Record.clear();
2456}
2457
2458void ModuleBitcodeWriter::writeDIExpression(const DIExpression *N,
2459 SmallVectorImpl<uint64_t> &Record,
2460 unsigned Abbrev) {
2461 Record.reserve(N: N->getElements().size() + 1);
2462 const uint64_t Version = 3 << 1;
2463 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
2464 Record.append(in_start: N->elements_begin(), in_end: N->elements_end());
2465
2466 Stream.EmitRecord(Code: bitc::METADATA_EXPRESSION, Vals: Record, Abbrev);
2467 Record.clear();
2468}
2469
2470void ModuleBitcodeWriter::writeDIGlobalVariableExpression(
2471 const DIGlobalVariableExpression *N, SmallVectorImpl<uint64_t> &Record,
2472 unsigned Abbrev) {
2473 Record.push_back(Elt: N->isDistinct());
2474 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getVariable()));
2475 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getExpression()));
2476
2477 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_VAR_EXPR, Vals: Record, Abbrev);
2478 Record.clear();
2479}
2480
2481void ModuleBitcodeWriter::writeDIObjCProperty(const DIObjCProperty *N,
2482 SmallVectorImpl<uint64_t> &Record,
2483 unsigned Abbrev) {
2484 Record.push_back(Elt: N->isDistinct());
2485 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2486 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2487 Record.push_back(Elt: N->getLine());
2488 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSetterName()));
2489 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawGetterName()));
2490 Record.push_back(Elt: N->getAttributes());
2491 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2492
2493 Stream.EmitRecord(Code: bitc::METADATA_OBJC_PROPERTY, Vals: Record, Abbrev);
2494 Record.clear();
2495}
2496
2497void ModuleBitcodeWriter::writeDIImportedEntity(
2498 const DIImportedEntity *N, SmallVectorImpl<uint64_t> &Record,
2499 unsigned Abbrev) {
2500 Record.push_back(Elt: N->isDistinct());
2501 Record.push_back(Elt: N->getTag());
2502 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2503 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getEntity()));
2504 Record.push_back(Elt: N->getLine());
2505 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2506 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFile()));
2507 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2508
2509 Stream.EmitRecord(Code: bitc::METADATA_IMPORTED_ENTITY, Vals: Record, Abbrev);
2510 Record.clear();
2511}
2512
2513unsigned ModuleBitcodeWriter::createNamedMetadataAbbrev() {
2514 auto Abbv = std::make_shared<BitCodeAbbrev>();
2515 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_NAME));
2516 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2517 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2518 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
2519}
2520
2521void ModuleBitcodeWriter::writeNamedMetadata(
2522 SmallVectorImpl<uint64_t> &Record) {
2523 if (M.named_metadata_empty())
2524 return;
2525
2526 unsigned Abbrev = createNamedMetadataAbbrev();
2527 for (const NamedMDNode &NMD : M.named_metadata()) {
2528 // Write name.
2529 StringRef Str = NMD.getName();
2530 Record.append(in_start: Str.bytes_begin(), in_end: Str.bytes_end());
2531 Stream.EmitRecord(Code: bitc::METADATA_NAME, Vals: Record, Abbrev);
2532 Record.clear();
2533
2534 // Write named metadata operands.
2535 for (const MDNode *N : NMD.operands())
2536 Record.push_back(Elt: VE.getMetadataID(MD: N));
2537 Stream.EmitRecord(Code: bitc::METADATA_NAMED_NODE, Vals: Record, Abbrev: 0);
2538 Record.clear();
2539 }
2540}
2541
2542unsigned ModuleBitcodeWriter::createMetadataStringsAbbrev() {
2543 auto Abbv = std::make_shared<BitCodeAbbrev>();
2544 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_STRINGS));
2545 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // # of strings
2546 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // offset to chars
2547 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
2548 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
2549}
2550
2551/// Write out a record for MDString.
2552///
2553/// All the metadata strings in a metadata block are emitted in a single
2554/// record. The sizes and strings themselves are shoved into a blob.
2555void ModuleBitcodeWriter::writeMetadataStrings(
2556 ArrayRef<const Metadata *> Strings, SmallVectorImpl<uint64_t> &Record) {
2557 if (Strings.empty())
2558 return;
2559
2560 // Start the record with the number of strings.
2561 Record.push_back(Elt: bitc::METADATA_STRINGS);
2562 Record.push_back(Elt: Strings.size());
2563
2564 // Emit the sizes of the strings in the blob.
2565 SmallString<256> Blob;
2566 {
2567 BitstreamWriter W(Blob);
2568 for (const Metadata *MD : Strings)
2569 W.EmitVBR(Val: cast<MDString>(Val: MD)->getLength(), NumBits: 6);
2570 W.FlushToWord();
2571 }
2572
2573 // Add the offset to the strings to the record.
2574 Record.push_back(Elt: Blob.size());
2575
2576 // Add the strings to the blob.
2577 for (const Metadata *MD : Strings)
2578 Blob.append(RHS: cast<MDString>(Val: MD)->getString());
2579
2580 // Emit the final record.
2581 Stream.EmitRecordWithBlob(Abbrev: createMetadataStringsAbbrev(), Vals: Record, Blob);
2582 Record.clear();
2583}
2584
2585// Generates an enum to use as an index in the Abbrev array of Metadata record.
2586enum MetadataAbbrev : unsigned {
2587#define HANDLE_MDNODE_LEAF(CLASS) CLASS##AbbrevID,
2588#include "llvm/IR/Metadata.def"
2589 LastPlusOne
2590};
2591
2592void ModuleBitcodeWriter::writeMetadataRecords(
2593 ArrayRef<const Metadata *> MDs, SmallVectorImpl<uint64_t> &Record,
2594 std::vector<unsigned> *MDAbbrevs, std::vector<uint64_t> *IndexPos) {
2595 if (MDs.empty())
2596 return;
2597
2598 // Initialize MDNode abbreviations.
2599#define HANDLE_MDNODE_LEAF(CLASS) unsigned CLASS##Abbrev = 0;
2600#include "llvm/IR/Metadata.def"
2601
2602 for (const Metadata *MD : MDs) {
2603 if (IndexPos)
2604 IndexPos->push_back(x: Stream.GetCurrentBitNo());
2605 if (const MDNode *N = dyn_cast<MDNode>(Val: MD)) {
2606 assert(N->isResolved() && "Expected forward references to be resolved");
2607
2608 switch (N->getMetadataID()) {
2609 default:
2610 llvm_unreachable("Invalid MDNode subclass");
2611#define HANDLE_MDNODE_LEAF(CLASS) \
2612 case Metadata::CLASS##Kind: \
2613 if (MDAbbrevs) \
2614 write##CLASS(cast<CLASS>(N), Record, \
2615 (*MDAbbrevs)[MetadataAbbrev::CLASS##AbbrevID]); \
2616 else \
2617 write##CLASS(cast<CLASS>(N), Record, CLASS##Abbrev); \
2618 continue;
2619#include "llvm/IR/Metadata.def"
2620 }
2621 }
2622 if (auto *AL = dyn_cast<DIArgList>(Val: MD)) {
2623 writeDIArgList(N: AL, Record);
2624 continue;
2625 }
2626 writeValueAsMetadata(MD: cast<ValueAsMetadata>(Val: MD), Record);
2627 }
2628}
2629
2630void ModuleBitcodeWriter::writeModuleMetadata() {
2631 if (!VE.hasMDs() && M.named_metadata_empty())
2632 return;
2633
2634 Stream.EnterSubblock(BlockID: bitc::METADATA_BLOCK_ID, CodeLen: 4);
2635 SmallVector<uint64_t, 64> Record;
2636
2637 // Emit all abbrevs upfront, so that the reader can jump in the middle of the
2638 // block and load any metadata.
2639 std::vector<unsigned> MDAbbrevs;
2640
2641 MDAbbrevs.resize(new_size: MetadataAbbrev::LastPlusOne);
2642 MDAbbrevs[MetadataAbbrev::DILocationAbbrevID] = createDILocationAbbrev();
2643 MDAbbrevs[MetadataAbbrev::GenericDINodeAbbrevID] =
2644 createGenericDINodeAbbrev();
2645
2646 auto Abbv = std::make_shared<BitCodeAbbrev>();
2647 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_INDEX_OFFSET));
2648 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2649 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2650 unsigned OffsetAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2651
2652 Abbv = std::make_shared<BitCodeAbbrev>();
2653 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_INDEX));
2654 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2655 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
2656 unsigned IndexAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2657
2658 // Emit MDStrings together upfront.
2659 writeMetadataStrings(Strings: VE.getMDStrings(), Record);
2660
2661 // We only emit an index for the metadata record if we have more than a given
2662 // (naive) threshold of metadatas, otherwise it is not worth it.
2663 if (VE.getNonMDStrings().size() > IndexThreshold) {
2664 // Write a placeholder value in for the offset of the metadata index,
2665 // which is written after the records, so that it can include
2666 // the offset of each entry. The placeholder offset will be
2667 // updated after all records are emitted.
2668 uint64_t Vals[] = {0, 0};
2669 Stream.EmitRecord(Code: bitc::METADATA_INDEX_OFFSET, Vals, Abbrev: OffsetAbbrev);
2670 }
2671
2672 // Compute and save the bit offset to the current position, which will be
2673 // patched when we emit the index later. We can simply subtract the 64-bit
2674 // fixed size from the current bit number to get the location to backpatch.
2675 uint64_t IndexOffsetRecordBitPos = Stream.GetCurrentBitNo();
2676
2677 // This index will contain the bitpos for each individual record.
2678 std::vector<uint64_t> IndexPos;
2679 IndexPos.reserve(n: VE.getNonMDStrings().size());
2680
2681 // Write all the records
2682 writeMetadataRecords(MDs: VE.getNonMDStrings(), Record, MDAbbrevs: &MDAbbrevs, IndexPos: &IndexPos);
2683
2684 if (VE.getNonMDStrings().size() > IndexThreshold) {
2685 // Now that we have emitted all the records we will emit the index. But
2686 // first
2687 // backpatch the forward reference so that the reader can skip the records
2688 // efficiently.
2689 Stream.BackpatchWord64(BitNo: IndexOffsetRecordBitPos - 64,
2690 Val: Stream.GetCurrentBitNo() - IndexOffsetRecordBitPos);
2691
2692 // Delta encode the index.
2693 uint64_t PreviousValue = IndexOffsetRecordBitPos;
2694 for (auto &Elt : IndexPos) {
2695 auto EltDelta = Elt - PreviousValue;
2696 PreviousValue = Elt;
2697 Elt = EltDelta;
2698 }
2699 // Emit the index record.
2700 Stream.EmitRecord(Code: bitc::METADATA_INDEX, Vals: IndexPos, Abbrev: IndexAbbrev);
2701 IndexPos.clear();
2702 }
2703
2704 // Write the named metadata now.
2705 writeNamedMetadata(Record);
2706
2707 auto AddDeclAttachedMetadata = [&](const GlobalObject &GO) {
2708 SmallVector<uint64_t, 4> Record;
2709 Record.push_back(Elt: VE.getValueID(V: &GO));
2710 pushGlobalMetadataAttachment(Record, GO);
2711 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_DECL_ATTACHMENT, Vals: Record);
2712 };
2713 for (const Function &F : M)
2714 if (F.isDeclaration() && F.hasMetadata())
2715 AddDeclAttachedMetadata(F);
2716 for (const GlobalIFunc &GI : M.ifuncs())
2717 if (GI.hasMetadata())
2718 AddDeclAttachedMetadata(GI);
2719 // FIXME: Only store metadata for declarations here, and move data for global
2720 // variable definitions to a separate block (PR28134).
2721 for (const GlobalVariable &GV : M.globals())
2722 if (GV.hasMetadata())
2723 AddDeclAttachedMetadata(GV);
2724
2725 Stream.ExitBlock();
2726}
2727
2728void ModuleBitcodeWriter::writeFunctionMetadata(const Function &F) {
2729 if (!VE.hasMDs())
2730 return;
2731
2732 Stream.EnterSubblock(BlockID: bitc::METADATA_BLOCK_ID, CodeLen: 3);
2733 SmallVector<uint64_t, 64> Record;
2734 writeMetadataStrings(Strings: VE.getMDStrings(), Record);
2735 writeMetadataRecords(MDs: VE.getNonMDStrings(), Record);
2736 Stream.ExitBlock();
2737}
2738
2739void ModuleBitcodeWriter::pushGlobalMetadataAttachment(
2740 SmallVectorImpl<uint64_t> &Record, const GlobalObject &GO) {
2741 // [n x [id, mdnode]]
2742 SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2743 GO.getAllMetadata(MDs);
2744 for (const auto &I : MDs) {
2745 Record.push_back(Elt: I.first);
2746 Record.push_back(Elt: VE.getMetadataID(MD: I.second));
2747 }
2748}
2749
2750void ModuleBitcodeWriter::writeFunctionMetadataAttachment(const Function &F) {
2751 Stream.EnterSubblock(BlockID: bitc::METADATA_ATTACHMENT_ID, CodeLen: 3);
2752
2753 SmallVector<uint64_t, 64> Record;
2754
2755 if (F.hasMetadata()) {
2756 pushGlobalMetadataAttachment(Record, GO: F);
2757 Stream.EmitRecord(Code: bitc::METADATA_ATTACHMENT, Vals: Record, Abbrev: 0);
2758 Record.clear();
2759 }
2760
2761 // Write metadata attachments
2762 // METADATA_ATTACHMENT - [m x [value, [n x [id, mdnode]]]
2763 SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2764 for (const BasicBlock &BB : F)
2765 for (const Instruction &I : BB) {
2766 MDs.clear();
2767 I.getAllMetadataOtherThanDebugLoc(MDs);
2768
2769 // If no metadata, ignore instruction.
2770 if (MDs.empty()) continue;
2771
2772 Record.push_back(Elt: VE.getInstructionID(I: &I));
2773
2774 for (const auto &[ID, MD] : MDs) {
2775 Record.push_back(Elt: ID);
2776 Record.push_back(Elt: VE.getMetadataID(MD));
2777 }
2778 Stream.EmitRecord(Code: bitc::METADATA_ATTACHMENT, Vals: Record, Abbrev: 0);
2779 Record.clear();
2780 }
2781
2782 Stream.ExitBlock();
2783}
2784
2785void ModuleBitcodeWriter::writeModuleMetadataKinds() {
2786 SmallVector<uint64_t, 64> Record;
2787
2788 // Write metadata kinds
2789 // METADATA_KIND - [n x [id, name]]
2790 SmallVector<StringRef, 8> Names;
2791 M.getMDKindNames(Result&: Names);
2792
2793 if (Names.empty()) return;
2794
2795 Stream.EnterSubblock(BlockID: bitc::METADATA_KIND_BLOCK_ID, CodeLen: 3);
2796
2797 for (unsigned MDKindID = 0, e = Names.size(); MDKindID != e; ++MDKindID) {
2798 Record.push_back(Elt: MDKindID);
2799 StringRef KName = Names[MDKindID];
2800 Record.append(in_start: KName.begin(), in_end: KName.end());
2801
2802 Stream.EmitRecord(Code: bitc::METADATA_KIND, Vals: Record, Abbrev: 0);
2803 Record.clear();
2804 }
2805
2806 Stream.ExitBlock();
2807}
2808
2809void ModuleBitcodeWriter::writeOperandBundleTags() {
2810 // Write metadata kinds
2811 //
2812 // OPERAND_BUNDLE_TAGS_BLOCK_ID : N x OPERAND_BUNDLE_TAG
2813 //
2814 // OPERAND_BUNDLE_TAG - [strchr x N]
2815
2816 SmallVector<StringRef, 8> Tags;
2817 M.getOperandBundleTags(Result&: Tags);
2818
2819 if (Tags.empty())
2820 return;
2821
2822 Stream.EnterSubblock(BlockID: bitc::OPERAND_BUNDLE_TAGS_BLOCK_ID, CodeLen: 3);
2823
2824 SmallVector<uint64_t, 64> Record;
2825
2826 for (auto Tag : Tags) {
2827 Record.append(in_start: Tag.begin(), in_end: Tag.end());
2828
2829 Stream.EmitRecord(Code: bitc::OPERAND_BUNDLE_TAG, Vals: Record, Abbrev: 0);
2830 Record.clear();
2831 }
2832
2833 Stream.ExitBlock();
2834}
2835
2836void ModuleBitcodeWriter::writeSyncScopeNames() {
2837 SmallVector<StringRef, 8> SSNs;
2838 M.getContext().getSyncScopeNames(SSNs);
2839 if (SSNs.empty())
2840 return;
2841
2842 Stream.EnterSubblock(BlockID: bitc::SYNC_SCOPE_NAMES_BLOCK_ID, CodeLen: 2);
2843
2844 SmallVector<uint64_t, 64> Record;
2845 for (auto SSN : SSNs) {
2846 Record.append(in_start: SSN.begin(), in_end: SSN.end());
2847 Stream.EmitRecord(Code: bitc::SYNC_SCOPE_NAME, Vals: Record, Abbrev: 0);
2848 Record.clear();
2849 }
2850
2851 Stream.ExitBlock();
2852}
2853
2854void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal,
2855 bool isGlobal) {
2856 if (FirstVal == LastVal) return;
2857
2858 Stream.EnterSubblock(BlockID: bitc::CONSTANTS_BLOCK_ID, CodeLen: 4);
2859
2860 unsigned AggregateAbbrev = 0;
2861 unsigned String8Abbrev = 0;
2862 unsigned CString7Abbrev = 0;
2863 unsigned CString6Abbrev = 0;
2864 // If this is a constant pool for the module, emit module-specific abbrevs.
2865 if (isGlobal) {
2866 // Abbrev for CST_CODE_AGGREGATE.
2867 auto Abbv = std::make_shared<BitCodeAbbrev>();
2868 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_AGGREGATE));
2869 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2870 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(Value: LastVal+1)));
2871 AggregateAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2872
2873 // Abbrev for CST_CODE_STRING.
2874 Abbv = std::make_shared<BitCodeAbbrev>();
2875 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_STRING));
2876 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2877 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2878 String8Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2879 // Abbrev for CST_CODE_CSTRING.
2880 Abbv = std::make_shared<BitCodeAbbrev>();
2881 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2882 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2883 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
2884 CString7Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2885 // Abbrev for CST_CODE_CSTRING.
2886 Abbv = std::make_shared<BitCodeAbbrev>();
2887 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2888 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2889 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
2890 CString6Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2891 }
2892
2893 SmallVector<uint64_t, 64> Record;
2894
2895 const ValueEnumerator::ValueList &Vals = VE.getValues();
2896 Type *LastTy = nullptr;
2897 for (unsigned i = FirstVal; i != LastVal; ++i) {
2898 const Value *V = Vals[i].first;
2899 // If we need to switch types, do so now.
2900 if (V->getType() != LastTy) {
2901 LastTy = V->getType();
2902 Record.push_back(Elt: VE.getTypeID(T: LastTy));
2903 Stream.EmitRecord(Code: bitc::CST_CODE_SETTYPE, Vals: Record,
2904 Abbrev: CONSTANTS_SETTYPE_ABBREV);
2905 Record.clear();
2906 }
2907
2908 if (const InlineAsm *IA = dyn_cast<InlineAsm>(Val: V)) {
2909 Record.push_back(Elt: VE.getTypeID(T: IA->getFunctionType()));
2910 Record.push_back(
2911 Elt: unsigned(IA->hasSideEffects()) | unsigned(IA->isAlignStack()) << 1 |
2912 unsigned(IA->getDialect() & 1) << 2 | unsigned(IA->canThrow()) << 3);
2913
2914 // Add the asm string.
2915 StringRef AsmStr = IA->getAsmString();
2916 Record.push_back(Elt: AsmStr.size());
2917 Record.append(in_start: AsmStr.begin(), in_end: AsmStr.end());
2918
2919 // Add the constraint string.
2920 StringRef ConstraintStr = IA->getConstraintString();
2921 Record.push_back(Elt: ConstraintStr.size());
2922 Record.append(in_start: ConstraintStr.begin(), in_end: ConstraintStr.end());
2923 Stream.EmitRecord(Code: bitc::CST_CODE_INLINEASM, Vals: Record);
2924 Record.clear();
2925 continue;
2926 }
2927 const Constant *C = cast<Constant>(Val: V);
2928 unsigned Code = -1U;
2929 unsigned AbbrevToUse = 0;
2930 if (C->isNullValue()) {
2931 Code = bitc::CST_CODE_NULL;
2932 } else if (isa<PoisonValue>(Val: C)) {
2933 Code = bitc::CST_CODE_POISON;
2934 } else if (isa<UndefValue>(Val: C)) {
2935 Code = bitc::CST_CODE_UNDEF;
2936 } else if (const ConstantInt *IV = dyn_cast<ConstantInt>(Val: C)) {
2937 if (IV->getBitWidth() <= 64) {
2938 uint64_t V = IV->getSExtValue();
2939 emitSignedInt64(Vals&: Record, V);
2940 Code = bitc::CST_CODE_INTEGER;
2941 AbbrevToUse = CONSTANTS_INTEGER_ABBREV;
2942 } else { // Wide integers, > 64 bits in size.
2943 emitWideAPInt(Vals&: Record, A: IV->getValue());
2944 Code = bitc::CST_CODE_WIDE_INTEGER;
2945 }
2946 } else if (const ConstantByte *BV = dyn_cast<ConstantByte>(Val: C)) {
2947 if (BV->getBitWidth() <= 64) {
2948 uint64_t V = BV->getSExtValue();
2949 emitSignedInt64(Vals&: Record, V);
2950 Code = bitc::CST_CODE_BYTE;
2951 AbbrevToUse = CONSTANTS_BYTE_ABBREV;
2952 } else { // Wide bytes, > 64 bits in size.
2953 emitWideAPInt(Vals&: Record, A: BV->getValue());
2954 Code = bitc::CST_CODE_WIDE_BYTE;
2955 }
2956 } else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
2957 Code = bitc::CST_CODE_FLOAT;
2958 Type *Ty = CFP->getType()->getScalarType();
2959 if (Ty->isHalfTy() || Ty->isBFloatTy() || Ty->isFloatTy() ||
2960 Ty->isDoubleTy()) {
2961 Record.push_back(Elt: CFP->getValueAPF().bitcastToAPInt().getZExtValue());
2962 } else if (Ty->isX86_FP80Ty()) {
2963 // api needed to prevent premature destruction
2964 // bits are not in the same order as a normal i80 APInt, compensate.
2965 APInt api = CFP->getValueAPF().bitcastToAPInt();
2966 const uint64_t *p = api.getRawData();
2967 Record.push_back(Elt: (p[1] << 48) | (p[0] >> 16));
2968 Record.push_back(Elt: p[0] & 0xffffLL);
2969 } else if (Ty->isFP128Ty() || Ty->isPPC_FP128Ty()) {
2970 APInt api = CFP->getValueAPF().bitcastToAPInt();
2971 const uint64_t *p = api.getRawData();
2972 Record.push_back(Elt: p[0]);
2973 Record.push_back(Elt: p[1]);
2974 } else {
2975 assert(0 && "Unknown FP type!");
2976 }
2977 } else if (isa<ConstantDataSequential>(Val: C) &&
2978 cast<ConstantDataSequential>(Val: C)->isString()) {
2979 const ConstantDataSequential *Str = cast<ConstantDataSequential>(Val: C);
2980 // Emit constant strings specially.
2981 uint64_t NumElts = Str->getNumElements();
2982 // If this is a null-terminated string, use the denser CSTRING encoding.
2983 if (Str->isCString()) {
2984 Code = bitc::CST_CODE_CSTRING;
2985 --NumElts; // Don't encode the null, which isn't allowed by char6.
2986 } else {
2987 Code = bitc::CST_CODE_STRING;
2988 AbbrevToUse = String8Abbrev;
2989 }
2990 bool isCStr7 = Code == bitc::CST_CODE_CSTRING;
2991 bool isCStrChar6 = Code == bitc::CST_CODE_CSTRING;
2992 for (uint64_t i = 0; i != NumElts; ++i) {
2993 unsigned char V = Str->getElementAsInteger(i);
2994 Record.push_back(Elt: V);
2995 isCStr7 &= (V & 128) == 0;
2996 if (isCStrChar6)
2997 isCStrChar6 = BitCodeAbbrevOp::isChar6(C: V);
2998 }
2999
3000 if (isCStrChar6)
3001 AbbrevToUse = CString6Abbrev;
3002 else if (isCStr7)
3003 AbbrevToUse = CString7Abbrev;
3004 } else if (const ConstantDataSequential *CDS =
3005 dyn_cast<ConstantDataSequential>(Val: C)) {
3006 Code = bitc::CST_CODE_DATA;
3007 Type *EltTy = CDS->getElementType();
3008 if (isa<IntegerType>(Val: EltTy) || isa<ByteType>(Val: EltTy)) {
3009 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i)
3010 Record.push_back(Elt: CDS->getElementAsInteger(i));
3011 } else {
3012 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i)
3013 Record.push_back(
3014 Elt: CDS->getElementAsAPFloat(i).bitcastToAPInt().getLimitedValue());
3015 }
3016 } else if (isa<ConstantAggregate>(Val: C)) {
3017 Code = bitc::CST_CODE_AGGREGATE;
3018 for (const Value *Op : C->operands())
3019 Record.push_back(Elt: VE.getValueID(V: Op));
3020 AbbrevToUse = AggregateAbbrev;
3021 } else if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: C)) {
3022 switch (CE->getOpcode()) {
3023 default:
3024 if (Instruction::isCast(Opcode: CE->getOpcode())) {
3025 Code = bitc::CST_CODE_CE_CAST;
3026 Record.push_back(Elt: getEncodedCastOpcode(Opcode: CE->getOpcode()));
3027 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
3028 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3029 AbbrevToUse = CONSTANTS_CE_CAST_Abbrev;
3030 } else {
3031 assert(CE->getNumOperands() == 2 && "Unknown constant expr!");
3032 Code = bitc::CST_CODE_CE_BINOP;
3033 Record.push_back(Elt: getEncodedBinaryOpcode(Opcode: CE->getOpcode()));
3034 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3035 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3036 uint64_t Flags = getOptimizationFlags(V: CE);
3037 if (Flags != 0)
3038 Record.push_back(Elt: Flags);
3039 }
3040 break;
3041 case Instruction::FNeg: {
3042 assert(CE->getNumOperands() == 1 && "Unknown constant expr!");
3043 Code = bitc::CST_CODE_CE_UNOP;
3044 Record.push_back(Elt: getEncodedUnaryOpcode(Opcode: CE->getOpcode()));
3045 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3046 uint64_t Flags = getOptimizationFlags(V: CE);
3047 if (Flags != 0)
3048 Record.push_back(Elt: Flags);
3049 break;
3050 }
3051 case Instruction::GetElementPtr: {
3052 Code = bitc::CST_CODE_CE_GEP;
3053 const auto *GO = cast<GEPOperator>(Val: C);
3054 Record.push_back(Elt: VE.getTypeID(T: GO->getSourceElementType()));
3055 Record.push_back(Elt: getOptimizationFlags(V: GO));
3056 if (std::optional<ConstantRange> Range = GO->getInRange()) {
3057 Code = bitc::CST_CODE_CE_GEP_WITH_INRANGE;
3058 emitConstantRange(Record, CR: *Range, /*EmitBitWidth=*/true);
3059 }
3060 for (const Value *Op : CE->operands()) {
3061 Record.push_back(Elt: VE.getTypeID(T: Op->getType()));
3062 Record.push_back(Elt: VE.getValueID(V: Op));
3063 }
3064 break;
3065 }
3066 case Instruction::ExtractElement:
3067 Code = bitc::CST_CODE_CE_EXTRACTELT;
3068 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
3069 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3070 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 1)->getType()));
3071 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3072 break;
3073 case Instruction::InsertElement:
3074 Code = bitc::CST_CODE_CE_INSERTELT;
3075 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3076 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3077 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 2)->getType()));
3078 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 2)));
3079 break;
3080 case Instruction::ShuffleVector:
3081 // If the return type and argument types are the same, this is a
3082 // standard shufflevector instruction. If the types are different,
3083 // then the shuffle is widening or truncating the input vectors, and
3084 // the argument type must also be encoded.
3085 if (C->getType() == C->getOperand(i: 0)->getType()) {
3086 Code = bitc::CST_CODE_CE_SHUFFLEVEC;
3087 } else {
3088 Code = bitc::CST_CODE_CE_SHUFVEC_EX;
3089 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
3090 }
3091 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3092 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3093 Record.push_back(Elt: VE.getValueID(V: CE->getShuffleMaskForBitcode()));
3094 break;
3095 }
3096 } else if (const BlockAddress *BA = dyn_cast<BlockAddress>(Val: C)) {
3097 Code = bitc::CST_CODE_BLOCKADDRESS;
3098 Record.push_back(Elt: VE.getTypeID(T: BA->getFunction()->getType()));
3099 Record.push_back(Elt: VE.getValueID(V: BA->getFunction()));
3100 Record.push_back(Elt: VE.getGlobalBasicBlockID(BB: BA->getBasicBlock()));
3101 } else if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(Val: C)) {
3102 Code = bitc::CST_CODE_DSO_LOCAL_EQUIVALENT;
3103 Record.push_back(Elt: VE.getTypeID(T: Equiv->getGlobalValue()->getType()));
3104 Record.push_back(Elt: VE.getValueID(V: Equiv->getGlobalValue()));
3105 } else if (const auto *NC = dyn_cast<NoCFIValue>(Val: C)) {
3106 Code = bitc::CST_CODE_NO_CFI_VALUE;
3107 Record.push_back(Elt: VE.getTypeID(T: NC->getGlobalValue()->getType()));
3108 Record.push_back(Elt: VE.getValueID(V: NC->getGlobalValue()));
3109 } else if (const auto *CPA = dyn_cast<ConstantPtrAuth>(Val: C)) {
3110 Code = bitc::CST_CODE_PTRAUTH2;
3111 Record.push_back(Elt: VE.getValueID(V: CPA->getPointer()));
3112 Record.push_back(Elt: VE.getValueID(V: CPA->getKey()));
3113 Record.push_back(Elt: VE.getValueID(V: CPA->getDiscriminator()));
3114 Record.push_back(Elt: VE.getValueID(V: CPA->getAddrDiscriminator()));
3115 Record.push_back(Elt: VE.getValueID(V: CPA->getDeactivationSymbol()));
3116 } else {
3117#ifndef NDEBUG
3118 C->dump();
3119#endif
3120 llvm_unreachable("Unknown constant!");
3121 }
3122 Stream.EmitRecord(Code, Vals: Record, Abbrev: AbbrevToUse);
3123 Record.clear();
3124 }
3125
3126 Stream.ExitBlock();
3127}
3128
3129void ModuleBitcodeWriter::writeModuleConstants() {
3130 const ValueEnumerator::ValueList &Vals = VE.getValues();
3131
3132 // Find the first constant to emit, which is the first non-globalvalue value.
3133 // We know globalvalues have been emitted by WriteModuleInfo.
3134 for (unsigned i = 0, e = Vals.size(); i != e; ++i) {
3135 if (!isa<GlobalValue>(Val: Vals[i].first)) {
3136 writeConstants(FirstVal: i, LastVal: Vals.size(), isGlobal: true);
3137 return;
3138 }
3139 }
3140}
3141
3142/// pushValueAndType - The file has to encode both the value and type id for
3143/// many values, because we need to know what type to create for forward
3144/// references. However, most operands are not forward references, so this type
3145/// field is not needed.
3146///
3147/// This function adds V's value ID to Vals. If the value ID is higher than the
3148/// instruction ID, then it is a forward reference, and it also includes the
3149/// type ID. The value ID that is written is encoded relative to the InstID.
3150bool ModuleBitcodeWriter::pushValueAndType(const Value *V, unsigned InstID,
3151 SmallVectorImpl<unsigned> &Vals) {
3152 unsigned ValID = VE.getValueID(V);
3153 // Make encoding relative to the InstID.
3154 Vals.push_back(Elt: InstID - ValID);
3155 if (ValID >= InstID) {
3156 Vals.push_back(Elt: VE.getTypeID(T: V->getType()));
3157 return true;
3158 }
3159 return false;
3160}
3161
3162bool ModuleBitcodeWriter::pushValueOrMetadata(const Value *V, unsigned InstID,
3163 SmallVectorImpl<unsigned> &Vals) {
3164 bool IsMetadata = V->getType()->isMetadataTy();
3165 if (IsMetadata) {
3166 Vals.push_back(Elt: bitc::OB_METADATA);
3167 Metadata *MD = cast<MetadataAsValue>(Val: V)->getMetadata();
3168 unsigned ValID = VE.getMetadataID(MD);
3169 Vals.push_back(Elt: InstID - ValID);
3170 return false;
3171 }
3172 return pushValueAndType(V, InstID, Vals);
3173}
3174
3175void ModuleBitcodeWriter::writeOperandBundles(const CallBase &CS,
3176 unsigned InstID) {
3177 SmallVector<unsigned, 64> Record;
3178 LLVMContext &C = CS.getContext();
3179
3180 for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i) {
3181 const auto &Bundle = CS.getOperandBundleAt(Index: i);
3182 Record.push_back(Elt: C.getOperandBundleTagID(Tag: Bundle.getTagName()));
3183
3184 for (auto &Input : Bundle.Inputs)
3185 pushValueOrMetadata(V: Input, InstID, Vals&: Record);
3186
3187 Stream.EmitRecord(Code: bitc::FUNC_CODE_OPERAND_BUNDLE, Vals: Record);
3188 Record.clear();
3189 }
3190}
3191
3192/// pushValue - Like pushValueAndType, but where the type of the value is
3193/// omitted (perhaps it was already encoded in an earlier operand).
3194void ModuleBitcodeWriter::pushValue(const Value *V, unsigned InstID,
3195 SmallVectorImpl<unsigned> &Vals) {
3196 unsigned ValID = VE.getValueID(V);
3197 Vals.push_back(Elt: InstID - ValID);
3198}
3199
3200void ModuleBitcodeWriter::pushValueSigned(const Value *V, unsigned InstID,
3201 SmallVectorImpl<uint64_t> &Vals) {
3202 unsigned ValID = VE.getValueID(V);
3203 int64_t diff = ((int32_t)InstID - (int32_t)ValID);
3204 emitSignedInt64(Vals, V: diff);
3205}
3206
3207/// WriteInstruction - Emit an instruction to the specified stream.
3208void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
3209 unsigned InstID,
3210 SmallVectorImpl<unsigned> &Vals) {
3211 unsigned Code = 0;
3212 unsigned AbbrevToUse = 0;
3213 VE.setInstructionID(&I);
3214 switch (I.getOpcode()) {
3215 default:
3216 if (Instruction::isCast(Opcode: I.getOpcode())) {
3217 Code = bitc::FUNC_CODE_INST_CAST;
3218 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3219 AbbrevToUse = FUNCTION_INST_CAST_ABBREV;
3220 Vals.push_back(Elt: VE.getTypeID(T: I.getType()));
3221 Vals.push_back(Elt: getEncodedCastOpcode(Opcode: I.getOpcode()));
3222 uint64_t Flags = getOptimizationFlags(V: &I);
3223 if (Flags != 0) {
3224 if (AbbrevToUse == FUNCTION_INST_CAST_ABBREV)
3225 AbbrevToUse = FUNCTION_INST_CAST_FLAGS_ABBREV;
3226 Vals.push_back(Elt: Flags);
3227 }
3228 } else {
3229 assert(isa<BinaryOperator>(I) && "Unknown instruction!");
3230 Code = bitc::FUNC_CODE_INST_BINOP;
3231 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3232 AbbrevToUse = FUNCTION_INST_BINOP_ABBREV;
3233 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3234 Vals.push_back(Elt: getEncodedBinaryOpcode(Opcode: I.getOpcode()));
3235 uint64_t Flags = getOptimizationFlags(V: &I);
3236 if (Flags != 0) {
3237 if (AbbrevToUse == FUNCTION_INST_BINOP_ABBREV)
3238 AbbrevToUse = FUNCTION_INST_BINOP_FLAGS_ABBREV;
3239 Vals.push_back(Elt: Flags);
3240 }
3241 }
3242 break;
3243 case Instruction::FNeg: {
3244 Code = bitc::FUNC_CODE_INST_UNOP;
3245 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3246 AbbrevToUse = FUNCTION_INST_UNOP_ABBREV;
3247 Vals.push_back(Elt: getEncodedUnaryOpcode(Opcode: I.getOpcode()));
3248 uint64_t Flags = getOptimizationFlags(V: &I);
3249 if (Flags != 0) {
3250 if (AbbrevToUse == FUNCTION_INST_UNOP_ABBREV)
3251 AbbrevToUse = FUNCTION_INST_UNOP_FLAGS_ABBREV;
3252 Vals.push_back(Elt: Flags);
3253 }
3254 break;
3255 }
3256 case Instruction::GetElementPtr: {
3257 Code = bitc::FUNC_CODE_INST_GEP;
3258 AbbrevToUse = FUNCTION_INST_GEP_ABBREV;
3259 auto &GEPInst = cast<GetElementPtrInst>(Val: I);
3260 Vals.push_back(Elt: getOptimizationFlags(V: &I));
3261 Vals.push_back(Elt: VE.getTypeID(T: GEPInst.getSourceElementType()));
3262 for (const Value *Op : I.operands())
3263 pushValueAndType(V: Op, InstID, Vals);
3264 break;
3265 }
3266 case Instruction::ExtractValue: {
3267 Code = bitc::FUNC_CODE_INST_EXTRACTVAL;
3268 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3269 const ExtractValueInst *EVI = cast<ExtractValueInst>(Val: &I);
3270 Vals.append(in_start: EVI->idx_begin(), in_end: EVI->idx_end());
3271 break;
3272 }
3273 case Instruction::InsertValue: {
3274 Code = bitc::FUNC_CODE_INST_INSERTVAL;
3275 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3276 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3277 const InsertValueInst *IVI = cast<InsertValueInst>(Val: &I);
3278 Vals.append(in_start: IVI->idx_begin(), in_end: IVI->idx_end());
3279 break;
3280 }
3281 case Instruction::Select: {
3282 Code = bitc::FUNC_CODE_INST_VSELECT;
3283 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3284 pushValue(V: I.getOperand(i: 2), InstID, Vals);
3285 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3286 uint64_t Flags = getOptimizationFlags(V: &I);
3287 if (Flags != 0)
3288 Vals.push_back(Elt: Flags);
3289 break;
3290 }
3291 case Instruction::ExtractElement:
3292 Code = bitc::FUNC_CODE_INST_EXTRACTELT;
3293 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3294 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3295 break;
3296 case Instruction::InsertElement:
3297 Code = bitc::FUNC_CODE_INST_INSERTELT;
3298 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3299 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3300 pushValueAndType(V: I.getOperand(i: 2), InstID, Vals);
3301 break;
3302 case Instruction::ShuffleVector:
3303 Code = bitc::FUNC_CODE_INST_SHUFFLEVEC;
3304 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3305 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3306 pushValue(V: cast<ShuffleVectorInst>(Val: I).getShuffleMaskForBitcode(), InstID,
3307 Vals);
3308 break;
3309 case Instruction::ICmp:
3310 case Instruction::FCmp: {
3311 // compare returning Int1Ty or vector of Int1Ty
3312 Code = bitc::FUNC_CODE_INST_CMP2;
3313 AbbrevToUse = FUNCTION_INST_CMP_ABBREV;
3314 if (pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3315 AbbrevToUse = 0;
3316 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3317 Vals.push_back(Elt: cast<CmpInst>(Val: I).getPredicate());
3318 uint64_t Flags = getOptimizationFlags(V: &I);
3319 if (Flags != 0) {
3320 Vals.push_back(Elt: Flags);
3321 if (AbbrevToUse)
3322 AbbrevToUse = FUNCTION_INST_CMP_FLAGS_ABBREV;
3323 }
3324 break;
3325 }
3326
3327 case Instruction::Ret:
3328 {
3329 Code = bitc::FUNC_CODE_INST_RET;
3330 unsigned NumOperands = I.getNumOperands();
3331 if (NumOperands == 0)
3332 AbbrevToUse = FUNCTION_INST_RET_VOID_ABBREV;
3333 else if (NumOperands == 1) {
3334 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3335 AbbrevToUse = FUNCTION_INST_RET_VAL_ABBREV;
3336 } else {
3337 for (const Value *Op : I.operands())
3338 pushValueAndType(V: Op, InstID, Vals);
3339 }
3340 }
3341 break;
3342 case Instruction::UncondBr: {
3343 Code = bitc::FUNC_CODE_INST_BR;
3344 AbbrevToUse = FUNCTION_INST_BR_UNCOND_ABBREV;
3345 const UncondBrInst &II = cast<UncondBrInst>(Val: I);
3346 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 0)));
3347 } break;
3348 case Instruction::CondBr: {
3349 Code = bitc::FUNC_CODE_INST_BR;
3350 AbbrevToUse = FUNCTION_INST_BR_COND_ABBREV;
3351 const CondBrInst &II = cast<CondBrInst>(Val: I);
3352 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 0)));
3353 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 1)));
3354 pushValue(V: II.getCondition(), InstID, Vals);
3355 } break;
3356 case Instruction::Switch:
3357 {
3358 Code = bitc::FUNC_CODE_INST_SWITCH;
3359 const SwitchInst &SI = cast<SwitchInst>(Val: I);
3360 Vals.push_back(Elt: VE.getTypeID(T: SI.getCondition()->getType()));
3361 pushValue(V: SI.getCondition(), InstID, Vals);
3362 Vals.push_back(Elt: VE.getValueID(V: SI.getDefaultDest()));
3363 for (auto Case : SI.cases()) {
3364 Vals.push_back(Elt: VE.getValueID(V: Case.getCaseValue()));
3365 Vals.push_back(Elt: VE.getValueID(V: Case.getCaseSuccessor()));
3366 }
3367 }
3368 break;
3369 case Instruction::IndirectBr:
3370 Code = bitc::FUNC_CODE_INST_INDIRECTBR;
3371 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType()));
3372 // Encode the address operand as relative, but not the basic blocks.
3373 pushValue(V: I.getOperand(i: 0), InstID, Vals);
3374 for (const Value *Op : drop_begin(RangeOrContainer: I.operands()))
3375 Vals.push_back(Elt: VE.getValueID(V: Op));
3376 break;
3377
3378 case Instruction::Invoke: {
3379 const InvokeInst *II = cast<InvokeInst>(Val: &I);
3380 const Value *Callee = II->getCalledOperand();
3381 FunctionType *FTy = II->getFunctionType();
3382
3383 if (II->hasOperandBundles())
3384 writeOperandBundles(CS: *II, InstID);
3385
3386 Code = bitc::FUNC_CODE_INST_INVOKE;
3387
3388 Vals.push_back(Elt: VE.getAttributeListID(PAL: II->getAttributes()));
3389 Vals.push_back(Elt: II->getCallingConv() | 1 << 13);
3390 Vals.push_back(Elt: VE.getValueID(V: II->getNormalDest()));
3391 Vals.push_back(Elt: VE.getValueID(V: II->getUnwindDest()));
3392 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3393 pushValueAndType(V: Callee, InstID, Vals);
3394
3395 // Emit value #'s for the fixed parameters.
3396 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3397 pushValue(V: I.getOperand(i), InstID, Vals); // fixed param.
3398
3399 // Emit type/value pairs for varargs params.
3400 if (FTy->isVarArg()) {
3401 for (unsigned i = FTy->getNumParams(), e = II->arg_size(); i != e; ++i)
3402 pushValueAndType(V: I.getOperand(i), InstID, Vals); // vararg
3403 }
3404 break;
3405 }
3406 case Instruction::Resume:
3407 Code = bitc::FUNC_CODE_INST_RESUME;
3408 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3409 break;
3410 case Instruction::CleanupRet: {
3411 Code = bitc::FUNC_CODE_INST_CLEANUPRET;
3412 const auto &CRI = cast<CleanupReturnInst>(Val: I);
3413 pushValue(V: CRI.getCleanupPad(), InstID, Vals);
3414 if (CRI.hasUnwindDest())
3415 Vals.push_back(Elt: VE.getValueID(V: CRI.getUnwindDest()));
3416 break;
3417 }
3418 case Instruction::CatchRet: {
3419 Code = bitc::FUNC_CODE_INST_CATCHRET;
3420 const auto &CRI = cast<CatchReturnInst>(Val: I);
3421 pushValue(V: CRI.getCatchPad(), InstID, Vals);
3422 Vals.push_back(Elt: VE.getValueID(V: CRI.getSuccessor()));
3423 break;
3424 }
3425 case Instruction::CleanupPad:
3426 case Instruction::CatchPad: {
3427 const auto &FuncletPad = cast<FuncletPadInst>(Val: I);
3428 Code = isa<CatchPadInst>(Val: FuncletPad) ? bitc::FUNC_CODE_INST_CATCHPAD
3429 : bitc::FUNC_CODE_INST_CLEANUPPAD;
3430 pushValue(V: FuncletPad.getParentPad(), InstID, Vals);
3431
3432 unsigned NumArgOperands = FuncletPad.arg_size();
3433 Vals.push_back(Elt: NumArgOperands);
3434 for (unsigned Op = 0; Op != NumArgOperands; ++Op)
3435 pushValueAndType(V: FuncletPad.getArgOperand(i: Op), InstID, Vals);
3436 break;
3437 }
3438 case Instruction::CatchSwitch: {
3439 Code = bitc::FUNC_CODE_INST_CATCHSWITCH;
3440 const auto &CatchSwitch = cast<CatchSwitchInst>(Val: I);
3441
3442 pushValue(V: CatchSwitch.getParentPad(), InstID, Vals);
3443
3444 unsigned NumHandlers = CatchSwitch.getNumHandlers();
3445 Vals.push_back(Elt: NumHandlers);
3446 for (const BasicBlock *CatchPadBB : CatchSwitch.handlers())
3447 Vals.push_back(Elt: VE.getValueID(V: CatchPadBB));
3448
3449 if (CatchSwitch.hasUnwindDest())
3450 Vals.push_back(Elt: VE.getValueID(V: CatchSwitch.getUnwindDest()));
3451 break;
3452 }
3453 case Instruction::CallBr: {
3454 const CallBrInst *CBI = cast<CallBrInst>(Val: &I);
3455 const Value *Callee = CBI->getCalledOperand();
3456 FunctionType *FTy = CBI->getFunctionType();
3457
3458 if (CBI->hasOperandBundles())
3459 writeOperandBundles(CS: *CBI, InstID);
3460
3461 Code = bitc::FUNC_CODE_INST_CALLBR;
3462
3463 Vals.push_back(Elt: VE.getAttributeListID(PAL: CBI->getAttributes()));
3464
3465 Vals.push_back(Elt: CBI->getCallingConv() << bitc::CALL_CCONV |
3466 1 << bitc::CALL_EXPLICIT_TYPE);
3467
3468 Vals.push_back(Elt: VE.getValueID(V: CBI->getDefaultDest()));
3469 Vals.push_back(Elt: CBI->getNumIndirectDests());
3470 for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i)
3471 Vals.push_back(Elt: VE.getValueID(V: CBI->getIndirectDest(i)));
3472
3473 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3474 pushValueAndType(V: Callee, InstID, Vals);
3475
3476 // Emit value #'s for the fixed parameters.
3477 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3478 pushValue(V: I.getOperand(i), InstID, Vals); // fixed param.
3479
3480 // Emit type/value pairs for varargs params.
3481 if (FTy->isVarArg()) {
3482 for (unsigned i = FTy->getNumParams(), e = CBI->arg_size(); i != e; ++i)
3483 pushValueAndType(V: I.getOperand(i), InstID, Vals); // vararg
3484 }
3485 break;
3486 }
3487 case Instruction::Unreachable:
3488 Code = bitc::FUNC_CODE_INST_UNREACHABLE;
3489 AbbrevToUse = FUNCTION_INST_UNREACHABLE_ABBREV;
3490 break;
3491
3492 case Instruction::PHI: {
3493 const PHINode &PN = cast<PHINode>(Val: I);
3494 Code = bitc::FUNC_CODE_INST_PHI;
3495 // With the newer instruction encoding, forward references could give
3496 // negative valued IDs. This is most common for PHIs, so we use
3497 // signed VBRs.
3498 SmallVector<uint64_t, 128> Vals64;
3499 Vals64.push_back(Elt: VE.getTypeID(T: PN.getType()));
3500 for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) {
3501 pushValueSigned(V: PN.getIncomingValue(i), InstID, Vals&: Vals64);
3502 Vals64.push_back(Elt: VE.getValueID(V: PN.getIncomingBlock(i)));
3503 }
3504
3505 uint64_t Flags = getOptimizationFlags(V: &I);
3506 if (Flags != 0)
3507 Vals64.push_back(Elt: Flags);
3508
3509 // Emit a Vals64 vector and exit.
3510 Stream.EmitRecord(Code, Vals: Vals64, Abbrev: AbbrevToUse);
3511 Vals64.clear();
3512 return;
3513 }
3514
3515 case Instruction::LandingPad: {
3516 const LandingPadInst &LP = cast<LandingPadInst>(Val: I);
3517 Code = bitc::FUNC_CODE_INST_LANDINGPAD;
3518 Vals.push_back(Elt: VE.getTypeID(T: LP.getType()));
3519 Vals.push_back(Elt: LP.isCleanup());
3520 Vals.push_back(Elt: LP.getNumClauses());
3521 for (unsigned I = 0, E = LP.getNumClauses(); I != E; ++I) {
3522 if (LP.isCatch(Idx: I))
3523 Vals.push_back(Elt: LandingPadInst::Catch);
3524 else
3525 Vals.push_back(Elt: LandingPadInst::Filter);
3526 pushValueAndType(V: LP.getClause(Idx: I), InstID, Vals);
3527 }
3528 break;
3529 }
3530
3531 case Instruction::Alloca: {
3532 Code = bitc::FUNC_CODE_INST_ALLOCA;
3533 const AllocaInst &AI = cast<AllocaInst>(Val: I);
3534 Vals.push_back(Elt: VE.getTypeID(T: AI.getAllocatedType()));
3535 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType()));
3536 Vals.push_back(Elt: VE.getValueID(V: I.getOperand(i: 0))); // size.
3537 using APV = AllocaPackedValues;
3538 unsigned Record = 0;
3539 unsigned EncodedAlign = getEncodedAlign(Alignment: AI.getAlign());
3540 Bitfield::set<APV::AlignLower>(
3541 Packed&: Record, Value: EncodedAlign & ((1 << APV::AlignLower::Bits) - 1));
3542 Bitfield::set<APV::AlignUpper>(Packed&: Record,
3543 Value: EncodedAlign >> APV::AlignLower::Bits);
3544 Bitfield::set<APV::UsedWithInAlloca>(Packed&: Record, Value: AI.isUsedWithInAlloca());
3545 Bitfield::set<APV::ExplicitType>(Packed&: Record, Value: true);
3546 Bitfield::set<APV::SwiftError>(Packed&: Record, Value: AI.isSwiftError());
3547 Vals.push_back(Elt: Record);
3548
3549 unsigned AS = AI.getAddressSpace();
3550 if (AS != M.getDataLayout().getAllocaAddrSpace())
3551 Vals.push_back(Elt: AS);
3552 break;
3553 }
3554
3555 case Instruction::Load:
3556 if (cast<LoadInst>(Val: I).isAtomic()) {
3557 Code = bitc::FUNC_CODE_INST_LOADATOMIC;
3558 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3559 } else {
3560 Code = bitc::FUNC_CODE_INST_LOAD;
3561 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals)) // ptr
3562 AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
3563 }
3564 Vals.push_back(Elt: VE.getTypeID(T: I.getType()));
3565 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<LoadInst>(Val: I).getAlign()));
3566 Vals.push_back(Elt: cast<LoadInst>(Val: I).isVolatile());
3567 if (cast<LoadInst>(Val: I).isAtomic()) {
3568 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<LoadInst>(Val: I).getOrdering()));
3569 Vals.push_back(Elt: getEncodedSyncScopeID(SSID: cast<LoadInst>(Val: I).getSyncScopeID()));
3570 }
3571 break;
3572 case Instruction::Store:
3573 if (cast<StoreInst>(Val: I).isAtomic()) {
3574 Code = bitc::FUNC_CODE_INST_STOREATOMIC;
3575 } else {
3576 Code = bitc::FUNC_CODE_INST_STORE;
3577 AbbrevToUse = FUNCTION_INST_STORE_ABBREV;
3578 }
3579 if (pushValueAndType(V: I.getOperand(i: 1), InstID, Vals)) // ptrty + ptr
3580 AbbrevToUse = 0;
3581 if (pushValueAndType(V: I.getOperand(i: 0), InstID, Vals)) // valty + val
3582 AbbrevToUse = 0;
3583 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<StoreInst>(Val: I).getAlign()));
3584 Vals.push_back(Elt: cast<StoreInst>(Val: I).isVolatile());
3585 if (cast<StoreInst>(Val: I).isAtomic()) {
3586 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<StoreInst>(Val: I).getOrdering()));
3587 Vals.push_back(
3588 Elt: getEncodedSyncScopeID(SSID: cast<StoreInst>(Val: I).getSyncScopeID()));
3589 }
3590 break;
3591 case Instruction::AtomicCmpXchg:
3592 Code = bitc::FUNC_CODE_INST_CMPXCHG;
3593 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals); // ptrty + ptr
3594 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals); // cmp.
3595 pushValue(V: I.getOperand(i: 2), InstID, Vals); // newval.
3596 Vals.push_back(Elt: cast<AtomicCmpXchgInst>(Val: I).isVolatile());
3597 Vals.push_back(
3598 Elt: getEncodedOrdering(Ordering: cast<AtomicCmpXchgInst>(Val: I).getSuccessOrdering()));
3599 Vals.push_back(
3600 Elt: getEncodedSyncScopeID(SSID: cast<AtomicCmpXchgInst>(Val: I).getSyncScopeID()));
3601 Vals.push_back(
3602 Elt: getEncodedOrdering(Ordering: cast<AtomicCmpXchgInst>(Val: I).getFailureOrdering()));
3603 Vals.push_back(Elt: cast<AtomicCmpXchgInst>(Val: I).isWeak());
3604 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<AtomicCmpXchgInst>(Val: I).getAlign()));
3605 break;
3606 case Instruction::AtomicRMW:
3607 Code = bitc::FUNC_CODE_INST_ATOMICRMW;
3608 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals); // ptrty + ptr
3609 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals); // valty + val
3610 Vals.push_back(Elt: getEncodedRMWOperation(I: cast<AtomicRMWInst>(Val: I)));
3611 Vals.push_back(Elt: cast<AtomicRMWInst>(Val: I).isVolatile());
3612 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<AtomicRMWInst>(Val: I).getOrdering()));
3613 Vals.push_back(
3614 Elt: getEncodedSyncScopeID(SSID: cast<AtomicRMWInst>(Val: I).getSyncScopeID()));
3615 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<AtomicRMWInst>(Val: I).getAlign()));
3616 break;
3617 case Instruction::Fence:
3618 Code = bitc::FUNC_CODE_INST_FENCE;
3619 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<FenceInst>(Val: I).getOrdering()));
3620 Vals.push_back(Elt: getEncodedSyncScopeID(SSID: cast<FenceInst>(Val: I).getSyncScopeID()));
3621 break;
3622 case Instruction::Call: {
3623 const CallInst &CI = cast<CallInst>(Val: I);
3624 FunctionType *FTy = CI.getFunctionType();
3625
3626 if (CI.hasOperandBundles())
3627 writeOperandBundles(CS: CI, InstID);
3628
3629 Code = bitc::FUNC_CODE_INST_CALL;
3630
3631 Vals.push_back(Elt: VE.getAttributeListID(PAL: CI.getAttributes()));
3632
3633 unsigned Flags = getOptimizationFlags(V: &I);
3634 Vals.push_back(Elt: CI.getCallingConv() << bitc::CALL_CCONV |
3635 unsigned(CI.isTailCall()) << bitc::CALL_TAIL |
3636 unsigned(CI.isMustTailCall()) << bitc::CALL_MUSTTAIL |
3637 1 << bitc::CALL_EXPLICIT_TYPE |
3638 unsigned(CI.isNoTailCall()) << bitc::CALL_NOTAIL |
3639 unsigned(Flags != 0) << bitc::CALL_FMF);
3640 if (Flags != 0)
3641 Vals.push_back(Elt: Flags);
3642
3643 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3644 pushValueAndType(V: CI.getCalledOperand(), InstID, Vals); // Callee
3645
3646 // Emit value #'s for the fixed parameters.
3647 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3648 pushValue(V: CI.getArgOperand(i), InstID, Vals); // fixed param.
3649
3650 // Emit type/value pairs for varargs params.
3651 if (FTy->isVarArg()) {
3652 for (unsigned i = FTy->getNumParams(), e = CI.arg_size(); i != e; ++i)
3653 pushValueAndType(V: CI.getArgOperand(i), InstID, Vals); // varargs
3654 }
3655 break;
3656 }
3657 case Instruction::VAArg:
3658 Code = bitc::FUNC_CODE_INST_VAARG;
3659 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType())); // valistty
3660 pushValue(V: I.getOperand(i: 0), InstID, Vals); // valist.
3661 Vals.push_back(Elt: VE.getTypeID(T: I.getType())); // restype.
3662 break;
3663 case Instruction::Freeze:
3664 Code = bitc::FUNC_CODE_INST_FREEZE;
3665 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3666 break;
3667 }
3668
3669 Stream.EmitRecord(Code, Vals, Abbrev: AbbrevToUse);
3670 Vals.clear();
3671}
3672
3673/// Write a GlobalValue VST to the module. The purpose of this data structure is
3674/// to allow clients to efficiently find the function body.
3675void ModuleBitcodeWriter::writeGlobalValueSymbolTable(
3676 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3677 // Get the offset of the VST we are writing, and backpatch it into
3678 // the VST forward declaration record.
3679 uint64_t VSTOffset = Stream.GetCurrentBitNo();
3680 // The BitcodeStartBit was the stream offset of the identification block.
3681 VSTOffset -= bitcodeStartBit();
3682 assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned");
3683 // Note that we add 1 here because the offset is relative to one word
3684 // before the start of the identification block, which was historically
3685 // always the start of the regular bitcode header.
3686 Stream.BackpatchWord(BitNo: VSTOffsetPlaceholder, Val: VSTOffset / 32 + 1);
3687
3688 Stream.EnterSubblock(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, CodeLen: 4);
3689
3690 auto Abbv = std::make_shared<BitCodeAbbrev>();
3691 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY));
3692 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
3693 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset
3694 unsigned FnEntryAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
3695
3696 for (const Function &F : M) {
3697 uint64_t Record[2];
3698
3699 if (F.isDeclaration())
3700 continue;
3701
3702 Record[0] = VE.getValueID(V: &F);
3703
3704 // Save the word offset of the function (from the start of the
3705 // actual bitcode written to the stream).
3706 uint64_t BitcodeIndex = FunctionToBitcodeIndex[&F] - bitcodeStartBit();
3707 assert((BitcodeIndex & 31) == 0 && "function block not 32-bit aligned");
3708 // Note that we add 1 here because the offset is relative to one word
3709 // before the start of the identification block, which was historically
3710 // always the start of the regular bitcode header.
3711 Record[1] = BitcodeIndex / 32 + 1;
3712
3713 Stream.EmitRecord(Code: bitc::VST_CODE_FNENTRY, Vals: Record, Abbrev: FnEntryAbbrev);
3714 }
3715
3716 Stream.ExitBlock();
3717}
3718
3719/// Emit names for arguments, instructions and basic blocks in a function.
3720void ModuleBitcodeWriter::writeFunctionLevelValueSymbolTable(
3721 const ValueSymbolTable &VST) {
3722 if (VST.empty())
3723 return;
3724
3725 Stream.EnterSubblock(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, CodeLen: 4);
3726
3727 // FIXME: Set up the abbrev, we know how many values there are!
3728 // FIXME: We know if the type names can use 7-bit ascii.
3729 SmallVector<uint64_t, 64> NameVals;
3730
3731 for (const ValueName &Name : VST) {
3732 // Figure out the encoding to use for the name.
3733 StringEncoding Bits = getStringEncoding(Str: Name.getKey());
3734
3735 unsigned AbbrevToUse = VST_ENTRY_8_ABBREV;
3736 NameVals.push_back(Elt: VE.getValueID(V: Name.getValue()));
3737
3738 // VST_CODE_ENTRY: [valueid, namechar x N]
3739 // VST_CODE_BBENTRY: [bbid, namechar x N]
3740 unsigned Code;
3741 if (isa<BasicBlock>(Val: Name.getValue())) {
3742 Code = bitc::VST_CODE_BBENTRY;
3743 if (Bits == SE_Char6)
3744 AbbrevToUse = VST_BBENTRY_6_ABBREV;
3745 } else {
3746 Code = bitc::VST_CODE_ENTRY;
3747 if (Bits == SE_Char6)
3748 AbbrevToUse = VST_ENTRY_6_ABBREV;
3749 else if (Bits == SE_Fixed7)
3750 AbbrevToUse = VST_ENTRY_7_ABBREV;
3751 }
3752
3753 for (const auto P : Name.getKey())
3754 NameVals.push_back(Elt: (unsigned char)P);
3755
3756 // Emit the finished record.
3757 Stream.EmitRecord(Code, Vals: NameVals, Abbrev: AbbrevToUse);
3758 NameVals.clear();
3759 }
3760
3761 Stream.ExitBlock();
3762}
3763
3764void ModuleBitcodeWriter::writeUseList(UseListOrder &&Order) {
3765 assert(Order.Shuffle.size() >= 2 && "Shuffle too small");
3766 unsigned Code;
3767 if (isa<BasicBlock>(Val: Order.V))
3768 Code = bitc::USELIST_CODE_BB;
3769 else
3770 Code = bitc::USELIST_CODE_DEFAULT;
3771
3772 SmallVector<uint64_t, 64> Record(Order.Shuffle.begin(), Order.Shuffle.end());
3773 Record.push_back(Elt: VE.getValueID(V: Order.V));
3774 Stream.EmitRecord(Code, Vals: Record);
3775}
3776
3777void ModuleBitcodeWriter::writeUseListBlock(const Function *F) {
3778 assert(VE.shouldPreserveUseListOrder() &&
3779 "Expected to be preserving use-list order");
3780
3781 auto hasMore = [&]() {
3782 return !VE.UseListOrders.empty() && VE.UseListOrders.back().F == F;
3783 };
3784 if (!hasMore())
3785 // Nothing to do.
3786 return;
3787
3788 Stream.EnterSubblock(BlockID: bitc::USELIST_BLOCK_ID, CodeLen: 3);
3789 while (hasMore()) {
3790 writeUseList(Order: std::move(VE.UseListOrders.back()));
3791 VE.UseListOrders.pop_back();
3792 }
3793 Stream.ExitBlock();
3794}
3795
3796/// Emit a function body to the module stream.
3797void ModuleBitcodeWriter::writeFunction(
3798 const Function &F,
3799 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3800 // Save the bitcode index of the start of this function block for recording
3801 // in the VST.
3802 FunctionToBitcodeIndex[&F] = Stream.GetCurrentBitNo();
3803
3804 Stream.EnterSubblock(BlockID: bitc::FUNCTION_BLOCK_ID, CodeLen: 5);
3805 VE.incorporateFunction(F);
3806
3807 SmallVector<unsigned, 64> Vals;
3808
3809 // Emit the number of basic blocks, so the reader can create them ahead of
3810 // time.
3811 Vals.push_back(Elt: VE.getBasicBlocks().size());
3812 Stream.EmitRecord(Code: bitc::FUNC_CODE_DECLAREBLOCKS, Vals);
3813 Vals.clear();
3814
3815 // If there are function-local constants, emit them now.
3816 unsigned CstStart, CstEnd;
3817 VE.getFunctionConstantRange(Start&: CstStart, End&: CstEnd);
3818 writeConstants(FirstVal: CstStart, LastVal: CstEnd, isGlobal: false);
3819
3820 // If there is function-local metadata, emit it now.
3821 writeFunctionMetadata(F);
3822
3823 // Keep a running idea of what the instruction ID is.
3824 unsigned InstID = CstEnd;
3825
3826 bool NeedsMetadataAttachment = F.hasMetadata();
3827
3828 DILocation *LastDL = nullptr;
3829 SmallSetVector<Function *, 4> BlockAddressUsers;
3830
3831 // Finally, emit all the instructions, in order.
3832 for (const BasicBlock &BB : F) {
3833 for (const Instruction &I : BB) {
3834 writeInstruction(I, InstID, Vals);
3835
3836 if (!I.getType()->isVoidTy())
3837 ++InstID;
3838
3839 // If the instruction has metadata, write a metadata attachment later.
3840 NeedsMetadataAttachment |= I.hasMetadataOtherThanDebugLoc();
3841
3842 // If the instruction has a debug location, emit it.
3843 if (DILocation *DL = I.getDebugLoc()) {
3844 if (DL == LastDL) {
3845 // Just repeat the same debug loc as last time.
3846 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_LOC_AGAIN, Vals);
3847 } else {
3848 Vals.push_back(Elt: DL->getLine());
3849 Vals.push_back(Elt: DL->getColumn());
3850 Vals.push_back(Elt: VE.getMetadataOrNullID(MD: DL->getScope()));
3851 Vals.push_back(Elt: VE.getMetadataOrNullID(MD: DL->getInlinedAt()));
3852 Vals.push_back(Elt: DL->isImplicitCode());
3853 Vals.push_back(Elt: DL->getAtomGroup());
3854 Vals.push_back(Elt: DL->getAtomRank());
3855 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_LOC, Vals,
3856 Abbrev: FUNCTION_DEBUG_LOC_ABBREV);
3857 Vals.clear();
3858 LastDL = DL;
3859 }
3860 }
3861
3862 // If the instruction has DbgRecords attached to it, emit them. Note that
3863 // they come after the instruction so that it's easy to attach them again
3864 // when reading the bitcode, even though conceptually the debug locations
3865 // start "before" the instruction.
3866 if (I.hasDbgRecords()) {
3867 /// Try to push the value only (unwrapped), otherwise push the
3868 /// metadata wrapped value. Returns true if the value was pushed
3869 /// without the ValueAsMetadata wrapper.
3870 auto PushValueOrMetadata = [&Vals, InstID,
3871 this](Metadata *RawLocation) {
3872 assert(RawLocation &&
3873 "RawLocation unexpectedly null in DbgVariableRecord");
3874 if (ValueAsMetadata *VAM = dyn_cast<ValueAsMetadata>(Val: RawLocation)) {
3875 SmallVector<unsigned, 2> ValAndType;
3876 // If the value is a fwd-ref the type is also pushed. We don't
3877 // want the type, so fwd-refs are kept wrapped (pushValueAndType
3878 // returns false if the value is pushed without type).
3879 if (!pushValueAndType(V: VAM->getValue(), InstID, Vals&: ValAndType)) {
3880 Vals.push_back(Elt: ValAndType[0]);
3881 return true;
3882 }
3883 }
3884 // The metadata is a DIArgList, or ValueAsMetadata wrapping a
3885 // fwd-ref. Push the metadata ID.
3886 Vals.push_back(Elt: VE.getMetadataID(MD: RawLocation));
3887 return false;
3888 };
3889
3890 // Write out non-instruction debug information attached to this
3891 // instruction. Write it after the instruction so that it's easy to
3892 // re-attach to the instruction reading the records in.
3893 for (DbgRecord &DR : I.DebugMarker->getDbgRecordRange()) {
3894 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
3895 Vals.push_back(Elt: VE.getMetadataID(MD: &*DLR->getDebugLoc()));
3896 Vals.push_back(Elt: VE.getMetadataID(MD: DLR->getLabel()));
3897 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_LABEL, Vals);
3898 Vals.clear();
3899 continue;
3900 }
3901
3902 // First 3 fields are common to all kinds:
3903 // DILocation, DILocalVariable, DIExpression
3904 // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE)
3905 // ..., LocationMetadata
3906 // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE - abbrev'd)
3907 // ..., Value
3908 // dbg_declare (FUNC_CODE_DEBUG_RECORD_DECLARE)
3909 // ..., LocationMetadata
3910 // dbg_assign (FUNC_CODE_DEBUG_RECORD_ASSIGN)
3911 // ..., LocationMetadata, DIAssignID, DIExpression, LocationMetadata
3912 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
3913 Vals.push_back(Elt: VE.getMetadataID(MD: &*DVR.getDebugLoc()));
3914 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getVariable()));
3915 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getExpression()));
3916 if (DVR.isDbgValue()) {
3917 if (PushValueOrMetadata(DVR.getRawLocation()))
3918 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE, Vals,
3919 Abbrev: FUNCTION_DEBUG_RECORD_VALUE_ABBREV);
3920 else
3921 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_VALUE, Vals);
3922 } else if (DVR.isDbgDeclare()) {
3923 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3924 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_DECLARE, Vals);
3925 } else if (DVR.isDbgDeclareValue()) {
3926 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3927 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_DECLARE_VALUE, Vals);
3928 } else {
3929 assert(DVR.isDbgAssign() && "Unexpected DbgRecord kind");
3930 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3931 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getAssignID()));
3932 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getAddressExpression()));
3933 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawAddress()));
3934 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_ASSIGN, Vals);
3935 }
3936 Vals.clear();
3937 }
3938 }
3939 }
3940
3941 if (BlockAddress *BA = BlockAddress::lookup(BB: &BB)) {
3942 SmallVector<Value *> Worklist{BA};
3943 SmallPtrSet<Value *, 8> Visited{BA};
3944 while (!Worklist.empty()) {
3945 Value *V = Worklist.pop_back_val();
3946 for (User *U : V->users()) {
3947 if (auto *I = dyn_cast<Instruction>(Val: U)) {
3948 Function *P = I->getFunction();
3949 if (P != &F)
3950 BlockAddressUsers.insert(X: P);
3951 } else if (isa<Constant>(Val: U) && !isa<GlobalValue>(Val: U) &&
3952 Visited.insert(Ptr: U).second)
3953 Worklist.push_back(Elt: U);
3954 }
3955 }
3956 }
3957 }
3958
3959 if (!BlockAddressUsers.empty()) {
3960 Vals.resize(N: BlockAddressUsers.size());
3961 for (auto I : llvm::enumerate(First&: BlockAddressUsers))
3962 Vals[I.index()] = VE.getValueID(V: I.value());
3963 Stream.EmitRecord(Code: bitc::FUNC_CODE_BLOCKADDR_USERS, Vals);
3964 Vals.clear();
3965 }
3966
3967 // Emit names for all the instructions etc.
3968 if (auto *Symtab = F.getValueSymbolTable())
3969 writeFunctionLevelValueSymbolTable(VST: *Symtab);
3970
3971 if (NeedsMetadataAttachment)
3972 writeFunctionMetadataAttachment(F);
3973 if (VE.shouldPreserveUseListOrder())
3974 writeUseListBlock(F: &F);
3975 VE.purgeFunction();
3976 Stream.ExitBlock();
3977}
3978
3979// Emit blockinfo, which defines the standard abbreviations etc.
3980void ModuleBitcodeWriter::writeBlockInfo() {
3981 // We only want to emit block info records for blocks that have multiple
3982 // instances: CONSTANTS_BLOCK, FUNCTION_BLOCK and VALUE_SYMTAB_BLOCK.
3983 // Other blocks can define their abbrevs inline.
3984 Stream.EnterBlockInfoBlock();
3985
3986 // Encode type indices using fixed size based on number of types.
3987 BitCodeAbbrevOp TypeAbbrevOp(BitCodeAbbrevOp::Fixed,
3988 VE.computeBitsRequiredForTypeIndices());
3989 // Encode value indices as 6-bit VBR.
3990 BitCodeAbbrevOp ValAbbrevOp(BitCodeAbbrevOp::VBR, 6);
3991
3992 { // 8-bit fixed-width VST_CODE_ENTRY/VST_CODE_BBENTRY strings.
3993 auto Abbv = std::make_shared<BitCodeAbbrev>();
3994 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3));
3995 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3996 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3997 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
3998 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3999 VST_ENTRY_8_ABBREV)
4000 llvm_unreachable("Unexpected abbrev ordering!");
4001 }
4002
4003 { // 7-bit fixed width VST_CODE_ENTRY strings.
4004 auto Abbv = std::make_shared<BitCodeAbbrev>();
4005 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
4006 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4007 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4008 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
4009 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
4010 VST_ENTRY_7_ABBREV)
4011 llvm_unreachable("Unexpected abbrev ordering!");
4012 }
4013 { // 6-bit char6 VST_CODE_ENTRY strings.
4014 auto Abbv = std::make_shared<BitCodeAbbrev>();
4015 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
4016 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4017 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4018 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
4019 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
4020 VST_ENTRY_6_ABBREV)
4021 llvm_unreachable("Unexpected abbrev ordering!");
4022 }
4023 { // 6-bit char6 VST_CODE_BBENTRY strings.
4024 auto Abbv = std::make_shared<BitCodeAbbrev>();
4025 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_BBENTRY));
4026 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4027 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4028 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
4029 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
4030 VST_BBENTRY_6_ABBREV)
4031 llvm_unreachable("Unexpected abbrev ordering!");
4032 }
4033
4034 { // SETTYPE abbrev for CONSTANTS_BLOCK.
4035 auto Abbv = std::make_shared<BitCodeAbbrev>();
4036 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_SETTYPE));
4037 Abbv->Add(OpInfo: TypeAbbrevOp);
4038 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4039 CONSTANTS_SETTYPE_ABBREV)
4040 llvm_unreachable("Unexpected abbrev ordering!");
4041 }
4042
4043 { // INTEGER abbrev for CONSTANTS_BLOCK.
4044 auto Abbv = std::make_shared<BitCodeAbbrev>();
4045 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_INTEGER));
4046 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4047 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4048 CONSTANTS_INTEGER_ABBREV)
4049 llvm_unreachable("Unexpected abbrev ordering!");
4050 }
4051
4052 { // BYTE abbrev for CONSTANTS_BLOCK.
4053 auto Abbv = std::make_shared<BitCodeAbbrev>();
4054 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_BYTE));
4055 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4056 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4057 CONSTANTS_BYTE_ABBREV)
4058 llvm_unreachable("Unexpected abbrev ordering!");
4059 }
4060
4061 { // CE_CAST abbrev for CONSTANTS_BLOCK.
4062 auto Abbv = std::make_shared<BitCodeAbbrev>();
4063 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CE_CAST));
4064 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // cast opc
4065 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // typeid
4066 VE.computeBitsRequiredForTypeIndices()));
4067 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
4068
4069 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4070 CONSTANTS_CE_CAST_Abbrev)
4071 llvm_unreachable("Unexpected abbrev ordering!");
4072 }
4073 { // NULL abbrev for CONSTANTS_BLOCK.
4074 auto Abbv = std::make_shared<BitCodeAbbrev>();
4075 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_NULL));
4076 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4077 CONSTANTS_NULL_Abbrev)
4078 llvm_unreachable("Unexpected abbrev ordering!");
4079 }
4080
4081 // FIXME: This should only use space for first class types!
4082
4083 { // INST_LOAD abbrev for FUNCTION_BLOCK.
4084 auto Abbv = std::make_shared<BitCodeAbbrev>();
4085 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_LOAD));
4086 Abbv->Add(OpInfo: ValAbbrevOp); // Ptr
4087 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4088 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Align
4089 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
4090 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4091 FUNCTION_INST_LOAD_ABBREV)
4092 llvm_unreachable("Unexpected abbrev ordering!");
4093 }
4094 {
4095 auto Abbv = std::make_shared<BitCodeAbbrev>();
4096 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_STORE));
4097 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4098 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4099 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // align
4100 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
4101 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4102 FUNCTION_INST_STORE_ABBREV)
4103 llvm_unreachable("Unexpected abbrev ordering!");
4104 }
4105 { // INST_UNOP abbrev for FUNCTION_BLOCK.
4106 auto Abbv = std::make_shared<BitCodeAbbrev>();
4107 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
4108 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4109 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4110 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4111 FUNCTION_INST_UNOP_ABBREV)
4112 llvm_unreachable("Unexpected abbrev ordering!");
4113 }
4114 { // INST_UNOP_FLAGS abbrev for FUNCTION_BLOCK.
4115 auto Abbv = std::make_shared<BitCodeAbbrev>();
4116 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
4117 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4118 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4119 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4120 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4121 FUNCTION_INST_UNOP_FLAGS_ABBREV)
4122 llvm_unreachable("Unexpected abbrev ordering!");
4123 }
4124 { // INST_BINOP abbrev for FUNCTION_BLOCK.
4125 auto Abbv = std::make_shared<BitCodeAbbrev>();
4126 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
4127 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4128 Abbv->Add(OpInfo: ValAbbrevOp); // RHS
4129 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4130 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4131 FUNCTION_INST_BINOP_ABBREV)
4132 llvm_unreachable("Unexpected abbrev ordering!");
4133 }
4134 { // INST_BINOP_FLAGS abbrev for FUNCTION_BLOCK.
4135 auto Abbv = std::make_shared<BitCodeAbbrev>();
4136 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
4137 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4138 Abbv->Add(OpInfo: ValAbbrevOp); // RHS
4139 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4140 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4141 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4142 FUNCTION_INST_BINOP_FLAGS_ABBREV)
4143 llvm_unreachable("Unexpected abbrev ordering!");
4144 }
4145 { // INST_CAST abbrev for FUNCTION_BLOCK.
4146 auto Abbv = std::make_shared<BitCodeAbbrev>();
4147 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
4148 Abbv->Add(OpInfo: ValAbbrevOp); // OpVal
4149 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4150 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4151 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4152 FUNCTION_INST_CAST_ABBREV)
4153 llvm_unreachable("Unexpected abbrev ordering!");
4154 }
4155 { // INST_CAST_FLAGS abbrev for FUNCTION_BLOCK.
4156 auto Abbv = std::make_shared<BitCodeAbbrev>();
4157 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
4158 Abbv->Add(OpInfo: ValAbbrevOp); // OpVal
4159 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4160 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4161 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 9)); // flags
4162 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4163 FUNCTION_INST_CAST_FLAGS_ABBREV)
4164 llvm_unreachable("Unexpected abbrev ordering!");
4165 }
4166
4167 { // INST_RET abbrev for FUNCTION_BLOCK.
4168 auto Abbv = std::make_shared<BitCodeAbbrev>();
4169 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
4170 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4171 FUNCTION_INST_RET_VOID_ABBREV)
4172 llvm_unreachable("Unexpected abbrev ordering!");
4173 }
4174 { // INST_RET abbrev for FUNCTION_BLOCK.
4175 auto Abbv = std::make_shared<BitCodeAbbrev>();
4176 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
4177 Abbv->Add(OpInfo: ValAbbrevOp);
4178 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4179 FUNCTION_INST_RET_VAL_ABBREV)
4180 llvm_unreachable("Unexpected abbrev ordering!");
4181 }
4182 {
4183 auto Abbv = std::make_shared<BitCodeAbbrev>();
4184 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BR));
4185 // TODO: Use different abbrev for absolute value reference (succ0)?
4186 Abbv->Add(OpInfo: ValAbbrevOp); // succ0
4187 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4188 FUNCTION_INST_BR_UNCOND_ABBREV)
4189 llvm_unreachable("Unexpected abbrev ordering!");
4190 }
4191 {
4192 auto Abbv = std::make_shared<BitCodeAbbrev>();
4193 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BR));
4194 // TODO: Use different abbrev for absolute value references (succ0, succ1)?
4195 Abbv->Add(OpInfo: ValAbbrevOp); // succ0
4196 Abbv->Add(OpInfo: ValAbbrevOp); // succ1
4197 Abbv->Add(OpInfo: ValAbbrevOp); // cond
4198 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4199 FUNCTION_INST_BR_COND_ABBREV)
4200 llvm_unreachable("Unexpected abbrev ordering!");
4201 }
4202 { // INST_UNREACHABLE abbrev for FUNCTION_BLOCK.
4203 auto Abbv = std::make_shared<BitCodeAbbrev>();
4204 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNREACHABLE));
4205 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4206 FUNCTION_INST_UNREACHABLE_ABBREV)
4207 llvm_unreachable("Unexpected abbrev ordering!");
4208 }
4209 {
4210 auto Abbv = std::make_shared<BitCodeAbbrev>();
4211 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_GEP));
4212 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // flags
4213 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4214 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4215 Abbv->Add(OpInfo: ValAbbrevOp);
4216 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4217 FUNCTION_INST_GEP_ABBREV)
4218 llvm_unreachable("Unexpected abbrev ordering!");
4219 }
4220 {
4221 auto Abbv = std::make_shared<BitCodeAbbrev>();
4222 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CMP2));
4223 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4224 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4225 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // pred
4226 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4227 FUNCTION_INST_CMP_ABBREV)
4228 llvm_unreachable("Unexpected abbrev ordering!");
4229 }
4230 {
4231 auto Abbv = std::make_shared<BitCodeAbbrev>();
4232 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CMP2));
4233 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4234 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4235 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // pred
4236 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4237 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4238 FUNCTION_INST_CMP_FLAGS_ABBREV)
4239 llvm_unreachable("Unexpected abbrev ordering!");
4240 }
4241 {
4242 auto Abbv = std::make_shared<BitCodeAbbrev>();
4243 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE));
4244 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // dbgloc
4245 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // var
4246 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // expr
4247 Abbv->Add(OpInfo: ValAbbrevOp); // val
4248 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4249 FUNCTION_DEBUG_RECORD_VALUE_ABBREV)
4250 llvm_unreachable("Unexpected abbrev ordering! 1");
4251 }
4252 {
4253 auto Abbv = std::make_shared<BitCodeAbbrev>();
4254 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_LOC));
4255 // NOTE: No IsDistinct field for FUNC_CODE_DEBUG_LOC.
4256 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4257 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4258 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4259 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4260 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
4261 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Atom group.
4262 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 3)); // Atom rank.
4263 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4264 FUNCTION_DEBUG_LOC_ABBREV)
4265 llvm_unreachable("Unexpected abbrev ordering!");
4266 }
4267 Stream.ExitBlock();
4268}
4269
4270/// Write the module path strings, currently only used when generating
4271/// a combined index file.
4272void IndexBitcodeWriter::writeModStrings() {
4273 Stream.EnterSubblock(BlockID: bitc::MODULE_STRTAB_BLOCK_ID, CodeLen: 3);
4274
4275 // TODO: See which abbrev sizes we actually need to emit
4276
4277 // 8-bit fixed-width MST_ENTRY strings.
4278 auto Abbv = std::make_shared<BitCodeAbbrev>();
4279 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4280 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4281 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4282 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
4283 unsigned Abbrev8Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4284
4285 // 7-bit fixed width MST_ENTRY strings.
4286 Abbv = std::make_shared<BitCodeAbbrev>();
4287 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4288 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4289 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4290 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
4291 unsigned Abbrev7Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4292
4293 // 6-bit char6 MST_ENTRY strings.
4294 Abbv = std::make_shared<BitCodeAbbrev>();
4295 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4296 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4297 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4298 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
4299 unsigned Abbrev6Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4300
4301 // Module Hash, 160 bits SHA1. Optionally, emitted after each MST_CODE_ENTRY.
4302 Abbv = std::make_shared<BitCodeAbbrev>();
4303 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_HASH));
4304 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4305 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4306 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4307 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4308 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4309 unsigned AbbrevHash = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4310
4311 SmallVector<unsigned, 64> Vals;
4312 forEachModule(Callback: [&](const StringMapEntry<ModuleHash> &MPSE) {
4313 StringRef Key = MPSE.getKey();
4314 const auto &Hash = MPSE.getValue();
4315 StringEncoding Bits = getStringEncoding(Str: Key);
4316 unsigned AbbrevToUse = Abbrev8Bit;
4317 if (Bits == SE_Char6)
4318 AbbrevToUse = Abbrev6Bit;
4319 else if (Bits == SE_Fixed7)
4320 AbbrevToUse = Abbrev7Bit;
4321
4322 auto ModuleId = ModuleIdMap.size();
4323 ModuleIdMap[Key] = ModuleId;
4324 Vals.push_back(Elt: ModuleId);
4325 // Use bytes_begin/end() for unsigned char iteration.
4326 Vals.append(in_start: Key.bytes_begin(), in_end: Key.bytes_end());
4327
4328 // Emit the finished record.
4329 Stream.EmitRecord(Code: bitc::MST_CODE_ENTRY, Vals, Abbrev: AbbrevToUse);
4330
4331 // Emit an optional hash for the module now
4332 if (llvm::any_of(Range: Hash, P: [](uint32_t H) { return H; })) {
4333 Vals.assign(in_start: Hash.begin(), in_end: Hash.end());
4334 // Emit the hash record.
4335 Stream.EmitRecord(Code: bitc::MST_CODE_HASH, Vals, Abbrev: AbbrevHash);
4336 }
4337
4338 Vals.clear();
4339 });
4340 Stream.ExitBlock();
4341}
4342
4343/// Write the function type metadata related records that need to appear before
4344/// a function summary entry (whether per-module or combined).
4345template <typename Fn>
4346static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream,
4347 FunctionSummary *FS,
4348 Fn GetValueID) {
4349 if (!FS->type_tests().empty())
4350 Stream.EmitRecord(Code: bitc::FS_TYPE_TESTS, Vals: FS->type_tests());
4351
4352 SmallVector<uint64_t, 64> Record;
4353
4354 auto WriteVFuncIdVec = [&](uint64_t Ty,
4355 ArrayRef<FunctionSummary::VFuncId> VFs) {
4356 if (VFs.empty())
4357 return;
4358 Record.clear();
4359 for (auto &VF : VFs) {
4360 Record.push_back(Elt: VF.GUID);
4361 Record.push_back(Elt: VF.Offset);
4362 }
4363 Stream.EmitRecord(Code: Ty, Vals: Record);
4364 };
4365
4366 WriteVFuncIdVec(bitc::FS_TYPE_TEST_ASSUME_VCALLS,
4367 FS->type_test_assume_vcalls());
4368 WriteVFuncIdVec(bitc::FS_TYPE_CHECKED_LOAD_VCALLS,
4369 FS->type_checked_load_vcalls());
4370
4371 auto WriteConstVCallVec = [&](uint64_t Ty,
4372 ArrayRef<FunctionSummary::ConstVCall> VCs) {
4373 for (auto &VC : VCs) {
4374 Record.clear();
4375 Record.push_back(Elt: VC.VFunc.GUID);
4376 Record.push_back(Elt: VC.VFunc.Offset);
4377 llvm::append_range(C&: Record, R: VC.Args);
4378 Stream.EmitRecord(Code: Ty, Vals: Record);
4379 }
4380 };
4381
4382 WriteConstVCallVec(bitc::FS_TYPE_TEST_ASSUME_CONST_VCALL,
4383 FS->type_test_assume_const_vcalls());
4384 WriteConstVCallVec(bitc::FS_TYPE_CHECKED_LOAD_CONST_VCALL,
4385 FS->type_checked_load_const_vcalls());
4386
4387 auto WriteRange = [&](ConstantRange Range) {
4388 Range = Range.sextOrTrunc(BitWidth: FunctionSummary::ParamAccess::RangeWidth);
4389 assert(Range.getLower().getNumWords() == 1);
4390 assert(Range.getUpper().getNumWords() == 1);
4391 emitSignedInt64(Vals&: Record, V: *Range.getLower().getRawData());
4392 emitSignedInt64(Vals&: Record, V: *Range.getUpper().getRawData());
4393 };
4394
4395 if (!FS->paramAccesses().empty()) {
4396 Record.clear();
4397 for (auto &Arg : FS->paramAccesses()) {
4398 size_t UndoSize = Record.size();
4399 Record.push_back(Elt: Arg.ParamNo);
4400 WriteRange(Arg.Use);
4401 Record.push_back(Elt: Arg.Calls.size());
4402 for (auto &Call : Arg.Calls) {
4403 Record.push_back(Elt: Call.ParamNo);
4404 std::optional<unsigned> ValueID = GetValueID(Call.Callee);
4405 if (!ValueID) {
4406 // If ValueID is unknown we can't drop just this call, we must drop
4407 // entire parameter.
4408 Record.resize(N: UndoSize);
4409 break;
4410 }
4411 Record.push_back(Elt: *ValueID);
4412 WriteRange(Call.Offsets);
4413 }
4414 }
4415 if (!Record.empty())
4416 Stream.EmitRecord(Code: bitc::FS_PARAM_ACCESS, Vals: Record);
4417 }
4418}
4419
4420/// Collect type IDs from type tests used by function.
4421static void
4422getReferencedTypeIds(FunctionSummary *FS,
4423 std::set<GlobalValue::GUID> &ReferencedTypeIds) {
4424 if (!FS->type_tests().empty())
4425 for (auto &TT : FS->type_tests())
4426 ReferencedTypeIds.insert(x: TT);
4427
4428 auto GetReferencedTypesFromVFuncIdVec =
4429 [&](ArrayRef<FunctionSummary::VFuncId> VFs) {
4430 for (auto &VF : VFs)
4431 ReferencedTypeIds.insert(x: VF.GUID);
4432 };
4433
4434 GetReferencedTypesFromVFuncIdVec(FS->type_test_assume_vcalls());
4435 GetReferencedTypesFromVFuncIdVec(FS->type_checked_load_vcalls());
4436
4437 auto GetReferencedTypesFromConstVCallVec =
4438 [&](ArrayRef<FunctionSummary::ConstVCall> VCs) {
4439 for (auto &VC : VCs)
4440 ReferencedTypeIds.insert(x: VC.VFunc.GUID);
4441 };
4442
4443 GetReferencedTypesFromConstVCallVec(FS->type_test_assume_const_vcalls());
4444 GetReferencedTypesFromConstVCallVec(FS->type_checked_load_const_vcalls());
4445}
4446
4447static void writeWholeProgramDevirtResolutionByArg(
4448 SmallVector<uint64_t, 64> &NameVals, const std::vector<uint64_t> &args,
4449 const WholeProgramDevirtResolution::ByArg &ByArg) {
4450 NameVals.push_back(Elt: args.size());
4451 llvm::append_range(C&: NameVals, R: args);
4452
4453 NameVals.push_back(Elt: ByArg.TheKind);
4454 NameVals.push_back(Elt: ByArg.Info);
4455 NameVals.push_back(Elt: ByArg.Byte);
4456 NameVals.push_back(Elt: ByArg.Bit);
4457}
4458
4459static void writeWholeProgramDevirtResolution(
4460 SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
4461 uint64_t Id, const WholeProgramDevirtResolution &Wpd) {
4462 NameVals.push_back(Elt: Id);
4463
4464 NameVals.push_back(Elt: Wpd.TheKind);
4465 NameVals.push_back(Elt: StrtabBuilder.add(S: Wpd.SingleImplName));
4466 NameVals.push_back(Elt: Wpd.SingleImplName.size());
4467
4468 NameVals.push_back(Elt: Wpd.ResByArg.size());
4469 for (auto &A : Wpd.ResByArg)
4470 writeWholeProgramDevirtResolutionByArg(NameVals, args: A.first, ByArg: A.second);
4471}
4472
4473static void writeTypeIdSummaryRecord(SmallVector<uint64_t, 64> &NameVals,
4474 StringTableBuilder &StrtabBuilder,
4475 StringRef Id,
4476 const TypeIdSummary &Summary) {
4477 NameVals.push_back(Elt: StrtabBuilder.add(S: Id));
4478 NameVals.push_back(Elt: Id.size());
4479
4480 NameVals.push_back(Elt: Summary.TTRes.TheKind);
4481 NameVals.push_back(Elt: Summary.TTRes.SizeM1BitWidth);
4482 NameVals.push_back(Elt: Summary.TTRes.AlignLog2);
4483 NameVals.push_back(Elt: Summary.TTRes.SizeM1);
4484 NameVals.push_back(Elt: Summary.TTRes.BitMask);
4485 NameVals.push_back(Elt: Summary.TTRes.InlineBits);
4486
4487 for (auto &W : Summary.WPDRes)
4488 writeWholeProgramDevirtResolution(NameVals, StrtabBuilder, Id: W.first,
4489 Wpd: W.second);
4490}
4491
4492static void writeTypeIdCompatibleVtableSummaryRecord(
4493 SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
4494 StringRef Id, const TypeIdCompatibleVtableInfo &Summary,
4495 ValueEnumerator &VE) {
4496 NameVals.push_back(Elt: StrtabBuilder.add(S: Id));
4497 NameVals.push_back(Elt: Id.size());
4498
4499 for (auto &P : Summary) {
4500 NameVals.push_back(Elt: P.AddressPointOffset);
4501 NameVals.push_back(Elt: VE.getValueID(V: P.VTableVI.getValue()));
4502 }
4503}
4504
4505// Adds the allocation contexts to the CallStacks map. We simply use the
4506// size at the time the context was added as the CallStackId. This works because
4507// when we look up the call stacks later on we process the function summaries
4508// and their allocation records in the same exact order.
4509static void collectMemProfCallStacks(
4510 FunctionSummary *FS, std::function<LinearFrameId(unsigned)> GetStackIndex,
4511 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> &CallStacks) {
4512 // The interfaces in ProfileData/MemProf.h use a type alias for a stack frame
4513 // id offset into the index of the full stack frames. The ModuleSummaryIndex
4514 // currently uses unsigned. Make sure these stay in sync.
4515 static_assert(std::is_same_v<LinearFrameId, unsigned>);
4516 for (auto &AI : FS->allocs()) {
4517 for (auto &MIB : AI.MIBs) {
4518 SmallVector<unsigned> StackIdIndices;
4519 StackIdIndices.reserve(N: MIB.StackIdIndices.size());
4520 for (auto Id : MIB.StackIdIndices)
4521 StackIdIndices.push_back(Elt: GetStackIndex(Id));
4522 // The CallStackId is the size at the time this context was inserted.
4523 CallStacks.insert(KV: {CallStacks.size(), StackIdIndices});
4524 }
4525 }
4526}
4527
4528// Build the radix tree from the accumulated CallStacks, write out the resulting
4529// linearized radix tree array, and return the map of call stack positions into
4530// this array for use when writing the allocation records. The returned map is
4531// indexed by a CallStackId which in this case is implicitly determined by the
4532// order of function summaries and their allocation infos being written.
4533static DenseMap<CallStackId, LinearCallStackId> writeMemoryProfileRadixTree(
4534 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> &&CallStacks,
4535 BitstreamWriter &Stream, unsigned RadixAbbrev) {
4536 assert(!CallStacks.empty());
4537 DenseMap<unsigned, FrameStat> FrameHistogram =
4538 computeFrameHistogram<LinearFrameId>(MemProfCallStackData&: CallStacks);
4539 CallStackRadixTreeBuilder<LinearFrameId> Builder;
4540 // We don't need a MemProfFrameIndexes map as we have already converted the
4541 // full stack id hash to a linear offset into the StackIds array.
4542 Builder.build(MemProfCallStackData: std::move(CallStacks), /*MemProfFrameIndexes=*/nullptr,
4543 FrameHistogram);
4544 Stream.EmitRecord(Code: bitc::FS_CONTEXT_RADIX_TREE_ARRAY, Vals: Builder.getRadixArray(),
4545 Abbrev: RadixAbbrev);
4546 return Builder.takeCallStackPos();
4547}
4548
4549static void writeFunctionHeapProfileRecords(
4550 BitstreamWriter &Stream, FunctionSummary *FS, unsigned CallsiteAbbrev,
4551 unsigned AllocAbbrev, unsigned ContextIdAbbvId, bool PerModule,
4552 std::function<unsigned(const ValueInfo &VI)> GetValueID,
4553 std::function<unsigned(unsigned)> GetStackIndex,
4554 bool WriteContextSizeInfoIndex,
4555 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
4556 CallStackId &CallStackCount) {
4557 SmallVector<uint64_t> Record;
4558
4559 for (auto &CI : FS->callsites()) {
4560 Record.clear();
4561 // Per module callsite clones should always have a single entry of
4562 // value 0.
4563 assert(!PerModule || (CI.Clones.size() == 1 && CI.Clones[0] == 0));
4564 Record.push_back(Elt: GetValueID(CI.Callee));
4565 if (!PerModule) {
4566 Record.push_back(Elt: CI.StackIdIndices.size());
4567 Record.push_back(Elt: CI.Clones.size());
4568 }
4569 for (auto Id : CI.StackIdIndices)
4570 Record.push_back(Elt: GetStackIndex(Id));
4571 if (!PerModule)
4572 llvm::append_range(C&: Record, R: CI.Clones);
4573 Stream.EmitRecord(Code: PerModule ? bitc::FS_PERMODULE_CALLSITE_INFO
4574 : bitc::FS_COMBINED_CALLSITE_INFO,
4575 Vals: Record, Abbrev: CallsiteAbbrev);
4576 }
4577
4578 for (auto &AI : FS->allocs()) {
4579 Record.clear();
4580 // Per module alloc versions should always have a single entry of
4581 // value 0.
4582 assert(!PerModule || (AI.Versions.size() == 1 && AI.Versions[0] == 0));
4583 Record.push_back(Elt: AI.MIBs.size());
4584 if (!PerModule)
4585 Record.push_back(Elt: AI.Versions.size());
4586 for (auto &MIB : AI.MIBs) {
4587 Record.push_back(Elt: (uint8_t)MIB.AllocType);
4588 // The per-module summary always needs to include the alloc context, as we
4589 // use it during the thin link. For the combined index it is optional (see
4590 // comments where CombinedIndexMemProfContext is defined).
4591 if (PerModule || CombinedIndexMemProfContext) {
4592 // Record the index into the radix tree array for this context.
4593 assert(CallStackCount <= CallStackPos.size());
4594 Record.push_back(Elt: CallStackPos[CallStackCount++]);
4595 }
4596 }
4597 if (!PerModule)
4598 llvm::append_range(C&: Record, R: AI.Versions);
4599 assert(AI.ContextSizeInfos.empty() ||
4600 AI.ContextSizeInfos.size() == AI.MIBs.size());
4601 // Optionally emit the context size information if it exists.
4602 if (WriteContextSizeInfoIndex && !AI.ContextSizeInfos.empty()) {
4603 // The abbreviation id for the context ids record should have been created
4604 // if we are emitting the per-module index, which is where we write this
4605 // info.
4606 assert(ContextIdAbbvId);
4607 SmallVector<uint32_t> ContextIds;
4608 // At least one context id per ContextSizeInfos entry (MIB), broken into 2
4609 // halves.
4610 ContextIds.reserve(N: AI.ContextSizeInfos.size() * 2);
4611 for (auto &Infos : AI.ContextSizeInfos) {
4612 Record.push_back(Elt: Infos.size());
4613 for (auto [FullStackId, TotalSize] : Infos) {
4614 // The context ids are emitted separately as a fixed width array,
4615 // which is more efficient than a VBR given that these hashes are
4616 // typically close to 64-bits. The max fixed width entry is 32 bits so
4617 // it is split into 2.
4618 ContextIds.push_back(Elt: static_cast<uint32_t>(FullStackId >> 32));
4619 ContextIds.push_back(Elt: static_cast<uint32_t>(FullStackId));
4620 Record.push_back(Elt: TotalSize);
4621 }
4622 }
4623 // The context ids are expected by the reader to immediately precede the
4624 // associated alloc info record.
4625 Stream.EmitRecord(Code: bitc::FS_ALLOC_CONTEXT_IDS, Vals: ContextIds,
4626 Abbrev: ContextIdAbbvId);
4627 }
4628 Stream.EmitRecord(Code: PerModule
4629 ? bitc::FS_PERMODULE_ALLOC_INFO
4630 : (CombinedIndexMemProfContext
4631 ? bitc::FS_COMBINED_ALLOC_INFO
4632 : bitc::FS_COMBINED_ALLOC_INFO_NO_CONTEXT),
4633 Vals: Record, Abbrev: AllocAbbrev);
4634 }
4635}
4636
4637// Helper to emit a single function summary record.
4638void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
4639 SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
4640 unsigned ValueID, unsigned FSCallsProfileAbbrev, unsigned CallsiteAbbrev,
4641 unsigned AllocAbbrev, unsigned ContextIdAbbvId, const Function &F,
4642 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
4643 CallStackId &CallStackCount) {
4644 NameVals.push_back(Elt: ValueID);
4645
4646 FunctionSummary *FS = cast<FunctionSummary>(Val: Summary);
4647
4648 writeFunctionTypeMetadataRecords(
4649 Stream, FS, GetValueID: [&](const ValueInfo &VI) -> std::optional<unsigned> {
4650 return {VE.getValueID(V: VI.getValue())};
4651 });
4652
4653 auto SpecialRefCnts = FS->specialRefCounts();
4654 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: FS->flags()));
4655 NameVals.push_back(Elt: FS->instCount());
4656 NameVals.push_back(Elt: getEncodedFFlags(Flags: FS->fflags()));
4657 NameVals.push_back(Elt: FS->refs().size());
4658 NameVals.push_back(Elt: SpecialRefCnts.first); // rorefcnt
4659 NameVals.push_back(Elt: SpecialRefCnts.second); // worefcnt
4660
4661 for (auto &RI : FS->refs())
4662 NameVals.push_back(Elt: getValueId(VI: RI));
4663
4664 for (auto &ECI : FS->calls()) {
4665 NameVals.push_back(Elt: getValueId(VI: ECI.first));
4666 NameVals.push_back(Elt: getEncodedHotnessCallEdgeInfo(CI: ECI.second));
4667 }
4668
4669 // Emit the finished record.
4670 Stream.EmitRecord(Code: bitc::FS_PERMODULE_PROFILE, Vals: NameVals, Abbrev: FSCallsProfileAbbrev);
4671 NameVals.clear();
4672
4673 writeFunctionHeapProfileRecords(
4674 Stream, FS, CallsiteAbbrev, AllocAbbrev, ContextIdAbbvId,
4675 /*PerModule*/ true,
4676 /*GetValueId*/ GetValueID: [&](const ValueInfo &VI) { return getValueId(VI); },
4677 /*GetStackIndex*/ [&](unsigned I) { return I; },
4678 /*WriteContextSizeInfoIndex*/ true, CallStackPos, CallStackCount);
4679}
4680
4681// Collect the global value references in the given variable's initializer,
4682// and emit them in a summary record.
4683void ModuleBitcodeWriterBase::writeModuleLevelReferences(
4684 const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals,
4685 unsigned FSModRefsAbbrev, unsigned FSModVTableRefsAbbrev) {
4686 auto VI = Index->getValueInfo(GUID: V.getGUID());
4687 if (!VI || VI.getSummaryList().empty()) {
4688 // Only declarations should not have a summary (a declaration might however
4689 // have a summary if the def was in module level asm).
4690 assert(V.isDeclaration());
4691 return;
4692 }
4693 auto *Summary = VI.getSummaryList()[0].get();
4694 NameVals.push_back(Elt: VE.getValueID(V: &V));
4695 GlobalVarSummary *VS = cast<GlobalVarSummary>(Val: Summary);
4696 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: VS->flags()));
4697 NameVals.push_back(Elt: getEncodedGVarFlags(Flags: VS->varflags()));
4698
4699 auto VTableFuncs = VS->vTableFuncs();
4700 if (!VTableFuncs.empty())
4701 NameVals.push_back(Elt: VS->refs().size());
4702
4703 unsigned SizeBeforeRefs = NameVals.size();
4704 for (auto &RI : VS->refs())
4705 NameVals.push_back(Elt: VE.getValueID(V: RI.getValue()));
4706 // Sort the refs for determinism output, the vector returned by FS->refs() has
4707 // been initialized from a DenseSet.
4708 llvm::sort(C: drop_begin(RangeOrContainer&: NameVals, N: SizeBeforeRefs));
4709
4710 if (VTableFuncs.empty())
4711 Stream.EmitRecord(Code: bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, Vals: NameVals,
4712 Abbrev: FSModRefsAbbrev);
4713 else {
4714 // VTableFuncs pairs should already be sorted by offset.
4715 for (auto &P : VTableFuncs) {
4716 NameVals.push_back(Elt: VE.getValueID(V: P.FuncVI.getValue()));
4717 NameVals.push_back(Elt: P.VTableOffset);
4718 }
4719
4720 Stream.EmitRecord(Code: bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS, Vals: NameVals,
4721 Abbrev: FSModVTableRefsAbbrev);
4722 }
4723 NameVals.clear();
4724}
4725
4726/// Emit the per-module summary section alongside the rest of
4727/// the module's bitcode.
4728void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
4729 // By default we compile with ThinLTO if the module has a summary, but the
4730 // client can request full LTO with a module flag.
4731 bool IsThinLTO = true;
4732 if (auto *MD =
4733 mdconst::extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: "ThinLTO")))
4734 IsThinLTO = MD->getZExtValue();
4735 Stream.EnterSubblock(BlockID: IsThinLTO ? bitc::GLOBALVAL_SUMMARY_BLOCK_ID
4736 : bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID,
4737 CodeLen: 4);
4738
4739 Stream.EmitRecord(
4740 Code: bitc::FS_VERSION,
4741 Vals: ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
4742
4743 // Write the index flags.
4744 uint64_t Flags = 0;
4745 // Bits 1-3 are set only in the combined index, skip them.
4746 if (Index->enableSplitLTOUnit())
4747 Flags |= 0x8;
4748 if (Index->hasUnifiedLTO())
4749 Flags |= 0x200;
4750
4751 Stream.EmitRecord(Code: bitc::FS_FLAGS, Vals: ArrayRef<uint64_t>{Flags});
4752
4753 if (Index->begin() == Index->end()) {
4754 Stream.ExitBlock();
4755 return;
4756 }
4757
4758 auto Abbv = std::make_shared<BitCodeAbbrev>();
4759 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_VALUE_GUID));
4760 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4761 // GUIDS often use up most of 64-bits, so encode as two Fixed 32.
4762 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4763 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4764 unsigned ValueGuidAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4765
4766 for (const auto &GVI : valueIds()) {
4767 Stream.EmitRecord(Code: bitc::FS_VALUE_GUID,
4768 Vals: ArrayRef<uint32_t>{GVI.second,
4769 static_cast<uint32_t>(GVI.first >> 32),
4770 static_cast<uint32_t>(GVI.first)},
4771 Abbrev: ValueGuidAbbrev);
4772 }
4773
4774 if (!Index->stackIds().empty()) {
4775 auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4776 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_STACK_IDS));
4777 // numids x stackid
4778 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4779 // The stack ids are hashes that are close to 64 bits in size, so emitting
4780 // as a pair of 32-bit fixed-width values is more efficient than a VBR.
4781 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4782 unsigned StackIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(StackIdAbbv));
4783 SmallVector<uint32_t> Vals;
4784 Vals.reserve(N: Index->stackIds().size() * 2);
4785 for (auto Id : Index->stackIds()) {
4786 Vals.push_back(Elt: static_cast<uint32_t>(Id >> 32));
4787 Vals.push_back(Elt: static_cast<uint32_t>(Id));
4788 }
4789 Stream.EmitRecord(Code: bitc::FS_STACK_IDS, Vals, Abbrev: StackIdAbbvId);
4790 }
4791
4792 unsigned ContextIdAbbvId = 0;
4793 if (metadataMayIncludeContextSizeInfo()) {
4794 // n x context id
4795 auto ContextIdAbbv = std::make_shared<BitCodeAbbrev>();
4796 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_ALLOC_CONTEXT_IDS));
4797 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4798 // The context ids are hashes that are close to 64 bits in size, so emitting
4799 // as a pair of 32-bit fixed-width values is more efficient than a VBR if we
4800 // are emitting them for all MIBs. Otherwise we use VBR to better compress 0
4801 // values that are expected to more frequently occur in an alloc's memprof
4802 // summary.
4803 if (metadataIncludesAllContextSizeInfo())
4804 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4805 else
4806 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4807 ContextIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(ContextIdAbbv));
4808 }
4809
4810 // Abbrev for FS_PERMODULE_PROFILE.
4811 Abbv = std::make_shared<BitCodeAbbrev>();
4812 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE));
4813 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4814 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // flags
4815 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
4816 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
4817 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4818 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
4819 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
4820 // numrefs x valueid, n x (valueid, hotness+tailcall flags)
4821 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4822 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4823 unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4824
4825 // Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS.
4826 Abbv = std::make_shared<BitCodeAbbrev>();
4827 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS));
4828 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4829 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4830 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
4831 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4832 unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4833
4834 // Abbrev for FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS.
4835 Abbv = std::make_shared<BitCodeAbbrev>();
4836 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS));
4837 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4838 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4839 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4840 // numrefs x valueid, n x (valueid , offset)
4841 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4842 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4843 unsigned FSModVTableRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4844
4845 // Abbrev for FS_ALIAS.
4846 Abbv = std::make_shared<BitCodeAbbrev>();
4847 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_ALIAS));
4848 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4849 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4850 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4851 unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4852
4853 // Abbrev for FS_TYPE_ID_METADATA
4854 Abbv = std::make_shared<BitCodeAbbrev>();
4855 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_TYPE_ID_METADATA));
4856 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid strtab index
4857 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid length
4858 // n x (valueid , offset)
4859 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4860 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4861 unsigned TypeIdCompatibleVtableAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4862
4863 Abbv = std::make_shared<BitCodeAbbrev>();
4864 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_CALLSITE_INFO));
4865 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4866 // n x stackidindex
4867 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4868 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4869 unsigned CallsiteAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4870
4871 Abbv = std::make_shared<BitCodeAbbrev>();
4872 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_ALLOC_INFO));
4873 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib
4874 // n x (alloc type, context radix tree index)
4875 // optional: nummib x (numcontext x total size)
4876 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4877 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4878 unsigned AllocAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4879
4880 Abbv = std::make_shared<BitCodeAbbrev>();
4881 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_CONTEXT_RADIX_TREE_ARRAY));
4882 // n x entry
4883 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4884 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4885 unsigned RadixAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4886
4887 // First walk through all the functions and collect the allocation contexts in
4888 // their associated summaries, for use in constructing a radix tree of
4889 // contexts. Note that we need to do this in the same order as the functions
4890 // are processed further below since the call stack positions in the resulting
4891 // radix tree array are identified based on this order.
4892 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> CallStacks;
4893 for (const Function &F : M) {
4894 // Summary emission does not support anonymous functions, they have to be
4895 // renamed using the anonymous function renaming pass.
4896 if (!F.hasName())
4897 report_fatal_error(reason: "Unexpected anonymous function when writing summary");
4898
4899 ValueInfo VI = Index->getValueInfo(GUID: F.getGUID());
4900 if (!VI || VI.getSummaryList().empty()) {
4901 // Only declarations should not have a summary (a declaration might
4902 // however have a summary if the def was in module level asm).
4903 assert(F.isDeclaration());
4904 continue;
4905 }
4906 auto *Summary = VI.getSummaryList()[0].get();
4907 FunctionSummary *FS = cast<FunctionSummary>(Val: Summary);
4908 collectMemProfCallStacks(
4909 FS, /*GetStackIndex*/ [](unsigned I) { return I; }, CallStacks);
4910 }
4911 // Finalize the radix tree, write it out, and get the map of positions in the
4912 // linearized tree array.
4913 DenseMap<CallStackId, LinearCallStackId> CallStackPos;
4914 if (!CallStacks.empty()) {
4915 CallStackPos =
4916 writeMemoryProfileRadixTree(CallStacks: std::move(CallStacks), Stream, RadixAbbrev);
4917 }
4918
4919 // Keep track of the current index into the CallStackPos map.
4920 CallStackId CallStackCount = 0;
4921
4922 SmallVector<uint64_t, 64> NameVals;
4923 // Iterate over the list of functions instead of the Index to
4924 // ensure the ordering is stable.
4925 for (const Function &F : M) {
4926 // Summary emission does not support anonymous functions, they have to
4927 // renamed using the anonymous function renaming pass.
4928 if (!F.hasName())
4929 report_fatal_error(reason: "Unexpected anonymous function when writing summary");
4930
4931 ValueInfo VI = Index->getValueInfo(GUID: F.getGUID());
4932 if (!VI || VI.getSummaryList().empty()) {
4933 // Only declarations should not have a summary (a declaration might
4934 // however have a summary if the def was in module level asm).
4935 assert(F.isDeclaration());
4936 continue;
4937 }
4938 auto *Summary = VI.getSummaryList()[0].get();
4939 writePerModuleFunctionSummaryRecord(NameVals, Summary, ValueID: VE.getValueID(V: &F),
4940 FSCallsProfileAbbrev, CallsiteAbbrev,
4941 AllocAbbrev, ContextIdAbbvId, F,
4942 CallStackPos, CallStackCount);
4943 }
4944
4945 // Capture references from GlobalVariable initializers, which are outside
4946 // of a function scope.
4947 for (const GlobalVariable &G : M.globals())
4948 writeModuleLevelReferences(V: G, NameVals, FSModRefsAbbrev,
4949 FSModVTableRefsAbbrev);
4950
4951 for (const GlobalAlias &A : M.aliases()) {
4952 auto *Aliasee = A.getAliaseeObject();
4953 // Skip ifunc and nameless functions which don't have an entry in the
4954 // summary.
4955 if (!Aliasee->hasName() || isa<GlobalIFunc>(Val: Aliasee))
4956 continue;
4957 auto AliasId = VE.getValueID(V: &A);
4958 auto AliaseeId = VE.getValueID(V: Aliasee);
4959 NameVals.push_back(Elt: AliasId);
4960 auto *Summary = Index->getGlobalValueSummary(GV: A);
4961 AliasSummary *AS = cast<AliasSummary>(Val: Summary);
4962 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: AS->flags()));
4963 NameVals.push_back(Elt: AliaseeId);
4964 Stream.EmitRecord(Code: bitc::FS_ALIAS, Vals: NameVals, Abbrev: FSAliasAbbrev);
4965 NameVals.clear();
4966 }
4967
4968 for (auto &S : Index->typeIdCompatibleVtableMap()) {
4969 writeTypeIdCompatibleVtableSummaryRecord(NameVals, StrtabBuilder, Id: S.first,
4970 Summary: S.second, VE);
4971 Stream.EmitRecord(Code: bitc::FS_TYPE_ID_METADATA, Vals: NameVals,
4972 Abbrev: TypeIdCompatibleVtableAbbrev);
4973 NameVals.clear();
4974 }
4975
4976 if (Index->getBlockCount())
4977 Stream.EmitRecord(Code: bitc::FS_BLOCK_COUNT,
4978 Vals: ArrayRef<uint64_t>{Index->getBlockCount()});
4979
4980 Stream.ExitBlock();
4981}
4982
4983/// Emit the combined summary section into the combined index file.
4984void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
4985 Stream.EnterSubblock(BlockID: bitc::GLOBALVAL_SUMMARY_BLOCK_ID, CodeLen: 4);
4986 Stream.EmitRecord(
4987 Code: bitc::FS_VERSION,
4988 Vals: ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
4989
4990 // Write the index flags.
4991 Stream.EmitRecord(Code: bitc::FS_FLAGS, Vals: ArrayRef<uint64_t>{Index.getFlags()});
4992
4993 auto Abbv = std::make_shared<BitCodeAbbrev>();
4994 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_VALUE_GUID));
4995 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4996 // GUIDS often use up most of 64-bits, so encode as two Fixed 32.
4997 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4998 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4999 unsigned ValueGuidAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5000
5001 for (const auto &GVI : valueIds()) {
5002 Stream.EmitRecord(Code: bitc::FS_VALUE_GUID,
5003 Vals: ArrayRef<uint32_t>{GVI.second,
5004 static_cast<uint32_t>(GVI.first >> 32),
5005 static_cast<uint32_t>(GVI.first)},
5006 Abbrev: ValueGuidAbbrev);
5007 }
5008
5009 // Write the stack ids used by this index, which will be a subset of those in
5010 // the full index in the case of distributed indexes.
5011 if (!StackIds.empty()) {
5012 auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
5013 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_STACK_IDS));
5014 // numids x stackid
5015 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5016 // The stack ids are hashes that are close to 64 bits in size, so emitting
5017 // as a pair of 32-bit fixed-width values is more efficient than a VBR.
5018 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
5019 unsigned StackIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(StackIdAbbv));
5020 SmallVector<uint32_t> Vals;
5021 Vals.reserve(N: StackIds.size() * 2);
5022 for (auto Id : StackIds) {
5023 Vals.push_back(Elt: static_cast<uint32_t>(Id >> 32));
5024 Vals.push_back(Elt: static_cast<uint32_t>(Id));
5025 }
5026 Stream.EmitRecord(Code: bitc::FS_STACK_IDS, Vals, Abbrev: StackIdAbbvId);
5027 }
5028
5029 // Abbrev for FS_COMBINED_PROFILE.
5030 Abbv = std::make_shared<BitCodeAbbrev>();
5031 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_PROFILE));
5032 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5033 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
5034 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
5035 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
5036 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
5037 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // entrycount
5038 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
5039 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
5040 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
5041 // numrefs x valueid, n x (valueid, hotness+tailcall flags)
5042 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5043 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5044 unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5045
5046 // Abbrev for FS_COMBINED_GLOBALVAR_INIT_REFS.
5047 Abbv = std::make_shared<BitCodeAbbrev>();
5048 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS));
5049 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5050 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
5051 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
5052 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
5053 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5054 unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5055
5056 // Abbrev for FS_COMBINED_ALIAS.
5057 Abbv = std::make_shared<BitCodeAbbrev>();
5058 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_ALIAS));
5059 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5060 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
5061 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
5062 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5063 unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5064
5065 Abbv = std::make_shared<BitCodeAbbrev>();
5066 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_CALLSITE_INFO));
5067 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5068 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numstackindices
5069 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
5070 // numstackindices x stackidindex, numver x version
5071 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5072 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5073 unsigned CallsiteAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5074
5075 Abbv = std::make_shared<BitCodeAbbrev>();
5076 Abbv->Add(OpInfo: BitCodeAbbrevOp(CombinedIndexMemProfContext
5077 ? bitc::FS_COMBINED_ALLOC_INFO
5078 : bitc::FS_COMBINED_ALLOC_INFO_NO_CONTEXT));
5079 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib
5080 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
5081 // nummib x (alloc type, context radix tree index),
5082 // numver x version
5083 // optional: nummib x total size
5084 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5085 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5086 unsigned AllocAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5087
5088 auto shouldImportValueAsDecl = [&](GlobalValueSummary *GVS) -> bool {
5089 if (DecSummaries == nullptr)
5090 return false;
5091 return DecSummaries->count(Ptr: GVS);
5092 };
5093
5094 // The aliases are emitted as a post-pass, and will point to the value
5095 // id of the aliasee. Save them in a vector for post-processing.
5096 SmallVector<AliasSummary *, 64> Aliases;
5097
5098 // Save the value id for each summary for alias emission.
5099 DenseMap<const GlobalValueSummary *, unsigned> SummaryToValueIdMap;
5100
5101 SmallVector<uint64_t, 64> NameVals;
5102
5103 // Set that will be populated during call to writeFunctionTypeMetadataRecords
5104 // with the type ids referenced by this index file.
5105 std::set<GlobalValue::GUID> ReferencedTypeIds;
5106
5107 // For local linkage, we also emit the original name separately
5108 // immediately after the record.
5109 auto MaybeEmitOriginalName = [&](GlobalValueSummary &S) {
5110 // We don't need to emit the original name if we are writing the index for
5111 // distributed backends (in which case ModuleToSummariesForIndex is
5112 // non-null). The original name is only needed during the thin link, since
5113 // for SamplePGO the indirect call targets for local functions have
5114 // have the original name annotated in profile.
5115 // Continue to emit it when writing out the entire combined index, which is
5116 // used in testing the thin link via llvm-lto.
5117 if (ModuleToSummariesForIndex || !GlobalValue::isLocalLinkage(Linkage: S.linkage()))
5118 return;
5119 NameVals.push_back(Elt: S.getOriginalName());
5120 Stream.EmitRecord(Code: bitc::FS_COMBINED_ORIGINAL_NAME, Vals: NameVals);
5121 NameVals.clear();
5122 };
5123
5124 DenseMap<CallStackId, LinearCallStackId> CallStackPos;
5125 if (CombinedIndexMemProfContext) {
5126 Abbv = std::make_shared<BitCodeAbbrev>();
5127 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_CONTEXT_RADIX_TREE_ARRAY));
5128 // n x entry
5129 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5130 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5131 unsigned RadixAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5132
5133 // First walk through all the functions and collect the allocation contexts
5134 // in their associated summaries, for use in constructing a radix tree of
5135 // contexts. Note that we need to do this in the same order as the functions
5136 // are processed further below since the call stack positions in the
5137 // resulting radix tree array are identified based on this order.
5138 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> CallStacks;
5139 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
5140 // Don't collect this when invoked for an aliasee, as it is not needed for
5141 // the alias summary. If the aliasee is to be imported, we will invoke
5142 // this separately with IsAliasee=false.
5143 if (IsAliasee)
5144 return;
5145 GlobalValueSummary *S = I.second;
5146 assert(S);
5147 auto *FS = dyn_cast<FunctionSummary>(Val: S);
5148 if (!FS)
5149 return;
5150 collectMemProfCallStacks(
5151 FS,
5152 /*GetStackIndex*/
5153 [&](unsigned I) {
5154 // Get the corresponding index into the list of StackIds actually
5155 // being written for this combined index (which may be a subset in
5156 // the case of distributed indexes).
5157 assert(StackIdIndicesToIndex.contains(I));
5158 return StackIdIndicesToIndex[I];
5159 },
5160 CallStacks);
5161 });
5162 // Finalize the radix tree, write it out, and get the map of positions in
5163 // the linearized tree array.
5164 if (!CallStacks.empty()) {
5165 CallStackPos = writeMemoryProfileRadixTree(CallStacks: std::move(CallStacks), Stream,
5166 RadixAbbrev);
5167 }
5168 }
5169
5170 // Keep track of the current index into the CallStackPos map. Not used if
5171 // CombinedIndexMemProfContext is false.
5172 CallStackId CallStackCount = 0;
5173
5174 DenseSet<GlobalValue::GUID> DefOrUseGUIDs;
5175 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
5176 GlobalValueSummary *S = I.second;
5177 assert(S);
5178 DefOrUseGUIDs.insert(V: I.first);
5179 for (const ValueInfo &VI : S->refs())
5180 DefOrUseGUIDs.insert(V: VI.getGUID());
5181
5182 auto ValueId = getValueId(ValGUID: I.first);
5183 assert(ValueId);
5184 SummaryToValueIdMap[S] = *ValueId;
5185
5186 // If this is invoked for an aliasee, we want to record the above
5187 // mapping, but then not emit a summary entry (if the aliasee is
5188 // to be imported, we will invoke this separately with IsAliasee=false).
5189 if (IsAliasee)
5190 return;
5191
5192 if (auto *AS = dyn_cast<AliasSummary>(Val: S)) {
5193 // Will process aliases as a post-pass because the reader wants all
5194 // global to be loaded first.
5195 Aliases.push_back(Elt: AS);
5196 return;
5197 }
5198
5199 if (auto *VS = dyn_cast<GlobalVarSummary>(Val: S)) {
5200 NameVals.push_back(Elt: *ValueId);
5201 assert(ModuleIdMap.count(VS->modulePath()));
5202 NameVals.push_back(Elt: ModuleIdMap[VS->modulePath()]);
5203 NameVals.push_back(
5204 Elt: getEncodedGVSummaryFlags(Flags: VS->flags(), ImportAsDecl: shouldImportValueAsDecl(VS)));
5205 NameVals.push_back(Elt: getEncodedGVarFlags(Flags: VS->varflags()));
5206 for (auto &RI : VS->refs()) {
5207 auto RefValueId = getValueId(ValGUID: RI.getGUID());
5208 if (!RefValueId)
5209 continue;
5210 NameVals.push_back(Elt: *RefValueId);
5211 }
5212
5213 // Emit the finished record.
5214 Stream.EmitRecord(Code: bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, Vals: NameVals,
5215 Abbrev: FSModRefsAbbrev);
5216 NameVals.clear();
5217 MaybeEmitOriginalName(*S);
5218 return;
5219 }
5220
5221 auto GetValueId = [&](const ValueInfo &VI) -> std::optional<unsigned> {
5222 if (!VI)
5223 return std::nullopt;
5224 return getValueId(ValGUID: VI.getGUID());
5225 };
5226
5227 auto *FS = cast<FunctionSummary>(Val: S);
5228 writeFunctionTypeMetadataRecords(Stream, FS, GetValueID: GetValueId);
5229 getReferencedTypeIds(FS, ReferencedTypeIds);
5230
5231 NameVals.push_back(Elt: *ValueId);
5232 assert(ModuleIdMap.count(FS->modulePath()));
5233 NameVals.push_back(Elt: ModuleIdMap[FS->modulePath()]);
5234 NameVals.push_back(
5235 Elt: getEncodedGVSummaryFlags(Flags: FS->flags(), ImportAsDecl: shouldImportValueAsDecl(FS)));
5236 NameVals.push_back(Elt: FS->instCount());
5237 NameVals.push_back(Elt: getEncodedFFlags(Flags: FS->fflags()));
5238 // TODO: Stop writing entry count and bump bitcode version.
5239 NameVals.push_back(Elt: 0 /* EntryCount */);
5240
5241 // Fill in below
5242 NameVals.push_back(Elt: 0); // numrefs
5243 NameVals.push_back(Elt: 0); // rorefcnt
5244 NameVals.push_back(Elt: 0); // worefcnt
5245
5246 unsigned Count = 0, RORefCnt = 0, WORefCnt = 0;
5247 for (auto &RI : FS->refs()) {
5248 auto RefValueId = getValueId(ValGUID: RI.getGUID());
5249 if (!RefValueId)
5250 continue;
5251 NameVals.push_back(Elt: *RefValueId);
5252 if (RI.isReadOnly())
5253 RORefCnt++;
5254 else if (RI.isWriteOnly())
5255 WORefCnt++;
5256 Count++;
5257 }
5258 NameVals[6] = Count;
5259 NameVals[7] = RORefCnt;
5260 NameVals[8] = WORefCnt;
5261
5262 for (auto &EI : FS->calls()) {
5263 // If this GUID doesn't have a value id, it doesn't have a function
5264 // summary and we don't need to record any calls to it.
5265 std::optional<unsigned> CallValueId = GetValueId(EI.first);
5266 if (!CallValueId)
5267 continue;
5268 NameVals.push_back(Elt: *CallValueId);
5269 NameVals.push_back(Elt: getEncodedHotnessCallEdgeInfo(CI: EI.second));
5270 }
5271
5272 // Emit the finished record.
5273 Stream.EmitRecord(Code: bitc::FS_COMBINED_PROFILE, Vals: NameVals,
5274 Abbrev: FSCallsProfileAbbrev);
5275 NameVals.clear();
5276
5277 writeFunctionHeapProfileRecords(
5278 Stream, FS, CallsiteAbbrev, AllocAbbrev, /*ContextIdAbbvId*/ 0,
5279 /*PerModule*/ false,
5280 /*GetValueId*/
5281 GetValueID: [&](const ValueInfo &VI) -> unsigned {
5282 std::optional<unsigned> ValueID = GetValueId(VI);
5283 // This can happen in shared index files for distributed ThinLTO if
5284 // the callee function summary is not included. Record 0 which we
5285 // will have to deal with conservatively when doing any kind of
5286 // validation in the ThinLTO backends.
5287 if (!ValueID)
5288 return 0;
5289 return *ValueID;
5290 },
5291 /*GetStackIndex*/
5292 [&](unsigned I) {
5293 // Get the corresponding index into the list of StackIds actually
5294 // being written for this combined index (which may be a subset in
5295 // the case of distributed indexes).
5296 assert(StackIdIndicesToIndex.contains(I));
5297 return StackIdIndicesToIndex[I];
5298 },
5299 /*WriteContextSizeInfoIndex*/ false, CallStackPos, CallStackCount);
5300
5301 MaybeEmitOriginalName(*S);
5302 });
5303
5304 for (auto *AS : Aliases) {
5305 auto AliasValueId = SummaryToValueIdMap[AS];
5306 assert(AliasValueId);
5307 NameVals.push_back(Elt: AliasValueId);
5308 assert(ModuleIdMap.count(AS->modulePath()));
5309 NameVals.push_back(Elt: ModuleIdMap[AS->modulePath()]);
5310 NameVals.push_back(
5311 Elt: getEncodedGVSummaryFlags(Flags: AS->flags(), ImportAsDecl: shouldImportValueAsDecl(AS)));
5312 // Set value id to 0 when an alias is imported but the aliasee summary is
5313 // not contained in the index.
5314 auto AliaseeValueId =
5315 AS->hasAliasee() ? SummaryToValueIdMap[&AS->getAliasee()] : 0;
5316 NameVals.push_back(Elt: AliaseeValueId);
5317
5318 // Emit the finished record.
5319 Stream.EmitRecord(Code: bitc::FS_COMBINED_ALIAS, Vals: NameVals, Abbrev: FSAliasAbbrev);
5320 NameVals.clear();
5321 MaybeEmitOriginalName(*AS);
5322
5323 if (AS->hasAliasee())
5324 if (auto *FS = dyn_cast<FunctionSummary>(Val: &AS->getAliasee()))
5325 getReferencedTypeIds(FS, ReferencedTypeIds);
5326 }
5327
5328 SmallVector<std::pair<StringRef, GlobalValue::GUID>, 4> Functions;
5329 auto EmitCfiFunctions = [&](const CfiFunctionIndex &CfiIndex,
5330 bitc::GlobalValueSummarySymtabCodes Code) {
5331 if (CfiIndex.empty())
5332 return;
5333 for (GlobalValue::GUID GUID : DefOrUseGUIDs) {
5334 auto Names = CfiIndex.getNamesForGUID(GUID);
5335 for (StringRef Name : Names)
5336 Functions.push_back(Elt: {Name, GUID});
5337 }
5338 if (Functions.empty())
5339 return;
5340 llvm::sort(C&: Functions);
5341 for (const auto &Record : Functions) {
5342 NameVals.push_back(Elt: Record.second);
5343 NameVals.push_back(Elt: StrtabBuilder.add(S: Record.first));
5344 NameVals.push_back(Elt: Record.first.size());
5345 }
5346 Stream.EmitRecord(Code, Vals: NameVals);
5347 NameVals.clear();
5348 Functions.clear();
5349 };
5350
5351 EmitCfiFunctions(Index.cfiFunctionDefs(), bitc::FS_CFI_FUNCTION_DEFS);
5352 EmitCfiFunctions(Index.cfiFunctionDecls(), bitc::FS_CFI_FUNCTION_DECLS);
5353
5354 // Walk the GUIDs that were referenced, and write the
5355 // corresponding type id records.
5356 for (auto &T : ReferencedTypeIds) {
5357 auto TidIter = Index.typeIds().equal_range(x: T);
5358 for (const auto &[GUID, TypeIdPair] : make_range(p: TidIter)) {
5359 writeTypeIdSummaryRecord(NameVals, StrtabBuilder, Id: TypeIdPair.first,
5360 Summary: TypeIdPair.second);
5361 Stream.EmitRecord(Code: bitc::FS_TYPE_ID, Vals: NameVals);
5362 NameVals.clear();
5363 }
5364 }
5365
5366 if (Index.getBlockCount())
5367 Stream.EmitRecord(Code: bitc::FS_BLOCK_COUNT,
5368 Vals: ArrayRef<uint64_t>{Index.getBlockCount()});
5369
5370 Stream.ExitBlock();
5371}
5372
5373/// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the
5374/// current llvm version, and a record for the epoch number.
5375static void writeIdentificationBlock(BitstreamWriter &Stream) {
5376 Stream.EnterSubblock(BlockID: bitc::IDENTIFICATION_BLOCK_ID, CodeLen: 5);
5377
5378 // Write the "user readable" string identifying the bitcode producer
5379 auto Abbv = std::make_shared<BitCodeAbbrev>();
5380 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_STRING));
5381 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5382 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
5383 auto StringAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5384 writeStringRecord(Stream, Code: bitc::IDENTIFICATION_CODE_STRING,
5385 Str: "LLVM" LLVM_VERSION_STRING, AbbrevToUse: StringAbbrev);
5386
5387 // Write the epoch version
5388 Abbv = std::make_shared<BitCodeAbbrev>();
5389 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_EPOCH));
5390 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
5391 auto EpochAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5392 constexpr std::array<unsigned, 1> Vals = {._M_elems: {bitc::BITCODE_CURRENT_EPOCH}};
5393 Stream.EmitRecord(Code: bitc::IDENTIFICATION_CODE_EPOCH, Vals, Abbrev: EpochAbbrev);
5394 Stream.ExitBlock();
5395}
5396
5397void ModuleBitcodeWriter::writeModuleHash(StringRef View) {
5398 // Emit the module's hash.
5399 // MODULE_CODE_HASH: [5*i32]
5400 if (GenerateHash) {
5401 uint32_t Vals[5];
5402 Hasher.update(Data: ArrayRef<uint8_t>(
5403 reinterpret_cast<const uint8_t *>(View.data()), View.size()));
5404 std::array<uint8_t, 20> Hash = Hasher.result();
5405 for (int Pos = 0; Pos < 20; Pos += 4) {
5406 Vals[Pos / 4] = support::endian::read32be(P: Hash.data() + Pos);
5407 }
5408
5409 // Emit the finished record.
5410 Stream.EmitRecord(Code: bitc::MODULE_CODE_HASH, Vals);
5411
5412 if (ModHash)
5413 // Save the written hash value.
5414 llvm::copy(Range&: Vals, Out: std::begin(cont&: *ModHash));
5415 }
5416}
5417
5418void ModuleBitcodeWriter::write() {
5419 writeIdentificationBlock(Stream);
5420
5421 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5422 // We will want to write the module hash at this point. Block any flushing so
5423 // we can have access to the whole underlying data later.
5424 Stream.markAndBlockFlushing();
5425
5426 writeModuleVersion();
5427
5428 // Emit blockinfo, which defines the standard abbreviations etc.
5429 writeBlockInfo();
5430
5431 // Emit information describing all of the types in the module.
5432 writeTypeTable();
5433
5434 // Emit information about attribute groups.
5435 writeAttributeGroupTable();
5436
5437 // Emit information about parameter attributes.
5438 writeAttributeTable();
5439
5440 writeComdats();
5441
5442 // Emit top-level description of module, including target triple, inline asm,
5443 // descriptors for global variables, and function prototype info.
5444 writeModuleInfo();
5445
5446 // Emit constants.
5447 writeModuleConstants();
5448
5449 // Emit metadata kind names.
5450 writeModuleMetadataKinds();
5451
5452 // Emit metadata.
5453 writeModuleMetadata();
5454
5455 // Emit module-level use-lists.
5456 if (VE.shouldPreserveUseListOrder())
5457 writeUseListBlock(F: nullptr);
5458
5459 writeOperandBundleTags();
5460 writeSyncScopeNames();
5461
5462 // Emit function bodies.
5463 DenseMap<const Function *, uint64_t> FunctionToBitcodeIndex;
5464 for (const Function &F : M)
5465 if (!F.isDeclaration())
5466 writeFunction(F, FunctionToBitcodeIndex);
5467
5468 // Need to write after the above call to WriteFunction which populates
5469 // the summary information in the index.
5470 if (Index)
5471 writePerModuleGlobalValueSummary();
5472
5473 writeGlobalValueSymbolTable(FunctionToBitcodeIndex);
5474
5475 writeModuleHash(View: Stream.getMarkedBufferAndResumeFlushing());
5476
5477 Stream.ExitBlock();
5478}
5479
5480static void writeInt32ToBuffer(uint32_t Value, SmallVectorImpl<char> &Buffer,
5481 uint32_t &Position) {
5482 support::endian::write32le(P: &Buffer[Position], V: Value);
5483 Position += 4;
5484}
5485
5486/// If generating a bc file on darwin, we have to emit a
5487/// header and trailer to make it compatible with the system archiver. To do
5488/// this we emit the following header, and then emit a trailer that pads the
5489/// file out to be a multiple of 16 bytes.
5490///
5491/// struct bc_header {
5492/// uint32_t Magic; // 0x0B17C0DE
5493/// uint32_t Version; // Version, currently always 0.
5494/// uint32_t BitcodeOffset; // Offset to traditional bitcode file.
5495/// uint32_t BitcodeSize; // Size of traditional bitcode file.
5496/// uint32_t CPUType; // CPU specifier.
5497/// ... potentially more later ...
5498/// };
5499static void emitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer,
5500 const Triple &TT) {
5501 unsigned CPUType = ~0U;
5502
5503 // Match x86_64-*, i[3-9]86-*, powerpc-*, powerpc64-*, arm-*, thumb-*,
5504 // armv[0-9]-*, thumbv[0-9]-*, armv5te-*, or armv6t2-*. The CPUType is a magic
5505 // number from /usr/include/mach/machine.h. It is ok to reproduce the
5506 // specific constants here because they are implicitly part of the Darwin ABI.
5507 enum {
5508 DARWIN_CPU_ARCH_ABI64 = 0x01000000,
5509 DARWIN_CPU_TYPE_X86 = 7,
5510 DARWIN_CPU_TYPE_ARM = 12,
5511 DARWIN_CPU_TYPE_POWERPC = 18
5512 };
5513
5514 Triple::ArchType Arch = TT.getArch();
5515 if (Arch == Triple::x86_64)
5516 CPUType = DARWIN_CPU_TYPE_X86 | DARWIN_CPU_ARCH_ABI64;
5517 else if (Arch == Triple::x86)
5518 CPUType = DARWIN_CPU_TYPE_X86;
5519 else if (Arch == Triple::ppc)
5520 CPUType = DARWIN_CPU_TYPE_POWERPC;
5521 else if (Arch == Triple::ppc64)
5522 CPUType = DARWIN_CPU_TYPE_POWERPC | DARWIN_CPU_ARCH_ABI64;
5523 else if (Arch == Triple::arm || Arch == Triple::thumb)
5524 CPUType = DARWIN_CPU_TYPE_ARM;
5525
5526 // Traditional Bitcode starts after header.
5527 assert(Buffer.size() >= BWH_HeaderSize &&
5528 "Expected header size to be reserved");
5529 unsigned BCOffset = BWH_HeaderSize;
5530 unsigned BCSize = Buffer.size() - BWH_HeaderSize;
5531
5532 // Write the magic and version.
5533 unsigned Position = 0;
5534 writeInt32ToBuffer(Value: 0x0B17C0DE, Buffer, Position);
5535 writeInt32ToBuffer(Value: 0, Buffer, Position); // Version.
5536 writeInt32ToBuffer(Value: BCOffset, Buffer, Position);
5537 writeInt32ToBuffer(Value: BCSize, Buffer, Position);
5538 writeInt32ToBuffer(Value: CPUType, Buffer, Position);
5539
5540 // If the file is not a multiple of 16 bytes, insert dummy padding.
5541 while (Buffer.size() & 15)
5542 Buffer.push_back(Elt: 0);
5543}
5544
5545/// Helper to write the header common to all bitcode files.
5546static void writeBitcodeHeader(BitstreamWriter &Stream) {
5547 // Emit the file header.
5548 Stream.Emit(Val: (unsigned)'B', NumBits: 8);
5549 Stream.Emit(Val: (unsigned)'C', NumBits: 8);
5550 Stream.Emit(Val: 0x0, NumBits: 4);
5551 Stream.Emit(Val: 0xC, NumBits: 4);
5552 Stream.Emit(Val: 0xE, NumBits: 4);
5553 Stream.Emit(Val: 0xD, NumBits: 4);
5554}
5555
5556BitcodeWriter::BitcodeWriter(SmallVectorImpl<char> &Buffer)
5557 : Stream(new BitstreamWriter(Buffer)) {
5558 writeBitcodeHeader(Stream&: *Stream);
5559}
5560
5561BitcodeWriter::BitcodeWriter(raw_ostream &FS)
5562 : Stream(new BitstreamWriter(FS, FlushThreshold)) {
5563 writeBitcodeHeader(Stream&: *Stream);
5564}
5565
5566BitcodeWriter::~BitcodeWriter() { assert(WroteStrtab); }
5567
5568void BitcodeWriter::writeBlob(unsigned Block, unsigned Record, StringRef Blob) {
5569 Stream->EnterSubblock(BlockID: Block, CodeLen: 3);
5570
5571 auto Abbv = std::make_shared<BitCodeAbbrev>();
5572 Abbv->Add(OpInfo: BitCodeAbbrevOp(Record));
5573 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
5574 auto AbbrevNo = Stream->EmitAbbrev(Abbv: std::move(Abbv));
5575
5576 Stream->EmitRecordWithBlob(Abbrev: AbbrevNo, Vals: ArrayRef<uint64_t>{Record}, Blob);
5577
5578 Stream->ExitBlock();
5579}
5580
5581void BitcodeWriter::writeSymtab() {
5582 assert(!WroteStrtab && !WroteSymtab);
5583
5584 // If any module has module-level inline asm, we will require a registered asm
5585 // parser for the target so that we can create an accurate symbol table for
5586 // the module.
5587 for (Module *M : Mods) {
5588 if (M->getModuleInlineAsm().empty())
5589 continue;
5590
5591 std::string Err;
5592 const Triple TT(M->getTargetTriple());
5593 const Target *T = TargetRegistry::lookupTarget(TheTriple: TT, Error&: Err);
5594 if (!T || !T->hasMCAsmParser())
5595 return;
5596 }
5597
5598 WroteSymtab = true;
5599 SmallVector<char, 0> Symtab;
5600 // The irsymtab::build function may be unable to create a symbol table if the
5601 // module is malformed (e.g. it contains an invalid alias). Writing a symbol
5602 // table is not required for correctness, but we still want to be able to
5603 // write malformed modules to bitcode files, so swallow the error.
5604 if (Error E = irsymtab::build(Mods, Symtab, StrtabBuilder, Alloc)) {
5605 consumeError(Err: std::move(E));
5606 return;
5607 }
5608
5609 writeBlob(Block: bitc::SYMTAB_BLOCK_ID, Record: bitc::SYMTAB_BLOB,
5610 Blob: {Symtab.data(), Symtab.size()});
5611}
5612
5613void BitcodeWriter::writeStrtab() {
5614 assert(!WroteStrtab);
5615
5616 std::vector<char> Strtab;
5617 StrtabBuilder.finalizeInOrder();
5618 Strtab.resize(new_size: StrtabBuilder.getSize());
5619 StrtabBuilder.write(Buf: (uint8_t *)Strtab.data());
5620
5621 writeBlob(Block: bitc::STRTAB_BLOCK_ID, Record: bitc::STRTAB_BLOB,
5622 Blob: {Strtab.data(), Strtab.size()});
5623
5624 WroteStrtab = true;
5625}
5626
5627void BitcodeWriter::copyStrtab(StringRef Strtab) {
5628 writeBlob(Block: bitc::STRTAB_BLOCK_ID, Record: bitc::STRTAB_BLOB, Blob: Strtab);
5629 WroteStrtab = true;
5630}
5631
5632void BitcodeWriter::writeModule(const Module &M,
5633 bool ShouldPreserveUseListOrder,
5634 const ModuleSummaryIndex *Index,
5635 bool GenerateHash, ModuleHash *ModHash) {
5636 assert(!WroteStrtab);
5637
5638 // The Mods vector is used by irsymtab::build, which requires non-const
5639 // Modules in case it needs to materialize metadata. But the bitcode writer
5640 // requires that the module is materialized, so we can cast to non-const here,
5641 // after checking that it is in fact materialized.
5642 assert(M.isMaterialized());
5643 Mods.push_back(x: const_cast<Module *>(&M));
5644
5645 ModuleBitcodeWriter ModuleWriter(M, StrtabBuilder, *Stream,
5646 ShouldPreserveUseListOrder, Index,
5647 GenerateHash, ModHash);
5648 ModuleWriter.write();
5649}
5650
5651void BitcodeWriter::writeIndex(
5652 const ModuleSummaryIndex *Index,
5653 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex,
5654 const GVSummaryPtrSet *DecSummaries) {
5655 IndexBitcodeWriter IndexWriter(*Stream, StrtabBuilder, *Index, DecSummaries,
5656 ModuleToSummariesForIndex);
5657 IndexWriter.write();
5658}
5659
5660/// Write the specified module to the specified output stream.
5661void llvm::WriteBitcodeToFile(const Module &M, raw_ostream &Out,
5662 bool ShouldPreserveUseListOrder,
5663 const ModuleSummaryIndex *Index,
5664 bool GenerateHash, ModuleHash *ModHash) {
5665 auto Write = [&](BitcodeWriter &Writer) {
5666 Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash,
5667 ModHash);
5668 Writer.writeSymtab();
5669 Writer.writeStrtab();
5670 };
5671 Triple TT(M.getTargetTriple());
5672 if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) {
5673 // If this is darwin or another generic macho target, reserve space for the
5674 // header. Note that the header is computed *after* the output is known, so
5675 // we currently explicitly use a buffer, write to it, and then subsequently
5676 // flush to Out.
5677 SmallVector<char, 0> Buffer;
5678 Buffer.reserve(N: 256 * 1024);
5679 Buffer.insert(I: Buffer.begin(), NumToInsert: BWH_HeaderSize, Elt: 0);
5680 BitcodeWriter Writer(Buffer);
5681 Write(Writer);
5682 emitDarwinBCHeaderAndTrailer(Buffer, TT);
5683 Out.write(Ptr: Buffer.data(), Size: Buffer.size());
5684 } else {
5685 BitcodeWriter Writer(Out);
5686 Write(Writer);
5687 }
5688}
5689
5690void IndexBitcodeWriter::write() {
5691 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5692
5693 writeModuleVersion();
5694
5695 // Write the module paths in the combined index.
5696 writeModStrings();
5697
5698 // Write the summary combined index records.
5699 writeCombinedGlobalValueSummary();
5700
5701 Stream.ExitBlock();
5702}
5703
5704// Write the specified module summary index to the given raw output stream,
5705// where it will be written in a new bitcode block. This is used when
5706// writing the combined index file for ThinLTO. When writing a subset of the
5707// index for a distributed backend, provide a \p ModuleToSummariesForIndex map.
5708void llvm::writeIndexToFile(
5709 const ModuleSummaryIndex &Index, raw_ostream &Out,
5710 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex,
5711 const GVSummaryPtrSet *DecSummaries) {
5712 SmallVector<char, 0> Buffer;
5713 Buffer.reserve(N: 256 * 1024);
5714
5715 BitcodeWriter Writer(Buffer);
5716 Writer.writeIndex(Index: &Index, ModuleToSummariesForIndex, DecSummaries);
5717 Writer.writeStrtab();
5718
5719 Out.write(Ptr: (char *)&Buffer.front(), Size: Buffer.size());
5720}
5721
5722namespace {
5723
5724/// Class to manage the bitcode writing for a thin link bitcode file.
5725class ThinLinkBitcodeWriter : public ModuleBitcodeWriterBase {
5726 /// ModHash is for use in ThinLTO incremental build, generated while writing
5727 /// the module bitcode file.
5728 const ModuleHash *ModHash;
5729
5730public:
5731 ThinLinkBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
5732 BitstreamWriter &Stream,
5733 const ModuleSummaryIndex &Index,
5734 const ModuleHash &ModHash)
5735 : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
5736 /*ShouldPreserveUseListOrder=*/false, &Index),
5737 ModHash(&ModHash) {}
5738
5739 void write();
5740
5741private:
5742 void writeSimplifiedModuleInfo();
5743};
5744
5745} // end anonymous namespace
5746
5747// This function writes a simpilified module info for thin link bitcode file.
5748// It only contains the source file name along with the name(the offset and
5749// size in strtab) and linkage for global values. For the global value info
5750// entry, in order to keep linkage at offset 5, there are three zeros used
5751// as padding.
5752void ThinLinkBitcodeWriter::writeSimplifiedModuleInfo() {
5753 SmallVector<unsigned, 64> Vals;
5754 // Emit the module's source file name.
5755 {
5756 StringEncoding Bits = getStringEncoding(Str: M.getSourceFileName());
5757 BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
5758 if (Bits == SE_Char6)
5759 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
5760 else if (Bits == SE_Fixed7)
5761 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
5762
5763 // MODULE_CODE_SOURCE_FILENAME: [namechar x N]
5764 auto Abbv = std::make_shared<BitCodeAbbrev>();
5765 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
5766 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5767 Abbv->Add(OpInfo: AbbrevOpToUse);
5768 unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5769
5770 for (const auto P : M.getSourceFileName())
5771 Vals.push_back(Elt: (unsigned char)P);
5772
5773 Stream.EmitRecord(Code: bitc::MODULE_CODE_SOURCE_FILENAME, Vals, Abbrev: FilenameAbbrev);
5774 Vals.clear();
5775 }
5776
5777 // Emit the global variable information.
5778 for (const GlobalVariable &GV : M.globals()) {
5779 // GLOBALVAR: [strtab offset, strtab size, 0, 0, 0, linkage]
5780 Vals.push_back(Elt: StrtabBuilder.add(S: GV.getName()));
5781 Vals.push_back(Elt: GV.getName().size());
5782 Vals.push_back(Elt: 0);
5783 Vals.push_back(Elt: 0);
5784 Vals.push_back(Elt: 0);
5785 Vals.push_back(Elt: getEncodedLinkage(GV));
5786
5787 Stream.EmitRecord(Code: bitc::MODULE_CODE_GLOBALVAR, Vals);
5788 Vals.clear();
5789 }
5790
5791 // Emit the function proto information.
5792 for (const Function &F : M) {
5793 // FUNCTION: [strtab offset, strtab size, 0, 0, 0, linkage]
5794 Vals.push_back(Elt: StrtabBuilder.add(S: F.getName()));
5795 Vals.push_back(Elt: F.getName().size());
5796 Vals.push_back(Elt: 0);
5797 Vals.push_back(Elt: 0);
5798 Vals.push_back(Elt: 0);
5799 Vals.push_back(Elt: getEncodedLinkage(GV: F));
5800
5801 Stream.EmitRecord(Code: bitc::MODULE_CODE_FUNCTION, Vals);
5802 Vals.clear();
5803 }
5804
5805 // Emit the alias information.
5806 for (const GlobalAlias &A : M.aliases()) {
5807 // ALIAS: [strtab offset, strtab size, 0, 0, 0, linkage]
5808 Vals.push_back(Elt: StrtabBuilder.add(S: A.getName()));
5809 Vals.push_back(Elt: A.getName().size());
5810 Vals.push_back(Elt: 0);
5811 Vals.push_back(Elt: 0);
5812 Vals.push_back(Elt: 0);
5813 Vals.push_back(Elt: getEncodedLinkage(GV: A));
5814
5815 Stream.EmitRecord(Code: bitc::MODULE_CODE_ALIAS, Vals);
5816 Vals.clear();
5817 }
5818
5819 // Emit the ifunc information.
5820 for (const GlobalIFunc &I : M.ifuncs()) {
5821 // IFUNC: [strtab offset, strtab size, 0, 0, 0, linkage]
5822 Vals.push_back(Elt: StrtabBuilder.add(S: I.getName()));
5823 Vals.push_back(Elt: I.getName().size());
5824 Vals.push_back(Elt: 0);
5825 Vals.push_back(Elt: 0);
5826 Vals.push_back(Elt: 0);
5827 Vals.push_back(Elt: getEncodedLinkage(GV: I));
5828
5829 Stream.EmitRecord(Code: bitc::MODULE_CODE_IFUNC, Vals);
5830 Vals.clear();
5831 }
5832}
5833
5834void ThinLinkBitcodeWriter::write() {
5835 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5836
5837 writeModuleVersion();
5838
5839 writeSimplifiedModuleInfo();
5840
5841 writePerModuleGlobalValueSummary();
5842
5843 // Write module hash.
5844 Stream.EmitRecord(Code: bitc::MODULE_CODE_HASH, Vals: ArrayRef<uint32_t>(*ModHash));
5845
5846 Stream.ExitBlock();
5847}
5848
5849void BitcodeWriter::writeThinLinkBitcode(const Module &M,
5850 const ModuleSummaryIndex &Index,
5851 const ModuleHash &ModHash) {
5852 assert(!WroteStrtab);
5853
5854 // The Mods vector is used by irsymtab::build, which requires non-const
5855 // Modules in case it needs to materialize metadata. But the bitcode writer
5856 // requires that the module is materialized, so we can cast to non-const here,
5857 // after checking that it is in fact materialized.
5858 assert(M.isMaterialized());
5859 Mods.push_back(x: const_cast<Module *>(&M));
5860
5861 ThinLinkBitcodeWriter ThinLinkWriter(M, StrtabBuilder, *Stream, Index,
5862 ModHash);
5863 ThinLinkWriter.write();
5864}
5865
5866// Write the specified thin link bitcode file to the given raw output stream,
5867// where it will be written in a new bitcode block. This is used when
5868// writing the per-module index file for ThinLTO.
5869void llvm::writeThinLinkBitcodeToFile(const Module &M, raw_ostream &Out,
5870 const ModuleSummaryIndex &Index,
5871 const ModuleHash &ModHash) {
5872 SmallVector<char, 0> Buffer;
5873 Buffer.reserve(N: 256 * 1024);
5874
5875 BitcodeWriter Writer(Buffer);
5876 Writer.writeThinLinkBitcode(M, Index, ModHash);
5877 Writer.writeSymtab();
5878 Writer.writeStrtab();
5879
5880 Out.write(Ptr: (char *)&Buffer.front(), Size: Buffer.size());
5881}
5882
5883static const char *getSectionNameForBitcode(const Triple &T) {
5884 switch (T.getObjectFormat()) {
5885 case Triple::MachO:
5886 return "__LLVM,__bitcode";
5887 case Triple::COFF:
5888 case Triple::ELF:
5889 case Triple::Wasm:
5890 case Triple::UnknownObjectFormat:
5891 return ".llvmbc";
5892 case Triple::GOFF:
5893 llvm_unreachable("GOFF is not yet implemented");
5894 break;
5895 case Triple::SPIRV:
5896 if (T.getVendor() == Triple::AMD)
5897 return ".llvmbc";
5898 llvm_unreachable("SPIRV is not yet implemented");
5899 break;
5900 case Triple::XCOFF:
5901 llvm_unreachable("XCOFF is not yet implemented");
5902 break;
5903 case Triple::DXContainer:
5904 llvm_unreachable("DXContainer is not yet implemented");
5905 break;
5906 }
5907 llvm_unreachable("Unimplemented ObjectFormatType");
5908}
5909
5910static const char *getSectionNameForCommandline(const Triple &T) {
5911 switch (T.getObjectFormat()) {
5912 case Triple::MachO:
5913 return "__LLVM,__cmdline";
5914 case Triple::COFF:
5915 case Triple::ELF:
5916 case Triple::Wasm:
5917 case Triple::UnknownObjectFormat:
5918 return ".llvmcmd";
5919 case Triple::GOFF:
5920 llvm_unreachable("GOFF is not yet implemented");
5921 break;
5922 case Triple::SPIRV:
5923 if (T.getVendor() == Triple::AMD)
5924 return ".llvmcmd";
5925 llvm_unreachable("SPIRV is not yet implemented");
5926 break;
5927 case Triple::XCOFF:
5928 llvm_unreachable("XCOFF is not yet implemented");
5929 break;
5930 case Triple::DXContainer:
5931 llvm_unreachable("DXC is not yet implemented");
5932 break;
5933 }
5934 llvm_unreachable("Unimplemented ObjectFormatType");
5935}
5936
5937void llvm::embedBitcodeInModule(llvm::Module &M, llvm::MemoryBufferRef Buf,
5938 bool EmbedBitcode, bool EmbedCmdline,
5939 const std::vector<uint8_t> &CmdArgs) {
5940 // Save llvm.compiler.used and remove it.
5941 SmallVector<Constant *, 2> UsedArray;
5942 SmallVector<GlobalValue *, 4> UsedGlobals;
5943 GlobalVariable *Used = collectUsedGlobalVariables(M, Vec&: UsedGlobals, CompilerUsed: true);
5944 Type *UsedElementType = Used ? Used->getValueType()->getArrayElementType()
5945 : PointerType::getUnqual(C&: M.getContext());
5946 for (auto *GV : UsedGlobals) {
5947 if (GV->getName() != "llvm.embedded.module" &&
5948 GV->getName() != "llvm.cmdline")
5949 UsedArray.push_back(
5950 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
5951 }
5952 if (Used)
5953 Used->eraseFromParent();
5954
5955 // Embed the bitcode for the llvm module.
5956 std::string Data;
5957 ArrayRef<uint8_t> ModuleData;
5958 Triple T(M.getTargetTriple());
5959
5960 if (EmbedBitcode) {
5961 if (Buf.getBufferSize() == 0 ||
5962 !isBitcode(BufPtr: (const unsigned char *)Buf.getBufferStart(),
5963 BufEnd: (const unsigned char *)Buf.getBufferEnd())) {
5964 // If the input is LLVM Assembly, bitcode is produced by serializing
5965 // the module. Use-lists order need to be preserved in this case.
5966 llvm::raw_string_ostream OS(Data);
5967 llvm::WriteBitcodeToFile(M, Out&: OS, /* ShouldPreserveUseListOrder */ true);
5968 ModuleData =
5969 ArrayRef<uint8_t>((const uint8_t *)OS.str().data(), OS.str().size());
5970 } else
5971 // If the input is LLVM bitcode, write the input byte stream directly.
5972 ModuleData = ArrayRef<uint8_t>((const uint8_t *)Buf.getBufferStart(),
5973 Buf.getBufferSize());
5974 }
5975 llvm::Constant *ModuleConstant =
5976 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts: ModuleData);
5977 llvm::GlobalVariable *GV = new llvm::GlobalVariable(
5978 M, ModuleConstant->getType(), true, llvm::GlobalValue::PrivateLinkage,
5979 ModuleConstant);
5980 GV->setSection(getSectionNameForBitcode(T));
5981 // Set alignment to 1 to prevent padding between two contributions from input
5982 // sections after linking.
5983 GV->setAlignment(Align(1));
5984 UsedArray.push_back(
5985 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
5986 if (llvm::GlobalVariable *Old =
5987 M.getGlobalVariable(Name: "llvm.embedded.module", AllowInternal: true)) {
5988 assert(Old->hasZeroLiveUses() &&
5989 "llvm.embedded.module can only be used once in llvm.compiler.used");
5990 GV->takeName(V: Old);
5991 Old->eraseFromParent();
5992 } else {
5993 GV->setName("llvm.embedded.module");
5994 }
5995
5996 // Skip if only bitcode needs to be embedded.
5997 if (EmbedCmdline) {
5998 // Embed command-line options.
5999 ArrayRef<uint8_t> CmdData(const_cast<uint8_t *>(CmdArgs.data()),
6000 CmdArgs.size());
6001 llvm::Constant *CmdConstant =
6002 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts: CmdData);
6003 GV = new llvm::GlobalVariable(M, CmdConstant->getType(), true,
6004 llvm::GlobalValue::PrivateLinkage,
6005 CmdConstant);
6006 GV->setSection(getSectionNameForCommandline(T));
6007 GV->setAlignment(Align(1));
6008 UsedArray.push_back(
6009 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
6010 if (llvm::GlobalVariable *Old = M.getGlobalVariable(Name: "llvm.cmdline", AllowInternal: true)) {
6011 assert(Old->hasZeroLiveUses() &&
6012 "llvm.cmdline can only be used once in llvm.compiler.used");
6013 GV->takeName(V: Old);
6014 Old->eraseFromParent();
6015 } else {
6016 GV->setName("llvm.cmdline");
6017 }
6018 }
6019
6020 if (UsedArray.empty())
6021 return;
6022
6023 // Recreate llvm.compiler.used.
6024 ArrayType *ATy = ArrayType::get(ElementType: UsedElementType, NumElements: UsedArray.size());
6025 auto *NewUsed = new GlobalVariable(
6026 M, ATy, false, llvm::GlobalValue::AppendingLinkage,
6027 llvm::ConstantArray::get(T: ATy, V: UsedArray), "llvm.compiler.used");
6028 NewUsed->setSection("llvm.metadata");
6029}
6030