1//===- Bitcode/Writer/BitcodeWriter.cpp - Bitcode Writer ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Bitcode writer implementation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Bitcode/BitcodeWriter.h"
14#include "ValueEnumerator.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/DenseMap.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SetVector.h"
21#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/SmallString.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringMap.h"
25#include "llvm/ADT/StringRef.h"
26#include "llvm/Analysis/MemoryProfileInfo.h"
27#include "llvm/BinaryFormat/Dwarf.h"
28#include "llvm/Bitcode/BitcodeCommon.h"
29#include "llvm/Bitcode/BitcodeReader.h"
30#include "llvm/Bitcode/LLVMBitCodes.h"
31#include "llvm/Bitstream/BitCodes.h"
32#include "llvm/Bitstream/BitstreamWriter.h"
33#include "llvm/Config/llvm-config.h"
34#include "llvm/IR/Attributes.h"
35#include "llvm/IR/BasicBlock.h"
36#include "llvm/IR/Comdat.h"
37#include "llvm/IR/Constant.h"
38#include "llvm/IR/ConstantRangeList.h"
39#include "llvm/IR/Constants.h"
40#include "llvm/IR/DebugInfoMetadata.h"
41#include "llvm/IR/DebugLoc.h"
42#include "llvm/IR/DerivedTypes.h"
43#include "llvm/IR/Function.h"
44#include "llvm/IR/GlobalAlias.h"
45#include "llvm/IR/GlobalIFunc.h"
46#include "llvm/IR/GlobalObject.h"
47#include "llvm/IR/GlobalValue.h"
48#include "llvm/IR/GlobalVariable.h"
49#include "llvm/IR/InlineAsm.h"
50#include "llvm/IR/InstrTypes.h"
51#include "llvm/IR/Instruction.h"
52#include "llvm/IR/Instructions.h"
53#include "llvm/IR/LLVMContext.h"
54#include "llvm/IR/Metadata.h"
55#include "llvm/IR/Module.h"
56#include "llvm/IR/ModuleSummaryIndex.h"
57#include "llvm/IR/Operator.h"
58#include "llvm/IR/Type.h"
59#include "llvm/IR/UseListOrder.h"
60#include "llvm/IR/Value.h"
61#include "llvm/IR/ValueSymbolTable.h"
62#include "llvm/MC/StringTableBuilder.h"
63#include "llvm/MC/TargetRegistry.h"
64#include "llvm/Object/IRSymtab.h"
65#include "llvm/ProfileData/MemProf.h"
66#include "llvm/ProfileData/MemProfRadixTree.h"
67#include "llvm/Support/AtomicOrdering.h"
68#include "llvm/Support/Casting.h"
69#include "llvm/Support/CommandLine.h"
70#include "llvm/Support/Compiler.h"
71#include "llvm/Support/Endian.h"
72#include "llvm/Support/Error.h"
73#include "llvm/Support/ErrorHandling.h"
74#include "llvm/Support/MathExtras.h"
75#include "llvm/Support/SHA1.h"
76#include "llvm/Support/raw_ostream.h"
77#include "llvm/TargetParser/Triple.h"
78#include <algorithm>
79#include <cassert>
80#include <cstddef>
81#include <cstdint>
82#include <iterator>
83#include <map>
84#include <memory>
85#include <optional>
86#include <string>
87#include <utility>
88#include <vector>
89
90using namespace llvm;
91using namespace llvm::memprof;
92
93static cl::opt<unsigned>
94 IndexThreshold("bitcode-mdindex-threshold", cl::Hidden, cl::init(Val: 25),
95 cl::desc("Number of metadatas above which we emit an index "
96 "to enable lazy-loading"));
97static cl::opt<uint32_t> FlushThreshold(
98 "bitcode-flush-threshold", cl::Hidden, cl::init(Val: 512),
99 cl::desc("The threshold (unit M) for flushing LLVM bitcode."));
100
101// Since we only use the context information in the memprof summary records in
102// the LTO backends to do assertion checking, save time and space by only
103// serializing the context for non-NDEBUG builds.
104// TODO: Currently this controls writing context of the allocation info records,
105// which are larger and more expensive, but we should do this for the callsite
106// records as well.
107// FIXME: Convert to a const once this has undergone more sigificant testing.
108static cl::opt<bool>
109 CombinedIndexMemProfContext("combined-index-memprof-context", cl::Hidden,
110#ifdef NDEBUG
111 cl::init(Val: false),
112#else
113 cl::init(true),
114#endif
115 cl::desc(""));
116
117static cl::opt<bool> PreserveBitcodeUseListOrder(
118 "preserve-bc-uselistorder", cl::Hidden, cl::init(Val: true),
119 cl::desc("Preserve use-list order when writing LLVM bitcode."));
120
121namespace llvm {
122extern FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold;
123}
124
125namespace {
126
127/// These are manifest constants used by the bitcode writer. They do not need to
128/// be kept in sync with the reader, but need to be consistent within this file.
129enum {
130 // VALUE_SYMTAB_BLOCK abbrev id's.
131 VST_ENTRY_8_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
132 VST_ENTRY_7_ABBREV,
133 VST_ENTRY_6_ABBREV,
134 VST_BBENTRY_6_ABBREV,
135
136 // CONSTANTS_BLOCK abbrev id's.
137 CONSTANTS_SETTYPE_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
138 CONSTANTS_INTEGER_ABBREV,
139 CONSTANTS_BYTE_ABBREV,
140 CONSTANTS_CE_CAST_Abbrev,
141 CONSTANTS_NULL_Abbrev,
142
143 // FUNCTION_BLOCK abbrev id's.
144 FUNCTION_INST_LOAD_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
145 FUNCTION_INST_STORE_ABBREV,
146 FUNCTION_INST_UNOP_ABBREV,
147 FUNCTION_INST_UNOP_FLAGS_ABBREV,
148 FUNCTION_INST_BINOP_ABBREV,
149 FUNCTION_INST_BINOP_FLAGS_ABBREV,
150 FUNCTION_INST_CAST_ABBREV,
151 FUNCTION_INST_CAST_FLAGS_ABBREV,
152 FUNCTION_INST_RET_VOID_ABBREV,
153 FUNCTION_INST_RET_VAL_ABBREV,
154 FUNCTION_INST_BR_UNCOND_ABBREV,
155 FUNCTION_INST_BR_COND_ABBREV,
156 FUNCTION_INST_UNREACHABLE_ABBREV,
157 FUNCTION_INST_GEP_ABBREV,
158 FUNCTION_INST_CMP_ABBREV,
159 FUNCTION_INST_CMP_FLAGS_ABBREV,
160 FUNCTION_DEBUG_RECORD_VALUE_ABBREV,
161 FUNCTION_DEBUG_LOC_ABBREV,
162};
163
164/// Abstract class to manage the bitcode writing, subclassed for each bitcode
165/// file type.
166class BitcodeWriterBase {
167protected:
168 /// The stream created and owned by the client.
169 BitstreamWriter &Stream;
170
171 StringTableBuilder &StrtabBuilder;
172
173public:
174 /// Constructs a BitcodeWriterBase object that writes to the provided
175 /// \p Stream.
176 BitcodeWriterBase(BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder)
177 : Stream(Stream), StrtabBuilder(StrtabBuilder) {}
178
179protected:
180 void writeModuleVersion();
181};
182
183void BitcodeWriterBase::writeModuleVersion() {
184 // VERSION: [version#]
185 Stream.EmitRecord(Code: bitc::MODULE_CODE_VERSION, Vals: ArrayRef<uint64_t>{2});
186}
187
188/// Base class to manage the module bitcode writing, currently subclassed for
189/// ModuleBitcodeWriter and ThinLinkBitcodeWriter.
190class ModuleBitcodeWriterBase : public BitcodeWriterBase {
191protected:
192 /// The Module to write to bitcode.
193 const Module &M;
194
195 /// Enumerates ids for all values in the module.
196 ValueEnumerator VE;
197
198 /// Optional per-module index to write for ThinLTO.
199 const ModuleSummaryIndex *Index;
200
201 /// Map that holds the correspondence between GUIDs in the summary index,
202 /// that came from indirect call profiles, and a value id generated by this
203 /// class to use in the VST and summary block records.
204 std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
205
206 /// Tracks the last value id recorded in the GUIDToValueMap.
207 unsigned GlobalValueId;
208
209 /// Saves the offset of the VSTOffset record that must eventually be
210 /// backpatched with the offset of the actual VST.
211 uint64_t VSTOffsetPlaceholder = 0;
212
213public:
214 /// Constructs a ModuleBitcodeWriterBase object for the given Module,
215 /// writing to the provided \p Buffer.
216 ModuleBitcodeWriterBase(const Module &M, StringTableBuilder &StrtabBuilder,
217 BitstreamWriter &Stream,
218 bool ShouldPreserveUseListOrder,
219 const ModuleSummaryIndex *Index)
220 : BitcodeWriterBase(Stream, StrtabBuilder), M(M),
221 VE(M, PreserveBitcodeUseListOrder.getNumOccurrences()
222 ? PreserveBitcodeUseListOrder
223 : ShouldPreserveUseListOrder),
224 Index(Index) {
225 // Assign ValueIds to any callee values in the index that came from
226 // indirect call profiles and were recorded as a GUID not a Value*
227 // (which would have been assigned an ID by the ValueEnumerator).
228 // The starting ValueId is just after the number of values in the
229 // ValueEnumerator, so that they can be emitted in the VST.
230 GlobalValueId = VE.getValues().size();
231 if (!Index)
232 return;
233 for (const auto &GUIDSummaryLists : *Index)
234 // Examine all summaries for this GUID.
235 for (auto &Summary : GUIDSummaryLists.second.getSummaryList())
236 if (auto FS = dyn_cast<FunctionSummary>(Val: Summary.get())) {
237 // For each call in the function summary, see if the call
238 // is to a GUID (which means it is for an indirect call,
239 // otherwise we would have a Value for it). If so, synthesize
240 // a value id.
241 for (auto &CallEdge : FS->calls())
242 if (!CallEdge.first.haveGVs() || !CallEdge.first.getValue())
243 assignValueId(ValGUID: CallEdge.first.getGUID());
244
245 // For each referenced variables in the function summary, see if the
246 // variable is represented by a GUID (as opposed to a symbol to
247 // declarations or definitions in the module). If so, synthesize a
248 // value id.
249 for (auto &RefEdge : FS->refs())
250 if (!RefEdge.haveGVs() || !RefEdge.getValue())
251 assignValueId(ValGUID: RefEdge.getGUID());
252 }
253 }
254
255protected:
256 void writePerModuleGlobalValueSummary();
257
258private:
259 void writePerModuleFunctionSummaryRecord(
260 SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
261 unsigned ValueID, unsigned FSCallsProfileAbbrev, unsigned CallsiteAbbrev,
262 unsigned AllocAbbrev, unsigned ContextIdAbbvId, const Function &F,
263 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
264 CallStackId &CallStackCount);
265 void writeModuleLevelReferences(const GlobalVariable &V,
266 SmallVector<uint64_t, 64> &NameVals,
267 unsigned FSModRefsAbbrev,
268 unsigned FSModVTableRefsAbbrev);
269
270 void assignValueId(GlobalValue::GUID ValGUID) {
271 GUIDToValueIdMap[ValGUID] = ++GlobalValueId;
272 }
273
274 unsigned getValueId(GlobalValue::GUID ValGUID) {
275 const auto &VMI = GUIDToValueIdMap.find(x: ValGUID);
276 // Expect that any GUID value had a value Id assigned by an
277 // earlier call to assignValueId.
278 assert(VMI != GUIDToValueIdMap.end() &&
279 "GUID does not have assigned value Id");
280 return VMI->second;
281 }
282
283 // Helper to get the valueId for the type of value recorded in VI.
284 unsigned getValueId(ValueInfo VI) {
285 if (!VI.haveGVs() || !VI.getValue())
286 return getValueId(ValGUID: VI.getGUID());
287 return VE.getValueID(V: VI.getValue());
288 }
289
290 std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
291};
292
293/// Class to manage the bitcode writing for a module.
294class ModuleBitcodeWriter : public ModuleBitcodeWriterBase {
295 /// True if a module hash record should be written.
296 bool GenerateHash;
297
298 /// If non-null, when GenerateHash is true, the resulting hash is written
299 /// into ModHash.
300 ModuleHash *ModHash;
301
302 SHA1 Hasher;
303
304 /// The start bit of the identification block.
305 uint64_t BitcodeStartBit;
306
307public:
308 /// Constructs a ModuleBitcodeWriter object for the given Module,
309 /// writing to the provided \p Buffer.
310 ModuleBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
311 BitstreamWriter &Stream, bool ShouldPreserveUseListOrder,
312 const ModuleSummaryIndex *Index, bool GenerateHash,
313 ModuleHash *ModHash = nullptr)
314 : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
315 ShouldPreserveUseListOrder, Index),
316 GenerateHash(GenerateHash), ModHash(ModHash),
317 BitcodeStartBit(Stream.GetCurrentBitNo()) {}
318
319 /// Emit the current module to the bitstream.
320 void write();
321
322private:
323 uint64_t bitcodeStartBit() { return BitcodeStartBit; }
324
325 size_t addToStrtab(StringRef Str);
326
327 void writeAttributeGroupTable();
328 void writeAttributeTable();
329 void writeTypeTable();
330 void writeComdats();
331 void writeValueSymbolTableForwardDecl();
332 void writeModuleInfo();
333 void writeValueAsMetadata(const ValueAsMetadata *MD,
334 SmallVectorImpl<uint64_t> &Record);
335 void writeMDTuple(const MDTuple *N, SmallVectorImpl<uint64_t> &Record,
336 unsigned Abbrev);
337 unsigned createDILocationAbbrev();
338 void writeDILocation(const DILocation *N, SmallVectorImpl<uint64_t> &Record,
339 unsigned &Abbrev);
340 unsigned createGenericDINodeAbbrev();
341 void writeGenericDINode(const GenericDINode *N,
342 SmallVectorImpl<uint64_t> &Record, unsigned &Abbrev);
343 void writeDISubrange(const DISubrange *N, SmallVectorImpl<uint64_t> &Record,
344 unsigned Abbrev);
345 void writeDIGenericSubrange(const DIGenericSubrange *N,
346 SmallVectorImpl<uint64_t> &Record,
347 unsigned Abbrev);
348 void writeDIEnumerator(const DIEnumerator *N,
349 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
350 void writeDIBasicType(const DIBasicType *N, SmallVectorImpl<uint64_t> &Record,
351 unsigned Abbrev);
352 void writeDIFixedPointType(const DIFixedPointType *N,
353 SmallVectorImpl<uint64_t> &Record,
354 unsigned Abbrev);
355 void writeDIStringType(const DIStringType *N,
356 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
357 void writeDIDerivedType(const DIDerivedType *N,
358 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
359 void writeDISubrangeType(const DISubrangeType *N,
360 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
361 void writeDICompositeType(const DICompositeType *N,
362 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
363 void writeDISubroutineType(const DISubroutineType *N,
364 SmallVectorImpl<uint64_t> &Record,
365 unsigned Abbrev);
366 void writeDIFile(const DIFile *N, SmallVectorImpl<uint64_t> &Record,
367 unsigned Abbrev);
368 void writeDICompileUnit(const DICompileUnit *N,
369 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
370 void writeDISubprogram(const DISubprogram *N,
371 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
372 void writeDILexicalBlock(const DILexicalBlock *N,
373 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
374 void writeDILexicalBlockFile(const DILexicalBlockFile *N,
375 SmallVectorImpl<uint64_t> &Record,
376 unsigned Abbrev);
377 void writeDICommonBlock(const DICommonBlock *N,
378 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
379 void writeDINamespace(const DINamespace *N, SmallVectorImpl<uint64_t> &Record,
380 unsigned Abbrev);
381 void writeDIMacro(const DIMacro *N, SmallVectorImpl<uint64_t> &Record,
382 unsigned Abbrev);
383 void writeDIMacroFile(const DIMacroFile *N, SmallVectorImpl<uint64_t> &Record,
384 unsigned Abbrev);
385 void writeDIArgList(const DIArgList *N, SmallVectorImpl<uint64_t> &Record);
386 void writeDIModule(const DIModule *N, SmallVectorImpl<uint64_t> &Record,
387 unsigned Abbrev);
388 void writeDIAssignID(const DIAssignID *N, SmallVectorImpl<uint64_t> &Record,
389 unsigned Abbrev);
390 void writeDITemplateTypeParameter(const DITemplateTypeParameter *N,
391 SmallVectorImpl<uint64_t> &Record,
392 unsigned Abbrev);
393 void writeDITemplateValueParameter(const DITemplateValueParameter *N,
394 SmallVectorImpl<uint64_t> &Record,
395 unsigned Abbrev);
396 void writeDIGlobalVariable(const DIGlobalVariable *N,
397 SmallVectorImpl<uint64_t> &Record,
398 unsigned Abbrev);
399 void writeDILocalVariable(const DILocalVariable *N,
400 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
401 void writeDILabel(const DILabel *N,
402 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
403 void writeDIExpression(const DIExpression *N,
404 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
405 void writeDIGlobalVariableExpression(const DIGlobalVariableExpression *N,
406 SmallVectorImpl<uint64_t> &Record,
407 unsigned Abbrev);
408 void writeDIObjCProperty(const DIObjCProperty *N,
409 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
410 void writeDIImportedEntity(const DIImportedEntity *N,
411 SmallVectorImpl<uint64_t> &Record,
412 unsigned Abbrev);
413 unsigned createNamedMetadataAbbrev();
414 void writeNamedMetadata(SmallVectorImpl<uint64_t> &Record);
415 unsigned createMetadataStringsAbbrev();
416 void writeMetadataStrings(ArrayRef<const Metadata *> Strings,
417 SmallVectorImpl<uint64_t> &Record);
418 void writeMetadataRecords(ArrayRef<const Metadata *> MDs,
419 SmallVectorImpl<uint64_t> &Record,
420 std::vector<unsigned> *MDAbbrevs = nullptr,
421 std::vector<uint64_t> *IndexPos = nullptr);
422 void writeModuleMetadata();
423 void writeFunctionMetadata(const Function &F);
424 void writeFunctionMetadataAttachment(const Function &F);
425 void pushGlobalMetadataAttachment(SmallVectorImpl<uint64_t> &Record,
426 const GlobalObject &GO);
427 void writeModuleMetadataKinds();
428 void writeOperandBundleTags();
429 void writeSyncScopeNames();
430 void writeConstants(unsigned FirstVal, unsigned LastVal, bool isGlobal);
431 void writeModuleConstants();
432 bool pushValueAndType(const Value *V, unsigned InstID,
433 SmallVectorImpl<unsigned> &Vals);
434 bool pushValueOrMetadata(const Value *V, unsigned InstID,
435 SmallVectorImpl<unsigned> &Vals);
436 void writeOperandBundles(const CallBase &CB, unsigned InstID);
437 void pushValue(const Value *V, unsigned InstID,
438 SmallVectorImpl<unsigned> &Vals);
439 void pushValueSigned(const Value *V, unsigned InstID,
440 SmallVectorImpl<uint64_t> &Vals);
441 void writeInstruction(const Instruction &I, unsigned InstID,
442 SmallVectorImpl<unsigned> &Vals);
443 void writeFunctionLevelValueSymbolTable(const ValueSymbolTable &VST);
444 void writeGlobalValueSymbolTable(
445 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
446 void writeUseList(UseListOrder &&Order);
447 void writeUseListBlock(const Function *F);
448 void
449 writeFunction(const Function &F,
450 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
451 void writeBlockInfo();
452 void writeModuleHash(StringRef View);
453
454 unsigned getEncodedSyncScopeID(SyncScope::ID SSID) {
455 return unsigned(SSID);
456 }
457
458 unsigned getEncodedAlign(MaybeAlign Alignment) { return encode(A: Alignment); }
459};
460
461/// Class to manage the bitcode writing for a combined index.
462class IndexBitcodeWriter : public BitcodeWriterBase {
463 /// The combined index to write to bitcode.
464 const ModuleSummaryIndex &Index;
465
466 /// When writing combined summaries, provides the set of global value
467 /// summaries for which the value (function, function alias, etc) should be
468 /// imported as a declaration.
469 const GVSummaryPtrSet *DecSummaries = nullptr;
470
471 /// When writing a subset of the index for distributed backends, client
472 /// provides a map of modules to the corresponding GUIDs/summaries to write.
473 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex;
474
475 /// Map that holds the correspondence between the GUID used in the combined
476 /// index and a value id generated by this class to use in references.
477 std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
478
479 // The stack ids used by this index, which will be a subset of those in
480 // the full index in the case of distributed indexes.
481 std::vector<uint64_t> StackIds;
482
483 // Keep a map of the stack id indices used by records being written for this
484 // index to the index of the corresponding stack id in the above StackIds
485 // vector. Ensures we write each referenced stack id once.
486 DenseMap<unsigned, unsigned> StackIdIndicesToIndex;
487
488 /// Tracks the last value id recorded in the GUIDToValueMap.
489 unsigned GlobalValueId = 0;
490
491 /// Tracks the assignment of module paths in the module path string table to
492 /// an id assigned for use in summary references to the module path.
493 DenseMap<StringRef, uint64_t> ModuleIdMap;
494
495public:
496 /// Constructs a IndexBitcodeWriter object for the given combined index,
497 /// writing to the provided \p Buffer. When writing a subset of the index
498 /// for a distributed backend, provide a \p ModuleToSummariesForIndex map.
499 /// If provided, \p DecSummaries specifies the set of summaries for which
500 /// the corresponding functions or aliased functions should be imported as a
501 /// declaration (but not definition) for each module.
502 IndexBitcodeWriter(
503 BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder,
504 const ModuleSummaryIndex &Index,
505 const GVSummaryPtrSet *DecSummaries = nullptr,
506 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex = nullptr)
507 : BitcodeWriterBase(Stream, StrtabBuilder), Index(Index),
508 DecSummaries(DecSummaries),
509 ModuleToSummariesForIndex(ModuleToSummariesForIndex) {
510
511 // See if the StackIdIndex was already added to the StackId map and
512 // vector. If not, record it.
513 auto RecordStackIdReference = [&](unsigned StackIdIndex) {
514 // If the StackIdIndex is not yet in the map, the below insert ensures
515 // that it will point to the new StackIds vector entry we push to just
516 // below.
517 auto Inserted =
518 StackIdIndicesToIndex.insert(KV: {StackIdIndex, StackIds.size()});
519 if (Inserted.second)
520 StackIds.push_back(x: Index.getStackIdAtIndex(Index: StackIdIndex));
521 };
522
523 // Assign unique value ids to all summaries to be written, for use
524 // in writing out the call graph edges. Save the mapping from GUID
525 // to the new global value id to use when writing those edges, which
526 // are currently saved in the index in terms of GUID.
527 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
528 GUIDToValueIdMap[I.first] = ++GlobalValueId;
529 // If this is invoked for an aliasee, we want to record the above mapping,
530 // but not the information needed for its summary entry (if the aliasee is
531 // to be imported, we will invoke this separately with IsAliasee=false).
532 if (IsAliasee)
533 return;
534 auto *FS = dyn_cast<FunctionSummary>(Val: I.second);
535 if (!FS)
536 return;
537 // Record all stack id indices actually used in the summary entries being
538 // written, so that we can compact them in the case of distributed ThinLTO
539 // indexes.
540 for (auto &CI : FS->callsites()) {
541 // If the stack id list is empty, this callsite info was synthesized for
542 // a missing tail call frame. Ensure that the callee's GUID gets a value
543 // id. Normally we only generate these for defined summaries, which in
544 // the case of distributed ThinLTO is only the functions already defined
545 // in the module or that we want to import. We don't bother to include
546 // all the callee symbols as they aren't normally needed in the backend.
547 // However, for the synthesized callsite infos we do need the callee
548 // GUID in the backend so that we can correlate the identified callee
549 // with this callsite info (which for non-tail calls is done by the
550 // ordering of the callsite infos and verified via stack ids).
551 if (CI.StackIdIndices.empty()) {
552 GUIDToValueIdMap[CI.Callee.getGUID()] = ++GlobalValueId;
553 continue;
554 }
555 for (auto Idx : CI.StackIdIndices)
556 RecordStackIdReference(Idx);
557 }
558 if (CombinedIndexMemProfContext) {
559 for (auto &AI : FS->allocs())
560 for (auto &MIB : AI.MIBs)
561 for (auto Idx : MIB.StackIdIndices)
562 RecordStackIdReference(Idx);
563 }
564 });
565 }
566
567 /// The below iterator returns the GUID and associated summary.
568 using GVInfo = std::pair<GlobalValue::GUID, GlobalValueSummary *>;
569
570 /// Calls the callback for each value GUID and summary to be written to
571 /// bitcode. This hides the details of whether they are being pulled from the
572 /// entire index or just those in a provided ModuleToSummariesForIndex map.
573 template<typename Functor>
574 void forEachSummary(Functor Callback) {
575 if (ModuleToSummariesForIndex) {
576 for (auto &M : *ModuleToSummariesForIndex)
577 for (auto &Summary : M.second) {
578 Callback(Summary, false);
579 // Ensure aliasee is handled, e.g. for assigning a valueId,
580 // even if we are not importing the aliasee directly (the
581 // imported alias will contain a copy of aliasee).
582 if (auto *AS = dyn_cast<AliasSummary>(Val: Summary.getSecond()))
583 Callback({AS->getAliaseeGUID(), &AS->getAliasee()}, true);
584 }
585 } else {
586 for (auto &Summaries : Index)
587 for (auto &Summary : Summaries.second.getSummaryList())
588 Callback({Summaries.first, Summary.get()}, false);
589 }
590 }
591
592 /// Calls the callback for each entry in the modulePaths StringMap that
593 /// should be written to the module path string table. This hides the details
594 /// of whether they are being pulled from the entire index or just those in a
595 /// provided ModuleToSummariesForIndex map.
596 template <typename Functor> void forEachModule(Functor Callback) {
597 if (ModuleToSummariesForIndex) {
598 for (const auto &M : *ModuleToSummariesForIndex) {
599 const auto &MPI = Index.modulePaths().find(Key: M.first);
600 if (MPI == Index.modulePaths().end()) {
601 // This should only happen if the bitcode file was empty, in which
602 // case we shouldn't be importing (the ModuleToSummariesForIndex
603 // would only include the module we are writing and index for).
604 assert(ModuleToSummariesForIndex->size() == 1);
605 continue;
606 }
607 Callback(*MPI);
608 }
609 } else {
610 // Since StringMap iteration order isn't guaranteed, order by path string
611 // first.
612 // FIXME: Make this a vector of StringMapEntry instead to avoid the later
613 // map lookup.
614 std::vector<StringRef> ModulePaths;
615 for (auto &[ModPath, _] : Index.modulePaths())
616 ModulePaths.push_back(x: ModPath);
617 llvm::sort(C&: ModulePaths);
618 for (auto &ModPath : ModulePaths)
619 Callback(*Index.modulePaths().find(Key: ModPath));
620 }
621 }
622
623 /// Main entry point for writing a combined index to bitcode.
624 void write();
625
626private:
627 void writeModStrings();
628 void writeCombinedGlobalValueSummary();
629
630 std::optional<unsigned> getValueId(GlobalValue::GUID ValGUID) {
631 auto VMI = GUIDToValueIdMap.find(x: ValGUID);
632 if (VMI == GUIDToValueIdMap.end())
633 return std::nullopt;
634 return VMI->second;
635 }
636
637 std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
638};
639
640} // end anonymous namespace
641
642static unsigned getEncodedCastOpcode(unsigned Opcode) {
643 switch (Opcode) {
644 default: llvm_unreachable("Unknown cast instruction!");
645 case Instruction::Trunc : return bitc::CAST_TRUNC;
646 case Instruction::ZExt : return bitc::CAST_ZEXT;
647 case Instruction::SExt : return bitc::CAST_SEXT;
648 case Instruction::FPToUI : return bitc::CAST_FPTOUI;
649 case Instruction::FPToSI : return bitc::CAST_FPTOSI;
650 case Instruction::UIToFP : return bitc::CAST_UITOFP;
651 case Instruction::SIToFP : return bitc::CAST_SITOFP;
652 case Instruction::FPTrunc : return bitc::CAST_FPTRUNC;
653 case Instruction::FPExt : return bitc::CAST_FPEXT;
654 case Instruction::PtrToAddr: return bitc::CAST_PTRTOADDR;
655 case Instruction::PtrToInt: return bitc::CAST_PTRTOINT;
656 case Instruction::IntToPtr: return bitc::CAST_INTTOPTR;
657 case Instruction::BitCast : return bitc::CAST_BITCAST;
658 case Instruction::AddrSpaceCast: return bitc::CAST_ADDRSPACECAST;
659 }
660}
661
662static unsigned getEncodedUnaryOpcode(unsigned Opcode) {
663 switch (Opcode) {
664 default: llvm_unreachable("Unknown binary instruction!");
665 case Instruction::FNeg: return bitc::UNOP_FNEG;
666 }
667}
668
669static unsigned getEncodedBinaryOpcode(unsigned Opcode) {
670 switch (Opcode) {
671 default: llvm_unreachable("Unknown binary instruction!");
672 case Instruction::Add:
673 case Instruction::FAdd: return bitc::BINOP_ADD;
674 case Instruction::Sub:
675 case Instruction::FSub: return bitc::BINOP_SUB;
676 case Instruction::Mul:
677 case Instruction::FMul: return bitc::BINOP_MUL;
678 case Instruction::UDiv: return bitc::BINOP_UDIV;
679 case Instruction::FDiv:
680 case Instruction::SDiv: return bitc::BINOP_SDIV;
681 case Instruction::URem: return bitc::BINOP_UREM;
682 case Instruction::FRem:
683 case Instruction::SRem: return bitc::BINOP_SREM;
684 case Instruction::Shl: return bitc::BINOP_SHL;
685 case Instruction::LShr: return bitc::BINOP_LSHR;
686 case Instruction::AShr: return bitc::BINOP_ASHR;
687 case Instruction::And: return bitc::BINOP_AND;
688 case Instruction::Or: return bitc::BINOP_OR;
689 case Instruction::Xor: return bitc::BINOP_XOR;
690 }
691}
692
693static unsigned getEncodedRMWOperation(AtomicRMWInst::BinOp Op) {
694 switch (Op) {
695 default: llvm_unreachable("Unknown RMW operation!");
696 case AtomicRMWInst::Xchg: return bitc::RMW_XCHG;
697 case AtomicRMWInst::Add: return bitc::RMW_ADD;
698 case AtomicRMWInst::Sub: return bitc::RMW_SUB;
699 case AtomicRMWInst::And: return bitc::RMW_AND;
700 case AtomicRMWInst::Nand: return bitc::RMW_NAND;
701 case AtomicRMWInst::Or: return bitc::RMW_OR;
702 case AtomicRMWInst::Xor: return bitc::RMW_XOR;
703 case AtomicRMWInst::Max: return bitc::RMW_MAX;
704 case AtomicRMWInst::Min: return bitc::RMW_MIN;
705 case AtomicRMWInst::UMax: return bitc::RMW_UMAX;
706 case AtomicRMWInst::UMin: return bitc::RMW_UMIN;
707 case AtomicRMWInst::FAdd: return bitc::RMW_FADD;
708 case AtomicRMWInst::FSub: return bitc::RMW_FSUB;
709 case AtomicRMWInst::FMax: return bitc::RMW_FMAX;
710 case AtomicRMWInst::FMin: return bitc::RMW_FMIN;
711 case AtomicRMWInst::FMaximum:
712 return bitc::RMW_FMAXIMUM;
713 case AtomicRMWInst::FMinimum:
714 return bitc::RMW_FMINIMUM;
715 case AtomicRMWInst::FMaximumNum:
716 return bitc::RMW_FMAXIMUMNUM;
717 case AtomicRMWInst::FMinimumNum:
718 return bitc::RMW_FMINIMUMNUM;
719 case AtomicRMWInst::UIncWrap:
720 return bitc::RMW_UINC_WRAP;
721 case AtomicRMWInst::UDecWrap:
722 return bitc::RMW_UDEC_WRAP;
723 case AtomicRMWInst::USubCond:
724 return bitc::RMW_USUB_COND;
725 case AtomicRMWInst::USubSat:
726 return bitc::RMW_USUB_SAT;
727 }
728}
729
730static unsigned getEncodedOrdering(AtomicOrdering Ordering) {
731 switch (Ordering) {
732 case AtomicOrdering::NotAtomic: return bitc::ORDERING_NOTATOMIC;
733 case AtomicOrdering::Unordered: return bitc::ORDERING_UNORDERED;
734 case AtomicOrdering::Monotonic: return bitc::ORDERING_MONOTONIC;
735 case AtomicOrdering::Acquire: return bitc::ORDERING_ACQUIRE;
736 case AtomicOrdering::Release: return bitc::ORDERING_RELEASE;
737 case AtomicOrdering::AcquireRelease: return bitc::ORDERING_ACQREL;
738 case AtomicOrdering::SequentiallyConsistent: return bitc::ORDERING_SEQCST;
739 }
740 llvm_unreachable("Invalid ordering");
741}
742
743static void writeStringRecord(BitstreamWriter &Stream, unsigned Code,
744 StringRef Str, unsigned AbbrevToUse) {
745 SmallVector<unsigned, 64> Vals;
746
747 // Code: [strchar x N]
748 for (char C : Str) {
749 if (AbbrevToUse && !BitCodeAbbrevOp::isChar6(C))
750 AbbrevToUse = 0;
751 Vals.push_back(Elt: C);
752 }
753
754 // Emit the finished record.
755 Stream.EmitRecord(Code, Vals, Abbrev: AbbrevToUse);
756}
757
758static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
759 switch (Kind) {
760 case Attribute::Alignment:
761 return bitc::ATTR_KIND_ALIGNMENT;
762 case Attribute::AllocAlign:
763 return bitc::ATTR_KIND_ALLOC_ALIGN;
764 case Attribute::AllocSize:
765 return bitc::ATTR_KIND_ALLOC_SIZE;
766 case Attribute::AlwaysInline:
767 return bitc::ATTR_KIND_ALWAYS_INLINE;
768 case Attribute::Builtin:
769 return bitc::ATTR_KIND_BUILTIN;
770 case Attribute::ByVal:
771 return bitc::ATTR_KIND_BY_VAL;
772 case Attribute::Convergent:
773 return bitc::ATTR_KIND_CONVERGENT;
774 case Attribute::InAlloca:
775 return bitc::ATTR_KIND_IN_ALLOCA;
776 case Attribute::Cold:
777 return bitc::ATTR_KIND_COLD;
778 case Attribute::DisableSanitizerInstrumentation:
779 return bitc::ATTR_KIND_DISABLE_SANITIZER_INSTRUMENTATION;
780 case Attribute::FnRetThunkExtern:
781 return bitc::ATTR_KIND_FNRETTHUNK_EXTERN;
782 case Attribute::Flatten:
783 return bitc::ATTR_KIND_FLATTEN;
784 case Attribute::Hot:
785 return bitc::ATTR_KIND_HOT;
786 case Attribute::ElementType:
787 return bitc::ATTR_KIND_ELEMENTTYPE;
788 case Attribute::HybridPatchable:
789 return bitc::ATTR_KIND_HYBRID_PATCHABLE;
790 case Attribute::InlineHint:
791 return bitc::ATTR_KIND_INLINE_HINT;
792 case Attribute::InReg:
793 return bitc::ATTR_KIND_IN_REG;
794 case Attribute::JumpTable:
795 return bitc::ATTR_KIND_JUMP_TABLE;
796 case Attribute::MinSize:
797 return bitc::ATTR_KIND_MIN_SIZE;
798 case Attribute::AllocatedPointer:
799 return bitc::ATTR_KIND_ALLOCATED_POINTER;
800 case Attribute::AllocKind:
801 return bitc::ATTR_KIND_ALLOC_KIND;
802 case Attribute::Memory:
803 return bitc::ATTR_KIND_MEMORY;
804 case Attribute::NoFPClass:
805 return bitc::ATTR_KIND_NOFPCLASS;
806 case Attribute::Naked:
807 return bitc::ATTR_KIND_NAKED;
808 case Attribute::Nest:
809 return bitc::ATTR_KIND_NEST;
810 case Attribute::NoAlias:
811 return bitc::ATTR_KIND_NO_ALIAS;
812 case Attribute::NoBuiltin:
813 return bitc::ATTR_KIND_NO_BUILTIN;
814 case Attribute::NoCallback:
815 return bitc::ATTR_KIND_NO_CALLBACK;
816 case Attribute::NoDivergenceSource:
817 return bitc::ATTR_KIND_NO_DIVERGENCE_SOURCE;
818 case Attribute::NoDuplicate:
819 return bitc::ATTR_KIND_NO_DUPLICATE;
820 case Attribute::NoFree:
821 return bitc::ATTR_KIND_NOFREE;
822 case Attribute::NoImplicitFloat:
823 return bitc::ATTR_KIND_NO_IMPLICIT_FLOAT;
824 case Attribute::NoInline:
825 return bitc::ATTR_KIND_NO_INLINE;
826 case Attribute::NoRecurse:
827 return bitc::ATTR_KIND_NO_RECURSE;
828 case Attribute::NoMerge:
829 return bitc::ATTR_KIND_NO_MERGE;
830 case Attribute::NonLazyBind:
831 return bitc::ATTR_KIND_NON_LAZY_BIND;
832 case Attribute::NonNull:
833 return bitc::ATTR_KIND_NON_NULL;
834 case Attribute::Dereferenceable:
835 return bitc::ATTR_KIND_DEREFERENCEABLE;
836 case Attribute::DereferenceableOrNull:
837 return bitc::ATTR_KIND_DEREFERENCEABLE_OR_NULL;
838 case Attribute::NoRedZone:
839 return bitc::ATTR_KIND_NO_RED_ZONE;
840 case Attribute::NoReturn:
841 return bitc::ATTR_KIND_NO_RETURN;
842 case Attribute::NoSync:
843 return bitc::ATTR_KIND_NOSYNC;
844 case Attribute::NoCfCheck:
845 return bitc::ATTR_KIND_NOCF_CHECK;
846 case Attribute::NoProfile:
847 return bitc::ATTR_KIND_NO_PROFILE;
848 case Attribute::SkipProfile:
849 return bitc::ATTR_KIND_SKIP_PROFILE;
850 case Attribute::NoUnwind:
851 return bitc::ATTR_KIND_NO_UNWIND;
852 case Attribute::NoSanitizeBounds:
853 return bitc::ATTR_KIND_NO_SANITIZE_BOUNDS;
854 case Attribute::NoSanitizeCoverage:
855 return bitc::ATTR_KIND_NO_SANITIZE_COVERAGE;
856 case Attribute::NullPointerIsValid:
857 return bitc::ATTR_KIND_NULL_POINTER_IS_VALID;
858 case Attribute::OptimizeForDebugging:
859 return bitc::ATTR_KIND_OPTIMIZE_FOR_DEBUGGING;
860 case Attribute::OptForFuzzing:
861 return bitc::ATTR_KIND_OPT_FOR_FUZZING;
862 case Attribute::OptimizeForSize:
863 return bitc::ATTR_KIND_OPTIMIZE_FOR_SIZE;
864 case Attribute::OptimizeNone:
865 return bitc::ATTR_KIND_OPTIMIZE_NONE;
866 case Attribute::ReadNone:
867 return bitc::ATTR_KIND_READ_NONE;
868 case Attribute::ReadOnly:
869 return bitc::ATTR_KIND_READ_ONLY;
870 case Attribute::Returned:
871 return bitc::ATTR_KIND_RETURNED;
872 case Attribute::ReturnsTwice:
873 return bitc::ATTR_KIND_RETURNS_TWICE;
874 case Attribute::SExt:
875 return bitc::ATTR_KIND_S_EXT;
876 case Attribute::Speculatable:
877 return bitc::ATTR_KIND_SPECULATABLE;
878 case Attribute::StackAlignment:
879 return bitc::ATTR_KIND_STACK_ALIGNMENT;
880 case Attribute::StackProtect:
881 return bitc::ATTR_KIND_STACK_PROTECT;
882 case Attribute::StackProtectReq:
883 return bitc::ATTR_KIND_STACK_PROTECT_REQ;
884 case Attribute::StackProtectStrong:
885 return bitc::ATTR_KIND_STACK_PROTECT_STRONG;
886 case Attribute::SafeStack:
887 return bitc::ATTR_KIND_SAFESTACK;
888 case Attribute::ShadowCallStack:
889 return bitc::ATTR_KIND_SHADOWCALLSTACK;
890 case Attribute::StrictFP:
891 return bitc::ATTR_KIND_STRICT_FP;
892 case Attribute::StructRet:
893 return bitc::ATTR_KIND_STRUCT_RET;
894 case Attribute::SanitizeAddress:
895 return bitc::ATTR_KIND_SANITIZE_ADDRESS;
896 case Attribute::SanitizeAllocToken:
897 return bitc::ATTR_KIND_SANITIZE_ALLOC_TOKEN;
898 case Attribute::SanitizeHWAddress:
899 return bitc::ATTR_KIND_SANITIZE_HWADDRESS;
900 case Attribute::SanitizeThread:
901 return bitc::ATTR_KIND_SANITIZE_THREAD;
902 case Attribute::SanitizeType:
903 return bitc::ATTR_KIND_SANITIZE_TYPE;
904 case Attribute::SanitizeMemory:
905 return bitc::ATTR_KIND_SANITIZE_MEMORY;
906 case Attribute::SanitizeNumericalStability:
907 return bitc::ATTR_KIND_SANITIZE_NUMERICAL_STABILITY;
908 case Attribute::SanitizeRealtime:
909 return bitc::ATTR_KIND_SANITIZE_REALTIME;
910 case Attribute::SanitizeRealtimeBlocking:
911 return bitc::ATTR_KIND_SANITIZE_REALTIME_BLOCKING;
912 case Attribute::SpeculativeLoadHardening:
913 return bitc::ATTR_KIND_SPECULATIVE_LOAD_HARDENING;
914 case Attribute::SwiftError:
915 return bitc::ATTR_KIND_SWIFT_ERROR;
916 case Attribute::SwiftSelf:
917 return bitc::ATTR_KIND_SWIFT_SELF;
918 case Attribute::SwiftAsync:
919 return bitc::ATTR_KIND_SWIFT_ASYNC;
920 case Attribute::UWTable:
921 return bitc::ATTR_KIND_UW_TABLE;
922 case Attribute::VScaleRange:
923 return bitc::ATTR_KIND_VSCALE_RANGE;
924 case Attribute::WillReturn:
925 return bitc::ATTR_KIND_WILLRETURN;
926 case Attribute::WriteOnly:
927 return bitc::ATTR_KIND_WRITEONLY;
928 case Attribute::ZExt:
929 return bitc::ATTR_KIND_Z_EXT;
930 case Attribute::ImmArg:
931 return bitc::ATTR_KIND_IMMARG;
932 case Attribute::SanitizeMemTag:
933 return bitc::ATTR_KIND_SANITIZE_MEMTAG;
934 case Attribute::Preallocated:
935 return bitc::ATTR_KIND_PREALLOCATED;
936 case Attribute::NoUndef:
937 return bitc::ATTR_KIND_NOUNDEF;
938 case Attribute::ByRef:
939 return bitc::ATTR_KIND_BYREF;
940 case Attribute::MustProgress:
941 return bitc::ATTR_KIND_MUSTPROGRESS;
942 case Attribute::PresplitCoroutine:
943 return bitc::ATTR_KIND_PRESPLIT_COROUTINE;
944 case Attribute::Writable:
945 return bitc::ATTR_KIND_WRITABLE;
946 case Attribute::CoroDestroyOnlyWhenComplete:
947 return bitc::ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE;
948 case Attribute::CoroElideSafe:
949 return bitc::ATTR_KIND_CORO_ELIDE_SAFE;
950 case Attribute::DeadOnUnwind:
951 return bitc::ATTR_KIND_DEAD_ON_UNWIND;
952 case Attribute::Range:
953 return bitc::ATTR_KIND_RANGE;
954 case Attribute::Initializes:
955 return bitc::ATTR_KIND_INITIALIZES;
956 case Attribute::NoExt:
957 return bitc::ATTR_KIND_NO_EXT;
958 case Attribute::Captures:
959 return bitc::ATTR_KIND_CAPTURES;
960 case Attribute::DeadOnReturn:
961 return bitc::ATTR_KIND_DEAD_ON_RETURN;
962 case Attribute::NoCreateUndefOrPoison:
963 return bitc::ATTR_KIND_NO_CREATE_UNDEF_OR_POISON;
964 case Attribute::DenormalFPEnv:
965 return bitc::ATTR_KIND_DENORMAL_FPENV;
966 case Attribute::NoOutline:
967 return bitc::ATTR_KIND_NOOUTLINE;
968 case Attribute::EndAttrKinds:
969 llvm_unreachable("Can not encode end-attribute kinds marker.");
970 case Attribute::None:
971 llvm_unreachable("Can not encode none-attribute.");
972 case Attribute::EmptyKey:
973 case Attribute::TombstoneKey:
974 llvm_unreachable("Trying to encode EmptyKey/TombstoneKey");
975 }
976
977 llvm_unreachable("Trying to encode unknown attribute");
978}
979
980static void emitSignedInt64(SmallVectorImpl<uint64_t> &Vals, uint64_t V) {
981 if ((int64_t)V >= 0)
982 Vals.push_back(Elt: V << 1);
983 else
984 Vals.push_back(Elt: (-V << 1) | 1);
985}
986
987static void emitWideAPInt(SmallVectorImpl<uint64_t> &Vals, const APInt &A) {
988 // We have an arbitrary precision integer value to write whose
989 // bit width is > 64. However, in canonical unsigned integer
990 // format it is likely that the high bits are going to be zero.
991 // So, we only write the number of active words.
992 unsigned NumWords = A.getActiveWords();
993 const uint64_t *RawData = A.getRawData();
994 for (unsigned i = 0; i < NumWords; i++)
995 emitSignedInt64(Vals, V: RawData[i]);
996}
997
998static void emitConstantRange(SmallVectorImpl<uint64_t> &Record,
999 const ConstantRange &CR, bool EmitBitWidth) {
1000 unsigned BitWidth = CR.getBitWidth();
1001 if (EmitBitWidth)
1002 Record.push_back(Elt: BitWidth);
1003 if (BitWidth > 64) {
1004 Record.push_back(Elt: CR.getLower().getActiveWords() |
1005 (uint64_t(CR.getUpper().getActiveWords()) << 32));
1006 emitWideAPInt(Vals&: Record, A: CR.getLower());
1007 emitWideAPInt(Vals&: Record, A: CR.getUpper());
1008 } else {
1009 emitSignedInt64(Vals&: Record, V: CR.getLower().getSExtValue());
1010 emitSignedInt64(Vals&: Record, V: CR.getUpper().getSExtValue());
1011 }
1012}
1013
1014void ModuleBitcodeWriter::writeAttributeGroupTable() {
1015 const std::vector<ValueEnumerator::IndexAndAttrSet> &AttrGrps =
1016 VE.getAttributeGroups();
1017 if (AttrGrps.empty()) return;
1018
1019 Stream.EnterSubblock(BlockID: bitc::PARAMATTR_GROUP_BLOCK_ID, CodeLen: 3);
1020
1021 SmallVector<uint64_t, 64> Record;
1022 for (ValueEnumerator::IndexAndAttrSet Pair : AttrGrps) {
1023 unsigned AttrListIndex = Pair.first;
1024 AttributeSet AS = Pair.second;
1025 Record.push_back(Elt: VE.getAttributeGroupID(Group: Pair));
1026 Record.push_back(Elt: AttrListIndex);
1027
1028 for (Attribute Attr : AS) {
1029 if (Attr.isEnumAttribute()) {
1030 Record.push_back(Elt: 0);
1031 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1032 } else if (Attr.isIntAttribute()) {
1033 Record.push_back(Elt: 1);
1034 Attribute::AttrKind Kind = Attr.getKindAsEnum();
1035 Record.push_back(Elt: getAttrKindEncoding(Kind));
1036 if (Kind == Attribute::Memory) {
1037 // Version field for upgrading old memory effects.
1038 const uint64_t Version = 1;
1039 Record.push_back(Elt: (Version << 56) | Attr.getValueAsInt());
1040 } else {
1041 Record.push_back(Elt: Attr.getValueAsInt());
1042 }
1043 } else if (Attr.isStringAttribute()) {
1044 StringRef Kind = Attr.getKindAsString();
1045 StringRef Val = Attr.getValueAsString();
1046
1047 Record.push_back(Elt: Val.empty() ? 3 : 4);
1048 Record.append(in_start: Kind.begin(), in_end: Kind.end());
1049 Record.push_back(Elt: 0);
1050 if (!Val.empty()) {
1051 Record.append(in_start: Val.begin(), in_end: Val.end());
1052 Record.push_back(Elt: 0);
1053 }
1054 } else if (Attr.isTypeAttribute()) {
1055 Type *Ty = Attr.getValueAsType();
1056 Record.push_back(Elt: Ty ? 6 : 5);
1057 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1058 if (Ty)
1059 Record.push_back(Elt: VE.getTypeID(T: Attr.getValueAsType()));
1060 } else if (Attr.isConstantRangeAttribute()) {
1061 Record.push_back(Elt: 7);
1062 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1063 emitConstantRange(Record, CR: Attr.getValueAsConstantRange(),
1064 /*EmitBitWidth=*/true);
1065 } else {
1066 assert(Attr.isConstantRangeListAttribute());
1067 Record.push_back(Elt: 8);
1068 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1069 ArrayRef<ConstantRange> Val = Attr.getValueAsConstantRangeList();
1070 Record.push_back(Elt: Val.size());
1071 Record.push_back(Elt: Val[0].getBitWidth());
1072 for (auto &CR : Val)
1073 emitConstantRange(Record, CR, /*EmitBitWidth=*/false);
1074 }
1075 }
1076
1077 Stream.EmitRecord(Code: bitc::PARAMATTR_GRP_CODE_ENTRY, Vals: Record);
1078 Record.clear();
1079 }
1080
1081 Stream.ExitBlock();
1082}
1083
1084void ModuleBitcodeWriter::writeAttributeTable() {
1085 const std::vector<AttributeList> &Attrs = VE.getAttributeLists();
1086 if (Attrs.empty()) return;
1087
1088 Stream.EnterSubblock(BlockID: bitc::PARAMATTR_BLOCK_ID, CodeLen: 3);
1089
1090 SmallVector<uint64_t, 64> Record;
1091 for (const AttributeList &AL : Attrs) {
1092 for (unsigned i : AL.indexes()) {
1093 AttributeSet AS = AL.getAttributes(Index: i);
1094 if (AS.hasAttributes())
1095 Record.push_back(Elt: VE.getAttributeGroupID(Group: {i, AS}));
1096 }
1097
1098 Stream.EmitRecord(Code: bitc::PARAMATTR_CODE_ENTRY, Vals: Record);
1099 Record.clear();
1100 }
1101
1102 Stream.ExitBlock();
1103}
1104
1105/// WriteTypeTable - Write out the type table for a module.
1106void ModuleBitcodeWriter::writeTypeTable() {
1107 const ValueEnumerator::TypeList &TypeList = VE.getTypes();
1108
1109 Stream.EnterSubblock(BlockID: bitc::TYPE_BLOCK_ID_NEW, CodeLen: 4 /*count from # abbrevs */);
1110 SmallVector<uint64_t, 64> TypeVals;
1111
1112 uint64_t NumBits = VE.computeBitsRequiredForTypeIndices();
1113
1114 // Abbrev for TYPE_CODE_OPAQUE_POINTER.
1115 auto Abbv = std::make_shared<BitCodeAbbrev>();
1116 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_OPAQUE_POINTER));
1117 Abbv->Add(OpInfo: BitCodeAbbrevOp(0)); // Addrspace = 0
1118 unsigned OpaquePtrAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1119
1120 // Abbrev for TYPE_CODE_FUNCTION.
1121 Abbv = std::make_shared<BitCodeAbbrev>();
1122 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_FUNCTION));
1123 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isvararg
1124 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1125 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1126 unsigned FunctionAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1127
1128 // Abbrev for TYPE_CODE_STRUCT_ANON.
1129 Abbv = std::make_shared<BitCodeAbbrev>();
1130 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_ANON));
1131 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked
1132 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1133 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1134 unsigned StructAnonAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1135
1136 // Abbrev for TYPE_CODE_STRUCT_NAME.
1137 Abbv = std::make_shared<BitCodeAbbrev>();
1138 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAME));
1139 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1140 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
1141 unsigned StructNameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1142
1143 // Abbrev for TYPE_CODE_STRUCT_NAMED.
1144 Abbv = std::make_shared<BitCodeAbbrev>();
1145 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAMED));
1146 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked
1147 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1148 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1149 unsigned StructNamedAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1150
1151 // Abbrev for TYPE_CODE_ARRAY.
1152 Abbv = std::make_shared<BitCodeAbbrev>();
1153 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_ARRAY));
1154 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // size
1155 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1156 unsigned ArrayAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1157
1158 // Emit an entry count so the reader can reserve space.
1159 TypeVals.push_back(Elt: TypeList.size());
1160 Stream.EmitRecord(Code: bitc::TYPE_CODE_NUMENTRY, Vals: TypeVals);
1161 TypeVals.clear();
1162
1163 // Loop over all of the types, emitting each in turn.
1164 for (Type *T : TypeList) {
1165 int AbbrevToUse = 0;
1166 unsigned Code = 0;
1167
1168 switch (T->getTypeID()) {
1169 case Type::VoidTyID: Code = bitc::TYPE_CODE_VOID; break;
1170 case Type::HalfTyID: Code = bitc::TYPE_CODE_HALF; break;
1171 case Type::BFloatTyID: Code = bitc::TYPE_CODE_BFLOAT; break;
1172 case Type::FloatTyID: Code = bitc::TYPE_CODE_FLOAT; break;
1173 case Type::DoubleTyID: Code = bitc::TYPE_CODE_DOUBLE; break;
1174 case Type::X86_FP80TyID: Code = bitc::TYPE_CODE_X86_FP80; break;
1175 case Type::FP128TyID: Code = bitc::TYPE_CODE_FP128; break;
1176 case Type::PPC_FP128TyID: Code = bitc::TYPE_CODE_PPC_FP128; break;
1177 case Type::LabelTyID: Code = bitc::TYPE_CODE_LABEL; break;
1178 case Type::MetadataTyID:
1179 Code = bitc::TYPE_CODE_METADATA;
1180 break;
1181 case Type::X86_AMXTyID: Code = bitc::TYPE_CODE_X86_AMX; break;
1182 case Type::TokenTyID: Code = bitc::TYPE_CODE_TOKEN; break;
1183 case Type::ByteTyID:
1184 // BYTE: [width]
1185 Code = bitc::TYPE_CODE_BYTE;
1186 TypeVals.push_back(Elt: T->getByteBitWidth());
1187 break;
1188 case Type::IntegerTyID:
1189 // INTEGER: [width]
1190 Code = bitc::TYPE_CODE_INTEGER;
1191 TypeVals.push_back(Elt: cast<IntegerType>(Val: T)->getBitWidth());
1192 break;
1193 case Type::PointerTyID: {
1194 PointerType *PTy = cast<PointerType>(Val: T);
1195 unsigned AddressSpace = PTy->getAddressSpace();
1196 // OPAQUE_POINTER: [address space]
1197 Code = bitc::TYPE_CODE_OPAQUE_POINTER;
1198 TypeVals.push_back(Elt: AddressSpace);
1199 if (AddressSpace == 0)
1200 AbbrevToUse = OpaquePtrAbbrev;
1201 break;
1202 }
1203 case Type::FunctionTyID: {
1204 FunctionType *FT = cast<FunctionType>(Val: T);
1205 // FUNCTION: [isvararg, retty, paramty x N]
1206 Code = bitc::TYPE_CODE_FUNCTION;
1207 TypeVals.push_back(Elt: FT->isVarArg());
1208 TypeVals.push_back(Elt: VE.getTypeID(T: FT->getReturnType()));
1209 for (unsigned i = 0, e = FT->getNumParams(); i != e; ++i)
1210 TypeVals.push_back(Elt: VE.getTypeID(T: FT->getParamType(i)));
1211 AbbrevToUse = FunctionAbbrev;
1212 break;
1213 }
1214 case Type::StructTyID: {
1215 StructType *ST = cast<StructType>(Val: T);
1216 // STRUCT: [ispacked, eltty x N]
1217 TypeVals.push_back(Elt: ST->isPacked());
1218 // Output all of the element types.
1219 for (Type *ET : ST->elements())
1220 TypeVals.push_back(Elt: VE.getTypeID(T: ET));
1221
1222 if (ST->isLiteral()) {
1223 Code = bitc::TYPE_CODE_STRUCT_ANON;
1224 AbbrevToUse = StructAnonAbbrev;
1225 } else {
1226 if (ST->isOpaque()) {
1227 Code = bitc::TYPE_CODE_OPAQUE;
1228 } else {
1229 Code = bitc::TYPE_CODE_STRUCT_NAMED;
1230 AbbrevToUse = StructNamedAbbrev;
1231 }
1232
1233 // Emit the name if it is present.
1234 if (!ST->getName().empty())
1235 writeStringRecord(Stream, Code: bitc::TYPE_CODE_STRUCT_NAME, Str: ST->getName(),
1236 AbbrevToUse: StructNameAbbrev);
1237 }
1238 break;
1239 }
1240 case Type::ArrayTyID: {
1241 ArrayType *AT = cast<ArrayType>(Val: T);
1242 // ARRAY: [numelts, eltty]
1243 Code = bitc::TYPE_CODE_ARRAY;
1244 TypeVals.push_back(Elt: AT->getNumElements());
1245 TypeVals.push_back(Elt: VE.getTypeID(T: AT->getElementType()));
1246 AbbrevToUse = ArrayAbbrev;
1247 break;
1248 }
1249 case Type::FixedVectorTyID:
1250 case Type::ScalableVectorTyID: {
1251 VectorType *VT = cast<VectorType>(Val: T);
1252 // VECTOR [numelts, eltty] or
1253 // [numelts, eltty, scalable]
1254 Code = bitc::TYPE_CODE_VECTOR;
1255 TypeVals.push_back(Elt: VT->getElementCount().getKnownMinValue());
1256 TypeVals.push_back(Elt: VE.getTypeID(T: VT->getElementType()));
1257 if (isa<ScalableVectorType>(Val: VT))
1258 TypeVals.push_back(Elt: true);
1259 break;
1260 }
1261 case Type::TargetExtTyID: {
1262 TargetExtType *TET = cast<TargetExtType>(Val: T);
1263 Code = bitc::TYPE_CODE_TARGET_TYPE;
1264 writeStringRecord(Stream, Code: bitc::TYPE_CODE_STRUCT_NAME, Str: TET->getName(),
1265 AbbrevToUse: StructNameAbbrev);
1266 TypeVals.push_back(Elt: TET->getNumTypeParameters());
1267 for (Type *InnerTy : TET->type_params())
1268 TypeVals.push_back(Elt: VE.getTypeID(T: InnerTy));
1269 llvm::append_range(C&: TypeVals, R: TET->int_params());
1270 break;
1271 }
1272 case Type::TypedPointerTyID:
1273 llvm_unreachable("Typed pointers cannot be added to IR modules");
1274 }
1275
1276 // Emit the finished record.
1277 Stream.EmitRecord(Code, Vals: TypeVals, Abbrev: AbbrevToUse);
1278 TypeVals.clear();
1279 }
1280
1281 Stream.ExitBlock();
1282}
1283
1284static unsigned getEncodedLinkage(const GlobalValue::LinkageTypes Linkage) {
1285 switch (Linkage) {
1286 case GlobalValue::ExternalLinkage:
1287 return 0;
1288 case GlobalValue::WeakAnyLinkage:
1289 return 16;
1290 case GlobalValue::AppendingLinkage:
1291 return 2;
1292 case GlobalValue::InternalLinkage:
1293 return 3;
1294 case GlobalValue::LinkOnceAnyLinkage:
1295 return 18;
1296 case GlobalValue::ExternalWeakLinkage:
1297 return 7;
1298 case GlobalValue::CommonLinkage:
1299 return 8;
1300 case GlobalValue::PrivateLinkage:
1301 return 9;
1302 case GlobalValue::WeakODRLinkage:
1303 return 17;
1304 case GlobalValue::LinkOnceODRLinkage:
1305 return 19;
1306 case GlobalValue::AvailableExternallyLinkage:
1307 return 12;
1308 }
1309 llvm_unreachable("Invalid linkage");
1310}
1311
1312static unsigned getEncodedLinkage(const GlobalValue &GV) {
1313 return getEncodedLinkage(Linkage: GV.getLinkage());
1314}
1315
1316static uint64_t getEncodedFFlags(FunctionSummary::FFlags Flags) {
1317 uint64_t RawFlags = 0;
1318 RawFlags |= Flags.ReadNone;
1319 RawFlags |= (Flags.ReadOnly << 1);
1320 RawFlags |= (Flags.NoRecurse << 2);
1321 RawFlags |= (Flags.ReturnDoesNotAlias << 3);
1322 RawFlags |= (Flags.NoInline << 4);
1323 RawFlags |= (Flags.AlwaysInline << 5);
1324 RawFlags |= (Flags.NoUnwind << 6);
1325 RawFlags |= (Flags.MayThrow << 7);
1326 RawFlags |= (Flags.HasUnknownCall << 8);
1327 RawFlags |= (Flags.MustBeUnreachable << 9);
1328 return RawFlags;
1329}
1330
1331// Decode the flags for GlobalValue in the summary. See getDecodedGVSummaryFlags
1332// in BitcodeReader.cpp.
1333static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags,
1334 bool ImportAsDecl = false) {
1335 uint64_t RawFlags = 0;
1336
1337 RawFlags |= Flags.NotEligibleToImport; // bool
1338 RawFlags |= (Flags.Live << 1);
1339 RawFlags |= (Flags.DSOLocal << 2);
1340 RawFlags |= (Flags.CanAutoHide << 3);
1341
1342 // Linkage don't need to be remapped at that time for the summary. Any future
1343 // change to the getEncodedLinkage() function will need to be taken into
1344 // account here as well.
1345 RawFlags = (RawFlags << 4) | Flags.Linkage; // 4 bits
1346
1347 RawFlags |= (Flags.Visibility << 8); // 2 bits
1348
1349 unsigned ImportType = Flags.ImportType | ImportAsDecl;
1350 RawFlags |= (ImportType << 10); // 1 bit
1351
1352 RawFlags |= (Flags.NoRenameOnPromotion << 11); // 1 bit
1353
1354 return RawFlags;
1355}
1356
1357static uint64_t getEncodedGVarFlags(GlobalVarSummary::GVarFlags Flags) {
1358 uint64_t RawFlags = Flags.MaybeReadOnly | (Flags.MaybeWriteOnly << 1) |
1359 (Flags.Constant << 2) | Flags.VCallVisibility << 3;
1360 return RawFlags;
1361}
1362
1363static uint64_t getEncodedHotnessCallEdgeInfo(const CalleeInfo &CI) {
1364 uint64_t RawFlags = 0;
1365
1366 RawFlags |= CI.Hotness; // 3 bits
1367 RawFlags |= (CI.HasTailCall << 3); // 1 bit
1368
1369 return RawFlags;
1370}
1371
1372static unsigned getEncodedVisibility(const GlobalValue &GV) {
1373 switch (GV.getVisibility()) {
1374 case GlobalValue::DefaultVisibility: return 0;
1375 case GlobalValue::HiddenVisibility: return 1;
1376 case GlobalValue::ProtectedVisibility: return 2;
1377 }
1378 llvm_unreachable("Invalid visibility");
1379}
1380
1381static unsigned getEncodedDLLStorageClass(const GlobalValue &GV) {
1382 switch (GV.getDLLStorageClass()) {
1383 case GlobalValue::DefaultStorageClass: return 0;
1384 case GlobalValue::DLLImportStorageClass: return 1;
1385 case GlobalValue::DLLExportStorageClass: return 2;
1386 }
1387 llvm_unreachable("Invalid DLL storage class");
1388}
1389
1390static unsigned getEncodedThreadLocalMode(const GlobalValue &GV) {
1391 switch (GV.getThreadLocalMode()) {
1392 case GlobalVariable::NotThreadLocal: return 0;
1393 case GlobalVariable::GeneralDynamicTLSModel: return 1;
1394 case GlobalVariable::LocalDynamicTLSModel: return 2;
1395 case GlobalVariable::InitialExecTLSModel: return 3;
1396 case GlobalVariable::LocalExecTLSModel: return 4;
1397 }
1398 llvm_unreachable("Invalid TLS model");
1399}
1400
1401static unsigned getEncodedComdatSelectionKind(const Comdat &C) {
1402 switch (C.getSelectionKind()) {
1403 case Comdat::Any:
1404 return bitc::COMDAT_SELECTION_KIND_ANY;
1405 case Comdat::ExactMatch:
1406 return bitc::COMDAT_SELECTION_KIND_EXACT_MATCH;
1407 case Comdat::Largest:
1408 return bitc::COMDAT_SELECTION_KIND_LARGEST;
1409 case Comdat::NoDeduplicate:
1410 return bitc::COMDAT_SELECTION_KIND_NO_DUPLICATES;
1411 case Comdat::SameSize:
1412 return bitc::COMDAT_SELECTION_KIND_SAME_SIZE;
1413 }
1414 llvm_unreachable("Invalid selection kind");
1415}
1416
1417static unsigned getEncodedUnnamedAddr(const GlobalValue &GV) {
1418 switch (GV.getUnnamedAddr()) {
1419 case GlobalValue::UnnamedAddr::None: return 0;
1420 case GlobalValue::UnnamedAddr::Local: return 2;
1421 case GlobalValue::UnnamedAddr::Global: return 1;
1422 }
1423 llvm_unreachable("Invalid unnamed_addr");
1424}
1425
1426size_t ModuleBitcodeWriter::addToStrtab(StringRef Str) {
1427 if (GenerateHash)
1428 Hasher.update(Str);
1429 return StrtabBuilder.add(S: Str);
1430}
1431
1432void ModuleBitcodeWriter::writeComdats() {
1433 SmallVector<unsigned, 64> Vals;
1434 for (const Comdat *C : VE.getComdats()) {
1435 // COMDAT: [strtab offset, strtab size, selection_kind]
1436 Vals.push_back(Elt: addToStrtab(Str: C->getName()));
1437 Vals.push_back(Elt: C->getName().size());
1438 Vals.push_back(Elt: getEncodedComdatSelectionKind(C: *C));
1439 Stream.EmitRecord(Code: bitc::MODULE_CODE_COMDAT, Vals, /*AbbrevToUse=*/Abbrev: 0);
1440 Vals.clear();
1441 }
1442}
1443
1444/// Write a record that will eventually hold the word offset of the
1445/// module-level VST. For now the offset is 0, which will be backpatched
1446/// after the real VST is written. Saves the bit offset to backpatch.
1447void ModuleBitcodeWriter::writeValueSymbolTableForwardDecl() {
1448 // Write a placeholder value in for the offset of the real VST,
1449 // which is written after the function blocks so that it can include
1450 // the offset of each function. The placeholder offset will be
1451 // updated when the real VST is written.
1452 auto Abbv = std::make_shared<BitCodeAbbrev>();
1453 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_VSTOFFSET));
1454 // Blocks are 32-bit aligned, so we can use a 32-bit word offset to
1455 // hold the real VST offset. Must use fixed instead of VBR as we don't
1456 // know how many VBR chunks to reserve ahead of time.
1457 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
1458 unsigned VSTOffsetAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1459
1460 // Emit the placeholder
1461 uint64_t Vals[] = {bitc::MODULE_CODE_VSTOFFSET, 0};
1462 Stream.EmitRecordWithAbbrev(Abbrev: VSTOffsetAbbrev, Vals);
1463
1464 // Compute and save the bit offset to the placeholder, which will be
1465 // patched when the real VST is written. We can simply subtract the 32-bit
1466 // fixed size from the current bit number to get the location to backpatch.
1467 VSTOffsetPlaceholder = Stream.GetCurrentBitNo() - 32;
1468}
1469
1470enum StringEncoding { SE_Char6, SE_Fixed7, SE_Fixed8 };
1471
1472/// Determine the encoding to use for the given string name and length.
1473static StringEncoding getStringEncoding(StringRef Str) {
1474 bool isChar6 = true;
1475 for (char C : Str) {
1476 if (isChar6)
1477 isChar6 = BitCodeAbbrevOp::isChar6(C);
1478 if ((unsigned char)C & 128)
1479 // don't bother scanning the rest.
1480 return SE_Fixed8;
1481 }
1482 if (isChar6)
1483 return SE_Char6;
1484 return SE_Fixed7;
1485}
1486
1487static_assert(sizeof(GlobalValue::SanitizerMetadata) <= sizeof(unsigned),
1488 "Sanitizer Metadata is too large for naive serialization.");
1489static unsigned
1490serializeSanitizerMetadata(const GlobalValue::SanitizerMetadata &Meta) {
1491 return Meta.NoAddress | (Meta.NoHWAddress << 1) |
1492 (Meta.Memtag << 2) | (Meta.IsDynInit << 3);
1493}
1494
1495/// Emit top-level description of module, including target triple, inline asm,
1496/// descriptors for global variables, and function prototype info.
1497/// Returns the bit offset to backpatch with the location of the real VST.
1498void ModuleBitcodeWriter::writeModuleInfo() {
1499 // Emit various pieces of data attached to a module.
1500 if (!M.getTargetTriple().empty())
1501 writeStringRecord(Stream, Code: bitc::MODULE_CODE_TRIPLE,
1502 Str: M.getTargetTriple().str(), AbbrevToUse: 0 /*TODO*/);
1503 const std::string &DL = M.getDataLayoutStr();
1504 if (!DL.empty())
1505 writeStringRecord(Stream, Code: bitc::MODULE_CODE_DATALAYOUT, Str: DL, AbbrevToUse: 0 /*TODO*/);
1506 if (!M.getModuleInlineAsm().empty())
1507 writeStringRecord(Stream, Code: bitc::MODULE_CODE_ASM, Str: M.getModuleInlineAsm(),
1508 AbbrevToUse: 0 /*TODO*/);
1509
1510 // Emit information about sections and GC, computing how many there are. Also
1511 // compute the maximum alignment value.
1512 std::map<std::string, unsigned> SectionMap;
1513 std::map<std::string, unsigned> GCMap;
1514 MaybeAlign MaxGVarAlignment;
1515 unsigned MaxGlobalType = 0;
1516 for (const GlobalVariable &GV : M.globals()) {
1517 if (MaybeAlign A = GV.getAlign())
1518 MaxGVarAlignment = !MaxGVarAlignment ? *A : std::max(a: *MaxGVarAlignment, b: *A);
1519 MaxGlobalType = std::max(a: MaxGlobalType, b: VE.getTypeID(T: GV.getValueType()));
1520 if (GV.hasSection()) {
1521 // Give section names unique ID's.
1522 unsigned &Entry = SectionMap[std::string(GV.getSection())];
1523 if (!Entry) {
1524 writeStringRecord(Stream, Code: bitc::MODULE_CODE_SECTIONNAME, Str: GV.getSection(),
1525 AbbrevToUse: 0 /*TODO*/);
1526 Entry = SectionMap.size();
1527 }
1528 }
1529 }
1530 for (const Function &F : M) {
1531 if (F.hasSection()) {
1532 // Give section names unique ID's.
1533 unsigned &Entry = SectionMap[std::string(F.getSection())];
1534 if (!Entry) {
1535 writeStringRecord(Stream, Code: bitc::MODULE_CODE_SECTIONNAME, Str: F.getSection(),
1536 AbbrevToUse: 0 /*TODO*/);
1537 Entry = SectionMap.size();
1538 }
1539 }
1540 if (F.hasGC()) {
1541 // Same for GC names.
1542 unsigned &Entry = GCMap[F.getGC()];
1543 if (!Entry) {
1544 writeStringRecord(Stream, Code: bitc::MODULE_CODE_GCNAME, Str: F.getGC(),
1545 AbbrevToUse: 0 /*TODO*/);
1546 Entry = GCMap.size();
1547 }
1548 }
1549 }
1550
1551 // Emit abbrev for globals, now that we know # sections and max alignment.
1552 unsigned SimpleGVarAbbrev = 0;
1553 if (!M.global_empty()) {
1554 // Add an abbrev for common globals with no visibility or thread localness.
1555 auto Abbv = std::make_shared<BitCodeAbbrev>();
1556 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_GLOBALVAR));
1557 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1558 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1559 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1560 Log2_32_Ceil(Value: MaxGlobalType+1)));
1561 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // AddrSpace << 2
1562 //| explicitType << 1
1563 //| constant
1564 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Initializer.
1565 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // Linkage.
1566 if (!MaxGVarAlignment) // Alignment.
1567 Abbv->Add(OpInfo: BitCodeAbbrevOp(0));
1568 else {
1569 unsigned MaxEncAlignment = getEncodedAlign(Alignment: MaxGVarAlignment);
1570 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1571 Log2_32_Ceil(Value: MaxEncAlignment+1)));
1572 }
1573 if (SectionMap.empty()) // Section.
1574 Abbv->Add(OpInfo: BitCodeAbbrevOp(0));
1575 else
1576 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1577 Log2_32_Ceil(Value: SectionMap.size()+1)));
1578 // Don't bother emitting vis + thread local.
1579 SimpleGVarAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1580 }
1581
1582 SmallVector<unsigned, 64> Vals;
1583 // Emit the module's source file name.
1584 {
1585 StringEncoding Bits = getStringEncoding(Str: M.getSourceFileName());
1586 BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
1587 if (Bits == SE_Char6)
1588 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
1589 else if (Bits == SE_Fixed7)
1590 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
1591
1592 // MODULE_CODE_SOURCE_FILENAME: [namechar x N]
1593 auto Abbv = std::make_shared<BitCodeAbbrev>();
1594 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
1595 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1596 Abbv->Add(OpInfo: AbbrevOpToUse);
1597 unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1598
1599 for (const auto P : M.getSourceFileName())
1600 Vals.push_back(Elt: (unsigned char)P);
1601
1602 // Emit the finished record.
1603 Stream.EmitRecord(Code: bitc::MODULE_CODE_SOURCE_FILENAME, Vals, Abbrev: FilenameAbbrev);
1604 Vals.clear();
1605 }
1606
1607 // Emit the global variable information.
1608 for (const GlobalVariable &GV : M.globals()) {
1609 unsigned AbbrevToUse = 0;
1610
1611 // GLOBALVAR: [strtab offset, strtab size, type, isconst, initid,
1612 // linkage, alignment, section, visibility, threadlocal,
1613 // unnamed_addr, externally_initialized, dllstorageclass,
1614 // comdat, attributes, DSO_Local, GlobalSanitizer, code_model]
1615 Vals.push_back(Elt: addToStrtab(Str: GV.getName()));
1616 Vals.push_back(Elt: GV.getName().size());
1617 Vals.push_back(Elt: VE.getTypeID(T: GV.getValueType()));
1618 Vals.push_back(Elt: GV.getType()->getAddressSpace() << 2 | 2 | GV.isConstant());
1619 Vals.push_back(Elt: GV.isDeclaration() ? 0 :
1620 (VE.getValueID(V: GV.getInitializer()) + 1));
1621 Vals.push_back(Elt: getEncodedLinkage(GV));
1622 Vals.push_back(Elt: getEncodedAlign(Alignment: GV.getAlign()));
1623 Vals.push_back(Elt: GV.hasSection() ? SectionMap[std::string(GV.getSection())]
1624 : 0);
1625 if (GV.isThreadLocal() ||
1626 GV.getVisibility() != GlobalValue::DefaultVisibility ||
1627 GV.getUnnamedAddr() != GlobalValue::UnnamedAddr::None ||
1628 GV.isExternallyInitialized() ||
1629 GV.getDLLStorageClass() != GlobalValue::DefaultStorageClass ||
1630 GV.hasComdat() || GV.hasAttributes() || GV.isDSOLocal() ||
1631 GV.hasPartition() || GV.hasSanitizerMetadata() || GV.getCodeModel()) {
1632 Vals.push_back(Elt: getEncodedVisibility(GV));
1633 Vals.push_back(Elt: getEncodedThreadLocalMode(GV));
1634 Vals.push_back(Elt: getEncodedUnnamedAddr(GV));
1635 Vals.push_back(Elt: GV.isExternallyInitialized());
1636 Vals.push_back(Elt: getEncodedDLLStorageClass(GV));
1637 Vals.push_back(Elt: GV.hasComdat() ? VE.getComdatID(C: GV.getComdat()) : 0);
1638
1639 auto AL = GV.getAttributesAsList(index: AttributeList::FunctionIndex);
1640 Vals.push_back(Elt: VE.getAttributeListID(PAL: AL));
1641
1642 Vals.push_back(Elt: GV.isDSOLocal());
1643 Vals.push_back(Elt: addToStrtab(Str: GV.getPartition()));
1644 Vals.push_back(Elt: GV.getPartition().size());
1645
1646 Vals.push_back(Elt: (GV.hasSanitizerMetadata() ? serializeSanitizerMetadata(
1647 Meta: GV.getSanitizerMetadata())
1648 : 0));
1649 Vals.push_back(Elt: GV.getCodeModelRaw());
1650 } else {
1651 AbbrevToUse = SimpleGVarAbbrev;
1652 }
1653
1654 Stream.EmitRecord(Code: bitc::MODULE_CODE_GLOBALVAR, Vals, Abbrev: AbbrevToUse);
1655 Vals.clear();
1656 }
1657
1658 // Emit the function proto information.
1659 for (const Function &F : M) {
1660 // FUNCTION: [strtab offset, strtab size, type, callingconv, isproto,
1661 // linkage, paramattrs, alignment, section, visibility, gc,
1662 // unnamed_addr, prologuedata, dllstorageclass, comdat,
1663 // prefixdata, personalityfn, DSO_Local, addrspace,
1664 // partition_strtab, partition_size, prefalign]
1665 Vals.push_back(Elt: addToStrtab(Str: F.getName()));
1666 Vals.push_back(Elt: F.getName().size());
1667 Vals.push_back(Elt: VE.getTypeID(T: F.getFunctionType()));
1668 Vals.push_back(Elt: F.getCallingConv());
1669 Vals.push_back(Elt: F.isDeclaration());
1670 Vals.push_back(Elt: getEncodedLinkage(GV: F));
1671 Vals.push_back(Elt: VE.getAttributeListID(PAL: F.getAttributes()));
1672 Vals.push_back(Elt: getEncodedAlign(Alignment: F.getAlign()));
1673 Vals.push_back(Elt: F.hasSection() ? SectionMap[std::string(F.getSection())]
1674 : 0);
1675 Vals.push_back(Elt: getEncodedVisibility(GV: F));
1676 Vals.push_back(Elt: F.hasGC() ? GCMap[F.getGC()] : 0);
1677 Vals.push_back(Elt: getEncodedUnnamedAddr(GV: F));
1678 Vals.push_back(Elt: F.hasPrologueData() ? (VE.getValueID(V: F.getPrologueData()) + 1)
1679 : 0);
1680 Vals.push_back(Elt: getEncodedDLLStorageClass(GV: F));
1681 Vals.push_back(Elt: F.hasComdat() ? VE.getComdatID(C: F.getComdat()) : 0);
1682 Vals.push_back(Elt: F.hasPrefixData() ? (VE.getValueID(V: F.getPrefixData()) + 1)
1683 : 0);
1684 Vals.push_back(
1685 Elt: F.hasPersonalityFn() ? (VE.getValueID(V: F.getPersonalityFn()) + 1) : 0);
1686
1687 Vals.push_back(Elt: F.isDSOLocal());
1688 Vals.push_back(Elt: F.getAddressSpace());
1689 Vals.push_back(Elt: addToStrtab(Str: F.getPartition()));
1690 Vals.push_back(Elt: F.getPartition().size());
1691 Vals.push_back(Elt: getEncodedAlign(Alignment: F.getPreferredAlignment()));
1692
1693 unsigned AbbrevToUse = 0;
1694 Stream.EmitRecord(Code: bitc::MODULE_CODE_FUNCTION, Vals, Abbrev: AbbrevToUse);
1695 Vals.clear();
1696 }
1697
1698 // Emit the alias information.
1699 for (const GlobalAlias &A : M.aliases()) {
1700 // ALIAS: [strtab offset, strtab size, alias type, aliasee val#, linkage,
1701 // visibility, dllstorageclass, threadlocal, unnamed_addr,
1702 // DSO_Local]
1703 Vals.push_back(Elt: addToStrtab(Str: A.getName()));
1704 Vals.push_back(Elt: A.getName().size());
1705 Vals.push_back(Elt: VE.getTypeID(T: A.getValueType()));
1706 Vals.push_back(Elt: A.getType()->getAddressSpace());
1707 Vals.push_back(Elt: VE.getValueID(V: A.getAliasee()));
1708 Vals.push_back(Elt: getEncodedLinkage(GV: A));
1709 Vals.push_back(Elt: getEncodedVisibility(GV: A));
1710 Vals.push_back(Elt: getEncodedDLLStorageClass(GV: A));
1711 Vals.push_back(Elt: getEncodedThreadLocalMode(GV: A));
1712 Vals.push_back(Elt: getEncodedUnnamedAddr(GV: A));
1713 Vals.push_back(Elt: A.isDSOLocal());
1714 Vals.push_back(Elt: addToStrtab(Str: A.getPartition()));
1715 Vals.push_back(Elt: A.getPartition().size());
1716
1717 unsigned AbbrevToUse = 0;
1718 Stream.EmitRecord(Code: bitc::MODULE_CODE_ALIAS, Vals, Abbrev: AbbrevToUse);
1719 Vals.clear();
1720 }
1721
1722 // Emit the ifunc information.
1723 for (const GlobalIFunc &I : M.ifuncs()) {
1724 // IFUNC: [strtab offset, strtab size, ifunc type, address space, resolver
1725 // val#, linkage, visibility, DSO_Local]
1726 Vals.push_back(Elt: addToStrtab(Str: I.getName()));
1727 Vals.push_back(Elt: I.getName().size());
1728 Vals.push_back(Elt: VE.getTypeID(T: I.getValueType()));
1729 Vals.push_back(Elt: I.getType()->getAddressSpace());
1730 Vals.push_back(Elt: VE.getValueID(V: I.getResolver()));
1731 Vals.push_back(Elt: getEncodedLinkage(GV: I));
1732 Vals.push_back(Elt: getEncodedVisibility(GV: I));
1733 Vals.push_back(Elt: I.isDSOLocal());
1734 Vals.push_back(Elt: addToStrtab(Str: I.getPartition()));
1735 Vals.push_back(Elt: I.getPartition().size());
1736 Stream.EmitRecord(Code: bitc::MODULE_CODE_IFUNC, Vals);
1737 Vals.clear();
1738 }
1739
1740 writeValueSymbolTableForwardDecl();
1741}
1742
1743static uint64_t getOptimizationFlags(const Value *V) {
1744 uint64_t Flags = 0;
1745
1746 if (const auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V)) {
1747 if (OBO->hasNoSignedWrap())
1748 Flags |= 1 << bitc::OBO_NO_SIGNED_WRAP;
1749 if (OBO->hasNoUnsignedWrap())
1750 Flags |= 1 << bitc::OBO_NO_UNSIGNED_WRAP;
1751 } else if (const auto *PEO = dyn_cast<PossiblyExactOperator>(Val: V)) {
1752 if (PEO->isExact())
1753 Flags |= 1 << bitc::PEO_EXACT;
1754 } else if (const auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: V)) {
1755 if (PDI->isDisjoint())
1756 Flags |= 1 << bitc::PDI_DISJOINT;
1757 } else if (const auto *FPMO = dyn_cast<FPMathOperator>(Val: V)) {
1758 if (FPMO->hasAllowReassoc())
1759 Flags |= bitc::AllowReassoc;
1760 if (FPMO->hasNoNaNs())
1761 Flags |= bitc::NoNaNs;
1762 if (FPMO->hasNoInfs())
1763 Flags |= bitc::NoInfs;
1764 if (FPMO->hasNoSignedZeros())
1765 Flags |= bitc::NoSignedZeros;
1766 if (FPMO->hasAllowReciprocal())
1767 Flags |= bitc::AllowReciprocal;
1768 if (FPMO->hasAllowContract())
1769 Flags |= bitc::AllowContract;
1770 if (FPMO->hasApproxFunc())
1771 Flags |= bitc::ApproxFunc;
1772 } else if (const auto *NNI = dyn_cast<PossiblyNonNegInst>(Val: V)) {
1773 if (NNI->hasNonNeg())
1774 Flags |= 1 << bitc::PNNI_NON_NEG;
1775 } else if (const auto *TI = dyn_cast<TruncInst>(Val: V)) {
1776 if (TI->hasNoSignedWrap())
1777 Flags |= 1 << bitc::TIO_NO_SIGNED_WRAP;
1778 if (TI->hasNoUnsignedWrap())
1779 Flags |= 1 << bitc::TIO_NO_UNSIGNED_WRAP;
1780 } else if (const auto *GEP = dyn_cast<GEPOperator>(Val: V)) {
1781 if (GEP->isInBounds())
1782 Flags |= 1 << bitc::GEP_INBOUNDS;
1783 if (GEP->hasNoUnsignedSignedWrap())
1784 Flags |= 1 << bitc::GEP_NUSW;
1785 if (GEP->hasNoUnsignedWrap())
1786 Flags |= 1 << bitc::GEP_NUW;
1787 } else if (const auto *ICmp = dyn_cast<ICmpInst>(Val: V)) {
1788 if (ICmp->hasSameSign())
1789 Flags |= 1 << bitc::ICMP_SAME_SIGN;
1790 }
1791
1792 return Flags;
1793}
1794
1795void ModuleBitcodeWriter::writeValueAsMetadata(
1796 const ValueAsMetadata *MD, SmallVectorImpl<uint64_t> &Record) {
1797 // Mimic an MDNode with a value as one operand.
1798 Value *V = MD->getValue();
1799 Record.push_back(Elt: VE.getTypeID(T: V->getType()));
1800 Record.push_back(Elt: VE.getValueID(V));
1801 Stream.EmitRecord(Code: bitc::METADATA_VALUE, Vals: Record, Abbrev: 0);
1802 Record.clear();
1803}
1804
1805void ModuleBitcodeWriter::writeMDTuple(const MDTuple *N,
1806 SmallVectorImpl<uint64_t> &Record,
1807 unsigned Abbrev) {
1808 for (const MDOperand &MDO : N->operands()) {
1809 Metadata *MD = MDO;
1810 assert(!(MD && isa<LocalAsMetadata>(MD)) &&
1811 "Unexpected function-local metadata");
1812 Record.push_back(Elt: VE.getMetadataOrNullID(MD));
1813 }
1814 Stream.EmitRecord(Code: N->isDistinct() ? bitc::METADATA_DISTINCT_NODE
1815 : bitc::METADATA_NODE,
1816 Vals: Record, Abbrev);
1817 Record.clear();
1818}
1819
1820unsigned ModuleBitcodeWriter::createDILocationAbbrev() {
1821 // Assume the column is usually under 128, and always output the inlined-at
1822 // location (it's never more expensive than building an array size 1).
1823 auto Abbv = std::make_shared<BitCodeAbbrev>();
1824 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_LOCATION));
1825 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isDistinct
1826 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // line
1827 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // column
1828 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // scope
1829 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // inlinedAt
1830 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isImplicitCode
1831 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // atomGroup
1832 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // atomRank
1833 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
1834}
1835
1836void ModuleBitcodeWriter::writeDILocation(const DILocation *N,
1837 SmallVectorImpl<uint64_t> &Record,
1838 unsigned &Abbrev) {
1839 if (!Abbrev)
1840 Abbrev = createDILocationAbbrev();
1841
1842 Record.push_back(Elt: N->isDistinct());
1843 Record.push_back(Elt: N->getLine());
1844 Record.push_back(Elt: N->getColumn());
1845 Record.push_back(Elt: VE.getMetadataID(MD: N->getScope()));
1846 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getInlinedAt()));
1847 Record.push_back(Elt: N->isImplicitCode());
1848 Record.push_back(Elt: N->getAtomGroup());
1849 Record.push_back(Elt: N->getAtomRank());
1850 Stream.EmitRecord(Code: bitc::METADATA_LOCATION, Vals: Record, Abbrev);
1851 Record.clear();
1852}
1853
1854unsigned ModuleBitcodeWriter::createGenericDINodeAbbrev() {
1855 // Assume the column is usually under 128, and always output the inlined-at
1856 // location (it's never more expensive than building an array size 1).
1857 auto Abbv = std::make_shared<BitCodeAbbrev>();
1858 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_GENERIC_DEBUG));
1859 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1860 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1861 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1862 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1863 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1864 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1865 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
1866}
1867
1868void ModuleBitcodeWriter::writeGenericDINode(const GenericDINode *N,
1869 SmallVectorImpl<uint64_t> &Record,
1870 unsigned &Abbrev) {
1871 if (!Abbrev)
1872 Abbrev = createGenericDINodeAbbrev();
1873
1874 Record.push_back(Elt: N->isDistinct());
1875 Record.push_back(Elt: N->getTag());
1876 Record.push_back(Elt: 0); // Per-tag version field; unused for now.
1877
1878 for (auto &I : N->operands())
1879 Record.push_back(Elt: VE.getMetadataOrNullID(MD: I));
1880
1881 Stream.EmitRecord(Code: bitc::METADATA_GENERIC_DEBUG, Vals: Record, Abbrev);
1882 Record.clear();
1883}
1884
1885void ModuleBitcodeWriter::writeDISubrange(const DISubrange *N,
1886 SmallVectorImpl<uint64_t> &Record,
1887 unsigned Abbrev) {
1888 const uint64_t Version = 2 << 1;
1889 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
1890 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawCountNode()));
1891 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
1892 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
1893 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
1894
1895 Stream.EmitRecord(Code: bitc::METADATA_SUBRANGE, Vals: Record, Abbrev);
1896 Record.clear();
1897}
1898
1899void ModuleBitcodeWriter::writeDIGenericSubrange(
1900 const DIGenericSubrange *N, SmallVectorImpl<uint64_t> &Record,
1901 unsigned Abbrev) {
1902 Record.push_back(Elt: (uint64_t)N->isDistinct());
1903 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawCountNode()));
1904 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
1905 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
1906 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
1907
1908 Stream.EmitRecord(Code: bitc::METADATA_GENERIC_SUBRANGE, Vals: Record, Abbrev);
1909 Record.clear();
1910}
1911
1912void ModuleBitcodeWriter::writeDIEnumerator(const DIEnumerator *N,
1913 SmallVectorImpl<uint64_t> &Record,
1914 unsigned Abbrev) {
1915 const uint64_t IsBigInt = 1 << 2;
1916 Record.push_back(Elt: IsBigInt | (N->isUnsigned() << 1) | N->isDistinct());
1917 Record.push_back(Elt: N->getValue().getBitWidth());
1918 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1919 emitWideAPInt(Vals&: Record, A: N->getValue());
1920
1921 Stream.EmitRecord(Code: bitc::METADATA_ENUMERATOR, Vals: Record, Abbrev);
1922 Record.clear();
1923}
1924
1925void ModuleBitcodeWriter::writeDIBasicType(const DIBasicType *N,
1926 SmallVectorImpl<uint64_t> &Record,
1927 unsigned Abbrev) {
1928 const unsigned SizeIsMetadata = 0x2;
1929 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
1930 Record.push_back(Elt: N->getTag());
1931 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1932 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
1933 Record.push_back(Elt: N->getAlignInBits());
1934 Record.push_back(Elt: N->getEncoding());
1935 Record.push_back(Elt: N->getFlags());
1936 Record.push_back(Elt: N->getNumExtraInhabitants());
1937 Record.push_back(Elt: N->getDataSizeInBits());
1938
1939 Stream.EmitRecord(Code: bitc::METADATA_BASIC_TYPE, Vals: Record, Abbrev);
1940 Record.clear();
1941}
1942
1943void ModuleBitcodeWriter::writeDIFixedPointType(
1944 const DIFixedPointType *N, SmallVectorImpl<uint64_t> &Record,
1945 unsigned Abbrev) {
1946 const unsigned SizeIsMetadata = 0x2;
1947 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
1948 Record.push_back(Elt: N->getTag());
1949 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1950 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
1951 Record.push_back(Elt: N->getAlignInBits());
1952 Record.push_back(Elt: N->getEncoding());
1953 Record.push_back(Elt: N->getFlags());
1954 Record.push_back(Elt: N->getKind());
1955 Record.push_back(Elt: N->getFactorRaw());
1956
1957 auto WriteWideInt = [&](const APInt &Value) {
1958 // Write an encoded word that holds the number of active words and
1959 // the number of bits.
1960 uint64_t NumWords = Value.getActiveWords();
1961 uint64_t Encoded = (NumWords << 32) | Value.getBitWidth();
1962 Record.push_back(Elt: Encoded);
1963 emitWideAPInt(Vals&: Record, A: Value);
1964 };
1965
1966 WriteWideInt(N->getNumeratorRaw());
1967 WriteWideInt(N->getDenominatorRaw());
1968
1969 Stream.EmitRecord(Code: bitc::METADATA_FIXED_POINT_TYPE, Vals: Record, Abbrev);
1970 Record.clear();
1971}
1972
1973void ModuleBitcodeWriter::writeDIStringType(const DIStringType *N,
1974 SmallVectorImpl<uint64_t> &Record,
1975 unsigned Abbrev) {
1976 const unsigned SizeIsMetadata = 0x2;
1977 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
1978 Record.push_back(Elt: N->getTag());
1979 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1980 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLength()));
1981 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLengthExp()));
1982 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLocationExp()));
1983 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
1984 Record.push_back(Elt: N->getAlignInBits());
1985 Record.push_back(Elt: N->getEncoding());
1986
1987 Stream.EmitRecord(Code: bitc::METADATA_STRING_TYPE, Vals: Record, Abbrev);
1988 Record.clear();
1989}
1990
1991void ModuleBitcodeWriter::writeDIDerivedType(const DIDerivedType *N,
1992 SmallVectorImpl<uint64_t> &Record,
1993 unsigned Abbrev) {
1994 const unsigned SizeIsMetadata = 0x2;
1995 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
1996 Record.push_back(Elt: N->getTag());
1997 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1998 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
1999 Record.push_back(Elt: N->getLine());
2000 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2001 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2002 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2003 Record.push_back(Elt: N->getAlignInBits());
2004 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawOffsetInBits()));
2005 Record.push_back(Elt: N->getFlags());
2006 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getExtraData()));
2007
2008 // DWARF address space is encoded as N->getDWARFAddressSpace() + 1. 0 means
2009 // that there is no DWARF address space associated with DIDerivedType.
2010 if (const auto &DWARFAddressSpace = N->getDWARFAddressSpace())
2011 Record.push_back(Elt: *DWARFAddressSpace + 1);
2012 else
2013 Record.push_back(Elt: 0);
2014
2015 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2016
2017 if (auto PtrAuthData = N->getPtrAuthData())
2018 Record.push_back(Elt: PtrAuthData->RawData);
2019 else
2020 Record.push_back(Elt: 0);
2021
2022 Stream.EmitRecord(Code: bitc::METADATA_DERIVED_TYPE, Vals: Record, Abbrev);
2023 Record.clear();
2024}
2025
2026void ModuleBitcodeWriter::writeDISubrangeType(const DISubrangeType *N,
2027 SmallVectorImpl<uint64_t> &Record,
2028 unsigned Abbrev) {
2029 const unsigned SizeIsMetadata = 0x2;
2030 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2031 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2032 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2033 Record.push_back(Elt: N->getLine());
2034 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2035 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2036 Record.push_back(Elt: N->getAlignInBits());
2037 Record.push_back(Elt: N->getFlags());
2038 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2039 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
2040 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
2041 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
2042 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawBias()));
2043
2044 Stream.EmitRecord(Code: bitc::METADATA_SUBRANGE_TYPE, Vals: Record, Abbrev);
2045 Record.clear();
2046}
2047
2048void ModuleBitcodeWriter::writeDICompositeType(
2049 const DICompositeType *N, SmallVectorImpl<uint64_t> &Record,
2050 unsigned Abbrev) {
2051 const unsigned IsNotUsedInOldTypeRef = 0x2;
2052 const unsigned SizeIsMetadata = 0x4;
2053 Record.push_back(Elt: SizeIsMetadata | IsNotUsedInOldTypeRef |
2054 (unsigned)N->isDistinct());
2055 Record.push_back(Elt: N->getTag());
2056 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2057 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2058 Record.push_back(Elt: N->getLine());
2059 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2060 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2061 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2062 Record.push_back(Elt: N->getAlignInBits());
2063 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawOffsetInBits()));
2064 Record.push_back(Elt: N->getFlags());
2065 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2066 Record.push_back(Elt: N->getRuntimeLang());
2067 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getVTableHolder()));
2068 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams().get()));
2069 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawIdentifier()));
2070 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDiscriminator()));
2071 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawDataLocation()));
2072 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawAssociated()));
2073 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawAllocated()));
2074 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawRank()));
2075 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2076 Record.push_back(Elt: N->getNumExtraInhabitants());
2077 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSpecification()));
2078 Record.push_back(
2079 Elt: N->getEnumKind().value_or(u: dwarf::DW_APPLE_ENUM_KIND_invalid));
2080 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawBitStride()));
2081
2082 Stream.EmitRecord(Code: bitc::METADATA_COMPOSITE_TYPE, Vals: Record, Abbrev);
2083 Record.clear();
2084}
2085
2086void ModuleBitcodeWriter::writeDISubroutineType(
2087 const DISubroutineType *N, SmallVectorImpl<uint64_t> &Record,
2088 unsigned Abbrev) {
2089 const unsigned HasNoOldTypeRefs = 0x2;
2090 Record.push_back(Elt: HasNoOldTypeRefs | (unsigned)N->isDistinct());
2091 Record.push_back(Elt: N->getFlags());
2092 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTypeArray().get()));
2093 Record.push_back(Elt: N->getCC());
2094
2095 Stream.EmitRecord(Code: bitc::METADATA_SUBROUTINE_TYPE, Vals: Record, Abbrev);
2096 Record.clear();
2097}
2098
2099void ModuleBitcodeWriter::writeDIFile(const DIFile *N,
2100 SmallVectorImpl<uint64_t> &Record,
2101 unsigned Abbrev) {
2102 Record.push_back(Elt: N->isDistinct());
2103 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFilename()));
2104 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawDirectory()));
2105 if (N->getRawChecksum()) {
2106 Record.push_back(Elt: N->getRawChecksum()->Kind);
2107 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawChecksum()->Value));
2108 } else {
2109 // Maintain backwards compatibility with the old internal representation of
2110 // CSK_None in ChecksumKind by writing nulls here when Checksum is None.
2111 Record.push_back(Elt: 0);
2112 Record.push_back(Elt: VE.getMetadataOrNullID(MD: nullptr));
2113 }
2114 auto Source = N->getRawSource();
2115 if (Source)
2116 Record.push_back(Elt: VE.getMetadataOrNullID(MD: Source));
2117
2118 Stream.EmitRecord(Code: bitc::METADATA_FILE, Vals: Record, Abbrev);
2119 Record.clear();
2120}
2121
2122void ModuleBitcodeWriter::writeDICompileUnit(const DICompileUnit *N,
2123 SmallVectorImpl<uint64_t> &Record,
2124 unsigned Abbrev) {
2125 assert(N->isDistinct() && "Expected distinct compile units");
2126 Record.push_back(/* IsDistinct */ Elt: true);
2127
2128 auto Lang = N->getSourceLanguage();
2129 Record.push_back(Elt: Lang.getName());
2130 // Set bit so the MetadataLoader can distniguish between versioned and
2131 // unversioned names.
2132 if (Lang.hasVersionedName())
2133 Record.back() ^= (uint64_t(1) << 63);
2134
2135 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2136 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawProducer()));
2137 Record.push_back(Elt: N->isOptimized());
2138 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFlags()));
2139 Record.push_back(Elt: N->getRuntimeVersion());
2140 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSplitDebugFilename()));
2141 Record.push_back(Elt: N->getEmissionKind());
2142 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getEnumTypes().get()));
2143 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRetainedTypes().get()));
2144 Record.push_back(/* subprograms */ Elt: 0);
2145 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getGlobalVariables().get()));
2146 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getImportedEntities().get()));
2147 Record.push_back(Elt: N->getDWOId());
2148 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getMacros().get()));
2149 Record.push_back(Elt: N->getSplitDebugInlining());
2150 Record.push_back(Elt: N->getDebugInfoForProfiling());
2151 Record.push_back(Elt: (unsigned)N->getNameTableKind());
2152 Record.push_back(Elt: N->getRangesBaseAddress());
2153 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSysRoot()));
2154 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSDK()));
2155 Record.push_back(Elt: Lang.hasVersionedName() ? Lang.getVersion() : 0);
2156
2157 Stream.EmitRecord(Code: bitc::METADATA_COMPILE_UNIT, Vals: Record, Abbrev);
2158 Record.clear();
2159}
2160
2161void ModuleBitcodeWriter::writeDISubprogram(const DISubprogram *N,
2162 SmallVectorImpl<uint64_t> &Record,
2163 unsigned Abbrev) {
2164 const uint64_t HasUnitFlag = 1 << 1;
2165 const uint64_t HasSPFlagsFlag = 1 << 2;
2166 Record.push_back(Elt: uint64_t(N->isDistinct()) | HasUnitFlag | HasSPFlagsFlag);
2167 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2168 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2169 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLinkageName()));
2170 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2171 Record.push_back(Elt: N->getLine());
2172 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2173 Record.push_back(Elt: N->getScopeLine());
2174 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getContainingType()));
2175 Record.push_back(Elt: N->getSPFlags());
2176 Record.push_back(Elt: N->getVirtualIndex());
2177 Record.push_back(Elt: N->getFlags());
2178 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUnit()));
2179 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams().get()));
2180 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDeclaration()));
2181 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRetainedNodes().get()));
2182 Record.push_back(Elt: N->getThisAdjustment());
2183 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getThrownTypes().get()));
2184 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2185 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawTargetFuncName()));
2186 Record.push_back(Elt: N->getKeyInstructionsEnabled());
2187
2188 Stream.EmitRecord(Code: bitc::METADATA_SUBPROGRAM, Vals: Record, Abbrev);
2189 Record.clear();
2190}
2191
2192void ModuleBitcodeWriter::writeDILexicalBlock(const DILexicalBlock *N,
2193 SmallVectorImpl<uint64_t> &Record,
2194 unsigned Abbrev) {
2195 Record.push_back(Elt: N->isDistinct());
2196 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2197 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2198 Record.push_back(Elt: N->getLine());
2199 Record.push_back(Elt: N->getColumn());
2200
2201 Stream.EmitRecord(Code: bitc::METADATA_LEXICAL_BLOCK, Vals: Record, Abbrev);
2202 Record.clear();
2203}
2204
2205void ModuleBitcodeWriter::writeDILexicalBlockFile(
2206 const DILexicalBlockFile *N, SmallVectorImpl<uint64_t> &Record,
2207 unsigned Abbrev) {
2208 Record.push_back(Elt: N->isDistinct());
2209 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2210 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2211 Record.push_back(Elt: N->getDiscriminator());
2212
2213 Stream.EmitRecord(Code: bitc::METADATA_LEXICAL_BLOCK_FILE, Vals: Record, Abbrev);
2214 Record.clear();
2215}
2216
2217void ModuleBitcodeWriter::writeDICommonBlock(const DICommonBlock *N,
2218 SmallVectorImpl<uint64_t> &Record,
2219 unsigned Abbrev) {
2220 Record.push_back(Elt: N->isDistinct());
2221 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2222 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDecl()));
2223 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2224 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2225 Record.push_back(Elt: N->getLineNo());
2226
2227 Stream.EmitRecord(Code: bitc::METADATA_COMMON_BLOCK, Vals: Record, Abbrev);
2228 Record.clear();
2229}
2230
2231void ModuleBitcodeWriter::writeDINamespace(const DINamespace *N,
2232 SmallVectorImpl<uint64_t> &Record,
2233 unsigned Abbrev) {
2234 Record.push_back(Elt: N->isDistinct() | N->getExportSymbols() << 1);
2235 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2236 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2237
2238 Stream.EmitRecord(Code: bitc::METADATA_NAMESPACE, Vals: Record, Abbrev);
2239 Record.clear();
2240}
2241
2242void ModuleBitcodeWriter::writeDIMacro(const DIMacro *N,
2243 SmallVectorImpl<uint64_t> &Record,
2244 unsigned Abbrev) {
2245 Record.push_back(Elt: N->isDistinct());
2246 Record.push_back(Elt: N->getMacinfoType());
2247 Record.push_back(Elt: N->getLine());
2248 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2249 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawValue()));
2250
2251 Stream.EmitRecord(Code: bitc::METADATA_MACRO, Vals: Record, Abbrev);
2252 Record.clear();
2253}
2254
2255void ModuleBitcodeWriter::writeDIMacroFile(const DIMacroFile *N,
2256 SmallVectorImpl<uint64_t> &Record,
2257 unsigned Abbrev) {
2258 Record.push_back(Elt: N->isDistinct());
2259 Record.push_back(Elt: N->getMacinfoType());
2260 Record.push_back(Elt: N->getLine());
2261 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2262 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2263
2264 Stream.EmitRecord(Code: bitc::METADATA_MACRO_FILE, Vals: Record, Abbrev);
2265 Record.clear();
2266}
2267
2268void ModuleBitcodeWriter::writeDIArgList(const DIArgList *N,
2269 SmallVectorImpl<uint64_t> &Record) {
2270 Record.reserve(N: N->getArgs().size());
2271 for (ValueAsMetadata *MD : N->getArgs())
2272 Record.push_back(Elt: VE.getMetadataID(MD));
2273
2274 Stream.EmitRecord(Code: bitc::METADATA_ARG_LIST, Vals: Record);
2275 Record.clear();
2276}
2277
2278void ModuleBitcodeWriter::writeDIModule(const DIModule *N,
2279 SmallVectorImpl<uint64_t> &Record,
2280 unsigned Abbrev) {
2281 Record.push_back(Elt: N->isDistinct());
2282 for (auto &I : N->operands())
2283 Record.push_back(Elt: VE.getMetadataOrNullID(MD: I));
2284 Record.push_back(Elt: N->getLineNo());
2285 Record.push_back(Elt: N->getIsDecl());
2286
2287 Stream.EmitRecord(Code: bitc::METADATA_MODULE, Vals: Record, Abbrev);
2288 Record.clear();
2289}
2290
2291void ModuleBitcodeWriter::writeDIAssignID(const DIAssignID *N,
2292 SmallVectorImpl<uint64_t> &Record,
2293 unsigned Abbrev) {
2294 // There are no arguments for this metadata type.
2295 Record.push_back(Elt: N->isDistinct());
2296 Stream.EmitRecord(Code: bitc::METADATA_ASSIGN_ID, Vals: Record, Abbrev);
2297 Record.clear();
2298}
2299
2300void ModuleBitcodeWriter::writeDITemplateTypeParameter(
2301 const DITemplateTypeParameter *N, SmallVectorImpl<uint64_t> &Record,
2302 unsigned Abbrev) {
2303 Record.push_back(Elt: N->isDistinct());
2304 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2305 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2306 Record.push_back(Elt: N->isDefault());
2307
2308 Stream.EmitRecord(Code: bitc::METADATA_TEMPLATE_TYPE, Vals: Record, Abbrev);
2309 Record.clear();
2310}
2311
2312void ModuleBitcodeWriter::writeDITemplateValueParameter(
2313 const DITemplateValueParameter *N, SmallVectorImpl<uint64_t> &Record,
2314 unsigned Abbrev) {
2315 Record.push_back(Elt: N->isDistinct());
2316 Record.push_back(Elt: N->getTag());
2317 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2318 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2319 Record.push_back(Elt: N->isDefault());
2320 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getValue()));
2321
2322 Stream.EmitRecord(Code: bitc::METADATA_TEMPLATE_VALUE, Vals: Record, Abbrev);
2323 Record.clear();
2324}
2325
2326void ModuleBitcodeWriter::writeDIGlobalVariable(
2327 const DIGlobalVariable *N, SmallVectorImpl<uint64_t> &Record,
2328 unsigned Abbrev) {
2329 const uint64_t Version = 2 << 1;
2330 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
2331 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2332 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2333 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLinkageName()));
2334 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2335 Record.push_back(Elt: N->getLine());
2336 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2337 Record.push_back(Elt: N->isLocalToUnit());
2338 Record.push_back(Elt: N->isDefinition());
2339 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStaticDataMemberDeclaration()));
2340 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams()));
2341 Record.push_back(Elt: N->getAlignInBits());
2342 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2343
2344 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_VAR, Vals: Record, Abbrev);
2345 Record.clear();
2346}
2347
2348void ModuleBitcodeWriter::writeDILocalVariable(
2349 const DILocalVariable *N, SmallVectorImpl<uint64_t> &Record,
2350 unsigned Abbrev) {
2351 // In order to support all possible bitcode formats in BitcodeReader we need
2352 // to distinguish the following cases:
2353 // 1) Record has no artificial tag (Record[1]),
2354 // has no obsolete inlinedAt field (Record[9]).
2355 // In this case Record size will be 8, HasAlignment flag is false.
2356 // 2) Record has artificial tag (Record[1]),
2357 // has no obsolete inlignedAt field (Record[9]).
2358 // In this case Record size will be 9, HasAlignment flag is false.
2359 // 3) Record has both artificial tag (Record[1]) and
2360 // obsolete inlignedAt field (Record[9]).
2361 // In this case Record size will be 10, HasAlignment flag is false.
2362 // 4) Record has neither artificial tag, nor inlignedAt field, but
2363 // HasAlignment flag is true and Record[8] contains alignment value.
2364 const uint64_t HasAlignmentFlag = 1 << 1;
2365 Record.push_back(Elt: (uint64_t)N->isDistinct() | HasAlignmentFlag);
2366 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2367 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2368 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2369 Record.push_back(Elt: N->getLine());
2370 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2371 Record.push_back(Elt: N->getArg());
2372 Record.push_back(Elt: N->getFlags());
2373 Record.push_back(Elt: N->getAlignInBits());
2374 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2375
2376 Stream.EmitRecord(Code: bitc::METADATA_LOCAL_VAR, Vals: Record, Abbrev);
2377 Record.clear();
2378}
2379
2380void ModuleBitcodeWriter::writeDILabel(
2381 const DILabel *N, SmallVectorImpl<uint64_t> &Record,
2382 unsigned Abbrev) {
2383 uint64_t IsArtificialFlag = uint64_t(N->isArtificial()) << 1;
2384 Record.push_back(Elt: (uint64_t)N->isDistinct() | IsArtificialFlag);
2385 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2386 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2387 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2388 Record.push_back(Elt: N->getLine());
2389 Record.push_back(Elt: N->getColumn());
2390 Record.push_back(Elt: N->getCoroSuspendIdx().has_value()
2391 ? (uint64_t)N->getCoroSuspendIdx().value()
2392 : std::numeric_limits<uint64_t>::max());
2393
2394 Stream.EmitRecord(Code: bitc::METADATA_LABEL, Vals: Record, Abbrev);
2395 Record.clear();
2396}
2397
2398void ModuleBitcodeWriter::writeDIExpression(const DIExpression *N,
2399 SmallVectorImpl<uint64_t> &Record,
2400 unsigned Abbrev) {
2401 Record.reserve(N: N->getElements().size() + 1);
2402 const uint64_t Version = 3 << 1;
2403 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
2404 Record.append(in_start: N->elements_begin(), in_end: N->elements_end());
2405
2406 Stream.EmitRecord(Code: bitc::METADATA_EXPRESSION, Vals: Record, Abbrev);
2407 Record.clear();
2408}
2409
2410void ModuleBitcodeWriter::writeDIGlobalVariableExpression(
2411 const DIGlobalVariableExpression *N, SmallVectorImpl<uint64_t> &Record,
2412 unsigned Abbrev) {
2413 Record.push_back(Elt: N->isDistinct());
2414 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getVariable()));
2415 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getExpression()));
2416
2417 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_VAR_EXPR, Vals: Record, Abbrev);
2418 Record.clear();
2419}
2420
2421void ModuleBitcodeWriter::writeDIObjCProperty(const DIObjCProperty *N,
2422 SmallVectorImpl<uint64_t> &Record,
2423 unsigned Abbrev) {
2424 Record.push_back(Elt: N->isDistinct());
2425 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2426 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2427 Record.push_back(Elt: N->getLine());
2428 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSetterName()));
2429 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawGetterName()));
2430 Record.push_back(Elt: N->getAttributes());
2431 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2432
2433 Stream.EmitRecord(Code: bitc::METADATA_OBJC_PROPERTY, Vals: Record, Abbrev);
2434 Record.clear();
2435}
2436
2437void ModuleBitcodeWriter::writeDIImportedEntity(
2438 const DIImportedEntity *N, SmallVectorImpl<uint64_t> &Record,
2439 unsigned Abbrev) {
2440 Record.push_back(Elt: N->isDistinct());
2441 Record.push_back(Elt: N->getTag());
2442 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2443 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getEntity()));
2444 Record.push_back(Elt: N->getLine());
2445 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2446 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFile()));
2447 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2448
2449 Stream.EmitRecord(Code: bitc::METADATA_IMPORTED_ENTITY, Vals: Record, Abbrev);
2450 Record.clear();
2451}
2452
2453unsigned ModuleBitcodeWriter::createNamedMetadataAbbrev() {
2454 auto Abbv = std::make_shared<BitCodeAbbrev>();
2455 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_NAME));
2456 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2457 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2458 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
2459}
2460
2461void ModuleBitcodeWriter::writeNamedMetadata(
2462 SmallVectorImpl<uint64_t> &Record) {
2463 if (M.named_metadata_empty())
2464 return;
2465
2466 unsigned Abbrev = createNamedMetadataAbbrev();
2467 for (const NamedMDNode &NMD : M.named_metadata()) {
2468 // Write name.
2469 StringRef Str = NMD.getName();
2470 Record.append(in_start: Str.bytes_begin(), in_end: Str.bytes_end());
2471 Stream.EmitRecord(Code: bitc::METADATA_NAME, Vals: Record, Abbrev);
2472 Record.clear();
2473
2474 // Write named metadata operands.
2475 for (const MDNode *N : NMD.operands())
2476 Record.push_back(Elt: VE.getMetadataID(MD: N));
2477 Stream.EmitRecord(Code: bitc::METADATA_NAMED_NODE, Vals: Record, Abbrev: 0);
2478 Record.clear();
2479 }
2480}
2481
2482unsigned ModuleBitcodeWriter::createMetadataStringsAbbrev() {
2483 auto Abbv = std::make_shared<BitCodeAbbrev>();
2484 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_STRINGS));
2485 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // # of strings
2486 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // offset to chars
2487 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
2488 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
2489}
2490
2491/// Write out a record for MDString.
2492///
2493/// All the metadata strings in a metadata block are emitted in a single
2494/// record. The sizes and strings themselves are shoved into a blob.
2495void ModuleBitcodeWriter::writeMetadataStrings(
2496 ArrayRef<const Metadata *> Strings, SmallVectorImpl<uint64_t> &Record) {
2497 if (Strings.empty())
2498 return;
2499
2500 // Start the record with the number of strings.
2501 Record.push_back(Elt: bitc::METADATA_STRINGS);
2502 Record.push_back(Elt: Strings.size());
2503
2504 // Emit the sizes of the strings in the blob.
2505 SmallString<256> Blob;
2506 {
2507 BitstreamWriter W(Blob);
2508 for (const Metadata *MD : Strings)
2509 W.EmitVBR(Val: cast<MDString>(Val: MD)->getLength(), NumBits: 6);
2510 W.FlushToWord();
2511 }
2512
2513 // Add the offset to the strings to the record.
2514 Record.push_back(Elt: Blob.size());
2515
2516 // Add the strings to the blob.
2517 for (const Metadata *MD : Strings)
2518 Blob.append(RHS: cast<MDString>(Val: MD)->getString());
2519
2520 // Emit the final record.
2521 Stream.EmitRecordWithBlob(Abbrev: createMetadataStringsAbbrev(), Vals: Record, Blob);
2522 Record.clear();
2523}
2524
2525// Generates an enum to use as an index in the Abbrev array of Metadata record.
2526enum MetadataAbbrev : unsigned {
2527#define HANDLE_MDNODE_LEAF(CLASS) CLASS##AbbrevID,
2528#include "llvm/IR/Metadata.def"
2529 LastPlusOne
2530};
2531
2532void ModuleBitcodeWriter::writeMetadataRecords(
2533 ArrayRef<const Metadata *> MDs, SmallVectorImpl<uint64_t> &Record,
2534 std::vector<unsigned> *MDAbbrevs, std::vector<uint64_t> *IndexPos) {
2535 if (MDs.empty())
2536 return;
2537
2538 // Initialize MDNode abbreviations.
2539#define HANDLE_MDNODE_LEAF(CLASS) unsigned CLASS##Abbrev = 0;
2540#include "llvm/IR/Metadata.def"
2541
2542 for (const Metadata *MD : MDs) {
2543 if (IndexPos)
2544 IndexPos->push_back(x: Stream.GetCurrentBitNo());
2545 if (const MDNode *N = dyn_cast<MDNode>(Val: MD)) {
2546 assert(N->isResolved() && "Expected forward references to be resolved");
2547
2548 switch (N->getMetadataID()) {
2549 default:
2550 llvm_unreachable("Invalid MDNode subclass");
2551#define HANDLE_MDNODE_LEAF(CLASS) \
2552 case Metadata::CLASS##Kind: \
2553 if (MDAbbrevs) \
2554 write##CLASS(cast<CLASS>(N), Record, \
2555 (*MDAbbrevs)[MetadataAbbrev::CLASS##AbbrevID]); \
2556 else \
2557 write##CLASS(cast<CLASS>(N), Record, CLASS##Abbrev); \
2558 continue;
2559#include "llvm/IR/Metadata.def"
2560 }
2561 }
2562 if (auto *AL = dyn_cast<DIArgList>(Val: MD)) {
2563 writeDIArgList(N: AL, Record);
2564 continue;
2565 }
2566 writeValueAsMetadata(MD: cast<ValueAsMetadata>(Val: MD), Record);
2567 }
2568}
2569
2570void ModuleBitcodeWriter::writeModuleMetadata() {
2571 if (!VE.hasMDs() && M.named_metadata_empty())
2572 return;
2573
2574 Stream.EnterSubblock(BlockID: bitc::METADATA_BLOCK_ID, CodeLen: 4);
2575 SmallVector<uint64_t, 64> Record;
2576
2577 // Emit all abbrevs upfront, so that the reader can jump in the middle of the
2578 // block and load any metadata.
2579 std::vector<unsigned> MDAbbrevs;
2580
2581 MDAbbrevs.resize(new_size: MetadataAbbrev::LastPlusOne);
2582 MDAbbrevs[MetadataAbbrev::DILocationAbbrevID] = createDILocationAbbrev();
2583 MDAbbrevs[MetadataAbbrev::GenericDINodeAbbrevID] =
2584 createGenericDINodeAbbrev();
2585
2586 auto Abbv = std::make_shared<BitCodeAbbrev>();
2587 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_INDEX_OFFSET));
2588 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2589 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2590 unsigned OffsetAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2591
2592 Abbv = std::make_shared<BitCodeAbbrev>();
2593 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_INDEX));
2594 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2595 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
2596 unsigned IndexAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2597
2598 // Emit MDStrings together upfront.
2599 writeMetadataStrings(Strings: VE.getMDStrings(), Record);
2600
2601 // We only emit an index for the metadata record if we have more than a given
2602 // (naive) threshold of metadatas, otherwise it is not worth it.
2603 if (VE.getNonMDStrings().size() > IndexThreshold) {
2604 // Write a placeholder value in for the offset of the metadata index,
2605 // which is written after the records, so that it can include
2606 // the offset of each entry. The placeholder offset will be
2607 // updated after all records are emitted.
2608 uint64_t Vals[] = {0, 0};
2609 Stream.EmitRecord(Code: bitc::METADATA_INDEX_OFFSET, Vals, Abbrev: OffsetAbbrev);
2610 }
2611
2612 // Compute and save the bit offset to the current position, which will be
2613 // patched when we emit the index later. We can simply subtract the 64-bit
2614 // fixed size from the current bit number to get the location to backpatch.
2615 uint64_t IndexOffsetRecordBitPos = Stream.GetCurrentBitNo();
2616
2617 // This index will contain the bitpos for each individual record.
2618 std::vector<uint64_t> IndexPos;
2619 IndexPos.reserve(n: VE.getNonMDStrings().size());
2620
2621 // Write all the records
2622 writeMetadataRecords(MDs: VE.getNonMDStrings(), Record, MDAbbrevs: &MDAbbrevs, IndexPos: &IndexPos);
2623
2624 if (VE.getNonMDStrings().size() > IndexThreshold) {
2625 // Now that we have emitted all the records we will emit the index. But
2626 // first
2627 // backpatch the forward reference so that the reader can skip the records
2628 // efficiently.
2629 Stream.BackpatchWord64(BitNo: IndexOffsetRecordBitPos - 64,
2630 Val: Stream.GetCurrentBitNo() - IndexOffsetRecordBitPos);
2631
2632 // Delta encode the index.
2633 uint64_t PreviousValue = IndexOffsetRecordBitPos;
2634 for (auto &Elt : IndexPos) {
2635 auto EltDelta = Elt - PreviousValue;
2636 PreviousValue = Elt;
2637 Elt = EltDelta;
2638 }
2639 // Emit the index record.
2640 Stream.EmitRecord(Code: bitc::METADATA_INDEX, Vals: IndexPos, Abbrev: IndexAbbrev);
2641 IndexPos.clear();
2642 }
2643
2644 // Write the named metadata now.
2645 writeNamedMetadata(Record);
2646
2647 auto AddDeclAttachedMetadata = [&](const GlobalObject &GO) {
2648 SmallVector<uint64_t, 4> Record;
2649 Record.push_back(Elt: VE.getValueID(V: &GO));
2650 pushGlobalMetadataAttachment(Record, GO);
2651 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_DECL_ATTACHMENT, Vals: Record);
2652 };
2653 for (const Function &F : M)
2654 if (F.isDeclaration() && F.hasMetadata())
2655 AddDeclAttachedMetadata(F);
2656 for (const GlobalIFunc &GI : M.ifuncs())
2657 if (GI.hasMetadata())
2658 AddDeclAttachedMetadata(GI);
2659 // FIXME: Only store metadata for declarations here, and move data for global
2660 // variable definitions to a separate block (PR28134).
2661 for (const GlobalVariable &GV : M.globals())
2662 if (GV.hasMetadata())
2663 AddDeclAttachedMetadata(GV);
2664
2665 Stream.ExitBlock();
2666}
2667
2668void ModuleBitcodeWriter::writeFunctionMetadata(const Function &F) {
2669 if (!VE.hasMDs())
2670 return;
2671
2672 Stream.EnterSubblock(BlockID: bitc::METADATA_BLOCK_ID, CodeLen: 3);
2673 SmallVector<uint64_t, 64> Record;
2674 writeMetadataStrings(Strings: VE.getMDStrings(), Record);
2675 writeMetadataRecords(MDs: VE.getNonMDStrings(), Record);
2676 Stream.ExitBlock();
2677}
2678
2679void ModuleBitcodeWriter::pushGlobalMetadataAttachment(
2680 SmallVectorImpl<uint64_t> &Record, const GlobalObject &GO) {
2681 // [n x [id, mdnode]]
2682 SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2683 GO.getAllMetadata(MDs);
2684 for (const auto &I : MDs) {
2685 Record.push_back(Elt: I.first);
2686 Record.push_back(Elt: VE.getMetadataID(MD: I.second));
2687 }
2688}
2689
2690void ModuleBitcodeWriter::writeFunctionMetadataAttachment(const Function &F) {
2691 Stream.EnterSubblock(BlockID: bitc::METADATA_ATTACHMENT_ID, CodeLen: 3);
2692
2693 SmallVector<uint64_t, 64> Record;
2694
2695 if (F.hasMetadata()) {
2696 pushGlobalMetadataAttachment(Record, GO: F);
2697 Stream.EmitRecord(Code: bitc::METADATA_ATTACHMENT, Vals: Record, Abbrev: 0);
2698 Record.clear();
2699 }
2700
2701 // Write metadata attachments
2702 // METADATA_ATTACHMENT - [m x [value, [n x [id, mdnode]]]
2703 SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2704 for (const BasicBlock &BB : F)
2705 for (const Instruction &I : BB) {
2706 MDs.clear();
2707 I.getAllMetadataOtherThanDebugLoc(MDs);
2708
2709 // If no metadata, ignore instruction.
2710 if (MDs.empty()) continue;
2711
2712 Record.push_back(Elt: VE.getInstructionID(I: &I));
2713
2714 for (const auto &[ID, MD] : MDs) {
2715 Record.push_back(Elt: ID);
2716 Record.push_back(Elt: VE.getMetadataID(MD));
2717 }
2718 Stream.EmitRecord(Code: bitc::METADATA_ATTACHMENT, Vals: Record, Abbrev: 0);
2719 Record.clear();
2720 }
2721
2722 Stream.ExitBlock();
2723}
2724
2725void ModuleBitcodeWriter::writeModuleMetadataKinds() {
2726 SmallVector<uint64_t, 64> Record;
2727
2728 // Write metadata kinds
2729 // METADATA_KIND - [n x [id, name]]
2730 SmallVector<StringRef, 8> Names;
2731 M.getMDKindNames(Result&: Names);
2732
2733 if (Names.empty()) return;
2734
2735 Stream.EnterSubblock(BlockID: bitc::METADATA_KIND_BLOCK_ID, CodeLen: 3);
2736
2737 for (unsigned MDKindID = 0, e = Names.size(); MDKindID != e; ++MDKindID) {
2738 Record.push_back(Elt: MDKindID);
2739 StringRef KName = Names[MDKindID];
2740 Record.append(in_start: KName.begin(), in_end: KName.end());
2741
2742 Stream.EmitRecord(Code: bitc::METADATA_KIND, Vals: Record, Abbrev: 0);
2743 Record.clear();
2744 }
2745
2746 Stream.ExitBlock();
2747}
2748
2749void ModuleBitcodeWriter::writeOperandBundleTags() {
2750 // Write metadata kinds
2751 //
2752 // OPERAND_BUNDLE_TAGS_BLOCK_ID : N x OPERAND_BUNDLE_TAG
2753 //
2754 // OPERAND_BUNDLE_TAG - [strchr x N]
2755
2756 SmallVector<StringRef, 8> Tags;
2757 M.getOperandBundleTags(Result&: Tags);
2758
2759 if (Tags.empty())
2760 return;
2761
2762 Stream.EnterSubblock(BlockID: bitc::OPERAND_BUNDLE_TAGS_BLOCK_ID, CodeLen: 3);
2763
2764 SmallVector<uint64_t, 64> Record;
2765
2766 for (auto Tag : Tags) {
2767 Record.append(in_start: Tag.begin(), in_end: Tag.end());
2768
2769 Stream.EmitRecord(Code: bitc::OPERAND_BUNDLE_TAG, Vals: Record, Abbrev: 0);
2770 Record.clear();
2771 }
2772
2773 Stream.ExitBlock();
2774}
2775
2776void ModuleBitcodeWriter::writeSyncScopeNames() {
2777 SmallVector<StringRef, 8> SSNs;
2778 M.getContext().getSyncScopeNames(SSNs);
2779 if (SSNs.empty())
2780 return;
2781
2782 Stream.EnterSubblock(BlockID: bitc::SYNC_SCOPE_NAMES_BLOCK_ID, CodeLen: 2);
2783
2784 SmallVector<uint64_t, 64> Record;
2785 for (auto SSN : SSNs) {
2786 Record.append(in_start: SSN.begin(), in_end: SSN.end());
2787 Stream.EmitRecord(Code: bitc::SYNC_SCOPE_NAME, Vals: Record, Abbrev: 0);
2788 Record.clear();
2789 }
2790
2791 Stream.ExitBlock();
2792}
2793
2794void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal,
2795 bool isGlobal) {
2796 if (FirstVal == LastVal) return;
2797
2798 Stream.EnterSubblock(BlockID: bitc::CONSTANTS_BLOCK_ID, CodeLen: 4);
2799
2800 unsigned AggregateAbbrev = 0;
2801 unsigned String8Abbrev = 0;
2802 unsigned CString7Abbrev = 0;
2803 unsigned CString6Abbrev = 0;
2804 // If this is a constant pool for the module, emit module-specific abbrevs.
2805 if (isGlobal) {
2806 // Abbrev for CST_CODE_AGGREGATE.
2807 auto Abbv = std::make_shared<BitCodeAbbrev>();
2808 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_AGGREGATE));
2809 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2810 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(Value: LastVal+1)));
2811 AggregateAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2812
2813 // Abbrev for CST_CODE_STRING.
2814 Abbv = std::make_shared<BitCodeAbbrev>();
2815 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_STRING));
2816 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2817 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2818 String8Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2819 // Abbrev for CST_CODE_CSTRING.
2820 Abbv = std::make_shared<BitCodeAbbrev>();
2821 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2822 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2823 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
2824 CString7Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2825 // Abbrev for CST_CODE_CSTRING.
2826 Abbv = std::make_shared<BitCodeAbbrev>();
2827 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2828 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2829 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
2830 CString6Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2831 }
2832
2833 SmallVector<uint64_t, 64> Record;
2834
2835 const ValueEnumerator::ValueList &Vals = VE.getValues();
2836 Type *LastTy = nullptr;
2837 for (unsigned i = FirstVal; i != LastVal; ++i) {
2838 const Value *V = Vals[i].first;
2839 // If we need to switch types, do so now.
2840 if (V->getType() != LastTy) {
2841 LastTy = V->getType();
2842 Record.push_back(Elt: VE.getTypeID(T: LastTy));
2843 Stream.EmitRecord(Code: bitc::CST_CODE_SETTYPE, Vals: Record,
2844 Abbrev: CONSTANTS_SETTYPE_ABBREV);
2845 Record.clear();
2846 }
2847
2848 if (const InlineAsm *IA = dyn_cast<InlineAsm>(Val: V)) {
2849 Record.push_back(Elt: VE.getTypeID(T: IA->getFunctionType()));
2850 Record.push_back(
2851 Elt: unsigned(IA->hasSideEffects()) | unsigned(IA->isAlignStack()) << 1 |
2852 unsigned(IA->getDialect() & 1) << 2 | unsigned(IA->canThrow()) << 3);
2853
2854 // Add the asm string.
2855 StringRef AsmStr = IA->getAsmString();
2856 Record.push_back(Elt: AsmStr.size());
2857 Record.append(in_start: AsmStr.begin(), in_end: AsmStr.end());
2858
2859 // Add the constraint string.
2860 StringRef ConstraintStr = IA->getConstraintString();
2861 Record.push_back(Elt: ConstraintStr.size());
2862 Record.append(in_start: ConstraintStr.begin(), in_end: ConstraintStr.end());
2863 Stream.EmitRecord(Code: bitc::CST_CODE_INLINEASM, Vals: Record);
2864 Record.clear();
2865 continue;
2866 }
2867 const Constant *C = cast<Constant>(Val: V);
2868 unsigned Code = -1U;
2869 unsigned AbbrevToUse = 0;
2870 if (C->isNullValue()) {
2871 Code = bitc::CST_CODE_NULL;
2872 } else if (isa<PoisonValue>(Val: C)) {
2873 Code = bitc::CST_CODE_POISON;
2874 } else if (isa<UndefValue>(Val: C)) {
2875 Code = bitc::CST_CODE_UNDEF;
2876 } else if (const ConstantInt *IV = dyn_cast<ConstantInt>(Val: C)) {
2877 if (IV->getBitWidth() <= 64) {
2878 uint64_t V = IV->getSExtValue();
2879 emitSignedInt64(Vals&: Record, V);
2880 Code = bitc::CST_CODE_INTEGER;
2881 AbbrevToUse = CONSTANTS_INTEGER_ABBREV;
2882 } else { // Wide integers, > 64 bits in size.
2883 emitWideAPInt(Vals&: Record, A: IV->getValue());
2884 Code = bitc::CST_CODE_WIDE_INTEGER;
2885 }
2886 } else if (const ConstantByte *BV = dyn_cast<ConstantByte>(Val: C)) {
2887 if (BV->getBitWidth() <= 64) {
2888 uint64_t V = BV->getSExtValue();
2889 emitSignedInt64(Vals&: Record, V);
2890 Code = bitc::CST_CODE_BYTE;
2891 AbbrevToUse = CONSTANTS_BYTE_ABBREV;
2892 } else { // Wide bytes, > 64 bits in size.
2893 emitWideAPInt(Vals&: Record, A: BV->getValue());
2894 Code = bitc::CST_CODE_WIDE_BYTE;
2895 }
2896 } else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
2897 Code = bitc::CST_CODE_FLOAT;
2898 Type *Ty = CFP->getType()->getScalarType();
2899 if (Ty->isHalfTy() || Ty->isBFloatTy() || Ty->isFloatTy() ||
2900 Ty->isDoubleTy()) {
2901 Record.push_back(Elt: CFP->getValueAPF().bitcastToAPInt().getZExtValue());
2902 } else if (Ty->isX86_FP80Ty()) {
2903 // api needed to prevent premature destruction
2904 // bits are not in the same order as a normal i80 APInt, compensate.
2905 APInt api = CFP->getValueAPF().bitcastToAPInt();
2906 const uint64_t *p = api.getRawData();
2907 Record.push_back(Elt: (p[1] << 48) | (p[0] >> 16));
2908 Record.push_back(Elt: p[0] & 0xffffLL);
2909 } else if (Ty->isFP128Ty() || Ty->isPPC_FP128Ty()) {
2910 APInt api = CFP->getValueAPF().bitcastToAPInt();
2911 const uint64_t *p = api.getRawData();
2912 Record.push_back(Elt: p[0]);
2913 Record.push_back(Elt: p[1]);
2914 } else {
2915 assert(0 && "Unknown FP type!");
2916 }
2917 } else if (isa<ConstantDataSequential>(Val: C) &&
2918 cast<ConstantDataSequential>(Val: C)->isString()) {
2919 const ConstantDataSequential *Str = cast<ConstantDataSequential>(Val: C);
2920 // Emit constant strings specially.
2921 uint64_t NumElts = Str->getNumElements();
2922 // If this is a null-terminated string, use the denser CSTRING encoding.
2923 if (Str->isCString()) {
2924 Code = bitc::CST_CODE_CSTRING;
2925 --NumElts; // Don't encode the null, which isn't allowed by char6.
2926 } else {
2927 Code = bitc::CST_CODE_STRING;
2928 AbbrevToUse = String8Abbrev;
2929 }
2930 bool isCStr7 = Code == bitc::CST_CODE_CSTRING;
2931 bool isCStrChar6 = Code == bitc::CST_CODE_CSTRING;
2932 for (uint64_t i = 0; i != NumElts; ++i) {
2933 unsigned char V = Str->getElementAsInteger(i);
2934 Record.push_back(Elt: V);
2935 isCStr7 &= (V & 128) == 0;
2936 if (isCStrChar6)
2937 isCStrChar6 = BitCodeAbbrevOp::isChar6(C: V);
2938 }
2939
2940 if (isCStrChar6)
2941 AbbrevToUse = CString6Abbrev;
2942 else if (isCStr7)
2943 AbbrevToUse = CString7Abbrev;
2944 } else if (const ConstantDataSequential *CDS =
2945 dyn_cast<ConstantDataSequential>(Val: C)) {
2946 Code = bitc::CST_CODE_DATA;
2947 Type *EltTy = CDS->getElementType();
2948 if (isa<IntegerType>(Val: EltTy) || isa<ByteType>(Val: EltTy)) {
2949 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i)
2950 Record.push_back(Elt: CDS->getElementAsInteger(i));
2951 } else {
2952 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i)
2953 Record.push_back(
2954 Elt: CDS->getElementAsAPFloat(i).bitcastToAPInt().getLimitedValue());
2955 }
2956 } else if (isa<ConstantAggregate>(Val: C)) {
2957 Code = bitc::CST_CODE_AGGREGATE;
2958 for (const Value *Op : C->operands())
2959 Record.push_back(Elt: VE.getValueID(V: Op));
2960 AbbrevToUse = AggregateAbbrev;
2961 } else if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: C)) {
2962 switch (CE->getOpcode()) {
2963 default:
2964 if (Instruction::isCast(Opcode: CE->getOpcode())) {
2965 Code = bitc::CST_CODE_CE_CAST;
2966 Record.push_back(Elt: getEncodedCastOpcode(Opcode: CE->getOpcode()));
2967 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
2968 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2969 AbbrevToUse = CONSTANTS_CE_CAST_Abbrev;
2970 } else {
2971 assert(CE->getNumOperands() == 2 && "Unknown constant expr!");
2972 Code = bitc::CST_CODE_CE_BINOP;
2973 Record.push_back(Elt: getEncodedBinaryOpcode(Opcode: CE->getOpcode()));
2974 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2975 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
2976 uint64_t Flags = getOptimizationFlags(V: CE);
2977 if (Flags != 0)
2978 Record.push_back(Elt: Flags);
2979 }
2980 break;
2981 case Instruction::FNeg: {
2982 assert(CE->getNumOperands() == 1 && "Unknown constant expr!");
2983 Code = bitc::CST_CODE_CE_UNOP;
2984 Record.push_back(Elt: getEncodedUnaryOpcode(Opcode: CE->getOpcode()));
2985 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2986 uint64_t Flags = getOptimizationFlags(V: CE);
2987 if (Flags != 0)
2988 Record.push_back(Elt: Flags);
2989 break;
2990 }
2991 case Instruction::GetElementPtr: {
2992 Code = bitc::CST_CODE_CE_GEP;
2993 const auto *GO = cast<GEPOperator>(Val: C);
2994 Record.push_back(Elt: VE.getTypeID(T: GO->getSourceElementType()));
2995 Record.push_back(Elt: getOptimizationFlags(V: GO));
2996 if (std::optional<ConstantRange> Range = GO->getInRange()) {
2997 Code = bitc::CST_CODE_CE_GEP_WITH_INRANGE;
2998 emitConstantRange(Record, CR: *Range, /*EmitBitWidth=*/true);
2999 }
3000 for (const Value *Op : CE->operands()) {
3001 Record.push_back(Elt: VE.getTypeID(T: Op->getType()));
3002 Record.push_back(Elt: VE.getValueID(V: Op));
3003 }
3004 break;
3005 }
3006 case Instruction::ExtractElement:
3007 Code = bitc::CST_CODE_CE_EXTRACTELT;
3008 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
3009 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3010 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 1)->getType()));
3011 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3012 break;
3013 case Instruction::InsertElement:
3014 Code = bitc::CST_CODE_CE_INSERTELT;
3015 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3016 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3017 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 2)->getType()));
3018 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 2)));
3019 break;
3020 case Instruction::ShuffleVector:
3021 // If the return type and argument types are the same, this is a
3022 // standard shufflevector instruction. If the types are different,
3023 // then the shuffle is widening or truncating the input vectors, and
3024 // the argument type must also be encoded.
3025 if (C->getType() == C->getOperand(i: 0)->getType()) {
3026 Code = bitc::CST_CODE_CE_SHUFFLEVEC;
3027 } else {
3028 Code = bitc::CST_CODE_CE_SHUFVEC_EX;
3029 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
3030 }
3031 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3032 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3033 Record.push_back(Elt: VE.getValueID(V: CE->getShuffleMaskForBitcode()));
3034 break;
3035 }
3036 } else if (const BlockAddress *BA = dyn_cast<BlockAddress>(Val: C)) {
3037 Code = bitc::CST_CODE_BLOCKADDRESS;
3038 Record.push_back(Elt: VE.getTypeID(T: BA->getFunction()->getType()));
3039 Record.push_back(Elt: VE.getValueID(V: BA->getFunction()));
3040 Record.push_back(Elt: VE.getGlobalBasicBlockID(BB: BA->getBasicBlock()));
3041 } else if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(Val: C)) {
3042 Code = bitc::CST_CODE_DSO_LOCAL_EQUIVALENT;
3043 Record.push_back(Elt: VE.getTypeID(T: Equiv->getGlobalValue()->getType()));
3044 Record.push_back(Elt: VE.getValueID(V: Equiv->getGlobalValue()));
3045 } else if (const auto *NC = dyn_cast<NoCFIValue>(Val: C)) {
3046 Code = bitc::CST_CODE_NO_CFI_VALUE;
3047 Record.push_back(Elt: VE.getTypeID(T: NC->getGlobalValue()->getType()));
3048 Record.push_back(Elt: VE.getValueID(V: NC->getGlobalValue()));
3049 } else if (const auto *CPA = dyn_cast<ConstantPtrAuth>(Val: C)) {
3050 Code = bitc::CST_CODE_PTRAUTH2;
3051 Record.push_back(Elt: VE.getValueID(V: CPA->getPointer()));
3052 Record.push_back(Elt: VE.getValueID(V: CPA->getKey()));
3053 Record.push_back(Elt: VE.getValueID(V: CPA->getDiscriminator()));
3054 Record.push_back(Elt: VE.getValueID(V: CPA->getAddrDiscriminator()));
3055 Record.push_back(Elt: VE.getValueID(V: CPA->getDeactivationSymbol()));
3056 } else {
3057#ifndef NDEBUG
3058 C->dump();
3059#endif
3060 llvm_unreachable("Unknown constant!");
3061 }
3062 Stream.EmitRecord(Code, Vals: Record, Abbrev: AbbrevToUse);
3063 Record.clear();
3064 }
3065
3066 Stream.ExitBlock();
3067}
3068
3069void ModuleBitcodeWriter::writeModuleConstants() {
3070 const ValueEnumerator::ValueList &Vals = VE.getValues();
3071
3072 // Find the first constant to emit, which is the first non-globalvalue value.
3073 // We know globalvalues have been emitted by WriteModuleInfo.
3074 for (unsigned i = 0, e = Vals.size(); i != e; ++i) {
3075 if (!isa<GlobalValue>(Val: Vals[i].first)) {
3076 writeConstants(FirstVal: i, LastVal: Vals.size(), isGlobal: true);
3077 return;
3078 }
3079 }
3080}
3081
3082/// pushValueAndType - The file has to encode both the value and type id for
3083/// many values, because we need to know what type to create for forward
3084/// references. However, most operands are not forward references, so this type
3085/// field is not needed.
3086///
3087/// This function adds V's value ID to Vals. If the value ID is higher than the
3088/// instruction ID, then it is a forward reference, and it also includes the
3089/// type ID. The value ID that is written is encoded relative to the InstID.
3090bool ModuleBitcodeWriter::pushValueAndType(const Value *V, unsigned InstID,
3091 SmallVectorImpl<unsigned> &Vals) {
3092 unsigned ValID = VE.getValueID(V);
3093 // Make encoding relative to the InstID.
3094 Vals.push_back(Elt: InstID - ValID);
3095 if (ValID >= InstID) {
3096 Vals.push_back(Elt: VE.getTypeID(T: V->getType()));
3097 return true;
3098 }
3099 return false;
3100}
3101
3102bool ModuleBitcodeWriter::pushValueOrMetadata(const Value *V, unsigned InstID,
3103 SmallVectorImpl<unsigned> &Vals) {
3104 bool IsMetadata = V->getType()->isMetadataTy();
3105 if (IsMetadata) {
3106 Vals.push_back(Elt: bitc::OB_METADATA);
3107 Metadata *MD = cast<MetadataAsValue>(Val: V)->getMetadata();
3108 unsigned ValID = VE.getMetadataID(MD);
3109 Vals.push_back(Elt: InstID - ValID);
3110 return false;
3111 }
3112 return pushValueAndType(V, InstID, Vals);
3113}
3114
3115void ModuleBitcodeWriter::writeOperandBundles(const CallBase &CS,
3116 unsigned InstID) {
3117 SmallVector<unsigned, 64> Record;
3118 LLVMContext &C = CS.getContext();
3119
3120 for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i) {
3121 const auto &Bundle = CS.getOperandBundleAt(Index: i);
3122 Record.push_back(Elt: C.getOperandBundleTagID(Tag: Bundle.getTagName()));
3123
3124 for (auto &Input : Bundle.Inputs)
3125 pushValueOrMetadata(V: Input, InstID, Vals&: Record);
3126
3127 Stream.EmitRecord(Code: bitc::FUNC_CODE_OPERAND_BUNDLE, Vals: Record);
3128 Record.clear();
3129 }
3130}
3131
3132/// pushValue - Like pushValueAndType, but where the type of the value is
3133/// omitted (perhaps it was already encoded in an earlier operand).
3134void ModuleBitcodeWriter::pushValue(const Value *V, unsigned InstID,
3135 SmallVectorImpl<unsigned> &Vals) {
3136 unsigned ValID = VE.getValueID(V);
3137 Vals.push_back(Elt: InstID - ValID);
3138}
3139
3140void ModuleBitcodeWriter::pushValueSigned(const Value *V, unsigned InstID,
3141 SmallVectorImpl<uint64_t> &Vals) {
3142 unsigned ValID = VE.getValueID(V);
3143 int64_t diff = ((int32_t)InstID - (int32_t)ValID);
3144 emitSignedInt64(Vals, V: diff);
3145}
3146
3147/// WriteInstruction - Emit an instruction to the specified stream.
3148void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
3149 unsigned InstID,
3150 SmallVectorImpl<unsigned> &Vals) {
3151 unsigned Code = 0;
3152 unsigned AbbrevToUse = 0;
3153 VE.setInstructionID(&I);
3154 switch (I.getOpcode()) {
3155 default:
3156 if (Instruction::isCast(Opcode: I.getOpcode())) {
3157 Code = bitc::FUNC_CODE_INST_CAST;
3158 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3159 AbbrevToUse = FUNCTION_INST_CAST_ABBREV;
3160 Vals.push_back(Elt: VE.getTypeID(T: I.getType()));
3161 Vals.push_back(Elt: getEncodedCastOpcode(Opcode: I.getOpcode()));
3162 uint64_t Flags = getOptimizationFlags(V: &I);
3163 if (Flags != 0) {
3164 if (AbbrevToUse == FUNCTION_INST_CAST_ABBREV)
3165 AbbrevToUse = FUNCTION_INST_CAST_FLAGS_ABBREV;
3166 Vals.push_back(Elt: Flags);
3167 }
3168 } else {
3169 assert(isa<BinaryOperator>(I) && "Unknown instruction!");
3170 Code = bitc::FUNC_CODE_INST_BINOP;
3171 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3172 AbbrevToUse = FUNCTION_INST_BINOP_ABBREV;
3173 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3174 Vals.push_back(Elt: getEncodedBinaryOpcode(Opcode: I.getOpcode()));
3175 uint64_t Flags = getOptimizationFlags(V: &I);
3176 if (Flags != 0) {
3177 if (AbbrevToUse == FUNCTION_INST_BINOP_ABBREV)
3178 AbbrevToUse = FUNCTION_INST_BINOP_FLAGS_ABBREV;
3179 Vals.push_back(Elt: Flags);
3180 }
3181 }
3182 break;
3183 case Instruction::FNeg: {
3184 Code = bitc::FUNC_CODE_INST_UNOP;
3185 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3186 AbbrevToUse = FUNCTION_INST_UNOP_ABBREV;
3187 Vals.push_back(Elt: getEncodedUnaryOpcode(Opcode: I.getOpcode()));
3188 uint64_t Flags = getOptimizationFlags(V: &I);
3189 if (Flags != 0) {
3190 if (AbbrevToUse == FUNCTION_INST_UNOP_ABBREV)
3191 AbbrevToUse = FUNCTION_INST_UNOP_FLAGS_ABBREV;
3192 Vals.push_back(Elt: Flags);
3193 }
3194 break;
3195 }
3196 case Instruction::GetElementPtr: {
3197 Code = bitc::FUNC_CODE_INST_GEP;
3198 AbbrevToUse = FUNCTION_INST_GEP_ABBREV;
3199 auto &GEPInst = cast<GetElementPtrInst>(Val: I);
3200 Vals.push_back(Elt: getOptimizationFlags(V: &I));
3201 Vals.push_back(Elt: VE.getTypeID(T: GEPInst.getSourceElementType()));
3202 for (const Value *Op : I.operands())
3203 pushValueAndType(V: Op, InstID, Vals);
3204 break;
3205 }
3206 case Instruction::ExtractValue: {
3207 Code = bitc::FUNC_CODE_INST_EXTRACTVAL;
3208 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3209 const ExtractValueInst *EVI = cast<ExtractValueInst>(Val: &I);
3210 Vals.append(in_start: EVI->idx_begin(), in_end: EVI->idx_end());
3211 break;
3212 }
3213 case Instruction::InsertValue: {
3214 Code = bitc::FUNC_CODE_INST_INSERTVAL;
3215 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3216 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3217 const InsertValueInst *IVI = cast<InsertValueInst>(Val: &I);
3218 Vals.append(in_start: IVI->idx_begin(), in_end: IVI->idx_end());
3219 break;
3220 }
3221 case Instruction::Select: {
3222 Code = bitc::FUNC_CODE_INST_VSELECT;
3223 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3224 pushValue(V: I.getOperand(i: 2), InstID, Vals);
3225 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3226 uint64_t Flags = getOptimizationFlags(V: &I);
3227 if (Flags != 0)
3228 Vals.push_back(Elt: Flags);
3229 break;
3230 }
3231 case Instruction::ExtractElement:
3232 Code = bitc::FUNC_CODE_INST_EXTRACTELT;
3233 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3234 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3235 break;
3236 case Instruction::InsertElement:
3237 Code = bitc::FUNC_CODE_INST_INSERTELT;
3238 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3239 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3240 pushValueAndType(V: I.getOperand(i: 2), InstID, Vals);
3241 break;
3242 case Instruction::ShuffleVector:
3243 Code = bitc::FUNC_CODE_INST_SHUFFLEVEC;
3244 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3245 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3246 pushValue(V: cast<ShuffleVectorInst>(Val: I).getShuffleMaskForBitcode(), InstID,
3247 Vals);
3248 break;
3249 case Instruction::ICmp:
3250 case Instruction::FCmp: {
3251 // compare returning Int1Ty or vector of Int1Ty
3252 Code = bitc::FUNC_CODE_INST_CMP2;
3253 AbbrevToUse = FUNCTION_INST_CMP_ABBREV;
3254 if (pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3255 AbbrevToUse = 0;
3256 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3257 Vals.push_back(Elt: cast<CmpInst>(Val: I).getPredicate());
3258 uint64_t Flags = getOptimizationFlags(V: &I);
3259 if (Flags != 0) {
3260 Vals.push_back(Elt: Flags);
3261 if (AbbrevToUse)
3262 AbbrevToUse = FUNCTION_INST_CMP_FLAGS_ABBREV;
3263 }
3264 break;
3265 }
3266
3267 case Instruction::Ret:
3268 {
3269 Code = bitc::FUNC_CODE_INST_RET;
3270 unsigned NumOperands = I.getNumOperands();
3271 if (NumOperands == 0)
3272 AbbrevToUse = FUNCTION_INST_RET_VOID_ABBREV;
3273 else if (NumOperands == 1) {
3274 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3275 AbbrevToUse = FUNCTION_INST_RET_VAL_ABBREV;
3276 } else {
3277 for (const Value *Op : I.operands())
3278 pushValueAndType(V: Op, InstID, Vals);
3279 }
3280 }
3281 break;
3282 case Instruction::UncondBr: {
3283 Code = bitc::FUNC_CODE_INST_BR;
3284 AbbrevToUse = FUNCTION_INST_BR_UNCOND_ABBREV;
3285 const UncondBrInst &II = cast<UncondBrInst>(Val: I);
3286 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 0)));
3287 } break;
3288 case Instruction::CondBr: {
3289 Code = bitc::FUNC_CODE_INST_BR;
3290 AbbrevToUse = FUNCTION_INST_BR_COND_ABBREV;
3291 const CondBrInst &II = cast<CondBrInst>(Val: I);
3292 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 0)));
3293 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 1)));
3294 pushValue(V: II.getCondition(), InstID, Vals);
3295 } break;
3296 case Instruction::Switch:
3297 {
3298 Code = bitc::FUNC_CODE_INST_SWITCH;
3299 const SwitchInst &SI = cast<SwitchInst>(Val: I);
3300 Vals.push_back(Elt: VE.getTypeID(T: SI.getCondition()->getType()));
3301 pushValue(V: SI.getCondition(), InstID, Vals);
3302 Vals.push_back(Elt: VE.getValueID(V: SI.getDefaultDest()));
3303 for (auto Case : SI.cases()) {
3304 Vals.push_back(Elt: VE.getValueID(V: Case.getCaseValue()));
3305 Vals.push_back(Elt: VE.getValueID(V: Case.getCaseSuccessor()));
3306 }
3307 }
3308 break;
3309 case Instruction::IndirectBr:
3310 Code = bitc::FUNC_CODE_INST_INDIRECTBR;
3311 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType()));
3312 // Encode the address operand as relative, but not the basic blocks.
3313 pushValue(V: I.getOperand(i: 0), InstID, Vals);
3314 for (const Value *Op : drop_begin(RangeOrContainer: I.operands()))
3315 Vals.push_back(Elt: VE.getValueID(V: Op));
3316 break;
3317
3318 case Instruction::Invoke: {
3319 const InvokeInst *II = cast<InvokeInst>(Val: &I);
3320 const Value *Callee = II->getCalledOperand();
3321 FunctionType *FTy = II->getFunctionType();
3322
3323 if (II->hasOperandBundles())
3324 writeOperandBundles(CS: *II, InstID);
3325
3326 Code = bitc::FUNC_CODE_INST_INVOKE;
3327
3328 Vals.push_back(Elt: VE.getAttributeListID(PAL: II->getAttributes()));
3329 Vals.push_back(Elt: II->getCallingConv() | 1 << 13);
3330 Vals.push_back(Elt: VE.getValueID(V: II->getNormalDest()));
3331 Vals.push_back(Elt: VE.getValueID(V: II->getUnwindDest()));
3332 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3333 pushValueAndType(V: Callee, InstID, Vals);
3334
3335 // Emit value #'s for the fixed parameters.
3336 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3337 pushValue(V: I.getOperand(i), InstID, Vals); // fixed param.
3338
3339 // Emit type/value pairs for varargs params.
3340 if (FTy->isVarArg()) {
3341 for (unsigned i = FTy->getNumParams(), e = II->arg_size(); i != e; ++i)
3342 pushValueAndType(V: I.getOperand(i), InstID, Vals); // vararg
3343 }
3344 break;
3345 }
3346 case Instruction::Resume:
3347 Code = bitc::FUNC_CODE_INST_RESUME;
3348 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3349 break;
3350 case Instruction::CleanupRet: {
3351 Code = bitc::FUNC_CODE_INST_CLEANUPRET;
3352 const auto &CRI = cast<CleanupReturnInst>(Val: I);
3353 pushValue(V: CRI.getCleanupPad(), InstID, Vals);
3354 if (CRI.hasUnwindDest())
3355 Vals.push_back(Elt: VE.getValueID(V: CRI.getUnwindDest()));
3356 break;
3357 }
3358 case Instruction::CatchRet: {
3359 Code = bitc::FUNC_CODE_INST_CATCHRET;
3360 const auto &CRI = cast<CatchReturnInst>(Val: I);
3361 pushValue(V: CRI.getCatchPad(), InstID, Vals);
3362 Vals.push_back(Elt: VE.getValueID(V: CRI.getSuccessor()));
3363 break;
3364 }
3365 case Instruction::CleanupPad:
3366 case Instruction::CatchPad: {
3367 const auto &FuncletPad = cast<FuncletPadInst>(Val: I);
3368 Code = isa<CatchPadInst>(Val: FuncletPad) ? bitc::FUNC_CODE_INST_CATCHPAD
3369 : bitc::FUNC_CODE_INST_CLEANUPPAD;
3370 pushValue(V: FuncletPad.getParentPad(), InstID, Vals);
3371
3372 unsigned NumArgOperands = FuncletPad.arg_size();
3373 Vals.push_back(Elt: NumArgOperands);
3374 for (unsigned Op = 0; Op != NumArgOperands; ++Op)
3375 pushValueAndType(V: FuncletPad.getArgOperand(i: Op), InstID, Vals);
3376 break;
3377 }
3378 case Instruction::CatchSwitch: {
3379 Code = bitc::FUNC_CODE_INST_CATCHSWITCH;
3380 const auto &CatchSwitch = cast<CatchSwitchInst>(Val: I);
3381
3382 pushValue(V: CatchSwitch.getParentPad(), InstID, Vals);
3383
3384 unsigned NumHandlers = CatchSwitch.getNumHandlers();
3385 Vals.push_back(Elt: NumHandlers);
3386 for (const BasicBlock *CatchPadBB : CatchSwitch.handlers())
3387 Vals.push_back(Elt: VE.getValueID(V: CatchPadBB));
3388
3389 if (CatchSwitch.hasUnwindDest())
3390 Vals.push_back(Elt: VE.getValueID(V: CatchSwitch.getUnwindDest()));
3391 break;
3392 }
3393 case Instruction::CallBr: {
3394 const CallBrInst *CBI = cast<CallBrInst>(Val: &I);
3395 const Value *Callee = CBI->getCalledOperand();
3396 FunctionType *FTy = CBI->getFunctionType();
3397
3398 if (CBI->hasOperandBundles())
3399 writeOperandBundles(CS: *CBI, InstID);
3400
3401 Code = bitc::FUNC_CODE_INST_CALLBR;
3402
3403 Vals.push_back(Elt: VE.getAttributeListID(PAL: CBI->getAttributes()));
3404
3405 Vals.push_back(Elt: CBI->getCallingConv() << bitc::CALL_CCONV |
3406 1 << bitc::CALL_EXPLICIT_TYPE);
3407
3408 Vals.push_back(Elt: VE.getValueID(V: CBI->getDefaultDest()));
3409 Vals.push_back(Elt: CBI->getNumIndirectDests());
3410 for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i)
3411 Vals.push_back(Elt: VE.getValueID(V: CBI->getIndirectDest(i)));
3412
3413 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3414 pushValueAndType(V: Callee, InstID, Vals);
3415
3416 // Emit value #'s for the fixed parameters.
3417 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3418 pushValue(V: I.getOperand(i), InstID, Vals); // fixed param.
3419
3420 // Emit type/value pairs for varargs params.
3421 if (FTy->isVarArg()) {
3422 for (unsigned i = FTy->getNumParams(), e = CBI->arg_size(); i != e; ++i)
3423 pushValueAndType(V: I.getOperand(i), InstID, Vals); // vararg
3424 }
3425 break;
3426 }
3427 case Instruction::Unreachable:
3428 Code = bitc::FUNC_CODE_INST_UNREACHABLE;
3429 AbbrevToUse = FUNCTION_INST_UNREACHABLE_ABBREV;
3430 break;
3431
3432 case Instruction::PHI: {
3433 const PHINode &PN = cast<PHINode>(Val: I);
3434 Code = bitc::FUNC_CODE_INST_PHI;
3435 // With the newer instruction encoding, forward references could give
3436 // negative valued IDs. This is most common for PHIs, so we use
3437 // signed VBRs.
3438 SmallVector<uint64_t, 128> Vals64;
3439 Vals64.push_back(Elt: VE.getTypeID(T: PN.getType()));
3440 for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) {
3441 pushValueSigned(V: PN.getIncomingValue(i), InstID, Vals&: Vals64);
3442 Vals64.push_back(Elt: VE.getValueID(V: PN.getIncomingBlock(i)));
3443 }
3444
3445 uint64_t Flags = getOptimizationFlags(V: &I);
3446 if (Flags != 0)
3447 Vals64.push_back(Elt: Flags);
3448
3449 // Emit a Vals64 vector and exit.
3450 Stream.EmitRecord(Code, Vals: Vals64, Abbrev: AbbrevToUse);
3451 Vals64.clear();
3452 return;
3453 }
3454
3455 case Instruction::LandingPad: {
3456 const LandingPadInst &LP = cast<LandingPadInst>(Val: I);
3457 Code = bitc::FUNC_CODE_INST_LANDINGPAD;
3458 Vals.push_back(Elt: VE.getTypeID(T: LP.getType()));
3459 Vals.push_back(Elt: LP.isCleanup());
3460 Vals.push_back(Elt: LP.getNumClauses());
3461 for (unsigned I = 0, E = LP.getNumClauses(); I != E; ++I) {
3462 if (LP.isCatch(Idx: I))
3463 Vals.push_back(Elt: LandingPadInst::Catch);
3464 else
3465 Vals.push_back(Elt: LandingPadInst::Filter);
3466 pushValueAndType(V: LP.getClause(Idx: I), InstID, Vals);
3467 }
3468 break;
3469 }
3470
3471 case Instruction::Alloca: {
3472 Code = bitc::FUNC_CODE_INST_ALLOCA;
3473 const AllocaInst &AI = cast<AllocaInst>(Val: I);
3474 Vals.push_back(Elt: VE.getTypeID(T: AI.getAllocatedType()));
3475 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType()));
3476 Vals.push_back(Elt: VE.getValueID(V: I.getOperand(i: 0))); // size.
3477 using APV = AllocaPackedValues;
3478 unsigned Record = 0;
3479 unsigned EncodedAlign = getEncodedAlign(Alignment: AI.getAlign());
3480 Bitfield::set<APV::AlignLower>(
3481 Packed&: Record, Value: EncodedAlign & ((1 << APV::AlignLower::Bits) - 1));
3482 Bitfield::set<APV::AlignUpper>(Packed&: Record,
3483 Value: EncodedAlign >> APV::AlignLower::Bits);
3484 Bitfield::set<APV::UsedWithInAlloca>(Packed&: Record, Value: AI.isUsedWithInAlloca());
3485 Bitfield::set<APV::ExplicitType>(Packed&: Record, Value: true);
3486 Bitfield::set<APV::SwiftError>(Packed&: Record, Value: AI.isSwiftError());
3487 Vals.push_back(Elt: Record);
3488
3489 unsigned AS = AI.getAddressSpace();
3490 if (AS != M.getDataLayout().getAllocaAddrSpace())
3491 Vals.push_back(Elt: AS);
3492 break;
3493 }
3494
3495 case Instruction::Load:
3496 if (cast<LoadInst>(Val: I).isAtomic()) {
3497 Code = bitc::FUNC_CODE_INST_LOADATOMIC;
3498 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3499 } else {
3500 Code = bitc::FUNC_CODE_INST_LOAD;
3501 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals)) // ptr
3502 AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
3503 }
3504 Vals.push_back(Elt: VE.getTypeID(T: I.getType()));
3505 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<LoadInst>(Val: I).getAlign()));
3506 Vals.push_back(Elt: cast<LoadInst>(Val: I).isVolatile());
3507 if (cast<LoadInst>(Val: I).isAtomic()) {
3508 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<LoadInst>(Val: I).getOrdering()));
3509 Vals.push_back(Elt: getEncodedSyncScopeID(SSID: cast<LoadInst>(Val: I).getSyncScopeID()));
3510 }
3511 break;
3512 case Instruction::Store:
3513 if (cast<StoreInst>(Val: I).isAtomic()) {
3514 Code = bitc::FUNC_CODE_INST_STOREATOMIC;
3515 } else {
3516 Code = bitc::FUNC_CODE_INST_STORE;
3517 AbbrevToUse = FUNCTION_INST_STORE_ABBREV;
3518 }
3519 if (pushValueAndType(V: I.getOperand(i: 1), InstID, Vals)) // ptrty + ptr
3520 AbbrevToUse = 0;
3521 if (pushValueAndType(V: I.getOperand(i: 0), InstID, Vals)) // valty + val
3522 AbbrevToUse = 0;
3523 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<StoreInst>(Val: I).getAlign()));
3524 Vals.push_back(Elt: cast<StoreInst>(Val: I).isVolatile());
3525 if (cast<StoreInst>(Val: I).isAtomic()) {
3526 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<StoreInst>(Val: I).getOrdering()));
3527 Vals.push_back(
3528 Elt: getEncodedSyncScopeID(SSID: cast<StoreInst>(Val: I).getSyncScopeID()));
3529 }
3530 break;
3531 case Instruction::AtomicCmpXchg:
3532 Code = bitc::FUNC_CODE_INST_CMPXCHG;
3533 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals); // ptrty + ptr
3534 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals); // cmp.
3535 pushValue(V: I.getOperand(i: 2), InstID, Vals); // newval.
3536 Vals.push_back(Elt: cast<AtomicCmpXchgInst>(Val: I).isVolatile());
3537 Vals.push_back(
3538 Elt: getEncodedOrdering(Ordering: cast<AtomicCmpXchgInst>(Val: I).getSuccessOrdering()));
3539 Vals.push_back(
3540 Elt: getEncodedSyncScopeID(SSID: cast<AtomicCmpXchgInst>(Val: I).getSyncScopeID()));
3541 Vals.push_back(
3542 Elt: getEncodedOrdering(Ordering: cast<AtomicCmpXchgInst>(Val: I).getFailureOrdering()));
3543 Vals.push_back(Elt: cast<AtomicCmpXchgInst>(Val: I).isWeak());
3544 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<AtomicCmpXchgInst>(Val: I).getAlign()));
3545 break;
3546 case Instruction::AtomicRMW:
3547 Code = bitc::FUNC_CODE_INST_ATOMICRMW;
3548 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals); // ptrty + ptr
3549 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals); // valty + val
3550 Vals.push_back(
3551 Elt: getEncodedRMWOperation(Op: cast<AtomicRMWInst>(Val: I).getOperation()));
3552 Vals.push_back(Elt: cast<AtomicRMWInst>(Val: I).isVolatile());
3553 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<AtomicRMWInst>(Val: I).getOrdering()));
3554 Vals.push_back(
3555 Elt: getEncodedSyncScopeID(SSID: cast<AtomicRMWInst>(Val: I).getSyncScopeID()));
3556 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<AtomicRMWInst>(Val: I).getAlign()));
3557 break;
3558 case Instruction::Fence:
3559 Code = bitc::FUNC_CODE_INST_FENCE;
3560 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<FenceInst>(Val: I).getOrdering()));
3561 Vals.push_back(Elt: getEncodedSyncScopeID(SSID: cast<FenceInst>(Val: I).getSyncScopeID()));
3562 break;
3563 case Instruction::Call: {
3564 const CallInst &CI = cast<CallInst>(Val: I);
3565 FunctionType *FTy = CI.getFunctionType();
3566
3567 if (CI.hasOperandBundles())
3568 writeOperandBundles(CS: CI, InstID);
3569
3570 Code = bitc::FUNC_CODE_INST_CALL;
3571
3572 Vals.push_back(Elt: VE.getAttributeListID(PAL: CI.getAttributes()));
3573
3574 unsigned Flags = getOptimizationFlags(V: &I);
3575 Vals.push_back(Elt: CI.getCallingConv() << bitc::CALL_CCONV |
3576 unsigned(CI.isTailCall()) << bitc::CALL_TAIL |
3577 unsigned(CI.isMustTailCall()) << bitc::CALL_MUSTTAIL |
3578 1 << bitc::CALL_EXPLICIT_TYPE |
3579 unsigned(CI.isNoTailCall()) << bitc::CALL_NOTAIL |
3580 unsigned(Flags != 0) << bitc::CALL_FMF);
3581 if (Flags != 0)
3582 Vals.push_back(Elt: Flags);
3583
3584 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3585 pushValueAndType(V: CI.getCalledOperand(), InstID, Vals); // Callee
3586
3587 // Emit value #'s for the fixed parameters.
3588 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3589 pushValue(V: CI.getArgOperand(i), InstID, Vals); // fixed param.
3590
3591 // Emit type/value pairs for varargs params.
3592 if (FTy->isVarArg()) {
3593 for (unsigned i = FTy->getNumParams(), e = CI.arg_size(); i != e; ++i)
3594 pushValueAndType(V: CI.getArgOperand(i), InstID, Vals); // varargs
3595 }
3596 break;
3597 }
3598 case Instruction::VAArg:
3599 Code = bitc::FUNC_CODE_INST_VAARG;
3600 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType())); // valistty
3601 pushValue(V: I.getOperand(i: 0), InstID, Vals); // valist.
3602 Vals.push_back(Elt: VE.getTypeID(T: I.getType())); // restype.
3603 break;
3604 case Instruction::Freeze:
3605 Code = bitc::FUNC_CODE_INST_FREEZE;
3606 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3607 break;
3608 }
3609
3610 Stream.EmitRecord(Code, Vals, Abbrev: AbbrevToUse);
3611 Vals.clear();
3612}
3613
3614/// Write a GlobalValue VST to the module. The purpose of this data structure is
3615/// to allow clients to efficiently find the function body.
3616void ModuleBitcodeWriter::writeGlobalValueSymbolTable(
3617 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3618 // Get the offset of the VST we are writing, and backpatch it into
3619 // the VST forward declaration record.
3620 uint64_t VSTOffset = Stream.GetCurrentBitNo();
3621 // The BitcodeStartBit was the stream offset of the identification block.
3622 VSTOffset -= bitcodeStartBit();
3623 assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned");
3624 // Note that we add 1 here because the offset is relative to one word
3625 // before the start of the identification block, which was historically
3626 // always the start of the regular bitcode header.
3627 Stream.BackpatchWord(BitNo: VSTOffsetPlaceholder, Val: VSTOffset / 32 + 1);
3628
3629 Stream.EnterSubblock(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, CodeLen: 4);
3630
3631 auto Abbv = std::make_shared<BitCodeAbbrev>();
3632 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY));
3633 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
3634 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset
3635 unsigned FnEntryAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
3636
3637 for (const Function &F : M) {
3638 uint64_t Record[2];
3639
3640 if (F.isDeclaration())
3641 continue;
3642
3643 Record[0] = VE.getValueID(V: &F);
3644
3645 // Save the word offset of the function (from the start of the
3646 // actual bitcode written to the stream).
3647 uint64_t BitcodeIndex = FunctionToBitcodeIndex[&F] - bitcodeStartBit();
3648 assert((BitcodeIndex & 31) == 0 && "function block not 32-bit aligned");
3649 // Note that we add 1 here because the offset is relative to one word
3650 // before the start of the identification block, which was historically
3651 // always the start of the regular bitcode header.
3652 Record[1] = BitcodeIndex / 32 + 1;
3653
3654 Stream.EmitRecord(Code: bitc::VST_CODE_FNENTRY, Vals: Record, Abbrev: FnEntryAbbrev);
3655 }
3656
3657 Stream.ExitBlock();
3658}
3659
3660/// Emit names for arguments, instructions and basic blocks in a function.
3661void ModuleBitcodeWriter::writeFunctionLevelValueSymbolTable(
3662 const ValueSymbolTable &VST) {
3663 if (VST.empty())
3664 return;
3665
3666 Stream.EnterSubblock(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, CodeLen: 4);
3667
3668 // FIXME: Set up the abbrev, we know how many values there are!
3669 // FIXME: We know if the type names can use 7-bit ascii.
3670 SmallVector<uint64_t, 64> NameVals;
3671
3672 for (const ValueName &Name : VST) {
3673 // Figure out the encoding to use for the name.
3674 StringEncoding Bits = getStringEncoding(Str: Name.getKey());
3675
3676 unsigned AbbrevToUse = VST_ENTRY_8_ABBREV;
3677 NameVals.push_back(Elt: VE.getValueID(V: Name.getValue()));
3678
3679 // VST_CODE_ENTRY: [valueid, namechar x N]
3680 // VST_CODE_BBENTRY: [bbid, namechar x N]
3681 unsigned Code;
3682 if (isa<BasicBlock>(Val: Name.getValue())) {
3683 Code = bitc::VST_CODE_BBENTRY;
3684 if (Bits == SE_Char6)
3685 AbbrevToUse = VST_BBENTRY_6_ABBREV;
3686 } else {
3687 Code = bitc::VST_CODE_ENTRY;
3688 if (Bits == SE_Char6)
3689 AbbrevToUse = VST_ENTRY_6_ABBREV;
3690 else if (Bits == SE_Fixed7)
3691 AbbrevToUse = VST_ENTRY_7_ABBREV;
3692 }
3693
3694 for (const auto P : Name.getKey())
3695 NameVals.push_back(Elt: (unsigned char)P);
3696
3697 // Emit the finished record.
3698 Stream.EmitRecord(Code, Vals: NameVals, Abbrev: AbbrevToUse);
3699 NameVals.clear();
3700 }
3701
3702 Stream.ExitBlock();
3703}
3704
3705void ModuleBitcodeWriter::writeUseList(UseListOrder &&Order) {
3706 assert(Order.Shuffle.size() >= 2 && "Shuffle too small");
3707 unsigned Code;
3708 if (isa<BasicBlock>(Val: Order.V))
3709 Code = bitc::USELIST_CODE_BB;
3710 else
3711 Code = bitc::USELIST_CODE_DEFAULT;
3712
3713 SmallVector<uint64_t, 64> Record(Order.Shuffle.begin(), Order.Shuffle.end());
3714 Record.push_back(Elt: VE.getValueID(V: Order.V));
3715 Stream.EmitRecord(Code, Vals: Record);
3716}
3717
3718void ModuleBitcodeWriter::writeUseListBlock(const Function *F) {
3719 assert(VE.shouldPreserveUseListOrder() &&
3720 "Expected to be preserving use-list order");
3721
3722 auto hasMore = [&]() {
3723 return !VE.UseListOrders.empty() && VE.UseListOrders.back().F == F;
3724 };
3725 if (!hasMore())
3726 // Nothing to do.
3727 return;
3728
3729 Stream.EnterSubblock(BlockID: bitc::USELIST_BLOCK_ID, CodeLen: 3);
3730 while (hasMore()) {
3731 writeUseList(Order: std::move(VE.UseListOrders.back()));
3732 VE.UseListOrders.pop_back();
3733 }
3734 Stream.ExitBlock();
3735}
3736
3737/// Emit a function body to the module stream.
3738void ModuleBitcodeWriter::writeFunction(
3739 const Function &F,
3740 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3741 // Save the bitcode index of the start of this function block for recording
3742 // in the VST.
3743 FunctionToBitcodeIndex[&F] = Stream.GetCurrentBitNo();
3744
3745 Stream.EnterSubblock(BlockID: bitc::FUNCTION_BLOCK_ID, CodeLen: 5);
3746 VE.incorporateFunction(F);
3747
3748 SmallVector<unsigned, 64> Vals;
3749
3750 // Emit the number of basic blocks, so the reader can create them ahead of
3751 // time.
3752 Vals.push_back(Elt: VE.getBasicBlocks().size());
3753 Stream.EmitRecord(Code: bitc::FUNC_CODE_DECLAREBLOCKS, Vals);
3754 Vals.clear();
3755
3756 // If there are function-local constants, emit them now.
3757 unsigned CstStart, CstEnd;
3758 VE.getFunctionConstantRange(Start&: CstStart, End&: CstEnd);
3759 writeConstants(FirstVal: CstStart, LastVal: CstEnd, isGlobal: false);
3760
3761 // If there is function-local metadata, emit it now.
3762 writeFunctionMetadata(F);
3763
3764 // Keep a running idea of what the instruction ID is.
3765 unsigned InstID = CstEnd;
3766
3767 bool NeedsMetadataAttachment = F.hasMetadata();
3768
3769 DILocation *LastDL = nullptr;
3770 SmallSetVector<Function *, 4> BlockAddressUsers;
3771
3772 // Finally, emit all the instructions, in order.
3773 for (const BasicBlock &BB : F) {
3774 for (const Instruction &I : BB) {
3775 writeInstruction(I, InstID, Vals);
3776
3777 if (!I.getType()->isVoidTy())
3778 ++InstID;
3779
3780 // If the instruction has metadata, write a metadata attachment later.
3781 NeedsMetadataAttachment |= I.hasMetadataOtherThanDebugLoc();
3782
3783 // If the instruction has a debug location, emit it.
3784 if (DILocation *DL = I.getDebugLoc()) {
3785 if (DL == LastDL) {
3786 // Just repeat the same debug loc as last time.
3787 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_LOC_AGAIN, Vals);
3788 } else {
3789 Vals.push_back(Elt: DL->getLine());
3790 Vals.push_back(Elt: DL->getColumn());
3791 Vals.push_back(Elt: VE.getMetadataOrNullID(MD: DL->getScope()));
3792 Vals.push_back(Elt: VE.getMetadataOrNullID(MD: DL->getInlinedAt()));
3793 Vals.push_back(Elt: DL->isImplicitCode());
3794 Vals.push_back(Elt: DL->getAtomGroup());
3795 Vals.push_back(Elt: DL->getAtomRank());
3796 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_LOC, Vals,
3797 Abbrev: FUNCTION_DEBUG_LOC_ABBREV);
3798 Vals.clear();
3799 LastDL = DL;
3800 }
3801 }
3802
3803 // If the instruction has DbgRecords attached to it, emit them. Note that
3804 // they come after the instruction so that it's easy to attach them again
3805 // when reading the bitcode, even though conceptually the debug locations
3806 // start "before" the instruction.
3807 if (I.hasDbgRecords()) {
3808 /// Try to push the value only (unwrapped), otherwise push the
3809 /// metadata wrapped value. Returns true if the value was pushed
3810 /// without the ValueAsMetadata wrapper.
3811 auto PushValueOrMetadata = [&Vals, InstID,
3812 this](Metadata *RawLocation) {
3813 assert(RawLocation &&
3814 "RawLocation unexpectedly null in DbgVariableRecord");
3815 if (ValueAsMetadata *VAM = dyn_cast<ValueAsMetadata>(Val: RawLocation)) {
3816 SmallVector<unsigned, 2> ValAndType;
3817 // If the value is a fwd-ref the type is also pushed. We don't
3818 // want the type, so fwd-refs are kept wrapped (pushValueAndType
3819 // returns false if the value is pushed without type).
3820 if (!pushValueAndType(V: VAM->getValue(), InstID, Vals&: ValAndType)) {
3821 Vals.push_back(Elt: ValAndType[0]);
3822 return true;
3823 }
3824 }
3825 // The metadata is a DIArgList, or ValueAsMetadata wrapping a
3826 // fwd-ref. Push the metadata ID.
3827 Vals.push_back(Elt: VE.getMetadataID(MD: RawLocation));
3828 return false;
3829 };
3830
3831 // Write out non-instruction debug information attached to this
3832 // instruction. Write it after the instruction so that it's easy to
3833 // re-attach to the instruction reading the records in.
3834 for (DbgRecord &DR : I.DebugMarker->getDbgRecordRange()) {
3835 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
3836 Vals.push_back(Elt: VE.getMetadataID(MD: &*DLR->getDebugLoc()));
3837 Vals.push_back(Elt: VE.getMetadataID(MD: DLR->getLabel()));
3838 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_LABEL, Vals);
3839 Vals.clear();
3840 continue;
3841 }
3842
3843 // First 3 fields are common to all kinds:
3844 // DILocation, DILocalVariable, DIExpression
3845 // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE)
3846 // ..., LocationMetadata
3847 // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE - abbrev'd)
3848 // ..., Value
3849 // dbg_declare (FUNC_CODE_DEBUG_RECORD_DECLARE)
3850 // ..., LocationMetadata
3851 // dbg_assign (FUNC_CODE_DEBUG_RECORD_ASSIGN)
3852 // ..., LocationMetadata, DIAssignID, DIExpression, LocationMetadata
3853 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
3854 Vals.push_back(Elt: VE.getMetadataID(MD: &*DVR.getDebugLoc()));
3855 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getVariable()));
3856 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getExpression()));
3857 if (DVR.isDbgValue()) {
3858 if (PushValueOrMetadata(DVR.getRawLocation()))
3859 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE, Vals,
3860 Abbrev: FUNCTION_DEBUG_RECORD_VALUE_ABBREV);
3861 else
3862 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_VALUE, Vals);
3863 } else if (DVR.isDbgDeclare()) {
3864 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3865 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_DECLARE, Vals);
3866 } else if (DVR.isDbgDeclareValue()) {
3867 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3868 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_DECLARE_VALUE, Vals);
3869 } else {
3870 assert(DVR.isDbgAssign() && "Unexpected DbgRecord kind");
3871 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3872 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getAssignID()));
3873 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getAddressExpression()));
3874 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawAddress()));
3875 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_ASSIGN, Vals);
3876 }
3877 Vals.clear();
3878 }
3879 }
3880 }
3881
3882 if (BlockAddress *BA = BlockAddress::lookup(BB: &BB)) {
3883 SmallVector<Value *> Worklist{BA};
3884 SmallPtrSet<Value *, 8> Visited{BA};
3885 while (!Worklist.empty()) {
3886 Value *V = Worklist.pop_back_val();
3887 for (User *U : V->users()) {
3888 if (auto *I = dyn_cast<Instruction>(Val: U)) {
3889 Function *P = I->getFunction();
3890 if (P != &F)
3891 BlockAddressUsers.insert(X: P);
3892 } else if (isa<Constant>(Val: U) && !isa<GlobalValue>(Val: U) &&
3893 Visited.insert(Ptr: U).second)
3894 Worklist.push_back(Elt: U);
3895 }
3896 }
3897 }
3898 }
3899
3900 if (!BlockAddressUsers.empty()) {
3901 Vals.resize(N: BlockAddressUsers.size());
3902 for (auto I : llvm::enumerate(First&: BlockAddressUsers))
3903 Vals[I.index()] = VE.getValueID(V: I.value());
3904 Stream.EmitRecord(Code: bitc::FUNC_CODE_BLOCKADDR_USERS, Vals);
3905 Vals.clear();
3906 }
3907
3908 // Emit names for all the instructions etc.
3909 if (auto *Symtab = F.getValueSymbolTable())
3910 writeFunctionLevelValueSymbolTable(VST: *Symtab);
3911
3912 if (NeedsMetadataAttachment)
3913 writeFunctionMetadataAttachment(F);
3914 if (VE.shouldPreserveUseListOrder())
3915 writeUseListBlock(F: &F);
3916 VE.purgeFunction();
3917 Stream.ExitBlock();
3918}
3919
3920// Emit blockinfo, which defines the standard abbreviations etc.
3921void ModuleBitcodeWriter::writeBlockInfo() {
3922 // We only want to emit block info records for blocks that have multiple
3923 // instances: CONSTANTS_BLOCK, FUNCTION_BLOCK and VALUE_SYMTAB_BLOCK.
3924 // Other blocks can define their abbrevs inline.
3925 Stream.EnterBlockInfoBlock();
3926
3927 // Encode type indices using fixed size based on number of types.
3928 BitCodeAbbrevOp TypeAbbrevOp(BitCodeAbbrevOp::Fixed,
3929 VE.computeBitsRequiredForTypeIndices());
3930 // Encode value indices as 6-bit VBR.
3931 BitCodeAbbrevOp ValAbbrevOp(BitCodeAbbrevOp::VBR, 6);
3932
3933 { // 8-bit fixed-width VST_CODE_ENTRY/VST_CODE_BBENTRY strings.
3934 auto Abbv = std::make_shared<BitCodeAbbrev>();
3935 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3));
3936 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3937 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3938 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
3939 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3940 VST_ENTRY_8_ABBREV)
3941 llvm_unreachable("Unexpected abbrev ordering!");
3942 }
3943
3944 { // 7-bit fixed width VST_CODE_ENTRY strings.
3945 auto Abbv = std::make_shared<BitCodeAbbrev>();
3946 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
3947 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3948 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3949 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
3950 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3951 VST_ENTRY_7_ABBREV)
3952 llvm_unreachable("Unexpected abbrev ordering!");
3953 }
3954 { // 6-bit char6 VST_CODE_ENTRY strings.
3955 auto Abbv = std::make_shared<BitCodeAbbrev>();
3956 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
3957 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3958 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3959 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
3960 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3961 VST_ENTRY_6_ABBREV)
3962 llvm_unreachable("Unexpected abbrev ordering!");
3963 }
3964 { // 6-bit char6 VST_CODE_BBENTRY strings.
3965 auto Abbv = std::make_shared<BitCodeAbbrev>();
3966 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_BBENTRY));
3967 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3968 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3969 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
3970 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3971 VST_BBENTRY_6_ABBREV)
3972 llvm_unreachable("Unexpected abbrev ordering!");
3973 }
3974
3975 { // SETTYPE abbrev for CONSTANTS_BLOCK.
3976 auto Abbv = std::make_shared<BitCodeAbbrev>();
3977 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_SETTYPE));
3978 Abbv->Add(OpInfo: TypeAbbrevOp);
3979 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3980 CONSTANTS_SETTYPE_ABBREV)
3981 llvm_unreachable("Unexpected abbrev ordering!");
3982 }
3983
3984 { // INTEGER abbrev for CONSTANTS_BLOCK.
3985 auto Abbv = std::make_shared<BitCodeAbbrev>();
3986 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_INTEGER));
3987 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3988 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3989 CONSTANTS_INTEGER_ABBREV)
3990 llvm_unreachable("Unexpected abbrev ordering!");
3991 }
3992
3993 { // BYTE abbrev for CONSTANTS_BLOCK.
3994 auto Abbv = std::make_shared<BitCodeAbbrev>();
3995 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_BYTE));
3996 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3997 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3998 CONSTANTS_BYTE_ABBREV)
3999 llvm_unreachable("Unexpected abbrev ordering!");
4000 }
4001
4002 { // CE_CAST abbrev for CONSTANTS_BLOCK.
4003 auto Abbv = std::make_shared<BitCodeAbbrev>();
4004 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CE_CAST));
4005 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // cast opc
4006 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // typeid
4007 VE.computeBitsRequiredForTypeIndices()));
4008 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
4009
4010 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4011 CONSTANTS_CE_CAST_Abbrev)
4012 llvm_unreachable("Unexpected abbrev ordering!");
4013 }
4014 { // NULL abbrev for CONSTANTS_BLOCK.
4015 auto Abbv = std::make_shared<BitCodeAbbrev>();
4016 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_NULL));
4017 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4018 CONSTANTS_NULL_Abbrev)
4019 llvm_unreachable("Unexpected abbrev ordering!");
4020 }
4021
4022 // FIXME: This should only use space for first class types!
4023
4024 { // INST_LOAD abbrev for FUNCTION_BLOCK.
4025 auto Abbv = std::make_shared<BitCodeAbbrev>();
4026 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_LOAD));
4027 Abbv->Add(OpInfo: ValAbbrevOp); // Ptr
4028 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4029 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Align
4030 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
4031 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4032 FUNCTION_INST_LOAD_ABBREV)
4033 llvm_unreachable("Unexpected abbrev ordering!");
4034 }
4035 {
4036 auto Abbv = std::make_shared<BitCodeAbbrev>();
4037 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_STORE));
4038 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4039 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4040 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // align
4041 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
4042 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4043 FUNCTION_INST_STORE_ABBREV)
4044 llvm_unreachable("Unexpected abbrev ordering!");
4045 }
4046 { // INST_UNOP abbrev for FUNCTION_BLOCK.
4047 auto Abbv = std::make_shared<BitCodeAbbrev>();
4048 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
4049 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4050 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4051 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4052 FUNCTION_INST_UNOP_ABBREV)
4053 llvm_unreachable("Unexpected abbrev ordering!");
4054 }
4055 { // INST_UNOP_FLAGS abbrev for FUNCTION_BLOCK.
4056 auto Abbv = std::make_shared<BitCodeAbbrev>();
4057 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
4058 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4059 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4060 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4061 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4062 FUNCTION_INST_UNOP_FLAGS_ABBREV)
4063 llvm_unreachable("Unexpected abbrev ordering!");
4064 }
4065 { // INST_BINOP abbrev for FUNCTION_BLOCK.
4066 auto Abbv = std::make_shared<BitCodeAbbrev>();
4067 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
4068 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4069 Abbv->Add(OpInfo: ValAbbrevOp); // RHS
4070 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4071 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4072 FUNCTION_INST_BINOP_ABBREV)
4073 llvm_unreachable("Unexpected abbrev ordering!");
4074 }
4075 { // INST_BINOP_FLAGS abbrev for FUNCTION_BLOCK.
4076 auto Abbv = std::make_shared<BitCodeAbbrev>();
4077 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
4078 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4079 Abbv->Add(OpInfo: ValAbbrevOp); // RHS
4080 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4081 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4082 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4083 FUNCTION_INST_BINOP_FLAGS_ABBREV)
4084 llvm_unreachable("Unexpected abbrev ordering!");
4085 }
4086 { // INST_CAST abbrev for FUNCTION_BLOCK.
4087 auto Abbv = std::make_shared<BitCodeAbbrev>();
4088 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
4089 Abbv->Add(OpInfo: ValAbbrevOp); // OpVal
4090 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4091 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4092 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4093 FUNCTION_INST_CAST_ABBREV)
4094 llvm_unreachable("Unexpected abbrev ordering!");
4095 }
4096 { // INST_CAST_FLAGS abbrev for FUNCTION_BLOCK.
4097 auto Abbv = std::make_shared<BitCodeAbbrev>();
4098 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
4099 Abbv->Add(OpInfo: ValAbbrevOp); // OpVal
4100 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4101 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4102 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4103 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4104 FUNCTION_INST_CAST_FLAGS_ABBREV)
4105 llvm_unreachable("Unexpected abbrev ordering!");
4106 }
4107
4108 { // INST_RET abbrev for FUNCTION_BLOCK.
4109 auto Abbv = std::make_shared<BitCodeAbbrev>();
4110 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
4111 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4112 FUNCTION_INST_RET_VOID_ABBREV)
4113 llvm_unreachable("Unexpected abbrev ordering!");
4114 }
4115 { // INST_RET abbrev for FUNCTION_BLOCK.
4116 auto Abbv = std::make_shared<BitCodeAbbrev>();
4117 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
4118 Abbv->Add(OpInfo: ValAbbrevOp);
4119 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4120 FUNCTION_INST_RET_VAL_ABBREV)
4121 llvm_unreachable("Unexpected abbrev ordering!");
4122 }
4123 {
4124 auto Abbv = std::make_shared<BitCodeAbbrev>();
4125 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BR));
4126 // TODO: Use different abbrev for absolute value reference (succ0)?
4127 Abbv->Add(OpInfo: ValAbbrevOp); // succ0
4128 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4129 FUNCTION_INST_BR_UNCOND_ABBREV)
4130 llvm_unreachable("Unexpected abbrev ordering!");
4131 }
4132 {
4133 auto Abbv = std::make_shared<BitCodeAbbrev>();
4134 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BR));
4135 // TODO: Use different abbrev for absolute value references (succ0, succ1)?
4136 Abbv->Add(OpInfo: ValAbbrevOp); // succ0
4137 Abbv->Add(OpInfo: ValAbbrevOp); // succ1
4138 Abbv->Add(OpInfo: ValAbbrevOp); // cond
4139 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4140 FUNCTION_INST_BR_COND_ABBREV)
4141 llvm_unreachable("Unexpected abbrev ordering!");
4142 }
4143 { // INST_UNREACHABLE abbrev for FUNCTION_BLOCK.
4144 auto Abbv = std::make_shared<BitCodeAbbrev>();
4145 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNREACHABLE));
4146 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4147 FUNCTION_INST_UNREACHABLE_ABBREV)
4148 llvm_unreachable("Unexpected abbrev ordering!");
4149 }
4150 {
4151 auto Abbv = std::make_shared<BitCodeAbbrev>();
4152 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_GEP));
4153 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // flags
4154 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4155 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4156 Abbv->Add(OpInfo: ValAbbrevOp);
4157 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4158 FUNCTION_INST_GEP_ABBREV)
4159 llvm_unreachable("Unexpected abbrev ordering!");
4160 }
4161 {
4162 auto Abbv = std::make_shared<BitCodeAbbrev>();
4163 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CMP2));
4164 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4165 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4166 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // pred
4167 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4168 FUNCTION_INST_CMP_ABBREV)
4169 llvm_unreachable("Unexpected abbrev ordering!");
4170 }
4171 {
4172 auto Abbv = std::make_shared<BitCodeAbbrev>();
4173 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CMP2));
4174 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4175 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4176 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // pred
4177 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4178 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4179 FUNCTION_INST_CMP_FLAGS_ABBREV)
4180 llvm_unreachable("Unexpected abbrev ordering!");
4181 }
4182 {
4183 auto Abbv = std::make_shared<BitCodeAbbrev>();
4184 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE));
4185 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // dbgloc
4186 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // var
4187 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // expr
4188 Abbv->Add(OpInfo: ValAbbrevOp); // val
4189 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4190 FUNCTION_DEBUG_RECORD_VALUE_ABBREV)
4191 llvm_unreachable("Unexpected abbrev ordering! 1");
4192 }
4193 {
4194 auto Abbv = std::make_shared<BitCodeAbbrev>();
4195 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_LOC));
4196 // NOTE: No IsDistinct field for FUNC_CODE_DEBUG_LOC.
4197 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4198 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4199 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4200 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4201 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
4202 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Atom group.
4203 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 3)); // Atom rank.
4204 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4205 FUNCTION_DEBUG_LOC_ABBREV)
4206 llvm_unreachable("Unexpected abbrev ordering!");
4207 }
4208 Stream.ExitBlock();
4209}
4210
4211/// Write the module path strings, currently only used when generating
4212/// a combined index file.
4213void IndexBitcodeWriter::writeModStrings() {
4214 Stream.EnterSubblock(BlockID: bitc::MODULE_STRTAB_BLOCK_ID, CodeLen: 3);
4215
4216 // TODO: See which abbrev sizes we actually need to emit
4217
4218 // 8-bit fixed-width MST_ENTRY strings.
4219 auto Abbv = std::make_shared<BitCodeAbbrev>();
4220 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4221 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4222 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4223 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
4224 unsigned Abbrev8Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4225
4226 // 7-bit fixed width MST_ENTRY strings.
4227 Abbv = std::make_shared<BitCodeAbbrev>();
4228 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4229 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4230 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4231 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
4232 unsigned Abbrev7Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4233
4234 // 6-bit char6 MST_ENTRY strings.
4235 Abbv = std::make_shared<BitCodeAbbrev>();
4236 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4237 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4238 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4239 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
4240 unsigned Abbrev6Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4241
4242 // Module Hash, 160 bits SHA1. Optionally, emitted after each MST_CODE_ENTRY.
4243 Abbv = std::make_shared<BitCodeAbbrev>();
4244 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_HASH));
4245 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4246 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4247 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4248 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4249 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4250 unsigned AbbrevHash = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4251
4252 SmallVector<unsigned, 64> Vals;
4253 forEachModule(Callback: [&](const StringMapEntry<ModuleHash> &MPSE) {
4254 StringRef Key = MPSE.getKey();
4255 const auto &Hash = MPSE.getValue();
4256 StringEncoding Bits = getStringEncoding(Str: Key);
4257 unsigned AbbrevToUse = Abbrev8Bit;
4258 if (Bits == SE_Char6)
4259 AbbrevToUse = Abbrev6Bit;
4260 else if (Bits == SE_Fixed7)
4261 AbbrevToUse = Abbrev7Bit;
4262
4263 auto ModuleId = ModuleIdMap.size();
4264 ModuleIdMap[Key] = ModuleId;
4265 Vals.push_back(Elt: ModuleId);
4266 Vals.append(in_start: Key.begin(), in_end: Key.end());
4267
4268 // Emit the finished record.
4269 Stream.EmitRecord(Code: bitc::MST_CODE_ENTRY, Vals, Abbrev: AbbrevToUse);
4270
4271 // Emit an optional hash for the module now
4272 if (llvm::any_of(Range: Hash, P: [](uint32_t H) { return H; })) {
4273 Vals.assign(in_start: Hash.begin(), in_end: Hash.end());
4274 // Emit the hash record.
4275 Stream.EmitRecord(Code: bitc::MST_CODE_HASH, Vals, Abbrev: AbbrevHash);
4276 }
4277
4278 Vals.clear();
4279 });
4280 Stream.ExitBlock();
4281}
4282
4283/// Write the function type metadata related records that need to appear before
4284/// a function summary entry (whether per-module or combined).
4285template <typename Fn>
4286static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream,
4287 FunctionSummary *FS,
4288 Fn GetValueID) {
4289 if (!FS->type_tests().empty())
4290 Stream.EmitRecord(Code: bitc::FS_TYPE_TESTS, Vals: FS->type_tests());
4291
4292 SmallVector<uint64_t, 64> Record;
4293
4294 auto WriteVFuncIdVec = [&](uint64_t Ty,
4295 ArrayRef<FunctionSummary::VFuncId> VFs) {
4296 if (VFs.empty())
4297 return;
4298 Record.clear();
4299 for (auto &VF : VFs) {
4300 Record.push_back(Elt: VF.GUID);
4301 Record.push_back(Elt: VF.Offset);
4302 }
4303 Stream.EmitRecord(Code: Ty, Vals: Record);
4304 };
4305
4306 WriteVFuncIdVec(bitc::FS_TYPE_TEST_ASSUME_VCALLS,
4307 FS->type_test_assume_vcalls());
4308 WriteVFuncIdVec(bitc::FS_TYPE_CHECKED_LOAD_VCALLS,
4309 FS->type_checked_load_vcalls());
4310
4311 auto WriteConstVCallVec = [&](uint64_t Ty,
4312 ArrayRef<FunctionSummary::ConstVCall> VCs) {
4313 for (auto &VC : VCs) {
4314 Record.clear();
4315 Record.push_back(Elt: VC.VFunc.GUID);
4316 Record.push_back(Elt: VC.VFunc.Offset);
4317 llvm::append_range(C&: Record, R: VC.Args);
4318 Stream.EmitRecord(Code: Ty, Vals: Record);
4319 }
4320 };
4321
4322 WriteConstVCallVec(bitc::FS_TYPE_TEST_ASSUME_CONST_VCALL,
4323 FS->type_test_assume_const_vcalls());
4324 WriteConstVCallVec(bitc::FS_TYPE_CHECKED_LOAD_CONST_VCALL,
4325 FS->type_checked_load_const_vcalls());
4326
4327 auto WriteRange = [&](ConstantRange Range) {
4328 Range = Range.sextOrTrunc(BitWidth: FunctionSummary::ParamAccess::RangeWidth);
4329 assert(Range.getLower().getNumWords() == 1);
4330 assert(Range.getUpper().getNumWords() == 1);
4331 emitSignedInt64(Vals&: Record, V: *Range.getLower().getRawData());
4332 emitSignedInt64(Vals&: Record, V: *Range.getUpper().getRawData());
4333 };
4334
4335 if (!FS->paramAccesses().empty()) {
4336 Record.clear();
4337 for (auto &Arg : FS->paramAccesses()) {
4338 size_t UndoSize = Record.size();
4339 Record.push_back(Elt: Arg.ParamNo);
4340 WriteRange(Arg.Use);
4341 Record.push_back(Elt: Arg.Calls.size());
4342 for (auto &Call : Arg.Calls) {
4343 Record.push_back(Elt: Call.ParamNo);
4344 std::optional<unsigned> ValueID = GetValueID(Call.Callee);
4345 if (!ValueID) {
4346 // If ValueID is unknown we can't drop just this call, we must drop
4347 // entire parameter.
4348 Record.resize(N: UndoSize);
4349 break;
4350 }
4351 Record.push_back(Elt: *ValueID);
4352 WriteRange(Call.Offsets);
4353 }
4354 }
4355 if (!Record.empty())
4356 Stream.EmitRecord(Code: bitc::FS_PARAM_ACCESS, Vals: Record);
4357 }
4358}
4359
4360/// Collect type IDs from type tests used by function.
4361static void
4362getReferencedTypeIds(FunctionSummary *FS,
4363 std::set<GlobalValue::GUID> &ReferencedTypeIds) {
4364 if (!FS->type_tests().empty())
4365 for (auto &TT : FS->type_tests())
4366 ReferencedTypeIds.insert(x: TT);
4367
4368 auto GetReferencedTypesFromVFuncIdVec =
4369 [&](ArrayRef<FunctionSummary::VFuncId> VFs) {
4370 for (auto &VF : VFs)
4371 ReferencedTypeIds.insert(x: VF.GUID);
4372 };
4373
4374 GetReferencedTypesFromVFuncIdVec(FS->type_test_assume_vcalls());
4375 GetReferencedTypesFromVFuncIdVec(FS->type_checked_load_vcalls());
4376
4377 auto GetReferencedTypesFromConstVCallVec =
4378 [&](ArrayRef<FunctionSummary::ConstVCall> VCs) {
4379 for (auto &VC : VCs)
4380 ReferencedTypeIds.insert(x: VC.VFunc.GUID);
4381 };
4382
4383 GetReferencedTypesFromConstVCallVec(FS->type_test_assume_const_vcalls());
4384 GetReferencedTypesFromConstVCallVec(FS->type_checked_load_const_vcalls());
4385}
4386
4387static void writeWholeProgramDevirtResolutionByArg(
4388 SmallVector<uint64_t, 64> &NameVals, const std::vector<uint64_t> &args,
4389 const WholeProgramDevirtResolution::ByArg &ByArg) {
4390 NameVals.push_back(Elt: args.size());
4391 llvm::append_range(C&: NameVals, R: args);
4392
4393 NameVals.push_back(Elt: ByArg.TheKind);
4394 NameVals.push_back(Elt: ByArg.Info);
4395 NameVals.push_back(Elt: ByArg.Byte);
4396 NameVals.push_back(Elt: ByArg.Bit);
4397}
4398
4399static void writeWholeProgramDevirtResolution(
4400 SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
4401 uint64_t Id, const WholeProgramDevirtResolution &Wpd) {
4402 NameVals.push_back(Elt: Id);
4403
4404 NameVals.push_back(Elt: Wpd.TheKind);
4405 NameVals.push_back(Elt: StrtabBuilder.add(S: Wpd.SingleImplName));
4406 NameVals.push_back(Elt: Wpd.SingleImplName.size());
4407
4408 NameVals.push_back(Elt: Wpd.ResByArg.size());
4409 for (auto &A : Wpd.ResByArg)
4410 writeWholeProgramDevirtResolutionByArg(NameVals, args: A.first, ByArg: A.second);
4411}
4412
4413static void writeTypeIdSummaryRecord(SmallVector<uint64_t, 64> &NameVals,
4414 StringTableBuilder &StrtabBuilder,
4415 StringRef Id,
4416 const TypeIdSummary &Summary) {
4417 NameVals.push_back(Elt: StrtabBuilder.add(S: Id));
4418 NameVals.push_back(Elt: Id.size());
4419
4420 NameVals.push_back(Elt: Summary.TTRes.TheKind);
4421 NameVals.push_back(Elt: Summary.TTRes.SizeM1BitWidth);
4422 NameVals.push_back(Elt: Summary.TTRes.AlignLog2);
4423 NameVals.push_back(Elt: Summary.TTRes.SizeM1);
4424 NameVals.push_back(Elt: Summary.TTRes.BitMask);
4425 NameVals.push_back(Elt: Summary.TTRes.InlineBits);
4426
4427 for (auto &W : Summary.WPDRes)
4428 writeWholeProgramDevirtResolution(NameVals, StrtabBuilder, Id: W.first,
4429 Wpd: W.second);
4430}
4431
4432static void writeTypeIdCompatibleVtableSummaryRecord(
4433 SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
4434 StringRef Id, const TypeIdCompatibleVtableInfo &Summary,
4435 ValueEnumerator &VE) {
4436 NameVals.push_back(Elt: StrtabBuilder.add(S: Id));
4437 NameVals.push_back(Elt: Id.size());
4438
4439 for (auto &P : Summary) {
4440 NameVals.push_back(Elt: P.AddressPointOffset);
4441 NameVals.push_back(Elt: VE.getValueID(V: P.VTableVI.getValue()));
4442 }
4443}
4444
4445// Adds the allocation contexts to the CallStacks map. We simply use the
4446// size at the time the context was added as the CallStackId. This works because
4447// when we look up the call stacks later on we process the function summaries
4448// and their allocation records in the same exact order.
4449static void collectMemProfCallStacks(
4450 FunctionSummary *FS, std::function<LinearFrameId(unsigned)> GetStackIndex,
4451 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> &CallStacks) {
4452 // The interfaces in ProfileData/MemProf.h use a type alias for a stack frame
4453 // id offset into the index of the full stack frames. The ModuleSummaryIndex
4454 // currently uses unsigned. Make sure these stay in sync.
4455 static_assert(std::is_same_v<LinearFrameId, unsigned>);
4456 for (auto &AI : FS->allocs()) {
4457 for (auto &MIB : AI.MIBs) {
4458 SmallVector<unsigned> StackIdIndices;
4459 StackIdIndices.reserve(N: MIB.StackIdIndices.size());
4460 for (auto Id : MIB.StackIdIndices)
4461 StackIdIndices.push_back(Elt: GetStackIndex(Id));
4462 // The CallStackId is the size at the time this context was inserted.
4463 CallStacks.insert(KV: {CallStacks.size(), StackIdIndices});
4464 }
4465 }
4466}
4467
4468// Build the radix tree from the accumulated CallStacks, write out the resulting
4469// linearized radix tree array, and return the map of call stack positions into
4470// this array for use when writing the allocation records. The returned map is
4471// indexed by a CallStackId which in this case is implicitly determined by the
4472// order of function summaries and their allocation infos being written.
4473static DenseMap<CallStackId, LinearCallStackId> writeMemoryProfileRadixTree(
4474 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> &&CallStacks,
4475 BitstreamWriter &Stream, unsigned RadixAbbrev) {
4476 assert(!CallStacks.empty());
4477 DenseMap<unsigned, FrameStat> FrameHistogram =
4478 computeFrameHistogram<LinearFrameId>(MemProfCallStackData&: CallStacks);
4479 CallStackRadixTreeBuilder<LinearFrameId> Builder;
4480 // We don't need a MemProfFrameIndexes map as we have already converted the
4481 // full stack id hash to a linear offset into the StackIds array.
4482 Builder.build(MemProfCallStackData: std::move(CallStacks), /*MemProfFrameIndexes=*/nullptr,
4483 FrameHistogram);
4484 Stream.EmitRecord(Code: bitc::FS_CONTEXT_RADIX_TREE_ARRAY, Vals: Builder.getRadixArray(),
4485 Abbrev: RadixAbbrev);
4486 return Builder.takeCallStackPos();
4487}
4488
4489static void writeFunctionHeapProfileRecords(
4490 BitstreamWriter &Stream, FunctionSummary *FS, unsigned CallsiteAbbrev,
4491 unsigned AllocAbbrev, unsigned ContextIdAbbvId, bool PerModule,
4492 std::function<unsigned(const ValueInfo &VI)> GetValueID,
4493 std::function<unsigned(unsigned)> GetStackIndex,
4494 bool WriteContextSizeInfoIndex,
4495 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
4496 CallStackId &CallStackCount) {
4497 SmallVector<uint64_t> Record;
4498
4499 for (auto &CI : FS->callsites()) {
4500 Record.clear();
4501 // Per module callsite clones should always have a single entry of
4502 // value 0.
4503 assert(!PerModule || (CI.Clones.size() == 1 && CI.Clones[0] == 0));
4504 Record.push_back(Elt: GetValueID(CI.Callee));
4505 if (!PerModule) {
4506 Record.push_back(Elt: CI.StackIdIndices.size());
4507 Record.push_back(Elt: CI.Clones.size());
4508 }
4509 for (auto Id : CI.StackIdIndices)
4510 Record.push_back(Elt: GetStackIndex(Id));
4511 if (!PerModule)
4512 llvm::append_range(C&: Record, R: CI.Clones);
4513 Stream.EmitRecord(Code: PerModule ? bitc::FS_PERMODULE_CALLSITE_INFO
4514 : bitc::FS_COMBINED_CALLSITE_INFO,
4515 Vals: Record, Abbrev: CallsiteAbbrev);
4516 }
4517
4518 for (auto &AI : FS->allocs()) {
4519 Record.clear();
4520 // Per module alloc versions should always have a single entry of
4521 // value 0.
4522 assert(!PerModule || (AI.Versions.size() == 1 && AI.Versions[0] == 0));
4523 Record.push_back(Elt: AI.MIBs.size());
4524 if (!PerModule)
4525 Record.push_back(Elt: AI.Versions.size());
4526 for (auto &MIB : AI.MIBs) {
4527 Record.push_back(Elt: (uint8_t)MIB.AllocType);
4528 // The per-module summary always needs to include the alloc context, as we
4529 // use it during the thin link. For the combined index it is optional (see
4530 // comments where CombinedIndexMemProfContext is defined).
4531 if (PerModule || CombinedIndexMemProfContext) {
4532 // Record the index into the radix tree array for this context.
4533 assert(CallStackCount <= CallStackPos.size());
4534 Record.push_back(Elt: CallStackPos[CallStackCount++]);
4535 }
4536 }
4537 if (!PerModule)
4538 llvm::append_range(C&: Record, R: AI.Versions);
4539 assert(AI.ContextSizeInfos.empty() ||
4540 AI.ContextSizeInfos.size() == AI.MIBs.size());
4541 // Optionally emit the context size information if it exists.
4542 if (WriteContextSizeInfoIndex && !AI.ContextSizeInfos.empty()) {
4543 // The abbreviation id for the context ids record should have been created
4544 // if we are emitting the per-module index, which is where we write this
4545 // info.
4546 assert(ContextIdAbbvId);
4547 SmallVector<uint32_t> ContextIds;
4548 // At least one context id per ContextSizeInfos entry (MIB), broken into 2
4549 // halves.
4550 ContextIds.reserve(N: AI.ContextSizeInfos.size() * 2);
4551 for (auto &Infos : AI.ContextSizeInfos) {
4552 Record.push_back(Elt: Infos.size());
4553 for (auto [FullStackId, TotalSize] : Infos) {
4554 // The context ids are emitted separately as a fixed width array,
4555 // which is more efficient than a VBR given that these hashes are
4556 // typically close to 64-bits. The max fixed width entry is 32 bits so
4557 // it is split into 2.
4558 ContextIds.push_back(Elt: static_cast<uint32_t>(FullStackId >> 32));
4559 ContextIds.push_back(Elt: static_cast<uint32_t>(FullStackId));
4560 Record.push_back(Elt: TotalSize);
4561 }
4562 }
4563 // The context ids are expected by the reader to immediately precede the
4564 // associated alloc info record.
4565 Stream.EmitRecord(Code: bitc::FS_ALLOC_CONTEXT_IDS, Vals: ContextIds,
4566 Abbrev: ContextIdAbbvId);
4567 }
4568 Stream.EmitRecord(Code: PerModule
4569 ? bitc::FS_PERMODULE_ALLOC_INFO
4570 : (CombinedIndexMemProfContext
4571 ? bitc::FS_COMBINED_ALLOC_INFO
4572 : bitc::FS_COMBINED_ALLOC_INFO_NO_CONTEXT),
4573 Vals: Record, Abbrev: AllocAbbrev);
4574 }
4575}
4576
4577// Helper to emit a single function summary record.
4578void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
4579 SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
4580 unsigned ValueID, unsigned FSCallsProfileAbbrev, unsigned CallsiteAbbrev,
4581 unsigned AllocAbbrev, unsigned ContextIdAbbvId, const Function &F,
4582 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
4583 CallStackId &CallStackCount) {
4584 NameVals.push_back(Elt: ValueID);
4585
4586 FunctionSummary *FS = cast<FunctionSummary>(Val: Summary);
4587
4588 writeFunctionTypeMetadataRecords(
4589 Stream, FS, GetValueID: [&](const ValueInfo &VI) -> std::optional<unsigned> {
4590 return {VE.getValueID(V: VI.getValue())};
4591 });
4592
4593 auto SpecialRefCnts = FS->specialRefCounts();
4594 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: FS->flags()));
4595 NameVals.push_back(Elt: FS->instCount());
4596 NameVals.push_back(Elt: getEncodedFFlags(Flags: FS->fflags()));
4597 NameVals.push_back(Elt: FS->refs().size());
4598 NameVals.push_back(Elt: SpecialRefCnts.first); // rorefcnt
4599 NameVals.push_back(Elt: SpecialRefCnts.second); // worefcnt
4600
4601 for (auto &RI : FS->refs())
4602 NameVals.push_back(Elt: getValueId(VI: RI));
4603
4604 for (auto &ECI : FS->calls()) {
4605 NameVals.push_back(Elt: getValueId(VI: ECI.first));
4606 NameVals.push_back(Elt: getEncodedHotnessCallEdgeInfo(CI: ECI.second));
4607 }
4608
4609 // Emit the finished record.
4610 Stream.EmitRecord(Code: bitc::FS_PERMODULE_PROFILE, Vals: NameVals, Abbrev: FSCallsProfileAbbrev);
4611 NameVals.clear();
4612
4613 writeFunctionHeapProfileRecords(
4614 Stream, FS, CallsiteAbbrev, AllocAbbrev, ContextIdAbbvId,
4615 /*PerModule*/ true,
4616 /*GetValueId*/ GetValueID: [&](const ValueInfo &VI) { return getValueId(VI); },
4617 /*GetStackIndex*/ [&](unsigned I) { return I; },
4618 /*WriteContextSizeInfoIndex*/ true, CallStackPos, CallStackCount);
4619}
4620
4621// Collect the global value references in the given variable's initializer,
4622// and emit them in a summary record.
4623void ModuleBitcodeWriterBase::writeModuleLevelReferences(
4624 const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals,
4625 unsigned FSModRefsAbbrev, unsigned FSModVTableRefsAbbrev) {
4626 auto VI = Index->getValueInfo(GUID: V.getGUID());
4627 if (!VI || VI.getSummaryList().empty()) {
4628 // Only declarations should not have a summary (a declaration might however
4629 // have a summary if the def was in module level asm).
4630 assert(V.isDeclaration());
4631 return;
4632 }
4633 auto *Summary = VI.getSummaryList()[0].get();
4634 NameVals.push_back(Elt: VE.getValueID(V: &V));
4635 GlobalVarSummary *VS = cast<GlobalVarSummary>(Val: Summary);
4636 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: VS->flags()));
4637 NameVals.push_back(Elt: getEncodedGVarFlags(Flags: VS->varflags()));
4638
4639 auto VTableFuncs = VS->vTableFuncs();
4640 if (!VTableFuncs.empty())
4641 NameVals.push_back(Elt: VS->refs().size());
4642
4643 unsigned SizeBeforeRefs = NameVals.size();
4644 for (auto &RI : VS->refs())
4645 NameVals.push_back(Elt: VE.getValueID(V: RI.getValue()));
4646 // Sort the refs for determinism output, the vector returned by FS->refs() has
4647 // been initialized from a DenseSet.
4648 llvm::sort(C: drop_begin(RangeOrContainer&: NameVals, N: SizeBeforeRefs));
4649
4650 if (VTableFuncs.empty())
4651 Stream.EmitRecord(Code: bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, Vals: NameVals,
4652 Abbrev: FSModRefsAbbrev);
4653 else {
4654 // VTableFuncs pairs should already be sorted by offset.
4655 for (auto &P : VTableFuncs) {
4656 NameVals.push_back(Elt: VE.getValueID(V: P.FuncVI.getValue()));
4657 NameVals.push_back(Elt: P.VTableOffset);
4658 }
4659
4660 Stream.EmitRecord(Code: bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS, Vals: NameVals,
4661 Abbrev: FSModVTableRefsAbbrev);
4662 }
4663 NameVals.clear();
4664}
4665
4666/// Emit the per-module summary section alongside the rest of
4667/// the module's bitcode.
4668void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
4669 // By default we compile with ThinLTO if the module has a summary, but the
4670 // client can request full LTO with a module flag.
4671 bool IsThinLTO = true;
4672 if (auto *MD =
4673 mdconst::extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: "ThinLTO")))
4674 IsThinLTO = MD->getZExtValue();
4675 Stream.EnterSubblock(BlockID: IsThinLTO ? bitc::GLOBALVAL_SUMMARY_BLOCK_ID
4676 : bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID,
4677 CodeLen: 4);
4678
4679 Stream.EmitRecord(
4680 Code: bitc::FS_VERSION,
4681 Vals: ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
4682
4683 // Write the index flags.
4684 uint64_t Flags = 0;
4685 // Bits 1-3 are set only in the combined index, skip them.
4686 if (Index->enableSplitLTOUnit())
4687 Flags |= 0x8;
4688 if (Index->hasUnifiedLTO())
4689 Flags |= 0x200;
4690
4691 Stream.EmitRecord(Code: bitc::FS_FLAGS, Vals: ArrayRef<uint64_t>{Flags});
4692
4693 if (Index->begin() == Index->end()) {
4694 Stream.ExitBlock();
4695 return;
4696 }
4697
4698 auto Abbv = std::make_shared<BitCodeAbbrev>();
4699 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_VALUE_GUID));
4700 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4701 // GUIDS often use up most of 64-bits, so encode as two Fixed 32.
4702 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4703 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4704 unsigned ValueGuidAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4705
4706 for (const auto &GVI : valueIds()) {
4707 Stream.EmitRecord(Code: bitc::FS_VALUE_GUID,
4708 Vals: ArrayRef<uint32_t>{GVI.second,
4709 static_cast<uint32_t>(GVI.first >> 32),
4710 static_cast<uint32_t>(GVI.first)},
4711 Abbrev: ValueGuidAbbrev);
4712 }
4713
4714 if (!Index->stackIds().empty()) {
4715 auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4716 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_STACK_IDS));
4717 // numids x stackid
4718 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4719 // The stack ids are hashes that are close to 64 bits in size, so emitting
4720 // as a pair of 32-bit fixed-width values is more efficient than a VBR.
4721 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4722 unsigned StackIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(StackIdAbbv));
4723 SmallVector<uint32_t> Vals;
4724 Vals.reserve(N: Index->stackIds().size() * 2);
4725 for (auto Id : Index->stackIds()) {
4726 Vals.push_back(Elt: static_cast<uint32_t>(Id >> 32));
4727 Vals.push_back(Elt: static_cast<uint32_t>(Id));
4728 }
4729 Stream.EmitRecord(Code: bitc::FS_STACK_IDS, Vals, Abbrev: StackIdAbbvId);
4730 }
4731
4732 unsigned ContextIdAbbvId = 0;
4733 if (metadataMayIncludeContextSizeInfo()) {
4734 // n x context id
4735 auto ContextIdAbbv = std::make_shared<BitCodeAbbrev>();
4736 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_ALLOC_CONTEXT_IDS));
4737 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4738 // The context ids are hashes that are close to 64 bits in size, so emitting
4739 // as a pair of 32-bit fixed-width values is more efficient than a VBR if we
4740 // are emitting them for all MIBs. Otherwise we use VBR to better compress 0
4741 // values that are expected to more frequently occur in an alloc's memprof
4742 // summary.
4743 if (metadataIncludesAllContextSizeInfo())
4744 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4745 else
4746 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4747 ContextIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(ContextIdAbbv));
4748 }
4749
4750 // Abbrev for FS_PERMODULE_PROFILE.
4751 Abbv = std::make_shared<BitCodeAbbrev>();
4752 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE));
4753 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4754 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // flags
4755 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
4756 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
4757 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4758 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
4759 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
4760 // numrefs x valueid, n x (valueid, hotness+tailcall flags)
4761 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4762 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4763 unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4764
4765 // Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS.
4766 Abbv = std::make_shared<BitCodeAbbrev>();
4767 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS));
4768 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4769 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4770 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
4771 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4772 unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4773
4774 // Abbrev for FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS.
4775 Abbv = std::make_shared<BitCodeAbbrev>();
4776 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS));
4777 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4778 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4779 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4780 // numrefs x valueid, n x (valueid , offset)
4781 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4782 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4783 unsigned FSModVTableRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4784
4785 // Abbrev for FS_ALIAS.
4786 Abbv = std::make_shared<BitCodeAbbrev>();
4787 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_ALIAS));
4788 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4789 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4790 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4791 unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4792
4793 // Abbrev for FS_TYPE_ID_METADATA
4794 Abbv = std::make_shared<BitCodeAbbrev>();
4795 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_TYPE_ID_METADATA));
4796 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid strtab index
4797 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid length
4798 // n x (valueid , offset)
4799 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4800 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4801 unsigned TypeIdCompatibleVtableAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4802
4803 Abbv = std::make_shared<BitCodeAbbrev>();
4804 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_CALLSITE_INFO));
4805 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4806 // n x stackidindex
4807 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4808 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4809 unsigned CallsiteAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4810
4811 Abbv = std::make_shared<BitCodeAbbrev>();
4812 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_ALLOC_INFO));
4813 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib
4814 // n x (alloc type, context radix tree index)
4815 // optional: nummib x (numcontext x total size)
4816 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4817 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4818 unsigned AllocAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4819
4820 Abbv = std::make_shared<BitCodeAbbrev>();
4821 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_CONTEXT_RADIX_TREE_ARRAY));
4822 // n x entry
4823 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4824 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4825 unsigned RadixAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4826
4827 // First walk through all the functions and collect the allocation contexts in
4828 // their associated summaries, for use in constructing a radix tree of
4829 // contexts. Note that we need to do this in the same order as the functions
4830 // are processed further below since the call stack positions in the resulting
4831 // radix tree array are identified based on this order.
4832 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> CallStacks;
4833 for (const Function &F : M) {
4834 // Summary emission does not support anonymous functions, they have to be
4835 // renamed using the anonymous function renaming pass.
4836 if (!F.hasName())
4837 report_fatal_error(reason: "Unexpected anonymous function when writing summary");
4838
4839 ValueInfo VI = Index->getValueInfo(GUID: F.getGUID());
4840 if (!VI || VI.getSummaryList().empty()) {
4841 // Only declarations should not have a summary (a declaration might
4842 // however have a summary if the def was in module level asm).
4843 assert(F.isDeclaration());
4844 continue;
4845 }
4846 auto *Summary = VI.getSummaryList()[0].get();
4847 FunctionSummary *FS = cast<FunctionSummary>(Val: Summary);
4848 collectMemProfCallStacks(
4849 FS, /*GetStackIndex*/ [](unsigned I) { return I; }, CallStacks);
4850 }
4851 // Finalize the radix tree, write it out, and get the map of positions in the
4852 // linearized tree array.
4853 DenseMap<CallStackId, LinearCallStackId> CallStackPos;
4854 if (!CallStacks.empty()) {
4855 CallStackPos =
4856 writeMemoryProfileRadixTree(CallStacks: std::move(CallStacks), Stream, RadixAbbrev);
4857 }
4858
4859 // Keep track of the current index into the CallStackPos map.
4860 CallStackId CallStackCount = 0;
4861
4862 SmallVector<uint64_t, 64> NameVals;
4863 // Iterate over the list of functions instead of the Index to
4864 // ensure the ordering is stable.
4865 for (const Function &F : M) {
4866 // Summary emission does not support anonymous functions, they have to
4867 // renamed using the anonymous function renaming pass.
4868 if (!F.hasName())
4869 report_fatal_error(reason: "Unexpected anonymous function when writing summary");
4870
4871 ValueInfo VI = Index->getValueInfo(GUID: F.getGUID());
4872 if (!VI || VI.getSummaryList().empty()) {
4873 // Only declarations should not have a summary (a declaration might
4874 // however have a summary if the def was in module level asm).
4875 assert(F.isDeclaration());
4876 continue;
4877 }
4878 auto *Summary = VI.getSummaryList()[0].get();
4879 writePerModuleFunctionSummaryRecord(NameVals, Summary, ValueID: VE.getValueID(V: &F),
4880 FSCallsProfileAbbrev, CallsiteAbbrev,
4881 AllocAbbrev, ContextIdAbbvId, F,
4882 CallStackPos, CallStackCount);
4883 }
4884
4885 // Capture references from GlobalVariable initializers, which are outside
4886 // of a function scope.
4887 for (const GlobalVariable &G : M.globals())
4888 writeModuleLevelReferences(V: G, NameVals, FSModRefsAbbrev,
4889 FSModVTableRefsAbbrev);
4890
4891 for (const GlobalAlias &A : M.aliases()) {
4892 auto *Aliasee = A.getAliaseeObject();
4893 // Skip ifunc and nameless functions which don't have an entry in the
4894 // summary.
4895 if (!Aliasee->hasName() || isa<GlobalIFunc>(Val: Aliasee))
4896 continue;
4897 auto AliasId = VE.getValueID(V: &A);
4898 auto AliaseeId = VE.getValueID(V: Aliasee);
4899 NameVals.push_back(Elt: AliasId);
4900 auto *Summary = Index->getGlobalValueSummary(GV: A);
4901 AliasSummary *AS = cast<AliasSummary>(Val: Summary);
4902 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: AS->flags()));
4903 NameVals.push_back(Elt: AliaseeId);
4904 Stream.EmitRecord(Code: bitc::FS_ALIAS, Vals: NameVals, Abbrev: FSAliasAbbrev);
4905 NameVals.clear();
4906 }
4907
4908 for (auto &S : Index->typeIdCompatibleVtableMap()) {
4909 writeTypeIdCompatibleVtableSummaryRecord(NameVals, StrtabBuilder, Id: S.first,
4910 Summary: S.second, VE);
4911 Stream.EmitRecord(Code: bitc::FS_TYPE_ID_METADATA, Vals: NameVals,
4912 Abbrev: TypeIdCompatibleVtableAbbrev);
4913 NameVals.clear();
4914 }
4915
4916 if (Index->getBlockCount())
4917 Stream.EmitRecord(Code: bitc::FS_BLOCK_COUNT,
4918 Vals: ArrayRef<uint64_t>{Index->getBlockCount()});
4919
4920 Stream.ExitBlock();
4921}
4922
4923/// Emit the combined summary section into the combined index file.
4924void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
4925 Stream.EnterSubblock(BlockID: bitc::GLOBALVAL_SUMMARY_BLOCK_ID, CodeLen: 4);
4926 Stream.EmitRecord(
4927 Code: bitc::FS_VERSION,
4928 Vals: ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
4929
4930 // Write the index flags.
4931 Stream.EmitRecord(Code: bitc::FS_FLAGS, Vals: ArrayRef<uint64_t>{Index.getFlags()});
4932
4933 auto Abbv = std::make_shared<BitCodeAbbrev>();
4934 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_VALUE_GUID));
4935 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4936 // GUIDS often use up most of 64-bits, so encode as two Fixed 32.
4937 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4938 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4939 unsigned ValueGuidAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4940
4941 for (const auto &GVI : valueIds()) {
4942 Stream.EmitRecord(Code: bitc::FS_VALUE_GUID,
4943 Vals: ArrayRef<uint32_t>{GVI.second,
4944 static_cast<uint32_t>(GVI.first >> 32),
4945 static_cast<uint32_t>(GVI.first)},
4946 Abbrev: ValueGuidAbbrev);
4947 }
4948
4949 // Write the stack ids used by this index, which will be a subset of those in
4950 // the full index in the case of distributed indexes.
4951 if (!StackIds.empty()) {
4952 auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4953 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_STACK_IDS));
4954 // numids x stackid
4955 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4956 // The stack ids are hashes that are close to 64 bits in size, so emitting
4957 // as a pair of 32-bit fixed-width values is more efficient than a VBR.
4958 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4959 unsigned StackIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(StackIdAbbv));
4960 SmallVector<uint32_t> Vals;
4961 Vals.reserve(N: StackIds.size() * 2);
4962 for (auto Id : StackIds) {
4963 Vals.push_back(Elt: static_cast<uint32_t>(Id >> 32));
4964 Vals.push_back(Elt: static_cast<uint32_t>(Id));
4965 }
4966 Stream.EmitRecord(Code: bitc::FS_STACK_IDS, Vals, Abbrev: StackIdAbbvId);
4967 }
4968
4969 // Abbrev for FS_COMBINED_PROFILE.
4970 Abbv = std::make_shared<BitCodeAbbrev>();
4971 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_PROFILE));
4972 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4973 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
4974 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4975 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
4976 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
4977 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // entrycount
4978 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4979 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
4980 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
4981 // numrefs x valueid, n x (valueid, hotness+tailcall flags)
4982 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4983 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4984 unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4985
4986 // Abbrev for FS_COMBINED_GLOBALVAR_INIT_REFS.
4987 Abbv = std::make_shared<BitCodeAbbrev>();
4988 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS));
4989 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4990 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
4991 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4992 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
4993 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4994 unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4995
4996 // Abbrev for FS_COMBINED_ALIAS.
4997 Abbv = std::make_shared<BitCodeAbbrev>();
4998 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_ALIAS));
4999 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5000 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
5001 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
5002 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5003 unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5004
5005 Abbv = std::make_shared<BitCodeAbbrev>();
5006 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_CALLSITE_INFO));
5007 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5008 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numstackindices
5009 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
5010 // numstackindices x stackidindex, numver x version
5011 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5012 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5013 unsigned CallsiteAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5014
5015 Abbv = std::make_shared<BitCodeAbbrev>();
5016 Abbv->Add(OpInfo: BitCodeAbbrevOp(CombinedIndexMemProfContext
5017 ? bitc::FS_COMBINED_ALLOC_INFO
5018 : bitc::FS_COMBINED_ALLOC_INFO_NO_CONTEXT));
5019 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib
5020 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
5021 // nummib x (alloc type, context radix tree index),
5022 // numver x version
5023 // optional: nummib x total size
5024 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5025 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5026 unsigned AllocAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5027
5028 auto shouldImportValueAsDecl = [&](GlobalValueSummary *GVS) -> bool {
5029 if (DecSummaries == nullptr)
5030 return false;
5031 return DecSummaries->count(x: GVS);
5032 };
5033
5034 // The aliases are emitted as a post-pass, and will point to the value
5035 // id of the aliasee. Save them in a vector for post-processing.
5036 SmallVector<AliasSummary *, 64> Aliases;
5037
5038 // Save the value id for each summary for alias emission.
5039 DenseMap<const GlobalValueSummary *, unsigned> SummaryToValueIdMap;
5040
5041 SmallVector<uint64_t, 64> NameVals;
5042
5043 // Set that will be populated during call to writeFunctionTypeMetadataRecords
5044 // with the type ids referenced by this index file.
5045 std::set<GlobalValue::GUID> ReferencedTypeIds;
5046
5047 // For local linkage, we also emit the original name separately
5048 // immediately after the record.
5049 auto MaybeEmitOriginalName = [&](GlobalValueSummary &S) {
5050 // We don't need to emit the original name if we are writing the index for
5051 // distributed backends (in which case ModuleToSummariesForIndex is
5052 // non-null). The original name is only needed during the thin link, since
5053 // for SamplePGO the indirect call targets for local functions have
5054 // have the original name annotated in profile.
5055 // Continue to emit it when writing out the entire combined index, which is
5056 // used in testing the thin link via llvm-lto.
5057 if (ModuleToSummariesForIndex || !GlobalValue::isLocalLinkage(Linkage: S.linkage()))
5058 return;
5059 NameVals.push_back(Elt: S.getOriginalName());
5060 Stream.EmitRecord(Code: bitc::FS_COMBINED_ORIGINAL_NAME, Vals: NameVals);
5061 NameVals.clear();
5062 };
5063
5064 DenseMap<CallStackId, LinearCallStackId> CallStackPos;
5065 if (CombinedIndexMemProfContext) {
5066 Abbv = std::make_shared<BitCodeAbbrev>();
5067 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_CONTEXT_RADIX_TREE_ARRAY));
5068 // n x entry
5069 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5070 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5071 unsigned RadixAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5072
5073 // First walk through all the functions and collect the allocation contexts
5074 // in their associated summaries, for use in constructing a radix tree of
5075 // contexts. Note that we need to do this in the same order as the functions
5076 // are processed further below since the call stack positions in the
5077 // resulting radix tree array are identified based on this order.
5078 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> CallStacks;
5079 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
5080 // Don't collect this when invoked for an aliasee, as it is not needed for
5081 // the alias summary. If the aliasee is to be imported, we will invoke
5082 // this separately with IsAliasee=false.
5083 if (IsAliasee)
5084 return;
5085 GlobalValueSummary *S = I.second;
5086 assert(S);
5087 auto *FS = dyn_cast<FunctionSummary>(Val: S);
5088 if (!FS)
5089 return;
5090 collectMemProfCallStacks(
5091 FS,
5092 /*GetStackIndex*/
5093 [&](unsigned I) {
5094 // Get the corresponding index into the list of StackIds actually
5095 // being written for this combined index (which may be a subset in
5096 // the case of distributed indexes).
5097 assert(StackIdIndicesToIndex.contains(I));
5098 return StackIdIndicesToIndex[I];
5099 },
5100 CallStacks);
5101 });
5102 // Finalize the radix tree, write it out, and get the map of positions in
5103 // the linearized tree array.
5104 if (!CallStacks.empty()) {
5105 CallStackPos = writeMemoryProfileRadixTree(CallStacks: std::move(CallStacks), Stream,
5106 RadixAbbrev);
5107 }
5108 }
5109
5110 // Keep track of the current index into the CallStackPos map. Not used if
5111 // CombinedIndexMemProfContext is false.
5112 CallStackId CallStackCount = 0;
5113
5114 DenseSet<GlobalValue::GUID> DefOrUseGUIDs;
5115 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
5116 GlobalValueSummary *S = I.second;
5117 assert(S);
5118 DefOrUseGUIDs.insert(V: I.first);
5119 for (const ValueInfo &VI : S->refs())
5120 DefOrUseGUIDs.insert(V: VI.getGUID());
5121
5122 auto ValueId = getValueId(ValGUID: I.first);
5123 assert(ValueId);
5124 SummaryToValueIdMap[S] = *ValueId;
5125
5126 // If this is invoked for an aliasee, we want to record the above
5127 // mapping, but then not emit a summary entry (if the aliasee is
5128 // to be imported, we will invoke this separately with IsAliasee=false).
5129 if (IsAliasee)
5130 return;
5131
5132 if (auto *AS = dyn_cast<AliasSummary>(Val: S)) {
5133 // Will process aliases as a post-pass because the reader wants all
5134 // global to be loaded first.
5135 Aliases.push_back(Elt: AS);
5136 return;
5137 }
5138
5139 if (auto *VS = dyn_cast<GlobalVarSummary>(Val: S)) {
5140 NameVals.push_back(Elt: *ValueId);
5141 assert(ModuleIdMap.count(VS->modulePath()));
5142 NameVals.push_back(Elt: ModuleIdMap[VS->modulePath()]);
5143 NameVals.push_back(
5144 Elt: getEncodedGVSummaryFlags(Flags: VS->flags(), ImportAsDecl: shouldImportValueAsDecl(VS)));
5145 NameVals.push_back(Elt: getEncodedGVarFlags(Flags: VS->varflags()));
5146 for (auto &RI : VS->refs()) {
5147 auto RefValueId = getValueId(ValGUID: RI.getGUID());
5148 if (!RefValueId)
5149 continue;
5150 NameVals.push_back(Elt: *RefValueId);
5151 }
5152
5153 // Emit the finished record.
5154 Stream.EmitRecord(Code: bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, Vals: NameVals,
5155 Abbrev: FSModRefsAbbrev);
5156 NameVals.clear();
5157 MaybeEmitOriginalName(*S);
5158 return;
5159 }
5160
5161 auto GetValueId = [&](const ValueInfo &VI) -> std::optional<unsigned> {
5162 if (!VI)
5163 return std::nullopt;
5164 return getValueId(ValGUID: VI.getGUID());
5165 };
5166
5167 auto *FS = cast<FunctionSummary>(Val: S);
5168 writeFunctionTypeMetadataRecords(Stream, FS, GetValueID: GetValueId);
5169 getReferencedTypeIds(FS, ReferencedTypeIds);
5170
5171 NameVals.push_back(Elt: *ValueId);
5172 assert(ModuleIdMap.count(FS->modulePath()));
5173 NameVals.push_back(Elt: ModuleIdMap[FS->modulePath()]);
5174 NameVals.push_back(
5175 Elt: getEncodedGVSummaryFlags(Flags: FS->flags(), ImportAsDecl: shouldImportValueAsDecl(FS)));
5176 NameVals.push_back(Elt: FS->instCount());
5177 NameVals.push_back(Elt: getEncodedFFlags(Flags: FS->fflags()));
5178 // TODO: Stop writing entry count and bump bitcode version.
5179 NameVals.push_back(Elt: 0 /* EntryCount */);
5180
5181 // Fill in below
5182 NameVals.push_back(Elt: 0); // numrefs
5183 NameVals.push_back(Elt: 0); // rorefcnt
5184 NameVals.push_back(Elt: 0); // worefcnt
5185
5186 unsigned Count = 0, RORefCnt = 0, WORefCnt = 0;
5187 for (auto &RI : FS->refs()) {
5188 auto RefValueId = getValueId(ValGUID: RI.getGUID());
5189 if (!RefValueId)
5190 continue;
5191 NameVals.push_back(Elt: *RefValueId);
5192 if (RI.isReadOnly())
5193 RORefCnt++;
5194 else if (RI.isWriteOnly())
5195 WORefCnt++;
5196 Count++;
5197 }
5198 NameVals[6] = Count;
5199 NameVals[7] = RORefCnt;
5200 NameVals[8] = WORefCnt;
5201
5202 for (auto &EI : FS->calls()) {
5203 // If this GUID doesn't have a value id, it doesn't have a function
5204 // summary and we don't need to record any calls to it.
5205 std::optional<unsigned> CallValueId = GetValueId(EI.first);
5206 if (!CallValueId)
5207 continue;
5208 NameVals.push_back(Elt: *CallValueId);
5209 NameVals.push_back(Elt: getEncodedHotnessCallEdgeInfo(CI: EI.second));
5210 }
5211
5212 // Emit the finished record.
5213 Stream.EmitRecord(Code: bitc::FS_COMBINED_PROFILE, Vals: NameVals,
5214 Abbrev: FSCallsProfileAbbrev);
5215 NameVals.clear();
5216
5217 writeFunctionHeapProfileRecords(
5218 Stream, FS, CallsiteAbbrev, AllocAbbrev, /*ContextIdAbbvId*/ 0,
5219 /*PerModule*/ false,
5220 /*GetValueId*/
5221 GetValueID: [&](const ValueInfo &VI) -> unsigned {
5222 std::optional<unsigned> ValueID = GetValueId(VI);
5223 // This can happen in shared index files for distributed ThinLTO if
5224 // the callee function summary is not included. Record 0 which we
5225 // will have to deal with conservatively when doing any kind of
5226 // validation in the ThinLTO backends.
5227 if (!ValueID)
5228 return 0;
5229 return *ValueID;
5230 },
5231 /*GetStackIndex*/
5232 [&](unsigned I) {
5233 // Get the corresponding index into the list of StackIds actually
5234 // being written for this combined index (which may be a subset in
5235 // the case of distributed indexes).
5236 assert(StackIdIndicesToIndex.contains(I));
5237 return StackIdIndicesToIndex[I];
5238 },
5239 /*WriteContextSizeInfoIndex*/ false, CallStackPos, CallStackCount);
5240
5241 MaybeEmitOriginalName(*S);
5242 });
5243
5244 for (auto *AS : Aliases) {
5245 auto AliasValueId = SummaryToValueIdMap[AS];
5246 assert(AliasValueId);
5247 NameVals.push_back(Elt: AliasValueId);
5248 assert(ModuleIdMap.count(AS->modulePath()));
5249 NameVals.push_back(Elt: ModuleIdMap[AS->modulePath()]);
5250 NameVals.push_back(
5251 Elt: getEncodedGVSummaryFlags(Flags: AS->flags(), ImportAsDecl: shouldImportValueAsDecl(AS)));
5252 // Set value id to 0 when an alias is imported but the aliasee summary is
5253 // not contained in the index.
5254 auto AliaseeValueId =
5255 AS->hasAliasee() ? SummaryToValueIdMap[&AS->getAliasee()] : 0;
5256 NameVals.push_back(Elt: AliaseeValueId);
5257
5258 // Emit the finished record.
5259 Stream.EmitRecord(Code: bitc::FS_COMBINED_ALIAS, Vals: NameVals, Abbrev: FSAliasAbbrev);
5260 NameVals.clear();
5261 MaybeEmitOriginalName(*AS);
5262
5263 if (AS->hasAliasee())
5264 if (auto *FS = dyn_cast<FunctionSummary>(Val: &AS->getAliasee()))
5265 getReferencedTypeIds(FS, ReferencedTypeIds);
5266 }
5267
5268 SmallVector<StringRef, 4> Functions;
5269 auto EmitCfiFunctions = [&](const CfiFunctionIndex &CfiIndex,
5270 bitc::GlobalValueSummarySymtabCodes Code) {
5271 if (CfiIndex.empty())
5272 return;
5273 for (GlobalValue::GUID GUID : DefOrUseGUIDs) {
5274 auto Defs = CfiIndex.forGuid(GUID);
5275 llvm::append_range(C&: Functions, R&: Defs);
5276 }
5277 if (Functions.empty())
5278 return;
5279 llvm::sort(C&: Functions);
5280 for (const auto &S : Functions) {
5281 NameVals.push_back(Elt: StrtabBuilder.add(S));
5282 NameVals.push_back(Elt: S.size());
5283 }
5284 Stream.EmitRecord(Code, Vals: NameVals);
5285 NameVals.clear();
5286 Functions.clear();
5287 };
5288
5289 EmitCfiFunctions(Index.cfiFunctionDefs(), bitc::FS_CFI_FUNCTION_DEFS);
5290 EmitCfiFunctions(Index.cfiFunctionDecls(), bitc::FS_CFI_FUNCTION_DECLS);
5291
5292 // Walk the GUIDs that were referenced, and write the
5293 // corresponding type id records.
5294 for (auto &T : ReferencedTypeIds) {
5295 auto TidIter = Index.typeIds().equal_range(x: T);
5296 for (const auto &[GUID, TypeIdPair] : make_range(p: TidIter)) {
5297 writeTypeIdSummaryRecord(NameVals, StrtabBuilder, Id: TypeIdPair.first,
5298 Summary: TypeIdPair.second);
5299 Stream.EmitRecord(Code: bitc::FS_TYPE_ID, Vals: NameVals);
5300 NameVals.clear();
5301 }
5302 }
5303
5304 if (Index.getBlockCount())
5305 Stream.EmitRecord(Code: bitc::FS_BLOCK_COUNT,
5306 Vals: ArrayRef<uint64_t>{Index.getBlockCount()});
5307
5308 Stream.ExitBlock();
5309}
5310
5311/// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the
5312/// current llvm version, and a record for the epoch number.
5313static void writeIdentificationBlock(BitstreamWriter &Stream) {
5314 Stream.EnterSubblock(BlockID: bitc::IDENTIFICATION_BLOCK_ID, CodeLen: 5);
5315
5316 // Write the "user readable" string identifying the bitcode producer
5317 auto Abbv = std::make_shared<BitCodeAbbrev>();
5318 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_STRING));
5319 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5320 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
5321 auto StringAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5322 writeStringRecord(Stream, Code: bitc::IDENTIFICATION_CODE_STRING,
5323 Str: "LLVM" LLVM_VERSION_STRING, AbbrevToUse: StringAbbrev);
5324
5325 // Write the epoch version
5326 Abbv = std::make_shared<BitCodeAbbrev>();
5327 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_EPOCH));
5328 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
5329 auto EpochAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5330 constexpr std::array<unsigned, 1> Vals = {._M_elems: {bitc::BITCODE_CURRENT_EPOCH}};
5331 Stream.EmitRecord(Code: bitc::IDENTIFICATION_CODE_EPOCH, Vals, Abbrev: EpochAbbrev);
5332 Stream.ExitBlock();
5333}
5334
5335void ModuleBitcodeWriter::writeModuleHash(StringRef View) {
5336 // Emit the module's hash.
5337 // MODULE_CODE_HASH: [5*i32]
5338 if (GenerateHash) {
5339 uint32_t Vals[5];
5340 Hasher.update(Data: ArrayRef<uint8_t>(
5341 reinterpret_cast<const uint8_t *>(View.data()), View.size()));
5342 std::array<uint8_t, 20> Hash = Hasher.result();
5343 for (int Pos = 0; Pos < 20; Pos += 4) {
5344 Vals[Pos / 4] = support::endian::read32be(P: Hash.data() + Pos);
5345 }
5346
5347 // Emit the finished record.
5348 Stream.EmitRecord(Code: bitc::MODULE_CODE_HASH, Vals);
5349
5350 if (ModHash)
5351 // Save the written hash value.
5352 llvm::copy(Range&: Vals, Out: std::begin(cont&: *ModHash));
5353 }
5354}
5355
5356void ModuleBitcodeWriter::write() {
5357 writeIdentificationBlock(Stream);
5358
5359 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5360 // We will want to write the module hash at this point. Block any flushing so
5361 // we can have access to the whole underlying data later.
5362 Stream.markAndBlockFlushing();
5363
5364 writeModuleVersion();
5365
5366 // Emit blockinfo, which defines the standard abbreviations etc.
5367 writeBlockInfo();
5368
5369 // Emit information describing all of the types in the module.
5370 writeTypeTable();
5371
5372 // Emit information about attribute groups.
5373 writeAttributeGroupTable();
5374
5375 // Emit information about parameter attributes.
5376 writeAttributeTable();
5377
5378 writeComdats();
5379
5380 // Emit top-level description of module, including target triple, inline asm,
5381 // descriptors for global variables, and function prototype info.
5382 writeModuleInfo();
5383
5384 // Emit constants.
5385 writeModuleConstants();
5386
5387 // Emit metadata kind names.
5388 writeModuleMetadataKinds();
5389
5390 // Emit metadata.
5391 writeModuleMetadata();
5392
5393 // Emit module-level use-lists.
5394 if (VE.shouldPreserveUseListOrder())
5395 writeUseListBlock(F: nullptr);
5396
5397 writeOperandBundleTags();
5398 writeSyncScopeNames();
5399
5400 // Emit function bodies.
5401 DenseMap<const Function *, uint64_t> FunctionToBitcodeIndex;
5402 for (const Function &F : M)
5403 if (!F.isDeclaration())
5404 writeFunction(F, FunctionToBitcodeIndex);
5405
5406 // Need to write after the above call to WriteFunction which populates
5407 // the summary information in the index.
5408 if (Index)
5409 writePerModuleGlobalValueSummary();
5410
5411 writeGlobalValueSymbolTable(FunctionToBitcodeIndex);
5412
5413 writeModuleHash(View: Stream.getMarkedBufferAndResumeFlushing());
5414
5415 Stream.ExitBlock();
5416}
5417
5418static void writeInt32ToBuffer(uint32_t Value, SmallVectorImpl<char> &Buffer,
5419 uint32_t &Position) {
5420 support::endian::write32le(P: &Buffer[Position], V: Value);
5421 Position += 4;
5422}
5423
5424/// If generating a bc file on darwin, we have to emit a
5425/// header and trailer to make it compatible with the system archiver. To do
5426/// this we emit the following header, and then emit a trailer that pads the
5427/// file out to be a multiple of 16 bytes.
5428///
5429/// struct bc_header {
5430/// uint32_t Magic; // 0x0B17C0DE
5431/// uint32_t Version; // Version, currently always 0.
5432/// uint32_t BitcodeOffset; // Offset to traditional bitcode file.
5433/// uint32_t BitcodeSize; // Size of traditional bitcode file.
5434/// uint32_t CPUType; // CPU specifier.
5435/// ... potentially more later ...
5436/// };
5437static void emitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer,
5438 const Triple &TT) {
5439 unsigned CPUType = ~0U;
5440
5441 // Match x86_64-*, i[3-9]86-*, powerpc-*, powerpc64-*, arm-*, thumb-*,
5442 // armv[0-9]-*, thumbv[0-9]-*, armv5te-*, or armv6t2-*. The CPUType is a magic
5443 // number from /usr/include/mach/machine.h. It is ok to reproduce the
5444 // specific constants here because they are implicitly part of the Darwin ABI.
5445 enum {
5446 DARWIN_CPU_ARCH_ABI64 = 0x01000000,
5447 DARWIN_CPU_TYPE_X86 = 7,
5448 DARWIN_CPU_TYPE_ARM = 12,
5449 DARWIN_CPU_TYPE_POWERPC = 18
5450 };
5451
5452 Triple::ArchType Arch = TT.getArch();
5453 if (Arch == Triple::x86_64)
5454 CPUType = DARWIN_CPU_TYPE_X86 | DARWIN_CPU_ARCH_ABI64;
5455 else if (Arch == Triple::x86)
5456 CPUType = DARWIN_CPU_TYPE_X86;
5457 else if (Arch == Triple::ppc)
5458 CPUType = DARWIN_CPU_TYPE_POWERPC;
5459 else if (Arch == Triple::ppc64)
5460 CPUType = DARWIN_CPU_TYPE_POWERPC | DARWIN_CPU_ARCH_ABI64;
5461 else if (Arch == Triple::arm || Arch == Triple::thumb)
5462 CPUType = DARWIN_CPU_TYPE_ARM;
5463
5464 // Traditional Bitcode starts after header.
5465 assert(Buffer.size() >= BWH_HeaderSize &&
5466 "Expected header size to be reserved");
5467 unsigned BCOffset = BWH_HeaderSize;
5468 unsigned BCSize = Buffer.size() - BWH_HeaderSize;
5469
5470 // Write the magic and version.
5471 unsigned Position = 0;
5472 writeInt32ToBuffer(Value: 0x0B17C0DE, Buffer, Position);
5473 writeInt32ToBuffer(Value: 0, Buffer, Position); // Version.
5474 writeInt32ToBuffer(Value: BCOffset, Buffer, Position);
5475 writeInt32ToBuffer(Value: BCSize, Buffer, Position);
5476 writeInt32ToBuffer(Value: CPUType, Buffer, Position);
5477
5478 // If the file is not a multiple of 16 bytes, insert dummy padding.
5479 while (Buffer.size() & 15)
5480 Buffer.push_back(Elt: 0);
5481}
5482
5483/// Helper to write the header common to all bitcode files.
5484static void writeBitcodeHeader(BitstreamWriter &Stream) {
5485 // Emit the file header.
5486 Stream.Emit(Val: (unsigned)'B', NumBits: 8);
5487 Stream.Emit(Val: (unsigned)'C', NumBits: 8);
5488 Stream.Emit(Val: 0x0, NumBits: 4);
5489 Stream.Emit(Val: 0xC, NumBits: 4);
5490 Stream.Emit(Val: 0xE, NumBits: 4);
5491 Stream.Emit(Val: 0xD, NumBits: 4);
5492}
5493
5494BitcodeWriter::BitcodeWriter(SmallVectorImpl<char> &Buffer)
5495 : Stream(new BitstreamWriter(Buffer)) {
5496 writeBitcodeHeader(Stream&: *Stream);
5497}
5498
5499BitcodeWriter::BitcodeWriter(raw_ostream &FS)
5500 : Stream(new BitstreamWriter(FS, FlushThreshold)) {
5501 writeBitcodeHeader(Stream&: *Stream);
5502}
5503
5504BitcodeWriter::~BitcodeWriter() { assert(WroteStrtab); }
5505
5506void BitcodeWriter::writeBlob(unsigned Block, unsigned Record, StringRef Blob) {
5507 Stream->EnterSubblock(BlockID: Block, CodeLen: 3);
5508
5509 auto Abbv = std::make_shared<BitCodeAbbrev>();
5510 Abbv->Add(OpInfo: BitCodeAbbrevOp(Record));
5511 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
5512 auto AbbrevNo = Stream->EmitAbbrev(Abbv: std::move(Abbv));
5513
5514 Stream->EmitRecordWithBlob(Abbrev: AbbrevNo, Vals: ArrayRef<uint64_t>{Record}, Blob);
5515
5516 Stream->ExitBlock();
5517}
5518
5519void BitcodeWriter::writeSymtab() {
5520 assert(!WroteStrtab && !WroteSymtab);
5521
5522 // If any module has module-level inline asm, we will require a registered asm
5523 // parser for the target so that we can create an accurate symbol table for
5524 // the module.
5525 for (Module *M : Mods) {
5526 if (M->getModuleInlineAsm().empty())
5527 continue;
5528
5529 std::string Err;
5530 const Triple TT(M->getTargetTriple());
5531 const Target *T = TargetRegistry::lookupTarget(TheTriple: TT, Error&: Err);
5532 if (!T || !T->hasMCAsmParser())
5533 return;
5534 }
5535
5536 WroteSymtab = true;
5537 SmallVector<char, 0> Symtab;
5538 // The irsymtab::build function may be unable to create a symbol table if the
5539 // module is malformed (e.g. it contains an invalid alias). Writing a symbol
5540 // table is not required for correctness, but we still want to be able to
5541 // write malformed modules to bitcode files, so swallow the error.
5542 if (Error E = irsymtab::build(Mods, Symtab, StrtabBuilder, Alloc)) {
5543 consumeError(Err: std::move(E));
5544 return;
5545 }
5546
5547 writeBlob(Block: bitc::SYMTAB_BLOCK_ID, Record: bitc::SYMTAB_BLOB,
5548 Blob: {Symtab.data(), Symtab.size()});
5549}
5550
5551void BitcodeWriter::writeStrtab() {
5552 assert(!WroteStrtab);
5553
5554 std::vector<char> Strtab;
5555 StrtabBuilder.finalizeInOrder();
5556 Strtab.resize(new_size: StrtabBuilder.getSize());
5557 StrtabBuilder.write(Buf: (uint8_t *)Strtab.data());
5558
5559 writeBlob(Block: bitc::STRTAB_BLOCK_ID, Record: bitc::STRTAB_BLOB,
5560 Blob: {Strtab.data(), Strtab.size()});
5561
5562 WroteStrtab = true;
5563}
5564
5565void BitcodeWriter::copyStrtab(StringRef Strtab) {
5566 writeBlob(Block: bitc::STRTAB_BLOCK_ID, Record: bitc::STRTAB_BLOB, Blob: Strtab);
5567 WroteStrtab = true;
5568}
5569
5570void BitcodeWriter::writeModule(const Module &M,
5571 bool ShouldPreserveUseListOrder,
5572 const ModuleSummaryIndex *Index,
5573 bool GenerateHash, ModuleHash *ModHash) {
5574 assert(!WroteStrtab);
5575
5576 // The Mods vector is used by irsymtab::build, which requires non-const
5577 // Modules in case it needs to materialize metadata. But the bitcode writer
5578 // requires that the module is materialized, so we can cast to non-const here,
5579 // after checking that it is in fact materialized.
5580 assert(M.isMaterialized());
5581 Mods.push_back(x: const_cast<Module *>(&M));
5582
5583 ModuleBitcodeWriter ModuleWriter(M, StrtabBuilder, *Stream,
5584 ShouldPreserveUseListOrder, Index,
5585 GenerateHash, ModHash);
5586 ModuleWriter.write();
5587}
5588
5589void BitcodeWriter::writeIndex(
5590 const ModuleSummaryIndex *Index,
5591 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex,
5592 const GVSummaryPtrSet *DecSummaries) {
5593 IndexBitcodeWriter IndexWriter(*Stream, StrtabBuilder, *Index, DecSummaries,
5594 ModuleToSummariesForIndex);
5595 IndexWriter.write();
5596}
5597
5598/// Write the specified module to the specified output stream.
5599void llvm::WriteBitcodeToFile(const Module &M, raw_ostream &Out,
5600 bool ShouldPreserveUseListOrder,
5601 const ModuleSummaryIndex *Index,
5602 bool GenerateHash, ModuleHash *ModHash) {
5603 auto Write = [&](BitcodeWriter &Writer) {
5604 Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash,
5605 ModHash);
5606 Writer.writeSymtab();
5607 Writer.writeStrtab();
5608 };
5609 Triple TT(M.getTargetTriple());
5610 if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) {
5611 // If this is darwin or another generic macho target, reserve space for the
5612 // header. Note that the header is computed *after* the output is known, so
5613 // we currently explicitly use a buffer, write to it, and then subsequently
5614 // flush to Out.
5615 SmallVector<char, 0> Buffer;
5616 Buffer.reserve(N: 256 * 1024);
5617 Buffer.insert(I: Buffer.begin(), NumToInsert: BWH_HeaderSize, Elt: 0);
5618 BitcodeWriter Writer(Buffer);
5619 Write(Writer);
5620 emitDarwinBCHeaderAndTrailer(Buffer, TT);
5621 Out.write(Ptr: Buffer.data(), Size: Buffer.size());
5622 } else {
5623 BitcodeWriter Writer(Out);
5624 Write(Writer);
5625 }
5626}
5627
5628void IndexBitcodeWriter::write() {
5629 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5630
5631 writeModuleVersion();
5632
5633 // Write the module paths in the combined index.
5634 writeModStrings();
5635
5636 // Write the summary combined index records.
5637 writeCombinedGlobalValueSummary();
5638
5639 Stream.ExitBlock();
5640}
5641
5642// Write the specified module summary index to the given raw output stream,
5643// where it will be written in a new bitcode block. This is used when
5644// writing the combined index file for ThinLTO. When writing a subset of the
5645// index for a distributed backend, provide a \p ModuleToSummariesForIndex map.
5646void llvm::writeIndexToFile(
5647 const ModuleSummaryIndex &Index, raw_ostream &Out,
5648 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex,
5649 const GVSummaryPtrSet *DecSummaries) {
5650 SmallVector<char, 0> Buffer;
5651 Buffer.reserve(N: 256 * 1024);
5652
5653 BitcodeWriter Writer(Buffer);
5654 Writer.writeIndex(Index: &Index, ModuleToSummariesForIndex, DecSummaries);
5655 Writer.writeStrtab();
5656
5657 Out.write(Ptr: (char *)&Buffer.front(), Size: Buffer.size());
5658}
5659
5660namespace {
5661
5662/// Class to manage the bitcode writing for a thin link bitcode file.
5663class ThinLinkBitcodeWriter : public ModuleBitcodeWriterBase {
5664 /// ModHash is for use in ThinLTO incremental build, generated while writing
5665 /// the module bitcode file.
5666 const ModuleHash *ModHash;
5667
5668public:
5669 ThinLinkBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
5670 BitstreamWriter &Stream,
5671 const ModuleSummaryIndex &Index,
5672 const ModuleHash &ModHash)
5673 : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
5674 /*ShouldPreserveUseListOrder=*/false, &Index),
5675 ModHash(&ModHash) {}
5676
5677 void write();
5678
5679private:
5680 void writeSimplifiedModuleInfo();
5681};
5682
5683} // end anonymous namespace
5684
5685// This function writes a simpilified module info for thin link bitcode file.
5686// It only contains the source file name along with the name(the offset and
5687// size in strtab) and linkage for global values. For the global value info
5688// entry, in order to keep linkage at offset 5, there are three zeros used
5689// as padding.
5690void ThinLinkBitcodeWriter::writeSimplifiedModuleInfo() {
5691 SmallVector<unsigned, 64> Vals;
5692 // Emit the module's source file name.
5693 {
5694 StringEncoding Bits = getStringEncoding(Str: M.getSourceFileName());
5695 BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
5696 if (Bits == SE_Char6)
5697 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
5698 else if (Bits == SE_Fixed7)
5699 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
5700
5701 // MODULE_CODE_SOURCE_FILENAME: [namechar x N]
5702 auto Abbv = std::make_shared<BitCodeAbbrev>();
5703 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
5704 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5705 Abbv->Add(OpInfo: AbbrevOpToUse);
5706 unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5707
5708 for (const auto P : M.getSourceFileName())
5709 Vals.push_back(Elt: (unsigned char)P);
5710
5711 Stream.EmitRecord(Code: bitc::MODULE_CODE_SOURCE_FILENAME, Vals, Abbrev: FilenameAbbrev);
5712 Vals.clear();
5713 }
5714
5715 // Emit the global variable information.
5716 for (const GlobalVariable &GV : M.globals()) {
5717 // GLOBALVAR: [strtab offset, strtab size, 0, 0, 0, linkage]
5718 Vals.push_back(Elt: StrtabBuilder.add(S: GV.getName()));
5719 Vals.push_back(Elt: GV.getName().size());
5720 Vals.push_back(Elt: 0);
5721 Vals.push_back(Elt: 0);
5722 Vals.push_back(Elt: 0);
5723 Vals.push_back(Elt: getEncodedLinkage(GV));
5724
5725 Stream.EmitRecord(Code: bitc::MODULE_CODE_GLOBALVAR, Vals);
5726 Vals.clear();
5727 }
5728
5729 // Emit the function proto information.
5730 for (const Function &F : M) {
5731 // FUNCTION: [strtab offset, strtab size, 0, 0, 0, linkage]
5732 Vals.push_back(Elt: StrtabBuilder.add(S: F.getName()));
5733 Vals.push_back(Elt: F.getName().size());
5734 Vals.push_back(Elt: 0);
5735 Vals.push_back(Elt: 0);
5736 Vals.push_back(Elt: 0);
5737 Vals.push_back(Elt: getEncodedLinkage(GV: F));
5738
5739 Stream.EmitRecord(Code: bitc::MODULE_CODE_FUNCTION, Vals);
5740 Vals.clear();
5741 }
5742
5743 // Emit the alias information.
5744 for (const GlobalAlias &A : M.aliases()) {
5745 // ALIAS: [strtab offset, strtab size, 0, 0, 0, linkage]
5746 Vals.push_back(Elt: StrtabBuilder.add(S: A.getName()));
5747 Vals.push_back(Elt: A.getName().size());
5748 Vals.push_back(Elt: 0);
5749 Vals.push_back(Elt: 0);
5750 Vals.push_back(Elt: 0);
5751 Vals.push_back(Elt: getEncodedLinkage(GV: A));
5752
5753 Stream.EmitRecord(Code: bitc::MODULE_CODE_ALIAS, Vals);
5754 Vals.clear();
5755 }
5756
5757 // Emit the ifunc information.
5758 for (const GlobalIFunc &I : M.ifuncs()) {
5759 // IFUNC: [strtab offset, strtab size, 0, 0, 0, linkage]
5760 Vals.push_back(Elt: StrtabBuilder.add(S: I.getName()));
5761 Vals.push_back(Elt: I.getName().size());
5762 Vals.push_back(Elt: 0);
5763 Vals.push_back(Elt: 0);
5764 Vals.push_back(Elt: 0);
5765 Vals.push_back(Elt: getEncodedLinkage(GV: I));
5766
5767 Stream.EmitRecord(Code: bitc::MODULE_CODE_IFUNC, Vals);
5768 Vals.clear();
5769 }
5770}
5771
5772void ThinLinkBitcodeWriter::write() {
5773 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5774
5775 writeModuleVersion();
5776
5777 writeSimplifiedModuleInfo();
5778
5779 writePerModuleGlobalValueSummary();
5780
5781 // Write module hash.
5782 Stream.EmitRecord(Code: bitc::MODULE_CODE_HASH, Vals: ArrayRef<uint32_t>(*ModHash));
5783
5784 Stream.ExitBlock();
5785}
5786
5787void BitcodeWriter::writeThinLinkBitcode(const Module &M,
5788 const ModuleSummaryIndex &Index,
5789 const ModuleHash &ModHash) {
5790 assert(!WroteStrtab);
5791
5792 // The Mods vector is used by irsymtab::build, which requires non-const
5793 // Modules in case it needs to materialize metadata. But the bitcode writer
5794 // requires that the module is materialized, so we can cast to non-const here,
5795 // after checking that it is in fact materialized.
5796 assert(M.isMaterialized());
5797 Mods.push_back(x: const_cast<Module *>(&M));
5798
5799 ThinLinkBitcodeWriter ThinLinkWriter(M, StrtabBuilder, *Stream, Index,
5800 ModHash);
5801 ThinLinkWriter.write();
5802}
5803
5804// Write the specified thin link bitcode file to the given raw output stream,
5805// where it will be written in a new bitcode block. This is used when
5806// writing the per-module index file for ThinLTO.
5807void llvm::writeThinLinkBitcodeToFile(const Module &M, raw_ostream &Out,
5808 const ModuleSummaryIndex &Index,
5809 const ModuleHash &ModHash) {
5810 SmallVector<char, 0> Buffer;
5811 Buffer.reserve(N: 256 * 1024);
5812
5813 BitcodeWriter Writer(Buffer);
5814 Writer.writeThinLinkBitcode(M, Index, ModHash);
5815 Writer.writeSymtab();
5816 Writer.writeStrtab();
5817
5818 Out.write(Ptr: (char *)&Buffer.front(), Size: Buffer.size());
5819}
5820
5821static const char *getSectionNameForBitcode(const Triple &T) {
5822 switch (T.getObjectFormat()) {
5823 case Triple::MachO:
5824 return "__LLVM,__bitcode";
5825 case Triple::COFF:
5826 case Triple::ELF:
5827 case Triple::Wasm:
5828 case Triple::UnknownObjectFormat:
5829 return ".llvmbc";
5830 case Triple::GOFF:
5831 llvm_unreachable("GOFF is not yet implemented");
5832 break;
5833 case Triple::SPIRV:
5834 if (T.getVendor() == Triple::AMD)
5835 return ".llvmbc";
5836 llvm_unreachable("SPIRV is not yet implemented");
5837 break;
5838 case Triple::XCOFF:
5839 llvm_unreachable("XCOFF is not yet implemented");
5840 break;
5841 case Triple::DXContainer:
5842 llvm_unreachable("DXContainer is not yet implemented");
5843 break;
5844 }
5845 llvm_unreachable("Unimplemented ObjectFormatType");
5846}
5847
5848static const char *getSectionNameForCommandline(const Triple &T) {
5849 switch (T.getObjectFormat()) {
5850 case Triple::MachO:
5851 return "__LLVM,__cmdline";
5852 case Triple::COFF:
5853 case Triple::ELF:
5854 case Triple::Wasm:
5855 case Triple::UnknownObjectFormat:
5856 return ".llvmcmd";
5857 case Triple::GOFF:
5858 llvm_unreachable("GOFF is not yet implemented");
5859 break;
5860 case Triple::SPIRV:
5861 if (T.getVendor() == Triple::AMD)
5862 return ".llvmcmd";
5863 llvm_unreachable("SPIRV is not yet implemented");
5864 break;
5865 case Triple::XCOFF:
5866 llvm_unreachable("XCOFF is not yet implemented");
5867 break;
5868 case Triple::DXContainer:
5869 llvm_unreachable("DXC is not yet implemented");
5870 break;
5871 }
5872 llvm_unreachable("Unimplemented ObjectFormatType");
5873}
5874
5875void llvm::embedBitcodeInModule(llvm::Module &M, llvm::MemoryBufferRef Buf,
5876 bool EmbedBitcode, bool EmbedCmdline,
5877 const std::vector<uint8_t> &CmdArgs) {
5878 // Save llvm.compiler.used and remove it.
5879 SmallVector<Constant *, 2> UsedArray;
5880 SmallVector<GlobalValue *, 4> UsedGlobals;
5881 GlobalVariable *Used = collectUsedGlobalVariables(M, Vec&: UsedGlobals, CompilerUsed: true);
5882 Type *UsedElementType = Used ? Used->getValueType()->getArrayElementType()
5883 : PointerType::getUnqual(C&: M.getContext());
5884 for (auto *GV : UsedGlobals) {
5885 if (GV->getName() != "llvm.embedded.module" &&
5886 GV->getName() != "llvm.cmdline")
5887 UsedArray.push_back(
5888 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
5889 }
5890 if (Used)
5891 Used->eraseFromParent();
5892
5893 // Embed the bitcode for the llvm module.
5894 std::string Data;
5895 ArrayRef<uint8_t> ModuleData;
5896 Triple T(M.getTargetTriple());
5897
5898 if (EmbedBitcode) {
5899 if (Buf.getBufferSize() == 0 ||
5900 !isBitcode(BufPtr: (const unsigned char *)Buf.getBufferStart(),
5901 BufEnd: (const unsigned char *)Buf.getBufferEnd())) {
5902 // If the input is LLVM Assembly, bitcode is produced by serializing
5903 // the module. Use-lists order need to be preserved in this case.
5904 llvm::raw_string_ostream OS(Data);
5905 llvm::WriteBitcodeToFile(M, Out&: OS, /* ShouldPreserveUseListOrder */ true);
5906 ModuleData =
5907 ArrayRef<uint8_t>((const uint8_t *)OS.str().data(), OS.str().size());
5908 } else
5909 // If the input is LLVM bitcode, write the input byte stream directly.
5910 ModuleData = ArrayRef<uint8_t>((const uint8_t *)Buf.getBufferStart(),
5911 Buf.getBufferSize());
5912 }
5913 llvm::Constant *ModuleConstant =
5914 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts: ModuleData);
5915 llvm::GlobalVariable *GV = new llvm::GlobalVariable(
5916 M, ModuleConstant->getType(), true, llvm::GlobalValue::PrivateLinkage,
5917 ModuleConstant);
5918 GV->setSection(getSectionNameForBitcode(T));
5919 // Set alignment to 1 to prevent padding between two contributions from input
5920 // sections after linking.
5921 GV->setAlignment(Align(1));
5922 UsedArray.push_back(
5923 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
5924 if (llvm::GlobalVariable *Old =
5925 M.getGlobalVariable(Name: "llvm.embedded.module", AllowInternal: true)) {
5926 assert(Old->hasZeroLiveUses() &&
5927 "llvm.embedded.module can only be used once in llvm.compiler.used");
5928 GV->takeName(V: Old);
5929 Old->eraseFromParent();
5930 } else {
5931 GV->setName("llvm.embedded.module");
5932 }
5933
5934 // Skip if only bitcode needs to be embedded.
5935 if (EmbedCmdline) {
5936 // Embed command-line options.
5937 ArrayRef<uint8_t> CmdData(const_cast<uint8_t *>(CmdArgs.data()),
5938 CmdArgs.size());
5939 llvm::Constant *CmdConstant =
5940 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts: CmdData);
5941 GV = new llvm::GlobalVariable(M, CmdConstant->getType(), true,
5942 llvm::GlobalValue::PrivateLinkage,
5943 CmdConstant);
5944 GV->setSection(getSectionNameForCommandline(T));
5945 GV->setAlignment(Align(1));
5946 UsedArray.push_back(
5947 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
5948 if (llvm::GlobalVariable *Old = M.getGlobalVariable(Name: "llvm.cmdline", AllowInternal: true)) {
5949 assert(Old->hasZeroLiveUses() &&
5950 "llvm.cmdline can only be used once in llvm.compiler.used");
5951 GV->takeName(V: Old);
5952 Old->eraseFromParent();
5953 } else {
5954 GV->setName("llvm.cmdline");
5955 }
5956 }
5957
5958 if (UsedArray.empty())
5959 return;
5960
5961 // Recreate llvm.compiler.used.
5962 ArrayType *ATy = ArrayType::get(ElementType: UsedElementType, NumElements: UsedArray.size());
5963 auto *NewUsed = new GlobalVariable(
5964 M, ATy, false, llvm::GlobalValue::AppendingLinkage,
5965 llvm::ConstantArray::get(T: ATy, V: UsedArray), "llvm.compiler.used");
5966 NewUsed->setSection("llvm.metadata");
5967}
5968