1//===- Bitcode/Writer/BitcodeWriter.cpp - Bitcode Writer ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Bitcode writer implementation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Bitcode/BitcodeWriter.h"
14#include "ValueEnumerator.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/DenseMap.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SetVector.h"
21#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/SmallString.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringMap.h"
25#include "llvm/ADT/StringRef.h"
26#include "llvm/Analysis/MemoryProfileInfo.h"
27#include "llvm/BinaryFormat/Dwarf.h"
28#include "llvm/Bitcode/BitcodeCommon.h"
29#include "llvm/Bitcode/BitcodeReader.h"
30#include "llvm/Bitcode/LLVMBitCodes.h"
31#include "llvm/Bitstream/BitCodes.h"
32#include "llvm/Bitstream/BitstreamWriter.h"
33#include "llvm/Config/llvm-config.h"
34#include "llvm/IR/Attributes.h"
35#include "llvm/IR/BasicBlock.h"
36#include "llvm/IR/Comdat.h"
37#include "llvm/IR/Constant.h"
38#include "llvm/IR/ConstantRangeList.h"
39#include "llvm/IR/Constants.h"
40#include "llvm/IR/DebugInfoMetadata.h"
41#include "llvm/IR/DebugLoc.h"
42#include "llvm/IR/DerivedTypes.h"
43#include "llvm/IR/Function.h"
44#include "llvm/IR/GlobalAlias.h"
45#include "llvm/IR/GlobalIFunc.h"
46#include "llvm/IR/GlobalObject.h"
47#include "llvm/IR/GlobalValue.h"
48#include "llvm/IR/GlobalVariable.h"
49#include "llvm/IR/InlineAsm.h"
50#include "llvm/IR/InstrTypes.h"
51#include "llvm/IR/Instruction.h"
52#include "llvm/IR/Instructions.h"
53#include "llvm/IR/LLVMContext.h"
54#include "llvm/IR/Metadata.h"
55#include "llvm/IR/Module.h"
56#include "llvm/IR/ModuleSummaryIndex.h"
57#include "llvm/IR/Operator.h"
58#include "llvm/IR/Type.h"
59#include "llvm/IR/UseListOrder.h"
60#include "llvm/IR/Value.h"
61#include "llvm/IR/ValueSymbolTable.h"
62#include "llvm/MC/StringTableBuilder.h"
63#include "llvm/MC/TargetRegistry.h"
64#include "llvm/Object/IRSymtab.h"
65#include "llvm/ProfileData/MemProf.h"
66#include "llvm/ProfileData/MemProfRadixTree.h"
67#include "llvm/Support/AtomicOrdering.h"
68#include "llvm/Support/Casting.h"
69#include "llvm/Support/CommandLine.h"
70#include "llvm/Support/Compiler.h"
71#include "llvm/Support/Endian.h"
72#include "llvm/Support/Error.h"
73#include "llvm/Support/ErrorHandling.h"
74#include "llvm/Support/MathExtras.h"
75#include "llvm/Support/SHA1.h"
76#include "llvm/Support/raw_ostream.h"
77#include "llvm/TargetParser/Triple.h"
78#include <algorithm>
79#include <cassert>
80#include <cstddef>
81#include <cstdint>
82#include <iterator>
83#include <map>
84#include <memory>
85#include <optional>
86#include <string>
87#include <utility>
88#include <vector>
89
90using namespace llvm;
91using namespace llvm::memprof;
92
93static cl::opt<unsigned>
94 IndexThreshold("bitcode-mdindex-threshold", cl::Hidden, cl::init(Val: 25),
95 cl::desc("Number of metadatas above which we emit an index "
96 "to enable lazy-loading"));
97static cl::opt<uint32_t> FlushThreshold(
98 "bitcode-flush-threshold", cl::Hidden, cl::init(Val: 512),
99 cl::desc("The threshold (unit M) for flushing LLVM bitcode."));
100
101// Since we only use the context information in the memprof summary records in
102// the LTO backends to do assertion checking, save time and space by only
103// serializing the context for non-NDEBUG builds.
104// TODO: Currently this controls writing context of the allocation info records,
105// which are larger and more expensive, but we should do this for the callsite
106// records as well.
107// FIXME: Convert to a const once this has undergone more sigificant testing.
108static cl::opt<bool>
109 CombinedIndexMemProfContext("combined-index-memprof-context", cl::Hidden,
110#ifdef NDEBUG
111 cl::init(Val: false),
112#else
113 cl::init(true),
114#endif
115 cl::desc(""));
116
117static cl::opt<bool> PreserveBitcodeUseListOrder(
118 "preserve-bc-uselistorder", cl::Hidden, cl::init(Val: true),
119 cl::desc("Preserve use-list order when writing LLVM bitcode."));
120
121namespace llvm {
122extern FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold;
123}
124
125namespace {
126
127/// These are manifest constants used by the bitcode writer. They do not need to
128/// be kept in sync with the reader, but need to be consistent within this file.
129enum {
130 // VALUE_SYMTAB_BLOCK abbrev id's.
131 VST_ENTRY_8_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
132 VST_ENTRY_7_ABBREV,
133 VST_ENTRY_6_ABBREV,
134 VST_BBENTRY_6_ABBREV,
135
136 // CONSTANTS_BLOCK abbrev id's.
137 CONSTANTS_SETTYPE_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
138 CONSTANTS_INTEGER_ABBREV,
139 CONSTANTS_CE_CAST_Abbrev,
140 CONSTANTS_NULL_Abbrev,
141
142 // FUNCTION_BLOCK abbrev id's.
143 FUNCTION_INST_LOAD_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
144 FUNCTION_INST_STORE_ABBREV,
145 FUNCTION_INST_UNOP_ABBREV,
146 FUNCTION_INST_UNOP_FLAGS_ABBREV,
147 FUNCTION_INST_BINOP_ABBREV,
148 FUNCTION_INST_BINOP_FLAGS_ABBREV,
149 FUNCTION_INST_CAST_ABBREV,
150 FUNCTION_INST_CAST_FLAGS_ABBREV,
151 FUNCTION_INST_RET_VOID_ABBREV,
152 FUNCTION_INST_RET_VAL_ABBREV,
153 FUNCTION_INST_BR_UNCOND_ABBREV,
154 FUNCTION_INST_BR_COND_ABBREV,
155 FUNCTION_INST_UNREACHABLE_ABBREV,
156 FUNCTION_INST_GEP_ABBREV,
157 FUNCTION_INST_CMP_ABBREV,
158 FUNCTION_INST_CMP_FLAGS_ABBREV,
159 FUNCTION_DEBUG_RECORD_VALUE_ABBREV,
160 FUNCTION_DEBUG_LOC_ABBREV,
161};
162
163/// Abstract class to manage the bitcode writing, subclassed for each bitcode
164/// file type.
165class BitcodeWriterBase {
166protected:
167 /// The stream created and owned by the client.
168 BitstreamWriter &Stream;
169
170 StringTableBuilder &StrtabBuilder;
171
172public:
173 /// Constructs a BitcodeWriterBase object that writes to the provided
174 /// \p Stream.
175 BitcodeWriterBase(BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder)
176 : Stream(Stream), StrtabBuilder(StrtabBuilder) {}
177
178protected:
179 void writeModuleVersion();
180};
181
182void BitcodeWriterBase::writeModuleVersion() {
183 // VERSION: [version#]
184 Stream.EmitRecord(Code: bitc::MODULE_CODE_VERSION, Vals: ArrayRef<uint64_t>{2});
185}
186
187/// Base class to manage the module bitcode writing, currently subclassed for
188/// ModuleBitcodeWriter and ThinLinkBitcodeWriter.
189class ModuleBitcodeWriterBase : public BitcodeWriterBase {
190protected:
191 /// The Module to write to bitcode.
192 const Module &M;
193
194 /// Enumerates ids for all values in the module.
195 ValueEnumerator VE;
196
197 /// Optional per-module index to write for ThinLTO.
198 const ModuleSummaryIndex *Index;
199
200 /// Map that holds the correspondence between GUIDs in the summary index,
201 /// that came from indirect call profiles, and a value id generated by this
202 /// class to use in the VST and summary block records.
203 std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
204
205 /// Tracks the last value id recorded in the GUIDToValueMap.
206 unsigned GlobalValueId;
207
208 /// Saves the offset of the VSTOffset record that must eventually be
209 /// backpatched with the offset of the actual VST.
210 uint64_t VSTOffsetPlaceholder = 0;
211
212public:
213 /// Constructs a ModuleBitcodeWriterBase object for the given Module,
214 /// writing to the provided \p Buffer.
215 ModuleBitcodeWriterBase(const Module &M, StringTableBuilder &StrtabBuilder,
216 BitstreamWriter &Stream,
217 bool ShouldPreserveUseListOrder,
218 const ModuleSummaryIndex *Index)
219 : BitcodeWriterBase(Stream, StrtabBuilder), M(M),
220 VE(M, PreserveBitcodeUseListOrder.getNumOccurrences()
221 ? PreserveBitcodeUseListOrder
222 : ShouldPreserveUseListOrder),
223 Index(Index) {
224 // Assign ValueIds to any callee values in the index that came from
225 // indirect call profiles and were recorded as a GUID not a Value*
226 // (which would have been assigned an ID by the ValueEnumerator).
227 // The starting ValueId is just after the number of values in the
228 // ValueEnumerator, so that they can be emitted in the VST.
229 GlobalValueId = VE.getValues().size();
230 if (!Index)
231 return;
232 for (const auto &GUIDSummaryLists : *Index)
233 // Examine all summaries for this GUID.
234 for (auto &Summary : GUIDSummaryLists.second.getSummaryList())
235 if (auto FS = dyn_cast<FunctionSummary>(Val: Summary.get())) {
236 // For each call in the function summary, see if the call
237 // is to a GUID (which means it is for an indirect call,
238 // otherwise we would have a Value for it). If so, synthesize
239 // a value id.
240 for (auto &CallEdge : FS->calls())
241 if (!CallEdge.first.haveGVs() || !CallEdge.first.getValue())
242 assignValueId(ValGUID: CallEdge.first.getGUID());
243
244 // For each referenced variables in the function summary, see if the
245 // variable is represented by a GUID (as opposed to a symbol to
246 // declarations or definitions in the module). If so, synthesize a
247 // value id.
248 for (auto &RefEdge : FS->refs())
249 if (!RefEdge.haveGVs() || !RefEdge.getValue())
250 assignValueId(ValGUID: RefEdge.getGUID());
251 }
252 }
253
254protected:
255 void writePerModuleGlobalValueSummary();
256
257private:
258 void writePerModuleFunctionSummaryRecord(
259 SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
260 unsigned ValueID, unsigned FSCallsProfileAbbrev, unsigned CallsiteAbbrev,
261 unsigned AllocAbbrev, unsigned ContextIdAbbvId, const Function &F,
262 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
263 CallStackId &CallStackCount);
264 void writeModuleLevelReferences(const GlobalVariable &V,
265 SmallVector<uint64_t, 64> &NameVals,
266 unsigned FSModRefsAbbrev,
267 unsigned FSModVTableRefsAbbrev);
268
269 void assignValueId(GlobalValue::GUID ValGUID) {
270 GUIDToValueIdMap[ValGUID] = ++GlobalValueId;
271 }
272
273 unsigned getValueId(GlobalValue::GUID ValGUID) {
274 const auto &VMI = GUIDToValueIdMap.find(x: ValGUID);
275 // Expect that any GUID value had a value Id assigned by an
276 // earlier call to assignValueId.
277 assert(VMI != GUIDToValueIdMap.end() &&
278 "GUID does not have assigned value Id");
279 return VMI->second;
280 }
281
282 // Helper to get the valueId for the type of value recorded in VI.
283 unsigned getValueId(ValueInfo VI) {
284 if (!VI.haveGVs() || !VI.getValue())
285 return getValueId(ValGUID: VI.getGUID());
286 return VE.getValueID(V: VI.getValue());
287 }
288
289 std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
290};
291
292/// Class to manage the bitcode writing for a module.
293class ModuleBitcodeWriter : public ModuleBitcodeWriterBase {
294 /// True if a module hash record should be written.
295 bool GenerateHash;
296
297 /// If non-null, when GenerateHash is true, the resulting hash is written
298 /// into ModHash.
299 ModuleHash *ModHash;
300
301 SHA1 Hasher;
302
303 /// The start bit of the identification block.
304 uint64_t BitcodeStartBit;
305
306public:
307 /// Constructs a ModuleBitcodeWriter object for the given Module,
308 /// writing to the provided \p Buffer.
309 ModuleBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
310 BitstreamWriter &Stream, bool ShouldPreserveUseListOrder,
311 const ModuleSummaryIndex *Index, bool GenerateHash,
312 ModuleHash *ModHash = nullptr)
313 : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
314 ShouldPreserveUseListOrder, Index),
315 GenerateHash(GenerateHash), ModHash(ModHash),
316 BitcodeStartBit(Stream.GetCurrentBitNo()) {}
317
318 /// Emit the current module to the bitstream.
319 void write();
320
321private:
322 uint64_t bitcodeStartBit() { return BitcodeStartBit; }
323
324 size_t addToStrtab(StringRef Str);
325
326 void writeAttributeGroupTable();
327 void writeAttributeTable();
328 void writeTypeTable();
329 void writeComdats();
330 void writeValueSymbolTableForwardDecl();
331 void writeModuleInfo();
332 void writeValueAsMetadata(const ValueAsMetadata *MD,
333 SmallVectorImpl<uint64_t> &Record);
334 void writeMDTuple(const MDTuple *N, SmallVectorImpl<uint64_t> &Record,
335 unsigned Abbrev);
336 unsigned createDILocationAbbrev();
337 void writeDILocation(const DILocation *N, SmallVectorImpl<uint64_t> &Record,
338 unsigned &Abbrev);
339 unsigned createGenericDINodeAbbrev();
340 void writeGenericDINode(const GenericDINode *N,
341 SmallVectorImpl<uint64_t> &Record, unsigned &Abbrev);
342 void writeDISubrange(const DISubrange *N, SmallVectorImpl<uint64_t> &Record,
343 unsigned Abbrev);
344 void writeDIGenericSubrange(const DIGenericSubrange *N,
345 SmallVectorImpl<uint64_t> &Record,
346 unsigned Abbrev);
347 void writeDIEnumerator(const DIEnumerator *N,
348 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
349 void writeDIBasicType(const DIBasicType *N, SmallVectorImpl<uint64_t> &Record,
350 unsigned Abbrev);
351 void writeDIFixedPointType(const DIFixedPointType *N,
352 SmallVectorImpl<uint64_t> &Record,
353 unsigned Abbrev);
354 void writeDIStringType(const DIStringType *N,
355 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
356 void writeDIDerivedType(const DIDerivedType *N,
357 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
358 void writeDISubrangeType(const DISubrangeType *N,
359 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
360 void writeDICompositeType(const DICompositeType *N,
361 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
362 void writeDISubroutineType(const DISubroutineType *N,
363 SmallVectorImpl<uint64_t> &Record,
364 unsigned Abbrev);
365 void writeDIFile(const DIFile *N, SmallVectorImpl<uint64_t> &Record,
366 unsigned Abbrev);
367 void writeDICompileUnit(const DICompileUnit *N,
368 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
369 void writeDISubprogram(const DISubprogram *N,
370 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
371 void writeDILexicalBlock(const DILexicalBlock *N,
372 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
373 void writeDILexicalBlockFile(const DILexicalBlockFile *N,
374 SmallVectorImpl<uint64_t> &Record,
375 unsigned Abbrev);
376 void writeDICommonBlock(const DICommonBlock *N,
377 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
378 void writeDINamespace(const DINamespace *N, SmallVectorImpl<uint64_t> &Record,
379 unsigned Abbrev);
380 void writeDIMacro(const DIMacro *N, SmallVectorImpl<uint64_t> &Record,
381 unsigned Abbrev);
382 void writeDIMacroFile(const DIMacroFile *N, SmallVectorImpl<uint64_t> &Record,
383 unsigned Abbrev);
384 void writeDIArgList(const DIArgList *N, SmallVectorImpl<uint64_t> &Record);
385 void writeDIModule(const DIModule *N, SmallVectorImpl<uint64_t> &Record,
386 unsigned Abbrev);
387 void writeDIAssignID(const DIAssignID *N, SmallVectorImpl<uint64_t> &Record,
388 unsigned Abbrev);
389 void writeDITemplateTypeParameter(const DITemplateTypeParameter *N,
390 SmallVectorImpl<uint64_t> &Record,
391 unsigned Abbrev);
392 void writeDITemplateValueParameter(const DITemplateValueParameter *N,
393 SmallVectorImpl<uint64_t> &Record,
394 unsigned Abbrev);
395 void writeDIGlobalVariable(const DIGlobalVariable *N,
396 SmallVectorImpl<uint64_t> &Record,
397 unsigned Abbrev);
398 void writeDILocalVariable(const DILocalVariable *N,
399 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
400 void writeDILabel(const DILabel *N,
401 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
402 void writeDIExpression(const DIExpression *N,
403 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
404 void writeDIGlobalVariableExpression(const DIGlobalVariableExpression *N,
405 SmallVectorImpl<uint64_t> &Record,
406 unsigned Abbrev);
407 void writeDIObjCProperty(const DIObjCProperty *N,
408 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
409 void writeDIImportedEntity(const DIImportedEntity *N,
410 SmallVectorImpl<uint64_t> &Record,
411 unsigned Abbrev);
412 unsigned createNamedMetadataAbbrev();
413 void writeNamedMetadata(SmallVectorImpl<uint64_t> &Record);
414 unsigned createMetadataStringsAbbrev();
415 void writeMetadataStrings(ArrayRef<const Metadata *> Strings,
416 SmallVectorImpl<uint64_t> &Record);
417 void writeMetadataRecords(ArrayRef<const Metadata *> MDs,
418 SmallVectorImpl<uint64_t> &Record,
419 std::vector<unsigned> *MDAbbrevs = nullptr,
420 std::vector<uint64_t> *IndexPos = nullptr);
421 void writeModuleMetadata();
422 void writeFunctionMetadata(const Function &F);
423 void writeFunctionMetadataAttachment(const Function &F);
424 void pushGlobalMetadataAttachment(SmallVectorImpl<uint64_t> &Record,
425 const GlobalObject &GO);
426 void writeModuleMetadataKinds();
427 void writeOperandBundleTags();
428 void writeSyncScopeNames();
429 void writeConstants(unsigned FirstVal, unsigned LastVal, bool isGlobal);
430 void writeModuleConstants();
431 bool pushValueAndType(const Value *V, unsigned InstID,
432 SmallVectorImpl<unsigned> &Vals);
433 bool pushValueOrMetadata(const Value *V, unsigned InstID,
434 SmallVectorImpl<unsigned> &Vals);
435 void writeOperandBundles(const CallBase &CB, unsigned InstID);
436 void pushValue(const Value *V, unsigned InstID,
437 SmallVectorImpl<unsigned> &Vals);
438 void pushValueSigned(const Value *V, unsigned InstID,
439 SmallVectorImpl<uint64_t> &Vals);
440 void writeInstruction(const Instruction &I, unsigned InstID,
441 SmallVectorImpl<unsigned> &Vals);
442 void writeFunctionLevelValueSymbolTable(const ValueSymbolTable &VST);
443 void writeGlobalValueSymbolTable(
444 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
445 void writeUseList(UseListOrder &&Order);
446 void writeUseListBlock(const Function *F);
447 void
448 writeFunction(const Function &F,
449 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
450 void writeBlockInfo();
451 void writeModuleHash(StringRef View);
452
453 unsigned getEncodedSyncScopeID(SyncScope::ID SSID) {
454 return unsigned(SSID);
455 }
456
457 unsigned getEncodedAlign(MaybeAlign Alignment) { return encode(A: Alignment); }
458};
459
460/// Class to manage the bitcode writing for a combined index.
461class IndexBitcodeWriter : public BitcodeWriterBase {
462 /// The combined index to write to bitcode.
463 const ModuleSummaryIndex &Index;
464
465 /// When writing combined summaries, provides the set of global value
466 /// summaries for which the value (function, function alias, etc) should be
467 /// imported as a declaration.
468 const GVSummaryPtrSet *DecSummaries = nullptr;
469
470 /// When writing a subset of the index for distributed backends, client
471 /// provides a map of modules to the corresponding GUIDs/summaries to write.
472 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex;
473
474 /// Map that holds the correspondence between the GUID used in the combined
475 /// index and a value id generated by this class to use in references.
476 std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
477
478 // The stack ids used by this index, which will be a subset of those in
479 // the full index in the case of distributed indexes.
480 std::vector<uint64_t> StackIds;
481
482 // Keep a map of the stack id indices used by records being written for this
483 // index to the index of the corresponding stack id in the above StackIds
484 // vector. Ensures we write each referenced stack id once.
485 DenseMap<unsigned, unsigned> StackIdIndicesToIndex;
486
487 /// Tracks the last value id recorded in the GUIDToValueMap.
488 unsigned GlobalValueId = 0;
489
490 /// Tracks the assignment of module paths in the module path string table to
491 /// an id assigned for use in summary references to the module path.
492 DenseMap<StringRef, uint64_t> ModuleIdMap;
493
494public:
495 /// Constructs a IndexBitcodeWriter object for the given combined index,
496 /// writing to the provided \p Buffer. When writing a subset of the index
497 /// for a distributed backend, provide a \p ModuleToSummariesForIndex map.
498 /// If provided, \p DecSummaries specifies the set of summaries for which
499 /// the corresponding functions or aliased functions should be imported as a
500 /// declaration (but not definition) for each module.
501 IndexBitcodeWriter(
502 BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder,
503 const ModuleSummaryIndex &Index,
504 const GVSummaryPtrSet *DecSummaries = nullptr,
505 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex = nullptr)
506 : BitcodeWriterBase(Stream, StrtabBuilder), Index(Index),
507 DecSummaries(DecSummaries),
508 ModuleToSummariesForIndex(ModuleToSummariesForIndex) {
509
510 // See if the StackIdIndex was already added to the StackId map and
511 // vector. If not, record it.
512 auto RecordStackIdReference = [&](unsigned StackIdIndex) {
513 // If the StackIdIndex is not yet in the map, the below insert ensures
514 // that it will point to the new StackIds vector entry we push to just
515 // below.
516 auto Inserted =
517 StackIdIndicesToIndex.insert(KV: {StackIdIndex, StackIds.size()});
518 if (Inserted.second)
519 StackIds.push_back(x: Index.getStackIdAtIndex(Index: StackIdIndex));
520 };
521
522 // Assign unique value ids to all summaries to be written, for use
523 // in writing out the call graph edges. Save the mapping from GUID
524 // to the new global value id to use when writing those edges, which
525 // are currently saved in the index in terms of GUID.
526 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
527 GUIDToValueIdMap[I.first] = ++GlobalValueId;
528 // If this is invoked for an aliasee, we want to record the above mapping,
529 // but not the information needed for its summary entry (if the aliasee is
530 // to be imported, we will invoke this separately with IsAliasee=false).
531 if (IsAliasee)
532 return;
533 auto *FS = dyn_cast<FunctionSummary>(Val: I.second);
534 if (!FS)
535 return;
536 // Record all stack id indices actually used in the summary entries being
537 // written, so that we can compact them in the case of distributed ThinLTO
538 // indexes.
539 for (auto &CI : FS->callsites()) {
540 // If the stack id list is empty, this callsite info was synthesized for
541 // a missing tail call frame. Ensure that the callee's GUID gets a value
542 // id. Normally we only generate these for defined summaries, which in
543 // the case of distributed ThinLTO is only the functions already defined
544 // in the module or that we want to import. We don't bother to include
545 // all the callee symbols as they aren't normally needed in the backend.
546 // However, for the synthesized callsite infos we do need the callee
547 // GUID in the backend so that we can correlate the identified callee
548 // with this callsite info (which for non-tail calls is done by the
549 // ordering of the callsite infos and verified via stack ids).
550 if (CI.StackIdIndices.empty()) {
551 GUIDToValueIdMap[CI.Callee.getGUID()] = ++GlobalValueId;
552 continue;
553 }
554 for (auto Idx : CI.StackIdIndices)
555 RecordStackIdReference(Idx);
556 }
557 if (CombinedIndexMemProfContext) {
558 for (auto &AI : FS->allocs())
559 for (auto &MIB : AI.MIBs)
560 for (auto Idx : MIB.StackIdIndices)
561 RecordStackIdReference(Idx);
562 }
563 });
564 }
565
566 /// The below iterator returns the GUID and associated summary.
567 using GVInfo = std::pair<GlobalValue::GUID, GlobalValueSummary *>;
568
569 /// Calls the callback for each value GUID and summary to be written to
570 /// bitcode. This hides the details of whether they are being pulled from the
571 /// entire index or just those in a provided ModuleToSummariesForIndex map.
572 template<typename Functor>
573 void forEachSummary(Functor Callback) {
574 if (ModuleToSummariesForIndex) {
575 for (auto &M : *ModuleToSummariesForIndex)
576 for (auto &Summary : M.second) {
577 Callback(Summary, false);
578 // Ensure aliasee is handled, e.g. for assigning a valueId,
579 // even if we are not importing the aliasee directly (the
580 // imported alias will contain a copy of aliasee).
581 if (auto *AS = dyn_cast<AliasSummary>(Val: Summary.getSecond()))
582 Callback({AS->getAliaseeGUID(), &AS->getAliasee()}, true);
583 }
584 } else {
585 for (auto &Summaries : Index)
586 for (auto &Summary : Summaries.second.getSummaryList())
587 Callback({Summaries.first, Summary.get()}, false);
588 }
589 }
590
591 /// Calls the callback for each entry in the modulePaths StringMap that
592 /// should be written to the module path string table. This hides the details
593 /// of whether they are being pulled from the entire index or just those in a
594 /// provided ModuleToSummariesForIndex map.
595 template <typename Functor> void forEachModule(Functor Callback) {
596 if (ModuleToSummariesForIndex) {
597 for (const auto &M : *ModuleToSummariesForIndex) {
598 const auto &MPI = Index.modulePaths().find(Key: M.first);
599 if (MPI == Index.modulePaths().end()) {
600 // This should only happen if the bitcode file was empty, in which
601 // case we shouldn't be importing (the ModuleToSummariesForIndex
602 // would only include the module we are writing and index for).
603 assert(ModuleToSummariesForIndex->size() == 1);
604 continue;
605 }
606 Callback(*MPI);
607 }
608 } else {
609 // Since StringMap iteration order isn't guaranteed, order by path string
610 // first.
611 // FIXME: Make this a vector of StringMapEntry instead to avoid the later
612 // map lookup.
613 std::vector<StringRef> ModulePaths;
614 for (auto &[ModPath, _] : Index.modulePaths())
615 ModulePaths.push_back(x: ModPath);
616 llvm::sort(C&: ModulePaths);
617 for (auto &ModPath : ModulePaths)
618 Callback(*Index.modulePaths().find(Key: ModPath));
619 }
620 }
621
622 /// Main entry point for writing a combined index to bitcode.
623 void write();
624
625private:
626 void writeModStrings();
627 void writeCombinedGlobalValueSummary();
628
629 std::optional<unsigned> getValueId(GlobalValue::GUID ValGUID) {
630 auto VMI = GUIDToValueIdMap.find(x: ValGUID);
631 if (VMI == GUIDToValueIdMap.end())
632 return std::nullopt;
633 return VMI->second;
634 }
635
636 std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
637};
638
639} // end anonymous namespace
640
641static unsigned getEncodedCastOpcode(unsigned Opcode) {
642 switch (Opcode) {
643 default: llvm_unreachable("Unknown cast instruction!");
644 case Instruction::Trunc : return bitc::CAST_TRUNC;
645 case Instruction::ZExt : return bitc::CAST_ZEXT;
646 case Instruction::SExt : return bitc::CAST_SEXT;
647 case Instruction::FPToUI : return bitc::CAST_FPTOUI;
648 case Instruction::FPToSI : return bitc::CAST_FPTOSI;
649 case Instruction::UIToFP : return bitc::CAST_UITOFP;
650 case Instruction::SIToFP : return bitc::CAST_SITOFP;
651 case Instruction::FPTrunc : return bitc::CAST_FPTRUNC;
652 case Instruction::FPExt : return bitc::CAST_FPEXT;
653 case Instruction::PtrToAddr: return bitc::CAST_PTRTOADDR;
654 case Instruction::PtrToInt: return bitc::CAST_PTRTOINT;
655 case Instruction::IntToPtr: return bitc::CAST_INTTOPTR;
656 case Instruction::BitCast : return bitc::CAST_BITCAST;
657 case Instruction::AddrSpaceCast: return bitc::CAST_ADDRSPACECAST;
658 }
659}
660
661static unsigned getEncodedUnaryOpcode(unsigned Opcode) {
662 switch (Opcode) {
663 default: llvm_unreachable("Unknown binary instruction!");
664 case Instruction::FNeg: return bitc::UNOP_FNEG;
665 }
666}
667
668static unsigned getEncodedBinaryOpcode(unsigned Opcode) {
669 switch (Opcode) {
670 default: llvm_unreachable("Unknown binary instruction!");
671 case Instruction::Add:
672 case Instruction::FAdd: return bitc::BINOP_ADD;
673 case Instruction::Sub:
674 case Instruction::FSub: return bitc::BINOP_SUB;
675 case Instruction::Mul:
676 case Instruction::FMul: return bitc::BINOP_MUL;
677 case Instruction::UDiv: return bitc::BINOP_UDIV;
678 case Instruction::FDiv:
679 case Instruction::SDiv: return bitc::BINOP_SDIV;
680 case Instruction::URem: return bitc::BINOP_UREM;
681 case Instruction::FRem:
682 case Instruction::SRem: return bitc::BINOP_SREM;
683 case Instruction::Shl: return bitc::BINOP_SHL;
684 case Instruction::LShr: return bitc::BINOP_LSHR;
685 case Instruction::AShr: return bitc::BINOP_ASHR;
686 case Instruction::And: return bitc::BINOP_AND;
687 case Instruction::Or: return bitc::BINOP_OR;
688 case Instruction::Xor: return bitc::BINOP_XOR;
689 }
690}
691
692static unsigned getEncodedRMWOperation(AtomicRMWInst::BinOp Op) {
693 switch (Op) {
694 default: llvm_unreachable("Unknown RMW operation!");
695 case AtomicRMWInst::Xchg: return bitc::RMW_XCHG;
696 case AtomicRMWInst::Add: return bitc::RMW_ADD;
697 case AtomicRMWInst::Sub: return bitc::RMW_SUB;
698 case AtomicRMWInst::And: return bitc::RMW_AND;
699 case AtomicRMWInst::Nand: return bitc::RMW_NAND;
700 case AtomicRMWInst::Or: return bitc::RMW_OR;
701 case AtomicRMWInst::Xor: return bitc::RMW_XOR;
702 case AtomicRMWInst::Max: return bitc::RMW_MAX;
703 case AtomicRMWInst::Min: return bitc::RMW_MIN;
704 case AtomicRMWInst::UMax: return bitc::RMW_UMAX;
705 case AtomicRMWInst::UMin: return bitc::RMW_UMIN;
706 case AtomicRMWInst::FAdd: return bitc::RMW_FADD;
707 case AtomicRMWInst::FSub: return bitc::RMW_FSUB;
708 case AtomicRMWInst::FMax: return bitc::RMW_FMAX;
709 case AtomicRMWInst::FMin: return bitc::RMW_FMIN;
710 case AtomicRMWInst::FMaximum:
711 return bitc::RMW_FMAXIMUM;
712 case AtomicRMWInst::FMinimum:
713 return bitc::RMW_FMINIMUM;
714 case AtomicRMWInst::UIncWrap:
715 return bitc::RMW_UINC_WRAP;
716 case AtomicRMWInst::UDecWrap:
717 return bitc::RMW_UDEC_WRAP;
718 case AtomicRMWInst::USubCond:
719 return bitc::RMW_USUB_COND;
720 case AtomicRMWInst::USubSat:
721 return bitc::RMW_USUB_SAT;
722 }
723}
724
725static unsigned getEncodedOrdering(AtomicOrdering Ordering) {
726 switch (Ordering) {
727 case AtomicOrdering::NotAtomic: return bitc::ORDERING_NOTATOMIC;
728 case AtomicOrdering::Unordered: return bitc::ORDERING_UNORDERED;
729 case AtomicOrdering::Monotonic: return bitc::ORDERING_MONOTONIC;
730 case AtomicOrdering::Acquire: return bitc::ORDERING_ACQUIRE;
731 case AtomicOrdering::Release: return bitc::ORDERING_RELEASE;
732 case AtomicOrdering::AcquireRelease: return bitc::ORDERING_ACQREL;
733 case AtomicOrdering::SequentiallyConsistent: return bitc::ORDERING_SEQCST;
734 }
735 llvm_unreachable("Invalid ordering");
736}
737
738static void writeStringRecord(BitstreamWriter &Stream, unsigned Code,
739 StringRef Str, unsigned AbbrevToUse) {
740 SmallVector<unsigned, 64> Vals;
741
742 // Code: [strchar x N]
743 for (char C : Str) {
744 if (AbbrevToUse && !BitCodeAbbrevOp::isChar6(C))
745 AbbrevToUse = 0;
746 Vals.push_back(Elt: C);
747 }
748
749 // Emit the finished record.
750 Stream.EmitRecord(Code, Vals, Abbrev: AbbrevToUse);
751}
752
753static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
754 switch (Kind) {
755 case Attribute::Alignment:
756 return bitc::ATTR_KIND_ALIGNMENT;
757 case Attribute::AllocAlign:
758 return bitc::ATTR_KIND_ALLOC_ALIGN;
759 case Attribute::AllocSize:
760 return bitc::ATTR_KIND_ALLOC_SIZE;
761 case Attribute::AlwaysInline:
762 return bitc::ATTR_KIND_ALWAYS_INLINE;
763 case Attribute::Builtin:
764 return bitc::ATTR_KIND_BUILTIN;
765 case Attribute::ByVal:
766 return bitc::ATTR_KIND_BY_VAL;
767 case Attribute::Convergent:
768 return bitc::ATTR_KIND_CONVERGENT;
769 case Attribute::InAlloca:
770 return bitc::ATTR_KIND_IN_ALLOCA;
771 case Attribute::Cold:
772 return bitc::ATTR_KIND_COLD;
773 case Attribute::DisableSanitizerInstrumentation:
774 return bitc::ATTR_KIND_DISABLE_SANITIZER_INSTRUMENTATION;
775 case Attribute::FnRetThunkExtern:
776 return bitc::ATTR_KIND_FNRETTHUNK_EXTERN;
777 case Attribute::Hot:
778 return bitc::ATTR_KIND_HOT;
779 case Attribute::ElementType:
780 return bitc::ATTR_KIND_ELEMENTTYPE;
781 case Attribute::HybridPatchable:
782 return bitc::ATTR_KIND_HYBRID_PATCHABLE;
783 case Attribute::InlineHint:
784 return bitc::ATTR_KIND_INLINE_HINT;
785 case Attribute::InReg:
786 return bitc::ATTR_KIND_IN_REG;
787 case Attribute::JumpTable:
788 return bitc::ATTR_KIND_JUMP_TABLE;
789 case Attribute::MinSize:
790 return bitc::ATTR_KIND_MIN_SIZE;
791 case Attribute::AllocatedPointer:
792 return bitc::ATTR_KIND_ALLOCATED_POINTER;
793 case Attribute::AllocKind:
794 return bitc::ATTR_KIND_ALLOC_KIND;
795 case Attribute::Memory:
796 return bitc::ATTR_KIND_MEMORY;
797 case Attribute::NoFPClass:
798 return bitc::ATTR_KIND_NOFPCLASS;
799 case Attribute::Naked:
800 return bitc::ATTR_KIND_NAKED;
801 case Attribute::Nest:
802 return bitc::ATTR_KIND_NEST;
803 case Attribute::NoAlias:
804 return bitc::ATTR_KIND_NO_ALIAS;
805 case Attribute::NoBuiltin:
806 return bitc::ATTR_KIND_NO_BUILTIN;
807 case Attribute::NoCallback:
808 return bitc::ATTR_KIND_NO_CALLBACK;
809 case Attribute::NoDivergenceSource:
810 return bitc::ATTR_KIND_NO_DIVERGENCE_SOURCE;
811 case Attribute::NoDuplicate:
812 return bitc::ATTR_KIND_NO_DUPLICATE;
813 case Attribute::NoFree:
814 return bitc::ATTR_KIND_NOFREE;
815 case Attribute::NoImplicitFloat:
816 return bitc::ATTR_KIND_NO_IMPLICIT_FLOAT;
817 case Attribute::NoInline:
818 return bitc::ATTR_KIND_NO_INLINE;
819 case Attribute::NoRecurse:
820 return bitc::ATTR_KIND_NO_RECURSE;
821 case Attribute::NoMerge:
822 return bitc::ATTR_KIND_NO_MERGE;
823 case Attribute::NonLazyBind:
824 return bitc::ATTR_KIND_NON_LAZY_BIND;
825 case Attribute::NonNull:
826 return bitc::ATTR_KIND_NON_NULL;
827 case Attribute::Dereferenceable:
828 return bitc::ATTR_KIND_DEREFERENCEABLE;
829 case Attribute::DereferenceableOrNull:
830 return bitc::ATTR_KIND_DEREFERENCEABLE_OR_NULL;
831 case Attribute::NoRedZone:
832 return bitc::ATTR_KIND_NO_RED_ZONE;
833 case Attribute::NoReturn:
834 return bitc::ATTR_KIND_NO_RETURN;
835 case Attribute::NoSync:
836 return bitc::ATTR_KIND_NOSYNC;
837 case Attribute::NoCfCheck:
838 return bitc::ATTR_KIND_NOCF_CHECK;
839 case Attribute::NoProfile:
840 return bitc::ATTR_KIND_NO_PROFILE;
841 case Attribute::SkipProfile:
842 return bitc::ATTR_KIND_SKIP_PROFILE;
843 case Attribute::NoUnwind:
844 return bitc::ATTR_KIND_NO_UNWIND;
845 case Attribute::NoSanitizeBounds:
846 return bitc::ATTR_KIND_NO_SANITIZE_BOUNDS;
847 case Attribute::NoSanitizeCoverage:
848 return bitc::ATTR_KIND_NO_SANITIZE_COVERAGE;
849 case Attribute::NullPointerIsValid:
850 return bitc::ATTR_KIND_NULL_POINTER_IS_VALID;
851 case Attribute::OptimizeForDebugging:
852 return bitc::ATTR_KIND_OPTIMIZE_FOR_DEBUGGING;
853 case Attribute::OptForFuzzing:
854 return bitc::ATTR_KIND_OPT_FOR_FUZZING;
855 case Attribute::OptimizeForSize:
856 return bitc::ATTR_KIND_OPTIMIZE_FOR_SIZE;
857 case Attribute::OptimizeNone:
858 return bitc::ATTR_KIND_OPTIMIZE_NONE;
859 case Attribute::ReadNone:
860 return bitc::ATTR_KIND_READ_NONE;
861 case Attribute::ReadOnly:
862 return bitc::ATTR_KIND_READ_ONLY;
863 case Attribute::Returned:
864 return bitc::ATTR_KIND_RETURNED;
865 case Attribute::ReturnsTwice:
866 return bitc::ATTR_KIND_RETURNS_TWICE;
867 case Attribute::SExt:
868 return bitc::ATTR_KIND_S_EXT;
869 case Attribute::Speculatable:
870 return bitc::ATTR_KIND_SPECULATABLE;
871 case Attribute::StackAlignment:
872 return bitc::ATTR_KIND_STACK_ALIGNMENT;
873 case Attribute::StackProtect:
874 return bitc::ATTR_KIND_STACK_PROTECT;
875 case Attribute::StackProtectReq:
876 return bitc::ATTR_KIND_STACK_PROTECT_REQ;
877 case Attribute::StackProtectStrong:
878 return bitc::ATTR_KIND_STACK_PROTECT_STRONG;
879 case Attribute::SafeStack:
880 return bitc::ATTR_KIND_SAFESTACK;
881 case Attribute::ShadowCallStack:
882 return bitc::ATTR_KIND_SHADOWCALLSTACK;
883 case Attribute::StrictFP:
884 return bitc::ATTR_KIND_STRICT_FP;
885 case Attribute::StructRet:
886 return bitc::ATTR_KIND_STRUCT_RET;
887 case Attribute::SanitizeAddress:
888 return bitc::ATTR_KIND_SANITIZE_ADDRESS;
889 case Attribute::SanitizeAllocToken:
890 return bitc::ATTR_KIND_SANITIZE_ALLOC_TOKEN;
891 case Attribute::SanitizeHWAddress:
892 return bitc::ATTR_KIND_SANITIZE_HWADDRESS;
893 case Attribute::SanitizeThread:
894 return bitc::ATTR_KIND_SANITIZE_THREAD;
895 case Attribute::SanitizeType:
896 return bitc::ATTR_KIND_SANITIZE_TYPE;
897 case Attribute::SanitizeMemory:
898 return bitc::ATTR_KIND_SANITIZE_MEMORY;
899 case Attribute::SanitizeNumericalStability:
900 return bitc::ATTR_KIND_SANITIZE_NUMERICAL_STABILITY;
901 case Attribute::SanitizeRealtime:
902 return bitc::ATTR_KIND_SANITIZE_REALTIME;
903 case Attribute::SanitizeRealtimeBlocking:
904 return bitc::ATTR_KIND_SANITIZE_REALTIME_BLOCKING;
905 case Attribute::SpeculativeLoadHardening:
906 return bitc::ATTR_KIND_SPECULATIVE_LOAD_HARDENING;
907 case Attribute::SwiftError:
908 return bitc::ATTR_KIND_SWIFT_ERROR;
909 case Attribute::SwiftSelf:
910 return bitc::ATTR_KIND_SWIFT_SELF;
911 case Attribute::SwiftAsync:
912 return bitc::ATTR_KIND_SWIFT_ASYNC;
913 case Attribute::UWTable:
914 return bitc::ATTR_KIND_UW_TABLE;
915 case Attribute::VScaleRange:
916 return bitc::ATTR_KIND_VSCALE_RANGE;
917 case Attribute::WillReturn:
918 return bitc::ATTR_KIND_WILLRETURN;
919 case Attribute::WriteOnly:
920 return bitc::ATTR_KIND_WRITEONLY;
921 case Attribute::ZExt:
922 return bitc::ATTR_KIND_Z_EXT;
923 case Attribute::ImmArg:
924 return bitc::ATTR_KIND_IMMARG;
925 case Attribute::SanitizeMemTag:
926 return bitc::ATTR_KIND_SANITIZE_MEMTAG;
927 case Attribute::Preallocated:
928 return bitc::ATTR_KIND_PREALLOCATED;
929 case Attribute::NoUndef:
930 return bitc::ATTR_KIND_NOUNDEF;
931 case Attribute::ByRef:
932 return bitc::ATTR_KIND_BYREF;
933 case Attribute::MustProgress:
934 return bitc::ATTR_KIND_MUSTPROGRESS;
935 case Attribute::PresplitCoroutine:
936 return bitc::ATTR_KIND_PRESPLIT_COROUTINE;
937 case Attribute::Writable:
938 return bitc::ATTR_KIND_WRITABLE;
939 case Attribute::CoroDestroyOnlyWhenComplete:
940 return bitc::ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE;
941 case Attribute::CoroElideSafe:
942 return bitc::ATTR_KIND_CORO_ELIDE_SAFE;
943 case Attribute::DeadOnUnwind:
944 return bitc::ATTR_KIND_DEAD_ON_UNWIND;
945 case Attribute::Range:
946 return bitc::ATTR_KIND_RANGE;
947 case Attribute::Initializes:
948 return bitc::ATTR_KIND_INITIALIZES;
949 case Attribute::NoExt:
950 return bitc::ATTR_KIND_NO_EXT;
951 case Attribute::Captures:
952 return bitc::ATTR_KIND_CAPTURES;
953 case Attribute::DeadOnReturn:
954 return bitc::ATTR_KIND_DEAD_ON_RETURN;
955 case Attribute::NoCreateUndefOrPoison:
956 return bitc::ATTR_KIND_NO_CREATE_UNDEF_OR_POISON;
957 case Attribute::DenormalFPEnv:
958 return bitc::ATTR_KIND_DENORMAL_FPENV;
959 case Attribute::NoOutline:
960 return bitc::ATTR_KIND_NOOUTLINE;
961 case Attribute::EndAttrKinds:
962 llvm_unreachable("Can not encode end-attribute kinds marker.");
963 case Attribute::None:
964 llvm_unreachable("Can not encode none-attribute.");
965 case Attribute::EmptyKey:
966 case Attribute::TombstoneKey:
967 llvm_unreachable("Trying to encode EmptyKey/TombstoneKey");
968 }
969
970 llvm_unreachable("Trying to encode unknown attribute");
971}
972
973static void emitSignedInt64(SmallVectorImpl<uint64_t> &Vals, uint64_t V) {
974 if ((int64_t)V >= 0)
975 Vals.push_back(Elt: V << 1);
976 else
977 Vals.push_back(Elt: (-V << 1) | 1);
978}
979
980static void emitWideAPInt(SmallVectorImpl<uint64_t> &Vals, const APInt &A) {
981 // We have an arbitrary precision integer value to write whose
982 // bit width is > 64. However, in canonical unsigned integer
983 // format it is likely that the high bits are going to be zero.
984 // So, we only write the number of active words.
985 unsigned NumWords = A.getActiveWords();
986 const uint64_t *RawData = A.getRawData();
987 for (unsigned i = 0; i < NumWords; i++)
988 emitSignedInt64(Vals, V: RawData[i]);
989}
990
991static void emitConstantRange(SmallVectorImpl<uint64_t> &Record,
992 const ConstantRange &CR, bool EmitBitWidth) {
993 unsigned BitWidth = CR.getBitWidth();
994 if (EmitBitWidth)
995 Record.push_back(Elt: BitWidth);
996 if (BitWidth > 64) {
997 Record.push_back(Elt: CR.getLower().getActiveWords() |
998 (uint64_t(CR.getUpper().getActiveWords()) << 32));
999 emitWideAPInt(Vals&: Record, A: CR.getLower());
1000 emitWideAPInt(Vals&: Record, A: CR.getUpper());
1001 } else {
1002 emitSignedInt64(Vals&: Record, V: CR.getLower().getSExtValue());
1003 emitSignedInt64(Vals&: Record, V: CR.getUpper().getSExtValue());
1004 }
1005}
1006
1007void ModuleBitcodeWriter::writeAttributeGroupTable() {
1008 const std::vector<ValueEnumerator::IndexAndAttrSet> &AttrGrps =
1009 VE.getAttributeGroups();
1010 if (AttrGrps.empty()) return;
1011
1012 Stream.EnterSubblock(BlockID: bitc::PARAMATTR_GROUP_BLOCK_ID, CodeLen: 3);
1013
1014 SmallVector<uint64_t, 64> Record;
1015 for (ValueEnumerator::IndexAndAttrSet Pair : AttrGrps) {
1016 unsigned AttrListIndex = Pair.first;
1017 AttributeSet AS = Pair.second;
1018 Record.push_back(Elt: VE.getAttributeGroupID(Group: Pair));
1019 Record.push_back(Elt: AttrListIndex);
1020
1021 for (Attribute Attr : AS) {
1022 if (Attr.isEnumAttribute()) {
1023 Record.push_back(Elt: 0);
1024 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1025 } else if (Attr.isIntAttribute()) {
1026 Record.push_back(Elt: 1);
1027 Attribute::AttrKind Kind = Attr.getKindAsEnum();
1028 Record.push_back(Elt: getAttrKindEncoding(Kind));
1029 if (Kind == Attribute::Memory) {
1030 // Version field for upgrading old memory effects.
1031 const uint64_t Version = 1;
1032 Record.push_back(Elt: (Version << 56) | Attr.getValueAsInt());
1033 } else {
1034 Record.push_back(Elt: Attr.getValueAsInt());
1035 }
1036 } else if (Attr.isStringAttribute()) {
1037 StringRef Kind = Attr.getKindAsString();
1038 StringRef Val = Attr.getValueAsString();
1039
1040 Record.push_back(Elt: Val.empty() ? 3 : 4);
1041 Record.append(in_start: Kind.begin(), in_end: Kind.end());
1042 Record.push_back(Elt: 0);
1043 if (!Val.empty()) {
1044 Record.append(in_start: Val.begin(), in_end: Val.end());
1045 Record.push_back(Elt: 0);
1046 }
1047 } else if (Attr.isTypeAttribute()) {
1048 Type *Ty = Attr.getValueAsType();
1049 Record.push_back(Elt: Ty ? 6 : 5);
1050 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1051 if (Ty)
1052 Record.push_back(Elt: VE.getTypeID(T: Attr.getValueAsType()));
1053 } else if (Attr.isConstantRangeAttribute()) {
1054 Record.push_back(Elt: 7);
1055 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1056 emitConstantRange(Record, CR: Attr.getValueAsConstantRange(),
1057 /*EmitBitWidth=*/true);
1058 } else {
1059 assert(Attr.isConstantRangeListAttribute());
1060 Record.push_back(Elt: 8);
1061 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1062 ArrayRef<ConstantRange> Val = Attr.getValueAsConstantRangeList();
1063 Record.push_back(Elt: Val.size());
1064 Record.push_back(Elt: Val[0].getBitWidth());
1065 for (auto &CR : Val)
1066 emitConstantRange(Record, CR, /*EmitBitWidth=*/false);
1067 }
1068 }
1069
1070 Stream.EmitRecord(Code: bitc::PARAMATTR_GRP_CODE_ENTRY, Vals: Record);
1071 Record.clear();
1072 }
1073
1074 Stream.ExitBlock();
1075}
1076
1077void ModuleBitcodeWriter::writeAttributeTable() {
1078 const std::vector<AttributeList> &Attrs = VE.getAttributeLists();
1079 if (Attrs.empty()) return;
1080
1081 Stream.EnterSubblock(BlockID: bitc::PARAMATTR_BLOCK_ID, CodeLen: 3);
1082
1083 SmallVector<uint64_t, 64> Record;
1084 for (const AttributeList &AL : Attrs) {
1085 for (unsigned i : AL.indexes()) {
1086 AttributeSet AS = AL.getAttributes(Index: i);
1087 if (AS.hasAttributes())
1088 Record.push_back(Elt: VE.getAttributeGroupID(Group: {i, AS}));
1089 }
1090
1091 Stream.EmitRecord(Code: bitc::PARAMATTR_CODE_ENTRY, Vals: Record);
1092 Record.clear();
1093 }
1094
1095 Stream.ExitBlock();
1096}
1097
1098/// WriteTypeTable - Write out the type table for a module.
1099void ModuleBitcodeWriter::writeTypeTable() {
1100 const ValueEnumerator::TypeList &TypeList = VE.getTypes();
1101
1102 Stream.EnterSubblock(BlockID: bitc::TYPE_BLOCK_ID_NEW, CodeLen: 4 /*count from # abbrevs */);
1103 SmallVector<uint64_t, 64> TypeVals;
1104
1105 uint64_t NumBits = VE.computeBitsRequiredForTypeIndices();
1106
1107 // Abbrev for TYPE_CODE_OPAQUE_POINTER.
1108 auto Abbv = std::make_shared<BitCodeAbbrev>();
1109 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_OPAQUE_POINTER));
1110 Abbv->Add(OpInfo: BitCodeAbbrevOp(0)); // Addrspace = 0
1111 unsigned OpaquePtrAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1112
1113 // Abbrev for TYPE_CODE_FUNCTION.
1114 Abbv = std::make_shared<BitCodeAbbrev>();
1115 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_FUNCTION));
1116 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isvararg
1117 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1118 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1119 unsigned FunctionAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1120
1121 // Abbrev for TYPE_CODE_STRUCT_ANON.
1122 Abbv = std::make_shared<BitCodeAbbrev>();
1123 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_ANON));
1124 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked
1125 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1126 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1127 unsigned StructAnonAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1128
1129 // Abbrev for TYPE_CODE_STRUCT_NAME.
1130 Abbv = std::make_shared<BitCodeAbbrev>();
1131 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAME));
1132 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1133 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
1134 unsigned StructNameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1135
1136 // Abbrev for TYPE_CODE_STRUCT_NAMED.
1137 Abbv = std::make_shared<BitCodeAbbrev>();
1138 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAMED));
1139 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked
1140 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1141 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1142 unsigned StructNamedAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1143
1144 // Abbrev for TYPE_CODE_ARRAY.
1145 Abbv = std::make_shared<BitCodeAbbrev>();
1146 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_ARRAY));
1147 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // size
1148 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1149 unsigned ArrayAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1150
1151 // Emit an entry count so the reader can reserve space.
1152 TypeVals.push_back(Elt: TypeList.size());
1153 Stream.EmitRecord(Code: bitc::TYPE_CODE_NUMENTRY, Vals: TypeVals);
1154 TypeVals.clear();
1155
1156 // Loop over all of the types, emitting each in turn.
1157 for (Type *T : TypeList) {
1158 int AbbrevToUse = 0;
1159 unsigned Code = 0;
1160
1161 switch (T->getTypeID()) {
1162 case Type::VoidTyID: Code = bitc::TYPE_CODE_VOID; break;
1163 case Type::HalfTyID: Code = bitc::TYPE_CODE_HALF; break;
1164 case Type::BFloatTyID: Code = bitc::TYPE_CODE_BFLOAT; break;
1165 case Type::FloatTyID: Code = bitc::TYPE_CODE_FLOAT; break;
1166 case Type::DoubleTyID: Code = bitc::TYPE_CODE_DOUBLE; break;
1167 case Type::X86_FP80TyID: Code = bitc::TYPE_CODE_X86_FP80; break;
1168 case Type::FP128TyID: Code = bitc::TYPE_CODE_FP128; break;
1169 case Type::PPC_FP128TyID: Code = bitc::TYPE_CODE_PPC_FP128; break;
1170 case Type::LabelTyID: Code = bitc::TYPE_CODE_LABEL; break;
1171 case Type::MetadataTyID:
1172 Code = bitc::TYPE_CODE_METADATA;
1173 break;
1174 case Type::X86_AMXTyID: Code = bitc::TYPE_CODE_X86_AMX; break;
1175 case Type::TokenTyID: Code = bitc::TYPE_CODE_TOKEN; break;
1176 case Type::IntegerTyID:
1177 // INTEGER: [width]
1178 Code = bitc::TYPE_CODE_INTEGER;
1179 TypeVals.push_back(Elt: cast<IntegerType>(Val: T)->getBitWidth());
1180 break;
1181 case Type::PointerTyID: {
1182 PointerType *PTy = cast<PointerType>(Val: T);
1183 unsigned AddressSpace = PTy->getAddressSpace();
1184 // OPAQUE_POINTER: [address space]
1185 Code = bitc::TYPE_CODE_OPAQUE_POINTER;
1186 TypeVals.push_back(Elt: AddressSpace);
1187 if (AddressSpace == 0)
1188 AbbrevToUse = OpaquePtrAbbrev;
1189 break;
1190 }
1191 case Type::FunctionTyID: {
1192 FunctionType *FT = cast<FunctionType>(Val: T);
1193 // FUNCTION: [isvararg, retty, paramty x N]
1194 Code = bitc::TYPE_CODE_FUNCTION;
1195 TypeVals.push_back(Elt: FT->isVarArg());
1196 TypeVals.push_back(Elt: VE.getTypeID(T: FT->getReturnType()));
1197 for (unsigned i = 0, e = FT->getNumParams(); i != e; ++i)
1198 TypeVals.push_back(Elt: VE.getTypeID(T: FT->getParamType(i)));
1199 AbbrevToUse = FunctionAbbrev;
1200 break;
1201 }
1202 case Type::StructTyID: {
1203 StructType *ST = cast<StructType>(Val: T);
1204 // STRUCT: [ispacked, eltty x N]
1205 TypeVals.push_back(Elt: ST->isPacked());
1206 // Output all of the element types.
1207 for (Type *ET : ST->elements())
1208 TypeVals.push_back(Elt: VE.getTypeID(T: ET));
1209
1210 if (ST->isLiteral()) {
1211 Code = bitc::TYPE_CODE_STRUCT_ANON;
1212 AbbrevToUse = StructAnonAbbrev;
1213 } else {
1214 if (ST->isOpaque()) {
1215 Code = bitc::TYPE_CODE_OPAQUE;
1216 } else {
1217 Code = bitc::TYPE_CODE_STRUCT_NAMED;
1218 AbbrevToUse = StructNamedAbbrev;
1219 }
1220
1221 // Emit the name if it is present.
1222 if (!ST->getName().empty())
1223 writeStringRecord(Stream, Code: bitc::TYPE_CODE_STRUCT_NAME, Str: ST->getName(),
1224 AbbrevToUse: StructNameAbbrev);
1225 }
1226 break;
1227 }
1228 case Type::ArrayTyID: {
1229 ArrayType *AT = cast<ArrayType>(Val: T);
1230 // ARRAY: [numelts, eltty]
1231 Code = bitc::TYPE_CODE_ARRAY;
1232 TypeVals.push_back(Elt: AT->getNumElements());
1233 TypeVals.push_back(Elt: VE.getTypeID(T: AT->getElementType()));
1234 AbbrevToUse = ArrayAbbrev;
1235 break;
1236 }
1237 case Type::FixedVectorTyID:
1238 case Type::ScalableVectorTyID: {
1239 VectorType *VT = cast<VectorType>(Val: T);
1240 // VECTOR [numelts, eltty] or
1241 // [numelts, eltty, scalable]
1242 Code = bitc::TYPE_CODE_VECTOR;
1243 TypeVals.push_back(Elt: VT->getElementCount().getKnownMinValue());
1244 TypeVals.push_back(Elt: VE.getTypeID(T: VT->getElementType()));
1245 if (isa<ScalableVectorType>(Val: VT))
1246 TypeVals.push_back(Elt: true);
1247 break;
1248 }
1249 case Type::TargetExtTyID: {
1250 TargetExtType *TET = cast<TargetExtType>(Val: T);
1251 Code = bitc::TYPE_CODE_TARGET_TYPE;
1252 writeStringRecord(Stream, Code: bitc::TYPE_CODE_STRUCT_NAME, Str: TET->getName(),
1253 AbbrevToUse: StructNameAbbrev);
1254 TypeVals.push_back(Elt: TET->getNumTypeParameters());
1255 for (Type *InnerTy : TET->type_params())
1256 TypeVals.push_back(Elt: VE.getTypeID(T: InnerTy));
1257 llvm::append_range(C&: TypeVals, R: TET->int_params());
1258 break;
1259 }
1260 case Type::TypedPointerTyID:
1261 llvm_unreachable("Typed pointers cannot be added to IR modules");
1262 }
1263
1264 // Emit the finished record.
1265 Stream.EmitRecord(Code, Vals: TypeVals, Abbrev: AbbrevToUse);
1266 TypeVals.clear();
1267 }
1268
1269 Stream.ExitBlock();
1270}
1271
1272static unsigned getEncodedLinkage(const GlobalValue::LinkageTypes Linkage) {
1273 switch (Linkage) {
1274 case GlobalValue::ExternalLinkage:
1275 return 0;
1276 case GlobalValue::WeakAnyLinkage:
1277 return 16;
1278 case GlobalValue::AppendingLinkage:
1279 return 2;
1280 case GlobalValue::InternalLinkage:
1281 return 3;
1282 case GlobalValue::LinkOnceAnyLinkage:
1283 return 18;
1284 case GlobalValue::ExternalWeakLinkage:
1285 return 7;
1286 case GlobalValue::CommonLinkage:
1287 return 8;
1288 case GlobalValue::PrivateLinkage:
1289 return 9;
1290 case GlobalValue::WeakODRLinkage:
1291 return 17;
1292 case GlobalValue::LinkOnceODRLinkage:
1293 return 19;
1294 case GlobalValue::AvailableExternallyLinkage:
1295 return 12;
1296 }
1297 llvm_unreachable("Invalid linkage");
1298}
1299
1300static unsigned getEncodedLinkage(const GlobalValue &GV) {
1301 return getEncodedLinkage(Linkage: GV.getLinkage());
1302}
1303
1304static uint64_t getEncodedFFlags(FunctionSummary::FFlags Flags) {
1305 uint64_t RawFlags = 0;
1306 RawFlags |= Flags.ReadNone;
1307 RawFlags |= (Flags.ReadOnly << 1);
1308 RawFlags |= (Flags.NoRecurse << 2);
1309 RawFlags |= (Flags.ReturnDoesNotAlias << 3);
1310 RawFlags |= (Flags.NoInline << 4);
1311 RawFlags |= (Flags.AlwaysInline << 5);
1312 RawFlags |= (Flags.NoUnwind << 6);
1313 RawFlags |= (Flags.MayThrow << 7);
1314 RawFlags |= (Flags.HasUnknownCall << 8);
1315 RawFlags |= (Flags.MustBeUnreachable << 9);
1316 return RawFlags;
1317}
1318
1319// Decode the flags for GlobalValue in the summary. See getDecodedGVSummaryFlags
1320// in BitcodeReader.cpp.
1321static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags,
1322 bool ImportAsDecl = false) {
1323 uint64_t RawFlags = 0;
1324
1325 RawFlags |= Flags.NotEligibleToImport; // bool
1326 RawFlags |= (Flags.Live << 1);
1327 RawFlags |= (Flags.DSOLocal << 2);
1328 RawFlags |= (Flags.CanAutoHide << 3);
1329
1330 // Linkage don't need to be remapped at that time for the summary. Any future
1331 // change to the getEncodedLinkage() function will need to be taken into
1332 // account here as well.
1333 RawFlags = (RawFlags << 4) | Flags.Linkage; // 4 bits
1334
1335 RawFlags |= (Flags.Visibility << 8); // 2 bits
1336
1337 unsigned ImportType = Flags.ImportType | ImportAsDecl;
1338 RawFlags |= (ImportType << 10); // 1 bit
1339
1340 return RawFlags;
1341}
1342
1343static uint64_t getEncodedGVarFlags(GlobalVarSummary::GVarFlags Flags) {
1344 uint64_t RawFlags = Flags.MaybeReadOnly | (Flags.MaybeWriteOnly << 1) |
1345 (Flags.Constant << 2) | Flags.VCallVisibility << 3;
1346 return RawFlags;
1347}
1348
1349static uint64_t getEncodedHotnessCallEdgeInfo(const CalleeInfo &CI) {
1350 uint64_t RawFlags = 0;
1351
1352 RawFlags |= CI.Hotness; // 3 bits
1353 RawFlags |= (CI.HasTailCall << 3); // 1 bit
1354
1355 return RawFlags;
1356}
1357
1358static unsigned getEncodedVisibility(const GlobalValue &GV) {
1359 switch (GV.getVisibility()) {
1360 case GlobalValue::DefaultVisibility: return 0;
1361 case GlobalValue::HiddenVisibility: return 1;
1362 case GlobalValue::ProtectedVisibility: return 2;
1363 }
1364 llvm_unreachable("Invalid visibility");
1365}
1366
1367static unsigned getEncodedDLLStorageClass(const GlobalValue &GV) {
1368 switch (GV.getDLLStorageClass()) {
1369 case GlobalValue::DefaultStorageClass: return 0;
1370 case GlobalValue::DLLImportStorageClass: return 1;
1371 case GlobalValue::DLLExportStorageClass: return 2;
1372 }
1373 llvm_unreachable("Invalid DLL storage class");
1374}
1375
1376static unsigned getEncodedThreadLocalMode(const GlobalValue &GV) {
1377 switch (GV.getThreadLocalMode()) {
1378 case GlobalVariable::NotThreadLocal: return 0;
1379 case GlobalVariable::GeneralDynamicTLSModel: return 1;
1380 case GlobalVariable::LocalDynamicTLSModel: return 2;
1381 case GlobalVariable::InitialExecTLSModel: return 3;
1382 case GlobalVariable::LocalExecTLSModel: return 4;
1383 }
1384 llvm_unreachable("Invalid TLS model");
1385}
1386
1387static unsigned getEncodedComdatSelectionKind(const Comdat &C) {
1388 switch (C.getSelectionKind()) {
1389 case Comdat::Any:
1390 return bitc::COMDAT_SELECTION_KIND_ANY;
1391 case Comdat::ExactMatch:
1392 return bitc::COMDAT_SELECTION_KIND_EXACT_MATCH;
1393 case Comdat::Largest:
1394 return bitc::COMDAT_SELECTION_KIND_LARGEST;
1395 case Comdat::NoDeduplicate:
1396 return bitc::COMDAT_SELECTION_KIND_NO_DUPLICATES;
1397 case Comdat::SameSize:
1398 return bitc::COMDAT_SELECTION_KIND_SAME_SIZE;
1399 }
1400 llvm_unreachable("Invalid selection kind");
1401}
1402
1403static unsigned getEncodedUnnamedAddr(const GlobalValue &GV) {
1404 switch (GV.getUnnamedAddr()) {
1405 case GlobalValue::UnnamedAddr::None: return 0;
1406 case GlobalValue::UnnamedAddr::Local: return 2;
1407 case GlobalValue::UnnamedAddr::Global: return 1;
1408 }
1409 llvm_unreachable("Invalid unnamed_addr");
1410}
1411
1412size_t ModuleBitcodeWriter::addToStrtab(StringRef Str) {
1413 if (GenerateHash)
1414 Hasher.update(Str);
1415 return StrtabBuilder.add(S: Str);
1416}
1417
1418void ModuleBitcodeWriter::writeComdats() {
1419 SmallVector<unsigned, 64> Vals;
1420 for (const Comdat *C : VE.getComdats()) {
1421 // COMDAT: [strtab offset, strtab size, selection_kind]
1422 Vals.push_back(Elt: addToStrtab(Str: C->getName()));
1423 Vals.push_back(Elt: C->getName().size());
1424 Vals.push_back(Elt: getEncodedComdatSelectionKind(C: *C));
1425 Stream.EmitRecord(Code: bitc::MODULE_CODE_COMDAT, Vals, /*AbbrevToUse=*/Abbrev: 0);
1426 Vals.clear();
1427 }
1428}
1429
1430/// Write a record that will eventually hold the word offset of the
1431/// module-level VST. For now the offset is 0, which will be backpatched
1432/// after the real VST is written. Saves the bit offset to backpatch.
1433void ModuleBitcodeWriter::writeValueSymbolTableForwardDecl() {
1434 // Write a placeholder value in for the offset of the real VST,
1435 // which is written after the function blocks so that it can include
1436 // the offset of each function. The placeholder offset will be
1437 // updated when the real VST is written.
1438 auto Abbv = std::make_shared<BitCodeAbbrev>();
1439 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_VSTOFFSET));
1440 // Blocks are 32-bit aligned, so we can use a 32-bit word offset to
1441 // hold the real VST offset. Must use fixed instead of VBR as we don't
1442 // know how many VBR chunks to reserve ahead of time.
1443 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
1444 unsigned VSTOffsetAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1445
1446 // Emit the placeholder
1447 uint64_t Vals[] = {bitc::MODULE_CODE_VSTOFFSET, 0};
1448 Stream.EmitRecordWithAbbrev(Abbrev: VSTOffsetAbbrev, Vals);
1449
1450 // Compute and save the bit offset to the placeholder, which will be
1451 // patched when the real VST is written. We can simply subtract the 32-bit
1452 // fixed size from the current bit number to get the location to backpatch.
1453 VSTOffsetPlaceholder = Stream.GetCurrentBitNo() - 32;
1454}
1455
1456enum StringEncoding { SE_Char6, SE_Fixed7, SE_Fixed8 };
1457
1458/// Determine the encoding to use for the given string name and length.
1459static StringEncoding getStringEncoding(StringRef Str) {
1460 bool isChar6 = true;
1461 for (char C : Str) {
1462 if (isChar6)
1463 isChar6 = BitCodeAbbrevOp::isChar6(C);
1464 if ((unsigned char)C & 128)
1465 // don't bother scanning the rest.
1466 return SE_Fixed8;
1467 }
1468 if (isChar6)
1469 return SE_Char6;
1470 return SE_Fixed7;
1471}
1472
1473static_assert(sizeof(GlobalValue::SanitizerMetadata) <= sizeof(unsigned),
1474 "Sanitizer Metadata is too large for naive serialization.");
1475static unsigned
1476serializeSanitizerMetadata(const GlobalValue::SanitizerMetadata &Meta) {
1477 return Meta.NoAddress | (Meta.NoHWAddress << 1) |
1478 (Meta.Memtag << 2) | (Meta.IsDynInit << 3);
1479}
1480
1481/// Emit top-level description of module, including target triple, inline asm,
1482/// descriptors for global variables, and function prototype info.
1483/// Returns the bit offset to backpatch with the location of the real VST.
1484void ModuleBitcodeWriter::writeModuleInfo() {
1485 // Emit various pieces of data attached to a module.
1486 if (!M.getTargetTriple().empty())
1487 writeStringRecord(Stream, Code: bitc::MODULE_CODE_TRIPLE,
1488 Str: M.getTargetTriple().str(), AbbrevToUse: 0 /*TODO*/);
1489 const std::string &DL = M.getDataLayoutStr();
1490 if (!DL.empty())
1491 writeStringRecord(Stream, Code: bitc::MODULE_CODE_DATALAYOUT, Str: DL, AbbrevToUse: 0 /*TODO*/);
1492 if (!M.getModuleInlineAsm().empty())
1493 writeStringRecord(Stream, Code: bitc::MODULE_CODE_ASM, Str: M.getModuleInlineAsm(),
1494 AbbrevToUse: 0 /*TODO*/);
1495
1496 // Emit information about sections and GC, computing how many there are. Also
1497 // compute the maximum alignment value.
1498 std::map<std::string, unsigned> SectionMap;
1499 std::map<std::string, unsigned> GCMap;
1500 MaybeAlign MaxGVarAlignment;
1501 unsigned MaxGlobalType = 0;
1502 for (const GlobalVariable &GV : M.globals()) {
1503 if (MaybeAlign A = GV.getAlign())
1504 MaxGVarAlignment = !MaxGVarAlignment ? *A : std::max(a: *MaxGVarAlignment, b: *A);
1505 MaxGlobalType = std::max(a: MaxGlobalType, b: VE.getTypeID(T: GV.getValueType()));
1506 if (GV.hasSection()) {
1507 // Give section names unique ID's.
1508 unsigned &Entry = SectionMap[std::string(GV.getSection())];
1509 if (!Entry) {
1510 writeStringRecord(Stream, Code: bitc::MODULE_CODE_SECTIONNAME, Str: GV.getSection(),
1511 AbbrevToUse: 0 /*TODO*/);
1512 Entry = SectionMap.size();
1513 }
1514 }
1515 }
1516 for (const Function &F : M) {
1517 if (F.hasSection()) {
1518 // Give section names unique ID's.
1519 unsigned &Entry = SectionMap[std::string(F.getSection())];
1520 if (!Entry) {
1521 writeStringRecord(Stream, Code: bitc::MODULE_CODE_SECTIONNAME, Str: F.getSection(),
1522 AbbrevToUse: 0 /*TODO*/);
1523 Entry = SectionMap.size();
1524 }
1525 }
1526 if (F.hasGC()) {
1527 // Same for GC names.
1528 unsigned &Entry = GCMap[F.getGC()];
1529 if (!Entry) {
1530 writeStringRecord(Stream, Code: bitc::MODULE_CODE_GCNAME, Str: F.getGC(),
1531 AbbrevToUse: 0 /*TODO*/);
1532 Entry = GCMap.size();
1533 }
1534 }
1535 }
1536
1537 // Emit abbrev for globals, now that we know # sections and max alignment.
1538 unsigned SimpleGVarAbbrev = 0;
1539 if (!M.global_empty()) {
1540 // Add an abbrev for common globals with no visibility or thread localness.
1541 auto Abbv = std::make_shared<BitCodeAbbrev>();
1542 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_GLOBALVAR));
1543 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1544 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1545 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1546 Log2_32_Ceil(Value: MaxGlobalType+1)));
1547 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // AddrSpace << 2
1548 //| explicitType << 1
1549 //| constant
1550 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Initializer.
1551 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // Linkage.
1552 if (!MaxGVarAlignment) // Alignment.
1553 Abbv->Add(OpInfo: BitCodeAbbrevOp(0));
1554 else {
1555 unsigned MaxEncAlignment = getEncodedAlign(Alignment: MaxGVarAlignment);
1556 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1557 Log2_32_Ceil(Value: MaxEncAlignment+1)));
1558 }
1559 if (SectionMap.empty()) // Section.
1560 Abbv->Add(OpInfo: BitCodeAbbrevOp(0));
1561 else
1562 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1563 Log2_32_Ceil(Value: SectionMap.size()+1)));
1564 // Don't bother emitting vis + thread local.
1565 SimpleGVarAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1566 }
1567
1568 SmallVector<unsigned, 64> Vals;
1569 // Emit the module's source file name.
1570 {
1571 StringEncoding Bits = getStringEncoding(Str: M.getSourceFileName());
1572 BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
1573 if (Bits == SE_Char6)
1574 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
1575 else if (Bits == SE_Fixed7)
1576 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
1577
1578 // MODULE_CODE_SOURCE_FILENAME: [namechar x N]
1579 auto Abbv = std::make_shared<BitCodeAbbrev>();
1580 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
1581 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1582 Abbv->Add(OpInfo: AbbrevOpToUse);
1583 unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1584
1585 for (const auto P : M.getSourceFileName())
1586 Vals.push_back(Elt: (unsigned char)P);
1587
1588 // Emit the finished record.
1589 Stream.EmitRecord(Code: bitc::MODULE_CODE_SOURCE_FILENAME, Vals, Abbrev: FilenameAbbrev);
1590 Vals.clear();
1591 }
1592
1593 // Emit the global variable information.
1594 for (const GlobalVariable &GV : M.globals()) {
1595 unsigned AbbrevToUse = 0;
1596
1597 // GLOBALVAR: [strtab offset, strtab size, type, isconst, initid,
1598 // linkage, alignment, section, visibility, threadlocal,
1599 // unnamed_addr, externally_initialized, dllstorageclass,
1600 // comdat, attributes, DSO_Local, GlobalSanitizer, code_model]
1601 Vals.push_back(Elt: addToStrtab(Str: GV.getName()));
1602 Vals.push_back(Elt: GV.getName().size());
1603 Vals.push_back(Elt: VE.getTypeID(T: GV.getValueType()));
1604 Vals.push_back(Elt: GV.getType()->getAddressSpace() << 2 | 2 | GV.isConstant());
1605 Vals.push_back(Elt: GV.isDeclaration() ? 0 :
1606 (VE.getValueID(V: GV.getInitializer()) + 1));
1607 Vals.push_back(Elt: getEncodedLinkage(GV));
1608 Vals.push_back(Elt: getEncodedAlign(Alignment: GV.getAlign()));
1609 Vals.push_back(Elt: GV.hasSection() ? SectionMap[std::string(GV.getSection())]
1610 : 0);
1611 if (GV.isThreadLocal() ||
1612 GV.getVisibility() != GlobalValue::DefaultVisibility ||
1613 GV.getUnnamedAddr() != GlobalValue::UnnamedAddr::None ||
1614 GV.isExternallyInitialized() ||
1615 GV.getDLLStorageClass() != GlobalValue::DefaultStorageClass ||
1616 GV.hasComdat() || GV.hasAttributes() || GV.isDSOLocal() ||
1617 GV.hasPartition() || GV.hasSanitizerMetadata() || GV.getCodeModel()) {
1618 Vals.push_back(Elt: getEncodedVisibility(GV));
1619 Vals.push_back(Elt: getEncodedThreadLocalMode(GV));
1620 Vals.push_back(Elt: getEncodedUnnamedAddr(GV));
1621 Vals.push_back(Elt: GV.isExternallyInitialized());
1622 Vals.push_back(Elt: getEncodedDLLStorageClass(GV));
1623 Vals.push_back(Elt: GV.hasComdat() ? VE.getComdatID(C: GV.getComdat()) : 0);
1624
1625 auto AL = GV.getAttributesAsList(index: AttributeList::FunctionIndex);
1626 Vals.push_back(Elt: VE.getAttributeListID(PAL: AL));
1627
1628 Vals.push_back(Elt: GV.isDSOLocal());
1629 Vals.push_back(Elt: addToStrtab(Str: GV.getPartition()));
1630 Vals.push_back(Elt: GV.getPartition().size());
1631
1632 Vals.push_back(Elt: (GV.hasSanitizerMetadata() ? serializeSanitizerMetadata(
1633 Meta: GV.getSanitizerMetadata())
1634 : 0));
1635 Vals.push_back(Elt: GV.getCodeModelRaw());
1636 } else {
1637 AbbrevToUse = SimpleGVarAbbrev;
1638 }
1639
1640 Stream.EmitRecord(Code: bitc::MODULE_CODE_GLOBALVAR, Vals, Abbrev: AbbrevToUse);
1641 Vals.clear();
1642 }
1643
1644 // Emit the function proto information.
1645 for (const Function &F : M) {
1646 // FUNCTION: [strtab offset, strtab size, type, callingconv, isproto,
1647 // linkage, paramattrs, alignment, section, visibility, gc,
1648 // unnamed_addr, prologuedata, dllstorageclass, comdat,
1649 // prefixdata, personalityfn, DSO_Local, addrspace,
1650 // partition_strtab, partition_size, prefalign]
1651 Vals.push_back(Elt: addToStrtab(Str: F.getName()));
1652 Vals.push_back(Elt: F.getName().size());
1653 Vals.push_back(Elt: VE.getTypeID(T: F.getFunctionType()));
1654 Vals.push_back(Elt: F.getCallingConv());
1655 Vals.push_back(Elt: F.isDeclaration());
1656 Vals.push_back(Elt: getEncodedLinkage(GV: F));
1657 Vals.push_back(Elt: VE.getAttributeListID(PAL: F.getAttributes()));
1658 Vals.push_back(Elt: getEncodedAlign(Alignment: F.getAlign()));
1659 Vals.push_back(Elt: F.hasSection() ? SectionMap[std::string(F.getSection())]
1660 : 0);
1661 Vals.push_back(Elt: getEncodedVisibility(GV: F));
1662 Vals.push_back(Elt: F.hasGC() ? GCMap[F.getGC()] : 0);
1663 Vals.push_back(Elt: getEncodedUnnamedAddr(GV: F));
1664 Vals.push_back(Elt: F.hasPrologueData() ? (VE.getValueID(V: F.getPrologueData()) + 1)
1665 : 0);
1666 Vals.push_back(Elt: getEncodedDLLStorageClass(GV: F));
1667 Vals.push_back(Elt: F.hasComdat() ? VE.getComdatID(C: F.getComdat()) : 0);
1668 Vals.push_back(Elt: F.hasPrefixData() ? (VE.getValueID(V: F.getPrefixData()) + 1)
1669 : 0);
1670 Vals.push_back(
1671 Elt: F.hasPersonalityFn() ? (VE.getValueID(V: F.getPersonalityFn()) + 1) : 0);
1672
1673 Vals.push_back(Elt: F.isDSOLocal());
1674 Vals.push_back(Elt: F.getAddressSpace());
1675 Vals.push_back(Elt: addToStrtab(Str: F.getPartition()));
1676 Vals.push_back(Elt: F.getPartition().size());
1677 Vals.push_back(Elt: getEncodedAlign(Alignment: F.getPreferredAlignment()));
1678
1679 unsigned AbbrevToUse = 0;
1680 Stream.EmitRecord(Code: bitc::MODULE_CODE_FUNCTION, Vals, Abbrev: AbbrevToUse);
1681 Vals.clear();
1682 }
1683
1684 // Emit the alias information.
1685 for (const GlobalAlias &A : M.aliases()) {
1686 // ALIAS: [strtab offset, strtab size, alias type, aliasee val#, linkage,
1687 // visibility, dllstorageclass, threadlocal, unnamed_addr,
1688 // DSO_Local]
1689 Vals.push_back(Elt: addToStrtab(Str: A.getName()));
1690 Vals.push_back(Elt: A.getName().size());
1691 Vals.push_back(Elt: VE.getTypeID(T: A.getValueType()));
1692 Vals.push_back(Elt: A.getType()->getAddressSpace());
1693 Vals.push_back(Elt: VE.getValueID(V: A.getAliasee()));
1694 Vals.push_back(Elt: getEncodedLinkage(GV: A));
1695 Vals.push_back(Elt: getEncodedVisibility(GV: A));
1696 Vals.push_back(Elt: getEncodedDLLStorageClass(GV: A));
1697 Vals.push_back(Elt: getEncodedThreadLocalMode(GV: A));
1698 Vals.push_back(Elt: getEncodedUnnamedAddr(GV: A));
1699 Vals.push_back(Elt: A.isDSOLocal());
1700 Vals.push_back(Elt: addToStrtab(Str: A.getPartition()));
1701 Vals.push_back(Elt: A.getPartition().size());
1702
1703 unsigned AbbrevToUse = 0;
1704 Stream.EmitRecord(Code: bitc::MODULE_CODE_ALIAS, Vals, Abbrev: AbbrevToUse);
1705 Vals.clear();
1706 }
1707
1708 // Emit the ifunc information.
1709 for (const GlobalIFunc &I : M.ifuncs()) {
1710 // IFUNC: [strtab offset, strtab size, ifunc type, address space, resolver
1711 // val#, linkage, visibility, DSO_Local]
1712 Vals.push_back(Elt: addToStrtab(Str: I.getName()));
1713 Vals.push_back(Elt: I.getName().size());
1714 Vals.push_back(Elt: VE.getTypeID(T: I.getValueType()));
1715 Vals.push_back(Elt: I.getType()->getAddressSpace());
1716 Vals.push_back(Elt: VE.getValueID(V: I.getResolver()));
1717 Vals.push_back(Elt: getEncodedLinkage(GV: I));
1718 Vals.push_back(Elt: getEncodedVisibility(GV: I));
1719 Vals.push_back(Elt: I.isDSOLocal());
1720 Vals.push_back(Elt: addToStrtab(Str: I.getPartition()));
1721 Vals.push_back(Elt: I.getPartition().size());
1722 Stream.EmitRecord(Code: bitc::MODULE_CODE_IFUNC, Vals);
1723 Vals.clear();
1724 }
1725
1726 writeValueSymbolTableForwardDecl();
1727}
1728
1729static uint64_t getOptimizationFlags(const Value *V) {
1730 uint64_t Flags = 0;
1731
1732 if (const auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V)) {
1733 if (OBO->hasNoSignedWrap())
1734 Flags |= 1 << bitc::OBO_NO_SIGNED_WRAP;
1735 if (OBO->hasNoUnsignedWrap())
1736 Flags |= 1 << bitc::OBO_NO_UNSIGNED_WRAP;
1737 } else if (const auto *PEO = dyn_cast<PossiblyExactOperator>(Val: V)) {
1738 if (PEO->isExact())
1739 Flags |= 1 << bitc::PEO_EXACT;
1740 } else if (const auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: V)) {
1741 if (PDI->isDisjoint())
1742 Flags |= 1 << bitc::PDI_DISJOINT;
1743 } else if (const auto *FPMO = dyn_cast<FPMathOperator>(Val: V)) {
1744 if (FPMO->hasAllowReassoc())
1745 Flags |= bitc::AllowReassoc;
1746 if (FPMO->hasNoNaNs())
1747 Flags |= bitc::NoNaNs;
1748 if (FPMO->hasNoInfs())
1749 Flags |= bitc::NoInfs;
1750 if (FPMO->hasNoSignedZeros())
1751 Flags |= bitc::NoSignedZeros;
1752 if (FPMO->hasAllowReciprocal())
1753 Flags |= bitc::AllowReciprocal;
1754 if (FPMO->hasAllowContract())
1755 Flags |= bitc::AllowContract;
1756 if (FPMO->hasApproxFunc())
1757 Flags |= bitc::ApproxFunc;
1758 } else if (const auto *NNI = dyn_cast<PossiblyNonNegInst>(Val: V)) {
1759 if (NNI->hasNonNeg())
1760 Flags |= 1 << bitc::PNNI_NON_NEG;
1761 } else if (const auto *TI = dyn_cast<TruncInst>(Val: V)) {
1762 if (TI->hasNoSignedWrap())
1763 Flags |= 1 << bitc::TIO_NO_SIGNED_WRAP;
1764 if (TI->hasNoUnsignedWrap())
1765 Flags |= 1 << bitc::TIO_NO_UNSIGNED_WRAP;
1766 } else if (const auto *GEP = dyn_cast<GEPOperator>(Val: V)) {
1767 if (GEP->isInBounds())
1768 Flags |= 1 << bitc::GEP_INBOUNDS;
1769 if (GEP->hasNoUnsignedSignedWrap())
1770 Flags |= 1 << bitc::GEP_NUSW;
1771 if (GEP->hasNoUnsignedWrap())
1772 Flags |= 1 << bitc::GEP_NUW;
1773 } else if (const auto *ICmp = dyn_cast<ICmpInst>(Val: V)) {
1774 if (ICmp->hasSameSign())
1775 Flags |= 1 << bitc::ICMP_SAME_SIGN;
1776 }
1777
1778 return Flags;
1779}
1780
1781void ModuleBitcodeWriter::writeValueAsMetadata(
1782 const ValueAsMetadata *MD, SmallVectorImpl<uint64_t> &Record) {
1783 // Mimic an MDNode with a value as one operand.
1784 Value *V = MD->getValue();
1785 Record.push_back(Elt: VE.getTypeID(T: V->getType()));
1786 Record.push_back(Elt: VE.getValueID(V));
1787 Stream.EmitRecord(Code: bitc::METADATA_VALUE, Vals: Record, Abbrev: 0);
1788 Record.clear();
1789}
1790
1791void ModuleBitcodeWriter::writeMDTuple(const MDTuple *N,
1792 SmallVectorImpl<uint64_t> &Record,
1793 unsigned Abbrev) {
1794 for (const MDOperand &MDO : N->operands()) {
1795 Metadata *MD = MDO;
1796 assert(!(MD && isa<LocalAsMetadata>(MD)) &&
1797 "Unexpected function-local metadata");
1798 Record.push_back(Elt: VE.getMetadataOrNullID(MD));
1799 }
1800 Stream.EmitRecord(Code: N->isDistinct() ? bitc::METADATA_DISTINCT_NODE
1801 : bitc::METADATA_NODE,
1802 Vals: Record, Abbrev);
1803 Record.clear();
1804}
1805
1806unsigned ModuleBitcodeWriter::createDILocationAbbrev() {
1807 // Assume the column is usually under 128, and always output the inlined-at
1808 // location (it's never more expensive than building an array size 1).
1809 auto Abbv = std::make_shared<BitCodeAbbrev>();
1810 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_LOCATION));
1811 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isDistinct
1812 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // line
1813 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // column
1814 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // scope
1815 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // inlinedAt
1816 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isImplicitCode
1817 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // atomGroup
1818 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // atomRank
1819 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
1820}
1821
1822void ModuleBitcodeWriter::writeDILocation(const DILocation *N,
1823 SmallVectorImpl<uint64_t> &Record,
1824 unsigned &Abbrev) {
1825 if (!Abbrev)
1826 Abbrev = createDILocationAbbrev();
1827
1828 Record.push_back(Elt: N->isDistinct());
1829 Record.push_back(Elt: N->getLine());
1830 Record.push_back(Elt: N->getColumn());
1831 Record.push_back(Elt: VE.getMetadataID(MD: N->getScope()));
1832 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getInlinedAt()));
1833 Record.push_back(Elt: N->isImplicitCode());
1834 Record.push_back(Elt: N->getAtomGroup());
1835 Record.push_back(Elt: N->getAtomRank());
1836 Stream.EmitRecord(Code: bitc::METADATA_LOCATION, Vals: Record, Abbrev);
1837 Record.clear();
1838}
1839
1840unsigned ModuleBitcodeWriter::createGenericDINodeAbbrev() {
1841 // Assume the column is usually under 128, and always output the inlined-at
1842 // location (it's never more expensive than building an array size 1).
1843 auto Abbv = std::make_shared<BitCodeAbbrev>();
1844 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_GENERIC_DEBUG));
1845 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1846 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1847 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1848 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1849 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1850 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1851 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
1852}
1853
1854void ModuleBitcodeWriter::writeGenericDINode(const GenericDINode *N,
1855 SmallVectorImpl<uint64_t> &Record,
1856 unsigned &Abbrev) {
1857 if (!Abbrev)
1858 Abbrev = createGenericDINodeAbbrev();
1859
1860 Record.push_back(Elt: N->isDistinct());
1861 Record.push_back(Elt: N->getTag());
1862 Record.push_back(Elt: 0); // Per-tag version field; unused for now.
1863
1864 for (auto &I : N->operands())
1865 Record.push_back(Elt: VE.getMetadataOrNullID(MD: I));
1866
1867 Stream.EmitRecord(Code: bitc::METADATA_GENERIC_DEBUG, Vals: Record, Abbrev);
1868 Record.clear();
1869}
1870
1871void ModuleBitcodeWriter::writeDISubrange(const DISubrange *N,
1872 SmallVectorImpl<uint64_t> &Record,
1873 unsigned Abbrev) {
1874 const uint64_t Version = 2 << 1;
1875 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
1876 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawCountNode()));
1877 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
1878 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
1879 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
1880
1881 Stream.EmitRecord(Code: bitc::METADATA_SUBRANGE, Vals: Record, Abbrev);
1882 Record.clear();
1883}
1884
1885void ModuleBitcodeWriter::writeDIGenericSubrange(
1886 const DIGenericSubrange *N, SmallVectorImpl<uint64_t> &Record,
1887 unsigned Abbrev) {
1888 Record.push_back(Elt: (uint64_t)N->isDistinct());
1889 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawCountNode()));
1890 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
1891 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
1892 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
1893
1894 Stream.EmitRecord(Code: bitc::METADATA_GENERIC_SUBRANGE, Vals: Record, Abbrev);
1895 Record.clear();
1896}
1897
1898void ModuleBitcodeWriter::writeDIEnumerator(const DIEnumerator *N,
1899 SmallVectorImpl<uint64_t> &Record,
1900 unsigned Abbrev) {
1901 const uint64_t IsBigInt = 1 << 2;
1902 Record.push_back(Elt: IsBigInt | (N->isUnsigned() << 1) | N->isDistinct());
1903 Record.push_back(Elt: N->getValue().getBitWidth());
1904 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1905 emitWideAPInt(Vals&: Record, A: N->getValue());
1906
1907 Stream.EmitRecord(Code: bitc::METADATA_ENUMERATOR, Vals: Record, Abbrev);
1908 Record.clear();
1909}
1910
1911void ModuleBitcodeWriter::writeDIBasicType(const DIBasicType *N,
1912 SmallVectorImpl<uint64_t> &Record,
1913 unsigned Abbrev) {
1914 const unsigned SizeIsMetadata = 0x2;
1915 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
1916 Record.push_back(Elt: N->getTag());
1917 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1918 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
1919 Record.push_back(Elt: N->getAlignInBits());
1920 Record.push_back(Elt: N->getEncoding());
1921 Record.push_back(Elt: N->getFlags());
1922 Record.push_back(Elt: N->getNumExtraInhabitants());
1923 Record.push_back(Elt: N->getDataSizeInBits());
1924
1925 Stream.EmitRecord(Code: bitc::METADATA_BASIC_TYPE, Vals: Record, Abbrev);
1926 Record.clear();
1927}
1928
1929void ModuleBitcodeWriter::writeDIFixedPointType(
1930 const DIFixedPointType *N, SmallVectorImpl<uint64_t> &Record,
1931 unsigned Abbrev) {
1932 const unsigned SizeIsMetadata = 0x2;
1933 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
1934 Record.push_back(Elt: N->getTag());
1935 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1936 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
1937 Record.push_back(Elt: N->getAlignInBits());
1938 Record.push_back(Elt: N->getEncoding());
1939 Record.push_back(Elt: N->getFlags());
1940 Record.push_back(Elt: N->getKind());
1941 Record.push_back(Elt: N->getFactorRaw());
1942
1943 auto WriteWideInt = [&](const APInt &Value) {
1944 // Write an encoded word that holds the number of active words and
1945 // the number of bits.
1946 uint64_t NumWords = Value.getActiveWords();
1947 uint64_t Encoded = (NumWords << 32) | Value.getBitWidth();
1948 Record.push_back(Elt: Encoded);
1949 emitWideAPInt(Vals&: Record, A: Value);
1950 };
1951
1952 WriteWideInt(N->getNumeratorRaw());
1953 WriteWideInt(N->getDenominatorRaw());
1954
1955 Stream.EmitRecord(Code: bitc::METADATA_FIXED_POINT_TYPE, Vals: Record, Abbrev);
1956 Record.clear();
1957}
1958
1959void ModuleBitcodeWriter::writeDIStringType(const DIStringType *N,
1960 SmallVectorImpl<uint64_t> &Record,
1961 unsigned Abbrev) {
1962 const unsigned SizeIsMetadata = 0x2;
1963 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
1964 Record.push_back(Elt: N->getTag());
1965 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1966 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLength()));
1967 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLengthExp()));
1968 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLocationExp()));
1969 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
1970 Record.push_back(Elt: N->getAlignInBits());
1971 Record.push_back(Elt: N->getEncoding());
1972
1973 Stream.EmitRecord(Code: bitc::METADATA_STRING_TYPE, Vals: Record, Abbrev);
1974 Record.clear();
1975}
1976
1977void ModuleBitcodeWriter::writeDIDerivedType(const DIDerivedType *N,
1978 SmallVectorImpl<uint64_t> &Record,
1979 unsigned Abbrev) {
1980 const unsigned SizeIsMetadata = 0x2;
1981 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
1982 Record.push_back(Elt: N->getTag());
1983 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1984 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
1985 Record.push_back(Elt: N->getLine());
1986 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
1987 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
1988 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
1989 Record.push_back(Elt: N->getAlignInBits());
1990 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawOffsetInBits()));
1991 Record.push_back(Elt: N->getFlags());
1992 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getExtraData()));
1993
1994 // DWARF address space is encoded as N->getDWARFAddressSpace() + 1. 0 means
1995 // that there is no DWARF address space associated with DIDerivedType.
1996 if (const auto &DWARFAddressSpace = N->getDWARFAddressSpace())
1997 Record.push_back(Elt: *DWARFAddressSpace + 1);
1998 else
1999 Record.push_back(Elt: 0);
2000
2001 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2002
2003 if (auto PtrAuthData = N->getPtrAuthData())
2004 Record.push_back(Elt: PtrAuthData->RawData);
2005 else
2006 Record.push_back(Elt: 0);
2007
2008 Stream.EmitRecord(Code: bitc::METADATA_DERIVED_TYPE, Vals: Record, Abbrev);
2009 Record.clear();
2010}
2011
2012void ModuleBitcodeWriter::writeDISubrangeType(const DISubrangeType *N,
2013 SmallVectorImpl<uint64_t> &Record,
2014 unsigned Abbrev) {
2015 const unsigned SizeIsMetadata = 0x2;
2016 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2017 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2018 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2019 Record.push_back(Elt: N->getLine());
2020 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2021 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2022 Record.push_back(Elt: N->getAlignInBits());
2023 Record.push_back(Elt: N->getFlags());
2024 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2025 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
2026 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
2027 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
2028 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawBias()));
2029
2030 Stream.EmitRecord(Code: bitc::METADATA_SUBRANGE_TYPE, Vals: Record, Abbrev);
2031 Record.clear();
2032}
2033
2034void ModuleBitcodeWriter::writeDICompositeType(
2035 const DICompositeType *N, SmallVectorImpl<uint64_t> &Record,
2036 unsigned Abbrev) {
2037 const unsigned IsNotUsedInOldTypeRef = 0x2;
2038 const unsigned SizeIsMetadata = 0x4;
2039 Record.push_back(Elt: SizeIsMetadata | IsNotUsedInOldTypeRef |
2040 (unsigned)N->isDistinct());
2041 Record.push_back(Elt: N->getTag());
2042 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2043 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2044 Record.push_back(Elt: N->getLine());
2045 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2046 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2047 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2048 Record.push_back(Elt: N->getAlignInBits());
2049 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawOffsetInBits()));
2050 Record.push_back(Elt: N->getFlags());
2051 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2052 Record.push_back(Elt: N->getRuntimeLang());
2053 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getVTableHolder()));
2054 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams().get()));
2055 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawIdentifier()));
2056 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDiscriminator()));
2057 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawDataLocation()));
2058 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawAssociated()));
2059 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawAllocated()));
2060 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawRank()));
2061 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2062 Record.push_back(Elt: N->getNumExtraInhabitants());
2063 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSpecification()));
2064 Record.push_back(
2065 Elt: N->getEnumKind().value_or(u: dwarf::DW_APPLE_ENUM_KIND_invalid));
2066 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawBitStride()));
2067
2068 Stream.EmitRecord(Code: bitc::METADATA_COMPOSITE_TYPE, Vals: Record, Abbrev);
2069 Record.clear();
2070}
2071
2072void ModuleBitcodeWriter::writeDISubroutineType(
2073 const DISubroutineType *N, SmallVectorImpl<uint64_t> &Record,
2074 unsigned Abbrev) {
2075 const unsigned HasNoOldTypeRefs = 0x2;
2076 Record.push_back(Elt: HasNoOldTypeRefs | (unsigned)N->isDistinct());
2077 Record.push_back(Elt: N->getFlags());
2078 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTypeArray().get()));
2079 Record.push_back(Elt: N->getCC());
2080
2081 Stream.EmitRecord(Code: bitc::METADATA_SUBROUTINE_TYPE, Vals: Record, Abbrev);
2082 Record.clear();
2083}
2084
2085void ModuleBitcodeWriter::writeDIFile(const DIFile *N,
2086 SmallVectorImpl<uint64_t> &Record,
2087 unsigned Abbrev) {
2088 Record.push_back(Elt: N->isDistinct());
2089 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFilename()));
2090 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawDirectory()));
2091 if (N->getRawChecksum()) {
2092 Record.push_back(Elt: N->getRawChecksum()->Kind);
2093 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawChecksum()->Value));
2094 } else {
2095 // Maintain backwards compatibility with the old internal representation of
2096 // CSK_None in ChecksumKind by writing nulls here when Checksum is None.
2097 Record.push_back(Elt: 0);
2098 Record.push_back(Elt: VE.getMetadataOrNullID(MD: nullptr));
2099 }
2100 auto Source = N->getRawSource();
2101 if (Source)
2102 Record.push_back(Elt: VE.getMetadataOrNullID(MD: Source));
2103
2104 Stream.EmitRecord(Code: bitc::METADATA_FILE, Vals: Record, Abbrev);
2105 Record.clear();
2106}
2107
2108void ModuleBitcodeWriter::writeDICompileUnit(const DICompileUnit *N,
2109 SmallVectorImpl<uint64_t> &Record,
2110 unsigned Abbrev) {
2111 assert(N->isDistinct() && "Expected distinct compile units");
2112 Record.push_back(/* IsDistinct */ Elt: true);
2113
2114 auto Lang = N->getSourceLanguage();
2115 Record.push_back(Elt: Lang.getName());
2116 // Set bit so the MetadataLoader can distniguish between versioned and
2117 // unversioned names.
2118 if (Lang.hasVersionedName())
2119 Record.back() ^= (uint64_t(1) << 63);
2120
2121 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2122 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawProducer()));
2123 Record.push_back(Elt: N->isOptimized());
2124 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFlags()));
2125 Record.push_back(Elt: N->getRuntimeVersion());
2126 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSplitDebugFilename()));
2127 Record.push_back(Elt: N->getEmissionKind());
2128 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getEnumTypes().get()));
2129 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRetainedTypes().get()));
2130 Record.push_back(/* subprograms */ Elt: 0);
2131 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getGlobalVariables().get()));
2132 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getImportedEntities().get()));
2133 Record.push_back(Elt: N->getDWOId());
2134 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getMacros().get()));
2135 Record.push_back(Elt: N->getSplitDebugInlining());
2136 Record.push_back(Elt: N->getDebugInfoForProfiling());
2137 Record.push_back(Elt: (unsigned)N->getNameTableKind());
2138 Record.push_back(Elt: N->getRangesBaseAddress());
2139 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSysRoot()));
2140 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSDK()));
2141 Record.push_back(Elt: Lang.hasVersionedName() ? Lang.getVersion() : 0);
2142
2143 Stream.EmitRecord(Code: bitc::METADATA_COMPILE_UNIT, Vals: Record, Abbrev);
2144 Record.clear();
2145}
2146
2147void ModuleBitcodeWriter::writeDISubprogram(const DISubprogram *N,
2148 SmallVectorImpl<uint64_t> &Record,
2149 unsigned Abbrev) {
2150 const uint64_t HasUnitFlag = 1 << 1;
2151 const uint64_t HasSPFlagsFlag = 1 << 2;
2152 Record.push_back(Elt: uint64_t(N->isDistinct()) | HasUnitFlag | HasSPFlagsFlag);
2153 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2154 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2155 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLinkageName()));
2156 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2157 Record.push_back(Elt: N->getLine());
2158 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2159 Record.push_back(Elt: N->getScopeLine());
2160 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getContainingType()));
2161 Record.push_back(Elt: N->getSPFlags());
2162 Record.push_back(Elt: N->getVirtualIndex());
2163 Record.push_back(Elt: N->getFlags());
2164 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUnit()));
2165 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams().get()));
2166 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDeclaration()));
2167 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRetainedNodes().get()));
2168 Record.push_back(Elt: N->getThisAdjustment());
2169 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getThrownTypes().get()));
2170 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2171 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawTargetFuncName()));
2172 Record.push_back(Elt: N->getKeyInstructionsEnabled());
2173
2174 Stream.EmitRecord(Code: bitc::METADATA_SUBPROGRAM, Vals: Record, Abbrev);
2175 Record.clear();
2176}
2177
2178void ModuleBitcodeWriter::writeDILexicalBlock(const DILexicalBlock *N,
2179 SmallVectorImpl<uint64_t> &Record,
2180 unsigned Abbrev) {
2181 Record.push_back(Elt: N->isDistinct());
2182 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2183 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2184 Record.push_back(Elt: N->getLine());
2185 Record.push_back(Elt: N->getColumn());
2186
2187 Stream.EmitRecord(Code: bitc::METADATA_LEXICAL_BLOCK, Vals: Record, Abbrev);
2188 Record.clear();
2189}
2190
2191void ModuleBitcodeWriter::writeDILexicalBlockFile(
2192 const DILexicalBlockFile *N, SmallVectorImpl<uint64_t> &Record,
2193 unsigned Abbrev) {
2194 Record.push_back(Elt: N->isDistinct());
2195 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2196 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2197 Record.push_back(Elt: N->getDiscriminator());
2198
2199 Stream.EmitRecord(Code: bitc::METADATA_LEXICAL_BLOCK_FILE, Vals: Record, Abbrev);
2200 Record.clear();
2201}
2202
2203void ModuleBitcodeWriter::writeDICommonBlock(const DICommonBlock *N,
2204 SmallVectorImpl<uint64_t> &Record,
2205 unsigned Abbrev) {
2206 Record.push_back(Elt: N->isDistinct());
2207 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2208 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDecl()));
2209 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2210 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2211 Record.push_back(Elt: N->getLineNo());
2212
2213 Stream.EmitRecord(Code: bitc::METADATA_COMMON_BLOCK, Vals: Record, Abbrev);
2214 Record.clear();
2215}
2216
2217void ModuleBitcodeWriter::writeDINamespace(const DINamespace *N,
2218 SmallVectorImpl<uint64_t> &Record,
2219 unsigned Abbrev) {
2220 Record.push_back(Elt: N->isDistinct() | N->getExportSymbols() << 1);
2221 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2222 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2223
2224 Stream.EmitRecord(Code: bitc::METADATA_NAMESPACE, Vals: Record, Abbrev);
2225 Record.clear();
2226}
2227
2228void ModuleBitcodeWriter::writeDIMacro(const DIMacro *N,
2229 SmallVectorImpl<uint64_t> &Record,
2230 unsigned Abbrev) {
2231 Record.push_back(Elt: N->isDistinct());
2232 Record.push_back(Elt: N->getMacinfoType());
2233 Record.push_back(Elt: N->getLine());
2234 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2235 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawValue()));
2236
2237 Stream.EmitRecord(Code: bitc::METADATA_MACRO, Vals: Record, Abbrev);
2238 Record.clear();
2239}
2240
2241void ModuleBitcodeWriter::writeDIMacroFile(const DIMacroFile *N,
2242 SmallVectorImpl<uint64_t> &Record,
2243 unsigned Abbrev) {
2244 Record.push_back(Elt: N->isDistinct());
2245 Record.push_back(Elt: N->getMacinfoType());
2246 Record.push_back(Elt: N->getLine());
2247 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2248 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2249
2250 Stream.EmitRecord(Code: bitc::METADATA_MACRO_FILE, Vals: Record, Abbrev);
2251 Record.clear();
2252}
2253
2254void ModuleBitcodeWriter::writeDIArgList(const DIArgList *N,
2255 SmallVectorImpl<uint64_t> &Record) {
2256 Record.reserve(N: N->getArgs().size());
2257 for (ValueAsMetadata *MD : N->getArgs())
2258 Record.push_back(Elt: VE.getMetadataID(MD));
2259
2260 Stream.EmitRecord(Code: bitc::METADATA_ARG_LIST, Vals: Record);
2261 Record.clear();
2262}
2263
2264void ModuleBitcodeWriter::writeDIModule(const DIModule *N,
2265 SmallVectorImpl<uint64_t> &Record,
2266 unsigned Abbrev) {
2267 Record.push_back(Elt: N->isDistinct());
2268 for (auto &I : N->operands())
2269 Record.push_back(Elt: VE.getMetadataOrNullID(MD: I));
2270 Record.push_back(Elt: N->getLineNo());
2271 Record.push_back(Elt: N->getIsDecl());
2272
2273 Stream.EmitRecord(Code: bitc::METADATA_MODULE, Vals: Record, Abbrev);
2274 Record.clear();
2275}
2276
2277void ModuleBitcodeWriter::writeDIAssignID(const DIAssignID *N,
2278 SmallVectorImpl<uint64_t> &Record,
2279 unsigned Abbrev) {
2280 // There are no arguments for this metadata type.
2281 Record.push_back(Elt: N->isDistinct());
2282 Stream.EmitRecord(Code: bitc::METADATA_ASSIGN_ID, Vals: Record, Abbrev);
2283 Record.clear();
2284}
2285
2286void ModuleBitcodeWriter::writeDITemplateTypeParameter(
2287 const DITemplateTypeParameter *N, SmallVectorImpl<uint64_t> &Record,
2288 unsigned Abbrev) {
2289 Record.push_back(Elt: N->isDistinct());
2290 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2291 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2292 Record.push_back(Elt: N->isDefault());
2293
2294 Stream.EmitRecord(Code: bitc::METADATA_TEMPLATE_TYPE, Vals: Record, Abbrev);
2295 Record.clear();
2296}
2297
2298void ModuleBitcodeWriter::writeDITemplateValueParameter(
2299 const DITemplateValueParameter *N, SmallVectorImpl<uint64_t> &Record,
2300 unsigned Abbrev) {
2301 Record.push_back(Elt: N->isDistinct());
2302 Record.push_back(Elt: N->getTag());
2303 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2304 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2305 Record.push_back(Elt: N->isDefault());
2306 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getValue()));
2307
2308 Stream.EmitRecord(Code: bitc::METADATA_TEMPLATE_VALUE, Vals: Record, Abbrev);
2309 Record.clear();
2310}
2311
2312void ModuleBitcodeWriter::writeDIGlobalVariable(
2313 const DIGlobalVariable *N, SmallVectorImpl<uint64_t> &Record,
2314 unsigned Abbrev) {
2315 const uint64_t Version = 2 << 1;
2316 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
2317 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2318 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2319 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLinkageName()));
2320 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2321 Record.push_back(Elt: N->getLine());
2322 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2323 Record.push_back(Elt: N->isLocalToUnit());
2324 Record.push_back(Elt: N->isDefinition());
2325 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStaticDataMemberDeclaration()));
2326 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams()));
2327 Record.push_back(Elt: N->getAlignInBits());
2328 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2329
2330 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_VAR, Vals: Record, Abbrev);
2331 Record.clear();
2332}
2333
2334void ModuleBitcodeWriter::writeDILocalVariable(
2335 const DILocalVariable *N, SmallVectorImpl<uint64_t> &Record,
2336 unsigned Abbrev) {
2337 // In order to support all possible bitcode formats in BitcodeReader we need
2338 // to distinguish the following cases:
2339 // 1) Record has no artificial tag (Record[1]),
2340 // has no obsolete inlinedAt field (Record[9]).
2341 // In this case Record size will be 8, HasAlignment flag is false.
2342 // 2) Record has artificial tag (Record[1]),
2343 // has no obsolete inlignedAt field (Record[9]).
2344 // In this case Record size will be 9, HasAlignment flag is false.
2345 // 3) Record has both artificial tag (Record[1]) and
2346 // obsolete inlignedAt field (Record[9]).
2347 // In this case Record size will be 10, HasAlignment flag is false.
2348 // 4) Record has neither artificial tag, nor inlignedAt field, but
2349 // HasAlignment flag is true and Record[8] contains alignment value.
2350 const uint64_t HasAlignmentFlag = 1 << 1;
2351 Record.push_back(Elt: (uint64_t)N->isDistinct() | HasAlignmentFlag);
2352 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2353 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2354 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2355 Record.push_back(Elt: N->getLine());
2356 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2357 Record.push_back(Elt: N->getArg());
2358 Record.push_back(Elt: N->getFlags());
2359 Record.push_back(Elt: N->getAlignInBits());
2360 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2361
2362 Stream.EmitRecord(Code: bitc::METADATA_LOCAL_VAR, Vals: Record, Abbrev);
2363 Record.clear();
2364}
2365
2366void ModuleBitcodeWriter::writeDILabel(
2367 const DILabel *N, SmallVectorImpl<uint64_t> &Record,
2368 unsigned Abbrev) {
2369 uint64_t IsArtificialFlag = uint64_t(N->isArtificial()) << 1;
2370 Record.push_back(Elt: (uint64_t)N->isDistinct() | IsArtificialFlag);
2371 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2372 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2373 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2374 Record.push_back(Elt: N->getLine());
2375 Record.push_back(Elt: N->getColumn());
2376 Record.push_back(Elt: N->getCoroSuspendIdx().has_value()
2377 ? (uint64_t)N->getCoroSuspendIdx().value()
2378 : std::numeric_limits<uint64_t>::max());
2379
2380 Stream.EmitRecord(Code: bitc::METADATA_LABEL, Vals: Record, Abbrev);
2381 Record.clear();
2382}
2383
2384void ModuleBitcodeWriter::writeDIExpression(const DIExpression *N,
2385 SmallVectorImpl<uint64_t> &Record,
2386 unsigned Abbrev) {
2387 Record.reserve(N: N->getElements().size() + 1);
2388 const uint64_t Version = 3 << 1;
2389 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
2390 Record.append(in_start: N->elements_begin(), in_end: N->elements_end());
2391
2392 Stream.EmitRecord(Code: bitc::METADATA_EXPRESSION, Vals: Record, Abbrev);
2393 Record.clear();
2394}
2395
2396void ModuleBitcodeWriter::writeDIGlobalVariableExpression(
2397 const DIGlobalVariableExpression *N, SmallVectorImpl<uint64_t> &Record,
2398 unsigned Abbrev) {
2399 Record.push_back(Elt: N->isDistinct());
2400 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getVariable()));
2401 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getExpression()));
2402
2403 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_VAR_EXPR, Vals: Record, Abbrev);
2404 Record.clear();
2405}
2406
2407void ModuleBitcodeWriter::writeDIObjCProperty(const DIObjCProperty *N,
2408 SmallVectorImpl<uint64_t> &Record,
2409 unsigned Abbrev) {
2410 Record.push_back(Elt: N->isDistinct());
2411 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2412 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2413 Record.push_back(Elt: N->getLine());
2414 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSetterName()));
2415 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawGetterName()));
2416 Record.push_back(Elt: N->getAttributes());
2417 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2418
2419 Stream.EmitRecord(Code: bitc::METADATA_OBJC_PROPERTY, Vals: Record, Abbrev);
2420 Record.clear();
2421}
2422
2423void ModuleBitcodeWriter::writeDIImportedEntity(
2424 const DIImportedEntity *N, SmallVectorImpl<uint64_t> &Record,
2425 unsigned Abbrev) {
2426 Record.push_back(Elt: N->isDistinct());
2427 Record.push_back(Elt: N->getTag());
2428 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2429 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getEntity()));
2430 Record.push_back(Elt: N->getLine());
2431 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2432 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFile()));
2433 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2434
2435 Stream.EmitRecord(Code: bitc::METADATA_IMPORTED_ENTITY, Vals: Record, Abbrev);
2436 Record.clear();
2437}
2438
2439unsigned ModuleBitcodeWriter::createNamedMetadataAbbrev() {
2440 auto Abbv = std::make_shared<BitCodeAbbrev>();
2441 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_NAME));
2442 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2443 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2444 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
2445}
2446
2447void ModuleBitcodeWriter::writeNamedMetadata(
2448 SmallVectorImpl<uint64_t> &Record) {
2449 if (M.named_metadata_empty())
2450 return;
2451
2452 unsigned Abbrev = createNamedMetadataAbbrev();
2453 for (const NamedMDNode &NMD : M.named_metadata()) {
2454 // Write name.
2455 StringRef Str = NMD.getName();
2456 Record.append(in_start: Str.bytes_begin(), in_end: Str.bytes_end());
2457 Stream.EmitRecord(Code: bitc::METADATA_NAME, Vals: Record, Abbrev);
2458 Record.clear();
2459
2460 // Write named metadata operands.
2461 for (const MDNode *N : NMD.operands())
2462 Record.push_back(Elt: VE.getMetadataID(MD: N));
2463 Stream.EmitRecord(Code: bitc::METADATA_NAMED_NODE, Vals: Record, Abbrev: 0);
2464 Record.clear();
2465 }
2466}
2467
2468unsigned ModuleBitcodeWriter::createMetadataStringsAbbrev() {
2469 auto Abbv = std::make_shared<BitCodeAbbrev>();
2470 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_STRINGS));
2471 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // # of strings
2472 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // offset to chars
2473 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
2474 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
2475}
2476
2477/// Write out a record for MDString.
2478///
2479/// All the metadata strings in a metadata block are emitted in a single
2480/// record. The sizes and strings themselves are shoved into a blob.
2481void ModuleBitcodeWriter::writeMetadataStrings(
2482 ArrayRef<const Metadata *> Strings, SmallVectorImpl<uint64_t> &Record) {
2483 if (Strings.empty())
2484 return;
2485
2486 // Start the record with the number of strings.
2487 Record.push_back(Elt: bitc::METADATA_STRINGS);
2488 Record.push_back(Elt: Strings.size());
2489
2490 // Emit the sizes of the strings in the blob.
2491 SmallString<256> Blob;
2492 {
2493 BitstreamWriter W(Blob);
2494 for (const Metadata *MD : Strings)
2495 W.EmitVBR(Val: cast<MDString>(Val: MD)->getLength(), NumBits: 6);
2496 W.FlushToWord();
2497 }
2498
2499 // Add the offset to the strings to the record.
2500 Record.push_back(Elt: Blob.size());
2501
2502 // Add the strings to the blob.
2503 for (const Metadata *MD : Strings)
2504 Blob.append(RHS: cast<MDString>(Val: MD)->getString());
2505
2506 // Emit the final record.
2507 Stream.EmitRecordWithBlob(Abbrev: createMetadataStringsAbbrev(), Vals: Record, Blob);
2508 Record.clear();
2509}
2510
2511// Generates an enum to use as an index in the Abbrev array of Metadata record.
2512enum MetadataAbbrev : unsigned {
2513#define HANDLE_MDNODE_LEAF(CLASS) CLASS##AbbrevID,
2514#include "llvm/IR/Metadata.def"
2515 LastPlusOne
2516};
2517
2518void ModuleBitcodeWriter::writeMetadataRecords(
2519 ArrayRef<const Metadata *> MDs, SmallVectorImpl<uint64_t> &Record,
2520 std::vector<unsigned> *MDAbbrevs, std::vector<uint64_t> *IndexPos) {
2521 if (MDs.empty())
2522 return;
2523
2524 // Initialize MDNode abbreviations.
2525#define HANDLE_MDNODE_LEAF(CLASS) unsigned CLASS##Abbrev = 0;
2526#include "llvm/IR/Metadata.def"
2527
2528 for (const Metadata *MD : MDs) {
2529 if (IndexPos)
2530 IndexPos->push_back(x: Stream.GetCurrentBitNo());
2531 if (const MDNode *N = dyn_cast<MDNode>(Val: MD)) {
2532 assert(N->isResolved() && "Expected forward references to be resolved");
2533
2534 switch (N->getMetadataID()) {
2535 default:
2536 llvm_unreachable("Invalid MDNode subclass");
2537#define HANDLE_MDNODE_LEAF(CLASS) \
2538 case Metadata::CLASS##Kind: \
2539 if (MDAbbrevs) \
2540 write##CLASS(cast<CLASS>(N), Record, \
2541 (*MDAbbrevs)[MetadataAbbrev::CLASS##AbbrevID]); \
2542 else \
2543 write##CLASS(cast<CLASS>(N), Record, CLASS##Abbrev); \
2544 continue;
2545#include "llvm/IR/Metadata.def"
2546 }
2547 }
2548 if (auto *AL = dyn_cast<DIArgList>(Val: MD)) {
2549 writeDIArgList(N: AL, Record);
2550 continue;
2551 }
2552 writeValueAsMetadata(MD: cast<ValueAsMetadata>(Val: MD), Record);
2553 }
2554}
2555
2556void ModuleBitcodeWriter::writeModuleMetadata() {
2557 if (!VE.hasMDs() && M.named_metadata_empty())
2558 return;
2559
2560 Stream.EnterSubblock(BlockID: bitc::METADATA_BLOCK_ID, CodeLen: 4);
2561 SmallVector<uint64_t, 64> Record;
2562
2563 // Emit all abbrevs upfront, so that the reader can jump in the middle of the
2564 // block and load any metadata.
2565 std::vector<unsigned> MDAbbrevs;
2566
2567 MDAbbrevs.resize(new_size: MetadataAbbrev::LastPlusOne);
2568 MDAbbrevs[MetadataAbbrev::DILocationAbbrevID] = createDILocationAbbrev();
2569 MDAbbrevs[MetadataAbbrev::GenericDINodeAbbrevID] =
2570 createGenericDINodeAbbrev();
2571
2572 auto Abbv = std::make_shared<BitCodeAbbrev>();
2573 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_INDEX_OFFSET));
2574 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2575 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2576 unsigned OffsetAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2577
2578 Abbv = std::make_shared<BitCodeAbbrev>();
2579 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_INDEX));
2580 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2581 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
2582 unsigned IndexAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2583
2584 // Emit MDStrings together upfront.
2585 writeMetadataStrings(Strings: VE.getMDStrings(), Record);
2586
2587 // We only emit an index for the metadata record if we have more than a given
2588 // (naive) threshold of metadatas, otherwise it is not worth it.
2589 if (VE.getNonMDStrings().size() > IndexThreshold) {
2590 // Write a placeholder value in for the offset of the metadata index,
2591 // which is written after the records, so that it can include
2592 // the offset of each entry. The placeholder offset will be
2593 // updated after all records are emitted.
2594 uint64_t Vals[] = {0, 0};
2595 Stream.EmitRecord(Code: bitc::METADATA_INDEX_OFFSET, Vals, Abbrev: OffsetAbbrev);
2596 }
2597
2598 // Compute and save the bit offset to the current position, which will be
2599 // patched when we emit the index later. We can simply subtract the 64-bit
2600 // fixed size from the current bit number to get the location to backpatch.
2601 uint64_t IndexOffsetRecordBitPos = Stream.GetCurrentBitNo();
2602
2603 // This index will contain the bitpos for each individual record.
2604 std::vector<uint64_t> IndexPos;
2605 IndexPos.reserve(n: VE.getNonMDStrings().size());
2606
2607 // Write all the records
2608 writeMetadataRecords(MDs: VE.getNonMDStrings(), Record, MDAbbrevs: &MDAbbrevs, IndexPos: &IndexPos);
2609
2610 if (VE.getNonMDStrings().size() > IndexThreshold) {
2611 // Now that we have emitted all the records we will emit the index. But
2612 // first
2613 // backpatch the forward reference so that the reader can skip the records
2614 // efficiently.
2615 Stream.BackpatchWord64(BitNo: IndexOffsetRecordBitPos - 64,
2616 Val: Stream.GetCurrentBitNo() - IndexOffsetRecordBitPos);
2617
2618 // Delta encode the index.
2619 uint64_t PreviousValue = IndexOffsetRecordBitPos;
2620 for (auto &Elt : IndexPos) {
2621 auto EltDelta = Elt - PreviousValue;
2622 PreviousValue = Elt;
2623 Elt = EltDelta;
2624 }
2625 // Emit the index record.
2626 Stream.EmitRecord(Code: bitc::METADATA_INDEX, Vals: IndexPos, Abbrev: IndexAbbrev);
2627 IndexPos.clear();
2628 }
2629
2630 // Write the named metadata now.
2631 writeNamedMetadata(Record);
2632
2633 auto AddDeclAttachedMetadata = [&](const GlobalObject &GO) {
2634 SmallVector<uint64_t, 4> Record;
2635 Record.push_back(Elt: VE.getValueID(V: &GO));
2636 pushGlobalMetadataAttachment(Record, GO);
2637 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_DECL_ATTACHMENT, Vals: Record);
2638 };
2639 for (const Function &F : M)
2640 if (F.isDeclaration() && F.hasMetadata())
2641 AddDeclAttachedMetadata(F);
2642 for (const GlobalIFunc &GI : M.ifuncs())
2643 if (GI.hasMetadata())
2644 AddDeclAttachedMetadata(GI);
2645 // FIXME: Only store metadata for declarations here, and move data for global
2646 // variable definitions to a separate block (PR28134).
2647 for (const GlobalVariable &GV : M.globals())
2648 if (GV.hasMetadata())
2649 AddDeclAttachedMetadata(GV);
2650
2651 Stream.ExitBlock();
2652}
2653
2654void ModuleBitcodeWriter::writeFunctionMetadata(const Function &F) {
2655 if (!VE.hasMDs())
2656 return;
2657
2658 Stream.EnterSubblock(BlockID: bitc::METADATA_BLOCK_ID, CodeLen: 3);
2659 SmallVector<uint64_t, 64> Record;
2660 writeMetadataStrings(Strings: VE.getMDStrings(), Record);
2661 writeMetadataRecords(MDs: VE.getNonMDStrings(), Record);
2662 Stream.ExitBlock();
2663}
2664
2665void ModuleBitcodeWriter::pushGlobalMetadataAttachment(
2666 SmallVectorImpl<uint64_t> &Record, const GlobalObject &GO) {
2667 // [n x [id, mdnode]]
2668 SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2669 GO.getAllMetadata(MDs);
2670 for (const auto &I : MDs) {
2671 Record.push_back(Elt: I.first);
2672 Record.push_back(Elt: VE.getMetadataID(MD: I.second));
2673 }
2674}
2675
2676void ModuleBitcodeWriter::writeFunctionMetadataAttachment(const Function &F) {
2677 Stream.EnterSubblock(BlockID: bitc::METADATA_ATTACHMENT_ID, CodeLen: 3);
2678
2679 SmallVector<uint64_t, 64> Record;
2680
2681 if (F.hasMetadata()) {
2682 pushGlobalMetadataAttachment(Record, GO: F);
2683 Stream.EmitRecord(Code: bitc::METADATA_ATTACHMENT, Vals: Record, Abbrev: 0);
2684 Record.clear();
2685 }
2686
2687 // Write metadata attachments
2688 // METADATA_ATTACHMENT - [m x [value, [n x [id, mdnode]]]
2689 SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2690 for (const BasicBlock &BB : F)
2691 for (const Instruction &I : BB) {
2692 MDs.clear();
2693 I.getAllMetadataOtherThanDebugLoc(MDs);
2694
2695 // If no metadata, ignore instruction.
2696 if (MDs.empty()) continue;
2697
2698 Record.push_back(Elt: VE.getInstructionID(I: &I));
2699
2700 for (const auto &[ID, MD] : MDs) {
2701 Record.push_back(Elt: ID);
2702 Record.push_back(Elt: VE.getMetadataID(MD));
2703 }
2704 Stream.EmitRecord(Code: bitc::METADATA_ATTACHMENT, Vals: Record, Abbrev: 0);
2705 Record.clear();
2706 }
2707
2708 Stream.ExitBlock();
2709}
2710
2711void ModuleBitcodeWriter::writeModuleMetadataKinds() {
2712 SmallVector<uint64_t, 64> Record;
2713
2714 // Write metadata kinds
2715 // METADATA_KIND - [n x [id, name]]
2716 SmallVector<StringRef, 8> Names;
2717 M.getMDKindNames(Result&: Names);
2718
2719 if (Names.empty()) return;
2720
2721 Stream.EnterSubblock(BlockID: bitc::METADATA_KIND_BLOCK_ID, CodeLen: 3);
2722
2723 for (unsigned MDKindID = 0, e = Names.size(); MDKindID != e; ++MDKindID) {
2724 Record.push_back(Elt: MDKindID);
2725 StringRef KName = Names[MDKindID];
2726 Record.append(in_start: KName.begin(), in_end: KName.end());
2727
2728 Stream.EmitRecord(Code: bitc::METADATA_KIND, Vals: Record, Abbrev: 0);
2729 Record.clear();
2730 }
2731
2732 Stream.ExitBlock();
2733}
2734
2735void ModuleBitcodeWriter::writeOperandBundleTags() {
2736 // Write metadata kinds
2737 //
2738 // OPERAND_BUNDLE_TAGS_BLOCK_ID : N x OPERAND_BUNDLE_TAG
2739 //
2740 // OPERAND_BUNDLE_TAG - [strchr x N]
2741
2742 SmallVector<StringRef, 8> Tags;
2743 M.getOperandBundleTags(Result&: Tags);
2744
2745 if (Tags.empty())
2746 return;
2747
2748 Stream.EnterSubblock(BlockID: bitc::OPERAND_BUNDLE_TAGS_BLOCK_ID, CodeLen: 3);
2749
2750 SmallVector<uint64_t, 64> Record;
2751
2752 for (auto Tag : Tags) {
2753 Record.append(in_start: Tag.begin(), in_end: Tag.end());
2754
2755 Stream.EmitRecord(Code: bitc::OPERAND_BUNDLE_TAG, Vals: Record, Abbrev: 0);
2756 Record.clear();
2757 }
2758
2759 Stream.ExitBlock();
2760}
2761
2762void ModuleBitcodeWriter::writeSyncScopeNames() {
2763 SmallVector<StringRef, 8> SSNs;
2764 M.getContext().getSyncScopeNames(SSNs);
2765 if (SSNs.empty())
2766 return;
2767
2768 Stream.EnterSubblock(BlockID: bitc::SYNC_SCOPE_NAMES_BLOCK_ID, CodeLen: 2);
2769
2770 SmallVector<uint64_t, 64> Record;
2771 for (auto SSN : SSNs) {
2772 Record.append(in_start: SSN.begin(), in_end: SSN.end());
2773 Stream.EmitRecord(Code: bitc::SYNC_SCOPE_NAME, Vals: Record, Abbrev: 0);
2774 Record.clear();
2775 }
2776
2777 Stream.ExitBlock();
2778}
2779
2780void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal,
2781 bool isGlobal) {
2782 if (FirstVal == LastVal) return;
2783
2784 Stream.EnterSubblock(BlockID: bitc::CONSTANTS_BLOCK_ID, CodeLen: 4);
2785
2786 unsigned AggregateAbbrev = 0;
2787 unsigned String8Abbrev = 0;
2788 unsigned CString7Abbrev = 0;
2789 unsigned CString6Abbrev = 0;
2790 // If this is a constant pool for the module, emit module-specific abbrevs.
2791 if (isGlobal) {
2792 // Abbrev for CST_CODE_AGGREGATE.
2793 auto Abbv = std::make_shared<BitCodeAbbrev>();
2794 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_AGGREGATE));
2795 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2796 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(Value: LastVal+1)));
2797 AggregateAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2798
2799 // Abbrev for CST_CODE_STRING.
2800 Abbv = std::make_shared<BitCodeAbbrev>();
2801 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_STRING));
2802 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2803 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2804 String8Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2805 // Abbrev for CST_CODE_CSTRING.
2806 Abbv = std::make_shared<BitCodeAbbrev>();
2807 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2808 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2809 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
2810 CString7Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2811 // Abbrev for CST_CODE_CSTRING.
2812 Abbv = std::make_shared<BitCodeAbbrev>();
2813 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2814 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2815 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
2816 CString6Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2817 }
2818
2819 SmallVector<uint64_t, 64> Record;
2820
2821 const ValueEnumerator::ValueList &Vals = VE.getValues();
2822 Type *LastTy = nullptr;
2823 for (unsigned i = FirstVal; i != LastVal; ++i) {
2824 const Value *V = Vals[i].first;
2825 // If we need to switch types, do so now.
2826 if (V->getType() != LastTy) {
2827 LastTy = V->getType();
2828 Record.push_back(Elt: VE.getTypeID(T: LastTy));
2829 Stream.EmitRecord(Code: bitc::CST_CODE_SETTYPE, Vals: Record,
2830 Abbrev: CONSTANTS_SETTYPE_ABBREV);
2831 Record.clear();
2832 }
2833
2834 if (const InlineAsm *IA = dyn_cast<InlineAsm>(Val: V)) {
2835 Record.push_back(Elt: VE.getTypeID(T: IA->getFunctionType()));
2836 Record.push_back(
2837 Elt: unsigned(IA->hasSideEffects()) | unsigned(IA->isAlignStack()) << 1 |
2838 unsigned(IA->getDialect() & 1) << 2 | unsigned(IA->canThrow()) << 3);
2839
2840 // Add the asm string.
2841 StringRef AsmStr = IA->getAsmString();
2842 Record.push_back(Elt: AsmStr.size());
2843 Record.append(in_start: AsmStr.begin(), in_end: AsmStr.end());
2844
2845 // Add the constraint string.
2846 StringRef ConstraintStr = IA->getConstraintString();
2847 Record.push_back(Elt: ConstraintStr.size());
2848 Record.append(in_start: ConstraintStr.begin(), in_end: ConstraintStr.end());
2849 Stream.EmitRecord(Code: bitc::CST_CODE_INLINEASM, Vals: Record);
2850 Record.clear();
2851 continue;
2852 }
2853 const Constant *C = cast<Constant>(Val: V);
2854 unsigned Code = -1U;
2855 unsigned AbbrevToUse = 0;
2856 if (C->isNullValue()) {
2857 Code = bitc::CST_CODE_NULL;
2858 } else if (isa<PoisonValue>(Val: C)) {
2859 Code = bitc::CST_CODE_POISON;
2860 } else if (isa<UndefValue>(Val: C)) {
2861 Code = bitc::CST_CODE_UNDEF;
2862 } else if (const ConstantInt *IV = dyn_cast<ConstantInt>(Val: C)) {
2863 if (IV->getBitWidth() <= 64) {
2864 uint64_t V = IV->getSExtValue();
2865 emitSignedInt64(Vals&: Record, V);
2866 Code = bitc::CST_CODE_INTEGER;
2867 AbbrevToUse = CONSTANTS_INTEGER_ABBREV;
2868 } else { // Wide integers, > 64 bits in size.
2869 emitWideAPInt(Vals&: Record, A: IV->getValue());
2870 Code = bitc::CST_CODE_WIDE_INTEGER;
2871 }
2872 } else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
2873 Code = bitc::CST_CODE_FLOAT;
2874 Type *Ty = CFP->getType()->getScalarType();
2875 if (Ty->isHalfTy() || Ty->isBFloatTy() || Ty->isFloatTy() ||
2876 Ty->isDoubleTy()) {
2877 Record.push_back(Elt: CFP->getValueAPF().bitcastToAPInt().getZExtValue());
2878 } else if (Ty->isX86_FP80Ty()) {
2879 // api needed to prevent premature destruction
2880 // bits are not in the same order as a normal i80 APInt, compensate.
2881 APInt api = CFP->getValueAPF().bitcastToAPInt();
2882 const uint64_t *p = api.getRawData();
2883 Record.push_back(Elt: (p[1] << 48) | (p[0] >> 16));
2884 Record.push_back(Elt: p[0] & 0xffffLL);
2885 } else if (Ty->isFP128Ty() || Ty->isPPC_FP128Ty()) {
2886 APInt api = CFP->getValueAPF().bitcastToAPInt();
2887 const uint64_t *p = api.getRawData();
2888 Record.push_back(Elt: p[0]);
2889 Record.push_back(Elt: p[1]);
2890 } else {
2891 assert(0 && "Unknown FP type!");
2892 }
2893 } else if (isa<ConstantDataSequential>(Val: C) &&
2894 cast<ConstantDataSequential>(Val: C)->isString()) {
2895 const ConstantDataSequential *Str = cast<ConstantDataSequential>(Val: C);
2896 // Emit constant strings specially.
2897 uint64_t NumElts = Str->getNumElements();
2898 // If this is a null-terminated string, use the denser CSTRING encoding.
2899 if (Str->isCString()) {
2900 Code = bitc::CST_CODE_CSTRING;
2901 --NumElts; // Don't encode the null, which isn't allowed by char6.
2902 } else {
2903 Code = bitc::CST_CODE_STRING;
2904 AbbrevToUse = String8Abbrev;
2905 }
2906 bool isCStr7 = Code == bitc::CST_CODE_CSTRING;
2907 bool isCStrChar6 = Code == bitc::CST_CODE_CSTRING;
2908 for (uint64_t i = 0; i != NumElts; ++i) {
2909 unsigned char V = Str->getElementAsInteger(i);
2910 Record.push_back(Elt: V);
2911 isCStr7 &= (V & 128) == 0;
2912 if (isCStrChar6)
2913 isCStrChar6 = BitCodeAbbrevOp::isChar6(C: V);
2914 }
2915
2916 if (isCStrChar6)
2917 AbbrevToUse = CString6Abbrev;
2918 else if (isCStr7)
2919 AbbrevToUse = CString7Abbrev;
2920 } else if (const ConstantDataSequential *CDS =
2921 dyn_cast<ConstantDataSequential>(Val: C)) {
2922 Code = bitc::CST_CODE_DATA;
2923 Type *EltTy = CDS->getElementType();
2924 if (isa<IntegerType>(Val: EltTy)) {
2925 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i)
2926 Record.push_back(Elt: CDS->getElementAsInteger(i));
2927 } else {
2928 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i)
2929 Record.push_back(
2930 Elt: CDS->getElementAsAPFloat(i).bitcastToAPInt().getLimitedValue());
2931 }
2932 } else if (isa<ConstantAggregate>(Val: C)) {
2933 Code = bitc::CST_CODE_AGGREGATE;
2934 for (const Value *Op : C->operands())
2935 Record.push_back(Elt: VE.getValueID(V: Op));
2936 AbbrevToUse = AggregateAbbrev;
2937 } else if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: C)) {
2938 switch (CE->getOpcode()) {
2939 default:
2940 if (Instruction::isCast(Opcode: CE->getOpcode())) {
2941 Code = bitc::CST_CODE_CE_CAST;
2942 Record.push_back(Elt: getEncodedCastOpcode(Opcode: CE->getOpcode()));
2943 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
2944 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2945 AbbrevToUse = CONSTANTS_CE_CAST_Abbrev;
2946 } else {
2947 assert(CE->getNumOperands() == 2 && "Unknown constant expr!");
2948 Code = bitc::CST_CODE_CE_BINOP;
2949 Record.push_back(Elt: getEncodedBinaryOpcode(Opcode: CE->getOpcode()));
2950 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2951 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
2952 uint64_t Flags = getOptimizationFlags(V: CE);
2953 if (Flags != 0)
2954 Record.push_back(Elt: Flags);
2955 }
2956 break;
2957 case Instruction::FNeg: {
2958 assert(CE->getNumOperands() == 1 && "Unknown constant expr!");
2959 Code = bitc::CST_CODE_CE_UNOP;
2960 Record.push_back(Elt: getEncodedUnaryOpcode(Opcode: CE->getOpcode()));
2961 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2962 uint64_t Flags = getOptimizationFlags(V: CE);
2963 if (Flags != 0)
2964 Record.push_back(Elt: Flags);
2965 break;
2966 }
2967 case Instruction::GetElementPtr: {
2968 Code = bitc::CST_CODE_CE_GEP;
2969 const auto *GO = cast<GEPOperator>(Val: C);
2970 Record.push_back(Elt: VE.getTypeID(T: GO->getSourceElementType()));
2971 Record.push_back(Elt: getOptimizationFlags(V: GO));
2972 if (std::optional<ConstantRange> Range = GO->getInRange()) {
2973 Code = bitc::CST_CODE_CE_GEP_WITH_INRANGE;
2974 emitConstantRange(Record, CR: *Range, /*EmitBitWidth=*/true);
2975 }
2976 for (const Value *Op : CE->operands()) {
2977 Record.push_back(Elt: VE.getTypeID(T: Op->getType()));
2978 Record.push_back(Elt: VE.getValueID(V: Op));
2979 }
2980 break;
2981 }
2982 case Instruction::ExtractElement:
2983 Code = bitc::CST_CODE_CE_EXTRACTELT;
2984 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
2985 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2986 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 1)->getType()));
2987 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
2988 break;
2989 case Instruction::InsertElement:
2990 Code = bitc::CST_CODE_CE_INSERTELT;
2991 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2992 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
2993 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 2)->getType()));
2994 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 2)));
2995 break;
2996 case Instruction::ShuffleVector:
2997 // If the return type and argument types are the same, this is a
2998 // standard shufflevector instruction. If the types are different,
2999 // then the shuffle is widening or truncating the input vectors, and
3000 // the argument type must also be encoded.
3001 if (C->getType() == C->getOperand(i: 0)->getType()) {
3002 Code = bitc::CST_CODE_CE_SHUFFLEVEC;
3003 } else {
3004 Code = bitc::CST_CODE_CE_SHUFVEC_EX;
3005 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
3006 }
3007 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3008 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3009 Record.push_back(Elt: VE.getValueID(V: CE->getShuffleMaskForBitcode()));
3010 break;
3011 }
3012 } else if (const BlockAddress *BA = dyn_cast<BlockAddress>(Val: C)) {
3013 Code = bitc::CST_CODE_BLOCKADDRESS;
3014 Record.push_back(Elt: VE.getTypeID(T: BA->getFunction()->getType()));
3015 Record.push_back(Elt: VE.getValueID(V: BA->getFunction()));
3016 Record.push_back(Elt: VE.getGlobalBasicBlockID(BB: BA->getBasicBlock()));
3017 } else if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(Val: C)) {
3018 Code = bitc::CST_CODE_DSO_LOCAL_EQUIVALENT;
3019 Record.push_back(Elt: VE.getTypeID(T: Equiv->getGlobalValue()->getType()));
3020 Record.push_back(Elt: VE.getValueID(V: Equiv->getGlobalValue()));
3021 } else if (const auto *NC = dyn_cast<NoCFIValue>(Val: C)) {
3022 Code = bitc::CST_CODE_NO_CFI_VALUE;
3023 Record.push_back(Elt: VE.getTypeID(T: NC->getGlobalValue()->getType()));
3024 Record.push_back(Elt: VE.getValueID(V: NC->getGlobalValue()));
3025 } else if (const auto *CPA = dyn_cast<ConstantPtrAuth>(Val: C)) {
3026 Code = bitc::CST_CODE_PTRAUTH2;
3027 Record.push_back(Elt: VE.getValueID(V: CPA->getPointer()));
3028 Record.push_back(Elt: VE.getValueID(V: CPA->getKey()));
3029 Record.push_back(Elt: VE.getValueID(V: CPA->getDiscriminator()));
3030 Record.push_back(Elt: VE.getValueID(V: CPA->getAddrDiscriminator()));
3031 Record.push_back(Elt: VE.getValueID(V: CPA->getDeactivationSymbol()));
3032 } else {
3033#ifndef NDEBUG
3034 C->dump();
3035#endif
3036 llvm_unreachable("Unknown constant!");
3037 }
3038 Stream.EmitRecord(Code, Vals: Record, Abbrev: AbbrevToUse);
3039 Record.clear();
3040 }
3041
3042 Stream.ExitBlock();
3043}
3044
3045void ModuleBitcodeWriter::writeModuleConstants() {
3046 const ValueEnumerator::ValueList &Vals = VE.getValues();
3047
3048 // Find the first constant to emit, which is the first non-globalvalue value.
3049 // We know globalvalues have been emitted by WriteModuleInfo.
3050 for (unsigned i = 0, e = Vals.size(); i != e; ++i) {
3051 if (!isa<GlobalValue>(Val: Vals[i].first)) {
3052 writeConstants(FirstVal: i, LastVal: Vals.size(), isGlobal: true);
3053 return;
3054 }
3055 }
3056}
3057
3058/// pushValueAndType - The file has to encode both the value and type id for
3059/// many values, because we need to know what type to create for forward
3060/// references. However, most operands are not forward references, so this type
3061/// field is not needed.
3062///
3063/// This function adds V's value ID to Vals. If the value ID is higher than the
3064/// instruction ID, then it is a forward reference, and it also includes the
3065/// type ID. The value ID that is written is encoded relative to the InstID.
3066bool ModuleBitcodeWriter::pushValueAndType(const Value *V, unsigned InstID,
3067 SmallVectorImpl<unsigned> &Vals) {
3068 unsigned ValID = VE.getValueID(V);
3069 // Make encoding relative to the InstID.
3070 Vals.push_back(Elt: InstID - ValID);
3071 if (ValID >= InstID) {
3072 Vals.push_back(Elt: VE.getTypeID(T: V->getType()));
3073 return true;
3074 }
3075 return false;
3076}
3077
3078bool ModuleBitcodeWriter::pushValueOrMetadata(const Value *V, unsigned InstID,
3079 SmallVectorImpl<unsigned> &Vals) {
3080 bool IsMetadata = V->getType()->isMetadataTy();
3081 if (IsMetadata) {
3082 Vals.push_back(Elt: bitc::OB_METADATA);
3083 Metadata *MD = cast<MetadataAsValue>(Val: V)->getMetadata();
3084 unsigned ValID = VE.getMetadataID(MD);
3085 Vals.push_back(Elt: InstID - ValID);
3086 return false;
3087 }
3088 return pushValueAndType(V, InstID, Vals);
3089}
3090
3091void ModuleBitcodeWriter::writeOperandBundles(const CallBase &CS,
3092 unsigned InstID) {
3093 SmallVector<unsigned, 64> Record;
3094 LLVMContext &C = CS.getContext();
3095
3096 for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i) {
3097 const auto &Bundle = CS.getOperandBundleAt(Index: i);
3098 Record.push_back(Elt: C.getOperandBundleTagID(Tag: Bundle.getTagName()));
3099
3100 for (auto &Input : Bundle.Inputs)
3101 pushValueOrMetadata(V: Input, InstID, Vals&: Record);
3102
3103 Stream.EmitRecord(Code: bitc::FUNC_CODE_OPERAND_BUNDLE, Vals: Record);
3104 Record.clear();
3105 }
3106}
3107
3108/// pushValue - Like pushValueAndType, but where the type of the value is
3109/// omitted (perhaps it was already encoded in an earlier operand).
3110void ModuleBitcodeWriter::pushValue(const Value *V, unsigned InstID,
3111 SmallVectorImpl<unsigned> &Vals) {
3112 unsigned ValID = VE.getValueID(V);
3113 Vals.push_back(Elt: InstID - ValID);
3114}
3115
3116void ModuleBitcodeWriter::pushValueSigned(const Value *V, unsigned InstID,
3117 SmallVectorImpl<uint64_t> &Vals) {
3118 unsigned ValID = VE.getValueID(V);
3119 int64_t diff = ((int32_t)InstID - (int32_t)ValID);
3120 emitSignedInt64(Vals, V: diff);
3121}
3122
3123/// WriteInstruction - Emit an instruction to the specified stream.
3124void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
3125 unsigned InstID,
3126 SmallVectorImpl<unsigned> &Vals) {
3127 unsigned Code = 0;
3128 unsigned AbbrevToUse = 0;
3129 VE.setInstructionID(&I);
3130 switch (I.getOpcode()) {
3131 default:
3132 if (Instruction::isCast(Opcode: I.getOpcode())) {
3133 Code = bitc::FUNC_CODE_INST_CAST;
3134 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3135 AbbrevToUse = FUNCTION_INST_CAST_ABBREV;
3136 Vals.push_back(Elt: VE.getTypeID(T: I.getType()));
3137 Vals.push_back(Elt: getEncodedCastOpcode(Opcode: I.getOpcode()));
3138 uint64_t Flags = getOptimizationFlags(V: &I);
3139 if (Flags != 0) {
3140 if (AbbrevToUse == FUNCTION_INST_CAST_ABBREV)
3141 AbbrevToUse = FUNCTION_INST_CAST_FLAGS_ABBREV;
3142 Vals.push_back(Elt: Flags);
3143 }
3144 } else {
3145 assert(isa<BinaryOperator>(I) && "Unknown instruction!");
3146 Code = bitc::FUNC_CODE_INST_BINOP;
3147 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3148 AbbrevToUse = FUNCTION_INST_BINOP_ABBREV;
3149 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3150 Vals.push_back(Elt: getEncodedBinaryOpcode(Opcode: I.getOpcode()));
3151 uint64_t Flags = getOptimizationFlags(V: &I);
3152 if (Flags != 0) {
3153 if (AbbrevToUse == FUNCTION_INST_BINOP_ABBREV)
3154 AbbrevToUse = FUNCTION_INST_BINOP_FLAGS_ABBREV;
3155 Vals.push_back(Elt: Flags);
3156 }
3157 }
3158 break;
3159 case Instruction::FNeg: {
3160 Code = bitc::FUNC_CODE_INST_UNOP;
3161 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3162 AbbrevToUse = FUNCTION_INST_UNOP_ABBREV;
3163 Vals.push_back(Elt: getEncodedUnaryOpcode(Opcode: I.getOpcode()));
3164 uint64_t Flags = getOptimizationFlags(V: &I);
3165 if (Flags != 0) {
3166 if (AbbrevToUse == FUNCTION_INST_UNOP_ABBREV)
3167 AbbrevToUse = FUNCTION_INST_UNOP_FLAGS_ABBREV;
3168 Vals.push_back(Elt: Flags);
3169 }
3170 break;
3171 }
3172 case Instruction::GetElementPtr: {
3173 Code = bitc::FUNC_CODE_INST_GEP;
3174 AbbrevToUse = FUNCTION_INST_GEP_ABBREV;
3175 auto &GEPInst = cast<GetElementPtrInst>(Val: I);
3176 Vals.push_back(Elt: getOptimizationFlags(V: &I));
3177 Vals.push_back(Elt: VE.getTypeID(T: GEPInst.getSourceElementType()));
3178 for (const Value *Op : I.operands())
3179 pushValueAndType(V: Op, InstID, Vals);
3180 break;
3181 }
3182 case Instruction::ExtractValue: {
3183 Code = bitc::FUNC_CODE_INST_EXTRACTVAL;
3184 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3185 const ExtractValueInst *EVI = cast<ExtractValueInst>(Val: &I);
3186 Vals.append(in_start: EVI->idx_begin(), in_end: EVI->idx_end());
3187 break;
3188 }
3189 case Instruction::InsertValue: {
3190 Code = bitc::FUNC_CODE_INST_INSERTVAL;
3191 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3192 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3193 const InsertValueInst *IVI = cast<InsertValueInst>(Val: &I);
3194 Vals.append(in_start: IVI->idx_begin(), in_end: IVI->idx_end());
3195 break;
3196 }
3197 case Instruction::Select: {
3198 Code = bitc::FUNC_CODE_INST_VSELECT;
3199 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3200 pushValue(V: I.getOperand(i: 2), InstID, Vals);
3201 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3202 uint64_t Flags = getOptimizationFlags(V: &I);
3203 if (Flags != 0)
3204 Vals.push_back(Elt: Flags);
3205 break;
3206 }
3207 case Instruction::ExtractElement:
3208 Code = bitc::FUNC_CODE_INST_EXTRACTELT;
3209 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3210 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3211 break;
3212 case Instruction::InsertElement:
3213 Code = bitc::FUNC_CODE_INST_INSERTELT;
3214 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3215 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3216 pushValueAndType(V: I.getOperand(i: 2), InstID, Vals);
3217 break;
3218 case Instruction::ShuffleVector:
3219 Code = bitc::FUNC_CODE_INST_SHUFFLEVEC;
3220 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3221 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3222 pushValue(V: cast<ShuffleVectorInst>(Val: I).getShuffleMaskForBitcode(), InstID,
3223 Vals);
3224 break;
3225 case Instruction::ICmp:
3226 case Instruction::FCmp: {
3227 // compare returning Int1Ty or vector of Int1Ty
3228 Code = bitc::FUNC_CODE_INST_CMP2;
3229 AbbrevToUse = FUNCTION_INST_CMP_ABBREV;
3230 if (pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3231 AbbrevToUse = 0;
3232 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3233 Vals.push_back(Elt: cast<CmpInst>(Val: I).getPredicate());
3234 uint64_t Flags = getOptimizationFlags(V: &I);
3235 if (Flags != 0) {
3236 Vals.push_back(Elt: Flags);
3237 if (AbbrevToUse)
3238 AbbrevToUse = FUNCTION_INST_CMP_FLAGS_ABBREV;
3239 }
3240 break;
3241 }
3242
3243 case Instruction::Ret:
3244 {
3245 Code = bitc::FUNC_CODE_INST_RET;
3246 unsigned NumOperands = I.getNumOperands();
3247 if (NumOperands == 0)
3248 AbbrevToUse = FUNCTION_INST_RET_VOID_ABBREV;
3249 else if (NumOperands == 1) {
3250 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3251 AbbrevToUse = FUNCTION_INST_RET_VAL_ABBREV;
3252 } else {
3253 for (const Value *Op : I.operands())
3254 pushValueAndType(V: Op, InstID, Vals);
3255 }
3256 }
3257 break;
3258 case Instruction::Br:
3259 {
3260 Code = bitc::FUNC_CODE_INST_BR;
3261 AbbrevToUse = FUNCTION_INST_BR_UNCOND_ABBREV;
3262 const BranchInst &II = cast<BranchInst>(Val: I);
3263 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 0)));
3264 if (II.isConditional()) {
3265 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 1)));
3266 pushValue(V: II.getCondition(), InstID, Vals);
3267 AbbrevToUse = FUNCTION_INST_BR_COND_ABBREV;
3268 }
3269 }
3270 break;
3271 case Instruction::Switch:
3272 {
3273 Code = bitc::FUNC_CODE_INST_SWITCH;
3274 const SwitchInst &SI = cast<SwitchInst>(Val: I);
3275 Vals.push_back(Elt: VE.getTypeID(T: SI.getCondition()->getType()));
3276 pushValue(V: SI.getCondition(), InstID, Vals);
3277 Vals.push_back(Elt: VE.getValueID(V: SI.getDefaultDest()));
3278 for (auto Case : SI.cases()) {
3279 Vals.push_back(Elt: VE.getValueID(V: Case.getCaseValue()));
3280 Vals.push_back(Elt: VE.getValueID(V: Case.getCaseSuccessor()));
3281 }
3282 }
3283 break;
3284 case Instruction::IndirectBr:
3285 Code = bitc::FUNC_CODE_INST_INDIRECTBR;
3286 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType()));
3287 // Encode the address operand as relative, but not the basic blocks.
3288 pushValue(V: I.getOperand(i: 0), InstID, Vals);
3289 for (const Value *Op : drop_begin(RangeOrContainer: I.operands()))
3290 Vals.push_back(Elt: VE.getValueID(V: Op));
3291 break;
3292
3293 case Instruction::Invoke: {
3294 const InvokeInst *II = cast<InvokeInst>(Val: &I);
3295 const Value *Callee = II->getCalledOperand();
3296 FunctionType *FTy = II->getFunctionType();
3297
3298 if (II->hasOperandBundles())
3299 writeOperandBundles(CS: *II, InstID);
3300
3301 Code = bitc::FUNC_CODE_INST_INVOKE;
3302
3303 Vals.push_back(Elt: VE.getAttributeListID(PAL: II->getAttributes()));
3304 Vals.push_back(Elt: II->getCallingConv() | 1 << 13);
3305 Vals.push_back(Elt: VE.getValueID(V: II->getNormalDest()));
3306 Vals.push_back(Elt: VE.getValueID(V: II->getUnwindDest()));
3307 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3308 pushValueAndType(V: Callee, InstID, Vals);
3309
3310 // Emit value #'s for the fixed parameters.
3311 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3312 pushValue(V: I.getOperand(i), InstID, Vals); // fixed param.
3313
3314 // Emit type/value pairs for varargs params.
3315 if (FTy->isVarArg()) {
3316 for (unsigned i = FTy->getNumParams(), e = II->arg_size(); i != e; ++i)
3317 pushValueAndType(V: I.getOperand(i), InstID, Vals); // vararg
3318 }
3319 break;
3320 }
3321 case Instruction::Resume:
3322 Code = bitc::FUNC_CODE_INST_RESUME;
3323 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3324 break;
3325 case Instruction::CleanupRet: {
3326 Code = bitc::FUNC_CODE_INST_CLEANUPRET;
3327 const auto &CRI = cast<CleanupReturnInst>(Val: I);
3328 pushValue(V: CRI.getCleanupPad(), InstID, Vals);
3329 if (CRI.hasUnwindDest())
3330 Vals.push_back(Elt: VE.getValueID(V: CRI.getUnwindDest()));
3331 break;
3332 }
3333 case Instruction::CatchRet: {
3334 Code = bitc::FUNC_CODE_INST_CATCHRET;
3335 const auto &CRI = cast<CatchReturnInst>(Val: I);
3336 pushValue(V: CRI.getCatchPad(), InstID, Vals);
3337 Vals.push_back(Elt: VE.getValueID(V: CRI.getSuccessor()));
3338 break;
3339 }
3340 case Instruction::CleanupPad:
3341 case Instruction::CatchPad: {
3342 const auto &FuncletPad = cast<FuncletPadInst>(Val: I);
3343 Code = isa<CatchPadInst>(Val: FuncletPad) ? bitc::FUNC_CODE_INST_CATCHPAD
3344 : bitc::FUNC_CODE_INST_CLEANUPPAD;
3345 pushValue(V: FuncletPad.getParentPad(), InstID, Vals);
3346
3347 unsigned NumArgOperands = FuncletPad.arg_size();
3348 Vals.push_back(Elt: NumArgOperands);
3349 for (unsigned Op = 0; Op != NumArgOperands; ++Op)
3350 pushValueAndType(V: FuncletPad.getArgOperand(i: Op), InstID, Vals);
3351 break;
3352 }
3353 case Instruction::CatchSwitch: {
3354 Code = bitc::FUNC_CODE_INST_CATCHSWITCH;
3355 const auto &CatchSwitch = cast<CatchSwitchInst>(Val: I);
3356
3357 pushValue(V: CatchSwitch.getParentPad(), InstID, Vals);
3358
3359 unsigned NumHandlers = CatchSwitch.getNumHandlers();
3360 Vals.push_back(Elt: NumHandlers);
3361 for (const BasicBlock *CatchPadBB : CatchSwitch.handlers())
3362 Vals.push_back(Elt: VE.getValueID(V: CatchPadBB));
3363
3364 if (CatchSwitch.hasUnwindDest())
3365 Vals.push_back(Elt: VE.getValueID(V: CatchSwitch.getUnwindDest()));
3366 break;
3367 }
3368 case Instruction::CallBr: {
3369 const CallBrInst *CBI = cast<CallBrInst>(Val: &I);
3370 const Value *Callee = CBI->getCalledOperand();
3371 FunctionType *FTy = CBI->getFunctionType();
3372
3373 if (CBI->hasOperandBundles())
3374 writeOperandBundles(CS: *CBI, InstID);
3375
3376 Code = bitc::FUNC_CODE_INST_CALLBR;
3377
3378 Vals.push_back(Elt: VE.getAttributeListID(PAL: CBI->getAttributes()));
3379
3380 Vals.push_back(Elt: CBI->getCallingConv() << bitc::CALL_CCONV |
3381 1 << bitc::CALL_EXPLICIT_TYPE);
3382
3383 Vals.push_back(Elt: VE.getValueID(V: CBI->getDefaultDest()));
3384 Vals.push_back(Elt: CBI->getNumIndirectDests());
3385 for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i)
3386 Vals.push_back(Elt: VE.getValueID(V: CBI->getIndirectDest(i)));
3387
3388 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3389 pushValueAndType(V: Callee, InstID, Vals);
3390
3391 // Emit value #'s for the fixed parameters.
3392 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3393 pushValue(V: I.getOperand(i), InstID, Vals); // fixed param.
3394
3395 // Emit type/value pairs for varargs params.
3396 if (FTy->isVarArg()) {
3397 for (unsigned i = FTy->getNumParams(), e = CBI->arg_size(); i != e; ++i)
3398 pushValueAndType(V: I.getOperand(i), InstID, Vals); // vararg
3399 }
3400 break;
3401 }
3402 case Instruction::Unreachable:
3403 Code = bitc::FUNC_CODE_INST_UNREACHABLE;
3404 AbbrevToUse = FUNCTION_INST_UNREACHABLE_ABBREV;
3405 break;
3406
3407 case Instruction::PHI: {
3408 const PHINode &PN = cast<PHINode>(Val: I);
3409 Code = bitc::FUNC_CODE_INST_PHI;
3410 // With the newer instruction encoding, forward references could give
3411 // negative valued IDs. This is most common for PHIs, so we use
3412 // signed VBRs.
3413 SmallVector<uint64_t, 128> Vals64;
3414 Vals64.push_back(Elt: VE.getTypeID(T: PN.getType()));
3415 for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) {
3416 pushValueSigned(V: PN.getIncomingValue(i), InstID, Vals&: Vals64);
3417 Vals64.push_back(Elt: VE.getValueID(V: PN.getIncomingBlock(i)));
3418 }
3419
3420 uint64_t Flags = getOptimizationFlags(V: &I);
3421 if (Flags != 0)
3422 Vals64.push_back(Elt: Flags);
3423
3424 // Emit a Vals64 vector and exit.
3425 Stream.EmitRecord(Code, Vals: Vals64, Abbrev: AbbrevToUse);
3426 Vals64.clear();
3427 return;
3428 }
3429
3430 case Instruction::LandingPad: {
3431 const LandingPadInst &LP = cast<LandingPadInst>(Val: I);
3432 Code = bitc::FUNC_CODE_INST_LANDINGPAD;
3433 Vals.push_back(Elt: VE.getTypeID(T: LP.getType()));
3434 Vals.push_back(Elt: LP.isCleanup());
3435 Vals.push_back(Elt: LP.getNumClauses());
3436 for (unsigned I = 0, E = LP.getNumClauses(); I != E; ++I) {
3437 if (LP.isCatch(Idx: I))
3438 Vals.push_back(Elt: LandingPadInst::Catch);
3439 else
3440 Vals.push_back(Elt: LandingPadInst::Filter);
3441 pushValueAndType(V: LP.getClause(Idx: I), InstID, Vals);
3442 }
3443 break;
3444 }
3445
3446 case Instruction::Alloca: {
3447 Code = bitc::FUNC_CODE_INST_ALLOCA;
3448 const AllocaInst &AI = cast<AllocaInst>(Val: I);
3449 Vals.push_back(Elt: VE.getTypeID(T: AI.getAllocatedType()));
3450 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType()));
3451 Vals.push_back(Elt: VE.getValueID(V: I.getOperand(i: 0))); // size.
3452 using APV = AllocaPackedValues;
3453 unsigned Record = 0;
3454 unsigned EncodedAlign = getEncodedAlign(Alignment: AI.getAlign());
3455 Bitfield::set<APV::AlignLower>(
3456 Packed&: Record, Value: EncodedAlign & ((1 << APV::AlignLower::Bits) - 1));
3457 Bitfield::set<APV::AlignUpper>(Packed&: Record,
3458 Value: EncodedAlign >> APV::AlignLower::Bits);
3459 Bitfield::set<APV::UsedWithInAlloca>(Packed&: Record, Value: AI.isUsedWithInAlloca());
3460 Bitfield::set<APV::ExplicitType>(Packed&: Record, Value: true);
3461 Bitfield::set<APV::SwiftError>(Packed&: Record, Value: AI.isSwiftError());
3462 Vals.push_back(Elt: Record);
3463
3464 unsigned AS = AI.getAddressSpace();
3465 if (AS != M.getDataLayout().getAllocaAddrSpace())
3466 Vals.push_back(Elt: AS);
3467 break;
3468 }
3469
3470 case Instruction::Load:
3471 if (cast<LoadInst>(Val: I).isAtomic()) {
3472 Code = bitc::FUNC_CODE_INST_LOADATOMIC;
3473 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3474 } else {
3475 Code = bitc::FUNC_CODE_INST_LOAD;
3476 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals)) // ptr
3477 AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
3478 }
3479 Vals.push_back(Elt: VE.getTypeID(T: I.getType()));
3480 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<LoadInst>(Val: I).getAlign()));
3481 Vals.push_back(Elt: cast<LoadInst>(Val: I).isVolatile());
3482 if (cast<LoadInst>(Val: I).isAtomic()) {
3483 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<LoadInst>(Val: I).getOrdering()));
3484 Vals.push_back(Elt: getEncodedSyncScopeID(SSID: cast<LoadInst>(Val: I).getSyncScopeID()));
3485 }
3486 break;
3487 case Instruction::Store:
3488 if (cast<StoreInst>(Val: I).isAtomic()) {
3489 Code = bitc::FUNC_CODE_INST_STOREATOMIC;
3490 } else {
3491 Code = bitc::FUNC_CODE_INST_STORE;
3492 AbbrevToUse = FUNCTION_INST_STORE_ABBREV;
3493 }
3494 if (pushValueAndType(V: I.getOperand(i: 1), InstID, Vals)) // ptrty + ptr
3495 AbbrevToUse = 0;
3496 if (pushValueAndType(V: I.getOperand(i: 0), InstID, Vals)) // valty + val
3497 AbbrevToUse = 0;
3498 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<StoreInst>(Val: I).getAlign()));
3499 Vals.push_back(Elt: cast<StoreInst>(Val: I).isVolatile());
3500 if (cast<StoreInst>(Val: I).isAtomic()) {
3501 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<StoreInst>(Val: I).getOrdering()));
3502 Vals.push_back(
3503 Elt: getEncodedSyncScopeID(SSID: cast<StoreInst>(Val: I).getSyncScopeID()));
3504 }
3505 break;
3506 case Instruction::AtomicCmpXchg:
3507 Code = bitc::FUNC_CODE_INST_CMPXCHG;
3508 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals); // ptrty + ptr
3509 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals); // cmp.
3510 pushValue(V: I.getOperand(i: 2), InstID, Vals); // newval.
3511 Vals.push_back(Elt: cast<AtomicCmpXchgInst>(Val: I).isVolatile());
3512 Vals.push_back(
3513 Elt: getEncodedOrdering(Ordering: cast<AtomicCmpXchgInst>(Val: I).getSuccessOrdering()));
3514 Vals.push_back(
3515 Elt: getEncodedSyncScopeID(SSID: cast<AtomicCmpXchgInst>(Val: I).getSyncScopeID()));
3516 Vals.push_back(
3517 Elt: getEncodedOrdering(Ordering: cast<AtomicCmpXchgInst>(Val: I).getFailureOrdering()));
3518 Vals.push_back(Elt: cast<AtomicCmpXchgInst>(Val: I).isWeak());
3519 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<AtomicCmpXchgInst>(Val: I).getAlign()));
3520 break;
3521 case Instruction::AtomicRMW:
3522 Code = bitc::FUNC_CODE_INST_ATOMICRMW;
3523 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals); // ptrty + ptr
3524 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals); // valty + val
3525 Vals.push_back(
3526 Elt: getEncodedRMWOperation(Op: cast<AtomicRMWInst>(Val: I).getOperation()));
3527 Vals.push_back(Elt: cast<AtomicRMWInst>(Val: I).isVolatile());
3528 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<AtomicRMWInst>(Val: I).getOrdering()));
3529 Vals.push_back(
3530 Elt: getEncodedSyncScopeID(SSID: cast<AtomicRMWInst>(Val: I).getSyncScopeID()));
3531 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<AtomicRMWInst>(Val: I).getAlign()));
3532 break;
3533 case Instruction::Fence:
3534 Code = bitc::FUNC_CODE_INST_FENCE;
3535 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<FenceInst>(Val: I).getOrdering()));
3536 Vals.push_back(Elt: getEncodedSyncScopeID(SSID: cast<FenceInst>(Val: I).getSyncScopeID()));
3537 break;
3538 case Instruction::Call: {
3539 const CallInst &CI = cast<CallInst>(Val: I);
3540 FunctionType *FTy = CI.getFunctionType();
3541
3542 if (CI.hasOperandBundles())
3543 writeOperandBundles(CS: CI, InstID);
3544
3545 Code = bitc::FUNC_CODE_INST_CALL;
3546
3547 Vals.push_back(Elt: VE.getAttributeListID(PAL: CI.getAttributes()));
3548
3549 unsigned Flags = getOptimizationFlags(V: &I);
3550 Vals.push_back(Elt: CI.getCallingConv() << bitc::CALL_CCONV |
3551 unsigned(CI.isTailCall()) << bitc::CALL_TAIL |
3552 unsigned(CI.isMustTailCall()) << bitc::CALL_MUSTTAIL |
3553 1 << bitc::CALL_EXPLICIT_TYPE |
3554 unsigned(CI.isNoTailCall()) << bitc::CALL_NOTAIL |
3555 unsigned(Flags != 0) << bitc::CALL_FMF);
3556 if (Flags != 0)
3557 Vals.push_back(Elt: Flags);
3558
3559 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3560 pushValueAndType(V: CI.getCalledOperand(), InstID, Vals); // Callee
3561
3562 // Emit value #'s for the fixed parameters.
3563 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3564 pushValue(V: CI.getArgOperand(i), InstID, Vals); // fixed param.
3565
3566 // Emit type/value pairs for varargs params.
3567 if (FTy->isVarArg()) {
3568 for (unsigned i = FTy->getNumParams(), e = CI.arg_size(); i != e; ++i)
3569 pushValueAndType(V: CI.getArgOperand(i), InstID, Vals); // varargs
3570 }
3571 break;
3572 }
3573 case Instruction::VAArg:
3574 Code = bitc::FUNC_CODE_INST_VAARG;
3575 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType())); // valistty
3576 pushValue(V: I.getOperand(i: 0), InstID, Vals); // valist.
3577 Vals.push_back(Elt: VE.getTypeID(T: I.getType())); // restype.
3578 break;
3579 case Instruction::Freeze:
3580 Code = bitc::FUNC_CODE_INST_FREEZE;
3581 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3582 break;
3583 }
3584
3585 Stream.EmitRecord(Code, Vals, Abbrev: AbbrevToUse);
3586 Vals.clear();
3587}
3588
3589/// Write a GlobalValue VST to the module. The purpose of this data structure is
3590/// to allow clients to efficiently find the function body.
3591void ModuleBitcodeWriter::writeGlobalValueSymbolTable(
3592 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3593 // Get the offset of the VST we are writing, and backpatch it into
3594 // the VST forward declaration record.
3595 uint64_t VSTOffset = Stream.GetCurrentBitNo();
3596 // The BitcodeStartBit was the stream offset of the identification block.
3597 VSTOffset -= bitcodeStartBit();
3598 assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned");
3599 // Note that we add 1 here because the offset is relative to one word
3600 // before the start of the identification block, which was historically
3601 // always the start of the regular bitcode header.
3602 Stream.BackpatchWord(BitNo: VSTOffsetPlaceholder, Val: VSTOffset / 32 + 1);
3603
3604 Stream.EnterSubblock(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, CodeLen: 4);
3605
3606 auto Abbv = std::make_shared<BitCodeAbbrev>();
3607 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY));
3608 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
3609 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset
3610 unsigned FnEntryAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
3611
3612 for (const Function &F : M) {
3613 uint64_t Record[2];
3614
3615 if (F.isDeclaration())
3616 continue;
3617
3618 Record[0] = VE.getValueID(V: &F);
3619
3620 // Save the word offset of the function (from the start of the
3621 // actual bitcode written to the stream).
3622 uint64_t BitcodeIndex = FunctionToBitcodeIndex[&F] - bitcodeStartBit();
3623 assert((BitcodeIndex & 31) == 0 && "function block not 32-bit aligned");
3624 // Note that we add 1 here because the offset is relative to one word
3625 // before the start of the identification block, which was historically
3626 // always the start of the regular bitcode header.
3627 Record[1] = BitcodeIndex / 32 + 1;
3628
3629 Stream.EmitRecord(Code: bitc::VST_CODE_FNENTRY, Vals: Record, Abbrev: FnEntryAbbrev);
3630 }
3631
3632 Stream.ExitBlock();
3633}
3634
3635/// Emit names for arguments, instructions and basic blocks in a function.
3636void ModuleBitcodeWriter::writeFunctionLevelValueSymbolTable(
3637 const ValueSymbolTable &VST) {
3638 if (VST.empty())
3639 return;
3640
3641 Stream.EnterSubblock(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, CodeLen: 4);
3642
3643 // FIXME: Set up the abbrev, we know how many values there are!
3644 // FIXME: We know if the type names can use 7-bit ascii.
3645 SmallVector<uint64_t, 64> NameVals;
3646
3647 for (const ValueName &Name : VST) {
3648 // Figure out the encoding to use for the name.
3649 StringEncoding Bits = getStringEncoding(Str: Name.getKey());
3650
3651 unsigned AbbrevToUse = VST_ENTRY_8_ABBREV;
3652 NameVals.push_back(Elt: VE.getValueID(V: Name.getValue()));
3653
3654 // VST_CODE_ENTRY: [valueid, namechar x N]
3655 // VST_CODE_BBENTRY: [bbid, namechar x N]
3656 unsigned Code;
3657 if (isa<BasicBlock>(Val: Name.getValue())) {
3658 Code = bitc::VST_CODE_BBENTRY;
3659 if (Bits == SE_Char6)
3660 AbbrevToUse = VST_BBENTRY_6_ABBREV;
3661 } else {
3662 Code = bitc::VST_CODE_ENTRY;
3663 if (Bits == SE_Char6)
3664 AbbrevToUse = VST_ENTRY_6_ABBREV;
3665 else if (Bits == SE_Fixed7)
3666 AbbrevToUse = VST_ENTRY_7_ABBREV;
3667 }
3668
3669 for (const auto P : Name.getKey())
3670 NameVals.push_back(Elt: (unsigned char)P);
3671
3672 // Emit the finished record.
3673 Stream.EmitRecord(Code, Vals: NameVals, Abbrev: AbbrevToUse);
3674 NameVals.clear();
3675 }
3676
3677 Stream.ExitBlock();
3678}
3679
3680void ModuleBitcodeWriter::writeUseList(UseListOrder &&Order) {
3681 assert(Order.Shuffle.size() >= 2 && "Shuffle too small");
3682 unsigned Code;
3683 if (isa<BasicBlock>(Val: Order.V))
3684 Code = bitc::USELIST_CODE_BB;
3685 else
3686 Code = bitc::USELIST_CODE_DEFAULT;
3687
3688 SmallVector<uint64_t, 64> Record(Order.Shuffle.begin(), Order.Shuffle.end());
3689 Record.push_back(Elt: VE.getValueID(V: Order.V));
3690 Stream.EmitRecord(Code, Vals: Record);
3691}
3692
3693void ModuleBitcodeWriter::writeUseListBlock(const Function *F) {
3694 assert(VE.shouldPreserveUseListOrder() &&
3695 "Expected to be preserving use-list order");
3696
3697 auto hasMore = [&]() {
3698 return !VE.UseListOrders.empty() && VE.UseListOrders.back().F == F;
3699 };
3700 if (!hasMore())
3701 // Nothing to do.
3702 return;
3703
3704 Stream.EnterSubblock(BlockID: bitc::USELIST_BLOCK_ID, CodeLen: 3);
3705 while (hasMore()) {
3706 writeUseList(Order: std::move(VE.UseListOrders.back()));
3707 VE.UseListOrders.pop_back();
3708 }
3709 Stream.ExitBlock();
3710}
3711
3712/// Emit a function body to the module stream.
3713void ModuleBitcodeWriter::writeFunction(
3714 const Function &F,
3715 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3716 // Save the bitcode index of the start of this function block for recording
3717 // in the VST.
3718 FunctionToBitcodeIndex[&F] = Stream.GetCurrentBitNo();
3719
3720 Stream.EnterSubblock(BlockID: bitc::FUNCTION_BLOCK_ID, CodeLen: 5);
3721 VE.incorporateFunction(F);
3722
3723 SmallVector<unsigned, 64> Vals;
3724
3725 // Emit the number of basic blocks, so the reader can create them ahead of
3726 // time.
3727 Vals.push_back(Elt: VE.getBasicBlocks().size());
3728 Stream.EmitRecord(Code: bitc::FUNC_CODE_DECLAREBLOCKS, Vals);
3729 Vals.clear();
3730
3731 // If there are function-local constants, emit them now.
3732 unsigned CstStart, CstEnd;
3733 VE.getFunctionConstantRange(Start&: CstStart, End&: CstEnd);
3734 writeConstants(FirstVal: CstStart, LastVal: CstEnd, isGlobal: false);
3735
3736 // If there is function-local metadata, emit it now.
3737 writeFunctionMetadata(F);
3738
3739 // Keep a running idea of what the instruction ID is.
3740 unsigned InstID = CstEnd;
3741
3742 bool NeedsMetadataAttachment = F.hasMetadata();
3743
3744 DILocation *LastDL = nullptr;
3745 SmallSetVector<Function *, 4> BlockAddressUsers;
3746
3747 // Finally, emit all the instructions, in order.
3748 for (const BasicBlock &BB : F) {
3749 for (const Instruction &I : BB) {
3750 writeInstruction(I, InstID, Vals);
3751
3752 if (!I.getType()->isVoidTy())
3753 ++InstID;
3754
3755 // If the instruction has metadata, write a metadata attachment later.
3756 NeedsMetadataAttachment |= I.hasMetadataOtherThanDebugLoc();
3757
3758 // If the instruction has a debug location, emit it.
3759 if (DILocation *DL = I.getDebugLoc()) {
3760 if (DL == LastDL) {
3761 // Just repeat the same debug loc as last time.
3762 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_LOC_AGAIN, Vals);
3763 } else {
3764 Vals.push_back(Elt: DL->getLine());
3765 Vals.push_back(Elt: DL->getColumn());
3766 Vals.push_back(Elt: VE.getMetadataOrNullID(MD: DL->getScope()));
3767 Vals.push_back(Elt: VE.getMetadataOrNullID(MD: DL->getInlinedAt()));
3768 Vals.push_back(Elt: DL->isImplicitCode());
3769 Vals.push_back(Elt: DL->getAtomGroup());
3770 Vals.push_back(Elt: DL->getAtomRank());
3771 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_LOC, Vals,
3772 Abbrev: FUNCTION_DEBUG_LOC_ABBREV);
3773 Vals.clear();
3774 LastDL = DL;
3775 }
3776 }
3777
3778 // If the instruction has DbgRecords attached to it, emit them. Note that
3779 // they come after the instruction so that it's easy to attach them again
3780 // when reading the bitcode, even though conceptually the debug locations
3781 // start "before" the instruction.
3782 if (I.hasDbgRecords()) {
3783 /// Try to push the value only (unwrapped), otherwise push the
3784 /// metadata wrapped value. Returns true if the value was pushed
3785 /// without the ValueAsMetadata wrapper.
3786 auto PushValueOrMetadata = [&Vals, InstID,
3787 this](Metadata *RawLocation) {
3788 assert(RawLocation &&
3789 "RawLocation unexpectedly null in DbgVariableRecord");
3790 if (ValueAsMetadata *VAM = dyn_cast<ValueAsMetadata>(Val: RawLocation)) {
3791 SmallVector<unsigned, 2> ValAndType;
3792 // If the value is a fwd-ref the type is also pushed. We don't
3793 // want the type, so fwd-refs are kept wrapped (pushValueAndType
3794 // returns false if the value is pushed without type).
3795 if (!pushValueAndType(V: VAM->getValue(), InstID, Vals&: ValAndType)) {
3796 Vals.push_back(Elt: ValAndType[0]);
3797 return true;
3798 }
3799 }
3800 // The metadata is a DIArgList, or ValueAsMetadata wrapping a
3801 // fwd-ref. Push the metadata ID.
3802 Vals.push_back(Elt: VE.getMetadataID(MD: RawLocation));
3803 return false;
3804 };
3805
3806 // Write out non-instruction debug information attached to this
3807 // instruction. Write it after the instruction so that it's easy to
3808 // re-attach to the instruction reading the records in.
3809 for (DbgRecord &DR : I.DebugMarker->getDbgRecordRange()) {
3810 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
3811 Vals.push_back(Elt: VE.getMetadataID(MD: &*DLR->getDebugLoc()));
3812 Vals.push_back(Elt: VE.getMetadataID(MD: DLR->getLabel()));
3813 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_LABEL, Vals);
3814 Vals.clear();
3815 continue;
3816 }
3817
3818 // First 3 fields are common to all kinds:
3819 // DILocation, DILocalVariable, DIExpression
3820 // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE)
3821 // ..., LocationMetadata
3822 // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE - abbrev'd)
3823 // ..., Value
3824 // dbg_declare (FUNC_CODE_DEBUG_RECORD_DECLARE)
3825 // ..., LocationMetadata
3826 // dbg_assign (FUNC_CODE_DEBUG_RECORD_ASSIGN)
3827 // ..., LocationMetadata, DIAssignID, DIExpression, LocationMetadata
3828 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
3829 Vals.push_back(Elt: VE.getMetadataID(MD: &*DVR.getDebugLoc()));
3830 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getVariable()));
3831 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getExpression()));
3832 if (DVR.isDbgValue()) {
3833 if (PushValueOrMetadata(DVR.getRawLocation()))
3834 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE, Vals,
3835 Abbrev: FUNCTION_DEBUG_RECORD_VALUE_ABBREV);
3836 else
3837 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_VALUE, Vals);
3838 } else if (DVR.isDbgDeclare()) {
3839 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3840 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_DECLARE, Vals);
3841 } else if (DVR.isDbgDeclareValue()) {
3842 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3843 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_DECLARE_VALUE, Vals);
3844 } else {
3845 assert(DVR.isDbgAssign() && "Unexpected DbgRecord kind");
3846 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3847 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getAssignID()));
3848 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getAddressExpression()));
3849 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawAddress()));
3850 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_ASSIGN, Vals);
3851 }
3852 Vals.clear();
3853 }
3854 }
3855 }
3856
3857 if (BlockAddress *BA = BlockAddress::lookup(BB: &BB)) {
3858 SmallVector<Value *> Worklist{BA};
3859 SmallPtrSet<Value *, 8> Visited{BA};
3860 while (!Worklist.empty()) {
3861 Value *V = Worklist.pop_back_val();
3862 for (User *U : V->users()) {
3863 if (auto *I = dyn_cast<Instruction>(Val: U)) {
3864 Function *P = I->getFunction();
3865 if (P != &F)
3866 BlockAddressUsers.insert(X: P);
3867 } else if (isa<Constant>(Val: U) && !isa<GlobalValue>(Val: U) &&
3868 Visited.insert(Ptr: U).second)
3869 Worklist.push_back(Elt: U);
3870 }
3871 }
3872 }
3873 }
3874
3875 if (!BlockAddressUsers.empty()) {
3876 Vals.resize(N: BlockAddressUsers.size());
3877 for (auto I : llvm::enumerate(First&: BlockAddressUsers))
3878 Vals[I.index()] = VE.getValueID(V: I.value());
3879 Stream.EmitRecord(Code: bitc::FUNC_CODE_BLOCKADDR_USERS, Vals);
3880 Vals.clear();
3881 }
3882
3883 // Emit names for all the instructions etc.
3884 if (auto *Symtab = F.getValueSymbolTable())
3885 writeFunctionLevelValueSymbolTable(VST: *Symtab);
3886
3887 if (NeedsMetadataAttachment)
3888 writeFunctionMetadataAttachment(F);
3889 if (VE.shouldPreserveUseListOrder())
3890 writeUseListBlock(F: &F);
3891 VE.purgeFunction();
3892 Stream.ExitBlock();
3893}
3894
3895// Emit blockinfo, which defines the standard abbreviations etc.
3896void ModuleBitcodeWriter::writeBlockInfo() {
3897 // We only want to emit block info records for blocks that have multiple
3898 // instances: CONSTANTS_BLOCK, FUNCTION_BLOCK and VALUE_SYMTAB_BLOCK.
3899 // Other blocks can define their abbrevs inline.
3900 Stream.EnterBlockInfoBlock();
3901
3902 // Encode type indices using fixed size based on number of types.
3903 BitCodeAbbrevOp TypeAbbrevOp(BitCodeAbbrevOp::Fixed,
3904 VE.computeBitsRequiredForTypeIndices());
3905 // Encode value indices as 6-bit VBR.
3906 BitCodeAbbrevOp ValAbbrevOp(BitCodeAbbrevOp::VBR, 6);
3907
3908 { // 8-bit fixed-width VST_CODE_ENTRY/VST_CODE_BBENTRY strings.
3909 auto Abbv = std::make_shared<BitCodeAbbrev>();
3910 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3));
3911 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3912 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3913 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
3914 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3915 VST_ENTRY_8_ABBREV)
3916 llvm_unreachable("Unexpected abbrev ordering!");
3917 }
3918
3919 { // 7-bit fixed width VST_CODE_ENTRY strings.
3920 auto Abbv = std::make_shared<BitCodeAbbrev>();
3921 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
3922 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3923 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3924 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
3925 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3926 VST_ENTRY_7_ABBREV)
3927 llvm_unreachable("Unexpected abbrev ordering!");
3928 }
3929 { // 6-bit char6 VST_CODE_ENTRY strings.
3930 auto Abbv = std::make_shared<BitCodeAbbrev>();
3931 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
3932 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3933 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3934 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
3935 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3936 VST_ENTRY_6_ABBREV)
3937 llvm_unreachable("Unexpected abbrev ordering!");
3938 }
3939 { // 6-bit char6 VST_CODE_BBENTRY strings.
3940 auto Abbv = std::make_shared<BitCodeAbbrev>();
3941 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_BBENTRY));
3942 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3943 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3944 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
3945 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3946 VST_BBENTRY_6_ABBREV)
3947 llvm_unreachable("Unexpected abbrev ordering!");
3948 }
3949
3950 { // SETTYPE abbrev for CONSTANTS_BLOCK.
3951 auto Abbv = std::make_shared<BitCodeAbbrev>();
3952 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_SETTYPE));
3953 Abbv->Add(OpInfo: TypeAbbrevOp);
3954 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3955 CONSTANTS_SETTYPE_ABBREV)
3956 llvm_unreachable("Unexpected abbrev ordering!");
3957 }
3958
3959 { // INTEGER abbrev for CONSTANTS_BLOCK.
3960 auto Abbv = std::make_shared<BitCodeAbbrev>();
3961 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_INTEGER));
3962 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3963 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3964 CONSTANTS_INTEGER_ABBREV)
3965 llvm_unreachable("Unexpected abbrev ordering!");
3966 }
3967
3968 { // CE_CAST abbrev for CONSTANTS_BLOCK.
3969 auto Abbv = std::make_shared<BitCodeAbbrev>();
3970 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CE_CAST));
3971 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // cast opc
3972 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // typeid
3973 VE.computeBitsRequiredForTypeIndices()));
3974 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
3975
3976 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3977 CONSTANTS_CE_CAST_Abbrev)
3978 llvm_unreachable("Unexpected abbrev ordering!");
3979 }
3980 { // NULL abbrev for CONSTANTS_BLOCK.
3981 auto Abbv = std::make_shared<BitCodeAbbrev>();
3982 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_NULL));
3983 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3984 CONSTANTS_NULL_Abbrev)
3985 llvm_unreachable("Unexpected abbrev ordering!");
3986 }
3987
3988 // FIXME: This should only use space for first class types!
3989
3990 { // INST_LOAD abbrev for FUNCTION_BLOCK.
3991 auto Abbv = std::make_shared<BitCodeAbbrev>();
3992 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_LOAD));
3993 Abbv->Add(OpInfo: ValAbbrevOp); // Ptr
3994 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
3995 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Align
3996 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
3997 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
3998 FUNCTION_INST_LOAD_ABBREV)
3999 llvm_unreachable("Unexpected abbrev ordering!");
4000 }
4001 {
4002 auto Abbv = std::make_shared<BitCodeAbbrev>();
4003 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_STORE));
4004 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4005 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4006 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // align
4007 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
4008 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4009 FUNCTION_INST_STORE_ABBREV)
4010 llvm_unreachable("Unexpected abbrev ordering!");
4011 }
4012 { // INST_UNOP abbrev for FUNCTION_BLOCK.
4013 auto Abbv = std::make_shared<BitCodeAbbrev>();
4014 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
4015 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4016 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4017 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4018 FUNCTION_INST_UNOP_ABBREV)
4019 llvm_unreachable("Unexpected abbrev ordering!");
4020 }
4021 { // INST_UNOP_FLAGS abbrev for FUNCTION_BLOCK.
4022 auto Abbv = std::make_shared<BitCodeAbbrev>();
4023 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
4024 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4025 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4026 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4027 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4028 FUNCTION_INST_UNOP_FLAGS_ABBREV)
4029 llvm_unreachable("Unexpected abbrev ordering!");
4030 }
4031 { // INST_BINOP abbrev for FUNCTION_BLOCK.
4032 auto Abbv = std::make_shared<BitCodeAbbrev>();
4033 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
4034 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4035 Abbv->Add(OpInfo: ValAbbrevOp); // RHS
4036 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4037 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4038 FUNCTION_INST_BINOP_ABBREV)
4039 llvm_unreachable("Unexpected abbrev ordering!");
4040 }
4041 { // INST_BINOP_FLAGS abbrev for FUNCTION_BLOCK.
4042 auto Abbv = std::make_shared<BitCodeAbbrev>();
4043 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
4044 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4045 Abbv->Add(OpInfo: ValAbbrevOp); // RHS
4046 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4047 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4048 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4049 FUNCTION_INST_BINOP_FLAGS_ABBREV)
4050 llvm_unreachable("Unexpected abbrev ordering!");
4051 }
4052 { // INST_CAST abbrev for FUNCTION_BLOCK.
4053 auto Abbv = std::make_shared<BitCodeAbbrev>();
4054 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
4055 Abbv->Add(OpInfo: ValAbbrevOp); // OpVal
4056 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4057 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4058 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4059 FUNCTION_INST_CAST_ABBREV)
4060 llvm_unreachable("Unexpected abbrev ordering!");
4061 }
4062 { // INST_CAST_FLAGS abbrev for FUNCTION_BLOCK.
4063 auto Abbv = std::make_shared<BitCodeAbbrev>();
4064 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
4065 Abbv->Add(OpInfo: ValAbbrevOp); // OpVal
4066 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4067 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4068 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4069 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4070 FUNCTION_INST_CAST_FLAGS_ABBREV)
4071 llvm_unreachable("Unexpected abbrev ordering!");
4072 }
4073
4074 { // INST_RET abbrev for FUNCTION_BLOCK.
4075 auto Abbv = std::make_shared<BitCodeAbbrev>();
4076 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
4077 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4078 FUNCTION_INST_RET_VOID_ABBREV)
4079 llvm_unreachable("Unexpected abbrev ordering!");
4080 }
4081 { // INST_RET abbrev for FUNCTION_BLOCK.
4082 auto Abbv = std::make_shared<BitCodeAbbrev>();
4083 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
4084 Abbv->Add(OpInfo: ValAbbrevOp);
4085 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4086 FUNCTION_INST_RET_VAL_ABBREV)
4087 llvm_unreachable("Unexpected abbrev ordering!");
4088 }
4089 {
4090 auto Abbv = std::make_shared<BitCodeAbbrev>();
4091 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BR));
4092 // TODO: Use different abbrev for absolute value reference (succ0)?
4093 Abbv->Add(OpInfo: ValAbbrevOp); // succ0
4094 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4095 FUNCTION_INST_BR_UNCOND_ABBREV)
4096 llvm_unreachable("Unexpected abbrev ordering!");
4097 }
4098 {
4099 auto Abbv = std::make_shared<BitCodeAbbrev>();
4100 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BR));
4101 // TODO: Use different abbrev for absolute value references (succ0, succ1)?
4102 Abbv->Add(OpInfo: ValAbbrevOp); // succ0
4103 Abbv->Add(OpInfo: ValAbbrevOp); // succ1
4104 Abbv->Add(OpInfo: ValAbbrevOp); // cond
4105 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4106 FUNCTION_INST_BR_COND_ABBREV)
4107 llvm_unreachable("Unexpected abbrev ordering!");
4108 }
4109 { // INST_UNREACHABLE abbrev for FUNCTION_BLOCK.
4110 auto Abbv = std::make_shared<BitCodeAbbrev>();
4111 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNREACHABLE));
4112 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4113 FUNCTION_INST_UNREACHABLE_ABBREV)
4114 llvm_unreachable("Unexpected abbrev ordering!");
4115 }
4116 {
4117 auto Abbv = std::make_shared<BitCodeAbbrev>();
4118 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_GEP));
4119 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // flags
4120 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4121 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4122 Abbv->Add(OpInfo: ValAbbrevOp);
4123 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4124 FUNCTION_INST_GEP_ABBREV)
4125 llvm_unreachable("Unexpected abbrev ordering!");
4126 }
4127 {
4128 auto Abbv = std::make_shared<BitCodeAbbrev>();
4129 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CMP2));
4130 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4131 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4132 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // pred
4133 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4134 FUNCTION_INST_CMP_ABBREV)
4135 llvm_unreachable("Unexpected abbrev ordering!");
4136 }
4137 {
4138 auto Abbv = std::make_shared<BitCodeAbbrev>();
4139 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CMP2));
4140 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4141 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4142 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // pred
4143 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4144 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4145 FUNCTION_INST_CMP_FLAGS_ABBREV)
4146 llvm_unreachable("Unexpected abbrev ordering!");
4147 }
4148 {
4149 auto Abbv = std::make_shared<BitCodeAbbrev>();
4150 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE));
4151 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // dbgloc
4152 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // var
4153 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // expr
4154 Abbv->Add(OpInfo: ValAbbrevOp); // val
4155 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4156 FUNCTION_DEBUG_RECORD_VALUE_ABBREV)
4157 llvm_unreachable("Unexpected abbrev ordering! 1");
4158 }
4159 {
4160 auto Abbv = std::make_shared<BitCodeAbbrev>();
4161 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_LOC));
4162 // NOTE: No IsDistinct field for FUNC_CODE_DEBUG_LOC.
4163 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4164 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4165 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4166 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4167 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
4168 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Atom group.
4169 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 3)); // Atom rank.
4170 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4171 FUNCTION_DEBUG_LOC_ABBREV)
4172 llvm_unreachable("Unexpected abbrev ordering!");
4173 }
4174 Stream.ExitBlock();
4175}
4176
4177/// Write the module path strings, currently only used when generating
4178/// a combined index file.
4179void IndexBitcodeWriter::writeModStrings() {
4180 Stream.EnterSubblock(BlockID: bitc::MODULE_STRTAB_BLOCK_ID, CodeLen: 3);
4181
4182 // TODO: See which abbrev sizes we actually need to emit
4183
4184 // 8-bit fixed-width MST_ENTRY strings.
4185 auto Abbv = std::make_shared<BitCodeAbbrev>();
4186 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4187 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4188 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4189 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
4190 unsigned Abbrev8Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4191
4192 // 7-bit fixed width MST_ENTRY strings.
4193 Abbv = std::make_shared<BitCodeAbbrev>();
4194 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4195 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4196 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4197 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
4198 unsigned Abbrev7Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4199
4200 // 6-bit char6 MST_ENTRY strings.
4201 Abbv = std::make_shared<BitCodeAbbrev>();
4202 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4203 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4204 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4205 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
4206 unsigned Abbrev6Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4207
4208 // Module Hash, 160 bits SHA1. Optionally, emitted after each MST_CODE_ENTRY.
4209 Abbv = std::make_shared<BitCodeAbbrev>();
4210 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_HASH));
4211 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4212 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4213 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4214 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4215 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4216 unsigned AbbrevHash = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4217
4218 SmallVector<unsigned, 64> Vals;
4219 forEachModule(Callback: [&](const StringMapEntry<ModuleHash> &MPSE) {
4220 StringRef Key = MPSE.getKey();
4221 const auto &Hash = MPSE.getValue();
4222 StringEncoding Bits = getStringEncoding(Str: Key);
4223 unsigned AbbrevToUse = Abbrev8Bit;
4224 if (Bits == SE_Char6)
4225 AbbrevToUse = Abbrev6Bit;
4226 else if (Bits == SE_Fixed7)
4227 AbbrevToUse = Abbrev7Bit;
4228
4229 auto ModuleId = ModuleIdMap.size();
4230 ModuleIdMap[Key] = ModuleId;
4231 Vals.push_back(Elt: ModuleId);
4232 Vals.append(in_start: Key.begin(), in_end: Key.end());
4233
4234 // Emit the finished record.
4235 Stream.EmitRecord(Code: bitc::MST_CODE_ENTRY, Vals, Abbrev: AbbrevToUse);
4236
4237 // Emit an optional hash for the module now
4238 if (llvm::any_of(Range: Hash, P: [](uint32_t H) { return H; })) {
4239 Vals.assign(in_start: Hash.begin(), in_end: Hash.end());
4240 // Emit the hash record.
4241 Stream.EmitRecord(Code: bitc::MST_CODE_HASH, Vals, Abbrev: AbbrevHash);
4242 }
4243
4244 Vals.clear();
4245 });
4246 Stream.ExitBlock();
4247}
4248
4249/// Write the function type metadata related records that need to appear before
4250/// a function summary entry (whether per-module or combined).
4251template <typename Fn>
4252static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream,
4253 FunctionSummary *FS,
4254 Fn GetValueID) {
4255 if (!FS->type_tests().empty())
4256 Stream.EmitRecord(Code: bitc::FS_TYPE_TESTS, Vals: FS->type_tests());
4257
4258 SmallVector<uint64_t, 64> Record;
4259
4260 auto WriteVFuncIdVec = [&](uint64_t Ty,
4261 ArrayRef<FunctionSummary::VFuncId> VFs) {
4262 if (VFs.empty())
4263 return;
4264 Record.clear();
4265 for (auto &VF : VFs) {
4266 Record.push_back(Elt: VF.GUID);
4267 Record.push_back(Elt: VF.Offset);
4268 }
4269 Stream.EmitRecord(Code: Ty, Vals: Record);
4270 };
4271
4272 WriteVFuncIdVec(bitc::FS_TYPE_TEST_ASSUME_VCALLS,
4273 FS->type_test_assume_vcalls());
4274 WriteVFuncIdVec(bitc::FS_TYPE_CHECKED_LOAD_VCALLS,
4275 FS->type_checked_load_vcalls());
4276
4277 auto WriteConstVCallVec = [&](uint64_t Ty,
4278 ArrayRef<FunctionSummary::ConstVCall> VCs) {
4279 for (auto &VC : VCs) {
4280 Record.clear();
4281 Record.push_back(Elt: VC.VFunc.GUID);
4282 Record.push_back(Elt: VC.VFunc.Offset);
4283 llvm::append_range(C&: Record, R: VC.Args);
4284 Stream.EmitRecord(Code: Ty, Vals: Record);
4285 }
4286 };
4287
4288 WriteConstVCallVec(bitc::FS_TYPE_TEST_ASSUME_CONST_VCALL,
4289 FS->type_test_assume_const_vcalls());
4290 WriteConstVCallVec(bitc::FS_TYPE_CHECKED_LOAD_CONST_VCALL,
4291 FS->type_checked_load_const_vcalls());
4292
4293 auto WriteRange = [&](ConstantRange Range) {
4294 Range = Range.sextOrTrunc(BitWidth: FunctionSummary::ParamAccess::RangeWidth);
4295 assert(Range.getLower().getNumWords() == 1);
4296 assert(Range.getUpper().getNumWords() == 1);
4297 emitSignedInt64(Vals&: Record, V: *Range.getLower().getRawData());
4298 emitSignedInt64(Vals&: Record, V: *Range.getUpper().getRawData());
4299 };
4300
4301 if (!FS->paramAccesses().empty()) {
4302 Record.clear();
4303 for (auto &Arg : FS->paramAccesses()) {
4304 size_t UndoSize = Record.size();
4305 Record.push_back(Elt: Arg.ParamNo);
4306 WriteRange(Arg.Use);
4307 Record.push_back(Elt: Arg.Calls.size());
4308 for (auto &Call : Arg.Calls) {
4309 Record.push_back(Elt: Call.ParamNo);
4310 std::optional<unsigned> ValueID = GetValueID(Call.Callee);
4311 if (!ValueID) {
4312 // If ValueID is unknown we can't drop just this call, we must drop
4313 // entire parameter.
4314 Record.resize(N: UndoSize);
4315 break;
4316 }
4317 Record.push_back(Elt: *ValueID);
4318 WriteRange(Call.Offsets);
4319 }
4320 }
4321 if (!Record.empty())
4322 Stream.EmitRecord(Code: bitc::FS_PARAM_ACCESS, Vals: Record);
4323 }
4324}
4325
4326/// Collect type IDs from type tests used by function.
4327static void
4328getReferencedTypeIds(FunctionSummary *FS,
4329 std::set<GlobalValue::GUID> &ReferencedTypeIds) {
4330 if (!FS->type_tests().empty())
4331 for (auto &TT : FS->type_tests())
4332 ReferencedTypeIds.insert(x: TT);
4333
4334 auto GetReferencedTypesFromVFuncIdVec =
4335 [&](ArrayRef<FunctionSummary::VFuncId> VFs) {
4336 for (auto &VF : VFs)
4337 ReferencedTypeIds.insert(x: VF.GUID);
4338 };
4339
4340 GetReferencedTypesFromVFuncIdVec(FS->type_test_assume_vcalls());
4341 GetReferencedTypesFromVFuncIdVec(FS->type_checked_load_vcalls());
4342
4343 auto GetReferencedTypesFromConstVCallVec =
4344 [&](ArrayRef<FunctionSummary::ConstVCall> VCs) {
4345 for (auto &VC : VCs)
4346 ReferencedTypeIds.insert(x: VC.VFunc.GUID);
4347 };
4348
4349 GetReferencedTypesFromConstVCallVec(FS->type_test_assume_const_vcalls());
4350 GetReferencedTypesFromConstVCallVec(FS->type_checked_load_const_vcalls());
4351}
4352
4353static void writeWholeProgramDevirtResolutionByArg(
4354 SmallVector<uint64_t, 64> &NameVals, const std::vector<uint64_t> &args,
4355 const WholeProgramDevirtResolution::ByArg &ByArg) {
4356 NameVals.push_back(Elt: args.size());
4357 llvm::append_range(C&: NameVals, R: args);
4358
4359 NameVals.push_back(Elt: ByArg.TheKind);
4360 NameVals.push_back(Elt: ByArg.Info);
4361 NameVals.push_back(Elt: ByArg.Byte);
4362 NameVals.push_back(Elt: ByArg.Bit);
4363}
4364
4365static void writeWholeProgramDevirtResolution(
4366 SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
4367 uint64_t Id, const WholeProgramDevirtResolution &Wpd) {
4368 NameVals.push_back(Elt: Id);
4369
4370 NameVals.push_back(Elt: Wpd.TheKind);
4371 NameVals.push_back(Elt: StrtabBuilder.add(S: Wpd.SingleImplName));
4372 NameVals.push_back(Elt: Wpd.SingleImplName.size());
4373
4374 NameVals.push_back(Elt: Wpd.ResByArg.size());
4375 for (auto &A : Wpd.ResByArg)
4376 writeWholeProgramDevirtResolutionByArg(NameVals, args: A.first, ByArg: A.second);
4377}
4378
4379static void writeTypeIdSummaryRecord(SmallVector<uint64_t, 64> &NameVals,
4380 StringTableBuilder &StrtabBuilder,
4381 StringRef Id,
4382 const TypeIdSummary &Summary) {
4383 NameVals.push_back(Elt: StrtabBuilder.add(S: Id));
4384 NameVals.push_back(Elt: Id.size());
4385
4386 NameVals.push_back(Elt: Summary.TTRes.TheKind);
4387 NameVals.push_back(Elt: Summary.TTRes.SizeM1BitWidth);
4388 NameVals.push_back(Elt: Summary.TTRes.AlignLog2);
4389 NameVals.push_back(Elt: Summary.TTRes.SizeM1);
4390 NameVals.push_back(Elt: Summary.TTRes.BitMask);
4391 NameVals.push_back(Elt: Summary.TTRes.InlineBits);
4392
4393 for (auto &W : Summary.WPDRes)
4394 writeWholeProgramDevirtResolution(NameVals, StrtabBuilder, Id: W.first,
4395 Wpd: W.second);
4396}
4397
4398static void writeTypeIdCompatibleVtableSummaryRecord(
4399 SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
4400 StringRef Id, const TypeIdCompatibleVtableInfo &Summary,
4401 ValueEnumerator &VE) {
4402 NameVals.push_back(Elt: StrtabBuilder.add(S: Id));
4403 NameVals.push_back(Elt: Id.size());
4404
4405 for (auto &P : Summary) {
4406 NameVals.push_back(Elt: P.AddressPointOffset);
4407 NameVals.push_back(Elt: VE.getValueID(V: P.VTableVI.getValue()));
4408 }
4409}
4410
4411// Adds the allocation contexts to the CallStacks map. We simply use the
4412// size at the time the context was added as the CallStackId. This works because
4413// when we look up the call stacks later on we process the function summaries
4414// and their allocation records in the same exact order.
4415static void collectMemProfCallStacks(
4416 FunctionSummary *FS, std::function<LinearFrameId(unsigned)> GetStackIndex,
4417 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> &CallStacks) {
4418 // The interfaces in ProfileData/MemProf.h use a type alias for a stack frame
4419 // id offset into the index of the full stack frames. The ModuleSummaryIndex
4420 // currently uses unsigned. Make sure these stay in sync.
4421 static_assert(std::is_same_v<LinearFrameId, unsigned>);
4422 for (auto &AI : FS->allocs()) {
4423 for (auto &MIB : AI.MIBs) {
4424 SmallVector<unsigned> StackIdIndices;
4425 StackIdIndices.reserve(N: MIB.StackIdIndices.size());
4426 for (auto Id : MIB.StackIdIndices)
4427 StackIdIndices.push_back(Elt: GetStackIndex(Id));
4428 // The CallStackId is the size at the time this context was inserted.
4429 CallStacks.insert(KV: {CallStacks.size(), StackIdIndices});
4430 }
4431 }
4432}
4433
4434// Build the radix tree from the accumulated CallStacks, write out the resulting
4435// linearized radix tree array, and return the map of call stack positions into
4436// this array for use when writing the allocation records. The returned map is
4437// indexed by a CallStackId which in this case is implicitly determined by the
4438// order of function summaries and their allocation infos being written.
4439static DenseMap<CallStackId, LinearCallStackId> writeMemoryProfileRadixTree(
4440 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> &&CallStacks,
4441 BitstreamWriter &Stream, unsigned RadixAbbrev) {
4442 assert(!CallStacks.empty());
4443 DenseMap<unsigned, FrameStat> FrameHistogram =
4444 computeFrameHistogram<LinearFrameId>(MemProfCallStackData&: CallStacks);
4445 CallStackRadixTreeBuilder<LinearFrameId> Builder;
4446 // We don't need a MemProfFrameIndexes map as we have already converted the
4447 // full stack id hash to a linear offset into the StackIds array.
4448 Builder.build(MemProfCallStackData: std::move(CallStacks), /*MemProfFrameIndexes=*/nullptr,
4449 FrameHistogram);
4450 Stream.EmitRecord(Code: bitc::FS_CONTEXT_RADIX_TREE_ARRAY, Vals: Builder.getRadixArray(),
4451 Abbrev: RadixAbbrev);
4452 return Builder.takeCallStackPos();
4453}
4454
4455static void writeFunctionHeapProfileRecords(
4456 BitstreamWriter &Stream, FunctionSummary *FS, unsigned CallsiteAbbrev,
4457 unsigned AllocAbbrev, unsigned ContextIdAbbvId, bool PerModule,
4458 std::function<unsigned(const ValueInfo &VI)> GetValueID,
4459 std::function<unsigned(unsigned)> GetStackIndex,
4460 bool WriteContextSizeInfoIndex,
4461 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
4462 CallStackId &CallStackCount) {
4463 SmallVector<uint64_t> Record;
4464
4465 for (auto &CI : FS->callsites()) {
4466 Record.clear();
4467 // Per module callsite clones should always have a single entry of
4468 // value 0.
4469 assert(!PerModule || (CI.Clones.size() == 1 && CI.Clones[0] == 0));
4470 Record.push_back(Elt: GetValueID(CI.Callee));
4471 if (!PerModule) {
4472 Record.push_back(Elt: CI.StackIdIndices.size());
4473 Record.push_back(Elt: CI.Clones.size());
4474 }
4475 for (auto Id : CI.StackIdIndices)
4476 Record.push_back(Elt: GetStackIndex(Id));
4477 if (!PerModule)
4478 llvm::append_range(C&: Record, R: CI.Clones);
4479 Stream.EmitRecord(Code: PerModule ? bitc::FS_PERMODULE_CALLSITE_INFO
4480 : bitc::FS_COMBINED_CALLSITE_INFO,
4481 Vals: Record, Abbrev: CallsiteAbbrev);
4482 }
4483
4484 for (auto &AI : FS->allocs()) {
4485 Record.clear();
4486 // Per module alloc versions should always have a single entry of
4487 // value 0.
4488 assert(!PerModule || (AI.Versions.size() == 1 && AI.Versions[0] == 0));
4489 Record.push_back(Elt: AI.MIBs.size());
4490 if (!PerModule)
4491 Record.push_back(Elt: AI.Versions.size());
4492 for (auto &MIB : AI.MIBs) {
4493 Record.push_back(Elt: (uint8_t)MIB.AllocType);
4494 // The per-module summary always needs to include the alloc context, as we
4495 // use it during the thin link. For the combined index it is optional (see
4496 // comments where CombinedIndexMemProfContext is defined).
4497 if (PerModule || CombinedIndexMemProfContext) {
4498 // Record the index into the radix tree array for this context.
4499 assert(CallStackCount <= CallStackPos.size());
4500 Record.push_back(Elt: CallStackPos[CallStackCount++]);
4501 }
4502 }
4503 if (!PerModule)
4504 llvm::append_range(C&: Record, R: AI.Versions);
4505 assert(AI.ContextSizeInfos.empty() ||
4506 AI.ContextSizeInfos.size() == AI.MIBs.size());
4507 // Optionally emit the context size information if it exists.
4508 if (WriteContextSizeInfoIndex && !AI.ContextSizeInfos.empty()) {
4509 // The abbreviation id for the context ids record should have been created
4510 // if we are emitting the per-module index, which is where we write this
4511 // info.
4512 assert(ContextIdAbbvId);
4513 SmallVector<uint32_t> ContextIds;
4514 // At least one context id per ContextSizeInfos entry (MIB), broken into 2
4515 // halves.
4516 ContextIds.reserve(N: AI.ContextSizeInfos.size() * 2);
4517 for (auto &Infos : AI.ContextSizeInfos) {
4518 Record.push_back(Elt: Infos.size());
4519 for (auto [FullStackId, TotalSize] : Infos) {
4520 // The context ids are emitted separately as a fixed width array,
4521 // which is more efficient than a VBR given that these hashes are
4522 // typically close to 64-bits. The max fixed width entry is 32 bits so
4523 // it is split into 2.
4524 ContextIds.push_back(Elt: static_cast<uint32_t>(FullStackId >> 32));
4525 ContextIds.push_back(Elt: static_cast<uint32_t>(FullStackId));
4526 Record.push_back(Elt: TotalSize);
4527 }
4528 }
4529 // The context ids are expected by the reader to immediately precede the
4530 // associated alloc info record.
4531 Stream.EmitRecord(Code: bitc::FS_ALLOC_CONTEXT_IDS, Vals: ContextIds,
4532 Abbrev: ContextIdAbbvId);
4533 }
4534 Stream.EmitRecord(Code: PerModule
4535 ? bitc::FS_PERMODULE_ALLOC_INFO
4536 : (CombinedIndexMemProfContext
4537 ? bitc::FS_COMBINED_ALLOC_INFO
4538 : bitc::FS_COMBINED_ALLOC_INFO_NO_CONTEXT),
4539 Vals: Record, Abbrev: AllocAbbrev);
4540 }
4541}
4542
4543// Helper to emit a single function summary record.
4544void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
4545 SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
4546 unsigned ValueID, unsigned FSCallsProfileAbbrev, unsigned CallsiteAbbrev,
4547 unsigned AllocAbbrev, unsigned ContextIdAbbvId, const Function &F,
4548 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
4549 CallStackId &CallStackCount) {
4550 NameVals.push_back(Elt: ValueID);
4551
4552 FunctionSummary *FS = cast<FunctionSummary>(Val: Summary);
4553
4554 writeFunctionTypeMetadataRecords(
4555 Stream, FS, GetValueID: [&](const ValueInfo &VI) -> std::optional<unsigned> {
4556 return {VE.getValueID(V: VI.getValue())};
4557 });
4558
4559 writeFunctionHeapProfileRecords(
4560 Stream, FS, CallsiteAbbrev, AllocAbbrev, ContextIdAbbvId,
4561 /*PerModule*/ true,
4562 /*GetValueId*/ GetValueID: [&](const ValueInfo &VI) { return getValueId(VI); },
4563 /*GetStackIndex*/ [&](unsigned I) { return I; },
4564 /*WriteContextSizeInfoIndex*/ true, CallStackPos, CallStackCount);
4565
4566 auto SpecialRefCnts = FS->specialRefCounts();
4567 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: FS->flags()));
4568 NameVals.push_back(Elt: FS->instCount());
4569 NameVals.push_back(Elt: getEncodedFFlags(Flags: FS->fflags()));
4570 NameVals.push_back(Elt: FS->refs().size());
4571 NameVals.push_back(Elt: SpecialRefCnts.first); // rorefcnt
4572 NameVals.push_back(Elt: SpecialRefCnts.second); // worefcnt
4573
4574 for (auto &RI : FS->refs())
4575 NameVals.push_back(Elt: getValueId(VI: RI));
4576
4577 for (auto &ECI : FS->calls()) {
4578 NameVals.push_back(Elt: getValueId(VI: ECI.first));
4579 NameVals.push_back(Elt: getEncodedHotnessCallEdgeInfo(CI: ECI.second));
4580 }
4581
4582 // Emit the finished record.
4583 Stream.EmitRecord(Code: bitc::FS_PERMODULE_PROFILE, Vals: NameVals, Abbrev: FSCallsProfileAbbrev);
4584 NameVals.clear();
4585}
4586
4587// Collect the global value references in the given variable's initializer,
4588// and emit them in a summary record.
4589void ModuleBitcodeWriterBase::writeModuleLevelReferences(
4590 const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals,
4591 unsigned FSModRefsAbbrev, unsigned FSModVTableRefsAbbrev) {
4592 auto VI = Index->getValueInfo(GUID: V.getGUID());
4593 if (!VI || VI.getSummaryList().empty()) {
4594 // Only declarations should not have a summary (a declaration might however
4595 // have a summary if the def was in module level asm).
4596 assert(V.isDeclaration());
4597 return;
4598 }
4599 auto *Summary = VI.getSummaryList()[0].get();
4600 NameVals.push_back(Elt: VE.getValueID(V: &V));
4601 GlobalVarSummary *VS = cast<GlobalVarSummary>(Val: Summary);
4602 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: VS->flags()));
4603 NameVals.push_back(Elt: getEncodedGVarFlags(Flags: VS->varflags()));
4604
4605 auto VTableFuncs = VS->vTableFuncs();
4606 if (!VTableFuncs.empty())
4607 NameVals.push_back(Elt: VS->refs().size());
4608
4609 unsigned SizeBeforeRefs = NameVals.size();
4610 for (auto &RI : VS->refs())
4611 NameVals.push_back(Elt: VE.getValueID(V: RI.getValue()));
4612 // Sort the refs for determinism output, the vector returned by FS->refs() has
4613 // been initialized from a DenseSet.
4614 llvm::sort(C: drop_begin(RangeOrContainer&: NameVals, N: SizeBeforeRefs));
4615
4616 if (VTableFuncs.empty())
4617 Stream.EmitRecord(Code: bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, Vals: NameVals,
4618 Abbrev: FSModRefsAbbrev);
4619 else {
4620 // VTableFuncs pairs should already be sorted by offset.
4621 for (auto &P : VTableFuncs) {
4622 NameVals.push_back(Elt: VE.getValueID(V: P.FuncVI.getValue()));
4623 NameVals.push_back(Elt: P.VTableOffset);
4624 }
4625
4626 Stream.EmitRecord(Code: bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS, Vals: NameVals,
4627 Abbrev: FSModVTableRefsAbbrev);
4628 }
4629 NameVals.clear();
4630}
4631
4632/// Emit the per-module summary section alongside the rest of
4633/// the module's bitcode.
4634void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
4635 // By default we compile with ThinLTO if the module has a summary, but the
4636 // client can request full LTO with a module flag.
4637 bool IsThinLTO = true;
4638 if (auto *MD =
4639 mdconst::extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: "ThinLTO")))
4640 IsThinLTO = MD->getZExtValue();
4641 Stream.EnterSubblock(BlockID: IsThinLTO ? bitc::GLOBALVAL_SUMMARY_BLOCK_ID
4642 : bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID,
4643 CodeLen: 4);
4644
4645 Stream.EmitRecord(
4646 Code: bitc::FS_VERSION,
4647 Vals: ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
4648
4649 // Write the index flags.
4650 uint64_t Flags = 0;
4651 // Bits 1-3 are set only in the combined index, skip them.
4652 if (Index->enableSplitLTOUnit())
4653 Flags |= 0x8;
4654 if (Index->hasUnifiedLTO())
4655 Flags |= 0x200;
4656
4657 Stream.EmitRecord(Code: bitc::FS_FLAGS, Vals: ArrayRef<uint64_t>{Flags});
4658
4659 if (Index->begin() == Index->end()) {
4660 Stream.ExitBlock();
4661 return;
4662 }
4663
4664 auto Abbv = std::make_shared<BitCodeAbbrev>();
4665 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_VALUE_GUID));
4666 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4667 // GUIDS often use up most of 64-bits, so encode as two Fixed 32.
4668 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4669 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4670 unsigned ValueGuidAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4671
4672 for (const auto &GVI : valueIds()) {
4673 Stream.EmitRecord(Code: bitc::FS_VALUE_GUID,
4674 Vals: ArrayRef<uint32_t>{GVI.second,
4675 static_cast<uint32_t>(GVI.first >> 32),
4676 static_cast<uint32_t>(GVI.first)},
4677 Abbrev: ValueGuidAbbrev);
4678 }
4679
4680 if (!Index->stackIds().empty()) {
4681 auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4682 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_STACK_IDS));
4683 // numids x stackid
4684 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4685 // The stack ids are hashes that are close to 64 bits in size, so emitting
4686 // as a pair of 32-bit fixed-width values is more efficient than a VBR.
4687 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4688 unsigned StackIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(StackIdAbbv));
4689 SmallVector<uint32_t> Vals;
4690 Vals.reserve(N: Index->stackIds().size() * 2);
4691 for (auto Id : Index->stackIds()) {
4692 Vals.push_back(Elt: static_cast<uint32_t>(Id >> 32));
4693 Vals.push_back(Elt: static_cast<uint32_t>(Id));
4694 }
4695 Stream.EmitRecord(Code: bitc::FS_STACK_IDS, Vals, Abbrev: StackIdAbbvId);
4696 }
4697
4698 unsigned ContextIdAbbvId = 0;
4699 if (metadataMayIncludeContextSizeInfo()) {
4700 // n x context id
4701 auto ContextIdAbbv = std::make_shared<BitCodeAbbrev>();
4702 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_ALLOC_CONTEXT_IDS));
4703 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4704 // The context ids are hashes that are close to 64 bits in size, so emitting
4705 // as a pair of 32-bit fixed-width values is more efficient than a VBR if we
4706 // are emitting them for all MIBs. Otherwise we use VBR to better compress 0
4707 // values that are expected to more frequently occur in an alloc's memprof
4708 // summary.
4709 if (metadataIncludesAllContextSizeInfo())
4710 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4711 else
4712 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4713 ContextIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(ContextIdAbbv));
4714 }
4715
4716 // Abbrev for FS_PERMODULE_PROFILE.
4717 Abbv = std::make_shared<BitCodeAbbrev>();
4718 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE));
4719 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4720 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // flags
4721 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
4722 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
4723 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4724 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
4725 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
4726 // numrefs x valueid, n x (valueid, hotness+tailcall flags)
4727 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4728 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4729 unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4730
4731 // Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS.
4732 Abbv = std::make_shared<BitCodeAbbrev>();
4733 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS));
4734 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4735 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4736 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
4737 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4738 unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4739
4740 // Abbrev for FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS.
4741 Abbv = std::make_shared<BitCodeAbbrev>();
4742 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS));
4743 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4744 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4745 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4746 // numrefs x valueid, n x (valueid , offset)
4747 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4748 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4749 unsigned FSModVTableRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4750
4751 // Abbrev for FS_ALIAS.
4752 Abbv = std::make_shared<BitCodeAbbrev>();
4753 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_ALIAS));
4754 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4755 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4756 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4757 unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4758
4759 // Abbrev for FS_TYPE_ID_METADATA
4760 Abbv = std::make_shared<BitCodeAbbrev>();
4761 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_TYPE_ID_METADATA));
4762 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid strtab index
4763 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid length
4764 // n x (valueid , offset)
4765 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4766 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4767 unsigned TypeIdCompatibleVtableAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4768
4769 Abbv = std::make_shared<BitCodeAbbrev>();
4770 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_CALLSITE_INFO));
4771 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4772 // n x stackidindex
4773 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4774 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4775 unsigned CallsiteAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4776
4777 Abbv = std::make_shared<BitCodeAbbrev>();
4778 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_ALLOC_INFO));
4779 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib
4780 // n x (alloc type, context radix tree index)
4781 // optional: nummib x (numcontext x total size)
4782 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4783 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4784 unsigned AllocAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4785
4786 Abbv = std::make_shared<BitCodeAbbrev>();
4787 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_CONTEXT_RADIX_TREE_ARRAY));
4788 // n x entry
4789 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4790 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4791 unsigned RadixAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4792
4793 // First walk through all the functions and collect the allocation contexts in
4794 // their associated summaries, for use in constructing a radix tree of
4795 // contexts. Note that we need to do this in the same order as the functions
4796 // are processed further below since the call stack positions in the resulting
4797 // radix tree array are identified based on this order.
4798 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> CallStacks;
4799 for (const Function &F : M) {
4800 // Summary emission does not support anonymous functions, they have to be
4801 // renamed using the anonymous function renaming pass.
4802 if (!F.hasName())
4803 report_fatal_error(reason: "Unexpected anonymous function when writing summary");
4804
4805 ValueInfo VI = Index->getValueInfo(GUID: F.getGUID());
4806 if (!VI || VI.getSummaryList().empty()) {
4807 // Only declarations should not have a summary (a declaration might
4808 // however have a summary if the def was in module level asm).
4809 assert(F.isDeclaration());
4810 continue;
4811 }
4812 auto *Summary = VI.getSummaryList()[0].get();
4813 FunctionSummary *FS = cast<FunctionSummary>(Val: Summary);
4814 collectMemProfCallStacks(
4815 FS, /*GetStackIndex*/ [](unsigned I) { return I; }, CallStacks);
4816 }
4817 // Finalize the radix tree, write it out, and get the map of positions in the
4818 // linearized tree array.
4819 DenseMap<CallStackId, LinearCallStackId> CallStackPos;
4820 if (!CallStacks.empty()) {
4821 CallStackPos =
4822 writeMemoryProfileRadixTree(CallStacks: std::move(CallStacks), Stream, RadixAbbrev);
4823 }
4824
4825 // Keep track of the current index into the CallStackPos map.
4826 CallStackId CallStackCount = 0;
4827
4828 SmallVector<uint64_t, 64> NameVals;
4829 // Iterate over the list of functions instead of the Index to
4830 // ensure the ordering is stable.
4831 for (const Function &F : M) {
4832 // Summary emission does not support anonymous functions, they have to
4833 // renamed using the anonymous function renaming pass.
4834 if (!F.hasName())
4835 report_fatal_error(reason: "Unexpected anonymous function when writing summary");
4836
4837 ValueInfo VI = Index->getValueInfo(GUID: F.getGUID());
4838 if (!VI || VI.getSummaryList().empty()) {
4839 // Only declarations should not have a summary (a declaration might
4840 // however have a summary if the def was in module level asm).
4841 assert(F.isDeclaration());
4842 continue;
4843 }
4844 auto *Summary = VI.getSummaryList()[0].get();
4845 writePerModuleFunctionSummaryRecord(NameVals, Summary, ValueID: VE.getValueID(V: &F),
4846 FSCallsProfileAbbrev, CallsiteAbbrev,
4847 AllocAbbrev, ContextIdAbbvId, F,
4848 CallStackPos, CallStackCount);
4849 }
4850
4851 // Capture references from GlobalVariable initializers, which are outside
4852 // of a function scope.
4853 for (const GlobalVariable &G : M.globals())
4854 writeModuleLevelReferences(V: G, NameVals, FSModRefsAbbrev,
4855 FSModVTableRefsAbbrev);
4856
4857 for (const GlobalAlias &A : M.aliases()) {
4858 auto *Aliasee = A.getAliaseeObject();
4859 // Skip ifunc and nameless functions which don't have an entry in the
4860 // summary.
4861 if (!Aliasee->hasName() || isa<GlobalIFunc>(Val: Aliasee))
4862 continue;
4863 auto AliasId = VE.getValueID(V: &A);
4864 auto AliaseeId = VE.getValueID(V: Aliasee);
4865 NameVals.push_back(Elt: AliasId);
4866 auto *Summary = Index->getGlobalValueSummary(GV: A);
4867 AliasSummary *AS = cast<AliasSummary>(Val: Summary);
4868 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: AS->flags()));
4869 NameVals.push_back(Elt: AliaseeId);
4870 Stream.EmitRecord(Code: bitc::FS_ALIAS, Vals: NameVals, Abbrev: FSAliasAbbrev);
4871 NameVals.clear();
4872 }
4873
4874 for (auto &S : Index->typeIdCompatibleVtableMap()) {
4875 writeTypeIdCompatibleVtableSummaryRecord(NameVals, StrtabBuilder, Id: S.first,
4876 Summary: S.second, VE);
4877 Stream.EmitRecord(Code: bitc::FS_TYPE_ID_METADATA, Vals: NameVals,
4878 Abbrev: TypeIdCompatibleVtableAbbrev);
4879 NameVals.clear();
4880 }
4881
4882 if (Index->getBlockCount())
4883 Stream.EmitRecord(Code: bitc::FS_BLOCK_COUNT,
4884 Vals: ArrayRef<uint64_t>{Index->getBlockCount()});
4885
4886 Stream.ExitBlock();
4887}
4888
4889/// Emit the combined summary section into the combined index file.
4890void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
4891 Stream.EnterSubblock(BlockID: bitc::GLOBALVAL_SUMMARY_BLOCK_ID, CodeLen: 4);
4892 Stream.EmitRecord(
4893 Code: bitc::FS_VERSION,
4894 Vals: ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
4895
4896 // Write the index flags.
4897 Stream.EmitRecord(Code: bitc::FS_FLAGS, Vals: ArrayRef<uint64_t>{Index.getFlags()});
4898
4899 auto Abbv = std::make_shared<BitCodeAbbrev>();
4900 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_VALUE_GUID));
4901 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4902 // GUIDS often use up most of 64-bits, so encode as two Fixed 32.
4903 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4904 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4905 unsigned ValueGuidAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4906
4907 for (const auto &GVI : valueIds()) {
4908 Stream.EmitRecord(Code: bitc::FS_VALUE_GUID,
4909 Vals: ArrayRef<uint32_t>{GVI.second,
4910 static_cast<uint32_t>(GVI.first >> 32),
4911 static_cast<uint32_t>(GVI.first)},
4912 Abbrev: ValueGuidAbbrev);
4913 }
4914
4915 // Write the stack ids used by this index, which will be a subset of those in
4916 // the full index in the case of distributed indexes.
4917 if (!StackIds.empty()) {
4918 auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4919 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_STACK_IDS));
4920 // numids x stackid
4921 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4922 // The stack ids are hashes that are close to 64 bits in size, so emitting
4923 // as a pair of 32-bit fixed-width values is more efficient than a VBR.
4924 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4925 unsigned StackIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(StackIdAbbv));
4926 SmallVector<uint32_t> Vals;
4927 Vals.reserve(N: StackIds.size() * 2);
4928 for (auto Id : StackIds) {
4929 Vals.push_back(Elt: static_cast<uint32_t>(Id >> 32));
4930 Vals.push_back(Elt: static_cast<uint32_t>(Id));
4931 }
4932 Stream.EmitRecord(Code: bitc::FS_STACK_IDS, Vals, Abbrev: StackIdAbbvId);
4933 }
4934
4935 // Abbrev for FS_COMBINED_PROFILE.
4936 Abbv = std::make_shared<BitCodeAbbrev>();
4937 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_PROFILE));
4938 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4939 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
4940 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4941 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
4942 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
4943 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // entrycount
4944 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4945 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
4946 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
4947 // numrefs x valueid, n x (valueid, hotness+tailcall flags)
4948 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4949 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4950 unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4951
4952 // Abbrev for FS_COMBINED_GLOBALVAR_INIT_REFS.
4953 Abbv = std::make_shared<BitCodeAbbrev>();
4954 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS));
4955 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4956 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
4957 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4958 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
4959 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4960 unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4961
4962 // Abbrev for FS_COMBINED_ALIAS.
4963 Abbv = std::make_shared<BitCodeAbbrev>();
4964 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_ALIAS));
4965 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4966 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
4967 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4968 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4969 unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4970
4971 Abbv = std::make_shared<BitCodeAbbrev>();
4972 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_CALLSITE_INFO));
4973 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4974 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numstackindices
4975 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
4976 // numstackindices x stackidindex, numver x version
4977 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4978 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4979 unsigned CallsiteAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4980
4981 Abbv = std::make_shared<BitCodeAbbrev>();
4982 Abbv->Add(OpInfo: BitCodeAbbrevOp(CombinedIndexMemProfContext
4983 ? bitc::FS_COMBINED_ALLOC_INFO
4984 : bitc::FS_COMBINED_ALLOC_INFO_NO_CONTEXT));
4985 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib
4986 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
4987 // nummib x (alloc type, context radix tree index),
4988 // numver x version
4989 // optional: nummib x total size
4990 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4991 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4992 unsigned AllocAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4993
4994 auto shouldImportValueAsDecl = [&](GlobalValueSummary *GVS) -> bool {
4995 if (DecSummaries == nullptr)
4996 return false;
4997 return DecSummaries->count(x: GVS);
4998 };
4999
5000 // The aliases are emitted as a post-pass, and will point to the value
5001 // id of the aliasee. Save them in a vector for post-processing.
5002 SmallVector<AliasSummary *, 64> Aliases;
5003
5004 // Save the value id for each summary for alias emission.
5005 DenseMap<const GlobalValueSummary *, unsigned> SummaryToValueIdMap;
5006
5007 SmallVector<uint64_t, 64> NameVals;
5008
5009 // Set that will be populated during call to writeFunctionTypeMetadataRecords
5010 // with the type ids referenced by this index file.
5011 std::set<GlobalValue::GUID> ReferencedTypeIds;
5012
5013 // For local linkage, we also emit the original name separately
5014 // immediately after the record.
5015 auto MaybeEmitOriginalName = [&](GlobalValueSummary &S) {
5016 // We don't need to emit the original name if we are writing the index for
5017 // distributed backends (in which case ModuleToSummariesForIndex is
5018 // non-null). The original name is only needed during the thin link, since
5019 // for SamplePGO the indirect call targets for local functions have
5020 // have the original name annotated in profile.
5021 // Continue to emit it when writing out the entire combined index, which is
5022 // used in testing the thin link via llvm-lto.
5023 if (ModuleToSummariesForIndex || !GlobalValue::isLocalLinkage(Linkage: S.linkage()))
5024 return;
5025 NameVals.push_back(Elt: S.getOriginalName());
5026 Stream.EmitRecord(Code: bitc::FS_COMBINED_ORIGINAL_NAME, Vals: NameVals);
5027 NameVals.clear();
5028 };
5029
5030 DenseMap<CallStackId, LinearCallStackId> CallStackPos;
5031 if (CombinedIndexMemProfContext) {
5032 Abbv = std::make_shared<BitCodeAbbrev>();
5033 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_CONTEXT_RADIX_TREE_ARRAY));
5034 // n x entry
5035 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5036 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5037 unsigned RadixAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5038
5039 // First walk through all the functions and collect the allocation contexts
5040 // in their associated summaries, for use in constructing a radix tree of
5041 // contexts. Note that we need to do this in the same order as the functions
5042 // are processed further below since the call stack positions in the
5043 // resulting radix tree array are identified based on this order.
5044 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> CallStacks;
5045 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
5046 // Don't collect this when invoked for an aliasee, as it is not needed for
5047 // the alias summary. If the aliasee is to be imported, we will invoke
5048 // this separately with IsAliasee=false.
5049 if (IsAliasee)
5050 return;
5051 GlobalValueSummary *S = I.second;
5052 assert(S);
5053 auto *FS = dyn_cast<FunctionSummary>(Val: S);
5054 if (!FS)
5055 return;
5056 collectMemProfCallStacks(
5057 FS,
5058 /*GetStackIndex*/
5059 [&](unsigned I) {
5060 // Get the corresponding index into the list of StackIds actually
5061 // being written for this combined index (which may be a subset in
5062 // the case of distributed indexes).
5063 assert(StackIdIndicesToIndex.contains(I));
5064 return StackIdIndicesToIndex[I];
5065 },
5066 CallStacks);
5067 });
5068 // Finalize the radix tree, write it out, and get the map of positions in
5069 // the linearized tree array.
5070 if (!CallStacks.empty()) {
5071 CallStackPos = writeMemoryProfileRadixTree(CallStacks: std::move(CallStacks), Stream,
5072 RadixAbbrev);
5073 }
5074 }
5075
5076 // Keep track of the current index into the CallStackPos map. Not used if
5077 // CombinedIndexMemProfContext is false.
5078 CallStackId CallStackCount = 0;
5079
5080 DenseSet<GlobalValue::GUID> DefOrUseGUIDs;
5081 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
5082 GlobalValueSummary *S = I.second;
5083 assert(S);
5084 DefOrUseGUIDs.insert(V: I.first);
5085 for (const ValueInfo &VI : S->refs())
5086 DefOrUseGUIDs.insert(V: VI.getGUID());
5087
5088 auto ValueId = getValueId(ValGUID: I.first);
5089 assert(ValueId);
5090 SummaryToValueIdMap[S] = *ValueId;
5091
5092 // If this is invoked for an aliasee, we want to record the above
5093 // mapping, but then not emit a summary entry (if the aliasee is
5094 // to be imported, we will invoke this separately with IsAliasee=false).
5095 if (IsAliasee)
5096 return;
5097
5098 if (auto *AS = dyn_cast<AliasSummary>(Val: S)) {
5099 // Will process aliases as a post-pass because the reader wants all
5100 // global to be loaded first.
5101 Aliases.push_back(Elt: AS);
5102 return;
5103 }
5104
5105 if (auto *VS = dyn_cast<GlobalVarSummary>(Val: S)) {
5106 NameVals.push_back(Elt: *ValueId);
5107 assert(ModuleIdMap.count(VS->modulePath()));
5108 NameVals.push_back(Elt: ModuleIdMap[VS->modulePath()]);
5109 NameVals.push_back(
5110 Elt: getEncodedGVSummaryFlags(Flags: VS->flags(), ImportAsDecl: shouldImportValueAsDecl(VS)));
5111 NameVals.push_back(Elt: getEncodedGVarFlags(Flags: VS->varflags()));
5112 for (auto &RI : VS->refs()) {
5113 auto RefValueId = getValueId(ValGUID: RI.getGUID());
5114 if (!RefValueId)
5115 continue;
5116 NameVals.push_back(Elt: *RefValueId);
5117 }
5118
5119 // Emit the finished record.
5120 Stream.EmitRecord(Code: bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, Vals: NameVals,
5121 Abbrev: FSModRefsAbbrev);
5122 NameVals.clear();
5123 MaybeEmitOriginalName(*S);
5124 return;
5125 }
5126
5127 auto GetValueId = [&](const ValueInfo &VI) -> std::optional<unsigned> {
5128 if (!VI)
5129 return std::nullopt;
5130 return getValueId(ValGUID: VI.getGUID());
5131 };
5132
5133 auto *FS = cast<FunctionSummary>(Val: S);
5134 writeFunctionTypeMetadataRecords(Stream, FS, GetValueID: GetValueId);
5135 getReferencedTypeIds(FS, ReferencedTypeIds);
5136
5137 writeFunctionHeapProfileRecords(
5138 Stream, FS, CallsiteAbbrev, AllocAbbrev, /*ContextIdAbbvId*/ 0,
5139 /*PerModule*/ false,
5140 /*GetValueId*/
5141 GetValueID: [&](const ValueInfo &VI) -> unsigned {
5142 std::optional<unsigned> ValueID = GetValueId(VI);
5143 // This can happen in shared index files for distributed ThinLTO if
5144 // the callee function summary is not included. Record 0 which we
5145 // will have to deal with conservatively when doing any kind of
5146 // validation in the ThinLTO backends.
5147 if (!ValueID)
5148 return 0;
5149 return *ValueID;
5150 },
5151 /*GetStackIndex*/
5152 [&](unsigned I) {
5153 // Get the corresponding index into the list of StackIds actually
5154 // being written for this combined index (which may be a subset in
5155 // the case of distributed indexes).
5156 assert(StackIdIndicesToIndex.contains(I));
5157 return StackIdIndicesToIndex[I];
5158 },
5159 /*WriteContextSizeInfoIndex*/ false, CallStackPos, CallStackCount);
5160
5161 NameVals.push_back(Elt: *ValueId);
5162 assert(ModuleIdMap.count(FS->modulePath()));
5163 NameVals.push_back(Elt: ModuleIdMap[FS->modulePath()]);
5164 NameVals.push_back(
5165 Elt: getEncodedGVSummaryFlags(Flags: FS->flags(), ImportAsDecl: shouldImportValueAsDecl(FS)));
5166 NameVals.push_back(Elt: FS->instCount());
5167 NameVals.push_back(Elt: getEncodedFFlags(Flags: FS->fflags()));
5168 // TODO: Stop writing entry count and bump bitcode version.
5169 NameVals.push_back(Elt: 0 /* EntryCount */);
5170
5171 // Fill in below
5172 NameVals.push_back(Elt: 0); // numrefs
5173 NameVals.push_back(Elt: 0); // rorefcnt
5174 NameVals.push_back(Elt: 0); // worefcnt
5175
5176 unsigned Count = 0, RORefCnt = 0, WORefCnt = 0;
5177 for (auto &RI : FS->refs()) {
5178 auto RefValueId = getValueId(ValGUID: RI.getGUID());
5179 if (!RefValueId)
5180 continue;
5181 NameVals.push_back(Elt: *RefValueId);
5182 if (RI.isReadOnly())
5183 RORefCnt++;
5184 else if (RI.isWriteOnly())
5185 WORefCnt++;
5186 Count++;
5187 }
5188 NameVals[6] = Count;
5189 NameVals[7] = RORefCnt;
5190 NameVals[8] = WORefCnt;
5191
5192 for (auto &EI : FS->calls()) {
5193 // If this GUID doesn't have a value id, it doesn't have a function
5194 // summary and we don't need to record any calls to it.
5195 std::optional<unsigned> CallValueId = GetValueId(EI.first);
5196 if (!CallValueId)
5197 continue;
5198 NameVals.push_back(Elt: *CallValueId);
5199 NameVals.push_back(Elt: getEncodedHotnessCallEdgeInfo(CI: EI.second));
5200 }
5201
5202 // Emit the finished record.
5203 Stream.EmitRecord(Code: bitc::FS_COMBINED_PROFILE, Vals: NameVals,
5204 Abbrev: FSCallsProfileAbbrev);
5205 NameVals.clear();
5206 MaybeEmitOriginalName(*S);
5207 });
5208
5209 for (auto *AS : Aliases) {
5210 auto AliasValueId = SummaryToValueIdMap[AS];
5211 assert(AliasValueId);
5212 NameVals.push_back(Elt: AliasValueId);
5213 assert(ModuleIdMap.count(AS->modulePath()));
5214 NameVals.push_back(Elt: ModuleIdMap[AS->modulePath()]);
5215 NameVals.push_back(
5216 Elt: getEncodedGVSummaryFlags(Flags: AS->flags(), ImportAsDecl: shouldImportValueAsDecl(AS)));
5217 // Set value id to 0 when an alias is imported but the aliasee summary is
5218 // not contained in the index.
5219 auto AliaseeValueId =
5220 AS->hasAliasee() ? SummaryToValueIdMap[&AS->getAliasee()] : 0;
5221 NameVals.push_back(Elt: AliaseeValueId);
5222
5223 // Emit the finished record.
5224 Stream.EmitRecord(Code: bitc::FS_COMBINED_ALIAS, Vals: NameVals, Abbrev: FSAliasAbbrev);
5225 NameVals.clear();
5226 MaybeEmitOriginalName(*AS);
5227
5228 if (AS->hasAliasee())
5229 if (auto *FS = dyn_cast<FunctionSummary>(Val: &AS->getAliasee()))
5230 getReferencedTypeIds(FS, ReferencedTypeIds);
5231 }
5232
5233 SmallVector<StringRef, 4> Functions;
5234 auto EmitCfiFunctions = [&](const CfiFunctionIndex &CfiIndex,
5235 bitc::GlobalValueSummarySymtabCodes Code) {
5236 if (CfiIndex.empty())
5237 return;
5238 for (GlobalValue::GUID GUID : DefOrUseGUIDs) {
5239 auto Defs = CfiIndex.forGuid(GUID);
5240 llvm::append_range(C&: Functions, R&: Defs);
5241 }
5242 if (Functions.empty())
5243 return;
5244 llvm::sort(C&: Functions);
5245 for (const auto &S : Functions) {
5246 NameVals.push_back(Elt: StrtabBuilder.add(S));
5247 NameVals.push_back(Elt: S.size());
5248 }
5249 Stream.EmitRecord(Code, Vals: NameVals);
5250 NameVals.clear();
5251 Functions.clear();
5252 };
5253
5254 EmitCfiFunctions(Index.cfiFunctionDefs(), bitc::FS_CFI_FUNCTION_DEFS);
5255 EmitCfiFunctions(Index.cfiFunctionDecls(), bitc::FS_CFI_FUNCTION_DECLS);
5256
5257 // Walk the GUIDs that were referenced, and write the
5258 // corresponding type id records.
5259 for (auto &T : ReferencedTypeIds) {
5260 auto TidIter = Index.typeIds().equal_range(x: T);
5261 for (const auto &[GUID, TypeIdPair] : make_range(p: TidIter)) {
5262 writeTypeIdSummaryRecord(NameVals, StrtabBuilder, Id: TypeIdPair.first,
5263 Summary: TypeIdPair.second);
5264 Stream.EmitRecord(Code: bitc::FS_TYPE_ID, Vals: NameVals);
5265 NameVals.clear();
5266 }
5267 }
5268
5269 if (Index.getBlockCount())
5270 Stream.EmitRecord(Code: bitc::FS_BLOCK_COUNT,
5271 Vals: ArrayRef<uint64_t>{Index.getBlockCount()});
5272
5273 Stream.ExitBlock();
5274}
5275
5276/// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the
5277/// current llvm version, and a record for the epoch number.
5278static void writeIdentificationBlock(BitstreamWriter &Stream) {
5279 Stream.EnterSubblock(BlockID: bitc::IDENTIFICATION_BLOCK_ID, CodeLen: 5);
5280
5281 // Write the "user readable" string identifying the bitcode producer
5282 auto Abbv = std::make_shared<BitCodeAbbrev>();
5283 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_STRING));
5284 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5285 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
5286 auto StringAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5287 writeStringRecord(Stream, Code: bitc::IDENTIFICATION_CODE_STRING,
5288 Str: "LLVM" LLVM_VERSION_STRING, AbbrevToUse: StringAbbrev);
5289
5290 // Write the epoch version
5291 Abbv = std::make_shared<BitCodeAbbrev>();
5292 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_EPOCH));
5293 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
5294 auto EpochAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5295 constexpr std::array<unsigned, 1> Vals = {._M_elems: {bitc::BITCODE_CURRENT_EPOCH}};
5296 Stream.EmitRecord(Code: bitc::IDENTIFICATION_CODE_EPOCH, Vals, Abbrev: EpochAbbrev);
5297 Stream.ExitBlock();
5298}
5299
5300void ModuleBitcodeWriter::writeModuleHash(StringRef View) {
5301 // Emit the module's hash.
5302 // MODULE_CODE_HASH: [5*i32]
5303 if (GenerateHash) {
5304 uint32_t Vals[5];
5305 Hasher.update(Data: ArrayRef<uint8_t>(
5306 reinterpret_cast<const uint8_t *>(View.data()), View.size()));
5307 std::array<uint8_t, 20> Hash = Hasher.result();
5308 for (int Pos = 0; Pos < 20; Pos += 4) {
5309 Vals[Pos / 4] = support::endian::read32be(P: Hash.data() + Pos);
5310 }
5311
5312 // Emit the finished record.
5313 Stream.EmitRecord(Code: bitc::MODULE_CODE_HASH, Vals);
5314
5315 if (ModHash)
5316 // Save the written hash value.
5317 llvm::copy(Range&: Vals, Out: std::begin(cont&: *ModHash));
5318 }
5319}
5320
5321void ModuleBitcodeWriter::write() {
5322 writeIdentificationBlock(Stream);
5323
5324 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5325 // We will want to write the module hash at this point. Block any flushing so
5326 // we can have access to the whole underlying data later.
5327 Stream.markAndBlockFlushing();
5328
5329 writeModuleVersion();
5330
5331 // Emit blockinfo, which defines the standard abbreviations etc.
5332 writeBlockInfo();
5333
5334 // Emit information describing all of the types in the module.
5335 writeTypeTable();
5336
5337 // Emit information about attribute groups.
5338 writeAttributeGroupTable();
5339
5340 // Emit information about parameter attributes.
5341 writeAttributeTable();
5342
5343 writeComdats();
5344
5345 // Emit top-level description of module, including target triple, inline asm,
5346 // descriptors for global variables, and function prototype info.
5347 writeModuleInfo();
5348
5349 // Emit constants.
5350 writeModuleConstants();
5351
5352 // Emit metadata kind names.
5353 writeModuleMetadataKinds();
5354
5355 // Emit metadata.
5356 writeModuleMetadata();
5357
5358 // Emit module-level use-lists.
5359 if (VE.shouldPreserveUseListOrder())
5360 writeUseListBlock(F: nullptr);
5361
5362 writeOperandBundleTags();
5363 writeSyncScopeNames();
5364
5365 // Emit function bodies.
5366 DenseMap<const Function *, uint64_t> FunctionToBitcodeIndex;
5367 for (const Function &F : M)
5368 if (!F.isDeclaration())
5369 writeFunction(F, FunctionToBitcodeIndex);
5370
5371 // Need to write after the above call to WriteFunction which populates
5372 // the summary information in the index.
5373 if (Index)
5374 writePerModuleGlobalValueSummary();
5375
5376 writeGlobalValueSymbolTable(FunctionToBitcodeIndex);
5377
5378 writeModuleHash(View: Stream.getMarkedBufferAndResumeFlushing());
5379
5380 Stream.ExitBlock();
5381}
5382
5383static void writeInt32ToBuffer(uint32_t Value, SmallVectorImpl<char> &Buffer,
5384 uint32_t &Position) {
5385 support::endian::write32le(P: &Buffer[Position], V: Value);
5386 Position += 4;
5387}
5388
5389/// If generating a bc file on darwin, we have to emit a
5390/// header and trailer to make it compatible with the system archiver. To do
5391/// this we emit the following header, and then emit a trailer that pads the
5392/// file out to be a multiple of 16 bytes.
5393///
5394/// struct bc_header {
5395/// uint32_t Magic; // 0x0B17C0DE
5396/// uint32_t Version; // Version, currently always 0.
5397/// uint32_t BitcodeOffset; // Offset to traditional bitcode file.
5398/// uint32_t BitcodeSize; // Size of traditional bitcode file.
5399/// uint32_t CPUType; // CPU specifier.
5400/// ... potentially more later ...
5401/// };
5402static void emitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer,
5403 const Triple &TT) {
5404 unsigned CPUType = ~0U;
5405
5406 // Match x86_64-*, i[3-9]86-*, powerpc-*, powerpc64-*, arm-*, thumb-*,
5407 // armv[0-9]-*, thumbv[0-9]-*, armv5te-*, or armv6t2-*. The CPUType is a magic
5408 // number from /usr/include/mach/machine.h. It is ok to reproduce the
5409 // specific constants here because they are implicitly part of the Darwin ABI.
5410 enum {
5411 DARWIN_CPU_ARCH_ABI64 = 0x01000000,
5412 DARWIN_CPU_TYPE_X86 = 7,
5413 DARWIN_CPU_TYPE_ARM = 12,
5414 DARWIN_CPU_TYPE_POWERPC = 18
5415 };
5416
5417 Triple::ArchType Arch = TT.getArch();
5418 if (Arch == Triple::x86_64)
5419 CPUType = DARWIN_CPU_TYPE_X86 | DARWIN_CPU_ARCH_ABI64;
5420 else if (Arch == Triple::x86)
5421 CPUType = DARWIN_CPU_TYPE_X86;
5422 else if (Arch == Triple::ppc)
5423 CPUType = DARWIN_CPU_TYPE_POWERPC;
5424 else if (Arch == Triple::ppc64)
5425 CPUType = DARWIN_CPU_TYPE_POWERPC | DARWIN_CPU_ARCH_ABI64;
5426 else if (Arch == Triple::arm || Arch == Triple::thumb)
5427 CPUType = DARWIN_CPU_TYPE_ARM;
5428
5429 // Traditional Bitcode starts after header.
5430 assert(Buffer.size() >= BWH_HeaderSize &&
5431 "Expected header size to be reserved");
5432 unsigned BCOffset = BWH_HeaderSize;
5433 unsigned BCSize = Buffer.size() - BWH_HeaderSize;
5434
5435 // Write the magic and version.
5436 unsigned Position = 0;
5437 writeInt32ToBuffer(Value: 0x0B17C0DE, Buffer, Position);
5438 writeInt32ToBuffer(Value: 0, Buffer, Position); // Version.
5439 writeInt32ToBuffer(Value: BCOffset, Buffer, Position);
5440 writeInt32ToBuffer(Value: BCSize, Buffer, Position);
5441 writeInt32ToBuffer(Value: CPUType, Buffer, Position);
5442
5443 // If the file is not a multiple of 16 bytes, insert dummy padding.
5444 while (Buffer.size() & 15)
5445 Buffer.push_back(Elt: 0);
5446}
5447
5448/// Helper to write the header common to all bitcode files.
5449static void writeBitcodeHeader(BitstreamWriter &Stream) {
5450 // Emit the file header.
5451 Stream.Emit(Val: (unsigned)'B', NumBits: 8);
5452 Stream.Emit(Val: (unsigned)'C', NumBits: 8);
5453 Stream.Emit(Val: 0x0, NumBits: 4);
5454 Stream.Emit(Val: 0xC, NumBits: 4);
5455 Stream.Emit(Val: 0xE, NumBits: 4);
5456 Stream.Emit(Val: 0xD, NumBits: 4);
5457}
5458
5459BitcodeWriter::BitcodeWriter(SmallVectorImpl<char> &Buffer)
5460 : Stream(new BitstreamWriter(Buffer)) {
5461 writeBitcodeHeader(Stream&: *Stream);
5462}
5463
5464BitcodeWriter::BitcodeWriter(raw_ostream &FS)
5465 : Stream(new BitstreamWriter(FS, FlushThreshold)) {
5466 writeBitcodeHeader(Stream&: *Stream);
5467}
5468
5469BitcodeWriter::~BitcodeWriter() { assert(WroteStrtab); }
5470
5471void BitcodeWriter::writeBlob(unsigned Block, unsigned Record, StringRef Blob) {
5472 Stream->EnterSubblock(BlockID: Block, CodeLen: 3);
5473
5474 auto Abbv = std::make_shared<BitCodeAbbrev>();
5475 Abbv->Add(OpInfo: BitCodeAbbrevOp(Record));
5476 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
5477 auto AbbrevNo = Stream->EmitAbbrev(Abbv: std::move(Abbv));
5478
5479 Stream->EmitRecordWithBlob(Abbrev: AbbrevNo, Vals: ArrayRef<uint64_t>{Record}, Blob);
5480
5481 Stream->ExitBlock();
5482}
5483
5484void BitcodeWriter::writeSymtab() {
5485 assert(!WroteStrtab && !WroteSymtab);
5486
5487 // If any module has module-level inline asm, we will require a registered asm
5488 // parser for the target so that we can create an accurate symbol table for
5489 // the module.
5490 for (Module *M : Mods) {
5491 if (M->getModuleInlineAsm().empty())
5492 continue;
5493
5494 std::string Err;
5495 const Triple TT(M->getTargetTriple());
5496 const Target *T = TargetRegistry::lookupTarget(TheTriple: TT, Error&: Err);
5497 if (!T || !T->hasMCAsmParser())
5498 return;
5499 }
5500
5501 WroteSymtab = true;
5502 SmallVector<char, 0> Symtab;
5503 // The irsymtab::build function may be unable to create a symbol table if the
5504 // module is malformed (e.g. it contains an invalid alias). Writing a symbol
5505 // table is not required for correctness, but we still want to be able to
5506 // write malformed modules to bitcode files, so swallow the error.
5507 if (Error E = irsymtab::build(Mods, Symtab, StrtabBuilder, Alloc)) {
5508 consumeError(Err: std::move(E));
5509 return;
5510 }
5511
5512 writeBlob(Block: bitc::SYMTAB_BLOCK_ID, Record: bitc::SYMTAB_BLOB,
5513 Blob: {Symtab.data(), Symtab.size()});
5514}
5515
5516void BitcodeWriter::writeStrtab() {
5517 assert(!WroteStrtab);
5518
5519 std::vector<char> Strtab;
5520 StrtabBuilder.finalizeInOrder();
5521 Strtab.resize(new_size: StrtabBuilder.getSize());
5522 StrtabBuilder.write(Buf: (uint8_t *)Strtab.data());
5523
5524 writeBlob(Block: bitc::STRTAB_BLOCK_ID, Record: bitc::STRTAB_BLOB,
5525 Blob: {Strtab.data(), Strtab.size()});
5526
5527 WroteStrtab = true;
5528}
5529
5530void BitcodeWriter::copyStrtab(StringRef Strtab) {
5531 writeBlob(Block: bitc::STRTAB_BLOCK_ID, Record: bitc::STRTAB_BLOB, Blob: Strtab);
5532 WroteStrtab = true;
5533}
5534
5535void BitcodeWriter::writeModule(const Module &M,
5536 bool ShouldPreserveUseListOrder,
5537 const ModuleSummaryIndex *Index,
5538 bool GenerateHash, ModuleHash *ModHash) {
5539 assert(!WroteStrtab);
5540
5541 // The Mods vector is used by irsymtab::build, which requires non-const
5542 // Modules in case it needs to materialize metadata. But the bitcode writer
5543 // requires that the module is materialized, so we can cast to non-const here,
5544 // after checking that it is in fact materialized.
5545 assert(M.isMaterialized());
5546 Mods.push_back(x: const_cast<Module *>(&M));
5547
5548 ModuleBitcodeWriter ModuleWriter(M, StrtabBuilder, *Stream,
5549 ShouldPreserveUseListOrder, Index,
5550 GenerateHash, ModHash);
5551 ModuleWriter.write();
5552}
5553
5554void BitcodeWriter::writeIndex(
5555 const ModuleSummaryIndex *Index,
5556 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex,
5557 const GVSummaryPtrSet *DecSummaries) {
5558 IndexBitcodeWriter IndexWriter(*Stream, StrtabBuilder, *Index, DecSummaries,
5559 ModuleToSummariesForIndex);
5560 IndexWriter.write();
5561}
5562
5563/// Write the specified module to the specified output stream.
5564void llvm::WriteBitcodeToFile(const Module &M, raw_ostream &Out,
5565 bool ShouldPreserveUseListOrder,
5566 const ModuleSummaryIndex *Index,
5567 bool GenerateHash, ModuleHash *ModHash) {
5568 auto Write = [&](BitcodeWriter &Writer) {
5569 Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash,
5570 ModHash);
5571 Writer.writeSymtab();
5572 Writer.writeStrtab();
5573 };
5574 Triple TT(M.getTargetTriple());
5575 if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) {
5576 // If this is darwin or another generic macho target, reserve space for the
5577 // header. Note that the header is computed *after* the output is known, so
5578 // we currently explicitly use a buffer, write to it, and then subsequently
5579 // flush to Out.
5580 SmallVector<char, 0> Buffer;
5581 Buffer.reserve(N: 256 * 1024);
5582 Buffer.insert(I: Buffer.begin(), NumToInsert: BWH_HeaderSize, Elt: 0);
5583 BitcodeWriter Writer(Buffer);
5584 Write(Writer);
5585 emitDarwinBCHeaderAndTrailer(Buffer, TT);
5586 Out.write(Ptr: Buffer.data(), Size: Buffer.size());
5587 } else {
5588 BitcodeWriter Writer(Out);
5589 Write(Writer);
5590 }
5591}
5592
5593void IndexBitcodeWriter::write() {
5594 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5595
5596 writeModuleVersion();
5597
5598 // Write the module paths in the combined index.
5599 writeModStrings();
5600
5601 // Write the summary combined index records.
5602 writeCombinedGlobalValueSummary();
5603
5604 Stream.ExitBlock();
5605}
5606
5607// Write the specified module summary index to the given raw output stream,
5608// where it will be written in a new bitcode block. This is used when
5609// writing the combined index file for ThinLTO. When writing a subset of the
5610// index for a distributed backend, provide a \p ModuleToSummariesForIndex map.
5611void llvm::writeIndexToFile(
5612 const ModuleSummaryIndex &Index, raw_ostream &Out,
5613 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex,
5614 const GVSummaryPtrSet *DecSummaries) {
5615 SmallVector<char, 0> Buffer;
5616 Buffer.reserve(N: 256 * 1024);
5617
5618 BitcodeWriter Writer(Buffer);
5619 Writer.writeIndex(Index: &Index, ModuleToSummariesForIndex, DecSummaries);
5620 Writer.writeStrtab();
5621
5622 Out.write(Ptr: (char *)&Buffer.front(), Size: Buffer.size());
5623}
5624
5625namespace {
5626
5627/// Class to manage the bitcode writing for a thin link bitcode file.
5628class ThinLinkBitcodeWriter : public ModuleBitcodeWriterBase {
5629 /// ModHash is for use in ThinLTO incremental build, generated while writing
5630 /// the module bitcode file.
5631 const ModuleHash *ModHash;
5632
5633public:
5634 ThinLinkBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
5635 BitstreamWriter &Stream,
5636 const ModuleSummaryIndex &Index,
5637 const ModuleHash &ModHash)
5638 : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
5639 /*ShouldPreserveUseListOrder=*/false, &Index),
5640 ModHash(&ModHash) {}
5641
5642 void write();
5643
5644private:
5645 void writeSimplifiedModuleInfo();
5646};
5647
5648} // end anonymous namespace
5649
5650// This function writes a simpilified module info for thin link bitcode file.
5651// It only contains the source file name along with the name(the offset and
5652// size in strtab) and linkage for global values. For the global value info
5653// entry, in order to keep linkage at offset 5, there are three zeros used
5654// as padding.
5655void ThinLinkBitcodeWriter::writeSimplifiedModuleInfo() {
5656 SmallVector<unsigned, 64> Vals;
5657 // Emit the module's source file name.
5658 {
5659 StringEncoding Bits = getStringEncoding(Str: M.getSourceFileName());
5660 BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
5661 if (Bits == SE_Char6)
5662 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
5663 else if (Bits == SE_Fixed7)
5664 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
5665
5666 // MODULE_CODE_SOURCE_FILENAME: [namechar x N]
5667 auto Abbv = std::make_shared<BitCodeAbbrev>();
5668 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
5669 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5670 Abbv->Add(OpInfo: AbbrevOpToUse);
5671 unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5672
5673 for (const auto P : M.getSourceFileName())
5674 Vals.push_back(Elt: (unsigned char)P);
5675
5676 Stream.EmitRecord(Code: bitc::MODULE_CODE_SOURCE_FILENAME, Vals, Abbrev: FilenameAbbrev);
5677 Vals.clear();
5678 }
5679
5680 // Emit the global variable information.
5681 for (const GlobalVariable &GV : M.globals()) {
5682 // GLOBALVAR: [strtab offset, strtab size, 0, 0, 0, linkage]
5683 Vals.push_back(Elt: StrtabBuilder.add(S: GV.getName()));
5684 Vals.push_back(Elt: GV.getName().size());
5685 Vals.push_back(Elt: 0);
5686 Vals.push_back(Elt: 0);
5687 Vals.push_back(Elt: 0);
5688 Vals.push_back(Elt: getEncodedLinkage(GV));
5689
5690 Stream.EmitRecord(Code: bitc::MODULE_CODE_GLOBALVAR, Vals);
5691 Vals.clear();
5692 }
5693
5694 // Emit the function proto information.
5695 for (const Function &F : M) {
5696 // FUNCTION: [strtab offset, strtab size, 0, 0, 0, linkage]
5697 Vals.push_back(Elt: StrtabBuilder.add(S: F.getName()));
5698 Vals.push_back(Elt: F.getName().size());
5699 Vals.push_back(Elt: 0);
5700 Vals.push_back(Elt: 0);
5701 Vals.push_back(Elt: 0);
5702 Vals.push_back(Elt: getEncodedLinkage(GV: F));
5703
5704 Stream.EmitRecord(Code: bitc::MODULE_CODE_FUNCTION, Vals);
5705 Vals.clear();
5706 }
5707
5708 // Emit the alias information.
5709 for (const GlobalAlias &A : M.aliases()) {
5710 // ALIAS: [strtab offset, strtab size, 0, 0, 0, linkage]
5711 Vals.push_back(Elt: StrtabBuilder.add(S: A.getName()));
5712 Vals.push_back(Elt: A.getName().size());
5713 Vals.push_back(Elt: 0);
5714 Vals.push_back(Elt: 0);
5715 Vals.push_back(Elt: 0);
5716 Vals.push_back(Elt: getEncodedLinkage(GV: A));
5717
5718 Stream.EmitRecord(Code: bitc::MODULE_CODE_ALIAS, Vals);
5719 Vals.clear();
5720 }
5721
5722 // Emit the ifunc information.
5723 for (const GlobalIFunc &I : M.ifuncs()) {
5724 // IFUNC: [strtab offset, strtab size, 0, 0, 0, linkage]
5725 Vals.push_back(Elt: StrtabBuilder.add(S: I.getName()));
5726 Vals.push_back(Elt: I.getName().size());
5727 Vals.push_back(Elt: 0);
5728 Vals.push_back(Elt: 0);
5729 Vals.push_back(Elt: 0);
5730 Vals.push_back(Elt: getEncodedLinkage(GV: I));
5731
5732 Stream.EmitRecord(Code: bitc::MODULE_CODE_IFUNC, Vals);
5733 Vals.clear();
5734 }
5735}
5736
5737void ThinLinkBitcodeWriter::write() {
5738 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5739
5740 writeModuleVersion();
5741
5742 writeSimplifiedModuleInfo();
5743
5744 writePerModuleGlobalValueSummary();
5745
5746 // Write module hash.
5747 Stream.EmitRecord(Code: bitc::MODULE_CODE_HASH, Vals: ArrayRef<uint32_t>(*ModHash));
5748
5749 Stream.ExitBlock();
5750}
5751
5752void BitcodeWriter::writeThinLinkBitcode(const Module &M,
5753 const ModuleSummaryIndex &Index,
5754 const ModuleHash &ModHash) {
5755 assert(!WroteStrtab);
5756
5757 // The Mods vector is used by irsymtab::build, which requires non-const
5758 // Modules in case it needs to materialize metadata. But the bitcode writer
5759 // requires that the module is materialized, so we can cast to non-const here,
5760 // after checking that it is in fact materialized.
5761 assert(M.isMaterialized());
5762 Mods.push_back(x: const_cast<Module *>(&M));
5763
5764 ThinLinkBitcodeWriter ThinLinkWriter(M, StrtabBuilder, *Stream, Index,
5765 ModHash);
5766 ThinLinkWriter.write();
5767}
5768
5769// Write the specified thin link bitcode file to the given raw output stream,
5770// where it will be written in a new bitcode block. This is used when
5771// writing the per-module index file for ThinLTO.
5772void llvm::writeThinLinkBitcodeToFile(const Module &M, raw_ostream &Out,
5773 const ModuleSummaryIndex &Index,
5774 const ModuleHash &ModHash) {
5775 SmallVector<char, 0> Buffer;
5776 Buffer.reserve(N: 256 * 1024);
5777
5778 BitcodeWriter Writer(Buffer);
5779 Writer.writeThinLinkBitcode(M, Index, ModHash);
5780 Writer.writeSymtab();
5781 Writer.writeStrtab();
5782
5783 Out.write(Ptr: (char *)&Buffer.front(), Size: Buffer.size());
5784}
5785
5786static const char *getSectionNameForBitcode(const Triple &T) {
5787 switch (T.getObjectFormat()) {
5788 case Triple::MachO:
5789 return "__LLVM,__bitcode";
5790 case Triple::COFF:
5791 case Triple::ELF:
5792 case Triple::Wasm:
5793 case Triple::UnknownObjectFormat:
5794 return ".llvmbc";
5795 case Triple::GOFF:
5796 llvm_unreachable("GOFF is not yet implemented");
5797 break;
5798 case Triple::SPIRV:
5799 if (T.getVendor() == Triple::AMD)
5800 return ".llvmbc";
5801 llvm_unreachable("SPIRV is not yet implemented");
5802 break;
5803 case Triple::XCOFF:
5804 llvm_unreachable("XCOFF is not yet implemented");
5805 break;
5806 case Triple::DXContainer:
5807 llvm_unreachable("DXContainer is not yet implemented");
5808 break;
5809 }
5810 llvm_unreachable("Unimplemented ObjectFormatType");
5811}
5812
5813static const char *getSectionNameForCommandline(const Triple &T) {
5814 switch (T.getObjectFormat()) {
5815 case Triple::MachO:
5816 return "__LLVM,__cmdline";
5817 case Triple::COFF:
5818 case Triple::ELF:
5819 case Triple::Wasm:
5820 case Triple::UnknownObjectFormat:
5821 return ".llvmcmd";
5822 case Triple::GOFF:
5823 llvm_unreachable("GOFF is not yet implemented");
5824 break;
5825 case Triple::SPIRV:
5826 if (T.getVendor() == Triple::AMD)
5827 return ".llvmcmd";
5828 llvm_unreachable("SPIRV is not yet implemented");
5829 break;
5830 case Triple::XCOFF:
5831 llvm_unreachable("XCOFF is not yet implemented");
5832 break;
5833 case Triple::DXContainer:
5834 llvm_unreachable("DXC is not yet implemented");
5835 break;
5836 }
5837 llvm_unreachable("Unimplemented ObjectFormatType");
5838}
5839
5840void llvm::embedBitcodeInModule(llvm::Module &M, llvm::MemoryBufferRef Buf,
5841 bool EmbedBitcode, bool EmbedCmdline,
5842 const std::vector<uint8_t> &CmdArgs) {
5843 // Save llvm.compiler.used and remove it.
5844 SmallVector<Constant *, 2> UsedArray;
5845 SmallVector<GlobalValue *, 4> UsedGlobals;
5846 GlobalVariable *Used = collectUsedGlobalVariables(M, Vec&: UsedGlobals, CompilerUsed: true);
5847 Type *UsedElementType = Used ? Used->getValueType()->getArrayElementType()
5848 : PointerType::getUnqual(C&: M.getContext());
5849 for (auto *GV : UsedGlobals) {
5850 if (GV->getName() != "llvm.embedded.module" &&
5851 GV->getName() != "llvm.cmdline")
5852 UsedArray.push_back(
5853 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
5854 }
5855 if (Used)
5856 Used->eraseFromParent();
5857
5858 // Embed the bitcode for the llvm module.
5859 std::string Data;
5860 ArrayRef<uint8_t> ModuleData;
5861 Triple T(M.getTargetTriple());
5862
5863 if (EmbedBitcode) {
5864 if (Buf.getBufferSize() == 0 ||
5865 !isBitcode(BufPtr: (const unsigned char *)Buf.getBufferStart(),
5866 BufEnd: (const unsigned char *)Buf.getBufferEnd())) {
5867 // If the input is LLVM Assembly, bitcode is produced by serializing
5868 // the module. Use-lists order need to be preserved in this case.
5869 llvm::raw_string_ostream OS(Data);
5870 llvm::WriteBitcodeToFile(M, Out&: OS, /* ShouldPreserveUseListOrder */ true);
5871 ModuleData =
5872 ArrayRef<uint8_t>((const uint8_t *)OS.str().data(), OS.str().size());
5873 } else
5874 // If the input is LLVM bitcode, write the input byte stream directly.
5875 ModuleData = ArrayRef<uint8_t>((const uint8_t *)Buf.getBufferStart(),
5876 Buf.getBufferSize());
5877 }
5878 llvm::Constant *ModuleConstant =
5879 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts: ModuleData);
5880 llvm::GlobalVariable *GV = new llvm::GlobalVariable(
5881 M, ModuleConstant->getType(), true, llvm::GlobalValue::PrivateLinkage,
5882 ModuleConstant);
5883 GV->setSection(getSectionNameForBitcode(T));
5884 // Set alignment to 1 to prevent padding between two contributions from input
5885 // sections after linking.
5886 GV->setAlignment(Align(1));
5887 UsedArray.push_back(
5888 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
5889 if (llvm::GlobalVariable *Old =
5890 M.getGlobalVariable(Name: "llvm.embedded.module", AllowInternal: true)) {
5891 assert(Old->hasZeroLiveUses() &&
5892 "llvm.embedded.module can only be used once in llvm.compiler.used");
5893 GV->takeName(V: Old);
5894 Old->eraseFromParent();
5895 } else {
5896 GV->setName("llvm.embedded.module");
5897 }
5898
5899 // Skip if only bitcode needs to be embedded.
5900 if (EmbedCmdline) {
5901 // Embed command-line options.
5902 ArrayRef<uint8_t> CmdData(const_cast<uint8_t *>(CmdArgs.data()),
5903 CmdArgs.size());
5904 llvm::Constant *CmdConstant =
5905 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts: CmdData);
5906 GV = new llvm::GlobalVariable(M, CmdConstant->getType(), true,
5907 llvm::GlobalValue::PrivateLinkage,
5908 CmdConstant);
5909 GV->setSection(getSectionNameForCommandline(T));
5910 GV->setAlignment(Align(1));
5911 UsedArray.push_back(
5912 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
5913 if (llvm::GlobalVariable *Old = M.getGlobalVariable(Name: "llvm.cmdline", AllowInternal: true)) {
5914 assert(Old->hasZeroLiveUses() &&
5915 "llvm.cmdline can only be used once in llvm.compiler.used");
5916 GV->takeName(V: Old);
5917 Old->eraseFromParent();
5918 } else {
5919 GV->setName("llvm.cmdline");
5920 }
5921 }
5922
5923 if (UsedArray.empty())
5924 return;
5925
5926 // Recreate llvm.compiler.used.
5927 ArrayType *ATy = ArrayType::get(ElementType: UsedElementType, NumElements: UsedArray.size());
5928 auto *NewUsed = new GlobalVariable(
5929 M, ATy, false, llvm::GlobalValue::AppendingLinkage,
5930 llvm::ConstantArray::get(T: ATy, V: UsedArray), "llvm.compiler.used");
5931 NewUsed->setSection("llvm.metadata");
5932}
5933