1//===- Bitcode/Writer/BitcodeWriter.cpp - Bitcode Writer ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Bitcode writer implementation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Bitcode/BitcodeWriter.h"
14#include "ValueEnumerator.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/DenseMap.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SetVector.h"
21#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/SmallString.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringMap.h"
25#include "llvm/ADT/StringRef.h"
26#include "llvm/Analysis/MemoryProfileInfo.h"
27#include "llvm/BinaryFormat/Dwarf.h"
28#include "llvm/Bitcode/BitcodeCommon.h"
29#include "llvm/Bitcode/BitcodeReader.h"
30#include "llvm/Bitcode/LLVMBitCodes.h"
31#include "llvm/Bitstream/BitCodes.h"
32#include "llvm/Bitstream/BitstreamWriter.h"
33#include "llvm/Config/llvm-config.h"
34#include "llvm/IR/Attributes.h"
35#include "llvm/IR/BasicBlock.h"
36#include "llvm/IR/Comdat.h"
37#include "llvm/IR/Constant.h"
38#include "llvm/IR/ConstantRangeList.h"
39#include "llvm/IR/Constants.h"
40#include "llvm/IR/DebugInfoMetadata.h"
41#include "llvm/IR/DebugLoc.h"
42#include "llvm/IR/DerivedTypes.h"
43#include "llvm/IR/Function.h"
44#include "llvm/IR/GlobalAlias.h"
45#include "llvm/IR/GlobalIFunc.h"
46#include "llvm/IR/GlobalObject.h"
47#include "llvm/IR/GlobalValue.h"
48#include "llvm/IR/GlobalVariable.h"
49#include "llvm/IR/InlineAsm.h"
50#include "llvm/IR/InstrTypes.h"
51#include "llvm/IR/Instruction.h"
52#include "llvm/IR/Instructions.h"
53#include "llvm/IR/LLVMContext.h"
54#include "llvm/IR/Metadata.h"
55#include "llvm/IR/Module.h"
56#include "llvm/IR/ModuleSummaryIndex.h"
57#include "llvm/IR/Operator.h"
58#include "llvm/IR/Type.h"
59#include "llvm/IR/UseListOrder.h"
60#include "llvm/IR/Value.h"
61#include "llvm/IR/ValueSymbolTable.h"
62#include "llvm/MC/StringTableBuilder.h"
63#include "llvm/MC/TargetRegistry.h"
64#include "llvm/Object/IRSymtab.h"
65#include "llvm/ProfileData/MemProf.h"
66#include "llvm/ProfileData/MemProfRadixTree.h"
67#include "llvm/Support/AtomicOrdering.h"
68#include "llvm/Support/Casting.h"
69#include "llvm/Support/CommandLine.h"
70#include "llvm/Support/Compiler.h"
71#include "llvm/Support/Endian.h"
72#include "llvm/Support/Error.h"
73#include "llvm/Support/ErrorHandling.h"
74#include "llvm/Support/MathExtras.h"
75#include "llvm/Support/SHA1.h"
76#include "llvm/Support/raw_ostream.h"
77#include "llvm/TargetParser/Triple.h"
78#include <algorithm>
79#include <cassert>
80#include <cstddef>
81#include <cstdint>
82#include <iterator>
83#include <map>
84#include <memory>
85#include <optional>
86#include <string>
87#include <utility>
88#include <vector>
89
90using namespace llvm;
91using namespace llvm::memprof;
92
93static cl::opt<unsigned>
94 IndexThreshold("bitcode-mdindex-threshold", cl::Hidden, cl::init(Val: 25),
95 cl::desc("Number of metadatas above which we emit an index "
96 "to enable lazy-loading"));
97static cl::opt<uint32_t> FlushThreshold(
98 "bitcode-flush-threshold", cl::Hidden, cl::init(Val: 512),
99 cl::desc("The threshold (unit M) for flushing LLVM bitcode."));
100
101// Since we only use the context information in the memprof summary records in
102// the LTO backends to do assertion checking, save time and space by only
103// serializing the context for non-NDEBUG builds.
104// TODO: Currently this controls writing context of the allocation info records,
105// which are larger and more expensive, but we should do this for the callsite
106// records as well.
107// FIXME: Convert to a const once this has undergone more sigificant testing.
108static cl::opt<bool>
109 CombinedIndexMemProfContext("combined-index-memprof-context", cl::Hidden,
110#ifdef NDEBUG
111 cl::init(Val: false),
112#else
113 cl::init(true),
114#endif
115 cl::desc(""));
116
117static cl::opt<bool> PreserveBitcodeUseListOrder(
118 "preserve-bc-uselistorder", cl::Hidden, cl::init(Val: true),
119 cl::desc("Preserve use-list order when writing LLVM bitcode."));
120
121namespace llvm {
122extern FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold;
123}
124
125namespace {
126
127/// These are manifest constants used by the bitcode writer. They do not need to
128/// be kept in sync with the reader, but need to be consistent within this file.
129enum {
130 // VALUE_SYMTAB_BLOCK abbrev id's.
131 VST_ENTRY_8_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
132 VST_ENTRY_7_ABBREV,
133 VST_ENTRY_6_ABBREV,
134 VST_BBENTRY_6_ABBREV,
135
136 // CONSTANTS_BLOCK abbrev id's.
137 CONSTANTS_SETTYPE_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
138 CONSTANTS_INTEGER_ABBREV,
139 CONSTANTS_BYTE_ABBREV,
140 CONSTANTS_CE_CAST_Abbrev,
141 CONSTANTS_NULL_Abbrev,
142
143 // FUNCTION_BLOCK abbrev id's.
144 FUNCTION_INST_LOAD_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
145 FUNCTION_INST_STORE_ABBREV,
146 FUNCTION_INST_UNOP_ABBREV,
147 FUNCTION_INST_UNOP_FLAGS_ABBREV,
148 FUNCTION_INST_BINOP_ABBREV,
149 FUNCTION_INST_BINOP_FLAGS_ABBREV,
150 FUNCTION_INST_CAST_ABBREV,
151 FUNCTION_INST_CAST_FLAGS_ABBREV,
152 FUNCTION_INST_RET_VOID_ABBREV,
153 FUNCTION_INST_RET_VAL_ABBREV,
154 FUNCTION_INST_BR_UNCOND_ABBREV,
155 FUNCTION_INST_BR_COND_ABBREV,
156 FUNCTION_INST_UNREACHABLE_ABBREV,
157 FUNCTION_INST_GEP_ABBREV,
158 FUNCTION_INST_CMP_ABBREV,
159 FUNCTION_INST_CMP_FLAGS_ABBREV,
160 FUNCTION_DEBUG_RECORD_VALUE_ABBREV,
161 FUNCTION_DEBUG_LOC_ABBREV,
162 FUNCTION_DEBUG_LOC_LAYERS_ABBREV,
163};
164
165/// Abstract class to manage the bitcode writing, subclassed for each bitcode
166/// file type.
167class BitcodeWriterBase {
168protected:
169 /// The stream created and owned by the client.
170 BitstreamWriter &Stream;
171
172 StringTableBuilder &StrtabBuilder;
173
174public:
175 /// Constructs a BitcodeWriterBase object that writes to the provided
176 /// \p Stream.
177 BitcodeWriterBase(BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder)
178 : Stream(Stream), StrtabBuilder(StrtabBuilder) {}
179
180protected:
181 void writeModuleVersion();
182};
183
184void BitcodeWriterBase::writeModuleVersion() {
185 // VERSION: [version#]
186 Stream.EmitRecord(Code: bitc::MODULE_CODE_VERSION, Vals: ArrayRef<uint64_t>{2});
187}
188
189/// Base class to manage the module bitcode writing, currently subclassed for
190/// ModuleBitcodeWriter and ThinLinkBitcodeWriter.
191class ModuleBitcodeWriterBase : public BitcodeWriterBase {
192protected:
193 /// The Module to write to bitcode.
194 const Module &M;
195
196 /// Enumerates ids for all values in the module.
197 ValueEnumerator VE;
198
199 /// Optional per-module index to write for ThinLTO.
200 const ModuleSummaryIndex *Index;
201
202 /// Map that holds the correspondence between GUIDs in the summary index,
203 /// that came from indirect call profiles, and a value id generated by this
204 /// class to use in the VST and summary block records.
205 std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
206
207 /// Tracks the last value id recorded in the GUIDToValueMap.
208 unsigned GlobalValueId;
209
210 /// Saves the offset of the VSTOffset record that must eventually be
211 /// backpatched with the offset of the actual VST.
212 uint64_t VSTOffsetPlaceholder = 0;
213
214public:
215 /// Constructs a ModuleBitcodeWriterBase object for the given Module,
216 /// writing to the provided \p Buffer.
217 ModuleBitcodeWriterBase(const Module &M, StringTableBuilder &StrtabBuilder,
218 BitstreamWriter &Stream,
219 bool ShouldPreserveUseListOrder,
220 const ModuleSummaryIndex *Index)
221 : BitcodeWriterBase(Stream, StrtabBuilder), M(M),
222 VE(M, PreserveBitcodeUseListOrder.getNumOccurrences()
223 ? PreserveBitcodeUseListOrder
224 : ShouldPreserveUseListOrder),
225 Index(Index) {
226 // Assign ValueIds to any callee values in the index that came from
227 // indirect call profiles and were recorded as a GUID not a Value*
228 // (which would have been assigned an ID by the ValueEnumerator).
229 // The starting ValueId is just after the number of values in the
230 // ValueEnumerator, so that they can be emitted in the VST.
231 GlobalValueId = VE.getValues().size();
232 if (!Index)
233 return;
234 // Sort by GUID for deterministic value ID assignment.
235 for (const auto &GUIDSummaryLists :
236 Index->sortedGlobalValueSummariesRange())
237 // Examine all summaries for this GUID.
238 for (auto &Summary : GUIDSummaryLists.second.getSummaryList())
239 if (auto *FS = dyn_cast<FunctionSummary>(Val: Summary.get())) {
240 // For each call in the function summary, see if the call
241 // is to a GUID (which means it is for an indirect call,
242 // otherwise we would have a Value for it). If so, synthesize
243 // a value id.
244 for (auto &CallEdge : FS->calls())
245 if (!CallEdge.first.haveGVs() || !CallEdge.first.getValue())
246 assignValueId(ValGUID: CallEdge.first.getGUID());
247
248 // For each referenced variables in the function summary, see if the
249 // variable is represented by a GUID (as opposed to a symbol to
250 // declarations or definitions in the module). If so, synthesize a
251 // value id.
252 for (auto &RefEdge : FS->refs())
253 if (!RefEdge.haveGVs() || !RefEdge.getValue())
254 assignValueId(ValGUID: RefEdge.getGUID());
255 }
256 }
257
258protected:
259 void writePerModuleGlobalValueSummary();
260 void writeGUIDList();
261
262private:
263 void writePerModuleFunctionSummaryRecord(
264 SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
265 unsigned ValueID, unsigned FSCallsProfileAbbrev, unsigned CallsiteAbbrev,
266 unsigned AllocAbbrev, unsigned ContextIdAbbvId, const Function &F,
267 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
268 CallStackId &CallStackCount);
269 void writeModuleLevelReferences(const GlobalVariable &V,
270 SmallVector<uint64_t, 64> &NameVals,
271 unsigned FSModRefsAbbrev,
272 unsigned FSModVTableRefsAbbrev);
273
274 void assignValueId(GlobalValue::GUID ValGUID) {
275 GUIDToValueIdMap[ValGUID] = ++GlobalValueId;
276 }
277
278 unsigned getValueId(GlobalValue::GUID ValGUID) {
279 const auto &VMI = GUIDToValueIdMap.find(x: ValGUID);
280 // Expect that any GUID value had a value Id assigned by an
281 // earlier call to assignValueId.
282 assert(VMI != GUIDToValueIdMap.end() &&
283 "GUID does not have assigned value Id");
284 return VMI->second;
285 }
286
287 // Helper to get the valueId for the type of value recorded in VI.
288 unsigned getValueId(ValueInfo VI) {
289 if (!VI.haveGVs() || !VI.getValue())
290 return getValueId(ValGUID: VI.getGUID());
291 return VE.getValueID(V: VI.getValue());
292 }
293
294 std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
295};
296
297/// Class to manage the bitcode writing for a module.
298class ModuleBitcodeWriter : public ModuleBitcodeWriterBase {
299 /// True if a module hash record should be written.
300 bool GenerateHash;
301
302 /// If non-null, when GenerateHash is true, the resulting hash is written
303 /// into ModHash.
304 ModuleHash *ModHash;
305
306 SHA1 Hasher;
307
308 /// The start bit of the identification block.
309 uint64_t BitcodeStartBit;
310
311 /// Abbrev for DILocations that carry an irlayers operand. Locations without
312 /// layers use the shorter abbrev threaded through writeDILocation's \p
313 /// Abbrev parameter, so they pay nothing for a field they do not use.
314 unsigned DILocationLayersAbbrev = 0;
315
316public:
317 /// Constructs a ModuleBitcodeWriter object for the given Module,
318 /// writing to the provided \p Buffer.
319 ModuleBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
320 BitstreamWriter &Stream, bool ShouldPreserveUseListOrder,
321 const ModuleSummaryIndex *Index, bool GenerateHash,
322 ModuleHash *ModHash = nullptr)
323 : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
324 ShouldPreserveUseListOrder, Index),
325 GenerateHash(GenerateHash), ModHash(ModHash),
326 BitcodeStartBit(Stream.GetCurrentBitNo()) {}
327
328 /// Emit the current module to the bitstream.
329 void write();
330
331private:
332 uint64_t bitcodeStartBit() { return BitcodeStartBit; }
333
334 size_t addToStrtab(StringRef Str);
335
336 void writeAttributeGroupTable();
337 void writeAttributeTable();
338 void writeTypeTable();
339 void writeComdats();
340 void writeValueSymbolTableForwardDecl();
341 void writeModuleInfo();
342 void writeValueAsMetadata(const ValueAsMetadata *MD,
343 SmallVectorImpl<uint64_t> &Record);
344 void writeMDTuple(const MDTuple *N, SmallVectorImpl<uint64_t> &Record,
345 unsigned Abbrev);
346 unsigned createDILocationAbbrev(bool WithIRLayers);
347 void writeDILocation(const DILocation *N, SmallVectorImpl<uint64_t> &Record,
348 unsigned &Abbrev);
349 void writeDILayerLoc(const DILayerLoc *N, SmallVectorImpl<uint64_t> &Record,
350 unsigned Abbrev);
351 void writeDILayerLocList(const DILayerLocList *N,
352 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
353 unsigned createGenericDINodeAbbrev();
354 void writeGenericDINode(const GenericDINode *N,
355 SmallVectorImpl<uint64_t> &Record, unsigned &Abbrev);
356 void writeDISubrange(const DISubrange *N, SmallVectorImpl<uint64_t> &Record,
357 unsigned Abbrev);
358 void writeDIGenericSubrange(const DIGenericSubrange *N,
359 SmallVectorImpl<uint64_t> &Record,
360 unsigned Abbrev);
361 void writeDIEnumerator(const DIEnumerator *N,
362 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
363 void writeDIBasicType(const DIBasicType *N, SmallVectorImpl<uint64_t> &Record,
364 unsigned Abbrev);
365 void writeDIFixedPointType(const DIFixedPointType *N,
366 SmallVectorImpl<uint64_t> &Record,
367 unsigned Abbrev);
368 void writeDIStringType(const DIStringType *N,
369 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
370 void writeDIDerivedType(const DIDerivedType *N,
371 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
372 void writeDISubrangeType(const DISubrangeType *N,
373 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
374 void writeDICompositeType(const DICompositeType *N,
375 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
376 void writeDISubroutineType(const DISubroutineType *N,
377 SmallVectorImpl<uint64_t> &Record,
378 unsigned Abbrev);
379 void writeDIFile(const DIFile *N, SmallVectorImpl<uint64_t> &Record,
380 unsigned Abbrev);
381 void writeDICompileUnit(const DICompileUnit *N,
382 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
383 void writeDISubprogram(const DISubprogram *N,
384 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
385 void writeDILexicalBlock(const DILexicalBlock *N,
386 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
387 void writeDILexicalBlockFile(const DILexicalBlockFile *N,
388 SmallVectorImpl<uint64_t> &Record,
389 unsigned Abbrev);
390 void writeDICommonBlock(const DICommonBlock *N,
391 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
392 void writeDINamespace(const DINamespace *N, SmallVectorImpl<uint64_t> &Record,
393 unsigned Abbrev);
394 void writeDIMacro(const DIMacro *N, SmallVectorImpl<uint64_t> &Record,
395 unsigned Abbrev);
396 void writeDIMacroFile(const DIMacroFile *N, SmallVectorImpl<uint64_t> &Record,
397 unsigned Abbrev);
398 void writeDIArgList(const DIArgList *N, SmallVectorImpl<uint64_t> &Record);
399 void writeDIModule(const DIModule *N, SmallVectorImpl<uint64_t> &Record,
400 unsigned Abbrev);
401 void writeDIAssignID(const DIAssignID *N, SmallVectorImpl<uint64_t> &Record,
402 unsigned Abbrev);
403 void writeDITemplateTypeParameter(const DITemplateTypeParameter *N,
404 SmallVectorImpl<uint64_t> &Record,
405 unsigned Abbrev);
406 void writeDITemplateValueParameter(const DITemplateValueParameter *N,
407 SmallVectorImpl<uint64_t> &Record,
408 unsigned Abbrev);
409 void writeDIGlobalVariable(const DIGlobalVariable *N,
410 SmallVectorImpl<uint64_t> &Record,
411 unsigned Abbrev);
412 void writeDILocalVariable(const DILocalVariable *N,
413 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
414 void writeDILabel(const DILabel *N,
415 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
416 void writeDIExpression(const DIExpression *N,
417 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
418 void writeDIGlobalVariableExpression(const DIGlobalVariableExpression *N,
419 SmallVectorImpl<uint64_t> &Record,
420 unsigned Abbrev);
421 void writeDIObjCProperty(const DIObjCProperty *N,
422 SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
423 void writeDIProperty(const DIProperty *N, SmallVectorImpl<uint64_t> &Record,
424 unsigned Abbrev);
425 void writeDIImportedEntity(const DIImportedEntity *N,
426 SmallVectorImpl<uint64_t> &Record,
427 unsigned Abbrev);
428 unsigned createNamedMetadataAbbrev();
429 void writeNamedMetadata(SmallVectorImpl<uint64_t> &Record);
430 unsigned createMetadataStringsAbbrev();
431 void writeMetadataStrings(ArrayRef<const Metadata *> Strings,
432 SmallVectorImpl<uint64_t> &Record);
433 void writeMetadataRecords(ArrayRef<const Metadata *> MDs,
434 SmallVectorImpl<uint64_t> &Record,
435 std::vector<unsigned> *MDAbbrevs = nullptr,
436 std::vector<uint64_t> *IndexPos = nullptr);
437 void writeModuleMetadata();
438 void writeFunctionMetadata(const Function &F);
439 void writeFunctionMetadataAttachment(const Function &F);
440 void pushGlobalMetadataAttachment(SmallVectorImpl<uint64_t> &Record,
441 const GlobalObject &GO);
442 void writeModuleMetadataKinds();
443 void writeOperandBundleTags();
444 void writeSyncScopeNames();
445 void writeConstants(unsigned FirstVal, unsigned LastVal, bool isGlobal);
446 void writeModuleConstants();
447 bool pushValueAndType(const Value *V, unsigned InstID,
448 SmallVectorImpl<unsigned> &Vals);
449 bool pushValueOrMetadata(const Value *V, unsigned InstID,
450 SmallVectorImpl<unsigned> &Vals);
451 void writeOperandBundles(const CallBase &CB, unsigned InstID);
452 void pushValue(const Value *V, unsigned InstID,
453 SmallVectorImpl<unsigned> &Vals);
454 void pushValueSigned(const Value *V, unsigned InstID,
455 SmallVectorImpl<uint64_t> &Vals);
456 void writeInstruction(const Instruction &I, unsigned InstID,
457 SmallVectorImpl<unsigned> &Vals);
458 void writeFunctionLevelValueSymbolTable(const ValueSymbolTable &VST);
459 void writeGlobalValueSymbolTable(
460 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
461 void writeUseList(UseListOrder &&Order);
462 void writeUseListBlock(const Function *F);
463 void
464 writeFunction(const Function &F,
465 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
466 void writeBlockInfo();
467 void writeModuleHash(StringRef View);
468
469 unsigned getEncodedSyncScopeID(SyncScope::ID SSID) {
470 return unsigned(SSID);
471 }
472
473 unsigned getEncodedAlign(MaybeAlign Alignment) { return encode(A: Alignment); }
474};
475
476/// Class to manage the bitcode writing for a combined index.
477class IndexBitcodeWriter : public BitcodeWriterBase {
478 /// The combined index to write to bitcode.
479 const ModuleSummaryIndex &Index;
480
481 /// When writing combined summaries, provides the set of global value
482 /// summaries for which the value (function, function alias, etc) should be
483 /// imported as a declaration.
484 const GVSummaryPtrSet *DecSummaries = nullptr;
485
486 /// When writing a subset of the index for distributed backends, client
487 /// provides a map of modules to the corresponding GUIDs/summaries to write.
488 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex;
489
490 /// Map that holds the correspondence between the GUID used in the combined
491 /// index and a value id generated by this class to use in references.
492 std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
493
494 // The stack ids used by this index, which will be a subset of those in
495 // the full index in the case of distributed indexes.
496 std::vector<uint64_t> StackIds;
497
498 // Keep a map of the stack id indices used by records being written for this
499 // index to the index of the corresponding stack id in the above StackIds
500 // vector. Ensures we write each referenced stack id once.
501 DenseMap<unsigned, unsigned> StackIdIndicesToIndex;
502
503 /// Tracks the last value id recorded in the GUIDToValueMap.
504 unsigned GlobalValueId = 0;
505
506 /// Tracks the assignment of module paths in the module path string table to
507 /// an id assigned for use in summary references to the module path.
508 DenseMap<StringRef, uint64_t> ModuleIdMap;
509
510public:
511 /// Constructs a IndexBitcodeWriter object for the given combined index,
512 /// writing to the provided \p Buffer. When writing a subset of the index
513 /// for a distributed backend, provide a \p ModuleToSummariesForIndex map.
514 /// If provided, \p DecSummaries specifies the set of summaries for which
515 /// the corresponding functions or aliased functions should be imported as a
516 /// declaration (but not definition) for each module.
517 IndexBitcodeWriter(
518 BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder,
519 const ModuleSummaryIndex &Index,
520 const GVSummaryPtrSet *DecSummaries = nullptr,
521 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex = nullptr)
522 : BitcodeWriterBase(Stream, StrtabBuilder), Index(Index),
523 DecSummaries(DecSummaries),
524 ModuleToSummariesForIndex(ModuleToSummariesForIndex) {
525
526 // See if the StackIdIndex was already added to the StackId map and
527 // vector. If not, record it.
528 auto RecordStackIdReference = [&](unsigned StackIdIndex) {
529 // If the StackIdIndex is not yet in the map, the below insert ensures
530 // that it will point to the new StackIds vector entry we push to just
531 // below.
532 auto Inserted =
533 StackIdIndicesToIndex.insert(KV: {StackIdIndex, StackIds.size()});
534 if (Inserted.second)
535 StackIds.push_back(x: Index.getStackIdAtIndex(Index: StackIdIndex));
536 };
537
538 // Assign unique value ids to all summaries to be written, for use
539 // in writing out the call graph edges. Save the mapping from GUID
540 // to the new global value id to use when writing those edges, which
541 // are currently saved in the index in terms of GUID.
542 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
543 GUIDToValueIdMap[I.first] = ++GlobalValueId;
544 // If this is invoked for an aliasee, we want to record the above mapping,
545 // but not the information needed for its summary entry (if the aliasee is
546 // to be imported, we will invoke this separately with IsAliasee=false).
547 if (IsAliasee)
548 return;
549 auto *FS = dyn_cast<FunctionSummary>(Val: I.second);
550 if (!FS)
551 return;
552 // Record all stack id indices actually used in the summary entries being
553 // written, so that we can compact them in the case of distributed ThinLTO
554 // indexes.
555 for (auto &CI : FS->callsites()) {
556 // If the stack id list is empty, this callsite info was synthesized for
557 // a missing tail call frame. Ensure that the callee's GUID gets a value
558 // id. Normally we only generate these for defined summaries, which in
559 // the case of distributed ThinLTO is only the functions already defined
560 // in the module or that we want to import. We don't bother to include
561 // all the callee symbols as they aren't normally needed in the backend.
562 // However, for the synthesized callsite infos we do need the callee
563 // GUID in the backend so that we can correlate the identified callee
564 // with this callsite info (which for non-tail calls is done by the
565 // ordering of the callsite infos and verified via stack ids).
566 if (CI.StackIdIndices.empty()) {
567 GUIDToValueIdMap[CI.Callee.getGUID()] = ++GlobalValueId;
568 continue;
569 }
570 for (auto Idx : CI.StackIdIndices)
571 RecordStackIdReference(Idx);
572 }
573 if (CombinedIndexMemProfContext) {
574 for (auto &AI : FS->allocs())
575 for (auto &MIB : AI.MIBs)
576 for (auto Idx : MIB.StackIdIndices)
577 RecordStackIdReference(Idx);
578 }
579 });
580 }
581
582 /// The below iterator returns the GUID and associated summary.
583 using GVInfo = std::pair<GlobalValue::GUID, GlobalValueSummary *>;
584
585 /// Calls the callback for each value GUID and summary to be written to
586 /// bitcode. This hides the details of whether they are being pulled from the
587 /// entire index or just those in a provided ModuleToSummariesForIndex map.
588 template<typename Functor>
589 void forEachSummary(Functor Callback) {
590 if (ModuleToSummariesForIndex) {
591 for (auto &M : *ModuleToSummariesForIndex)
592 for (auto &[GUID, GVS] : M.second) {
593 Callback({GUID, GVS}, false);
594 // Ensure aliasee is handled, e.g. for assigning a valueId,
595 // even if we are not importing the aliasee directly (the
596 // imported alias will contain a copy of aliasee).
597 if (auto *AS = dyn_cast<AliasSummary>(Val: GVS))
598 Callback({AS->getAliaseeGUID(), &AS->getAliasee()}, true);
599 }
600 } else {
601 // Sort by GUID for deterministic output.
602 for (const auto &Summaries : Index.sortedGlobalValueSummariesRange())
603 for (auto &Summary : Summaries.second.getSummaryList())
604 Callback({Summaries.first, Summary.get()}, false);
605 }
606 }
607
608 /// Calls the callback for each entry in the modulePaths StringMap that
609 /// should be written to the module path string table. This hides the details
610 /// of whether they are being pulled from the entire index or just those in a
611 /// provided ModuleToSummariesForIndex map.
612 template <typename Functor> void forEachModule(Functor Callback) {
613 if (ModuleToSummariesForIndex) {
614 for (const auto &M : *ModuleToSummariesForIndex) {
615 const auto &MPI = Index.modulePaths().find(Key: M.first);
616 if (MPI == Index.modulePaths().end()) {
617 // This should only happen if the bitcode file was empty, in which
618 // case we shouldn't be importing (the ModuleToSummariesForIndex
619 // would only include the module we are writing and index for).
620 assert(ModuleToSummariesForIndex->size() == 1);
621 continue;
622 }
623 Callback(*MPI);
624 }
625 } else {
626 // Since StringMap iteration order isn't guaranteed, order by path string
627 // first.
628 // FIXME: Make this a vector of StringMapEntry instead to avoid the later
629 // map lookup.
630 std::vector<StringRef> ModulePaths;
631 for (auto &[ModPath, _] : Index.modulePaths())
632 ModulePaths.push_back(x: ModPath);
633 llvm::sort(C&: ModulePaths);
634 for (auto &ModPath : ModulePaths)
635 Callback(*Index.modulePaths().find(Key: ModPath));
636 }
637 }
638
639 /// Main entry point for writing a combined index to bitcode.
640 void write();
641
642private:
643 void writeModStrings();
644 void writeCombinedGlobalValueSummary();
645
646 std::optional<unsigned> getValueId(GlobalValue::GUID ValGUID) {
647 auto VMI = GUIDToValueIdMap.find(x: ValGUID);
648 if (VMI == GUIDToValueIdMap.end())
649 return std::nullopt;
650 return VMI->second;
651 }
652
653 std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
654};
655
656} // end anonymous namespace
657
658static unsigned getEncodedCastOpcode(unsigned Opcode) {
659 switch (Opcode) {
660 default: llvm_unreachable("Unknown cast instruction!");
661 case Instruction::Trunc : return bitc::CAST_TRUNC;
662 case Instruction::ZExt : return bitc::CAST_ZEXT;
663 case Instruction::SExt : return bitc::CAST_SEXT;
664 case Instruction::FPToUI : return bitc::CAST_FPTOUI;
665 case Instruction::FPToSI : return bitc::CAST_FPTOSI;
666 case Instruction::UIToFP : return bitc::CAST_UITOFP;
667 case Instruction::SIToFP : return bitc::CAST_SITOFP;
668 case Instruction::FPTrunc : return bitc::CAST_FPTRUNC;
669 case Instruction::FPExt : return bitc::CAST_FPEXT;
670 case Instruction::PtrToAddr: return bitc::CAST_PTRTOADDR;
671 case Instruction::PtrToInt: return bitc::CAST_PTRTOINT;
672 case Instruction::IntToPtr: return bitc::CAST_INTTOPTR;
673 case Instruction::BitCast : return bitc::CAST_BITCAST;
674 case Instruction::AddrSpaceCast: return bitc::CAST_ADDRSPACECAST;
675 }
676}
677
678static unsigned getEncodedUnaryOpcode(unsigned Opcode) {
679 switch (Opcode) {
680 default: llvm_unreachable("Unknown binary instruction!");
681 case Instruction::FNeg: return bitc::UNOP_FNEG;
682 }
683}
684
685static unsigned getEncodedBinaryOpcode(unsigned Opcode) {
686 switch (Opcode) {
687 default: llvm_unreachable("Unknown binary instruction!");
688 case Instruction::Add:
689 case Instruction::FAdd: return bitc::BINOP_ADD;
690 case Instruction::Sub:
691 case Instruction::FSub: return bitc::BINOP_SUB;
692 case Instruction::Mul:
693 case Instruction::FMul: return bitc::BINOP_MUL;
694 case Instruction::UDiv: return bitc::BINOP_UDIV;
695 case Instruction::FDiv:
696 case Instruction::SDiv: return bitc::BINOP_SDIV;
697 case Instruction::URem: return bitc::BINOP_UREM;
698 case Instruction::FRem:
699 case Instruction::SRem: return bitc::BINOP_SREM;
700 case Instruction::Shl: return bitc::BINOP_SHL;
701 case Instruction::LShr: return bitc::BINOP_LSHR;
702 case Instruction::AShr: return bitc::BINOP_ASHR;
703 case Instruction::And: return bitc::BINOP_AND;
704 case Instruction::Or: return bitc::BINOP_OR;
705 case Instruction::Xor: return bitc::BINOP_XOR;
706 }
707}
708
709static unsigned getEncodedRMWOperation(const AtomicRMWInst &I) {
710 unsigned Encoding = 0;
711 switch (I.getOperation()) {
712 default: llvm_unreachable("Unknown RMW operation!");
713 case AtomicRMWInst::Xchg:
714 Encoding = bitc::RMW_XCHG;
715 break;
716 case AtomicRMWInst::Add:
717 Encoding = bitc::RMW_ADD;
718 break;
719 case AtomicRMWInst::Sub:
720 Encoding = bitc::RMW_SUB;
721 break;
722 case AtomicRMWInst::And:
723 Encoding = bitc::RMW_AND;
724 break;
725 case AtomicRMWInst::Nand:
726 Encoding = bitc::RMW_NAND;
727 break;
728 case AtomicRMWInst::Or:
729 Encoding = bitc::RMW_OR;
730 break;
731 case AtomicRMWInst::Xor:
732 Encoding = bitc::RMW_XOR;
733 break;
734 case AtomicRMWInst::Max:
735 Encoding = bitc::RMW_MAX;
736 break;
737 case AtomicRMWInst::Min:
738 Encoding = bitc::RMW_MIN;
739 break;
740 case AtomicRMWInst::UMax:
741 Encoding = bitc::RMW_UMAX;
742 break;
743 case AtomicRMWInst::UMin:
744 Encoding = bitc::RMW_UMIN;
745 break;
746 case AtomicRMWInst::FAdd:
747 Encoding = bitc::RMW_FADD;
748 break;
749 case AtomicRMWInst::FSub:
750 Encoding = bitc::RMW_FSUB;
751 break;
752 case AtomicRMWInst::FMax:
753 Encoding = bitc::RMW_FMAX;
754 break;
755 case AtomicRMWInst::FMin:
756 Encoding = bitc::RMW_FMIN;
757 break;
758 case AtomicRMWInst::FMaximum:
759 Encoding = bitc::RMW_FMAXIMUM;
760 break;
761 case AtomicRMWInst::FMinimum:
762 Encoding = bitc::RMW_FMINIMUM;
763 break;
764 case AtomicRMWInst::FMaximumNum:
765 Encoding = bitc::RMW_FMAXIMUMNUM;
766 break;
767 case AtomicRMWInst::FMinimumNum:
768 Encoding = bitc::RMW_FMINIMUMNUM;
769 break;
770 case AtomicRMWInst::UIncWrap:
771 Encoding = bitc::RMW_UINC_WRAP;
772 break;
773 case AtomicRMWInst::UDecWrap:
774 Encoding = bitc::RMW_UDEC_WRAP;
775 break;
776 case AtomicRMWInst::USubCond:
777 Encoding = bitc::RMW_USUB_COND;
778 break;
779 case AtomicRMWInst::USubSat:
780 Encoding = bitc::RMW_USUB_SAT;
781 break;
782 }
783
784 if (I.isElementwise())
785 Encoding |= bitc::RMW_ELEMENTWISE_FLAG;
786 return Encoding;
787}
788
789static unsigned getEncodedOrdering(AtomicOrdering Ordering) {
790 switch (Ordering) {
791 case AtomicOrdering::NotAtomic: return bitc::ORDERING_NOTATOMIC;
792 case AtomicOrdering::Unordered: return bitc::ORDERING_UNORDERED;
793 case AtomicOrdering::Monotonic: return bitc::ORDERING_MONOTONIC;
794 case AtomicOrdering::Acquire: return bitc::ORDERING_ACQUIRE;
795 case AtomicOrdering::Release: return bitc::ORDERING_RELEASE;
796 case AtomicOrdering::AcquireRelease: return bitc::ORDERING_ACQREL;
797 case AtomicOrdering::SequentiallyConsistent: return bitc::ORDERING_SEQCST;
798 }
799 llvm_unreachable("Invalid ordering");
800}
801
802static void writeStringRecord(BitstreamWriter &Stream, unsigned Code,
803 StringRef Str, unsigned AbbrevToUse) {
804 SmallVector<unsigned, 64> Vals;
805
806 // Code: [strchar x N]
807 for (char C : Str) {
808 if (AbbrevToUse && !BitCodeAbbrevOp::isChar6(C))
809 AbbrevToUse = 0;
810 Vals.push_back(Elt: C);
811 }
812
813 // Emit the finished record.
814 Stream.EmitRecord(Code, Vals, Abbrev: AbbrevToUse);
815}
816
817static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
818 switch (Kind) {
819 case Attribute::Alignment:
820 return bitc::ATTR_KIND_ALIGNMENT;
821 case Attribute::AllocAlign:
822 return bitc::ATTR_KIND_ALLOC_ALIGN;
823 case Attribute::AllocSize:
824 return bitc::ATTR_KIND_ALLOC_SIZE;
825 case Attribute::AlwaysInline:
826 return bitc::ATTR_KIND_ALWAYS_INLINE;
827 case Attribute::Builtin:
828 return bitc::ATTR_KIND_BUILTIN;
829 case Attribute::ByVal:
830 return bitc::ATTR_KIND_BY_VAL;
831 case Attribute::Convergent:
832 return bitc::ATTR_KIND_CONVERGENT;
833 case Attribute::InAlloca:
834 return bitc::ATTR_KIND_IN_ALLOCA;
835 case Attribute::Cold:
836 return bitc::ATTR_KIND_COLD;
837 case Attribute::DisableSanitizerInstrumentation:
838 return bitc::ATTR_KIND_DISABLE_SANITIZER_INSTRUMENTATION;
839 case Attribute::FnRetThunkExtern:
840 return bitc::ATTR_KIND_FNRETTHUNK_EXTERN;
841 case Attribute::Flatten:
842 return bitc::ATTR_KIND_FLATTEN;
843 case Attribute::Hot:
844 return bitc::ATTR_KIND_HOT;
845 case Attribute::ElementType:
846 return bitc::ATTR_KIND_ELEMENTTYPE;
847 case Attribute::HybridPatchable:
848 return bitc::ATTR_KIND_HYBRID_PATCHABLE;
849 case Attribute::InlineHint:
850 return bitc::ATTR_KIND_INLINE_HINT;
851 case Attribute::InReg:
852 return bitc::ATTR_KIND_IN_REG;
853 case Attribute::JumpTable:
854 return bitc::ATTR_KIND_JUMP_TABLE;
855 case Attribute::MinSize:
856 return bitc::ATTR_KIND_MIN_SIZE;
857 case Attribute::AllocatedPointer:
858 return bitc::ATTR_KIND_ALLOCATED_POINTER;
859 case Attribute::AllocKind:
860 return bitc::ATTR_KIND_ALLOC_KIND;
861 case Attribute::Memory:
862 return bitc::ATTR_KIND_MEMORY;
863 case Attribute::NoFPClass:
864 return bitc::ATTR_KIND_NOFPCLASS;
865 case Attribute::Naked:
866 return bitc::ATTR_KIND_NAKED;
867 case Attribute::Nest:
868 return bitc::ATTR_KIND_NEST;
869 case Attribute::NoAlias:
870 return bitc::ATTR_KIND_NO_ALIAS;
871 case Attribute::NoBuiltin:
872 return bitc::ATTR_KIND_NO_BUILTIN;
873 case Attribute::NoCallback:
874 return bitc::ATTR_KIND_NO_CALLBACK;
875 case Attribute::NoDivergenceSource:
876 return bitc::ATTR_KIND_NO_DIVERGENCE_SOURCE;
877 case Attribute::NoDuplicate:
878 return bitc::ATTR_KIND_NO_DUPLICATE;
879 case Attribute::NoFree:
880 return bitc::ATTR_KIND_NOFREE;
881 case Attribute::NoFreeObj:
882 return bitc::ATTR_KIND_NOFREEOBJ;
883 case Attribute::NoImplicitFloat:
884 return bitc::ATTR_KIND_NO_IMPLICIT_FLOAT;
885 case Attribute::NoInline:
886 return bitc::ATTR_KIND_NO_INLINE;
887 case Attribute::NoRecurse:
888 return bitc::ATTR_KIND_NO_RECURSE;
889 case Attribute::NoMerge:
890 return bitc::ATTR_KIND_NO_MERGE;
891 case Attribute::NonLazyBind:
892 return bitc::ATTR_KIND_NON_LAZY_BIND;
893 case Attribute::NonNull:
894 return bitc::ATTR_KIND_NON_NULL;
895 case Attribute::Dereferenceable:
896 return bitc::ATTR_KIND_DEREFERENCEABLE;
897 case Attribute::DereferenceableOrNull:
898 return bitc::ATTR_KIND_DEREFERENCEABLE_OR_NULL;
899 case Attribute::NoRedZone:
900 return bitc::ATTR_KIND_NO_RED_ZONE;
901 case Attribute::NoReturn:
902 return bitc::ATTR_KIND_NO_RETURN;
903 case Attribute::NoSync:
904 return bitc::ATTR_KIND_NOSYNC;
905 case Attribute::NoCfCheck:
906 return bitc::ATTR_KIND_NOCF_CHECK;
907 case Attribute::NoProfile:
908 return bitc::ATTR_KIND_NO_PROFILE;
909 case Attribute::SkipProfile:
910 return bitc::ATTR_KIND_SKIP_PROFILE;
911 case Attribute::NoUnwind:
912 return bitc::ATTR_KIND_NO_UNWIND;
913 case Attribute::NoSanitizeBounds:
914 return bitc::ATTR_KIND_NO_SANITIZE_BOUNDS;
915 case Attribute::NoSanitizeCoverage:
916 return bitc::ATTR_KIND_NO_SANITIZE_COVERAGE;
917 case Attribute::NullPointerIsValid:
918 return bitc::ATTR_KIND_NULL_POINTER_IS_VALID;
919 case Attribute::OptimizeForDebugging:
920 return bitc::ATTR_KIND_OPTIMIZE_FOR_DEBUGGING;
921 case Attribute::OptForFuzzing:
922 return bitc::ATTR_KIND_OPT_FOR_FUZZING;
923 case Attribute::OptimizeForSize:
924 return bitc::ATTR_KIND_OPTIMIZE_FOR_SIZE;
925 case Attribute::OptimizeNone:
926 return bitc::ATTR_KIND_OPTIMIZE_NONE;
927 case Attribute::ReadNone:
928 return bitc::ATTR_KIND_READ_NONE;
929 case Attribute::ReadOnly:
930 return bitc::ATTR_KIND_READ_ONLY;
931 case Attribute::Returned:
932 return bitc::ATTR_KIND_RETURNED;
933 case Attribute::ReturnsTwice:
934 return bitc::ATTR_KIND_RETURNS_TWICE;
935 case Attribute::SExt:
936 return bitc::ATTR_KIND_S_EXT;
937 case Attribute::Speculatable:
938 return bitc::ATTR_KIND_SPECULATABLE;
939 case Attribute::StackAlignment:
940 return bitc::ATTR_KIND_STACK_ALIGNMENT;
941 case Attribute::StackProtect:
942 return bitc::ATTR_KIND_STACK_PROTECT;
943 case Attribute::StackProtectReq:
944 return bitc::ATTR_KIND_STACK_PROTECT_REQ;
945 case Attribute::StackProtectStrong:
946 return bitc::ATTR_KIND_STACK_PROTECT_STRONG;
947 case Attribute::SafeStack:
948 return bitc::ATTR_KIND_SAFESTACK;
949 case Attribute::ShadowCallStack:
950 return bitc::ATTR_KIND_SHADOWCALLSTACK;
951 case Attribute::StrictFP:
952 return bitc::ATTR_KIND_STRICT_FP;
953 case Attribute::StructRet:
954 return bitc::ATTR_KIND_STRUCT_RET;
955 case Attribute::SanitizeAddress:
956 return bitc::ATTR_KIND_SANITIZE_ADDRESS;
957 case Attribute::SanitizeAllocToken:
958 return bitc::ATTR_KIND_SANITIZE_ALLOC_TOKEN;
959 case Attribute::SanitizeHWAddress:
960 return bitc::ATTR_KIND_SANITIZE_HWADDRESS;
961 case Attribute::SanitizeThread:
962 return bitc::ATTR_KIND_SANITIZE_THREAD;
963 case Attribute::SanitizeType:
964 return bitc::ATTR_KIND_SANITIZE_TYPE;
965 case Attribute::SanitizeMemory:
966 return bitc::ATTR_KIND_SANITIZE_MEMORY;
967 case Attribute::SanitizeNumericalStability:
968 return bitc::ATTR_KIND_SANITIZE_NUMERICAL_STABILITY;
969 case Attribute::SanitizeRealtime:
970 return bitc::ATTR_KIND_SANITIZE_REALTIME;
971 case Attribute::SanitizeRealtimeBlocking:
972 return bitc::ATTR_KIND_SANITIZE_REALTIME_BLOCKING;
973 case Attribute::SpeculativeLoadHardening:
974 return bitc::ATTR_KIND_SPECULATIVE_LOAD_HARDENING;
975 case Attribute::SwiftError:
976 return bitc::ATTR_KIND_SWIFT_ERROR;
977 case Attribute::SwiftSelf:
978 return bitc::ATTR_KIND_SWIFT_SELF;
979 case Attribute::SwiftAsync:
980 return bitc::ATTR_KIND_SWIFT_ASYNC;
981 case Attribute::UWTable:
982 return bitc::ATTR_KIND_UW_TABLE;
983 case Attribute::VScaleRange:
984 return bitc::ATTR_KIND_VSCALE_RANGE;
985 case Attribute::WillReturn:
986 return bitc::ATTR_KIND_WILLRETURN;
987 case Attribute::WriteOnly:
988 return bitc::ATTR_KIND_WRITEONLY;
989 case Attribute::ZExt:
990 return bitc::ATTR_KIND_Z_EXT;
991 case Attribute::ImmArg:
992 return bitc::ATTR_KIND_IMMARG;
993 case Attribute::SanitizeMemTag:
994 return bitc::ATTR_KIND_SANITIZE_MEMTAG;
995 case Attribute::Preallocated:
996 return bitc::ATTR_KIND_PREALLOCATED;
997 case Attribute::NoUndef:
998 return bitc::ATTR_KIND_NOUNDEF;
999 case Attribute::ByRef:
1000 return bitc::ATTR_KIND_BYREF;
1001 case Attribute::MustProgress:
1002 return bitc::ATTR_KIND_MUSTPROGRESS;
1003 case Attribute::PresplitCoroutine:
1004 return bitc::ATTR_KIND_PRESPLIT_COROUTINE;
1005 case Attribute::Writable:
1006 return bitc::ATTR_KIND_WRITABLE;
1007 case Attribute::CoroDestroyOnlyWhenComplete:
1008 return bitc::ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE;
1009 case Attribute::CoroElideSafe:
1010 return bitc::ATTR_KIND_CORO_ELIDE_SAFE;
1011 case Attribute::DeadOnUnwind:
1012 return bitc::ATTR_KIND_DEAD_ON_UNWIND;
1013 case Attribute::Range:
1014 return bitc::ATTR_KIND_RANGE;
1015 case Attribute::Initializes:
1016 return bitc::ATTR_KIND_INITIALIZES;
1017 case Attribute::NoExt:
1018 return bitc::ATTR_KIND_NO_EXT;
1019 case Attribute::Captures:
1020 return bitc::ATTR_KIND_CAPTURES;
1021 case Attribute::DeadOnReturn:
1022 return bitc::ATTR_KIND_DEAD_ON_RETURN;
1023 case Attribute::NoCreateUndefOrPoison:
1024 return bitc::ATTR_KIND_NO_CREATE_UNDEF_OR_POISON;
1025 case Attribute::DenormalFPEnv:
1026 return bitc::ATTR_KIND_DENORMAL_FPENV;
1027 case Attribute::NoOutline:
1028 return bitc::ATTR_KIND_NOOUTLINE;
1029 case Attribute::NoIPA:
1030 return bitc::ATTR_KIND_NOIPA;
1031 case Attribute::EndAttrKinds:
1032 llvm_unreachable("Can not encode end-attribute kinds marker.");
1033 case Attribute::None:
1034 llvm_unreachable("Can not encode none-attribute.");
1035 case Attribute::EmptyKey:
1036 case Attribute::TombstoneKey:
1037 llvm_unreachable("Trying to encode EmptyKey/TombstoneKey");
1038 }
1039
1040 llvm_unreachable("Trying to encode unknown attribute");
1041}
1042
1043static void emitSignedInt64(SmallVectorImpl<uint64_t> &Vals, uint64_t V) {
1044 if ((int64_t)V >= 0)
1045 Vals.push_back(Elt: V << 1);
1046 else
1047 Vals.push_back(Elt: (-V << 1) | 1);
1048}
1049
1050static void emitWideAPInt(SmallVectorImpl<uint64_t> &Vals, const APInt &A) {
1051 // We have an arbitrary precision integer value to write whose
1052 // bit width is > 64. However, in canonical unsigned integer
1053 // format it is likely that the high bits are going to be zero.
1054 // So, we only write the number of active words.
1055 unsigned NumWords = A.getActiveWords();
1056 const uint64_t *RawData = A.getRawData();
1057 for (unsigned i = 0; i < NumWords; i++)
1058 emitSignedInt64(Vals, V: RawData[i]);
1059}
1060
1061static void emitConstantRange(SmallVectorImpl<uint64_t> &Record,
1062 const ConstantRange &CR, bool EmitBitWidth) {
1063 unsigned BitWidth = CR.getBitWidth();
1064 if (EmitBitWidth)
1065 Record.push_back(Elt: BitWidth);
1066 if (BitWidth > 64) {
1067 Record.push_back(Elt: CR.getLower().getActiveWords() |
1068 (uint64_t(CR.getUpper().getActiveWords()) << 32));
1069 emitWideAPInt(Vals&: Record, A: CR.getLower());
1070 emitWideAPInt(Vals&: Record, A: CR.getUpper());
1071 } else {
1072 emitSignedInt64(Vals&: Record, V: CR.getLower().getSExtValue());
1073 emitSignedInt64(Vals&: Record, V: CR.getUpper().getSExtValue());
1074 }
1075}
1076
1077void ModuleBitcodeWriter::writeAttributeGroupTable() {
1078 const std::vector<ValueEnumerator::IndexAndAttrSet> &AttrGrps =
1079 VE.getAttributeGroups();
1080 if (AttrGrps.empty()) return;
1081
1082 Stream.EnterSubblock(BlockID: bitc::PARAMATTR_GROUP_BLOCK_ID, CodeLen: 3);
1083
1084 SmallVector<uint64_t, 64> Record;
1085 for (ValueEnumerator::IndexAndAttrSet Pair : AttrGrps) {
1086 unsigned AttrListIndex = Pair.first;
1087 AttributeSet AS = Pair.second;
1088 Record.push_back(Elt: VE.getAttributeGroupID(Group: Pair));
1089 Record.push_back(Elt: AttrListIndex);
1090
1091 for (Attribute Attr : AS) {
1092 if (Attr.isEnumAttribute()) {
1093 Record.push_back(Elt: 0);
1094 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1095 } else if (Attr.isIntAttribute()) {
1096 Record.push_back(Elt: 1);
1097 Attribute::AttrKind Kind = Attr.getKindAsEnum();
1098 Record.push_back(Elt: getAttrKindEncoding(Kind));
1099 if (Kind == Attribute::Memory) {
1100 // Version field for upgrading old memory effects.
1101 const uint64_t Version = 2;
1102 Record.push_back(Elt: (Version << 56) | Attr.getValueAsInt());
1103 } else {
1104 Record.push_back(Elt: Attr.getValueAsInt());
1105 }
1106 } else if (Attr.isStringAttribute()) {
1107 StringRef Kind = Attr.getKindAsString();
1108 StringRef Val = Attr.getValueAsString();
1109
1110 Record.push_back(Elt: Val.empty() ? 3 : 4);
1111 Record.append(in_start: Kind.begin(), in_end: Kind.end());
1112 Record.push_back(Elt: 0);
1113 if (!Val.empty()) {
1114 Record.append(in_start: Val.begin(), in_end: Val.end());
1115 Record.push_back(Elt: 0);
1116 }
1117 } else if (Attr.isTypeAttribute()) {
1118 Type *Ty = Attr.getValueAsType();
1119 Record.push_back(Elt: Ty ? 6 : 5);
1120 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1121 if (Ty)
1122 Record.push_back(Elt: VE.getTypeID(T: Attr.getValueAsType()));
1123 } else if (Attr.isConstantRangeAttribute()) {
1124 Record.push_back(Elt: 7);
1125 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1126 emitConstantRange(Record, CR: Attr.getValueAsConstantRange(),
1127 /*EmitBitWidth=*/true);
1128 } else {
1129 assert(Attr.isConstantRangeListAttribute());
1130 Record.push_back(Elt: 8);
1131 Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
1132 ArrayRef<ConstantRange> Val = Attr.getValueAsConstantRangeList();
1133 Record.push_back(Elt: Val.size());
1134 Record.push_back(Elt: Val[0].getBitWidth());
1135 for (auto &CR : Val)
1136 emitConstantRange(Record, CR, /*EmitBitWidth=*/false);
1137 }
1138 }
1139
1140 Stream.EmitRecord(Code: bitc::PARAMATTR_GRP_CODE_ENTRY, Vals: Record);
1141 Record.clear();
1142 }
1143
1144 Stream.ExitBlock();
1145}
1146
1147void ModuleBitcodeWriter::writeAttributeTable() {
1148 const std::vector<AttributeList> &Attrs = VE.getAttributeLists();
1149 if (Attrs.empty()) return;
1150
1151 Stream.EnterSubblock(BlockID: bitc::PARAMATTR_BLOCK_ID, CodeLen: 3);
1152
1153 SmallVector<uint64_t, 64> Record;
1154 for (const AttributeList &AL : Attrs) {
1155 for (unsigned i : AL.indexes()) {
1156 AttributeSet AS = AL.getAttributes(Index: i);
1157 if (AS.hasAttributes())
1158 Record.push_back(Elt: VE.getAttributeGroupID(Group: {i, AS}));
1159 }
1160
1161 Stream.EmitRecord(Code: bitc::PARAMATTR_CODE_ENTRY, Vals: Record);
1162 Record.clear();
1163 }
1164
1165 Stream.ExitBlock();
1166}
1167
1168/// WriteTypeTable - Write out the type table for a module.
1169void ModuleBitcodeWriter::writeTypeTable() {
1170 const ValueEnumerator::TypeList &TypeList = VE.getTypes();
1171
1172 Stream.EnterSubblock(BlockID: bitc::TYPE_BLOCK_ID_NEW, CodeLen: 4 /*count from # abbrevs */);
1173 SmallVector<uint64_t, 64> TypeVals;
1174
1175 uint64_t NumBits = VE.computeBitsRequiredForTypeIndices();
1176
1177 // Abbrev for TYPE_CODE_OPAQUE_POINTER.
1178 auto Abbv = std::make_shared<BitCodeAbbrev>();
1179 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_OPAQUE_POINTER));
1180 Abbv->Add(OpInfo: BitCodeAbbrevOp(0)); // Addrspace = 0
1181 unsigned OpaquePtrAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1182
1183 // Abbrev for TYPE_CODE_FUNCTION.
1184 Abbv = std::make_shared<BitCodeAbbrev>();
1185 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_FUNCTION));
1186 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isvararg
1187 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1188 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1189 unsigned FunctionAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1190
1191 // Abbrev for TYPE_CODE_STRUCT_ANON.
1192 Abbv = std::make_shared<BitCodeAbbrev>();
1193 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_ANON));
1194 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked
1195 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1196 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1197 unsigned StructAnonAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1198
1199 // Abbrev for TYPE_CODE_STRUCT_NAME.
1200 Abbv = std::make_shared<BitCodeAbbrev>();
1201 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAME));
1202 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1203 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
1204 unsigned StructNameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1205
1206 // Abbrev for TYPE_CODE_STRUCT_NAMED.
1207 Abbv = std::make_shared<BitCodeAbbrev>();
1208 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAMED));
1209 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked
1210 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1211 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1212 unsigned StructNamedAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1213
1214 // Abbrev for TYPE_CODE_ARRAY.
1215 Abbv = std::make_shared<BitCodeAbbrev>();
1216 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_ARRAY));
1217 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // size
1218 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1219 unsigned ArrayAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1220
1221 // Emit an entry count so the reader can reserve space.
1222 TypeVals.push_back(Elt: TypeList.size());
1223 Stream.EmitRecord(Code: bitc::TYPE_CODE_NUMENTRY, Vals: TypeVals);
1224 TypeVals.clear();
1225
1226 // Loop over all of the types, emitting each in turn.
1227 for (Type *T : TypeList) {
1228 int AbbrevToUse = 0;
1229 unsigned Code = 0;
1230
1231 switch (T->getTypeID()) {
1232 case Type::VoidTyID: Code = bitc::TYPE_CODE_VOID; break;
1233 case Type::HalfTyID: Code = bitc::TYPE_CODE_HALF; break;
1234 case Type::BFloatTyID: Code = bitc::TYPE_CODE_BFLOAT; break;
1235 case Type::FloatTyID: Code = bitc::TYPE_CODE_FLOAT; break;
1236 case Type::DoubleTyID: Code = bitc::TYPE_CODE_DOUBLE; break;
1237 case Type::X86_FP80TyID: Code = bitc::TYPE_CODE_X86_FP80; break;
1238 case Type::FP128TyID: Code = bitc::TYPE_CODE_FP128; break;
1239 case Type::PPC_FP128TyID: Code = bitc::TYPE_CODE_PPC_FP128; break;
1240 case Type::LabelTyID: Code = bitc::TYPE_CODE_LABEL; break;
1241 case Type::MetadataTyID:
1242 Code = bitc::TYPE_CODE_METADATA;
1243 break;
1244 case Type::X86_AMXTyID: Code = bitc::TYPE_CODE_X86_AMX; break;
1245 case Type::TokenTyID: Code = bitc::TYPE_CODE_TOKEN; break;
1246 case Type::ByteTyID:
1247 // BYTE: [width]
1248 Code = bitc::TYPE_CODE_BYTE;
1249 TypeVals.push_back(Elt: T->getByteBitWidth());
1250 break;
1251 case Type::IntegerTyID:
1252 // INTEGER: [width]
1253 Code = bitc::TYPE_CODE_INTEGER;
1254 TypeVals.push_back(Elt: cast<IntegerType>(Val: T)->getBitWidth());
1255 break;
1256 case Type::PointerTyID: {
1257 PointerType *PTy = cast<PointerType>(Val: T);
1258 unsigned AddressSpace = PTy->getAddressSpace();
1259 // OPAQUE_POINTER: [address space]
1260 Code = bitc::TYPE_CODE_OPAQUE_POINTER;
1261 TypeVals.push_back(Elt: AddressSpace);
1262 if (AddressSpace == 0)
1263 AbbrevToUse = OpaquePtrAbbrev;
1264 break;
1265 }
1266 case Type::FunctionTyID: {
1267 FunctionType *FT = cast<FunctionType>(Val: T);
1268 // FUNCTION: [isvararg, retty, paramty x N]
1269 Code = bitc::TYPE_CODE_FUNCTION;
1270 TypeVals.push_back(Elt: FT->isVarArg());
1271 TypeVals.push_back(Elt: VE.getTypeID(T: FT->getReturnType()));
1272 for (unsigned i = 0, e = FT->getNumParams(); i != e; ++i)
1273 TypeVals.push_back(Elt: VE.getTypeID(T: FT->getParamType(i)));
1274 AbbrevToUse = FunctionAbbrev;
1275 break;
1276 }
1277 case Type::StructTyID: {
1278 StructType *ST = cast<StructType>(Val: T);
1279 // STRUCT: [ispacked, eltty x N]
1280 TypeVals.push_back(Elt: ST->isPacked());
1281 // Output all of the element types.
1282 for (Type *ET : ST->elements())
1283 TypeVals.push_back(Elt: VE.getTypeID(T: ET));
1284
1285 if (ST->isLiteral()) {
1286 Code = bitc::TYPE_CODE_STRUCT_ANON;
1287 AbbrevToUse = StructAnonAbbrev;
1288 } else {
1289 if (ST->isOpaque()) {
1290 Code = bitc::TYPE_CODE_OPAQUE;
1291 } else {
1292 Code = bitc::TYPE_CODE_STRUCT_NAMED;
1293 AbbrevToUse = StructNamedAbbrev;
1294 }
1295
1296 // Emit the name if it is present.
1297 if (!ST->getName().empty())
1298 writeStringRecord(Stream, Code: bitc::TYPE_CODE_STRUCT_NAME, Str: ST->getName(),
1299 AbbrevToUse: StructNameAbbrev);
1300 }
1301 break;
1302 }
1303 case Type::ArrayTyID: {
1304 ArrayType *AT = cast<ArrayType>(Val: T);
1305 // ARRAY: [numelts, eltty]
1306 Code = bitc::TYPE_CODE_ARRAY;
1307 TypeVals.push_back(Elt: AT->getNumElements());
1308 TypeVals.push_back(Elt: VE.getTypeID(T: AT->getElementType()));
1309 AbbrevToUse = ArrayAbbrev;
1310 break;
1311 }
1312 case Type::FixedVectorTyID:
1313 case Type::ScalableVectorTyID: {
1314 VectorType *VT = cast<VectorType>(Val: T);
1315 // VECTOR [numelts, eltty] or
1316 // [numelts, eltty, scalable]
1317 Code = bitc::TYPE_CODE_VECTOR;
1318 TypeVals.push_back(Elt: VT->getElementCount().getKnownMinValue());
1319 TypeVals.push_back(Elt: VE.getTypeID(T: VT->getElementType()));
1320 if (isa<ScalableVectorType>(Val: VT))
1321 TypeVals.push_back(Elt: true);
1322 break;
1323 }
1324 case Type::TargetExtTyID: {
1325 TargetExtType *TET = cast<TargetExtType>(Val: T);
1326 Code = bitc::TYPE_CODE_TARGET_TYPE;
1327 writeStringRecord(Stream, Code: bitc::TYPE_CODE_STRUCT_NAME, Str: TET->getName(),
1328 AbbrevToUse: StructNameAbbrev);
1329 TypeVals.push_back(Elt: TET->getNumTypeParameters());
1330 for (Type *InnerTy : TET->type_params())
1331 TypeVals.push_back(Elt: VE.getTypeID(T: InnerTy));
1332 llvm::append_range(C&: TypeVals, R: TET->int_params());
1333 break;
1334 }
1335 case Type::TypedPointerTyID:
1336 llvm_unreachable("Typed pointers cannot be added to IR modules");
1337 }
1338
1339 // Emit the finished record.
1340 Stream.EmitRecord(Code, Vals: TypeVals, Abbrev: AbbrevToUse);
1341 TypeVals.clear();
1342 }
1343
1344 Stream.ExitBlock();
1345}
1346
1347static unsigned getEncodedLinkage(const GlobalValue::LinkageTypes Linkage) {
1348 switch (Linkage) {
1349 case GlobalValue::ExternalLinkage:
1350 return 0;
1351 case GlobalValue::WeakAnyLinkage:
1352 return 16;
1353 case GlobalValue::AppendingLinkage:
1354 return 2;
1355 case GlobalValue::InternalLinkage:
1356 return 3;
1357 case GlobalValue::LinkOnceAnyLinkage:
1358 return 18;
1359 case GlobalValue::ExternalWeakLinkage:
1360 return 7;
1361 case GlobalValue::CommonLinkage:
1362 return 8;
1363 case GlobalValue::PrivateLinkage:
1364 return 9;
1365 case GlobalValue::WeakODRLinkage:
1366 return 17;
1367 case GlobalValue::LinkOnceODRLinkage:
1368 return 19;
1369 case GlobalValue::AvailableExternallyLinkage:
1370 return 12;
1371 }
1372 llvm_unreachable("Invalid linkage");
1373}
1374
1375static unsigned getEncodedLinkage(const GlobalValue &GV) {
1376 return getEncodedLinkage(Linkage: GV.getLinkage());
1377}
1378
1379static uint64_t getEncodedFFlags(FunctionSummary::FFlags Flags) {
1380 uint64_t RawFlags = 0;
1381 RawFlags |= Flags.ReadNone;
1382 RawFlags |= (Flags.ReadOnly << 1);
1383 RawFlags |= (Flags.NoRecurse << 2);
1384 RawFlags |= (Flags.ReturnDoesNotAlias << 3);
1385 RawFlags |= (Flags.NoInline << 4);
1386 RawFlags |= (Flags.AlwaysInline << 5);
1387 RawFlags |= (Flags.NoUnwind << 6);
1388 RawFlags |= (Flags.MayThrow << 7);
1389 RawFlags |= (Flags.HasUnknownCall << 8);
1390 RawFlags |= (Flags.MustBeUnreachable << 9);
1391 return RawFlags;
1392}
1393
1394// Decode the flags for GlobalValue in the summary. See getDecodedGVSummaryFlags
1395// in BitcodeReader.cpp.
1396static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags,
1397 bool ImportAsDecl = false) {
1398 uint64_t RawFlags = 0;
1399
1400 RawFlags |= Flags.NotEligibleToImport; // bool
1401 RawFlags |= (Flags.Live << 1);
1402 RawFlags |= (Flags.DSOLocal << 2);
1403 RawFlags |= (Flags.CanAutoHide << 3);
1404
1405 // Linkage don't need to be remapped at that time for the summary. Any future
1406 // change to the getEncodedLinkage() function will need to be taken into
1407 // account here as well.
1408 RawFlags = (RawFlags << 4) | Flags.Linkage; // 4 bits
1409
1410 RawFlags |= (Flags.Visibility << 8); // 2 bits
1411
1412 unsigned ImportType = Flags.ImportType | ImportAsDecl;
1413 RawFlags |= (ImportType << 10); // 1 bit
1414
1415 RawFlags |= (Flags.NoRenameOnPromotion << 11); // 1 bit
1416
1417 return RawFlags;
1418}
1419
1420static uint64_t getEncodedGVarFlags(GlobalVarSummary::GVarFlags Flags) {
1421 uint64_t RawFlags = Flags.MaybeReadOnly | (Flags.MaybeWriteOnly << 1) |
1422 (Flags.Constant << 2) | Flags.VCallVisibility << 3;
1423 return RawFlags;
1424}
1425
1426static uint64_t getEncodedHotnessCallEdgeInfo(const CalleeInfo &CI) {
1427 uint64_t RawFlags = 0;
1428
1429 RawFlags |= CI.Hotness; // 3 bits
1430 RawFlags |= (CI.HasTailCall << 3); // 1 bit
1431
1432 return RawFlags;
1433}
1434
1435static unsigned getEncodedVisibility(const GlobalValue &GV) {
1436 switch (GV.getVisibility()) {
1437 case GlobalValue::DefaultVisibility: return 0;
1438 case GlobalValue::HiddenVisibility: return 1;
1439 case GlobalValue::ProtectedVisibility: return 2;
1440 }
1441 llvm_unreachable("Invalid visibility");
1442}
1443
1444static unsigned getEncodedDLLStorageClass(const GlobalValue &GV) {
1445 switch (GV.getDLLStorageClass()) {
1446 case GlobalValue::DefaultStorageClass: return 0;
1447 case GlobalValue::DLLImportStorageClass: return 1;
1448 case GlobalValue::DLLExportStorageClass: return 2;
1449 }
1450 llvm_unreachable("Invalid DLL storage class");
1451}
1452
1453static unsigned getEncodedThreadLocalMode(const GlobalValue &GV) {
1454 switch (GV.getThreadLocalMode()) {
1455 case GlobalVariable::NotThreadLocal: return 0;
1456 case GlobalVariable::GeneralDynamicTLSModel: return 1;
1457 case GlobalVariable::LocalDynamicTLSModel: return 2;
1458 case GlobalVariable::InitialExecTLSModel: return 3;
1459 case GlobalVariable::LocalExecTLSModel: return 4;
1460 }
1461 llvm_unreachable("Invalid TLS model");
1462}
1463
1464static unsigned getEncodedComdatSelectionKind(const Comdat &C) {
1465 switch (C.getSelectionKind()) {
1466 case Comdat::Any:
1467 return bitc::COMDAT_SELECTION_KIND_ANY;
1468 case Comdat::ExactMatch:
1469 return bitc::COMDAT_SELECTION_KIND_EXACT_MATCH;
1470 case Comdat::Largest:
1471 return bitc::COMDAT_SELECTION_KIND_LARGEST;
1472 case Comdat::NoDeduplicate:
1473 return bitc::COMDAT_SELECTION_KIND_NO_DUPLICATES;
1474 case Comdat::SameSize:
1475 return bitc::COMDAT_SELECTION_KIND_SAME_SIZE;
1476 }
1477 llvm_unreachable("Invalid selection kind");
1478}
1479
1480static unsigned getEncodedUnnamedAddr(const GlobalValue &GV) {
1481 switch (GV.getUnnamedAddr()) {
1482 case GlobalValue::UnnamedAddr::None: return 0;
1483 case GlobalValue::UnnamedAddr::Local: return 2;
1484 case GlobalValue::UnnamedAddr::Global: return 1;
1485 }
1486 llvm_unreachable("Invalid unnamed_addr");
1487}
1488
1489size_t ModuleBitcodeWriter::addToStrtab(StringRef Str) {
1490 if (GenerateHash)
1491 Hasher.update(Str);
1492 return StrtabBuilder.add(S: Str);
1493}
1494
1495void ModuleBitcodeWriter::writeComdats() {
1496 SmallVector<unsigned, 64> Vals;
1497 for (const Comdat *C : VE.getComdats()) {
1498 // COMDAT: [strtab offset, strtab size, selection_kind]
1499 Vals.push_back(Elt: addToStrtab(Str: C->getName()));
1500 Vals.push_back(Elt: C->getName().size());
1501 Vals.push_back(Elt: getEncodedComdatSelectionKind(C: *C));
1502 Stream.EmitRecord(Code: bitc::MODULE_CODE_COMDAT, Vals, /*AbbrevToUse=*/Abbrev: 0);
1503 Vals.clear();
1504 }
1505}
1506
1507/// Write a record that will eventually hold the word offset of the
1508/// module-level VST. For now the offset is 0, which will be backpatched
1509/// after the real VST is written. Saves the bit offset to backpatch.
1510void ModuleBitcodeWriter::writeValueSymbolTableForwardDecl() {
1511 // Write a placeholder value in for the offset of the real VST,
1512 // which is written after the function blocks so that it can include
1513 // the offset of each function. The placeholder offset will be
1514 // updated when the real VST is written.
1515 auto Abbv = std::make_shared<BitCodeAbbrev>();
1516 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_VSTOFFSET));
1517 // Blocks are 32-bit aligned, so we can use a 32-bit word offset to
1518 // hold the real VST offset. Must use fixed instead of VBR as we don't
1519 // know how many VBR chunks to reserve ahead of time.
1520 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
1521 unsigned VSTOffsetAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1522
1523 // Emit the placeholder
1524 uint64_t Vals[] = {bitc::MODULE_CODE_VSTOFFSET, 0};
1525 Stream.EmitRecordWithAbbrev(Abbrev: VSTOffsetAbbrev, Vals);
1526
1527 // Compute and save the bit offset to the placeholder, which will be
1528 // patched when the real VST is written. We can simply subtract the 32-bit
1529 // fixed size from the current bit number to get the location to backpatch.
1530 VSTOffsetPlaceholder = Stream.GetCurrentBitNo() - 32;
1531}
1532
1533enum StringEncoding { SE_Char6, SE_Fixed7, SE_Fixed8 };
1534
1535/// Determine the encoding to use for the given string name and length.
1536static StringEncoding getStringEncoding(StringRef Str) {
1537 bool isChar6 = true;
1538 for (char C : Str) {
1539 if (isChar6)
1540 isChar6 = BitCodeAbbrevOp::isChar6(C);
1541 if ((unsigned char)C & 128)
1542 // don't bother scanning the rest.
1543 return SE_Fixed8;
1544 }
1545 if (isChar6)
1546 return SE_Char6;
1547 return SE_Fixed7;
1548}
1549
1550static_assert(sizeof(GlobalValue::SanitizerMetadata) <= sizeof(unsigned),
1551 "Sanitizer Metadata is too large for naive serialization.");
1552static unsigned
1553serializeSanitizerMetadata(const GlobalValue::SanitizerMetadata &Meta) {
1554 return Meta.NoAddress | (Meta.NoHWAddress << 1) |
1555 (Meta.Memtag << 2) | (Meta.IsDynInit << 3);
1556}
1557
1558/// Emit top-level description of module, including target triple, inline asm,
1559/// descriptors for global variables, and function prototype info.
1560/// Returns the bit offset to backpatch with the location of the real VST.
1561void ModuleBitcodeWriter::writeModuleInfo() {
1562 // Emit various pieces of data attached to a module.
1563 if (!M.getTargetTriple().empty())
1564 writeStringRecord(Stream, Code: bitc::MODULE_CODE_TRIPLE,
1565 Str: M.getTargetTriple().str(), AbbrevToUse: 0 /*TODO*/);
1566 const std::string &DL = M.getDataLayoutStr();
1567 if (!DL.empty())
1568 writeStringRecord(Stream, Code: bitc::MODULE_CODE_DATALAYOUT, Str: DL, AbbrevToUse: 0 /*TODO*/);
1569
1570 for (const Module::GlobalAsmFragment &Frag : M.getModuleInlineAsm()) {
1571 SmallVector<std::pair<StringRef, StringRef>> Props =
1572 Frag.Props.getAsStrings();
1573 for (auto [Key, Value] : Props) {
1574 SmallVector<unsigned, 64> Record;
1575 Record.append(in_start: Key.begin(), in_end: Key.end());
1576 Record.push_back(Elt: 0);
1577 Record.append(in_start: Value.begin(), in_end: Value.end());
1578 Stream.EmitRecord(Code: bitc::MODULE_CODE_ASM_PROPERTY, Vals: Record);
1579 }
1580 writeStringRecord(Stream, Code: bitc::MODULE_CODE_ASM, Str: Frag.Asm, AbbrevToUse: 0 /*TODO*/);
1581 }
1582
1583 // Emit information about sections and GC, computing how many there are. Also
1584 // compute the maximum alignment value.
1585 std::map<std::string, unsigned> SectionMap;
1586 std::map<std::string, unsigned> GCMap;
1587 MaybeAlign MaxGVarAlignment;
1588 unsigned MaxGlobalType = 0;
1589 for (const GlobalVariable &GV : M.globals()) {
1590 if (MaybeAlign A = GV.getAlign())
1591 MaxGVarAlignment = !MaxGVarAlignment ? *A : std::max(a: *MaxGVarAlignment, b: *A);
1592 MaxGlobalType = std::max(a: MaxGlobalType, b: VE.getTypeID(T: GV.getValueType()));
1593 if (GV.hasSection()) {
1594 // Give section names unique ID's.
1595 unsigned &Entry = SectionMap[std::string(GV.getSection())];
1596 if (!Entry) {
1597 writeStringRecord(Stream, Code: bitc::MODULE_CODE_SECTIONNAME, Str: GV.getSection(),
1598 AbbrevToUse: 0 /*TODO*/);
1599 Entry = SectionMap.size();
1600 }
1601 }
1602 }
1603 for (const Function &F : M) {
1604 if (F.hasSection()) {
1605 // Give section names unique ID's.
1606 unsigned &Entry = SectionMap[std::string(F.getSection())];
1607 if (!Entry) {
1608 writeStringRecord(Stream, Code: bitc::MODULE_CODE_SECTIONNAME, Str: F.getSection(),
1609 AbbrevToUse: 0 /*TODO*/);
1610 Entry = SectionMap.size();
1611 }
1612 }
1613 if (F.hasGC()) {
1614 // Same for GC names.
1615 unsigned &Entry = GCMap[F.getGC()];
1616 if (!Entry) {
1617 writeStringRecord(Stream, Code: bitc::MODULE_CODE_GCNAME, Str: F.getGC(),
1618 AbbrevToUse: 0 /*TODO*/);
1619 Entry = GCMap.size();
1620 }
1621 }
1622 }
1623
1624 // Emit abbrev for globals, now that we know # sections and max alignment.
1625 unsigned SimpleGVarAbbrev = 0;
1626 if (!M.global_empty()) {
1627 // Add an abbrev for common globals with no visibility or thread localness.
1628 auto Abbv = std::make_shared<BitCodeAbbrev>();
1629 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_GLOBALVAR));
1630 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1631 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1632 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1633 Log2_32_Ceil(Value: MaxGlobalType+1)));
1634 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // AddrSpace << 2
1635 //| explicitType << 1
1636 //| constant
1637 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Initializer.
1638 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // Linkage.
1639 if (!MaxGVarAlignment) // Alignment.
1640 Abbv->Add(OpInfo: BitCodeAbbrevOp(0));
1641 else {
1642 unsigned MaxEncAlignment = getEncodedAlign(Alignment: MaxGVarAlignment);
1643 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1644 Log2_32_Ceil(Value: MaxEncAlignment+1)));
1645 }
1646 if (SectionMap.empty()) // Section.
1647 Abbv->Add(OpInfo: BitCodeAbbrevOp(0));
1648 else
1649 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1650 Log2_32_Ceil(Value: SectionMap.size()+1)));
1651 // Don't bother emitting vis + thread local.
1652 SimpleGVarAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1653 }
1654
1655 SmallVector<unsigned, 64> Vals;
1656 // Emit the module's source file name.
1657 {
1658 StringEncoding Bits = getStringEncoding(Str: M.getSourceFileName());
1659 BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
1660 if (Bits == SE_Char6)
1661 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
1662 else if (Bits == SE_Fixed7)
1663 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
1664
1665 // MODULE_CODE_SOURCE_FILENAME: [namechar x N]
1666 auto Abbv = std::make_shared<BitCodeAbbrev>();
1667 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
1668 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1669 Abbv->Add(OpInfo: AbbrevOpToUse);
1670 unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1671
1672 for (const auto P : M.getSourceFileName())
1673 Vals.push_back(Elt: (unsigned char)P);
1674
1675 // Emit the finished record.
1676 Stream.EmitRecord(Code: bitc::MODULE_CODE_SOURCE_FILENAME, Vals, Abbrev: FilenameAbbrev);
1677 Vals.clear();
1678 }
1679
1680 writeGUIDList();
1681
1682 // Emit the global variable information.
1683 for (const GlobalVariable &GV : M.globals()) {
1684 unsigned AbbrevToUse = 0;
1685
1686 // GLOBALVAR: [strtab offset, strtab size, type, isconst, initid,
1687 // linkage, alignment, section, visibility, threadlocal,
1688 // unnamed_addr, externally_initialized, dllstorageclass,
1689 // comdat, attributes, DSO_Local, GlobalSanitizer, code_model]
1690 Vals.push_back(Elt: addToStrtab(Str: GV.getName()));
1691 Vals.push_back(Elt: GV.getName().size());
1692 Vals.push_back(Elt: VE.getTypeID(T: GV.getValueType()));
1693 Vals.push_back(Elt: GV.getType()->getAddressSpace() << 2 | 2 | GV.isConstant());
1694 Vals.push_back(Elt: GV.isDeclaration() ? 0 :
1695 (VE.getValueID(V: GV.getInitializer()) + 1));
1696 Vals.push_back(Elt: getEncodedLinkage(GV));
1697 Vals.push_back(Elt: getEncodedAlign(Alignment: GV.getAlign()));
1698 Vals.push_back(Elt: GV.hasSection() ? SectionMap[std::string(GV.getSection())]
1699 : 0);
1700 if (GV.isThreadLocal() ||
1701 GV.getVisibility() != GlobalValue::DefaultVisibility ||
1702 GV.getUnnamedAddr() != GlobalValue::UnnamedAddr::None ||
1703 GV.isExternallyInitialized() ||
1704 GV.getDLLStorageClass() != GlobalValue::DefaultStorageClass ||
1705 GV.hasComdat() || GV.hasAttributes() || GV.isDSOLocal() ||
1706 GV.hasPartition() || GV.hasSanitizerMetadata() || GV.getCodeModel()) {
1707 Vals.push_back(Elt: getEncodedVisibility(GV));
1708 Vals.push_back(Elt: getEncodedThreadLocalMode(GV));
1709 Vals.push_back(Elt: getEncodedUnnamedAddr(GV));
1710 Vals.push_back(Elt: GV.isExternallyInitialized());
1711 Vals.push_back(Elt: getEncodedDLLStorageClass(GV));
1712 Vals.push_back(Elt: GV.hasComdat() ? VE.getComdatID(C: GV.getComdat()) : 0);
1713
1714 auto AL = GV.getAttributesAsList(index: AttributeList::FunctionIndex);
1715 Vals.push_back(Elt: VE.getAttributeListID(PAL: AL));
1716
1717 Vals.push_back(Elt: GV.isDSOLocal());
1718 Vals.push_back(Elt: addToStrtab(Str: GV.getPartition()));
1719 Vals.push_back(Elt: GV.getPartition().size());
1720
1721 Vals.push_back(Elt: (GV.hasSanitizerMetadata() ? serializeSanitizerMetadata(
1722 Meta: GV.getSanitizerMetadata())
1723 : 0));
1724 Vals.push_back(Elt: GV.getCodeModelRaw());
1725 } else {
1726 AbbrevToUse = SimpleGVarAbbrev;
1727 }
1728
1729 Stream.EmitRecord(Code: bitc::MODULE_CODE_GLOBALVAR, Vals, Abbrev: AbbrevToUse);
1730 Vals.clear();
1731 }
1732
1733 // Emit the function proto information.
1734 for (const Function &F : M) {
1735 // FUNCTION: [strtab offset, strtab size, type, callingconv, isproto,
1736 // linkage, paramattrs, alignment, section, visibility, gc,
1737 // unnamed_addr, prologuedata, dllstorageclass, comdat,
1738 // prefixdata, personalityfn, DSO_Local, addrspace,
1739 // partition_strtab, partition_size, prefalign]
1740 Vals.push_back(Elt: addToStrtab(Str: F.getName()));
1741 Vals.push_back(Elt: F.getName().size());
1742 Vals.push_back(Elt: VE.getTypeID(T: F.getFunctionType()));
1743 Vals.push_back(Elt: F.getCallingConv());
1744 Vals.push_back(Elt: F.isDeclaration());
1745 Vals.push_back(Elt: getEncodedLinkage(GV: F));
1746 Vals.push_back(Elt: VE.getAttributeListID(PAL: F.getAttributes()));
1747 Vals.push_back(Elt: getEncodedAlign(Alignment: F.getAlign()));
1748 Vals.push_back(Elt: F.hasSection() ? SectionMap[std::string(F.getSection())]
1749 : 0);
1750 Vals.push_back(Elt: getEncodedVisibility(GV: F));
1751 Vals.push_back(Elt: F.hasGC() ? GCMap[F.getGC()] : 0);
1752 Vals.push_back(Elt: getEncodedUnnamedAddr(GV: F));
1753 Vals.push_back(Elt: F.hasPrologueData() ? (VE.getValueID(V: F.getPrologueData()) + 1)
1754 : 0);
1755 Vals.push_back(Elt: getEncodedDLLStorageClass(GV: F));
1756 Vals.push_back(Elt: F.hasComdat() ? VE.getComdatID(C: F.getComdat()) : 0);
1757 Vals.push_back(Elt: F.hasPrefixData() ? (VE.getValueID(V: F.getPrefixData()) + 1)
1758 : 0);
1759 Vals.push_back(
1760 Elt: F.hasPersonalityFn() ? (VE.getValueID(V: F.getPersonalityFn()) + 1) : 0);
1761
1762 Vals.push_back(Elt: F.isDSOLocal());
1763 Vals.push_back(Elt: F.getAddressSpace());
1764 Vals.push_back(Elt: addToStrtab(Str: F.getPartition()));
1765 Vals.push_back(Elt: F.getPartition().size());
1766 Vals.push_back(Elt: getEncodedAlign(Alignment: F.getPreferredAlignment()));
1767
1768 unsigned AbbrevToUse = 0;
1769 Stream.EmitRecord(Code: bitc::MODULE_CODE_FUNCTION, Vals, Abbrev: AbbrevToUse);
1770 Vals.clear();
1771 }
1772
1773 // Emit the alias information.
1774 for (const GlobalAlias &A : M.aliases()) {
1775 // ALIAS: [strtab offset, strtab size, alias type, aliasee val#, linkage,
1776 // visibility, dllstorageclass, threadlocal, unnamed_addr,
1777 // DSO_Local]
1778 Vals.push_back(Elt: addToStrtab(Str: A.getName()));
1779 Vals.push_back(Elt: A.getName().size());
1780 Vals.push_back(Elt: VE.getTypeID(T: A.getValueType()));
1781 Vals.push_back(Elt: A.getType()->getAddressSpace());
1782 Vals.push_back(Elt: VE.getValueID(V: A.getAliasee()));
1783 Vals.push_back(Elt: getEncodedLinkage(GV: A));
1784 Vals.push_back(Elt: getEncodedVisibility(GV: A));
1785 Vals.push_back(Elt: getEncodedDLLStorageClass(GV: A));
1786 Vals.push_back(Elt: getEncodedThreadLocalMode(GV: A));
1787 Vals.push_back(Elt: getEncodedUnnamedAddr(GV: A));
1788 Vals.push_back(Elt: A.isDSOLocal());
1789 Vals.push_back(Elt: addToStrtab(Str: A.getPartition()));
1790 Vals.push_back(Elt: A.getPartition().size());
1791
1792 unsigned AbbrevToUse = 0;
1793 Stream.EmitRecord(Code: bitc::MODULE_CODE_ALIAS, Vals, Abbrev: AbbrevToUse);
1794 Vals.clear();
1795 }
1796
1797 // Emit the ifunc information.
1798 for (const GlobalIFunc &I : M.ifuncs()) {
1799 // IFUNC: [strtab offset, strtab size, ifunc type, address space, resolver
1800 // val#, linkage, visibility, DSO_Local]
1801 Vals.push_back(Elt: addToStrtab(Str: I.getName()));
1802 Vals.push_back(Elt: I.getName().size());
1803 Vals.push_back(Elt: VE.getTypeID(T: I.getValueType()));
1804 Vals.push_back(Elt: I.getType()->getAddressSpace());
1805 Vals.push_back(Elt: VE.getValueID(V: I.getResolver()));
1806 Vals.push_back(Elt: getEncodedLinkage(GV: I));
1807 Vals.push_back(Elt: getEncodedVisibility(GV: I));
1808 Vals.push_back(Elt: I.isDSOLocal());
1809 Vals.push_back(Elt: addToStrtab(Str: I.getPartition()));
1810 Vals.push_back(Elt: I.getPartition().size());
1811 Stream.EmitRecord(Code: bitc::MODULE_CODE_IFUNC, Vals);
1812 Vals.clear();
1813 }
1814
1815 writeValueSymbolTableForwardDecl();
1816}
1817
1818static uint64_t getOptimizationFlags(const Value *V) {
1819 uint64_t Flags = 0;
1820
1821 if (const auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V)) {
1822 if (OBO->hasNoSignedWrap())
1823 Flags |= 1 << bitc::OBO_NO_SIGNED_WRAP;
1824 if (OBO->hasNoUnsignedWrap())
1825 Flags |= 1 << bitc::OBO_NO_UNSIGNED_WRAP;
1826 } else if (const auto *PEO = dyn_cast<PossiblyExactOperator>(Val: V)) {
1827 if (PEO->isExact())
1828 Flags |= 1 << bitc::PEO_EXACT;
1829 } else if (const auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: V)) {
1830 if (PDI->isDisjoint())
1831 Flags |= 1 << bitc::PDI_DISJOINT;
1832 } else if (const auto *FPMO = dyn_cast<FPMathOperator>(Val: V)) {
1833 if (FPMO->hasAllowReassoc())
1834 Flags |= bitc::AllowReassoc;
1835 if (FPMO->hasNoNaNs())
1836 Flags |= bitc::NoNaNs;
1837 if (FPMO->hasNoInfs())
1838 Flags |= bitc::NoInfs;
1839 if (FPMO->hasNoSignedZeros())
1840 Flags |= bitc::NoSignedZeros;
1841 if (FPMO->hasAllowReciprocal())
1842 Flags |= bitc::AllowReciprocal;
1843 if (FPMO->hasAllowContract())
1844 Flags |= bitc::AllowContract;
1845 if (FPMO->hasApproxFunc())
1846 Flags |= bitc::ApproxFunc;
1847
1848 // Handle uitofp.
1849 if (const auto *NNI = dyn_cast<PossiblyNonNegInst>(Val: V)) {
1850 Flags <<= 1;
1851 if (NNI->hasNonNeg())
1852 Flags |= 1 << bitc::PNNI_NON_NEG;
1853 }
1854 } else if (const auto *NNI = dyn_cast<PossiblyNonNegInst>(Val: V)) {
1855 if (NNI->hasNonNeg())
1856 Flags |= 1 << bitc::PNNI_NON_NEG;
1857 } else if (const auto *TI = dyn_cast<TruncInst>(Val: V)) {
1858 if (TI->hasNoSignedWrap())
1859 Flags |= 1 << bitc::TIO_NO_SIGNED_WRAP;
1860 if (TI->hasNoUnsignedWrap())
1861 Flags |= 1 << bitc::TIO_NO_UNSIGNED_WRAP;
1862 } else if (const auto *GEP = dyn_cast<GEPOperator>(Val: V)) {
1863 if (GEP->isInBounds())
1864 Flags |= 1 << bitc::GEP_INBOUNDS;
1865 if (GEP->hasNoUnsignedSignedWrap())
1866 Flags |= 1 << bitc::GEP_NUSW;
1867 if (GEP->hasNoUnsignedWrap())
1868 Flags |= 1 << bitc::GEP_NUW;
1869 } else if (const auto *ICmp = dyn_cast<ICmpInst>(Val: V)) {
1870 if (ICmp->hasSameSign())
1871 Flags |= 1 << bitc::ICMP_SAME_SIGN;
1872 } else if (const auto *ASC = dyn_cast<AddrSpaceCastInst>(Val: V)) {
1873 if (ASC->hasNonNull())
1874 Flags |= 1 << bitc::ASCI_NON_NULL;
1875 }
1876
1877 return Flags;
1878}
1879
1880void ModuleBitcodeWriter::writeValueAsMetadata(
1881 const ValueAsMetadata *MD, SmallVectorImpl<uint64_t> &Record) {
1882 // Mimic an MDNode with a value as one operand.
1883 Value *V = MD->getValue();
1884 Record.push_back(Elt: VE.getTypeID(T: V->getType()));
1885 Record.push_back(Elt: VE.getValueID(V));
1886 Stream.EmitRecord(Code: bitc::METADATA_VALUE, Vals: Record, Abbrev: 0);
1887 Record.clear();
1888}
1889
1890void ModuleBitcodeWriter::writeMDTuple(const MDTuple *N,
1891 SmallVectorImpl<uint64_t> &Record,
1892 unsigned Abbrev) {
1893 for (const MDOperand &MDO : N->operands()) {
1894 Metadata *MD = MDO;
1895 assert(!(MD && isa<LocalAsMetadata>(MD)) &&
1896 "Unexpected function-local metadata");
1897 Record.push_back(Elt: VE.getMetadataOrNullID(MD));
1898 }
1899 Stream.EmitRecord(Code: N->isDistinct() ? bitc::METADATA_DISTINCT_NODE
1900 : bitc::METADATA_NODE,
1901 Vals: Record, Abbrev);
1902 Record.clear();
1903}
1904
1905unsigned ModuleBitcodeWriter::createDILocationAbbrev(bool WithIRLayers) {
1906 // Assume the column is usually under 128, and always output the inlined-at
1907 // location (it's never more expensive than building an array size 1).
1908 //
1909 // Separate abbrev so a location without layers does not spend a VBR chunk
1910 // encoding a zero; writeDILocation picks between the two per record.
1911 auto Abbv = std::make_shared<BitCodeAbbrev>();
1912 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_LOCATION));
1913 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isDistinct
1914 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // line
1915 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // column
1916 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // scope
1917 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // inlinedAt
1918 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isImplicitCode
1919 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // atomGroup
1920 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // atomRank
1921 if (WithIRLayers)
1922 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // irlayers
1923 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
1924}
1925
1926void ModuleBitcodeWriter::writeDILocation(const DILocation *N,
1927 SmallVectorImpl<uint64_t> &Record,
1928 unsigned &Abbrev) {
1929 if (!Abbrev)
1930 Abbrev = createDILocationAbbrev(/*WithIRLayers=*/false);
1931
1932 Record.push_back(Elt: N->isDistinct());
1933 Record.push_back(Elt: N->getLine());
1934 Record.push_back(Elt: N->getColumn());
1935 Record.push_back(Elt: VE.getMetadataID(MD: N->getScope()));
1936 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getInlinedAt()));
1937 Record.push_back(Elt: N->isImplicitCode());
1938 Record.push_back(Elt: N->getAtomGroup());
1939 Record.push_back(Elt: N->getAtomRank());
1940
1941 unsigned AbbrevToUse = Abbrev;
1942 if (DILayerLocList *IRLayers = N->getIRLayers()) {
1943 if (!DILocationLayersAbbrev)
1944 DILocationLayersAbbrev = createDILocationAbbrev(/*WithIRLayers=*/true);
1945 AbbrevToUse = DILocationLayersAbbrev;
1946 Record.push_back(Elt: VE.getMetadataOrNullID(MD: IRLayers));
1947 }
1948
1949 Stream.EmitRecord(Code: bitc::METADATA_LOCATION, Vals: Record, Abbrev: AbbrevToUse);
1950 Record.clear();
1951}
1952
1953void ModuleBitcodeWriter::writeDILayerLoc(const DILayerLoc *N,
1954 SmallVectorImpl<uint64_t> &Record,
1955 unsigned Abbrev) {
1956 Record.push_back(Elt: N->isDistinct());
1957 Record.push_back(Elt: N->getLine());
1958 Record.push_back(Elt: N->getColumn());
1959 Record.push_back(Elt: VE.getMetadataID(MD: N->getRawFile()));
1960 Record.push_back(Elt: VE.getMetadataID(MD: N->getRawKind()));
1961 Stream.EmitRecord(Code: bitc::METADATA_LAYERLOC, Vals: Record, Abbrev);
1962 Record.clear();
1963}
1964
1965void ModuleBitcodeWriter::writeDILayerLocList(const DILayerLocList *N,
1966 SmallVectorImpl<uint64_t> &Record,
1967 unsigned Abbrev) {
1968 Record.push_back(Elt: N->isDistinct());
1969 for (const MDOperand &Op : N->layers())
1970 Record.push_back(Elt: VE.getMetadataID(MD: Op.get()));
1971 Stream.EmitRecord(Code: bitc::METADATA_LAYERLOCLIST, Vals: Record, Abbrev);
1972 Record.clear();
1973}
1974
1975unsigned ModuleBitcodeWriter::createGenericDINodeAbbrev() {
1976 // Assume the column is usually under 128, and always output the inlined-at
1977 // location (it's never more expensive than building an array size 1).
1978 auto Abbv = std::make_shared<BitCodeAbbrev>();
1979 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_GENERIC_DEBUG));
1980 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1981 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1982 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1983 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1984 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1985 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1986 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
1987}
1988
1989void ModuleBitcodeWriter::writeGenericDINode(const GenericDINode *N,
1990 SmallVectorImpl<uint64_t> &Record,
1991 unsigned &Abbrev) {
1992 if (!Abbrev)
1993 Abbrev = createGenericDINodeAbbrev();
1994
1995 Record.push_back(Elt: N->isDistinct());
1996 Record.push_back(Elt: N->getTag());
1997 Record.push_back(Elt: 0); // Per-tag version field; unused for now.
1998
1999 for (auto &I : N->operands())
2000 Record.push_back(Elt: VE.getMetadataOrNullID(MD: I));
2001
2002 Stream.EmitRecord(Code: bitc::METADATA_GENERIC_DEBUG, Vals: Record, Abbrev);
2003 Record.clear();
2004}
2005
2006void ModuleBitcodeWriter::writeDISubrange(const DISubrange *N,
2007 SmallVectorImpl<uint64_t> &Record,
2008 unsigned Abbrev) {
2009 const uint64_t Version = 2 << 1;
2010 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
2011 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawCountNode()));
2012 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
2013 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
2014 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
2015
2016 Stream.EmitRecord(Code: bitc::METADATA_SUBRANGE, Vals: Record, Abbrev);
2017 Record.clear();
2018}
2019
2020void ModuleBitcodeWriter::writeDIGenericSubrange(
2021 const DIGenericSubrange *N, SmallVectorImpl<uint64_t> &Record,
2022 unsigned Abbrev) {
2023 Record.push_back(Elt: (uint64_t)N->isDistinct());
2024 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawCountNode()));
2025 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
2026 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
2027 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
2028
2029 Stream.EmitRecord(Code: bitc::METADATA_GENERIC_SUBRANGE, Vals: Record, Abbrev);
2030 Record.clear();
2031}
2032
2033void ModuleBitcodeWriter::writeDIEnumerator(const DIEnumerator *N,
2034 SmallVectorImpl<uint64_t> &Record,
2035 unsigned Abbrev) {
2036 const uint64_t IsBigInt = 1 << 2;
2037 Record.push_back(Elt: IsBigInt | (N->isUnsigned() << 1) | N->isDistinct());
2038 Record.push_back(Elt: N->getValue().getBitWidth());
2039 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2040 emitWideAPInt(Vals&: Record, A: N->getValue());
2041
2042 Stream.EmitRecord(Code: bitc::METADATA_ENUMERATOR, Vals: Record, Abbrev);
2043 Record.clear();
2044}
2045
2046void ModuleBitcodeWriter::writeDIBasicType(const DIBasicType *N,
2047 SmallVectorImpl<uint64_t> &Record,
2048 unsigned Abbrev) {
2049 const unsigned SizeIsMetadata = 0x2;
2050 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2051 Record.push_back(Elt: N->getTag());
2052 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2053 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2054 Record.push_back(Elt: N->getAlignInBits());
2055 Record.push_back(Elt: N->getEncoding());
2056 Record.push_back(Elt: N->getFlags());
2057 Record.push_back(Elt: N->getNumExtraInhabitants());
2058 Record.push_back(Elt: N->getDataSizeInBits());
2059 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2060 Record.push_back(Elt: N->getLine());
2061 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2062
2063 Stream.EmitRecord(Code: bitc::METADATA_BASIC_TYPE, Vals: Record, Abbrev);
2064 Record.clear();
2065}
2066
2067void ModuleBitcodeWriter::writeDIFixedPointType(
2068 const DIFixedPointType *N, SmallVectorImpl<uint64_t> &Record,
2069 unsigned Abbrev) {
2070 const unsigned SizeIsMetadata = 0x2;
2071 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2072 Record.push_back(Elt: N->getTag());
2073 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2074 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2075 Record.push_back(Elt: N->getAlignInBits());
2076 Record.push_back(Elt: N->getEncoding());
2077 Record.push_back(Elt: N->getFlags());
2078 Record.push_back(Elt: N->getKind());
2079 Record.push_back(Elt: N->getFactorRaw());
2080
2081 auto WriteWideInt = [&](const APInt &Value) {
2082 // Write an encoded word that holds the number of active words and
2083 // the number of bits.
2084 uint64_t NumWords = Value.getActiveWords();
2085 uint64_t Encoded = (NumWords << 32) | Value.getBitWidth();
2086 Record.push_back(Elt: Encoded);
2087 emitWideAPInt(Vals&: Record, A: Value);
2088 };
2089
2090 WriteWideInt(N->getNumeratorRaw());
2091 WriteWideInt(N->getDenominatorRaw());
2092
2093 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2094 Record.push_back(Elt: N->getLine());
2095 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2096
2097 Stream.EmitRecord(Code: bitc::METADATA_FIXED_POINT_TYPE, Vals: Record, Abbrev);
2098 Record.clear();
2099}
2100
2101void ModuleBitcodeWriter::writeDIStringType(const DIStringType *N,
2102 SmallVectorImpl<uint64_t> &Record,
2103 unsigned Abbrev) {
2104 const unsigned SizeIsMetadata = 0x2;
2105 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2106 Record.push_back(Elt: N->getTag());
2107 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2108 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLength()));
2109 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLengthExp()));
2110 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLocationExp()));
2111 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2112 Record.push_back(Elt: N->getAlignInBits());
2113 Record.push_back(Elt: N->getEncoding());
2114 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawCharType()));
2115
2116 Stream.EmitRecord(Code: bitc::METADATA_STRING_TYPE, Vals: Record, Abbrev);
2117 Record.clear();
2118}
2119
2120void ModuleBitcodeWriter::writeDIDerivedType(const DIDerivedType *N,
2121 SmallVectorImpl<uint64_t> &Record,
2122 unsigned Abbrev) {
2123 const unsigned SizeIsMetadata = 0x2;
2124 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2125 Record.push_back(Elt: N->getTag());
2126 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2127 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2128 Record.push_back(Elt: N->getLine());
2129 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2130 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2131 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2132 Record.push_back(Elt: N->getAlignInBits());
2133 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawOffsetInBits()));
2134 Record.push_back(Elt: N->getFlags());
2135 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getExtraData()));
2136
2137 // DWARF address space is encoded as N->getDWARFAddressSpace() + 1. 0 means
2138 // that there is no DWARF address space associated with DIDerivedType.
2139 if (const auto &DWARFAddressSpace = N->getDWARFAddressSpace())
2140 Record.push_back(Elt: *DWARFAddressSpace + 1);
2141 else
2142 Record.push_back(Elt: 0);
2143
2144 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2145
2146 if (auto PtrAuthData = N->getPtrAuthData())
2147 Record.push_back(Elt: PtrAuthData->RawData);
2148 else
2149 Record.push_back(Elt: 0);
2150
2151 Stream.EmitRecord(Code: bitc::METADATA_DERIVED_TYPE, Vals: Record, Abbrev);
2152 Record.clear();
2153}
2154
2155void ModuleBitcodeWriter::writeDISubrangeType(const DISubrangeType *N,
2156 SmallVectorImpl<uint64_t> &Record,
2157 unsigned Abbrev) {
2158 const unsigned SizeIsMetadata = 0x2;
2159 Record.push_back(Elt: SizeIsMetadata | (unsigned)N->isDistinct());
2160 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2161 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2162 Record.push_back(Elt: N->getLine());
2163 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2164 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2165 Record.push_back(Elt: N->getAlignInBits());
2166 Record.push_back(Elt: N->getFlags());
2167 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2168 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
2169 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
2170 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
2171 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawBias()));
2172
2173 Stream.EmitRecord(Code: bitc::METADATA_SUBRANGE_TYPE, Vals: Record, Abbrev);
2174 Record.clear();
2175}
2176
2177void ModuleBitcodeWriter::writeDICompositeType(
2178 const DICompositeType *N, SmallVectorImpl<uint64_t> &Record,
2179 unsigned Abbrev) {
2180 const unsigned IsNotUsedInOldTypeRef = 0x2;
2181 const unsigned SizeIsMetadata = 0x4;
2182 Record.push_back(Elt: SizeIsMetadata | IsNotUsedInOldTypeRef |
2183 (unsigned)N->isDistinct());
2184 Record.push_back(Elt: N->getTag());
2185 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2186 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2187 Record.push_back(Elt: N->getLine());
2188 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2189 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
2190 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSizeInBits()));
2191 Record.push_back(Elt: N->getAlignInBits());
2192 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawOffsetInBits()));
2193 Record.push_back(Elt: N->getFlags());
2194 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2195 Record.push_back(Elt: N->getRuntimeLang());
2196 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getVTableHolder()));
2197 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams().get()));
2198 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawIdentifier()));
2199 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDiscriminator()));
2200 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawDataLocation()));
2201 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawAssociated()));
2202 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawAllocated()));
2203 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawRank()));
2204 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2205 Record.push_back(Elt: N->getNumExtraInhabitants());
2206 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSpecification()));
2207 Record.push_back(
2208 Elt: N->getEnumKind().value_or(u: dwarf::DW_APPLE_ENUM_KIND_invalid));
2209 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawBitStride()));
2210
2211 Stream.EmitRecord(Code: bitc::METADATA_COMPOSITE_TYPE, Vals: Record, Abbrev);
2212 Record.clear();
2213}
2214
2215void ModuleBitcodeWriter::writeDISubroutineType(
2216 const DISubroutineType *N, SmallVectorImpl<uint64_t> &Record,
2217 unsigned Abbrev) {
2218 const unsigned HasNoOldTypeRefs = 0x2;
2219 Record.push_back(Elt: HasNoOldTypeRefs | (unsigned)N->isDistinct());
2220 Record.push_back(Elt: N->getFlags());
2221 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTypeArray().get()));
2222 Record.push_back(Elt: N->getCC());
2223
2224 Stream.EmitRecord(Code: bitc::METADATA_SUBROUTINE_TYPE, Vals: Record, Abbrev);
2225 Record.clear();
2226}
2227
2228void ModuleBitcodeWriter::writeDIFile(const DIFile *N,
2229 SmallVectorImpl<uint64_t> &Record,
2230 unsigned Abbrev) {
2231 Record.push_back(Elt: N->isDistinct());
2232 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFilename()));
2233 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawDirectory()));
2234 if (N->getRawChecksum()) {
2235 Record.push_back(Elt: N->getRawChecksum()->Kind);
2236 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawChecksum()->Value));
2237 } else {
2238 // Maintain backwards compatibility with the old internal representation of
2239 // CSK_None in ChecksumKind by writing nulls here when Checksum is None.
2240 Record.push_back(Elt: 0);
2241 Record.push_back(Elt: VE.getMetadataOrNullID(MD: nullptr));
2242 }
2243 auto Source = N->getRawSource();
2244 if (Source)
2245 Record.push_back(Elt: VE.getMetadataOrNullID(MD: Source));
2246
2247 Stream.EmitRecord(Code: bitc::METADATA_FILE, Vals: Record, Abbrev);
2248 Record.clear();
2249}
2250
2251void ModuleBitcodeWriter::writeDICompileUnit(const DICompileUnit *N,
2252 SmallVectorImpl<uint64_t> &Record,
2253 unsigned Abbrev) {
2254 assert(N->isDistinct() && "Expected distinct compile units");
2255 Record.push_back(/* IsDistinct */ Elt: true);
2256
2257 auto Lang = N->getSourceLanguage();
2258 Record.push_back(Elt: Lang.getName());
2259 // Set bit so the MetadataLoader can distniguish between versioned and
2260 // unversioned names.
2261 if (Lang.hasVersionedName())
2262 Record.back() ^= (uint64_t(1) << 63);
2263
2264 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2265 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawProducer()));
2266 Record.push_back(Elt: N->isOptimized());
2267 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFlags()));
2268 Record.push_back(Elt: N->getRuntimeVersion());
2269 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSplitDebugFilename()));
2270 Record.push_back(Elt: N->getEmissionKind());
2271 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getEnumTypes().get()));
2272 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRetainedTypes().get()));
2273 Record.push_back(/* subprograms */ Elt: 0);
2274 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getGlobalVariables().get()));
2275 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getImportedEntities().get()));
2276 Record.push_back(Elt: N->getDWOId());
2277 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getMacros().get()));
2278 Record.push_back(Elt: N->getSplitDebugInlining());
2279 Record.push_back(Elt: N->getDebugInfoForProfiling());
2280 Record.push_back(Elt: (unsigned)N->getNameTableKind());
2281 Record.push_back(Elt: N->getRangesBaseAddress());
2282 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSysRoot()));
2283 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSDK()));
2284 Record.push_back(Elt: Lang.hasVersionedName() ? Lang.getVersion() : 0);
2285 Record.push_back(Elt: Lang.getDialect());
2286
2287 Stream.EmitRecord(Code: bitc::METADATA_COMPILE_UNIT, Vals: Record, Abbrev);
2288 Record.clear();
2289}
2290
2291void ModuleBitcodeWriter::writeDISubprogram(const DISubprogram *N,
2292 SmallVectorImpl<uint64_t> &Record,
2293 unsigned Abbrev) {
2294 const uint64_t HasUnitFlag = 1 << 1;
2295 const uint64_t HasSPFlagsFlag = 1 << 2;
2296 Record.push_back(Elt: uint64_t(N->isDistinct()) | HasUnitFlag | HasSPFlagsFlag);
2297 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2298 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2299 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLinkageName()));
2300 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2301 Record.push_back(Elt: N->getLine());
2302 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2303 Record.push_back(Elt: N->getScopeLine());
2304 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getContainingType()));
2305 Record.push_back(Elt: N->getSPFlags());
2306 Record.push_back(Elt: N->getVirtualIndex());
2307 Record.push_back(Elt: N->getFlags());
2308 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUnit()));
2309 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams().get()));
2310 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDeclaration()));
2311 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRetainedNodes().get()));
2312 Record.push_back(Elt: N->getThisAdjustment());
2313 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getThrownTypes().get()));
2314 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2315 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawTargetFuncName()));
2316 Record.push_back(Elt: N->getKeyInstructionsEnabled());
2317
2318 Stream.EmitRecord(Code: bitc::METADATA_SUBPROGRAM, Vals: Record, Abbrev);
2319 Record.clear();
2320}
2321
2322void ModuleBitcodeWriter::writeDILexicalBlock(const DILexicalBlock *N,
2323 SmallVectorImpl<uint64_t> &Record,
2324 unsigned Abbrev) {
2325 Record.push_back(Elt: N->isDistinct());
2326 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2327 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2328 Record.push_back(Elt: N->getLine());
2329 Record.push_back(Elt: N->getColumn());
2330
2331 Stream.EmitRecord(Code: bitc::METADATA_LEXICAL_BLOCK, Vals: Record, Abbrev);
2332 Record.clear();
2333}
2334
2335void ModuleBitcodeWriter::writeDILexicalBlockFile(
2336 const DILexicalBlockFile *N, SmallVectorImpl<uint64_t> &Record,
2337 unsigned Abbrev) {
2338 Record.push_back(Elt: N->isDistinct());
2339 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2340 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2341 Record.push_back(Elt: N->getDiscriminator());
2342
2343 Stream.EmitRecord(Code: bitc::METADATA_LEXICAL_BLOCK_FILE, Vals: Record, Abbrev);
2344 Record.clear();
2345}
2346
2347void ModuleBitcodeWriter::writeDICommonBlock(const DICommonBlock *N,
2348 SmallVectorImpl<uint64_t> &Record,
2349 unsigned Abbrev) {
2350 Record.push_back(Elt: N->isDistinct());
2351 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2352 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDecl()));
2353 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2354 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2355 Record.push_back(Elt: N->getLineNo());
2356
2357 Stream.EmitRecord(Code: bitc::METADATA_COMMON_BLOCK, Vals: Record, Abbrev);
2358 Record.clear();
2359}
2360
2361void ModuleBitcodeWriter::writeDINamespace(const DINamespace *N,
2362 SmallVectorImpl<uint64_t> &Record,
2363 unsigned Abbrev) {
2364 Record.push_back(Elt: N->isDistinct() | N->getExportSymbols() << 1);
2365 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2366 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2367
2368 Stream.EmitRecord(Code: bitc::METADATA_NAMESPACE, Vals: Record, Abbrev);
2369 Record.clear();
2370}
2371
2372void ModuleBitcodeWriter::writeDIMacro(const DIMacro *N,
2373 SmallVectorImpl<uint64_t> &Record,
2374 unsigned Abbrev) {
2375 Record.push_back(Elt: N->isDistinct());
2376 Record.push_back(Elt: N->getMacinfoType());
2377 Record.push_back(Elt: N->getLine());
2378 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2379 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawValue()));
2380
2381 Stream.EmitRecord(Code: bitc::METADATA_MACRO, Vals: Record, Abbrev);
2382 Record.clear();
2383}
2384
2385void ModuleBitcodeWriter::writeDIMacroFile(const DIMacroFile *N,
2386 SmallVectorImpl<uint64_t> &Record,
2387 unsigned Abbrev) {
2388 Record.push_back(Elt: N->isDistinct());
2389 Record.push_back(Elt: N->getMacinfoType());
2390 Record.push_back(Elt: N->getLine());
2391 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2392 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2393
2394 Stream.EmitRecord(Code: bitc::METADATA_MACRO_FILE, Vals: Record, Abbrev);
2395 Record.clear();
2396}
2397
2398void ModuleBitcodeWriter::writeDIArgList(const DIArgList *N,
2399 SmallVectorImpl<uint64_t> &Record) {
2400 Record.reserve(N: N->getArgs().size());
2401 for (ValueAsMetadata *MD : N->getArgs())
2402 Record.push_back(Elt: VE.getMetadataID(MD));
2403
2404 Stream.EmitRecord(Code: bitc::METADATA_ARG_LIST, Vals: Record);
2405 Record.clear();
2406}
2407
2408void ModuleBitcodeWriter::writeDIModule(const DIModule *N,
2409 SmallVectorImpl<uint64_t> &Record,
2410 unsigned Abbrev) {
2411 Record.push_back(Elt: N->isDistinct());
2412 for (auto &I : N->operands())
2413 Record.push_back(Elt: VE.getMetadataOrNullID(MD: I));
2414 Record.push_back(Elt: N->getLineNo());
2415 Record.push_back(Elt: N->getIsDecl());
2416
2417 Stream.EmitRecord(Code: bitc::METADATA_MODULE, Vals: Record, Abbrev);
2418 Record.clear();
2419}
2420
2421void ModuleBitcodeWriter::writeDIAssignID(const DIAssignID *N,
2422 SmallVectorImpl<uint64_t> &Record,
2423 unsigned Abbrev) {
2424 // There are no arguments for this metadata type.
2425 Record.push_back(Elt: N->isDistinct());
2426 Stream.EmitRecord(Code: bitc::METADATA_ASSIGN_ID, Vals: Record, Abbrev);
2427 Record.clear();
2428}
2429
2430void ModuleBitcodeWriter::writeDITemplateTypeParameter(
2431 const DITemplateTypeParameter *N, SmallVectorImpl<uint64_t> &Record,
2432 unsigned Abbrev) {
2433 Record.push_back(Elt: N->isDistinct());
2434 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2435 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2436 Record.push_back(Elt: N->isDefault());
2437
2438 Stream.EmitRecord(Code: bitc::METADATA_TEMPLATE_TYPE, Vals: Record, Abbrev);
2439 Record.clear();
2440}
2441
2442void ModuleBitcodeWriter::writeDITemplateValueParameter(
2443 const DITemplateValueParameter *N, SmallVectorImpl<uint64_t> &Record,
2444 unsigned Abbrev) {
2445 Record.push_back(Elt: N->isDistinct());
2446 Record.push_back(Elt: N->getTag());
2447 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2448 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2449 Record.push_back(Elt: N->isDefault());
2450 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getValue()));
2451
2452 Stream.EmitRecord(Code: bitc::METADATA_TEMPLATE_VALUE, Vals: Record, Abbrev);
2453 Record.clear();
2454}
2455
2456void ModuleBitcodeWriter::writeDIGlobalVariable(
2457 const DIGlobalVariable *N, SmallVectorImpl<uint64_t> &Record,
2458 unsigned Abbrev) {
2459 const uint64_t Version = 2 << 1;
2460 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
2461 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2462 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2463 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLinkageName()));
2464 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2465 Record.push_back(Elt: N->getLine());
2466 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2467 Record.push_back(Elt: N->isLocalToUnit());
2468 Record.push_back(Elt: N->isDefinition());
2469 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStaticDataMemberDeclaration()));
2470 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams()));
2471 Record.push_back(Elt: N->getAlignInBits());
2472 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2473
2474 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_VAR, Vals: Record, Abbrev);
2475 Record.clear();
2476}
2477
2478void ModuleBitcodeWriter::writeDILocalVariable(
2479 const DILocalVariable *N, SmallVectorImpl<uint64_t> &Record,
2480 unsigned Abbrev) {
2481 // In order to support all possible bitcode formats in BitcodeReader we need
2482 // to distinguish the following cases:
2483 // 1) Record has no artificial tag (Record[1]),
2484 // has no obsolete inlinedAt field (Record[9]).
2485 // In this case Record size will be 8, HasAlignment flag is false.
2486 // 2) Record has artificial tag (Record[1]),
2487 // has no obsolete inlignedAt field (Record[9]).
2488 // In this case Record size will be 9, HasAlignment flag is false.
2489 // 3) Record has both artificial tag (Record[1]) and
2490 // obsolete inlignedAt field (Record[9]).
2491 // In this case Record size will be 10, HasAlignment flag is false.
2492 // 4) Record has neither artificial tag, nor inlignedAt field, but
2493 // HasAlignment flag is true and Record[8] contains alignment value.
2494 const uint64_t HasAlignmentFlag = 1 << 1;
2495 Record.push_back(Elt: (uint64_t)N->isDistinct() | HasAlignmentFlag);
2496 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2497 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2498 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2499 Record.push_back(Elt: N->getLine());
2500 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2501 Record.push_back(Elt: N->getArg());
2502 Record.push_back(Elt: N->getFlags());
2503 Record.push_back(Elt: N->getAlignInBits());
2504 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2505
2506 Stream.EmitRecord(Code: bitc::METADATA_LOCAL_VAR, Vals: Record, Abbrev);
2507 Record.clear();
2508}
2509
2510void ModuleBitcodeWriter::writeDILabel(
2511 const DILabel *N, SmallVectorImpl<uint64_t> &Record,
2512 unsigned Abbrev) {
2513 uint64_t IsArtificialFlag = uint64_t(N->isArtificial()) << 1;
2514 Record.push_back(Elt: (uint64_t)N->isDistinct() | IsArtificialFlag);
2515 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2516 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2517 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2518 Record.push_back(Elt: N->getLine());
2519 Record.push_back(Elt: N->getColumn());
2520 Record.push_back(Elt: N->getCoroSuspendIdx().has_value()
2521 ? (uint64_t)N->getCoroSuspendIdx().value()
2522 : std::numeric_limits<uint64_t>::max());
2523
2524 Stream.EmitRecord(Code: bitc::METADATA_LABEL, Vals: Record, Abbrev);
2525 Record.clear();
2526}
2527
2528void ModuleBitcodeWriter::writeDIExpression(const DIExpression *N,
2529 SmallVectorImpl<uint64_t> &Record,
2530 unsigned Abbrev) {
2531 Record.reserve(N: N->getElements().size() + 1);
2532 const uint64_t Version = 3 << 1;
2533 Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
2534 Record.append(in_start: N->elements_begin(), in_end: N->elements_end());
2535
2536 Stream.EmitRecord(Code: bitc::METADATA_EXPRESSION, Vals: Record, Abbrev);
2537 Record.clear();
2538}
2539
2540void ModuleBitcodeWriter::writeDIGlobalVariableExpression(
2541 const DIGlobalVariableExpression *N, SmallVectorImpl<uint64_t> &Record,
2542 unsigned Abbrev) {
2543 Record.push_back(Elt: N->isDistinct());
2544 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getVariable()));
2545 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getExpression()));
2546
2547 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_VAR_EXPR, Vals: Record, Abbrev);
2548 Record.clear();
2549}
2550
2551void ModuleBitcodeWriter::writeDIObjCProperty(const DIObjCProperty *N,
2552 SmallVectorImpl<uint64_t> &Record,
2553 unsigned Abbrev) {
2554 Record.push_back(Elt: N->isDistinct());
2555 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2556 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2557 Record.push_back(Elt: N->getLine());
2558 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSetterName()));
2559 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawGetterName()));
2560 Record.push_back(Elt: N->getAttributes());
2561 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2562
2563 Stream.EmitRecord(Code: bitc::METADATA_OBJC_PROPERTY, Vals: Record, Abbrev);
2564 Record.clear();
2565}
2566
2567void ModuleBitcodeWriter::writeDIProperty(const DIProperty *N,
2568 SmallVectorImpl<uint64_t> &Record,
2569 unsigned Abbrev) {
2570 Record.push_back(Elt: N->isDistinct());
2571 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2572 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2573 Record.push_back(Elt: N->getLine());
2574 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2575 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBackingStorage()));
2576
2577 Stream.EmitRecord(Code: bitc::METADATA_PROPERTY, Vals: Record, Abbrev);
2578 Record.clear();
2579}
2580
2581void ModuleBitcodeWriter::writeDIImportedEntity(
2582 const DIImportedEntity *N, SmallVectorImpl<uint64_t> &Record,
2583 unsigned Abbrev) {
2584 Record.push_back(Elt: N->isDistinct());
2585 Record.push_back(Elt: N->getTag());
2586 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2587 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getEntity()));
2588 Record.push_back(Elt: N->getLine());
2589 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2590 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFile()));
2591 Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2592
2593 Stream.EmitRecord(Code: bitc::METADATA_IMPORTED_ENTITY, Vals: Record, Abbrev);
2594 Record.clear();
2595}
2596
2597unsigned ModuleBitcodeWriter::createNamedMetadataAbbrev() {
2598 auto Abbv = std::make_shared<BitCodeAbbrev>();
2599 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_NAME));
2600 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2601 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2602 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
2603}
2604
2605void ModuleBitcodeWriter::writeNamedMetadata(
2606 SmallVectorImpl<uint64_t> &Record) {
2607 if (M.named_metadata_empty())
2608 return;
2609
2610 unsigned Abbrev = createNamedMetadataAbbrev();
2611 for (const NamedMDNode &NMD : M.named_metadata()) {
2612 // Write name.
2613 StringRef Str = NMD.getName();
2614 Record.append(in_start: Str.bytes_begin(), in_end: Str.bytes_end());
2615 Stream.EmitRecord(Code: bitc::METADATA_NAME, Vals: Record, Abbrev);
2616 Record.clear();
2617
2618 // Write named metadata operands.
2619 for (const MDNode *N : NMD.operands())
2620 Record.push_back(Elt: VE.getMetadataID(MD: N));
2621 Stream.EmitRecord(Code: bitc::METADATA_NAMED_NODE, Vals: Record, Abbrev: 0);
2622 Record.clear();
2623 }
2624}
2625
2626unsigned ModuleBitcodeWriter::createMetadataStringsAbbrev() {
2627 auto Abbv = std::make_shared<BitCodeAbbrev>();
2628 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_STRINGS));
2629 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // # of strings
2630 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // offset to chars
2631 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
2632 return Stream.EmitAbbrev(Abbv: std::move(Abbv));
2633}
2634
2635/// Write out a record for MDString.
2636///
2637/// All the metadata strings in a metadata block are emitted in a single
2638/// record. The sizes and strings themselves are shoved into a blob.
2639void ModuleBitcodeWriter::writeMetadataStrings(
2640 ArrayRef<const Metadata *> Strings, SmallVectorImpl<uint64_t> &Record) {
2641 if (Strings.empty())
2642 return;
2643
2644 // Start the record with the number of strings.
2645 Record.push_back(Elt: bitc::METADATA_STRINGS);
2646 Record.push_back(Elt: Strings.size());
2647
2648 // Emit the sizes of the strings in the blob.
2649 SmallString<256> Blob;
2650 {
2651 BitstreamWriter W(Blob);
2652 for (const Metadata *MD : Strings)
2653 W.EmitVBR(Val: cast<MDString>(Val: MD)->getLength(), NumBits: 6);
2654 W.FlushToWord();
2655 }
2656
2657 // Add the offset to the strings to the record.
2658 Record.push_back(Elt: Blob.size());
2659
2660 // Add the strings to the blob.
2661 for (const Metadata *MD : Strings)
2662 Blob.append(RHS: cast<MDString>(Val: MD)->getString());
2663
2664 // Emit the final record.
2665 Stream.EmitRecordWithBlob(Abbrev: createMetadataStringsAbbrev(), Vals: Record, Blob);
2666 Record.clear();
2667}
2668
2669// Generates an enum to use as an index in the Abbrev array of Metadata record.
2670enum MetadataAbbrev : unsigned {
2671#define HANDLE_MDNODE_LEAF(CLASS) CLASS##AbbrevID,
2672#include "llvm/IR/Metadata.def"
2673 LastPlusOne
2674};
2675
2676void ModuleBitcodeWriter::writeMetadataRecords(
2677 ArrayRef<const Metadata *> MDs, SmallVectorImpl<uint64_t> &Record,
2678 std::vector<unsigned> *MDAbbrevs, std::vector<uint64_t> *IndexPos) {
2679 if (MDs.empty())
2680 return;
2681
2682 // Initialize MDNode abbreviations.
2683#define HANDLE_MDNODE_LEAF(CLASS) unsigned CLASS##Abbrev = 0;
2684#include "llvm/IR/Metadata.def"
2685
2686 for (const Metadata *MD : MDs) {
2687 if (IndexPos)
2688 IndexPos->push_back(x: Stream.GetCurrentBitNo());
2689 if (const MDNode *N = dyn_cast<MDNode>(Val: MD)) {
2690 assert(N->isResolved() && "Expected forward references to be resolved");
2691
2692 switch (N->getMetadataID()) {
2693 default:
2694 llvm_unreachable("Invalid MDNode subclass");
2695#define HANDLE_MDNODE_LEAF(CLASS) \
2696 case Metadata::CLASS##Kind: \
2697 if (MDAbbrevs) \
2698 write##CLASS(cast<CLASS>(N), Record, \
2699 (*MDAbbrevs)[MetadataAbbrev::CLASS##AbbrevID]); \
2700 else \
2701 write##CLASS(cast<CLASS>(N), Record, CLASS##Abbrev); \
2702 continue;
2703#include "llvm/IR/Metadata.def"
2704 }
2705 }
2706 if (auto *AL = dyn_cast<DIArgList>(Val: MD)) {
2707 writeDIArgList(N: AL, Record);
2708 continue;
2709 }
2710 writeValueAsMetadata(MD: cast<ValueAsMetadata>(Val: MD), Record);
2711 }
2712}
2713
2714void ModuleBitcodeWriter::writeModuleMetadata() {
2715 if (!VE.hasMDs() && M.named_metadata_empty())
2716 return;
2717
2718 Stream.EnterSubblock(BlockID: bitc::METADATA_BLOCK_ID, CodeLen: 4);
2719 SmallVector<uint64_t, 64> Record;
2720
2721 // Emit all abbrevs upfront, so that the reader can jump in the middle of the
2722 // block and load any metadata.
2723 std::vector<unsigned> MDAbbrevs;
2724
2725 MDAbbrevs.resize(new_size: MetadataAbbrev::LastPlusOne);
2726 MDAbbrevs[MetadataAbbrev::DILocationAbbrevID] =
2727 createDILocationAbbrev(/*WithIRLayers=*/false);
2728 DILocationLayersAbbrev = createDILocationAbbrev(/*WithIRLayers=*/true);
2729 MDAbbrevs[MetadataAbbrev::GenericDINodeAbbrevID] =
2730 createGenericDINodeAbbrev();
2731
2732 auto Abbv = std::make_shared<BitCodeAbbrev>();
2733 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_INDEX_OFFSET));
2734 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2735 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2736 unsigned OffsetAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2737
2738 Abbv = std::make_shared<BitCodeAbbrev>();
2739 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_INDEX));
2740 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2741 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
2742 unsigned IndexAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2743
2744 // Emit MDStrings together upfront.
2745 writeMetadataStrings(Strings: VE.getMDStrings(), Record);
2746
2747 // We only emit an index for the metadata record if we have more than a given
2748 // (naive) threshold of metadatas, otherwise it is not worth it.
2749 if (VE.getNonMDStrings().size() > IndexThreshold) {
2750 // Write a placeholder value in for the offset of the metadata index,
2751 // which is written after the records, so that it can include
2752 // the offset of each entry. The placeholder offset will be
2753 // updated after all records are emitted.
2754 uint64_t Vals[] = {0, 0};
2755 Stream.EmitRecord(Code: bitc::METADATA_INDEX_OFFSET, Vals, Abbrev: OffsetAbbrev);
2756 }
2757
2758 // Compute and save the bit offset to the current position, which will be
2759 // patched when we emit the index later. We can simply subtract the 64-bit
2760 // fixed size from the current bit number to get the location to backpatch.
2761 uint64_t IndexOffsetRecordBitPos = Stream.GetCurrentBitNo();
2762
2763 // This index will contain the bitpos for each individual record.
2764 std::vector<uint64_t> IndexPos;
2765 IndexPos.reserve(n: VE.getNonMDStrings().size());
2766
2767 // Write all the records
2768 writeMetadataRecords(MDs: VE.getNonMDStrings(), Record, MDAbbrevs: &MDAbbrevs, IndexPos: &IndexPos);
2769
2770 if (VE.getNonMDStrings().size() > IndexThreshold) {
2771 // Now that we have emitted all the records we will emit the index. But
2772 // first
2773 // backpatch the forward reference so that the reader can skip the records
2774 // efficiently.
2775 Stream.BackpatchWord64(BitNo: IndexOffsetRecordBitPos - 64,
2776 Val: Stream.GetCurrentBitNo() - IndexOffsetRecordBitPos);
2777
2778 // Delta encode the index.
2779 uint64_t PreviousValue = IndexOffsetRecordBitPos;
2780 for (auto &Elt : IndexPos) {
2781 auto EltDelta = Elt - PreviousValue;
2782 PreviousValue = Elt;
2783 Elt = EltDelta;
2784 }
2785 // Emit the index record.
2786 Stream.EmitRecord(Code: bitc::METADATA_INDEX, Vals: IndexPos, Abbrev: IndexAbbrev);
2787 IndexPos.clear();
2788 }
2789
2790 // Write the named metadata now.
2791 writeNamedMetadata(Record);
2792
2793 auto AddDeclAttachedMetadata = [&](const GlobalObject &GO) {
2794 SmallVector<uint64_t, 4> Record;
2795 Record.push_back(Elt: VE.getValueID(V: &GO));
2796 pushGlobalMetadataAttachment(Record, GO);
2797 Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_DECL_ATTACHMENT, Vals: Record);
2798 };
2799 for (const Function &F : M)
2800 if (F.isDeclaration() && F.hasMetadata())
2801 AddDeclAttachedMetadata(F);
2802 for (const GlobalIFunc &GI : M.ifuncs())
2803 if (GI.hasMetadata())
2804 AddDeclAttachedMetadata(GI);
2805 // FIXME: Only store metadata for declarations here, and move data for global
2806 // variable definitions to a separate block (PR28134).
2807 for (const GlobalVariable &GV : M.globals())
2808 if (GV.hasMetadata())
2809 AddDeclAttachedMetadata(GV);
2810
2811 Stream.ExitBlock();
2812}
2813
2814void ModuleBitcodeWriter::writeFunctionMetadata(const Function &F) {
2815 if (!VE.hasMDs())
2816 return;
2817
2818 Stream.EnterSubblock(BlockID: bitc::METADATA_BLOCK_ID, CodeLen: 3);
2819 // New block, new abbrev id space. Unlike the module block this one is not
2820 // randomly accessed, so the irlayers abbrev can be created on first use.
2821 DILocationLayersAbbrev = 0;
2822 SmallVector<uint64_t, 64> Record;
2823 writeMetadataStrings(Strings: VE.getMDStrings(), Record);
2824 writeMetadataRecords(MDs: VE.getNonMDStrings(), Record);
2825 Stream.ExitBlock();
2826}
2827
2828void ModuleBitcodeWriter::pushGlobalMetadataAttachment(
2829 SmallVectorImpl<uint64_t> &Record, const GlobalObject &GO) {
2830 // [n x [id, mdnode]]
2831 SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2832 GO.getAllMetadata(MDs);
2833 for (const auto &I : MDs) {
2834 Record.push_back(Elt: I.first);
2835 Record.push_back(Elt: VE.getMetadataID(MD: I.second));
2836 }
2837}
2838
2839void ModuleBitcodeWriter::writeFunctionMetadataAttachment(const Function &F) {
2840 Stream.EnterSubblock(BlockID: bitc::METADATA_ATTACHMENT_ID, CodeLen: 3);
2841
2842 SmallVector<uint64_t, 64> Record;
2843
2844 if (F.hasMetadata()) {
2845 pushGlobalMetadataAttachment(Record, GO: F);
2846 Stream.EmitRecord(Code: bitc::METADATA_ATTACHMENT, Vals: Record, Abbrev: 0);
2847 Record.clear();
2848 }
2849
2850 // Write metadata attachments
2851 // METADATA_ATTACHMENT - [m x [value, [n x [id, mdnode]]]
2852 SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2853 for (const BasicBlock &BB : F)
2854 for (const Instruction &I : BB) {
2855 MDs.clear();
2856 I.getAllMetadataOtherThanDebugLoc(MDs);
2857
2858 // If no metadata, ignore instruction.
2859 if (MDs.empty()) continue;
2860
2861 Record.push_back(Elt: VE.getInstructionID(I: &I));
2862
2863 for (const auto &[ID, MD] : MDs) {
2864 Record.push_back(Elt: ID);
2865 Record.push_back(Elt: VE.getMetadataID(MD));
2866 }
2867 Stream.EmitRecord(Code: bitc::METADATA_ATTACHMENT, Vals: Record, Abbrev: 0);
2868 Record.clear();
2869 }
2870
2871 Stream.ExitBlock();
2872}
2873
2874void ModuleBitcodeWriter::writeModuleMetadataKinds() {
2875 SmallVector<uint64_t, 64> Record;
2876
2877 // Write metadata kinds
2878 // METADATA_KIND - [n x [id, name]]
2879 SmallVector<StringRef, 8> Names;
2880 M.getMDKindNames(Result&: Names);
2881
2882 if (Names.empty()) return;
2883
2884 Stream.EnterSubblock(BlockID: bitc::METADATA_KIND_BLOCK_ID, CodeLen: 3);
2885
2886 for (unsigned MDKindID = 0, e = Names.size(); MDKindID != e; ++MDKindID) {
2887 Record.push_back(Elt: MDKindID);
2888 StringRef KName = Names[MDKindID];
2889 Record.append(in_start: KName.begin(), in_end: KName.end());
2890
2891 Stream.EmitRecord(Code: bitc::METADATA_KIND, Vals: Record, Abbrev: 0);
2892 Record.clear();
2893 }
2894
2895 Stream.ExitBlock();
2896}
2897
2898void ModuleBitcodeWriter::writeOperandBundleTags() {
2899 // Write metadata kinds
2900 //
2901 // OPERAND_BUNDLE_TAGS_BLOCK_ID : N x OPERAND_BUNDLE_TAG
2902 //
2903 // OPERAND_BUNDLE_TAG - [strchr x N]
2904
2905 SmallVector<StringRef, 8> Tags;
2906 M.getOperandBundleTags(Result&: Tags);
2907
2908 if (Tags.empty())
2909 return;
2910
2911 Stream.EnterSubblock(BlockID: bitc::OPERAND_BUNDLE_TAGS_BLOCK_ID, CodeLen: 3);
2912
2913 SmallVector<uint64_t, 64> Record;
2914
2915 for (auto Tag : Tags) {
2916 Record.append(in_start: Tag.begin(), in_end: Tag.end());
2917
2918 Stream.EmitRecord(Code: bitc::OPERAND_BUNDLE_TAG, Vals: Record, Abbrev: 0);
2919 Record.clear();
2920 }
2921
2922 Stream.ExitBlock();
2923}
2924
2925void ModuleBitcodeWriter::writeSyncScopeNames() {
2926 SmallVector<StringRef, 8> SSNs;
2927 M.getContext().getSyncScopeNames(SSNs);
2928 if (SSNs.empty())
2929 return;
2930
2931 Stream.EnterSubblock(BlockID: bitc::SYNC_SCOPE_NAMES_BLOCK_ID, CodeLen: 2);
2932
2933 SmallVector<uint64_t, 64> Record;
2934 for (auto SSN : SSNs) {
2935 Record.append(in_start: SSN.begin(), in_end: SSN.end());
2936 Stream.EmitRecord(Code: bitc::SYNC_SCOPE_NAME, Vals: Record, Abbrev: 0);
2937 Record.clear();
2938 }
2939
2940 Stream.ExitBlock();
2941}
2942
2943void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal,
2944 bool isGlobal) {
2945 if (FirstVal == LastVal) return;
2946
2947 Stream.EnterSubblock(BlockID: bitc::CONSTANTS_BLOCK_ID, CodeLen: 4);
2948
2949 unsigned AggregateAbbrev = 0;
2950 unsigned String8Abbrev = 0;
2951 unsigned CString7Abbrev = 0;
2952 unsigned CString6Abbrev = 0;
2953 // If this is a constant pool for the module, emit module-specific abbrevs.
2954 if (isGlobal) {
2955 // Abbrev for CST_CODE_AGGREGATE.
2956 auto Abbv = std::make_shared<BitCodeAbbrev>();
2957 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_AGGREGATE));
2958 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2959 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(Value: LastVal+1)));
2960 AggregateAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2961
2962 // Abbrev for CST_CODE_STRING.
2963 Abbv = std::make_shared<BitCodeAbbrev>();
2964 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_STRING));
2965 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2966 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2967 String8Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2968 // Abbrev for CST_CODE_CSTRING.
2969 Abbv = std::make_shared<BitCodeAbbrev>();
2970 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2971 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2972 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
2973 CString7Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2974 // Abbrev for CST_CODE_CSTRING.
2975 Abbv = std::make_shared<BitCodeAbbrev>();
2976 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2977 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2978 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
2979 CString6Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2980 }
2981
2982 SmallVector<uint64_t, 64> Record;
2983
2984 const ValueEnumerator::ValueList &Vals = VE.getValues();
2985 Type *LastTy = nullptr;
2986 for (unsigned i = FirstVal; i != LastVal; ++i) {
2987 const Value *V = Vals[i].first;
2988 // If we need to switch types, do so now.
2989 if (V->getType() != LastTy) {
2990 LastTy = V->getType();
2991 Record.push_back(Elt: VE.getTypeID(T: LastTy));
2992 Stream.EmitRecord(Code: bitc::CST_CODE_SETTYPE, Vals: Record,
2993 Abbrev: CONSTANTS_SETTYPE_ABBREV);
2994 Record.clear();
2995 }
2996
2997 if (const InlineAsm *IA = dyn_cast<InlineAsm>(Val: V)) {
2998 Record.push_back(Elt: VE.getTypeID(T: IA->getFunctionType()));
2999 Record.push_back(
3000 Elt: unsigned(IA->hasSideEffects()) | unsigned(IA->isAlignStack()) << 1 |
3001 unsigned(IA->getDialect() & 1) << 2 | unsigned(IA->canThrow()) << 3);
3002
3003 // Add the asm string.
3004 StringRef AsmStr = IA->getAsmString();
3005 Record.push_back(Elt: AsmStr.size());
3006 Record.append(in_start: AsmStr.begin(), in_end: AsmStr.end());
3007
3008 // Add the constraint string.
3009 StringRef ConstraintStr = IA->getConstraintString();
3010 Record.push_back(Elt: ConstraintStr.size());
3011 Record.append(in_start: ConstraintStr.begin(), in_end: ConstraintStr.end());
3012 Stream.EmitRecord(Code: bitc::CST_CODE_INLINEASM, Vals: Record);
3013 Record.clear();
3014 continue;
3015 }
3016 const Constant *C = cast<Constant>(Val: V);
3017 unsigned Code = -1U;
3018 unsigned AbbrevToUse = 0;
3019 if (C->isNullValue()) {
3020 Code = bitc::CST_CODE_NULL;
3021 } else if (isa<PoisonValue>(Val: C)) {
3022 Code = bitc::CST_CODE_POISON;
3023 } else if (isa<UndefValue>(Val: C)) {
3024 Code = bitc::CST_CODE_UNDEF;
3025 } else if (const ConstantInt *IV = dyn_cast<ConstantInt>(Val: C)) {
3026 if (IV->getBitWidth() <= 64) {
3027 uint64_t V = IV->getSExtValue();
3028 emitSignedInt64(Vals&: Record, V);
3029 Code = bitc::CST_CODE_INTEGER;
3030 AbbrevToUse = CONSTANTS_INTEGER_ABBREV;
3031 } else { // Wide integers, > 64 bits in size.
3032 emitWideAPInt(Vals&: Record, A: IV->getValue());
3033 Code = bitc::CST_CODE_WIDE_INTEGER;
3034 }
3035 } else if (const ConstantByte *BV = dyn_cast<ConstantByte>(Val: C)) {
3036 if (BV->getBitWidth() <= 64) {
3037 uint64_t V = BV->getSExtValue();
3038 emitSignedInt64(Vals&: Record, V);
3039 Code = bitc::CST_CODE_BYTE;
3040 AbbrevToUse = CONSTANTS_BYTE_ABBREV;
3041 } else { // Wide bytes, > 64 bits in size.
3042 emitWideAPInt(Vals&: Record, A: BV->getValue());
3043 Code = bitc::CST_CODE_WIDE_BYTE;
3044 }
3045 } else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
3046 Code = bitc::CST_CODE_FLOAT;
3047 Type *Ty = CFP->getType()->getScalarType();
3048 if (Ty->isHalfTy() || Ty->isBFloatTy() || Ty->isFloatTy() ||
3049 Ty->isDoubleTy()) {
3050 Record.push_back(Elt: CFP->getValueAPF().bitcastToAPInt().getZExtValue());
3051 } else if (Ty->isX86_FP80Ty()) {
3052 // api needed to prevent premature destruction
3053 // bits are not in the same order as a normal i80 APInt, compensate.
3054 APInt api = CFP->getValueAPF().bitcastToAPInt();
3055 const uint64_t *p = api.getRawData();
3056 Record.push_back(Elt: (p[1] << 48) | (p[0] >> 16));
3057 Record.push_back(Elt: p[0] & 0xffffLL);
3058 } else if (Ty->isFP128Ty() || Ty->isPPC_FP128Ty()) {
3059 APInt api = CFP->getValueAPF().bitcastToAPInt();
3060 const uint64_t *p = api.getRawData();
3061 Record.push_back(Elt: p[0]);
3062 Record.push_back(Elt: p[1]);
3063 } else {
3064 assert(0 && "Unknown FP type!");
3065 }
3066 } else if (isa<ConstantDataSequential>(Val: C) &&
3067 cast<ConstantDataSequential>(Val: C)->isString()) {
3068 const ConstantDataSequential *Str = cast<ConstantDataSequential>(Val: C);
3069 // Emit constant strings specially.
3070 uint64_t NumElts = Str->getNumElements();
3071 // If this is a null-terminated string, use the denser CSTRING encoding.
3072 if (Str->isCString()) {
3073 Code = bitc::CST_CODE_CSTRING;
3074 --NumElts; // Don't encode the null, which isn't allowed by char6.
3075 } else {
3076 Code = bitc::CST_CODE_STRING;
3077 AbbrevToUse = String8Abbrev;
3078 }
3079 bool isCStr7 = Code == bitc::CST_CODE_CSTRING;
3080 bool isCStrChar6 = Code == bitc::CST_CODE_CSTRING;
3081 for (uint64_t i = 0; i != NumElts; ++i) {
3082 unsigned char V = Str->getElementAsInteger(i);
3083 Record.push_back(Elt: V);
3084 isCStr7 &= (V & 128) == 0;
3085 if (isCStrChar6)
3086 isCStrChar6 = BitCodeAbbrevOp::isChar6(C: V);
3087 }
3088
3089 if (isCStrChar6)
3090 AbbrevToUse = CString6Abbrev;
3091 else if (isCStr7)
3092 AbbrevToUse = CString7Abbrev;
3093 } else if (const ConstantDataSequential *CDS =
3094 dyn_cast<ConstantDataSequential>(Val: C)) {
3095 Code = bitc::CST_CODE_DATA;
3096 Type *EltTy = CDS->getElementType();
3097 if (isa<IntegerType>(Val: EltTy) || isa<ByteType>(Val: EltTy)) {
3098 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i)
3099 Record.push_back(Elt: CDS->getElementAsInteger(i));
3100 } else {
3101 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i)
3102 Record.push_back(
3103 Elt: CDS->getElementAsAPFloat(i).bitcastToAPInt().getLimitedValue());
3104 }
3105 } else if (isa<ConstantAggregate>(Val: C)) {
3106 Code = bitc::CST_CODE_AGGREGATE;
3107 for (const Value *Op : C->operands())
3108 Record.push_back(Elt: VE.getValueID(V: Op));
3109 AbbrevToUse = AggregateAbbrev;
3110 } else if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: C)) {
3111 switch (CE->getOpcode()) {
3112 default:
3113 if (Instruction::isCast(Opcode: CE->getOpcode())) {
3114 Code = bitc::CST_CODE_CE_CAST;
3115 Record.push_back(Elt: getEncodedCastOpcode(Opcode: CE->getOpcode()));
3116 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
3117 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3118 AbbrevToUse = CONSTANTS_CE_CAST_Abbrev;
3119 } else {
3120 assert(CE->getNumOperands() == 2 && "Unknown constant expr!");
3121 Code = bitc::CST_CODE_CE_BINOP;
3122 Record.push_back(Elt: getEncodedBinaryOpcode(Opcode: CE->getOpcode()));
3123 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3124 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3125 uint64_t Flags = getOptimizationFlags(V: CE);
3126 if (Flags != 0)
3127 Record.push_back(Elt: Flags);
3128 }
3129 break;
3130 case Instruction::FNeg: {
3131 assert(CE->getNumOperands() == 1 && "Unknown constant expr!");
3132 Code = bitc::CST_CODE_CE_UNOP;
3133 Record.push_back(Elt: getEncodedUnaryOpcode(Opcode: CE->getOpcode()));
3134 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3135 uint64_t Flags = getOptimizationFlags(V: CE);
3136 if (Flags != 0)
3137 Record.push_back(Elt: Flags);
3138 break;
3139 }
3140 case Instruction::GetElementPtr: {
3141 Code = bitc::CST_CODE_CE_GEP;
3142 const auto *GO = cast<GEPOperator>(Val: C);
3143 Record.push_back(Elt: VE.getTypeID(T: GO->getSourceElementType()));
3144 Record.push_back(Elt: getOptimizationFlags(V: GO));
3145 if (std::optional<ConstantRange> Range = GO->getInRange()) {
3146 Code = bitc::CST_CODE_CE_GEP_WITH_INRANGE;
3147 emitConstantRange(Record, CR: *Range, /*EmitBitWidth=*/true);
3148 }
3149 for (const Value *Op : CE->operands()) {
3150 Record.push_back(Elt: VE.getTypeID(T: Op->getType()));
3151 Record.push_back(Elt: VE.getValueID(V: Op));
3152 }
3153 break;
3154 }
3155 case Instruction::ExtractElement:
3156 Code = bitc::CST_CODE_CE_EXTRACTELT;
3157 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
3158 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3159 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 1)->getType()));
3160 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3161 break;
3162 case Instruction::InsertElement:
3163 Code = bitc::CST_CODE_CE_INSERTELT;
3164 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3165 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3166 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 2)->getType()));
3167 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 2)));
3168 break;
3169 case Instruction::ShuffleVector:
3170 // If the return type and argument types are the same, this is a
3171 // standard shufflevector instruction. If the types are different,
3172 // then the shuffle is widening or truncating the input vectors, and
3173 // the argument type must also be encoded.
3174 if (C->getType() == C->getOperand(i: 0)->getType()) {
3175 Code = bitc::CST_CODE_CE_SHUFFLEVEC;
3176 } else {
3177 Code = bitc::CST_CODE_CE_SHUFVEC_EX;
3178 Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
3179 }
3180 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
3181 Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
3182 Record.push_back(Elt: VE.getValueID(V: CE->getShuffleMaskForBitcode()));
3183 break;
3184 }
3185 } else if (const BlockAddress *BA = dyn_cast<BlockAddress>(Val: C)) {
3186 Code = bitc::CST_CODE_BLOCKADDRESS;
3187 Record.push_back(Elt: VE.getTypeID(T: BA->getFunction()->getType()));
3188 Record.push_back(Elt: VE.getValueID(V: BA->getFunction()));
3189 Record.push_back(Elt: VE.getGlobalBasicBlockID(BB: BA->getBasicBlock()));
3190 } else if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(Val: C)) {
3191 Code = bitc::CST_CODE_DSO_LOCAL_EQUIVALENT;
3192 Record.push_back(Elt: VE.getTypeID(T: Equiv->getGlobalValue()->getType()));
3193 Record.push_back(Elt: VE.getValueID(V: Equiv->getGlobalValue()));
3194 } else if (const auto *NC = dyn_cast<NoCFIValue>(Val: C)) {
3195 Code = bitc::CST_CODE_NO_CFI_VALUE;
3196 Record.push_back(Elt: VE.getTypeID(T: NC->getGlobalValue()->getType()));
3197 Record.push_back(Elt: VE.getValueID(V: NC->getGlobalValue()));
3198 } else if (const auto *CPA = dyn_cast<ConstantPtrAuth>(Val: C)) {
3199 Code = bitc::CST_CODE_PTRAUTH2;
3200 Record.push_back(Elt: VE.getValueID(V: CPA->getPointer()));
3201 Record.push_back(Elt: VE.getValueID(V: CPA->getKey()));
3202 Record.push_back(Elt: VE.getValueID(V: CPA->getDiscriminator()));
3203 Record.push_back(Elt: VE.getValueID(V: CPA->getAddrDiscriminator()));
3204 Record.push_back(Elt: VE.getValueID(V: CPA->getDeactivationSymbol()));
3205 } else {
3206#ifndef NDEBUG
3207 C->dump();
3208#endif
3209 llvm_unreachable("Unknown constant!");
3210 }
3211 Stream.EmitRecord(Code, Vals: Record, Abbrev: AbbrevToUse);
3212 Record.clear();
3213 }
3214
3215 Stream.ExitBlock();
3216}
3217
3218void ModuleBitcodeWriter::writeModuleConstants() {
3219 const ValueEnumerator::ValueList &Vals = VE.getValues();
3220
3221 // Find the first constant to emit, which is the first non-globalvalue value.
3222 // We know globalvalues have been emitted by WriteModuleInfo.
3223 for (unsigned i = 0, e = Vals.size(); i != e; ++i) {
3224 if (!isa<GlobalValue>(Val: Vals[i].first)) {
3225 writeConstants(FirstVal: i, LastVal: Vals.size(), isGlobal: true);
3226 return;
3227 }
3228 }
3229}
3230
3231/// pushValueAndType - The file has to encode both the value and type id for
3232/// many values, because we need to know what type to create for forward
3233/// references. However, most operands are not forward references, so this type
3234/// field is not needed.
3235///
3236/// This function adds V's value ID to Vals. If the value ID is higher than the
3237/// instruction ID, then it is a forward reference, and it also includes the
3238/// type ID. The value ID that is written is encoded relative to the InstID.
3239bool ModuleBitcodeWriter::pushValueAndType(const Value *V, unsigned InstID,
3240 SmallVectorImpl<unsigned> &Vals) {
3241 unsigned ValID = VE.getValueID(V);
3242 // Make encoding relative to the InstID.
3243 Vals.push_back(Elt: InstID - ValID);
3244 if (ValID >= InstID) {
3245 Vals.push_back(Elt: VE.getTypeID(T: V->getType()));
3246 return true;
3247 }
3248 return false;
3249}
3250
3251bool ModuleBitcodeWriter::pushValueOrMetadata(const Value *V, unsigned InstID,
3252 SmallVectorImpl<unsigned> &Vals) {
3253 bool IsMetadata = V->getType()->isMetadataTy();
3254 if (IsMetadata) {
3255 Vals.push_back(Elt: bitc::OB_METADATA);
3256 Metadata *MD = cast<MetadataAsValue>(Val: V)->getMetadata();
3257 unsigned ValID = VE.getMetadataID(MD);
3258 Vals.push_back(Elt: InstID - ValID);
3259 return false;
3260 }
3261 return pushValueAndType(V, InstID, Vals);
3262}
3263
3264void ModuleBitcodeWriter::writeOperandBundles(const CallBase &CS,
3265 unsigned InstID) {
3266 SmallVector<unsigned, 64> Record;
3267 LLVMContext &C = CS.getContext();
3268
3269 for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i) {
3270 const auto &Bundle = CS.getOperandBundleAt(Index: i);
3271 Record.push_back(Elt: C.getOperandBundleTagID(Tag: Bundle.getTagName()));
3272
3273 for (auto &Input : Bundle.Inputs)
3274 pushValueOrMetadata(V: Input, InstID, Vals&: Record);
3275
3276 Stream.EmitRecord(Code: bitc::FUNC_CODE_OPERAND_BUNDLE, Vals: Record);
3277 Record.clear();
3278 }
3279}
3280
3281/// pushValue - Like pushValueAndType, but where the type of the value is
3282/// omitted (perhaps it was already encoded in an earlier operand).
3283void ModuleBitcodeWriter::pushValue(const Value *V, unsigned InstID,
3284 SmallVectorImpl<unsigned> &Vals) {
3285 unsigned ValID = VE.getValueID(V);
3286 Vals.push_back(Elt: InstID - ValID);
3287}
3288
3289void ModuleBitcodeWriter::pushValueSigned(const Value *V, unsigned InstID,
3290 SmallVectorImpl<uint64_t> &Vals) {
3291 unsigned ValID = VE.getValueID(V);
3292 int64_t diff = ((int32_t)InstID - (int32_t)ValID);
3293 emitSignedInt64(Vals, V: diff);
3294}
3295
3296/// WriteInstruction - Emit an instruction to the specified stream.
3297void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
3298 unsigned InstID,
3299 SmallVectorImpl<unsigned> &Vals) {
3300 unsigned Code = 0;
3301 unsigned AbbrevToUse = 0;
3302 VE.setInstructionID(&I);
3303 switch (I.getOpcode()) {
3304 default:
3305 if (Instruction::isCast(Opcode: I.getOpcode())) {
3306 Code = bitc::FUNC_CODE_INST_CAST;
3307 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3308 AbbrevToUse = FUNCTION_INST_CAST_ABBREV;
3309 Vals.push_back(Elt: VE.getTypeID(T: I.getType()));
3310 Vals.push_back(Elt: getEncodedCastOpcode(Opcode: I.getOpcode()));
3311 uint64_t Flags = getOptimizationFlags(V: &I);
3312 if (Flags != 0) {
3313 if (AbbrevToUse == FUNCTION_INST_CAST_ABBREV)
3314 AbbrevToUse = FUNCTION_INST_CAST_FLAGS_ABBREV;
3315 Vals.push_back(Elt: Flags);
3316 }
3317 } else {
3318 assert(isa<BinaryOperator>(I) && "Unknown instruction!");
3319 Code = bitc::FUNC_CODE_INST_BINOP;
3320 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3321 AbbrevToUse = FUNCTION_INST_BINOP_ABBREV;
3322 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3323 Vals.push_back(Elt: getEncodedBinaryOpcode(Opcode: I.getOpcode()));
3324 uint64_t Flags = getOptimizationFlags(V: &I);
3325 if (Flags != 0) {
3326 if (AbbrevToUse == FUNCTION_INST_BINOP_ABBREV)
3327 AbbrevToUse = FUNCTION_INST_BINOP_FLAGS_ABBREV;
3328 Vals.push_back(Elt: Flags);
3329 }
3330 }
3331 break;
3332 case Instruction::FNeg: {
3333 Code = bitc::FUNC_CODE_INST_UNOP;
3334 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3335 AbbrevToUse = FUNCTION_INST_UNOP_ABBREV;
3336 Vals.push_back(Elt: getEncodedUnaryOpcode(Opcode: I.getOpcode()));
3337 uint64_t Flags = getOptimizationFlags(V: &I);
3338 if (Flags != 0) {
3339 if (AbbrevToUse == FUNCTION_INST_UNOP_ABBREV)
3340 AbbrevToUse = FUNCTION_INST_UNOP_FLAGS_ABBREV;
3341 Vals.push_back(Elt: Flags);
3342 }
3343 break;
3344 }
3345 case Instruction::GetElementPtr: {
3346 Code = bitc::FUNC_CODE_INST_GEP;
3347 AbbrevToUse = FUNCTION_INST_GEP_ABBREV;
3348 auto &GEPInst = cast<GetElementPtrInst>(Val: I);
3349 Vals.push_back(Elt: getOptimizationFlags(V: &I));
3350 Vals.push_back(Elt: VE.getTypeID(T: GEPInst.getSourceElementType()));
3351 for (const Value *Op : I.operands())
3352 pushValueAndType(V: Op, InstID, Vals);
3353 break;
3354 }
3355 case Instruction::ExtractValue: {
3356 Code = bitc::FUNC_CODE_INST_EXTRACTVAL;
3357 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3358 const ExtractValueInst *EVI = cast<ExtractValueInst>(Val: &I);
3359 Vals.append(in_start: EVI->idx_begin(), in_end: EVI->idx_end());
3360 break;
3361 }
3362 case Instruction::InsertValue: {
3363 Code = bitc::FUNC_CODE_INST_INSERTVAL;
3364 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3365 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3366 const InsertValueInst *IVI = cast<InsertValueInst>(Val: &I);
3367 Vals.append(in_start: IVI->idx_begin(), in_end: IVI->idx_end());
3368 break;
3369 }
3370 case Instruction::Select: {
3371 Code = bitc::FUNC_CODE_INST_VSELECT;
3372 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3373 pushValue(V: I.getOperand(i: 2), InstID, Vals);
3374 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3375 uint64_t Flags = getOptimizationFlags(V: &I);
3376 if (Flags != 0)
3377 Vals.push_back(Elt: Flags);
3378 break;
3379 }
3380 case Instruction::ExtractElement:
3381 Code = bitc::FUNC_CODE_INST_EXTRACTELT;
3382 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3383 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3384 break;
3385 case Instruction::InsertElement:
3386 Code = bitc::FUNC_CODE_INST_INSERTELT;
3387 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3388 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3389 pushValueAndType(V: I.getOperand(i: 2), InstID, Vals);
3390 break;
3391 case Instruction::BitExtract:
3392 Code = bitc::FUNC_CODE_INST_BITEXTRACT;
3393 Vals.push_back(Elt: VE.getTypeID(T: I.getType()));
3394 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3395 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3396 break;
3397 case Instruction::BitInsert:
3398 Code = bitc::FUNC_CODE_INST_BITINSERT;
3399 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3400 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
3401 pushValueAndType(V: I.getOperand(i: 2), InstID, Vals);
3402 break;
3403 case Instruction::ShuffleVector:
3404 Code = bitc::FUNC_CODE_INST_SHUFFLEVEC;
3405 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3406 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3407 pushValue(V: cast<ShuffleVectorInst>(Val: I).getShuffleMaskForBitcode(), InstID,
3408 Vals);
3409 break;
3410 case Instruction::ICmp:
3411 case Instruction::FCmp: {
3412 // compare returning Int1Ty or vector of Int1Ty
3413 Code = bitc::FUNC_CODE_INST_CMP2;
3414 AbbrevToUse = FUNCTION_INST_CMP_ABBREV;
3415 if (pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3416 AbbrevToUse = 0;
3417 pushValue(V: I.getOperand(i: 1), InstID, Vals);
3418 Vals.push_back(Elt: cast<CmpInst>(Val: I).getPredicate());
3419 uint64_t Flags = getOptimizationFlags(V: &I);
3420 if (Flags != 0) {
3421 Vals.push_back(Elt: Flags);
3422 if (AbbrevToUse)
3423 AbbrevToUse = FUNCTION_INST_CMP_FLAGS_ABBREV;
3424 }
3425 break;
3426 }
3427
3428 case Instruction::Ret:
3429 {
3430 Code = bitc::FUNC_CODE_INST_RET;
3431 unsigned NumOperands = I.getNumOperands();
3432 if (NumOperands == 0)
3433 AbbrevToUse = FUNCTION_INST_RET_VOID_ABBREV;
3434 else if (NumOperands == 1) {
3435 if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3436 AbbrevToUse = FUNCTION_INST_RET_VAL_ABBREV;
3437 } else {
3438 for (const Value *Op : I.operands())
3439 pushValueAndType(V: Op, InstID, Vals);
3440 }
3441 }
3442 break;
3443 case Instruction::UncondBr: {
3444 Code = bitc::FUNC_CODE_INST_BR;
3445 AbbrevToUse = FUNCTION_INST_BR_UNCOND_ABBREV;
3446 const UncondBrInst &II = cast<UncondBrInst>(Val: I);
3447 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 0)));
3448 } break;
3449 case Instruction::CondBr: {
3450 Code = bitc::FUNC_CODE_INST_BR;
3451 AbbrevToUse = FUNCTION_INST_BR_COND_ABBREV;
3452 const CondBrInst &II = cast<CondBrInst>(Val: I);
3453 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 0)));
3454 Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 1)));
3455 pushValue(V: II.getCondition(), InstID, Vals);
3456 } break;
3457 case Instruction::Switch:
3458 {
3459 Code = bitc::FUNC_CODE_INST_SWITCH;
3460 const SwitchInst &SI = cast<SwitchInst>(Val: I);
3461 Vals.push_back(Elt: VE.getTypeID(T: SI.getCondition()->getType()));
3462 pushValue(V: SI.getCondition(), InstID, Vals);
3463 Vals.push_back(Elt: VE.getValueID(V: SI.getDefaultDest()));
3464 for (auto Case : SI.cases()) {
3465 Vals.push_back(Elt: VE.getValueID(V: Case.getCaseValue()));
3466 Vals.push_back(Elt: VE.getValueID(V: Case.getCaseSuccessor()));
3467 }
3468 }
3469 break;
3470 case Instruction::IndirectBr:
3471 Code = bitc::FUNC_CODE_INST_INDIRECTBR;
3472 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType()));
3473 // Encode the address operand as relative, but not the basic blocks.
3474 pushValue(V: I.getOperand(i: 0), InstID, Vals);
3475 for (const Value *Op : drop_begin(RangeOrContainer: I.operands()))
3476 Vals.push_back(Elt: VE.getValueID(V: Op));
3477 break;
3478
3479 case Instruction::Invoke: {
3480 const InvokeInst *II = cast<InvokeInst>(Val: &I);
3481 const Value *Callee = II->getCalledOperand();
3482 FunctionType *FTy = II->getFunctionType();
3483
3484 if (II->hasOperandBundles())
3485 writeOperandBundles(CS: *II, InstID);
3486
3487 Code = bitc::FUNC_CODE_INST_INVOKE;
3488
3489 Vals.push_back(Elt: VE.getAttributeListID(PAL: II->getAttributes()));
3490 Vals.push_back(Elt: II->getCallingConv() | 1 << 13);
3491 Vals.push_back(Elt: VE.getValueID(V: II->getNormalDest()));
3492 Vals.push_back(Elt: VE.getValueID(V: II->getUnwindDest()));
3493 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3494 pushValueAndType(V: Callee, InstID, Vals);
3495
3496 // Emit value #'s for the fixed parameters.
3497 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3498 pushValue(V: I.getOperand(i), InstID, Vals); // fixed param.
3499
3500 // Emit type/value pairs for varargs params.
3501 if (FTy->isVarArg()) {
3502 for (unsigned i = FTy->getNumParams(), e = II->arg_size(); i != e; ++i)
3503 pushValueAndType(V: I.getOperand(i), InstID, Vals); // vararg
3504 }
3505 break;
3506 }
3507 case Instruction::Resume:
3508 Code = bitc::FUNC_CODE_INST_RESUME;
3509 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3510 break;
3511 case Instruction::CleanupRet: {
3512 Code = bitc::FUNC_CODE_INST_CLEANUPRET;
3513 const auto &CRI = cast<CleanupReturnInst>(Val: I);
3514 pushValue(V: CRI.getCleanupPad(), InstID, Vals);
3515 if (CRI.hasUnwindDest())
3516 Vals.push_back(Elt: VE.getValueID(V: CRI.getUnwindDest()));
3517 break;
3518 }
3519 case Instruction::CatchRet: {
3520 Code = bitc::FUNC_CODE_INST_CATCHRET;
3521 const auto &CRI = cast<CatchReturnInst>(Val: I);
3522 pushValue(V: CRI.getCatchPad(), InstID, Vals);
3523 Vals.push_back(Elt: VE.getValueID(V: CRI.getSuccessor()));
3524 break;
3525 }
3526 case Instruction::CleanupPad:
3527 case Instruction::CatchPad: {
3528 const auto &FuncletPad = cast<FuncletPadInst>(Val: I);
3529 Code = isa<CatchPadInst>(Val: FuncletPad) ? bitc::FUNC_CODE_INST_CATCHPAD
3530 : bitc::FUNC_CODE_INST_CLEANUPPAD;
3531 pushValue(V: FuncletPad.getParentPad(), InstID, Vals);
3532
3533 unsigned NumArgOperands = FuncletPad.arg_size();
3534 Vals.push_back(Elt: NumArgOperands);
3535 for (unsigned Op = 0; Op != NumArgOperands; ++Op)
3536 pushValueAndType(V: FuncletPad.getArgOperand(i: Op), InstID, Vals);
3537 break;
3538 }
3539 case Instruction::CatchSwitch: {
3540 Code = bitc::FUNC_CODE_INST_CATCHSWITCH;
3541 const auto &CatchSwitch = cast<CatchSwitchInst>(Val: I);
3542
3543 pushValue(V: CatchSwitch.getParentPad(), InstID, Vals);
3544
3545 unsigned NumHandlers = CatchSwitch.getNumHandlers();
3546 Vals.push_back(Elt: NumHandlers);
3547 for (const BasicBlock *CatchPadBB : CatchSwitch.handlers())
3548 Vals.push_back(Elt: VE.getValueID(V: CatchPadBB));
3549
3550 if (CatchSwitch.hasUnwindDest())
3551 Vals.push_back(Elt: VE.getValueID(V: CatchSwitch.getUnwindDest()));
3552 break;
3553 }
3554 case Instruction::CallBr: {
3555 const CallBrInst *CBI = cast<CallBrInst>(Val: &I);
3556 const Value *Callee = CBI->getCalledOperand();
3557 FunctionType *FTy = CBI->getFunctionType();
3558
3559 if (CBI->hasOperandBundles())
3560 writeOperandBundles(CS: *CBI, InstID);
3561
3562 Code = bitc::FUNC_CODE_INST_CALLBR;
3563
3564 Vals.push_back(Elt: VE.getAttributeListID(PAL: CBI->getAttributes()));
3565
3566 Vals.push_back(Elt: CBI->getCallingConv() << bitc::CALL_CCONV |
3567 1 << bitc::CALL_EXPLICIT_TYPE);
3568
3569 Vals.push_back(Elt: VE.getValueID(V: CBI->getDefaultDest()));
3570 Vals.push_back(Elt: CBI->getNumIndirectDests());
3571 for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i)
3572 Vals.push_back(Elt: VE.getValueID(V: CBI->getIndirectDest(i)));
3573
3574 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3575 pushValueAndType(V: Callee, InstID, Vals);
3576
3577 // Emit value #'s for the fixed parameters.
3578 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3579 pushValue(V: I.getOperand(i), InstID, Vals); // fixed param.
3580
3581 // Emit type/value pairs for varargs params.
3582 if (FTy->isVarArg()) {
3583 for (unsigned i = FTy->getNumParams(), e = CBI->arg_size(); i != e; ++i)
3584 pushValueAndType(V: I.getOperand(i), InstID, Vals); // vararg
3585 }
3586 break;
3587 }
3588 case Instruction::Unreachable:
3589 Code = bitc::FUNC_CODE_INST_UNREACHABLE;
3590 AbbrevToUse = FUNCTION_INST_UNREACHABLE_ABBREV;
3591 break;
3592
3593 case Instruction::PHI: {
3594 const PHINode &PN = cast<PHINode>(Val: I);
3595 Code = bitc::FUNC_CODE_INST_PHI;
3596 // With the newer instruction encoding, forward references could give
3597 // negative valued IDs. This is most common for PHIs, so we use
3598 // signed VBRs.
3599 SmallVector<uint64_t, 128> Vals64;
3600 Vals64.push_back(Elt: VE.getTypeID(T: PN.getType()));
3601 for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) {
3602 pushValueSigned(V: PN.getIncomingValue(i), InstID, Vals&: Vals64);
3603 Vals64.push_back(Elt: VE.getValueID(V: PN.getIncomingBlock(i)));
3604 }
3605
3606 uint64_t Flags = getOptimizationFlags(V: &I);
3607 if (Flags != 0)
3608 Vals64.push_back(Elt: Flags);
3609
3610 // Emit a Vals64 vector and exit.
3611 Stream.EmitRecord(Code, Vals: Vals64, Abbrev: AbbrevToUse);
3612 Vals64.clear();
3613 return;
3614 }
3615
3616 case Instruction::LandingPad: {
3617 const LandingPadInst &LP = cast<LandingPadInst>(Val: I);
3618 Code = bitc::FUNC_CODE_INST_LANDINGPAD;
3619 Vals.push_back(Elt: VE.getTypeID(T: LP.getType()));
3620 Vals.push_back(Elt: LP.isCleanup());
3621 Vals.push_back(Elt: LP.getNumClauses());
3622 for (unsigned I = 0, E = LP.getNumClauses(); I != E; ++I) {
3623 if (LP.isCatch(Idx: I))
3624 Vals.push_back(Elt: LandingPadInst::Catch);
3625 else
3626 Vals.push_back(Elt: LandingPadInst::Filter);
3627 pushValueAndType(V: LP.getClause(Idx: I), InstID, Vals);
3628 }
3629 break;
3630 }
3631
3632 case Instruction::Alloca: {
3633 Code = bitc::FUNC_CODE_INST_ALLOCA;
3634 const AllocaInst &AI = cast<AllocaInst>(Val: I);
3635 Vals.push_back(Elt: VE.getTypeID(T: AI.getAllocatedType()));
3636 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType()));
3637 Vals.push_back(Elt: VE.getValueID(V: I.getOperand(i: 0))); // size.
3638 using APV = AllocaPackedValues;
3639 unsigned Record = 0;
3640 unsigned EncodedAlign = getEncodedAlign(Alignment: AI.getAlign());
3641 Bitfield::set<APV::AlignLower>(
3642 Packed&: Record, Value: EncodedAlign & ((1 << APV::AlignLower::Bits) - 1));
3643 Bitfield::set<APV::AlignUpper>(Packed&: Record,
3644 Value: EncodedAlign >> APV::AlignLower::Bits);
3645 Bitfield::set<APV::UsedWithInAlloca>(Packed&: Record, Value: AI.isUsedWithInAlloca());
3646 Bitfield::set<APV::ExplicitType>(Packed&: Record, Value: true);
3647 Bitfield::set<APV::SwiftError>(Packed&: Record, Value: AI.isSwiftError());
3648 Vals.push_back(Elt: Record);
3649
3650 unsigned AS = AI.getAddressSpace();
3651 if (AS != M.getDataLayout().getAllocaAddrSpace())
3652 Vals.push_back(Elt: AS);
3653 break;
3654 }
3655
3656 case Instruction::Load: {
3657 const auto &LI = cast<LoadInst>(Val: I);
3658 if (LI.isAtomic()) {
3659 Code = bitc::FUNC_CODE_INST_LOADATOMIC;
3660 pushValueAndType(V: LI.getOperand(i_nocapture: 0), InstID, Vals);
3661 } else {
3662 Code = bitc::FUNC_CODE_INST_LOAD;
3663 if (!pushValueAndType(V: LI.getOperand(i_nocapture: 0), InstID, Vals)) // ptr
3664 AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
3665 }
3666 Vals.push_back(Elt: VE.getTypeID(T: LI.getType()));
3667 Vals.push_back(Elt: getEncodedAlign(Alignment: LI.getAlign()));
3668 Vals.push_back(Elt: LI.isVolatile());
3669 if (LI.isAtomic()) {
3670 Vals.push_back(Elt: getEncodedOrdering(Ordering: LI.getOrdering()));
3671 Vals.push_back(Elt: getEncodedSyncScopeID(SSID: LI.getSyncScopeID()));
3672 if (LI.isElementwise())
3673 Vals.push_back(Elt: 1);
3674 }
3675 break;
3676 }
3677
3678 case Instruction::Store: {
3679 const auto &SI = cast<StoreInst>(Val: I);
3680 if (SI.isAtomic()) {
3681 Code = bitc::FUNC_CODE_INST_STOREATOMIC;
3682 } else {
3683 Code = bitc::FUNC_CODE_INST_STORE;
3684 AbbrevToUse = FUNCTION_INST_STORE_ABBREV;
3685 }
3686 if (pushValueAndType(V: I.getOperand(i: 1), InstID, Vals)) // ptrty + ptr
3687 AbbrevToUse = 0;
3688 if (pushValueAndType(V: I.getOperand(i: 0), InstID, Vals)) // valty + val
3689 AbbrevToUse = 0;
3690 Vals.push_back(Elt: getEncodedAlign(Alignment: SI.getAlign()));
3691 Vals.push_back(Elt: SI.isVolatile());
3692 if (SI.isAtomic()) {
3693 Vals.push_back(Elt: getEncodedOrdering(Ordering: SI.getOrdering()));
3694 Vals.push_back(Elt: getEncodedSyncScopeID(SSID: SI.getSyncScopeID()));
3695 if (SI.isElementwise())
3696 Vals.push_back(Elt: 1);
3697 }
3698 break;
3699 }
3700
3701 case Instruction::AtomicCmpXchg:
3702 Code = bitc::FUNC_CODE_INST_CMPXCHG;
3703 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals); // ptrty + ptr
3704 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals); // cmp.
3705 pushValue(V: I.getOperand(i: 2), InstID, Vals); // newval.
3706 Vals.push_back(Elt: cast<AtomicCmpXchgInst>(Val: I).isVolatile());
3707 Vals.push_back(
3708 Elt: getEncodedOrdering(Ordering: cast<AtomicCmpXchgInst>(Val: I).getSuccessOrdering()));
3709 Vals.push_back(
3710 Elt: getEncodedSyncScopeID(SSID: cast<AtomicCmpXchgInst>(Val: I).getSyncScopeID()));
3711 Vals.push_back(
3712 Elt: getEncodedOrdering(Ordering: cast<AtomicCmpXchgInst>(Val: I).getFailureOrdering()));
3713 Vals.push_back(Elt: cast<AtomicCmpXchgInst>(Val: I).isWeak());
3714 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<AtomicCmpXchgInst>(Val: I).getAlign()));
3715 break;
3716 case Instruction::AtomicRMW:
3717 Code = bitc::FUNC_CODE_INST_ATOMICRMW;
3718 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals); // ptrty + ptr
3719 pushValueAndType(V: I.getOperand(i: 1), InstID, Vals); // valty + val
3720 Vals.push_back(Elt: getEncodedRMWOperation(I: cast<AtomicRMWInst>(Val: I)));
3721 Vals.push_back(Elt: cast<AtomicRMWInst>(Val: I).isVolatile());
3722 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<AtomicRMWInst>(Val: I).getOrdering()));
3723 Vals.push_back(
3724 Elt: getEncodedSyncScopeID(SSID: cast<AtomicRMWInst>(Val: I).getSyncScopeID()));
3725 Vals.push_back(Elt: getEncodedAlign(Alignment: cast<AtomicRMWInst>(Val: I).getAlign()));
3726 break;
3727 case Instruction::Fence:
3728 Code = bitc::FUNC_CODE_INST_FENCE;
3729 Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<FenceInst>(Val: I).getOrdering()));
3730 Vals.push_back(Elt: getEncodedSyncScopeID(SSID: cast<FenceInst>(Val: I).getSyncScopeID()));
3731 break;
3732 case Instruction::Call: {
3733 const CallInst &CI = cast<CallInst>(Val: I);
3734 FunctionType *FTy = CI.getFunctionType();
3735
3736 if (CI.hasOperandBundles())
3737 writeOperandBundles(CS: CI, InstID);
3738
3739 Code = bitc::FUNC_CODE_INST_CALL;
3740
3741 Vals.push_back(Elt: VE.getAttributeListID(PAL: CI.getAttributes()));
3742
3743 unsigned Flags = getOptimizationFlags(V: &I);
3744 Vals.push_back(Elt: CI.getCallingConv() << bitc::CALL_CCONV |
3745 unsigned(CI.isTailCall()) << bitc::CALL_TAIL |
3746 unsigned(CI.isMustTailCall()) << bitc::CALL_MUSTTAIL |
3747 1 << bitc::CALL_EXPLICIT_TYPE |
3748 unsigned(CI.isNoTailCall()) << bitc::CALL_NOTAIL |
3749 unsigned(Flags != 0) << bitc::CALL_FMF);
3750 if (Flags != 0)
3751 Vals.push_back(Elt: Flags);
3752
3753 Vals.push_back(Elt: VE.getTypeID(T: FTy));
3754 pushValueAndType(V: CI.getCalledOperand(), InstID, Vals); // Callee
3755
3756 // Emit value #'s for the fixed parameters.
3757 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3758 pushValue(V: CI.getArgOperand(i), InstID, Vals); // fixed param.
3759
3760 // Emit type/value pairs for varargs params.
3761 if (FTy->isVarArg()) {
3762 for (unsigned i = FTy->getNumParams(), e = CI.arg_size(); i != e; ++i)
3763 pushValueAndType(V: CI.getArgOperand(i), InstID, Vals); // varargs
3764 }
3765 break;
3766 }
3767 case Instruction::VAArg:
3768 Code = bitc::FUNC_CODE_INST_VAARG;
3769 Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType())); // valistty
3770 pushValue(V: I.getOperand(i: 0), InstID, Vals); // valist.
3771 Vals.push_back(Elt: VE.getTypeID(T: I.getType())); // restype.
3772 break;
3773 case Instruction::Freeze:
3774 Code = bitc::FUNC_CODE_INST_FREEZE;
3775 pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3776 break;
3777 }
3778
3779 Stream.EmitRecord(Code, Vals, Abbrev: AbbrevToUse);
3780 Vals.clear();
3781}
3782
3783/// Write a GlobalValue VST to the module. The purpose of this data structure is
3784/// to allow clients to efficiently find the function body.
3785void ModuleBitcodeWriter::writeGlobalValueSymbolTable(
3786 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3787 // Get the offset of the VST we are writing, and backpatch it into
3788 // the VST forward declaration record.
3789 uint64_t VSTOffset = Stream.GetCurrentBitNo();
3790 // The BitcodeStartBit was the stream offset of the identification block.
3791 VSTOffset -= bitcodeStartBit();
3792 assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned");
3793 // Note that we add 1 here because the offset is relative to one word
3794 // before the start of the identification block, which was historically
3795 // always the start of the regular bitcode header.
3796 Stream.BackpatchWord(BitNo: VSTOffsetPlaceholder, Val: VSTOffset / 32 + 1);
3797
3798 Stream.EnterSubblock(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, CodeLen: 4);
3799
3800 auto Abbv = std::make_shared<BitCodeAbbrev>();
3801 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY));
3802 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
3803 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset
3804 unsigned FnEntryAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
3805
3806 for (const Function &F : M) {
3807 uint64_t Record[2];
3808
3809 if (F.isDeclaration())
3810 continue;
3811
3812 Record[0] = VE.getValueID(V: &F);
3813
3814 // Save the word offset of the function (from the start of the
3815 // actual bitcode written to the stream).
3816 uint64_t BitcodeIndex = FunctionToBitcodeIndex[&F] - bitcodeStartBit();
3817 assert((BitcodeIndex & 31) == 0 && "function block not 32-bit aligned");
3818 // Note that we add 1 here because the offset is relative to one word
3819 // before the start of the identification block, which was historically
3820 // always the start of the regular bitcode header.
3821 Record[1] = BitcodeIndex / 32 + 1;
3822
3823 Stream.EmitRecord(Code: bitc::VST_CODE_FNENTRY, Vals: Record, Abbrev: FnEntryAbbrev);
3824 }
3825
3826 Stream.ExitBlock();
3827}
3828
3829/// Emit names for arguments, instructions and basic blocks in a function.
3830void ModuleBitcodeWriter::writeFunctionLevelValueSymbolTable(
3831 const ValueSymbolTable &VST) {
3832 if (VST.empty())
3833 return;
3834
3835 Stream.EnterSubblock(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, CodeLen: 4);
3836
3837 // FIXME: Set up the abbrev, we know how many values there are!
3838 // FIXME: We know if the type names can use 7-bit ascii.
3839 SmallVector<uint64_t, 64> NameVals;
3840
3841 for (const ValueName &Name : VST) {
3842 // Figure out the encoding to use for the name.
3843 StringEncoding Bits = getStringEncoding(Str: Name.getKey());
3844
3845 unsigned AbbrevToUse = VST_ENTRY_8_ABBREV;
3846 NameVals.push_back(Elt: VE.getValueID(V: Name.getValue()));
3847
3848 // VST_CODE_ENTRY: [valueid, namechar x N]
3849 // VST_CODE_BBENTRY: [bbid, namechar x N]
3850 unsigned Code;
3851 if (isa<BasicBlock>(Val: Name.getValue())) {
3852 Code = bitc::VST_CODE_BBENTRY;
3853 if (Bits == SE_Char6)
3854 AbbrevToUse = VST_BBENTRY_6_ABBREV;
3855 } else {
3856 Code = bitc::VST_CODE_ENTRY;
3857 if (Bits == SE_Char6)
3858 AbbrevToUse = VST_ENTRY_6_ABBREV;
3859 else if (Bits == SE_Fixed7)
3860 AbbrevToUse = VST_ENTRY_7_ABBREV;
3861 }
3862
3863 for (const auto P : Name.getKey())
3864 NameVals.push_back(Elt: (unsigned char)P);
3865
3866 // Emit the finished record.
3867 Stream.EmitRecord(Code, Vals: NameVals, Abbrev: AbbrevToUse);
3868 NameVals.clear();
3869 }
3870
3871 Stream.ExitBlock();
3872}
3873
3874void ModuleBitcodeWriter::writeUseList(UseListOrder &&Order) {
3875 assert(Order.Shuffle.size() >= 2 && "Shuffle too small");
3876 unsigned Code;
3877 if (isa<BasicBlock>(Val: Order.V))
3878 Code = bitc::USELIST_CODE_BB;
3879 else
3880 Code = bitc::USELIST_CODE_DEFAULT;
3881
3882 SmallVector<uint64_t, 64> Record(Order.Shuffle.begin(), Order.Shuffle.end());
3883 Record.push_back(Elt: VE.getValueID(V: Order.V));
3884 Stream.EmitRecord(Code, Vals: Record);
3885}
3886
3887void ModuleBitcodeWriter::writeUseListBlock(const Function *F) {
3888 assert(VE.shouldPreserveUseListOrder() &&
3889 "Expected to be preserving use-list order");
3890
3891 auto hasMore = [&]() {
3892 return !VE.UseListOrders.empty() && VE.UseListOrders.back().F == F;
3893 };
3894 if (!hasMore())
3895 // Nothing to do.
3896 return;
3897
3898 Stream.EnterSubblock(BlockID: bitc::USELIST_BLOCK_ID, CodeLen: 3);
3899 while (hasMore()) {
3900 writeUseList(Order: std::move(VE.UseListOrders.back()));
3901 VE.UseListOrders.pop_back();
3902 }
3903 Stream.ExitBlock();
3904}
3905
3906/// Emit a function body to the module stream.
3907void ModuleBitcodeWriter::writeFunction(
3908 const Function &F,
3909 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3910 // Save the bitcode index of the start of this function block for recording
3911 // in the VST.
3912 FunctionToBitcodeIndex[&F] = Stream.GetCurrentBitNo();
3913
3914 Stream.EnterSubblock(BlockID: bitc::FUNCTION_BLOCK_ID, CodeLen: 5);
3915 VE.incorporateFunction(F);
3916
3917 SmallVector<unsigned, 64> Vals;
3918
3919 // Emit the number of basic blocks, so the reader can create them ahead of
3920 // time.
3921 Vals.push_back(Elt: VE.getBasicBlocks().size());
3922 Stream.EmitRecord(Code: bitc::FUNC_CODE_DECLAREBLOCKS, Vals);
3923 Vals.clear();
3924
3925 // If there are function-local constants, emit them now.
3926 unsigned CstStart, CstEnd;
3927 VE.getFunctionConstantRange(Start&: CstStart, End&: CstEnd);
3928 writeConstants(FirstVal: CstStart, LastVal: CstEnd, isGlobal: false);
3929
3930 // If there is function-local metadata, emit it now.
3931 writeFunctionMetadata(F);
3932
3933 // Keep a running idea of what the instruction ID is.
3934 unsigned InstID = CstEnd;
3935
3936 bool NeedsMetadataAttachment = F.hasMetadata();
3937
3938 DILocation *LastDL = nullptr;
3939 SmallSetVector<Function *, 4> BlockAddressUsers;
3940
3941 // Finally, emit all the instructions, in order.
3942 for (const BasicBlock &BB : F) {
3943 for (const Instruction &I : BB) {
3944 writeInstruction(I, InstID, Vals);
3945
3946 if (!I.getType()->isVoidTy())
3947 ++InstID;
3948
3949 // If the instruction has metadata, write a metadata attachment later.
3950 NeedsMetadataAttachment |= I.hasMetadataOtherThanDebugLoc();
3951
3952 // If the instruction has a debug location, emit it.
3953 if (DILocation *DL = I.getDebugLoc()) {
3954 if (DL == LastDL) {
3955 // Just repeat the same debug loc as last time.
3956 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_LOC_AGAIN, Vals);
3957 } else {
3958 Vals.push_back(Elt: DL->getLine());
3959 Vals.push_back(Elt: DL->getColumn());
3960 Vals.push_back(Elt: VE.getMetadataOrNullID(MD: DL->getScope()));
3961 Vals.push_back(Elt: VE.getMetadataOrNullID(MD: DL->getInlinedAt()));
3962 Vals.push_back(Elt: DL->isImplicitCode());
3963 Vals.push_back(Elt: DL->getAtomGroup());
3964 Vals.push_back(Elt: DL->getAtomRank());
3965
3966 unsigned DLAbbrev = FUNCTION_DEBUG_LOC_ABBREV;
3967 if (DILayerLocList *IRLayers = DL->getIRLayers()) {
3968 DLAbbrev = FUNCTION_DEBUG_LOC_LAYERS_ABBREV;
3969 Vals.push_back(Elt: VE.getMetadataOrNullID(MD: IRLayers));
3970 }
3971
3972 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_LOC, Vals, Abbrev: DLAbbrev);
3973 Vals.clear();
3974 LastDL = DL;
3975 }
3976 }
3977
3978 // If the instruction has DbgRecords attached to it, emit them. Note that
3979 // they come after the instruction so that it's easy to attach them again
3980 // when reading the bitcode, even though conceptually the debug locations
3981 // start "before" the instruction.
3982 if (I.hasDbgRecords()) {
3983 /// Try to push the value only (unwrapped), otherwise push the
3984 /// metadata wrapped value. Returns true if the value was pushed
3985 /// without the ValueAsMetadata wrapper.
3986 auto PushValueOrMetadata = [&Vals, InstID,
3987 this](Metadata *RawLocation) {
3988 assert(RawLocation &&
3989 "RawLocation unexpectedly null in DbgVariableRecord");
3990 if (ValueAsMetadata *VAM = dyn_cast<ValueAsMetadata>(Val: RawLocation)) {
3991 SmallVector<unsigned, 2> ValAndType;
3992 // If the value is a fwd-ref the type is also pushed. We don't
3993 // want the type, so fwd-refs are kept wrapped (pushValueAndType
3994 // returns false if the value is pushed without type).
3995 if (!pushValueAndType(V: VAM->getValue(), InstID, Vals&: ValAndType)) {
3996 Vals.push_back(Elt: ValAndType[0]);
3997 return true;
3998 }
3999 }
4000 // The metadata is a DIArgList, or ValueAsMetadata wrapping a
4001 // fwd-ref. Push the metadata ID.
4002 Vals.push_back(Elt: VE.getMetadataID(MD: RawLocation));
4003 return false;
4004 };
4005
4006 // Write out non-instruction debug information attached to this
4007 // instruction. Write it after the instruction so that it's easy to
4008 // re-attach to the instruction reading the records in.
4009 for (DbgRecord &DR : I.getDbgMarker()->getDbgRecordRange()) {
4010 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
4011 Vals.push_back(Elt: VE.getMetadataID(MD: &*DLR->getDebugLoc()));
4012 Vals.push_back(Elt: VE.getMetadataID(MD: DLR->getLabel()));
4013 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_LABEL, Vals);
4014 Vals.clear();
4015 continue;
4016 }
4017
4018 // First 3 fields are common to all kinds:
4019 // DILocation, DILocalVariable, DIExpression
4020 // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE)
4021 // ..., LocationMetadata
4022 // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE - abbrev'd)
4023 // ..., Value
4024 // dbg_declare (FUNC_CODE_DEBUG_RECORD_DECLARE)
4025 // ..., LocationMetadata
4026 // dbg_assign (FUNC_CODE_DEBUG_RECORD_ASSIGN)
4027 // ..., LocationMetadata, DIAssignID, DIExpression, LocationMetadata
4028 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
4029 Vals.push_back(Elt: VE.getMetadataID(MD: &*DVR.getDebugLoc()));
4030 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getVariable()));
4031 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getExpression()));
4032 if (DVR.isDbgValue()) {
4033 if (PushValueOrMetadata(DVR.getRawLocation()))
4034 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE, Vals,
4035 Abbrev: FUNCTION_DEBUG_RECORD_VALUE_ABBREV);
4036 else
4037 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_VALUE, Vals);
4038 } else if (DVR.isDbgDeclare()) {
4039 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
4040 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_DECLARE, Vals);
4041 } else if (DVR.isDbgDeclareValue()) {
4042 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
4043 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_DECLARE_VALUE, Vals);
4044 } else {
4045 assert(DVR.isDbgAssign() && "Unexpected DbgRecord kind");
4046 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
4047 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getAssignID()));
4048 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getAddressExpression()));
4049 Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawAddress()));
4050 Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_ASSIGN, Vals);
4051 }
4052 Vals.clear();
4053 }
4054 }
4055 }
4056
4057 if (BlockAddress *BA = BlockAddress::lookup(BB: &BB)) {
4058 SmallVector<Value *> Worklist{BA};
4059 SmallPtrSet<Value *, 8> Visited{BA};
4060 while (!Worklist.empty()) {
4061 Value *V = Worklist.pop_back_val();
4062 for (User *U : V->users()) {
4063 if (auto *I = dyn_cast<Instruction>(Val: U)) {
4064 Function *P = I->getFunction();
4065 if (P != &F)
4066 BlockAddressUsers.insert(X: P);
4067 } else if (isa<Constant>(Val: U) && !isa<GlobalValue>(Val: U) &&
4068 Visited.insert(Ptr: U).second)
4069 Worklist.push_back(Elt: U);
4070 }
4071 }
4072 }
4073 }
4074
4075 if (!BlockAddressUsers.empty()) {
4076 Vals.resize(N: BlockAddressUsers.size());
4077 for (auto I : llvm::enumerate(First&: BlockAddressUsers))
4078 Vals[I.index()] = VE.getValueID(V: I.value());
4079 Stream.EmitRecord(Code: bitc::FUNC_CODE_BLOCKADDR_USERS, Vals);
4080 Vals.clear();
4081 }
4082
4083 // Emit names for all the instructions etc.
4084 if (auto *Symtab = F.getValueSymbolTable())
4085 writeFunctionLevelValueSymbolTable(VST: *Symtab);
4086
4087 if (NeedsMetadataAttachment)
4088 writeFunctionMetadataAttachment(F);
4089 if (VE.shouldPreserveUseListOrder())
4090 writeUseListBlock(F: &F);
4091 VE.purgeFunction();
4092 Stream.ExitBlock();
4093}
4094
4095// Emit blockinfo, which defines the standard abbreviations etc.
4096void ModuleBitcodeWriter::writeBlockInfo() {
4097 // We only want to emit block info records for blocks that have multiple
4098 // instances: CONSTANTS_BLOCK, FUNCTION_BLOCK and VALUE_SYMTAB_BLOCK.
4099 // Other blocks can define their abbrevs inline.
4100 Stream.EnterBlockInfoBlock();
4101
4102 // Encode type indices using fixed size based on number of types.
4103 BitCodeAbbrevOp TypeAbbrevOp(BitCodeAbbrevOp::Fixed,
4104 VE.computeBitsRequiredForTypeIndices());
4105 // Encode value indices as 6-bit VBR.
4106 BitCodeAbbrevOp ValAbbrevOp(BitCodeAbbrevOp::VBR, 6);
4107
4108 { // 8-bit fixed-width VST_CODE_ENTRY/VST_CODE_BBENTRY strings.
4109 auto Abbv = std::make_shared<BitCodeAbbrev>();
4110 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3));
4111 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4112 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4113 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
4114 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
4115 VST_ENTRY_8_ABBREV)
4116 llvm_unreachable("Unexpected abbrev ordering!");
4117 }
4118
4119 { // 7-bit fixed width VST_CODE_ENTRY strings.
4120 auto Abbv = std::make_shared<BitCodeAbbrev>();
4121 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
4122 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4123 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4124 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
4125 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
4126 VST_ENTRY_7_ABBREV)
4127 llvm_unreachable("Unexpected abbrev ordering!");
4128 }
4129 { // 6-bit char6 VST_CODE_ENTRY strings.
4130 auto Abbv = std::make_shared<BitCodeAbbrev>();
4131 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
4132 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4133 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4134 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
4135 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
4136 VST_ENTRY_6_ABBREV)
4137 llvm_unreachable("Unexpected abbrev ordering!");
4138 }
4139 { // 6-bit char6 VST_CODE_BBENTRY strings.
4140 auto Abbv = std::make_shared<BitCodeAbbrev>();
4141 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_BBENTRY));
4142 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4143 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4144 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
4145 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
4146 VST_BBENTRY_6_ABBREV)
4147 llvm_unreachable("Unexpected abbrev ordering!");
4148 }
4149
4150 { // SETTYPE abbrev for CONSTANTS_BLOCK.
4151 auto Abbv = std::make_shared<BitCodeAbbrev>();
4152 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_SETTYPE));
4153 Abbv->Add(OpInfo: TypeAbbrevOp);
4154 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4155 CONSTANTS_SETTYPE_ABBREV)
4156 llvm_unreachable("Unexpected abbrev ordering!");
4157 }
4158
4159 { // INTEGER abbrev for CONSTANTS_BLOCK.
4160 auto Abbv = std::make_shared<BitCodeAbbrev>();
4161 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_INTEGER));
4162 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4163 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4164 CONSTANTS_INTEGER_ABBREV)
4165 llvm_unreachable("Unexpected abbrev ordering!");
4166 }
4167
4168 { // BYTE abbrev for CONSTANTS_BLOCK.
4169 auto Abbv = std::make_shared<BitCodeAbbrev>();
4170 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_BYTE));
4171 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4172 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4173 CONSTANTS_BYTE_ABBREV)
4174 llvm_unreachable("Unexpected abbrev ordering!");
4175 }
4176
4177 { // CE_CAST abbrev for CONSTANTS_BLOCK.
4178 auto Abbv = std::make_shared<BitCodeAbbrev>();
4179 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CE_CAST));
4180 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // cast opc
4181 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // typeid
4182 VE.computeBitsRequiredForTypeIndices()));
4183 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
4184
4185 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4186 CONSTANTS_CE_CAST_Abbrev)
4187 llvm_unreachable("Unexpected abbrev ordering!");
4188 }
4189 { // NULL abbrev for CONSTANTS_BLOCK.
4190 auto Abbv = std::make_shared<BitCodeAbbrev>();
4191 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_NULL));
4192 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
4193 CONSTANTS_NULL_Abbrev)
4194 llvm_unreachable("Unexpected abbrev ordering!");
4195 }
4196
4197 // FIXME: This should only use space for first class types!
4198
4199 { // INST_LOAD abbrev for FUNCTION_BLOCK.
4200 auto Abbv = std::make_shared<BitCodeAbbrev>();
4201 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_LOAD));
4202 Abbv->Add(OpInfo: ValAbbrevOp); // Ptr
4203 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4204 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Align
4205 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
4206 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4207 FUNCTION_INST_LOAD_ABBREV)
4208 llvm_unreachable("Unexpected abbrev ordering!");
4209 }
4210 {
4211 auto Abbv = std::make_shared<BitCodeAbbrev>();
4212 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_STORE));
4213 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4214 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4215 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // align
4216 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
4217 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4218 FUNCTION_INST_STORE_ABBREV)
4219 llvm_unreachable("Unexpected abbrev ordering!");
4220 }
4221 { // INST_UNOP abbrev for FUNCTION_BLOCK.
4222 auto Abbv = std::make_shared<BitCodeAbbrev>();
4223 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
4224 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4225 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4226 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4227 FUNCTION_INST_UNOP_ABBREV)
4228 llvm_unreachable("Unexpected abbrev ordering!");
4229 }
4230 { // INST_UNOP_FLAGS abbrev for FUNCTION_BLOCK.
4231 auto Abbv = std::make_shared<BitCodeAbbrev>();
4232 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
4233 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4234 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4235 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4236 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4237 FUNCTION_INST_UNOP_FLAGS_ABBREV)
4238 llvm_unreachable("Unexpected abbrev ordering!");
4239 }
4240 { // INST_BINOP abbrev for FUNCTION_BLOCK.
4241 auto Abbv = std::make_shared<BitCodeAbbrev>();
4242 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
4243 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4244 Abbv->Add(OpInfo: ValAbbrevOp); // RHS
4245 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4246 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4247 FUNCTION_INST_BINOP_ABBREV)
4248 llvm_unreachable("Unexpected abbrev ordering!");
4249 }
4250 { // INST_BINOP_FLAGS abbrev for FUNCTION_BLOCK.
4251 auto Abbv = std::make_shared<BitCodeAbbrev>();
4252 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
4253 Abbv->Add(OpInfo: ValAbbrevOp); // LHS
4254 Abbv->Add(OpInfo: ValAbbrevOp); // RHS
4255 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4256 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4257 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4258 FUNCTION_INST_BINOP_FLAGS_ABBREV)
4259 llvm_unreachable("Unexpected abbrev ordering!");
4260 }
4261 { // INST_CAST abbrev for FUNCTION_BLOCK.
4262 auto Abbv = std::make_shared<BitCodeAbbrev>();
4263 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
4264 Abbv->Add(OpInfo: ValAbbrevOp); // OpVal
4265 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4266 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4267 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4268 FUNCTION_INST_CAST_ABBREV)
4269 llvm_unreachable("Unexpected abbrev ordering!");
4270 }
4271 { // INST_CAST_FLAGS abbrev for FUNCTION_BLOCK.
4272 auto Abbv = std::make_shared<BitCodeAbbrev>();
4273 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
4274 Abbv->Add(OpInfo: ValAbbrevOp); // OpVal
4275 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4276 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
4277 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 9)); // flags
4278 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4279 FUNCTION_INST_CAST_FLAGS_ABBREV)
4280 llvm_unreachable("Unexpected abbrev ordering!");
4281 }
4282
4283 { // INST_RET abbrev for FUNCTION_BLOCK.
4284 auto Abbv = std::make_shared<BitCodeAbbrev>();
4285 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
4286 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4287 FUNCTION_INST_RET_VOID_ABBREV)
4288 llvm_unreachable("Unexpected abbrev ordering!");
4289 }
4290 { // INST_RET abbrev for FUNCTION_BLOCK.
4291 auto Abbv = std::make_shared<BitCodeAbbrev>();
4292 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
4293 Abbv->Add(OpInfo: ValAbbrevOp);
4294 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4295 FUNCTION_INST_RET_VAL_ABBREV)
4296 llvm_unreachable("Unexpected abbrev ordering!");
4297 }
4298 {
4299 auto Abbv = std::make_shared<BitCodeAbbrev>();
4300 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BR));
4301 // TODO: Use different abbrev for absolute value reference (succ0)?
4302 Abbv->Add(OpInfo: ValAbbrevOp); // succ0
4303 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4304 FUNCTION_INST_BR_UNCOND_ABBREV)
4305 llvm_unreachable("Unexpected abbrev ordering!");
4306 }
4307 {
4308 auto Abbv = std::make_shared<BitCodeAbbrev>();
4309 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BR));
4310 // TODO: Use different abbrev for absolute value references (succ0, succ1)?
4311 Abbv->Add(OpInfo: ValAbbrevOp); // succ0
4312 Abbv->Add(OpInfo: ValAbbrevOp); // succ1
4313 Abbv->Add(OpInfo: ValAbbrevOp); // cond
4314 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4315 FUNCTION_INST_BR_COND_ABBREV)
4316 llvm_unreachable("Unexpected abbrev ordering!");
4317 }
4318 { // INST_UNREACHABLE abbrev for FUNCTION_BLOCK.
4319 auto Abbv = std::make_shared<BitCodeAbbrev>();
4320 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNREACHABLE));
4321 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4322 FUNCTION_INST_UNREACHABLE_ABBREV)
4323 llvm_unreachable("Unexpected abbrev ordering!");
4324 }
4325 {
4326 auto Abbv = std::make_shared<BitCodeAbbrev>();
4327 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_GEP));
4328 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // flags
4329 Abbv->Add(OpInfo: TypeAbbrevOp); // dest ty
4330 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4331 Abbv->Add(OpInfo: ValAbbrevOp);
4332 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4333 FUNCTION_INST_GEP_ABBREV)
4334 llvm_unreachable("Unexpected abbrev ordering!");
4335 }
4336 {
4337 auto Abbv = std::make_shared<BitCodeAbbrev>();
4338 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CMP2));
4339 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4340 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4341 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // pred
4342 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4343 FUNCTION_INST_CMP_ABBREV)
4344 llvm_unreachable("Unexpected abbrev ordering!");
4345 }
4346 {
4347 auto Abbv = std::make_shared<BitCodeAbbrev>();
4348 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CMP2));
4349 Abbv->Add(OpInfo: ValAbbrevOp); // op0
4350 Abbv->Add(OpInfo: ValAbbrevOp); // op1
4351 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // pred
4352 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
4353 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4354 FUNCTION_INST_CMP_FLAGS_ABBREV)
4355 llvm_unreachable("Unexpected abbrev ordering!");
4356 }
4357 {
4358 auto Abbv = std::make_shared<BitCodeAbbrev>();
4359 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE));
4360 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // dbgloc
4361 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // var
4362 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // expr
4363 Abbv->Add(OpInfo: ValAbbrevOp); // val
4364 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4365 FUNCTION_DEBUG_RECORD_VALUE_ABBREV)
4366 llvm_unreachable("Unexpected abbrev ordering! 1");
4367 }
4368 {
4369 auto Abbv = std::make_shared<BitCodeAbbrev>();
4370 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_LOC));
4371 // NOTE: No IsDistinct field for FUNC_CODE_DEBUG_LOC.
4372 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4373 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4374 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4375 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4376 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
4377 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Atom group.
4378 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 3)); // Atom rank.
4379 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4380 FUNCTION_DEBUG_LOC_ABBREV)
4381 llvm_unreachable("Unexpected abbrev ordering!");
4382 }
4383 { // DEBUG_LOC abbrev for FUNCTION_BLOCK, irlayers variant.
4384 // Separate abbrev so a location without layers does not spend a VBR chunk
4385 // encoding a zero; writeInstruction picks between the two per record.
4386 auto Abbv = std::make_shared<BitCodeAbbrev>();
4387 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_LOC));
4388 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4389 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4390 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4391 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4392 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
4393 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Atom group.
4394 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 3)); // Atom rank.
4395 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // irlayers.
4396 if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
4397 FUNCTION_DEBUG_LOC_LAYERS_ABBREV)
4398 llvm_unreachable("Unexpected abbrev ordering!");
4399 }
4400 Stream.ExitBlock();
4401}
4402
4403/// Write the module path strings, currently only used when generating
4404/// a combined index file.
4405void IndexBitcodeWriter::writeModStrings() {
4406 Stream.EnterSubblock(BlockID: bitc::MODULE_STRTAB_BLOCK_ID, CodeLen: 3);
4407
4408 // TODO: See which abbrev sizes we actually need to emit
4409
4410 // 8-bit fixed-width MST_ENTRY strings.
4411 auto Abbv = std::make_shared<BitCodeAbbrev>();
4412 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4413 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4414 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4415 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
4416 unsigned Abbrev8Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4417
4418 // 7-bit fixed width MST_ENTRY strings.
4419 Abbv = std::make_shared<BitCodeAbbrev>();
4420 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4421 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4422 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4423 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
4424 unsigned Abbrev7Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4425
4426 // 6-bit char6 MST_ENTRY strings.
4427 Abbv = std::make_shared<BitCodeAbbrev>();
4428 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4429 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4430 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4431 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
4432 unsigned Abbrev6Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4433
4434 // Module Hash, 160 bits SHA1. Optionally, emitted after each MST_CODE_ENTRY.
4435 Abbv = std::make_shared<BitCodeAbbrev>();
4436 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_HASH));
4437 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4438 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4439 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4440 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4441 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4442 unsigned AbbrevHash = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4443
4444 SmallVector<unsigned, 64> Vals;
4445 forEachModule(Callback: [&](const StringMapEntry<ModuleHash> &MPSE) {
4446 StringRef Key = MPSE.getKey();
4447 const auto &Hash = MPSE.getValue();
4448 StringEncoding Bits = getStringEncoding(Str: Key);
4449 unsigned AbbrevToUse = Abbrev8Bit;
4450 if (Bits == SE_Char6)
4451 AbbrevToUse = Abbrev6Bit;
4452 else if (Bits == SE_Fixed7)
4453 AbbrevToUse = Abbrev7Bit;
4454
4455 auto ModuleId = ModuleIdMap.size();
4456 ModuleIdMap[Key] = ModuleId;
4457 Vals.push_back(Elt: ModuleId);
4458 // Use bytes_begin/end() for unsigned char iteration.
4459 Vals.append(in_start: Key.bytes_begin(), in_end: Key.bytes_end());
4460
4461 // Emit the finished record.
4462 Stream.EmitRecord(Code: bitc::MST_CODE_ENTRY, Vals, Abbrev: AbbrevToUse);
4463
4464 // Emit an optional hash for the module now
4465 if (llvm::any_of(Range: Hash, P: [](uint32_t H) { return H; })) {
4466 Vals.assign(in_start: Hash.begin(), in_end: Hash.end());
4467 // Emit the hash record.
4468 Stream.EmitRecord(Code: bitc::MST_CODE_HASH, Vals, Abbrev: AbbrevHash);
4469 }
4470
4471 Vals.clear();
4472 });
4473 Stream.ExitBlock();
4474}
4475
4476/// Write the function type metadata related records that need to appear before
4477/// a function summary entry (whether per-module or combined).
4478template <typename Fn>
4479static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream,
4480 FunctionSummary *FS,
4481 Fn GetValueID) {
4482 if (!FS->type_tests().empty())
4483 Stream.EmitRecord(Code: bitc::FS_TYPE_TESTS, Vals: FS->type_tests());
4484
4485 SmallVector<uint64_t, 64> Record;
4486
4487 auto WriteVFuncIdVec = [&](uint64_t Ty,
4488 ArrayRef<FunctionSummary::VFuncId> VFs) {
4489 if (VFs.empty())
4490 return;
4491 Record.clear();
4492 for (auto &VF : VFs) {
4493 Record.push_back(Elt: VF.GUID);
4494 Record.push_back(Elt: VF.Offset);
4495 }
4496 Stream.EmitRecord(Code: Ty, Vals: Record);
4497 };
4498
4499 WriteVFuncIdVec(bitc::FS_TYPE_TEST_ASSUME_VCALLS,
4500 FS->type_test_assume_vcalls());
4501 WriteVFuncIdVec(bitc::FS_TYPE_CHECKED_LOAD_VCALLS,
4502 FS->type_checked_load_vcalls());
4503
4504 auto WriteConstVCallVec = [&](uint64_t Ty,
4505 ArrayRef<FunctionSummary::ConstVCall> VCs) {
4506 for (auto &VC : VCs) {
4507 Record.clear();
4508 Record.push_back(Elt: VC.VFunc.GUID);
4509 Record.push_back(Elt: VC.VFunc.Offset);
4510 llvm::append_range(C&: Record, R: VC.Args);
4511 Stream.EmitRecord(Code: Ty, Vals: Record);
4512 }
4513 };
4514
4515 WriteConstVCallVec(bitc::FS_TYPE_TEST_ASSUME_CONST_VCALL,
4516 FS->type_test_assume_const_vcalls());
4517 WriteConstVCallVec(bitc::FS_TYPE_CHECKED_LOAD_CONST_VCALL,
4518 FS->type_checked_load_const_vcalls());
4519
4520 auto WriteRange = [&](ConstantRange Range) {
4521 Range = Range.sextOrTrunc(BitWidth: FunctionSummary::ParamAccess::RangeWidth);
4522 assert(Range.getLower().getNumWords() == 1);
4523 assert(Range.getUpper().getNumWords() == 1);
4524 emitSignedInt64(Vals&: Record, V: *Range.getLower().getRawData());
4525 emitSignedInt64(Vals&: Record, V: *Range.getUpper().getRawData());
4526 };
4527
4528 if (!FS->paramAccesses().empty()) {
4529 Record.clear();
4530 for (auto &Arg : FS->paramAccesses()) {
4531 size_t UndoSize = Record.size();
4532 Record.push_back(Elt: Arg.ParamNo);
4533 WriteRange(Arg.Use);
4534 Record.push_back(Elt: Arg.Calls.size());
4535 for (auto &Call : Arg.Calls) {
4536 Record.push_back(Elt: Call.ParamNo);
4537 std::optional<unsigned> ValueID = GetValueID(Call.Callee);
4538 if (!ValueID) {
4539 // If ValueID is unknown we can't drop just this call, we must drop
4540 // entire parameter.
4541 Record.resize(N: UndoSize);
4542 break;
4543 }
4544 Record.push_back(Elt: *ValueID);
4545 WriteRange(Call.Offsets);
4546 }
4547 }
4548 if (!Record.empty())
4549 Stream.EmitRecord(Code: bitc::FS_PARAM_ACCESS, Vals: Record);
4550 }
4551}
4552
4553/// Collect type IDs from type tests used by function.
4554static void
4555getReferencedTypeIds(FunctionSummary *FS,
4556 std::set<GlobalValue::GUID> &ReferencedTypeIds) {
4557 if (!FS->type_tests().empty())
4558 for (auto &TT : FS->type_tests())
4559 ReferencedTypeIds.insert(x: TT);
4560
4561 auto GetReferencedTypesFromVFuncIdVec =
4562 [&](ArrayRef<FunctionSummary::VFuncId> VFs) {
4563 for (auto &VF : VFs)
4564 ReferencedTypeIds.insert(x: VF.GUID);
4565 };
4566
4567 GetReferencedTypesFromVFuncIdVec(FS->type_test_assume_vcalls());
4568 GetReferencedTypesFromVFuncIdVec(FS->type_checked_load_vcalls());
4569
4570 auto GetReferencedTypesFromConstVCallVec =
4571 [&](ArrayRef<FunctionSummary::ConstVCall> VCs) {
4572 for (auto &VC : VCs)
4573 ReferencedTypeIds.insert(x: VC.VFunc.GUID);
4574 };
4575
4576 GetReferencedTypesFromConstVCallVec(FS->type_test_assume_const_vcalls());
4577 GetReferencedTypesFromConstVCallVec(FS->type_checked_load_const_vcalls());
4578}
4579
4580static void writeWholeProgramDevirtResolutionByArg(
4581 SmallVector<uint64_t, 64> &NameVals, const std::vector<uint64_t> &args,
4582 const WholeProgramDevirtResolution::ByArg &ByArg) {
4583 NameVals.push_back(Elt: args.size());
4584 llvm::append_range(C&: NameVals, R: args);
4585
4586 NameVals.push_back(Elt: ByArg.TheKind);
4587 NameVals.push_back(Elt: ByArg.Info);
4588 NameVals.push_back(Elt: ByArg.Byte);
4589 NameVals.push_back(Elt: ByArg.Bit);
4590}
4591
4592static void writeWholeProgramDevirtResolution(
4593 SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
4594 uint64_t Id, const WholeProgramDevirtResolution &Wpd) {
4595 NameVals.push_back(Elt: Id);
4596
4597 NameVals.push_back(Elt: Wpd.TheKind);
4598 NameVals.push_back(Elt: StrtabBuilder.add(S: Wpd.SingleImplName));
4599 NameVals.push_back(Elt: Wpd.SingleImplName.size());
4600
4601 NameVals.push_back(Elt: Wpd.ResByArg.size());
4602 for (auto &A : Wpd.ResByArg)
4603 writeWholeProgramDevirtResolutionByArg(NameVals, args: A.first, ByArg: A.second);
4604}
4605
4606static void writeTypeIdSummaryRecord(SmallVector<uint64_t, 64> &NameVals,
4607 StringTableBuilder &StrtabBuilder,
4608 StringRef Id,
4609 const TypeIdSummary &Summary) {
4610 NameVals.push_back(Elt: StrtabBuilder.add(S: Id));
4611 NameVals.push_back(Elt: Id.size());
4612
4613 NameVals.push_back(Elt: Summary.TTRes.TheKind);
4614 NameVals.push_back(Elt: Summary.TTRes.SizeM1BitWidth);
4615 NameVals.push_back(Elt: Summary.TTRes.AlignLog2);
4616 NameVals.push_back(Elt: Summary.TTRes.SizeM1);
4617 NameVals.push_back(Elt: Summary.TTRes.BitMask);
4618 NameVals.push_back(Elt: Summary.TTRes.InlineBits);
4619
4620 for (auto &W : Summary.WPDRes)
4621 writeWholeProgramDevirtResolution(NameVals, StrtabBuilder, Id: W.first,
4622 Wpd: W.second);
4623}
4624
4625static void writeTypeIdCompatibleVtableSummaryRecord(
4626 SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
4627 StringRef Id, const TypeIdCompatibleVtableInfo &Summary,
4628 ValueEnumerator &VE) {
4629 NameVals.push_back(Elt: StrtabBuilder.add(S: Id));
4630 NameVals.push_back(Elt: Id.size());
4631
4632 for (auto &P : Summary) {
4633 NameVals.push_back(Elt: P.AddressPointOffset);
4634 NameVals.push_back(Elt: VE.getValueID(V: P.VTableVI.getValue()));
4635 }
4636}
4637
4638// Adds the allocation contexts to the CallStacks map. We simply use the
4639// size at the time the context was added as the CallStackId. This works because
4640// when we look up the call stacks later on we process the function summaries
4641// and their allocation records in the same exact order.
4642static void collectMemProfCallStacks(
4643 FunctionSummary *FS, std::function<LinearFrameId(unsigned)> GetStackIndex,
4644 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> &CallStacks) {
4645 // The interfaces in ProfileData/MemProf.h use a type alias for a stack frame
4646 // id offset into the index of the full stack frames. The ModuleSummaryIndex
4647 // currently uses unsigned. Make sure these stay in sync.
4648 static_assert(std::is_same_v<LinearFrameId, unsigned>);
4649 for (auto &AI : FS->allocs()) {
4650 for (auto &MIB : AI.MIBs) {
4651 SmallVector<unsigned> StackIdIndices;
4652 StackIdIndices.reserve(N: MIB.StackIdIndices.size());
4653 for (auto Id : MIB.StackIdIndices)
4654 StackIdIndices.push_back(Elt: GetStackIndex(Id));
4655 // The CallStackId is the size at the time this context was inserted.
4656 CallStacks.insert(KV: {CallStacks.size(), StackIdIndices});
4657 }
4658 }
4659}
4660
4661// Build the radix tree from the accumulated CallStacks, write out the resulting
4662// linearized radix tree array, and return the map of call stack positions into
4663// this array for use when writing the allocation records. The returned map is
4664// indexed by a CallStackId which in this case is implicitly determined by the
4665// order of function summaries and their allocation infos being written.
4666static DenseMap<CallStackId, LinearCallStackId> writeMemoryProfileRadixTree(
4667 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> &&CallStacks,
4668 BitstreamWriter &Stream, unsigned RadixAbbrev) {
4669 assert(!CallStacks.empty());
4670 DenseMap<unsigned, FrameStat> FrameHistogram =
4671 computeFrameHistogram<LinearFrameId>(MemProfCallStackData&: CallStacks);
4672 CallStackRadixTreeBuilder<LinearFrameId> Builder;
4673 // We don't need a MemProfFrameIndexes map as we have already converted the
4674 // full stack id hash to a linear offset into the StackIds array.
4675 Builder.build(MemProfCallStackData: std::move(CallStacks), /*MemProfFrameIndexes=*/nullptr,
4676 FrameHistogram);
4677 Stream.EmitRecord(Code: bitc::FS_CONTEXT_RADIX_TREE_ARRAY, Vals: Builder.getRadixArray(),
4678 Abbrev: RadixAbbrev);
4679 return Builder.takeCallStackPos();
4680}
4681
4682static void writeFunctionHeapProfileRecords(
4683 BitstreamWriter &Stream, FunctionSummary *FS, unsigned CallsiteAbbrev,
4684 unsigned AllocAbbrev, unsigned ContextIdAbbvId, bool PerModule,
4685 std::function<unsigned(const ValueInfo &VI)> GetValueID,
4686 std::function<unsigned(unsigned)> GetStackIndex,
4687 bool WriteContextSizeInfoIndex,
4688 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
4689 CallStackId &CallStackCount) {
4690 SmallVector<uint64_t> Record;
4691
4692 for (auto &CI : FS->callsites()) {
4693 Record.clear();
4694 // Per module callsite clones should always have a single entry of
4695 // value 0.
4696 assert(!PerModule || (CI.Clones.size() == 1 && CI.Clones[0] == 0));
4697 Record.push_back(Elt: GetValueID(CI.Callee));
4698 if (!PerModule) {
4699 Record.push_back(Elt: CI.StackIdIndices.size());
4700 Record.push_back(Elt: CI.Clones.size());
4701 }
4702 for (auto Id : CI.StackIdIndices)
4703 Record.push_back(Elt: GetStackIndex(Id));
4704 if (!PerModule)
4705 llvm::append_range(C&: Record, R: CI.Clones);
4706 Stream.EmitRecord(Code: PerModule ? bitc::FS_PERMODULE_CALLSITE_INFO
4707 : bitc::FS_COMBINED_CALLSITE_INFO,
4708 Vals: Record, Abbrev: CallsiteAbbrev);
4709 }
4710
4711 for (auto &AI : FS->allocs()) {
4712 Record.clear();
4713 // Per module alloc versions should always have a single entry of
4714 // value 0.
4715 assert(!PerModule || (AI.Versions.size() == 1 && AI.Versions[0] == 0));
4716 Record.push_back(Elt: AI.MIBs.size());
4717 if (!PerModule)
4718 Record.push_back(Elt: AI.Versions.size());
4719 for (auto &MIB : AI.MIBs) {
4720 Record.push_back(Elt: (uint8_t)MIB.AllocType);
4721 // The per-module summary always needs to include the alloc context, as we
4722 // use it during the thin link. For the combined index it is optional (see
4723 // comments where CombinedIndexMemProfContext is defined).
4724 if (PerModule || CombinedIndexMemProfContext) {
4725 // Record the index into the radix tree array for this context.
4726 assert(CallStackCount <= CallStackPos.size());
4727 Record.push_back(Elt: CallStackPos[CallStackCount++]);
4728 }
4729 }
4730 if (!PerModule)
4731 llvm::append_range(C&: Record, R: AI.Versions);
4732 assert(AI.ContextSizeInfos.empty() ||
4733 AI.ContextSizeInfos.size() == AI.MIBs.size());
4734 // Optionally emit the context size information if it exists.
4735 if (WriteContextSizeInfoIndex && !AI.ContextSizeInfos.empty()) {
4736 // The abbreviation id for the context ids record should have been created
4737 // if we are emitting the per-module index, which is where we write this
4738 // info.
4739 assert(ContextIdAbbvId);
4740 SmallVector<uint32_t> ContextIds;
4741 // At least one context id per ContextSizeInfos entry (MIB), broken into 2
4742 // halves.
4743 ContextIds.reserve(N: AI.ContextSizeInfos.size() * 2);
4744 for (auto &Infos : AI.ContextSizeInfos) {
4745 Record.push_back(Elt: Infos.size());
4746 for (auto [FullStackId, TotalSize] : Infos) {
4747 // The context ids are emitted separately as a fixed width array,
4748 // which is more efficient than a VBR given that these hashes are
4749 // typically close to 64-bits. The max fixed width entry is 32 bits so
4750 // it is split into 2.
4751 ContextIds.push_back(Elt: static_cast<uint32_t>(FullStackId >> 32));
4752 ContextIds.push_back(Elt: static_cast<uint32_t>(FullStackId));
4753 Record.push_back(Elt: TotalSize);
4754 }
4755 }
4756 // The context ids are expected by the reader to immediately precede the
4757 // associated alloc info record.
4758 Stream.EmitRecord(Code: bitc::FS_ALLOC_CONTEXT_IDS, Vals: ContextIds,
4759 Abbrev: ContextIdAbbvId);
4760 }
4761 Stream.EmitRecord(Code: PerModule
4762 ? bitc::FS_PERMODULE_ALLOC_INFO
4763 : (CombinedIndexMemProfContext
4764 ? bitc::FS_COMBINED_ALLOC_INFO
4765 : bitc::FS_COMBINED_ALLOC_INFO_NO_CONTEXT),
4766 Vals: Record, Abbrev: AllocAbbrev);
4767 }
4768}
4769
4770// Helper to emit a single function summary record.
4771void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
4772 SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
4773 unsigned ValueID, unsigned FSCallsProfileAbbrev, unsigned CallsiteAbbrev,
4774 unsigned AllocAbbrev, unsigned ContextIdAbbvId, const Function &F,
4775 DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
4776 CallStackId &CallStackCount) {
4777 NameVals.push_back(Elt: ValueID);
4778
4779 FunctionSummary *FS = cast<FunctionSummary>(Val: Summary);
4780
4781 writeFunctionTypeMetadataRecords(
4782 Stream, FS, GetValueID: [&](const ValueInfo &VI) -> std::optional<unsigned> {
4783 return {VE.getValueID(V: VI.getValue())};
4784 });
4785
4786 auto SpecialRefCnts = FS->specialRefCounts();
4787 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: FS->flags()));
4788 NameVals.push_back(Elt: FS->instCount());
4789 NameVals.push_back(Elt: getEncodedFFlags(Flags: FS->fflags()));
4790 NameVals.push_back(Elt: FS->refs().size());
4791 NameVals.push_back(Elt: SpecialRefCnts.first); // rorefcnt
4792 NameVals.push_back(Elt: SpecialRefCnts.second); // worefcnt
4793
4794 for (auto &RI : FS->refs())
4795 NameVals.push_back(Elt: getValueId(VI: RI));
4796
4797 for (auto &ECI : FS->calls()) {
4798 NameVals.push_back(Elt: getValueId(VI: ECI.first));
4799 NameVals.push_back(Elt: getEncodedHotnessCallEdgeInfo(CI: ECI.second));
4800 }
4801
4802 // Emit the finished record.
4803 Stream.EmitRecord(Code: bitc::FS_PERMODULE_PROFILE, Vals: NameVals, Abbrev: FSCallsProfileAbbrev);
4804 NameVals.clear();
4805
4806 writeFunctionHeapProfileRecords(
4807 Stream, FS, CallsiteAbbrev, AllocAbbrev, ContextIdAbbvId,
4808 /*PerModule*/ true,
4809 /*GetValueId*/ GetValueID: [&](const ValueInfo &VI) { return getValueId(VI); },
4810 /*GetStackIndex*/ [&](unsigned I) { return I; },
4811 /*WriteContextSizeInfoIndex*/ true, CallStackPos, CallStackCount);
4812}
4813
4814// Collect the global value references in the given variable's initializer,
4815// and emit them in a summary record.
4816void ModuleBitcodeWriterBase::writeModuleLevelReferences(
4817 const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals,
4818 unsigned FSModRefsAbbrev, unsigned FSModVTableRefsAbbrev) {
4819 // Be a little lenient here, to accomodate older files without GUIDs
4820 // already computed and assigned as metadata.
4821 GlobalValue::GUID GUID = V.getGUIDOrFallback();
4822
4823 auto VI = Index->getValueInfo(GUID);
4824 if (!VI || VI.getSummaryList().empty()) {
4825 // Only declarations should not have a summary (a declaration might however
4826 // have a summary if the def was in module level asm).
4827 assert(V.isDeclaration());
4828 return;
4829 }
4830 auto *Summary = VI.getSummaryList()[0].get();
4831 NameVals.push_back(Elt: VE.getValueID(V: &V));
4832 GlobalVarSummary *VS = cast<GlobalVarSummary>(Val: Summary);
4833 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: VS->flags()));
4834 NameVals.push_back(Elt: getEncodedGVarFlags(Flags: VS->varflags()));
4835
4836 auto VTableFuncs = VS->vTableFuncs();
4837 if (!VTableFuncs.empty())
4838 NameVals.push_back(Elt: VS->refs().size());
4839
4840 unsigned SizeBeforeRefs = NameVals.size();
4841 for (auto &RI : VS->refs())
4842 NameVals.push_back(Elt: VE.getValueID(V: RI.getValue()));
4843 // Sort the refs for determinism output, the vector returned by FS->refs() has
4844 // been initialized from a DenseSet.
4845 llvm::sort(C: drop_begin(RangeOrContainer&: NameVals, N: SizeBeforeRefs));
4846
4847 if (VTableFuncs.empty())
4848 Stream.EmitRecord(Code: bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, Vals: NameVals,
4849 Abbrev: FSModRefsAbbrev);
4850 else {
4851 // VTableFuncs pairs should already be sorted by offset.
4852 for (auto &P : VTableFuncs) {
4853 NameVals.push_back(Elt: VE.getValueID(V: P.FuncVI.getValue()));
4854 NameVals.push_back(Elt: P.VTableOffset);
4855 }
4856
4857 Stream.EmitRecord(Code: bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS, Vals: NameVals,
4858 Abbrev: FSModVTableRefsAbbrev);
4859 }
4860 NameVals.clear();
4861}
4862
4863/// Emit the per-module summary section alongside the rest of
4864/// the module's bitcode.
4865void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
4866 // By default we compile with ThinLTO if the module has a summary, but the
4867 // client can request full LTO with a module flag.
4868 bool IsThinLTO = true;
4869 if (auto *MD =
4870 mdconst::extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: "ThinLTO")))
4871 IsThinLTO = MD->getZExtValue();
4872 Stream.EnterSubblock(BlockID: IsThinLTO ? bitc::GLOBALVAL_SUMMARY_BLOCK_ID
4873 : bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID,
4874 CodeLen: 4);
4875
4876 Stream.EmitRecord(
4877 Code: bitc::FS_VERSION,
4878 Vals: ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
4879
4880 // Write the index flags.
4881 uint64_t Flags = 0;
4882 // Bits 1-3 are set only in the combined index, skip them.
4883 if (Index->enableSplitLTOUnit())
4884 Flags |= 0x8;
4885 if (Index->hasUnifiedLTO())
4886 Flags |= 0x200;
4887
4888 Stream.EmitRecord(Code: bitc::FS_FLAGS, Vals: ArrayRef<uint64_t>{Flags});
4889
4890 if (Index->begin() == Index->end()) {
4891 Stream.ExitBlock();
4892 return;
4893 }
4894
4895 auto Abbv = std::make_shared<BitCodeAbbrev>();
4896 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_VALUE_GUID));
4897 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4898 // GUIDS often use up most of 64-bits, so encode as two Fixed 32.
4899 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4900 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4901 unsigned ValueGuidAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4902
4903 for (const auto &GVI : valueIds()) {
4904 Stream.EmitRecord(Code: bitc::FS_VALUE_GUID,
4905 Vals: ArrayRef<uint32_t>{GVI.second,
4906 static_cast<uint32_t>(GVI.first >> 32),
4907 static_cast<uint32_t>(GVI.first)},
4908 Abbrev: ValueGuidAbbrev);
4909 }
4910
4911 if (!Index->stackIds().empty()) {
4912 auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4913 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_STACK_IDS));
4914 // numids x stackid
4915 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4916 // The stack ids are hashes that are close to 64 bits in size, so emitting
4917 // as a pair of 32-bit fixed-width values is more efficient than a VBR.
4918 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4919 unsigned StackIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(StackIdAbbv));
4920 SmallVector<uint32_t> Vals;
4921 Vals.reserve(N: Index->stackIds().size() * 2);
4922 for (auto Id : Index->stackIds()) {
4923 Vals.push_back(Elt: static_cast<uint32_t>(Id >> 32));
4924 Vals.push_back(Elt: static_cast<uint32_t>(Id));
4925 }
4926 Stream.EmitRecord(Code: bitc::FS_STACK_IDS, Vals, Abbrev: StackIdAbbvId);
4927 }
4928
4929 unsigned ContextIdAbbvId = 0;
4930 if (metadataMayIncludeContextSizeInfo()) {
4931 // n x context id
4932 auto ContextIdAbbv = std::make_shared<BitCodeAbbrev>();
4933 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_ALLOC_CONTEXT_IDS));
4934 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4935 // The context ids are hashes that are close to 64 bits in size, so emitting
4936 // as a pair of 32-bit fixed-width values is more efficient than a VBR if we
4937 // are emitting them for all MIBs. Otherwise we use VBR to better compress 0
4938 // values that are expected to more frequently occur in an alloc's memprof
4939 // summary.
4940 if (metadataIncludesAllContextSizeInfo())
4941 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4942 else
4943 ContextIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4944 ContextIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(ContextIdAbbv));
4945 }
4946
4947 // Abbrev for FS_PERMODULE_PROFILE.
4948 Abbv = std::make_shared<BitCodeAbbrev>();
4949 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE));
4950 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4951 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // flags
4952 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
4953 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
4954 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4955 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
4956 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
4957 // numrefs x valueid, n x (valueid, hotness+tailcall flags)
4958 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4959 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4960 unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4961
4962 // Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS.
4963 Abbv = std::make_shared<BitCodeAbbrev>();
4964 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS));
4965 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4966 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4967 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
4968 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4969 unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4970
4971 // Abbrev for FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS.
4972 Abbv = std::make_shared<BitCodeAbbrev>();
4973 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS));
4974 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4975 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4976 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4977 // numrefs x valueid, n x (valueid , offset)
4978 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4979 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4980 unsigned FSModVTableRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4981
4982 // Abbrev for FS_ALIAS.
4983 Abbv = std::make_shared<BitCodeAbbrev>();
4984 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_ALIAS));
4985 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4986 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4987 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4988 unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4989
4990 // Abbrev for FS_TYPE_ID_METADATA
4991 Abbv = std::make_shared<BitCodeAbbrev>();
4992 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_TYPE_ID_METADATA));
4993 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid strtab index
4994 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid length
4995 // n x (valueid , offset)
4996 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4997 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4998 unsigned TypeIdCompatibleVtableAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4999
5000 Abbv = std::make_shared<BitCodeAbbrev>();
5001 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_CALLSITE_INFO));
5002 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5003 // n x stackidindex
5004 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5005 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5006 unsigned CallsiteAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5007
5008 Abbv = std::make_shared<BitCodeAbbrev>();
5009 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_ALLOC_INFO));
5010 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib
5011 // n x (alloc type, context radix tree index)
5012 // optional: nummib x (numcontext x total size)
5013 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5014 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5015 unsigned AllocAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5016
5017 Abbv = std::make_shared<BitCodeAbbrev>();
5018 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_CONTEXT_RADIX_TREE_ARRAY));
5019 // n x entry
5020 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5021 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5022 unsigned RadixAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5023
5024 // First walk through all the functions and collect the allocation contexts in
5025 // their associated summaries, for use in constructing a radix tree of
5026 // contexts. Note that we need to do this in the same order as the functions
5027 // are processed further below since the call stack positions in the resulting
5028 // radix tree array are identified based on this order.
5029 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> CallStacks;
5030 for (const Function &F : M) {
5031 // Summary emission does not support anonymous functions, they have to be
5032 // renamed using the anonymous function renaming pass.
5033 if (!F.hasName())
5034 report_fatal_error(reason: "Unexpected anonymous function when writing summary");
5035
5036 // Be a little lenient here, to accomodate older files without GUIDs
5037 // already computed and assigned as metadata.
5038 GlobalValue::GUID GUID = F.getGUIDOrFallback();
5039
5040 ValueInfo VI = Index->getValueInfo(GUID);
5041 if (!VI || VI.getSummaryList().empty()) {
5042 // Only declarations should not have a summary (a declaration might
5043 // however have a summary if the def was in module level asm).
5044 if (!F.isDeclaration())
5045 reportFatalUsageError(reason: "expected function definition " + F.getName() +
5046 " to have an associated value info.");
5047 continue;
5048 }
5049 auto *Summary = VI.getSummaryList()[0].get();
5050 FunctionSummary *FS = cast<FunctionSummary>(Val: Summary);
5051 collectMemProfCallStacks(
5052 FS, /*GetStackIndex*/ [](unsigned I) { return I; }, CallStacks);
5053 }
5054 // Finalize the radix tree, write it out, and get the map of positions in the
5055 // linearized tree array.
5056 DenseMap<CallStackId, LinearCallStackId> CallStackPos;
5057 if (!CallStacks.empty()) {
5058 CallStackPos =
5059 writeMemoryProfileRadixTree(CallStacks: std::move(CallStacks), Stream, RadixAbbrev);
5060 }
5061
5062 // Keep track of the current index into the CallStackPos map.
5063 CallStackId CallStackCount = 0;
5064
5065 SmallVector<uint64_t, 64> NameVals;
5066 // Iterate over the list of functions instead of the Index to
5067 // ensure the ordering is stable.
5068 for (const Function &F : M) {
5069 // Summary emission does not support anonymous functions, they have to
5070 // renamed using the anonymous function renaming pass.
5071 if (!F.hasName())
5072 report_fatal_error(reason: "Unexpected anonymous function when writing summary");
5073
5074 GlobalValue::GUID GUID = F.getGUIDOrFallback();
5075
5076 ValueInfo VI = Index->getValueInfo(GUID);
5077 if (!VI || VI.getSummaryList().empty()) {
5078 // Only declarations should not have a summary (a declaration might
5079 // however have a summary if the def was in module level asm).
5080 assert(F.isDeclaration());
5081 continue;
5082 }
5083 auto *Summary = VI.getSummaryList()[0].get();
5084 writePerModuleFunctionSummaryRecord(NameVals, Summary, ValueID: VE.getValueID(V: &F),
5085 FSCallsProfileAbbrev, CallsiteAbbrev,
5086 AllocAbbrev, ContextIdAbbvId, F,
5087 CallStackPos, CallStackCount);
5088 }
5089
5090 // Capture references from GlobalVariable initializers, which are outside
5091 // of a function scope.
5092 for (const GlobalVariable &G : M.globals())
5093 writeModuleLevelReferences(V: G, NameVals, FSModRefsAbbrev,
5094 FSModVTableRefsAbbrev);
5095
5096 for (const GlobalAlias &A : M.aliases()) {
5097 auto *Aliasee = A.getAliaseeObject();
5098 // Skip ifunc and nameless functions which don't have an entry in the
5099 // summary.
5100 if (!Aliasee->hasName() || isa<GlobalIFunc>(Val: Aliasee))
5101 continue;
5102 auto AliasId = VE.getValueID(V: &A);
5103 auto AliaseeId = VE.getValueID(V: Aliasee);
5104 NameVals.push_back(Elt: AliasId);
5105 auto *Summary = Index->getGlobalValueSummary(GV: A);
5106 AliasSummary *AS = cast<AliasSummary>(Val: Summary);
5107 NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: AS->flags()));
5108 NameVals.push_back(Elt: AliaseeId);
5109 Stream.EmitRecord(Code: bitc::FS_ALIAS, Vals: NameVals, Abbrev: FSAliasAbbrev);
5110 NameVals.clear();
5111 }
5112
5113 for (auto &S : Index->typeIdCompatibleVtableMap()) {
5114 writeTypeIdCompatibleVtableSummaryRecord(NameVals, StrtabBuilder, Id: S.first,
5115 Summary: S.second, VE);
5116 Stream.EmitRecord(Code: bitc::FS_TYPE_ID_METADATA, Vals: NameVals,
5117 Abbrev: TypeIdCompatibleVtableAbbrev);
5118 NameVals.clear();
5119 }
5120
5121 if (Index->getBlockCount())
5122 Stream.EmitRecord(Code: bitc::FS_BLOCK_COUNT,
5123 Vals: ArrayRef<uint64_t>{Index->getBlockCount()});
5124
5125 Stream.ExitBlock();
5126}
5127
5128void ModuleBitcodeWriterBase::writeGUIDList() {
5129 const ValueEnumerator::ValueList &Vals = VE.getValues();
5130 const size_t Max = Vals.size();
5131
5132 std::vector<GlobalValue::GUID> GUIDs(Max, 0);
5133 for (const GlobalValue &GV : M.global_values()) {
5134 auto MaybeGUID = GV.getGUIDIfAssigned();
5135 if (!MaybeGUID)
5136 continue;
5137 auto GUID = *MaybeGUID;
5138
5139 const auto ValueID = VE.getValueID(V: &GV);
5140 GUIDs[ValueID] = GUID;
5141 }
5142
5143 auto Abbv = std::make_shared<BitCodeAbbrev>();
5144 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_GUIDLIST));
5145 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5146 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
5147 unsigned GUIDListAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5148
5149 SmallVector<uint32_t> RecordVals;
5150 RecordVals.reserve(N: Max * 2);
5151 for (auto GUID : GUIDs) {
5152 RecordVals.push_back(Elt: static_cast<uint32_t>(GUID >> 32));
5153 RecordVals.push_back(Elt: static_cast<uint32_t>(GUID));
5154 }
5155
5156 Stream.EmitRecord(Code: bitc::MODULE_CODE_GUIDLIST, Vals: RecordVals, Abbrev: GUIDListAbbrev);
5157}
5158
5159/// Emit the combined summary section into the combined index file.
5160void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
5161 Stream.EnterSubblock(BlockID: bitc::GLOBALVAL_SUMMARY_BLOCK_ID, CodeLen: 4);
5162 Stream.EmitRecord(
5163 Code: bitc::FS_VERSION,
5164 Vals: ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
5165
5166 // Write the index flags.
5167 Stream.EmitRecord(Code: bitc::FS_FLAGS, Vals: ArrayRef<uint64_t>{Index.getFlags()});
5168
5169 auto Abbv = std::make_shared<BitCodeAbbrev>();
5170 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_VALUE_GUID));
5171 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
5172 // GUIDS often use up most of 64-bits, so encode as two Fixed 32.
5173 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
5174 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
5175 unsigned ValueGuidAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5176
5177 for (const auto &GVI : valueIds()) {
5178 Stream.EmitRecord(Code: bitc::FS_VALUE_GUID,
5179 Vals: ArrayRef<uint32_t>{GVI.second,
5180 static_cast<uint32_t>(GVI.first >> 32),
5181 static_cast<uint32_t>(GVI.first)},
5182 Abbrev: ValueGuidAbbrev);
5183 }
5184
5185 // Write the stack ids used by this index, which will be a subset of those in
5186 // the full index in the case of distributed indexes.
5187 if (!StackIds.empty()) {
5188 auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
5189 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_STACK_IDS));
5190 // numids x stackid
5191 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5192 // The stack ids are hashes that are close to 64 bits in size, so emitting
5193 // as a pair of 32-bit fixed-width values is more efficient than a VBR.
5194 StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
5195 unsigned StackIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(StackIdAbbv));
5196 SmallVector<uint32_t> Vals;
5197 Vals.reserve(N: StackIds.size() * 2);
5198 for (auto Id : StackIds) {
5199 Vals.push_back(Elt: static_cast<uint32_t>(Id >> 32));
5200 Vals.push_back(Elt: static_cast<uint32_t>(Id));
5201 }
5202 Stream.EmitRecord(Code: bitc::FS_STACK_IDS, Vals, Abbrev: StackIdAbbvId);
5203 }
5204
5205 // Abbrev for FS_COMBINED_PROFILE.
5206 Abbv = std::make_shared<BitCodeAbbrev>();
5207 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_PROFILE));
5208 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5209 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
5210 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
5211 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
5212 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
5213 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // entrycount
5214 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
5215 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
5216 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
5217 // numrefs x valueid, n x (valueid, hotness+tailcall flags)
5218 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5219 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5220 unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5221
5222 // Abbrev for FS_COMBINED_GLOBALVAR_INIT_REFS.
5223 Abbv = std::make_shared<BitCodeAbbrev>();
5224 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS));
5225 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5226 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
5227 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
5228 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
5229 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5230 unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5231
5232 // Abbrev for FS_COMBINED_ALIAS.
5233 Abbv = std::make_shared<BitCodeAbbrev>();
5234 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_ALIAS));
5235 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5236 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
5237 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
5238 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5239 unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5240
5241 Abbv = std::make_shared<BitCodeAbbrev>();
5242 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_CALLSITE_INFO));
5243 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
5244 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numstackindices
5245 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
5246 // numstackindices x stackidindex, numver x version
5247 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5248 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5249 unsigned CallsiteAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5250
5251 Abbv = std::make_shared<BitCodeAbbrev>();
5252 Abbv->Add(OpInfo: BitCodeAbbrevOp(CombinedIndexMemProfContext
5253 ? bitc::FS_COMBINED_ALLOC_INFO
5254 : bitc::FS_COMBINED_ALLOC_INFO_NO_CONTEXT));
5255 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib
5256 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
5257 // nummib x (alloc type, context radix tree index),
5258 // numver x version
5259 // optional: nummib x total size
5260 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5261 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5262 unsigned AllocAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5263
5264 auto shouldImportValueAsDecl = [&](GlobalValueSummary *GVS) -> bool {
5265 if (DecSummaries == nullptr)
5266 return false;
5267 return DecSummaries->count(Ptr: GVS);
5268 };
5269
5270 // The aliases are emitted as a post-pass, and will point to the value
5271 // id of the aliasee. Save them in a vector for post-processing.
5272 SmallVector<AliasSummary *, 64> Aliases;
5273
5274 // Save the value id for each summary for alias emission.
5275 DenseMap<const GlobalValueSummary *, unsigned> SummaryToValueIdMap;
5276
5277 SmallVector<uint64_t, 64> NameVals;
5278
5279 // Set that will be populated during call to writeFunctionTypeMetadataRecords
5280 // with the type ids referenced by this index file.
5281 std::set<GlobalValue::GUID> ReferencedTypeIds;
5282
5283 // For local linkage, we also emit the original name separately
5284 // immediately after the record.
5285 auto MaybeEmitOriginalName = [&](GlobalValueSummary &S) {
5286 // We don't need to emit the original name if we are writing the index for
5287 // distributed backends (in which case ModuleToSummariesForIndex is
5288 // non-null). The original name is only needed during the thin link, since
5289 // for SamplePGO the indirect call targets for local functions have
5290 // have the original name annotated in profile.
5291 // Continue to emit it when writing out the entire combined index, which is
5292 // used in testing the thin link via llvm-lto.
5293 if (ModuleToSummariesForIndex || !GlobalValue::isLocalLinkage(Linkage: S.linkage()))
5294 return;
5295 NameVals.push_back(Elt: S.getOriginalName());
5296 Stream.EmitRecord(Code: bitc::FS_COMBINED_ORIGINAL_NAME, Vals: NameVals);
5297 NameVals.clear();
5298 };
5299
5300 DenseMap<CallStackId, LinearCallStackId> CallStackPos;
5301 if (CombinedIndexMemProfContext) {
5302 Abbv = std::make_shared<BitCodeAbbrev>();
5303 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_CONTEXT_RADIX_TREE_ARRAY));
5304 // n x entry
5305 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5306 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
5307 unsigned RadixAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5308
5309 // First walk through all the functions and collect the allocation contexts
5310 // in their associated summaries, for use in constructing a radix tree of
5311 // contexts. Note that we need to do this in the same order as the functions
5312 // are processed further below since the call stack positions in the
5313 // resulting radix tree array are identified based on this order.
5314 MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> CallStacks;
5315 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
5316 // Don't collect this when invoked for an aliasee, as it is not needed for
5317 // the alias summary. If the aliasee is to be imported, we will invoke
5318 // this separately with IsAliasee=false.
5319 if (IsAliasee)
5320 return;
5321 GlobalValueSummary *S = I.second;
5322 assert(S);
5323 auto *FS = dyn_cast<FunctionSummary>(Val: S);
5324 if (!FS)
5325 return;
5326 collectMemProfCallStacks(
5327 FS,
5328 /*GetStackIndex*/
5329 [&](unsigned I) {
5330 // Get the corresponding index into the list of StackIds actually
5331 // being written for this combined index (which may be a subset in
5332 // the case of distributed indexes).
5333 assert(StackIdIndicesToIndex.contains(I));
5334 return StackIdIndicesToIndex[I];
5335 },
5336 CallStacks);
5337 });
5338 // Finalize the radix tree, write it out, and get the map of positions in
5339 // the linearized tree array.
5340 if (!CallStacks.empty()) {
5341 CallStackPos = writeMemoryProfileRadixTree(CallStacks: std::move(CallStacks), Stream,
5342 RadixAbbrev);
5343 }
5344 }
5345
5346 // Keep track of the current index into the CallStackPos map. Not used if
5347 // CombinedIndexMemProfContext is false.
5348 CallStackId CallStackCount = 0;
5349
5350 DenseSet<GlobalValue::GUID> DefOrUseGUIDs;
5351 forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
5352 GlobalValueSummary *S = I.second;
5353 assert(S);
5354 DefOrUseGUIDs.insert(V: I.first);
5355 for (const ValueInfo &VI : S->refs())
5356 DefOrUseGUIDs.insert(V: VI.getGUID());
5357
5358 auto ValueId = getValueId(ValGUID: I.first);
5359 assert(ValueId);
5360 SummaryToValueIdMap[S] = *ValueId;
5361
5362 // If this is invoked for an aliasee, we want to record the above
5363 // mapping, but then not emit a summary entry (if the aliasee is
5364 // to be imported, we will invoke this separately with IsAliasee=false).
5365 if (IsAliasee)
5366 return;
5367
5368 if (auto *AS = dyn_cast<AliasSummary>(Val: S)) {
5369 // Will process aliases as a post-pass because the reader wants all
5370 // global to be loaded first.
5371 Aliases.push_back(Elt: AS);
5372 return;
5373 }
5374
5375 if (auto *VS = dyn_cast<GlobalVarSummary>(Val: S)) {
5376 NameVals.push_back(Elt: *ValueId);
5377 assert(ModuleIdMap.count(VS->modulePath()));
5378 NameVals.push_back(Elt: ModuleIdMap[VS->modulePath()]);
5379 NameVals.push_back(
5380 Elt: getEncodedGVSummaryFlags(Flags: VS->flags(), ImportAsDecl: shouldImportValueAsDecl(VS)));
5381 NameVals.push_back(Elt: getEncodedGVarFlags(Flags: VS->varflags()));
5382 for (auto &RI : VS->refs()) {
5383 auto RefValueId = getValueId(ValGUID: RI.getGUID());
5384 if (!RefValueId)
5385 continue;
5386 NameVals.push_back(Elt: *RefValueId);
5387 }
5388
5389 // Emit the finished record.
5390 Stream.EmitRecord(Code: bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, Vals: NameVals,
5391 Abbrev: FSModRefsAbbrev);
5392 NameVals.clear();
5393 MaybeEmitOriginalName(*S);
5394 return;
5395 }
5396
5397 auto GetValueId = [&](const ValueInfo &VI) -> std::optional<unsigned> {
5398 if (!VI)
5399 return std::nullopt;
5400 return getValueId(ValGUID: VI.getGUID());
5401 };
5402
5403 auto *FS = cast<FunctionSummary>(Val: S);
5404 writeFunctionTypeMetadataRecords(Stream, FS, GetValueID: GetValueId);
5405 getReferencedTypeIds(FS, ReferencedTypeIds);
5406
5407 NameVals.push_back(Elt: *ValueId);
5408 assert(ModuleIdMap.count(FS->modulePath()));
5409 NameVals.push_back(Elt: ModuleIdMap[FS->modulePath()]);
5410 NameVals.push_back(
5411 Elt: getEncodedGVSummaryFlags(Flags: FS->flags(), ImportAsDecl: shouldImportValueAsDecl(FS)));
5412 NameVals.push_back(Elt: FS->instCount());
5413 NameVals.push_back(Elt: getEncodedFFlags(Flags: FS->fflags()));
5414 // TODO: Stop writing entry count and bump bitcode version.
5415 NameVals.push_back(Elt: 0 /* EntryCount */);
5416
5417 // Fill in below
5418 NameVals.push_back(Elt: 0); // numrefs
5419 NameVals.push_back(Elt: 0); // rorefcnt
5420 NameVals.push_back(Elt: 0); // worefcnt
5421
5422 unsigned Count = 0, RORefCnt = 0, WORefCnt = 0;
5423 for (auto &RI : FS->refs()) {
5424 auto RefValueId = getValueId(ValGUID: RI.getGUID());
5425 if (!RefValueId)
5426 continue;
5427 NameVals.push_back(Elt: *RefValueId);
5428 if (RI.isReadOnly())
5429 RORefCnt++;
5430 else if (RI.isWriteOnly())
5431 WORefCnt++;
5432 Count++;
5433 }
5434 NameVals[6] = Count;
5435 NameVals[7] = RORefCnt;
5436 NameVals[8] = WORefCnt;
5437
5438 for (auto &EI : FS->calls()) {
5439 // If this GUID doesn't have a value id, it doesn't have a function
5440 // summary and we don't need to record any calls to it.
5441 std::optional<unsigned> CallValueId = GetValueId(EI.first);
5442 if (!CallValueId)
5443 continue;
5444 NameVals.push_back(Elt: *CallValueId);
5445 NameVals.push_back(Elt: getEncodedHotnessCallEdgeInfo(CI: EI.second));
5446 }
5447
5448 // Emit the finished record.
5449 Stream.EmitRecord(Code: bitc::FS_COMBINED_PROFILE, Vals: NameVals,
5450 Abbrev: FSCallsProfileAbbrev);
5451 NameVals.clear();
5452
5453 writeFunctionHeapProfileRecords(
5454 Stream, FS, CallsiteAbbrev, AllocAbbrev, /*ContextIdAbbvId*/ 0,
5455 /*PerModule*/ false,
5456 /*GetValueId*/
5457 GetValueID: [&](const ValueInfo &VI) -> unsigned {
5458 std::optional<unsigned> ValueID = GetValueId(VI);
5459 // This can happen in shared index files for distributed ThinLTO if
5460 // the callee function summary is not included. Record 0 which we
5461 // will have to deal with conservatively when doing any kind of
5462 // validation in the ThinLTO backends.
5463 if (!ValueID)
5464 return 0;
5465 return *ValueID;
5466 },
5467 /*GetStackIndex*/
5468 [&](unsigned I) {
5469 // Get the corresponding index into the list of StackIds actually
5470 // being written for this combined index (which may be a subset in
5471 // the case of distributed indexes).
5472 assert(StackIdIndicesToIndex.contains(I));
5473 return StackIdIndicesToIndex[I];
5474 },
5475 /*WriteContextSizeInfoIndex*/ false, CallStackPos, CallStackCount);
5476
5477 MaybeEmitOriginalName(*S);
5478 });
5479
5480 for (auto *AS : Aliases) {
5481 auto AliasValueId = SummaryToValueIdMap[AS];
5482 assert(AliasValueId);
5483 NameVals.push_back(Elt: AliasValueId);
5484 assert(ModuleIdMap.count(AS->modulePath()));
5485 NameVals.push_back(Elt: ModuleIdMap[AS->modulePath()]);
5486 NameVals.push_back(
5487 Elt: getEncodedGVSummaryFlags(Flags: AS->flags(), ImportAsDecl: shouldImportValueAsDecl(AS)));
5488 // Set value id to 0 when an alias is imported but the aliasee summary is
5489 // not contained in the index.
5490 auto AliaseeValueId =
5491 AS->hasAliasee() ? SummaryToValueIdMap[&AS->getAliasee()] : 0;
5492 NameVals.push_back(Elt: AliaseeValueId);
5493
5494 // Emit the finished record.
5495 Stream.EmitRecord(Code: bitc::FS_COMBINED_ALIAS, Vals: NameVals, Abbrev: FSAliasAbbrev);
5496 NameVals.clear();
5497 MaybeEmitOriginalName(*AS);
5498
5499 if (AS->hasAliasee())
5500 if (auto *FS = dyn_cast<FunctionSummary>(Val: &AS->getAliasee()))
5501 getReferencedTypeIds(FS, ReferencedTypeIds);
5502 }
5503
5504 SmallVector<std::pair<StringRef, GlobalValue::GUID>, 4> Functions;
5505 auto EmitCfiFunctions = [&](const CfiFunctionIndex &CfiIndex,
5506 bitc::GlobalValueSummarySymtabCodes Code) {
5507 if (CfiIndex.empty())
5508 return;
5509 for (GlobalValue::GUID GUID : DefOrUseGUIDs) {
5510 auto Names = CfiIndex.getNamesForGUID(GUID);
5511 for (StringRef Name : Names)
5512 Functions.push_back(Elt: {Name, GUID});
5513 }
5514 if (Functions.empty())
5515 return;
5516 llvm::sort(C&: Functions);
5517 for (const auto &Record : Functions) {
5518 NameVals.push_back(Elt: Record.second);
5519 NameVals.push_back(Elt: StrtabBuilder.add(S: Record.first));
5520 NameVals.push_back(Elt: Record.first.size());
5521 }
5522 Stream.EmitRecord(Code, Vals: NameVals);
5523 NameVals.clear();
5524 Functions.clear();
5525 };
5526
5527 EmitCfiFunctions(Index.cfiFunctionDefs(), bitc::FS_CFI_FUNCTION_DEFS);
5528 EmitCfiFunctions(Index.cfiFunctionDecls(), bitc::FS_CFI_FUNCTION_DECLS);
5529
5530 // Walk the GUIDs that were referenced, and write the
5531 // corresponding type id records.
5532 for (auto &T : ReferencedTypeIds) {
5533 auto TidIter = Index.typeIds().equal_range(x: T);
5534 for (const auto &[GUID, TypeIdPair] : make_range(p: TidIter)) {
5535 writeTypeIdSummaryRecord(NameVals, StrtabBuilder, Id: TypeIdPair.first,
5536 Summary: TypeIdPair.second);
5537 Stream.EmitRecord(Code: bitc::FS_TYPE_ID, Vals: NameVals);
5538 NameVals.clear();
5539 }
5540 }
5541
5542 if (Index.getBlockCount())
5543 Stream.EmitRecord(Code: bitc::FS_BLOCK_COUNT,
5544 Vals: ArrayRef<uint64_t>{Index.getBlockCount()});
5545
5546 Stream.ExitBlock();
5547}
5548
5549/// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the
5550/// current llvm version, and a record for the epoch number.
5551static void writeIdentificationBlock(BitstreamWriter &Stream) {
5552 Stream.EnterSubblock(BlockID: bitc::IDENTIFICATION_BLOCK_ID, CodeLen: 5);
5553
5554 // Write the "user readable" string identifying the bitcode producer
5555 auto Abbv = std::make_shared<BitCodeAbbrev>();
5556 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_STRING));
5557 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5558 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
5559 auto StringAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5560 writeStringRecord(Stream, Code: bitc::IDENTIFICATION_CODE_STRING,
5561 Str: "LLVM" LLVM_VERSION_STRING, AbbrevToUse: StringAbbrev);
5562
5563 // Write the epoch version
5564 Abbv = std::make_shared<BitCodeAbbrev>();
5565 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_EPOCH));
5566 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
5567 auto EpochAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5568 constexpr std::array<unsigned, 1> Vals = {._M_elems: {bitc::BITCODE_CURRENT_EPOCH}};
5569 Stream.EmitRecord(Code: bitc::IDENTIFICATION_CODE_EPOCH, Vals, Abbrev: EpochAbbrev);
5570 Stream.ExitBlock();
5571}
5572
5573void ModuleBitcodeWriter::writeModuleHash(StringRef View) {
5574 // Emit the module's hash.
5575 // MODULE_CODE_HASH: [5*i32]
5576 if (GenerateHash) {
5577 uint32_t Vals[5];
5578 Hasher.update(Data: ArrayRef<uint8_t>(
5579 reinterpret_cast<const uint8_t *>(View.data()), View.size()));
5580 std::array<uint8_t, 20> Hash = Hasher.result();
5581 for (int Pos = 0; Pos < 20; Pos += 4) {
5582 Vals[Pos / 4] = support::endian::read32be(P: Hash.data() + Pos);
5583 }
5584
5585 // Emit the finished record.
5586 Stream.EmitRecord(Code: bitc::MODULE_CODE_HASH, Vals);
5587
5588 if (ModHash)
5589 // Save the written hash value.
5590 llvm::copy(Range&: Vals, Out: std::begin(cont&: *ModHash));
5591 }
5592}
5593
5594void ModuleBitcodeWriter::write() {
5595 writeIdentificationBlock(Stream);
5596
5597 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5598 // We will want to write the module hash at this point. Block any flushing so
5599 // we can have access to the whole underlying data later.
5600 Stream.markAndBlockFlushing();
5601
5602 writeModuleVersion();
5603
5604 // Emit blockinfo, which defines the standard abbreviations etc.
5605 writeBlockInfo();
5606
5607 // Emit information describing all of the types in the module.
5608 writeTypeTable();
5609
5610 // Emit information about attribute groups.
5611 writeAttributeGroupTable();
5612
5613 // Emit information about parameter attributes.
5614 writeAttributeTable();
5615
5616 writeComdats();
5617
5618 // Emit top-level description of module, including target triple, inline asm,
5619 // descriptors for global variables, and function prototype info.
5620 writeModuleInfo();
5621
5622 // Emit constants.
5623 writeModuleConstants();
5624
5625 // Emit metadata kind names.
5626 writeModuleMetadataKinds();
5627
5628 // Emit metadata.
5629 writeModuleMetadata();
5630
5631 // Emit module-level use-lists.
5632 if (VE.shouldPreserveUseListOrder())
5633 writeUseListBlock(F: nullptr);
5634
5635 writeOperandBundleTags();
5636 writeSyncScopeNames();
5637
5638 // Emit function bodies.
5639 DenseMap<const Function *, uint64_t> FunctionToBitcodeIndex;
5640 for (const Function &F : M)
5641 if (!F.isDeclaration())
5642 writeFunction(F, FunctionToBitcodeIndex);
5643
5644 // Need to write after the above call to WriteFunction which populates
5645 // the summary information in the index.
5646 if (Index)
5647 writePerModuleGlobalValueSummary();
5648
5649 writeGlobalValueSymbolTable(FunctionToBitcodeIndex);
5650
5651 writeModuleHash(View: Stream.getMarkedBufferAndResumeFlushing());
5652
5653 Stream.ExitBlock();
5654}
5655
5656static void writeInt32ToBuffer(uint32_t Value, SmallVectorImpl<char> &Buffer,
5657 uint32_t &Position) {
5658 support::endian::write32le(P: &Buffer[Position], V: Value);
5659 Position += 4;
5660}
5661
5662/// If generating a bc file on darwin, we have to emit a
5663/// header and trailer to make it compatible with the system archiver. To do
5664/// this we emit the following header, and then emit a trailer that pads the
5665/// file out to be a multiple of 16 bytes.
5666///
5667/// struct bc_header {
5668/// uint32_t Magic; // 0x0B17C0DE
5669/// uint32_t Version; // Version, currently always 0.
5670/// uint32_t BitcodeOffset; // Offset to traditional bitcode file.
5671/// uint32_t BitcodeSize; // Size of traditional bitcode file.
5672/// uint32_t CPUType; // CPU specifier.
5673/// ... potentially more later ...
5674/// };
5675static void emitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer,
5676 const Triple &TT) {
5677 unsigned CPUType = ~0U;
5678
5679 // Match x86_64-*, i[3-9]86-*, powerpc-*, powerpc64-*, arm-*, thumb-*,
5680 // armv[0-9]-*, thumbv[0-9]-*, armv5te-*, or armv6t2-*. The CPUType is a magic
5681 // number from /usr/include/mach/machine.h. It is ok to reproduce the
5682 // specific constants here because they are implicitly part of the Darwin ABI.
5683 enum {
5684 DARWIN_CPU_ARCH_ABI64 = 0x01000000,
5685 DARWIN_CPU_TYPE_X86 = 7,
5686 DARWIN_CPU_TYPE_ARM = 12,
5687 DARWIN_CPU_TYPE_POWERPC = 18
5688 };
5689
5690 Triple::ArchType Arch = TT.getArch();
5691 if (Arch == Triple::x86_64)
5692 CPUType = DARWIN_CPU_TYPE_X86 | DARWIN_CPU_ARCH_ABI64;
5693 else if (Arch == Triple::x86)
5694 CPUType = DARWIN_CPU_TYPE_X86;
5695 else if (Arch == Triple::ppc)
5696 CPUType = DARWIN_CPU_TYPE_POWERPC;
5697 else if (Arch == Triple::ppc64)
5698 CPUType = DARWIN_CPU_TYPE_POWERPC | DARWIN_CPU_ARCH_ABI64;
5699 else if (Arch == Triple::arm || Arch == Triple::thumb)
5700 CPUType = DARWIN_CPU_TYPE_ARM;
5701
5702 // Traditional Bitcode starts after header.
5703 assert(Buffer.size() >= BWH_HeaderSize &&
5704 "Expected header size to be reserved");
5705 unsigned BCOffset = BWH_HeaderSize;
5706 unsigned BCSize = Buffer.size() - BWH_HeaderSize;
5707
5708 // Write the magic and version.
5709 unsigned Position = 0;
5710 writeInt32ToBuffer(Value: 0x0B17C0DE, Buffer, Position);
5711 writeInt32ToBuffer(Value: 0, Buffer, Position); // Version.
5712 writeInt32ToBuffer(Value: BCOffset, Buffer, Position);
5713 writeInt32ToBuffer(Value: BCSize, Buffer, Position);
5714 writeInt32ToBuffer(Value: CPUType, Buffer, Position);
5715
5716 // If the file is not a multiple of 16 bytes, insert dummy padding.
5717 while (Buffer.size() & 15)
5718 Buffer.push_back(Elt: 0);
5719}
5720
5721/// Helper to write the header common to all bitcode files.
5722static void writeBitcodeHeader(BitstreamWriter &Stream) {
5723 // Emit the file header.
5724 Stream.Emit(Val: (unsigned)'B', NumBits: 8);
5725 Stream.Emit(Val: (unsigned)'C', NumBits: 8);
5726 Stream.Emit(Val: 0x0, NumBits: 4);
5727 Stream.Emit(Val: 0xC, NumBits: 4);
5728 Stream.Emit(Val: 0xE, NumBits: 4);
5729 Stream.Emit(Val: 0xD, NumBits: 4);
5730}
5731
5732BitcodeWriter::BitcodeWriter(SmallVectorImpl<char> &Buffer)
5733 : Stream(new BitstreamWriter(Buffer)) {
5734 writeBitcodeHeader(Stream&: *Stream);
5735}
5736
5737BitcodeWriter::BitcodeWriter(raw_ostream &FS)
5738 : Stream(new BitstreamWriter(FS, FlushThreshold)) {
5739 writeBitcodeHeader(Stream&: *Stream);
5740}
5741
5742BitcodeWriter::~BitcodeWriter() { assert(WroteStrtab); }
5743
5744void BitcodeWriter::writeBlob(unsigned Block, unsigned Record, StringRef Blob) {
5745 Stream->EnterSubblock(BlockID: Block, CodeLen: 3);
5746
5747 auto Abbv = std::make_shared<BitCodeAbbrev>();
5748 Abbv->Add(OpInfo: BitCodeAbbrevOp(Record));
5749 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
5750 auto AbbrevNo = Stream->EmitAbbrev(Abbv: std::move(Abbv));
5751
5752 Stream->EmitRecordWithBlob(Abbrev: AbbrevNo, Vals: ArrayRef<uint64_t>{Record}, Blob);
5753
5754 Stream->ExitBlock();
5755}
5756
5757void BitcodeWriter::writeSymtab() {
5758 assert(!WroteStrtab && !WroteSymtab);
5759
5760 // If any module has module-level inline asm, we will require a registered asm
5761 // parser for the target so that we can create an accurate symbol table for
5762 // the module.
5763 for (Module *M : Mods) {
5764 if (M->getModuleInlineAsm().empty())
5765 continue;
5766
5767 std::string Err;
5768 const Triple TT(M->getTargetTriple());
5769 const Target *T = TargetRegistry::lookupTarget(TheTriple: TT, Error&: Err);
5770 if (!T || !T->hasMCAsmParser())
5771 return;
5772 }
5773
5774 WroteSymtab = true;
5775 SmallVector<char, 0> Symtab;
5776 // The irsymtab::build function may be unable to create a symbol table if the
5777 // module is malformed (e.g. it contains an invalid alias). Writing a symbol
5778 // table is not required for correctness, but we still want to be able to
5779 // write malformed modules to bitcode files, so swallow the error.
5780 if (Error E = irsymtab::build(Mods, Symtab, StrtabBuilder, Alloc)) {
5781 consumeError(Err: std::move(E));
5782 return;
5783 }
5784
5785 writeBlob(Block: bitc::SYMTAB_BLOCK_ID, Record: bitc::SYMTAB_BLOB,
5786 Blob: {Symtab.data(), Symtab.size()});
5787}
5788
5789void BitcodeWriter::writeStrtab() {
5790 assert(!WroteStrtab);
5791
5792 std::vector<char> Strtab;
5793 StrtabBuilder.finalizeInOrder();
5794 Strtab.resize(new_size: StrtabBuilder.getSize());
5795 StrtabBuilder.write(Buf: (uint8_t *)Strtab.data());
5796
5797 writeBlob(Block: bitc::STRTAB_BLOCK_ID, Record: bitc::STRTAB_BLOB,
5798 Blob: {Strtab.data(), Strtab.size()});
5799
5800 WroteStrtab = true;
5801}
5802
5803void BitcodeWriter::copyStrtab(StringRef Strtab) {
5804 writeBlob(Block: bitc::STRTAB_BLOCK_ID, Record: bitc::STRTAB_BLOB, Blob: Strtab);
5805 WroteStrtab = true;
5806}
5807
5808void BitcodeWriter::writeModule(const Module &M,
5809 bool ShouldPreserveUseListOrder,
5810 const ModuleSummaryIndex *Index,
5811 bool GenerateHash, ModuleHash *ModHash) {
5812 assert(!WroteStrtab);
5813
5814 // The Mods vector is used by irsymtab::build, which requires non-const
5815 // Modules in case it needs to materialize metadata. But the bitcode writer
5816 // requires that the module is materialized, so we can cast to non-const here,
5817 // after checking that it is in fact materialized.
5818 assert(M.isMaterialized());
5819 Mods.push_back(x: const_cast<Module *>(&M));
5820
5821 ModuleBitcodeWriter ModuleWriter(M, StrtabBuilder, *Stream,
5822 ShouldPreserveUseListOrder, Index,
5823 GenerateHash, ModHash);
5824 ModuleWriter.write();
5825}
5826
5827void BitcodeWriter::writeIndex(
5828 const ModuleSummaryIndex *Index,
5829 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex,
5830 const GVSummaryPtrSet *DecSummaries) {
5831 IndexBitcodeWriter IndexWriter(*Stream, StrtabBuilder, *Index, DecSummaries,
5832 ModuleToSummariesForIndex);
5833 IndexWriter.write();
5834}
5835
5836/// Write the specified module to the specified output stream.
5837void llvm::WriteBitcodeToFile(const Module &M, raw_ostream &Out,
5838 bool ShouldPreserveUseListOrder,
5839 const ModuleSummaryIndex *Index,
5840 bool GenerateHash, ModuleHash *ModHash) {
5841 auto Write = [&](BitcodeWriter &Writer) {
5842 Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash,
5843 ModHash);
5844 Writer.writeSymtab();
5845 Writer.writeStrtab();
5846 };
5847 Triple TT(M.getTargetTriple());
5848 if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) {
5849 // If this is darwin or another generic macho target, reserve space for the
5850 // header. Note that the header is computed *after* the output is known, so
5851 // we currently explicitly use a buffer, write to it, and then subsequently
5852 // flush to Out.
5853 SmallVector<char, 0> Buffer;
5854 Buffer.reserve(N: 256 * 1024);
5855 Buffer.insert(I: Buffer.begin(), NumToInsert: BWH_HeaderSize, Elt: 0);
5856 BitcodeWriter Writer(Buffer);
5857 Write(Writer);
5858 emitDarwinBCHeaderAndTrailer(Buffer, TT);
5859 Out.write(Ptr: Buffer.data(), Size: Buffer.size());
5860 } else {
5861 BitcodeWriter Writer(Out);
5862 Write(Writer);
5863 }
5864}
5865
5866void IndexBitcodeWriter::write() {
5867 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5868
5869 writeModuleVersion();
5870
5871 // Write the module paths in the combined index.
5872 writeModStrings();
5873
5874 // Write the summary combined index records.
5875 writeCombinedGlobalValueSummary();
5876
5877 Stream.ExitBlock();
5878}
5879
5880// Write the specified module summary index to the given raw output stream,
5881// where it will be written in a new bitcode block. This is used when
5882// writing the combined index file for ThinLTO. When writing a subset of the
5883// index for a distributed backend, provide a \p ModuleToSummariesForIndex map.
5884void llvm::writeIndexToFile(
5885 const ModuleSummaryIndex &Index, raw_ostream &Out,
5886 const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex,
5887 const GVSummaryPtrSet *DecSummaries) {
5888 SmallVector<char, 0> Buffer;
5889 Buffer.reserve(N: 256 * 1024);
5890
5891 BitcodeWriter Writer(Buffer);
5892 Writer.writeIndex(Index: &Index, ModuleToSummariesForIndex, DecSummaries);
5893 Writer.writeStrtab();
5894
5895 Out.write(Ptr: (char *)&Buffer.front(), Size: Buffer.size());
5896}
5897
5898namespace {
5899
5900/// Class to manage the bitcode writing for a thin link bitcode file.
5901class ThinLinkBitcodeWriter : public ModuleBitcodeWriterBase {
5902 /// ModHash is for use in ThinLTO incremental build, generated while writing
5903 /// the module bitcode file.
5904 const ModuleHash *ModHash;
5905
5906public:
5907 ThinLinkBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
5908 BitstreamWriter &Stream,
5909 const ModuleSummaryIndex &Index,
5910 const ModuleHash &ModHash)
5911 : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
5912 /*ShouldPreserveUseListOrder=*/false, &Index),
5913 ModHash(&ModHash) {}
5914
5915 void write();
5916
5917private:
5918 void writeSimplifiedModuleInfo();
5919};
5920
5921} // end anonymous namespace
5922
5923// This function writes a simpilified module info for thin link bitcode file.
5924// It only contains the source file name along with the name(the offset and
5925// size in strtab) and linkage for global values. For the global value info
5926// entry, in order to keep linkage at offset 5, there are three zeros used
5927// as padding.
5928void ThinLinkBitcodeWriter::writeSimplifiedModuleInfo() {
5929 SmallVector<unsigned, 64> Vals;
5930 // Emit the module's source file name.
5931 {
5932 StringEncoding Bits = getStringEncoding(Str: M.getSourceFileName());
5933 BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
5934 if (Bits == SE_Char6)
5935 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
5936 else if (Bits == SE_Fixed7)
5937 AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
5938
5939 // MODULE_CODE_SOURCE_FILENAME: [namechar x N]
5940 auto Abbv = std::make_shared<BitCodeAbbrev>();
5941 Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
5942 Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5943 Abbv->Add(OpInfo: AbbrevOpToUse);
5944 unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5945
5946 for (const auto P : M.getSourceFileName())
5947 Vals.push_back(Elt: (unsigned char)P);
5948
5949 Stream.EmitRecord(Code: bitc::MODULE_CODE_SOURCE_FILENAME, Vals, Abbrev: FilenameAbbrev);
5950 Vals.clear();
5951 }
5952
5953 writeGUIDList();
5954
5955 // Emit the global variable information.
5956 for (const GlobalVariable &GV : M.globals()) {
5957 // GLOBALVAR: [strtab offset, strtab size, 0, 0, 0, linkage]
5958 Vals.push_back(Elt: StrtabBuilder.add(S: GV.getName()));
5959 Vals.push_back(Elt: GV.getName().size());
5960 Vals.push_back(Elt: 0);
5961 Vals.push_back(Elt: 0);
5962 Vals.push_back(Elt: 0);
5963 Vals.push_back(Elt: getEncodedLinkage(GV));
5964
5965 Stream.EmitRecord(Code: bitc::MODULE_CODE_GLOBALVAR, Vals);
5966 Vals.clear();
5967 }
5968
5969 // Emit the function proto information.
5970 for (const Function &F : M) {
5971 // FUNCTION: [strtab offset, strtab size, 0, 0, 0, linkage]
5972 Vals.push_back(Elt: StrtabBuilder.add(S: F.getName()));
5973 Vals.push_back(Elt: F.getName().size());
5974 Vals.push_back(Elt: 0);
5975 Vals.push_back(Elt: 0);
5976 Vals.push_back(Elt: 0);
5977 Vals.push_back(Elt: getEncodedLinkage(GV: F));
5978
5979 Stream.EmitRecord(Code: bitc::MODULE_CODE_FUNCTION, Vals);
5980 Vals.clear();
5981 }
5982
5983 // Emit the alias information.
5984 for (const GlobalAlias &A : M.aliases()) {
5985 // ALIAS: [strtab offset, strtab size, 0, 0, 0, linkage]
5986 Vals.push_back(Elt: StrtabBuilder.add(S: A.getName()));
5987 Vals.push_back(Elt: A.getName().size());
5988 Vals.push_back(Elt: 0);
5989 Vals.push_back(Elt: 0);
5990 Vals.push_back(Elt: 0);
5991 Vals.push_back(Elt: getEncodedLinkage(GV: A));
5992
5993 Stream.EmitRecord(Code: bitc::MODULE_CODE_ALIAS, Vals);
5994 Vals.clear();
5995 }
5996
5997 // Emit the ifunc information.
5998 for (const GlobalIFunc &I : M.ifuncs()) {
5999 // IFUNC: [strtab offset, strtab size, 0, 0, 0, linkage]
6000 Vals.push_back(Elt: StrtabBuilder.add(S: I.getName()));
6001 Vals.push_back(Elt: I.getName().size());
6002 Vals.push_back(Elt: 0);
6003 Vals.push_back(Elt: 0);
6004 Vals.push_back(Elt: 0);
6005 Vals.push_back(Elt: getEncodedLinkage(GV: I));
6006
6007 Stream.EmitRecord(Code: bitc::MODULE_CODE_IFUNC, Vals);
6008 Vals.clear();
6009 }
6010}
6011
6012void ThinLinkBitcodeWriter::write() {
6013 Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
6014
6015 writeModuleVersion();
6016
6017 writeSimplifiedModuleInfo();
6018
6019 writePerModuleGlobalValueSummary();
6020
6021 // Write module hash.
6022 Stream.EmitRecord(Code: bitc::MODULE_CODE_HASH, Vals: ArrayRef<uint32_t>(*ModHash));
6023
6024 Stream.ExitBlock();
6025}
6026
6027void BitcodeWriter::writeThinLinkBitcode(const Module &M,
6028 const ModuleSummaryIndex &Index,
6029 const ModuleHash &ModHash) {
6030 assert(!WroteStrtab);
6031
6032 // The Mods vector is used by irsymtab::build, which requires non-const
6033 // Modules in case it needs to materialize metadata. But the bitcode writer
6034 // requires that the module is materialized, so we can cast to non-const here,
6035 // after checking that it is in fact materialized.
6036 assert(M.isMaterialized());
6037 Mods.push_back(x: const_cast<Module *>(&M));
6038
6039 ThinLinkBitcodeWriter ThinLinkWriter(M, StrtabBuilder, *Stream, Index,
6040 ModHash);
6041 ThinLinkWriter.write();
6042}
6043
6044// Write the specified thin link bitcode file to the given raw output stream,
6045// where it will be written in a new bitcode block. This is used when
6046// writing the per-module index file for ThinLTO.
6047void llvm::writeThinLinkBitcodeToFile(const Module &M, raw_ostream &Out,
6048 const ModuleSummaryIndex &Index,
6049 const ModuleHash &ModHash) {
6050 SmallVector<char, 0> Buffer;
6051 Buffer.reserve(N: 256 * 1024);
6052
6053 BitcodeWriter Writer(Buffer);
6054 Writer.writeThinLinkBitcode(M, Index, ModHash);
6055 Writer.writeSymtab();
6056 Writer.writeStrtab();
6057
6058 Out.write(Ptr: (char *)&Buffer.front(), Size: Buffer.size());
6059}
6060
6061static const char *getSectionNameForBitcode(const Triple &T) {
6062 switch (T.getObjectFormat()) {
6063 case Triple::MachO:
6064 return "__LLVM,__bitcode";
6065 case Triple::COFF:
6066 case Triple::ELF:
6067 case Triple::Wasm:
6068 case Triple::UnknownObjectFormat:
6069 return ".llvmbc";
6070 case Triple::GOFF:
6071 llvm_unreachable("GOFF is not yet implemented");
6072 break;
6073 case Triple::SPIRV:
6074 if (T.getVendor() == Triple::AMD)
6075 return ".llvmbc";
6076 llvm_unreachable("SPIRV is not yet implemented");
6077 break;
6078 case Triple::XCOFF:
6079 llvm_unreachable("XCOFF is not yet implemented");
6080 break;
6081 case Triple::DXContainer:
6082 llvm_unreachable("DXContainer is not yet implemented");
6083 break;
6084 }
6085 llvm_unreachable("Unimplemented ObjectFormatType");
6086}
6087
6088static const char *getSectionNameForCommandline(const Triple &T) {
6089 switch (T.getObjectFormat()) {
6090 case Triple::MachO:
6091 return "__LLVM,__cmdline";
6092 case Triple::COFF:
6093 case Triple::ELF:
6094 case Triple::Wasm:
6095 case Triple::UnknownObjectFormat:
6096 return ".llvmcmd";
6097 case Triple::GOFF:
6098 llvm_unreachable("GOFF is not yet implemented");
6099 break;
6100 case Triple::SPIRV:
6101 if (T.getVendor() == Triple::AMD)
6102 return ".llvmcmd";
6103 llvm_unreachable("SPIRV is not yet implemented");
6104 break;
6105 case Triple::XCOFF:
6106 llvm_unreachable("XCOFF is not yet implemented");
6107 break;
6108 case Triple::DXContainer:
6109 llvm_unreachable("DXC is not yet implemented");
6110 break;
6111 }
6112 llvm_unreachable("Unimplemented ObjectFormatType");
6113}
6114
6115void llvm::embedBitcodeInModule(llvm::Module &M, llvm::MemoryBufferRef Buf,
6116 bool EmbedBitcode, bool EmbedCmdline,
6117 const std::vector<uint8_t> &CmdArgs) {
6118 // Save llvm.compiler.used and remove it.
6119 SmallVector<Constant *, 2> UsedArray;
6120 SmallVector<GlobalValue *, 4> UsedGlobals;
6121 GlobalVariable *Used = collectUsedGlobalVariables(M, Vec&: UsedGlobals, CompilerUsed: true);
6122 Type *UsedElementType = Used ? Used->getValueType()->getArrayElementType()
6123 : PointerType::getUnqual(C&: M.getContext());
6124 for (auto *GV : UsedGlobals) {
6125 if (GV->getName() != "llvm.embedded.module" &&
6126 GV->getName() != "llvm.cmdline")
6127 UsedArray.push_back(
6128 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
6129 }
6130 if (Used)
6131 Used->eraseFromParent();
6132
6133 // Embed the bitcode for the llvm module.
6134 std::string Data;
6135 ArrayRef<uint8_t> ModuleData;
6136 Triple T(M.getTargetTriple());
6137
6138 if (EmbedBitcode) {
6139 if (Buf.getBufferSize() == 0 ||
6140 !isBitcode(BufPtr: (const unsigned char *)Buf.getBufferStart(),
6141 BufEnd: (const unsigned char *)Buf.getBufferEnd())) {
6142 // If the input is LLVM Assembly, bitcode is produced by serializing
6143 // the module. Use-lists order need to be preserved in this case.
6144 llvm::raw_string_ostream OS(Data);
6145 llvm::WriteBitcodeToFile(M, Out&: OS, /* ShouldPreserveUseListOrder */ true);
6146 ModuleData =
6147 ArrayRef<uint8_t>((const uint8_t *)OS.str().data(), OS.str().size());
6148 } else
6149 // If the input is LLVM bitcode, write the input byte stream directly.
6150 ModuleData = ArrayRef<uint8_t>((const uint8_t *)Buf.getBufferStart(),
6151 Buf.getBufferSize());
6152 }
6153 llvm::Constant *ModuleConstant =
6154 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts: ModuleData);
6155 llvm::GlobalVariable *GV = new llvm::GlobalVariable(
6156 M, ModuleConstant->getType(), true, llvm::GlobalValue::PrivateLinkage,
6157 ModuleConstant);
6158 GV->setSection(getSectionNameForBitcode(T));
6159 // Set alignment to 1 to prevent padding between two contributions from input
6160 // sections after linking.
6161 GV->setAlignment(Align(1));
6162 UsedArray.push_back(
6163 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
6164 if (llvm::GlobalVariable *Old =
6165 M.getGlobalVariable(Name: "llvm.embedded.module", AllowInternal: true)) {
6166 assert(Old->hasZeroLiveUses() &&
6167 "llvm.embedded.module can only be used once in llvm.compiler.used");
6168 GV->takeName(V: Old);
6169 Old->eraseFromParent();
6170 } else {
6171 GV->setName("llvm.embedded.module");
6172 }
6173
6174 // Skip if only bitcode needs to be embedded.
6175 if (EmbedCmdline) {
6176 // Embed command-line options.
6177 ArrayRef<uint8_t> CmdData(const_cast<uint8_t *>(CmdArgs.data()),
6178 CmdArgs.size());
6179 llvm::Constant *CmdConstant =
6180 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts: CmdData);
6181 GV = new llvm::GlobalVariable(M, CmdConstant->getType(), true,
6182 llvm::GlobalValue::PrivateLinkage,
6183 CmdConstant);
6184 GV->setSection(getSectionNameForCommandline(T));
6185 GV->setAlignment(Align(1));
6186 UsedArray.push_back(
6187 Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
6188 if (llvm::GlobalVariable *Old = M.getGlobalVariable(Name: "llvm.cmdline", AllowInternal: true)) {
6189 assert(Old->hasZeroLiveUses() &&
6190 "llvm.cmdline can only be used once in llvm.compiler.used");
6191 GV->takeName(V: Old);
6192 Old->eraseFromParent();
6193 } else {
6194 GV->setName("llvm.cmdline");
6195 }
6196 }
6197
6198 if (UsedArray.empty())
6199 return;
6200
6201 // Recreate llvm.compiler.used.
6202 ArrayType *ATy = ArrayType::get(ElementType: UsedElementType, NumElements: UsedArray.size());
6203 auto *NewUsed = new GlobalVariable(
6204 M, ATy, false, llvm::GlobalValue::AppendingLinkage,
6205 llvm::ConstantArray::get(T: ATy, V: UsedArray), "llvm.compiler.used");
6206 NewUsed->setSection("llvm.metadata");
6207}
6208