1//===- PDBFileBuilder.cpp - PDB File Creation -------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/DebugInfo/PDB/Native/PDBFileBuilder.h"
10#include "llvm/ADT/SmallString.h"
11#include "llvm/ADT/StringExtras.h"
12#include "llvm/DebugInfo/CodeView/GUID.h"
13#include "llvm/DebugInfo/MSF/MSFBuilder.h"
14#include "llvm/DebugInfo/MSF/MSFCommon.h"
15#include "llvm/DebugInfo/MSF/MappedBlockStream.h"
16#include "llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h"
17#include "llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h"
18#include "llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h"
19#include "llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h"
20#include "llvm/DebugInfo/PDB/Native/RawConstants.h"
21#include "llvm/DebugInfo/PDB/Native/RawError.h"
22#include "llvm/DebugInfo/PDB/Native/RawTypes.h"
23#include "llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h"
24#include "llvm/Support/BinaryStreamWriter.h"
25#include "llvm/Support/CRC.h"
26#include "llvm/Support/Path.h"
27#include "llvm/Support/TimeProfiler.h"
28#include "llvm/Support/xxhash.h"
29
30#include <ctime>
31
32using namespace llvm;
33using namespace llvm::codeview;
34using namespace llvm::msf;
35using namespace llvm::pdb;
36using namespace llvm::support;
37
38namespace llvm {
39class WritableBinaryStream;
40}
41
42PDBFileBuilder::PDBFileBuilder(BumpPtrAllocator &Allocator)
43 : Allocator(Allocator), InjectedSourceTable(2) {}
44
45PDBFileBuilder::~PDBFileBuilder() = default;
46
47Error PDBFileBuilder::initialize(uint32_t BlockSize) {
48 auto ExpectedMsf = MSFBuilder::create(Allocator, BlockSize);
49 if (!ExpectedMsf)
50 return ExpectedMsf.takeError();
51 Msf = std::make_unique<MSFBuilder>(args: std::move(*ExpectedMsf));
52 return Error::success();
53}
54
55MSFBuilder &PDBFileBuilder::getMsfBuilder() { return *Msf; }
56
57InfoStreamBuilder &PDBFileBuilder::getInfoBuilder() {
58 if (!Info)
59 Info = std::make_unique<InfoStreamBuilder>(args&: *Msf, args&: NamedStreams);
60 return *Info;
61}
62
63DbiStreamBuilder &PDBFileBuilder::getDbiBuilder() {
64 if (!Dbi)
65 Dbi = std::make_unique<DbiStreamBuilder>(args&: *Msf);
66 return *Dbi;
67}
68
69TpiStreamBuilder &PDBFileBuilder::getTpiBuilder() {
70 if (!Tpi)
71 Tpi = std::make_unique<TpiStreamBuilder>(args&: *Msf, args: StreamTPI);
72 return *Tpi;
73}
74
75TpiStreamBuilder &PDBFileBuilder::getIpiBuilder() {
76 if (!Ipi)
77 Ipi = std::make_unique<TpiStreamBuilder>(args&: *Msf, args: StreamIPI);
78 return *Ipi;
79}
80
81PDBStringTableBuilder &PDBFileBuilder::getStringTableBuilder() {
82 if (!Strings) {
83 Strings = std::make_unique<PDBStringTableBuilder>();
84 InjectedSourceHashTraits = StringTableHashTraits(*Strings);
85 }
86 return *Strings;
87}
88
89GSIStreamBuilder &PDBFileBuilder::getGsiBuilder() {
90 if (!Gsi)
91 Gsi = std::make_unique<GSIStreamBuilder>(args&: *Msf);
92 return *Gsi;
93}
94
95std::unique_ptr<SmallVector<char>> &PDBFileBuilder::getDXContainerData() {
96 if (!Dxc)
97 Dxc = std::make_unique<SmallVector<char>>();
98 return Dxc;
99}
100
101Expected<uint32_t> PDBFileBuilder::allocateNamedStream(StringRef Name,
102 uint32_t Size) {
103 auto ExpectedStream = Msf->addStream(Size);
104 if (ExpectedStream)
105 NamedStreams.set(Stream: Name, StreamNo: *ExpectedStream);
106 return ExpectedStream;
107}
108
109Error PDBFileBuilder::addNamedStream(StringRef Name, StringRef Data) {
110 Expected<uint32_t> ExpectedIndex = allocateNamedStream(Name, Size: Data.size());
111 if (!ExpectedIndex)
112 return ExpectedIndex.takeError();
113 assert(NamedStreamData.count(*ExpectedIndex) == 0);
114 NamedStreamData[*ExpectedIndex] = std::string(Data);
115 return Error::success();
116}
117
118void PDBFileBuilder::addInjectedSource(StringRef Name,
119 std::unique_ptr<MemoryBuffer> Buffer) {
120 // Stream names must be exact matches, since they get looked up in a hash
121 // table and the hash value is dependent on the exact contents of the string.
122 // link.exe lowercases a path and converts / to \, so we must do the same.
123 SmallString<64> VName;
124 sys::path::native(path: Name.lower(), result&: VName, style: sys::path::Style::windows_backslash);
125
126 uint32_t NI = getStringTableBuilder().insert(S: Name);
127 uint32_t VNI = getStringTableBuilder().insert(S: VName);
128
129 InjectedSourceDescriptor Desc;
130 Desc.Content = std::move(Buffer);
131 Desc.NameIndex = NI;
132 Desc.VNameIndex = VNI;
133 Desc.StreamName = "/src/files/";
134
135 Desc.StreamName += VName;
136
137 InjectedSources.push_back(Elt: std::move(Desc));
138}
139
140Error PDBFileBuilder::finalizeMsfLayout() {
141 llvm::TimeTraceScope timeScope("MSF layout");
142
143 if (Ipi && Ipi->getRecordCount() > 0) {
144 // In theory newer PDBs always have an ID stream, but by saying that we're
145 // only going to *really* have an ID stream if there is at least one ID
146 // record, we leave open the opportunity to test older PDBs such as those
147 // that don't have an ID stream.
148 auto &Info = getInfoBuilder();
149 Info.addFeature(Sig: PdbRaw_FeatureSig::VC140);
150 }
151
152 if (Dxc) {
153 if (auto EC = Msf->setStreamSize(Idx: StreamDXContainer, Size: Dxc->size()))
154 return EC;
155 } else {
156 Expected<uint32_t> SN = allocateNamedStream(Name: "/LinkInfo", Size: 0);
157 if (!SN)
158 return SN.takeError();
159 }
160
161 if (Gsi) {
162 if (auto EC = Gsi->finalizeMsfLayout())
163 return EC;
164 if (Dbi) {
165 Dbi->setPublicsStreamIndex(Gsi->getPublicsStreamIndex());
166 Dbi->setGlobalsStreamIndex(Gsi->getGlobalsStreamIndex());
167 Dbi->setSymbolRecordStreamIndex(Gsi->getRecordStreamIndex());
168 }
169 }
170 if (Tpi) {
171 if (auto EC = Tpi->finalizeMsfLayout())
172 return EC;
173 }
174 if (Dbi) {
175 if (auto EC = Dbi->finalizeMsfLayout())
176 return EC;
177 }
178 if (Strings) {
179 uint32_t StringsLen = Strings->calculateSerializedSize();
180 Expected<uint32_t> SN = allocateNamedStream(Name: "/names", Size: StringsLen);
181 if (!SN)
182 return SN.takeError();
183 }
184 if (Ipi) {
185 if (auto EC = Ipi->finalizeMsfLayout())
186 return EC;
187 }
188
189 // Do this last, since it relies on the named stream map being complete, and
190 // that can be updated by previous steps in the finalization.
191 if (Info) {
192 if (auto EC = Info->finalizeMsfLayout())
193 return EC;
194 }
195
196 if (!InjectedSources.empty()) {
197 for (const auto &IS : InjectedSources) {
198 JamCRC CRC(0);
199 CRC.update(Data: arrayRefFromStringRef(Input: IS.Content->getBuffer()));
200
201 SrcHeaderBlockEntry Entry;
202 ::memset(s: &Entry, c: 0, n: sizeof(SrcHeaderBlockEntry));
203 Entry.Size = sizeof(SrcHeaderBlockEntry);
204 Entry.FileSize = IS.Content->getBufferSize();
205 Entry.FileNI = IS.NameIndex;
206 Entry.VFileNI = IS.VNameIndex;
207 Entry.ObjNI = 1;
208 Entry.IsVirtual = 0;
209 Entry.Version =
210 static_cast<uint32_t>(PdbRaw_SrcHeaderBlockVer::SrcVerOne);
211 Entry.CRC = CRC.getCRC();
212 StringRef VName = getStringTableBuilder().getStringForId(Id: IS.VNameIndex);
213 InjectedSourceTable.set_as(K: VName, V: std::move(Entry),
214 Traits&: InjectedSourceHashTraits);
215 }
216
217 uint32_t SrcHeaderBlockSize =
218 sizeof(SrcHeaderBlockHeader) +
219 InjectedSourceTable.calculateSerializedLength();
220 Expected<uint32_t> SN =
221 allocateNamedStream(Name: "/src/headerblock", Size: SrcHeaderBlockSize);
222 if (!SN)
223 return SN.takeError();
224 for (const auto &IS : InjectedSources) {
225 SN = allocateNamedStream(Name: IS.StreamName, Size: IS.Content->getBufferSize());
226 if (!SN)
227 return SN.takeError();
228 }
229 }
230
231 // Do this last, since it relies on the named stream map being complete, and
232 // that can be updated by previous steps in the finalization.
233 if (Info) {
234 if (auto EC = Info->finalizeMsfLayout())
235 return EC;
236 }
237
238 return Error::success();
239}
240
241Expected<uint32_t> PDBFileBuilder::getNamedStreamIndex(StringRef Name) const {
242 uint32_t SN = 0;
243 if (!NamedStreams.get(Stream: Name, StreamNo&: SN))
244 return llvm::make_error<pdb::RawError>(Args: raw_error_code::no_stream);
245 return SN;
246}
247
248void PDBFileBuilder::commitSrcHeaderBlock(WritableBinaryStream &MsfBuffer,
249 const msf::MSFLayout &Layout) {
250 assert(!InjectedSourceTable.empty());
251
252 uint32_t SN = cantFail(ValOrErr: getNamedStreamIndex(Name: "/src/headerblock"));
253 auto Stream = WritableMappedBlockStream::createIndexedStream(
254 Layout, MsfData: MsfBuffer, StreamIndex: SN, Allocator);
255 BinaryStreamWriter Writer(*Stream);
256
257 SrcHeaderBlockHeader Header;
258 ::memset(s: &Header, c: 0, n: sizeof(Header));
259 Header.Version = static_cast<uint32_t>(PdbRaw_SrcHeaderBlockVer::SrcVerOne);
260 Header.Size = Writer.bytesRemaining();
261
262 cantFail(Err: Writer.writeObject(Obj: Header));
263 cantFail(Err: InjectedSourceTable.commit(Writer));
264
265 assert(Writer.bytesRemaining() == 0);
266}
267
268void PDBFileBuilder::commitInjectedSources(WritableBinaryStream &MsfBuffer,
269 const msf::MSFLayout &Layout) {
270 if (InjectedSourceTable.empty())
271 return;
272
273 llvm::TimeTraceScope timeScope("Commit injected sources");
274 commitSrcHeaderBlock(MsfBuffer, Layout);
275
276 for (const auto &IS : InjectedSources) {
277 uint32_t SN = cantFail(ValOrErr: getNamedStreamIndex(Name: IS.StreamName));
278
279 auto SourceStream = WritableMappedBlockStream::createIndexedStream(
280 Layout, MsfData: MsfBuffer, StreamIndex: SN, Allocator);
281 BinaryStreamWriter SourceWriter(*SourceStream);
282 assert(SourceWriter.bytesRemaining() == IS.Content->getBufferSize());
283 cantFail(Err: SourceWriter.writeBytes(
284 Buffer: arrayRefFromStringRef(Input: IS.Content->getBuffer())));
285 }
286}
287
288Error PDBFileBuilder::commit(StringRef Filename, codeview::GUID *Guid) {
289 assert(!Filename.empty());
290 if (auto EC = finalizeMsfLayout())
291 return EC;
292
293 MSFLayout Layout;
294 Expected<FileBufferByteStream> ExpectedMsfBuffer =
295 Msf->commit(Path: Filename, Layout);
296 if (!ExpectedMsfBuffer)
297 return ExpectedMsfBuffer.takeError();
298 FileBufferByteStream Buffer = std::move(*ExpectedMsfBuffer);
299
300 if (Strings) {
301 auto ExpectedSN = getNamedStreamIndex(Name: "/names");
302 if (!ExpectedSN)
303 return ExpectedSN.takeError();
304
305 auto NS = WritableMappedBlockStream::createIndexedStream(
306 Layout, MsfData: Buffer, StreamIndex: *ExpectedSN, Allocator);
307 BinaryStreamWriter NSWriter(*NS);
308 if (auto EC = Strings->commit(Writer&: NSWriter))
309 return EC;
310 }
311 {
312 llvm::TimeTraceScope timeScope("Named stream data");
313 for (const auto &NSE : NamedStreamData) {
314 if (NSE.second.empty())
315 continue;
316
317 auto NS = WritableMappedBlockStream::createIndexedStream(
318 Layout, MsfData: Buffer, StreamIndex: NSE.first, Allocator);
319 BinaryStreamWriter NSW(*NS);
320 if (auto EC = NSW.writeBytes(Buffer: arrayRefFromStringRef(Input: NSE.second)))
321 return EC;
322 }
323 }
324
325 if (Info) {
326 if (auto EC = Info->commit(Layout, Buffer))
327 return EC;
328 }
329
330 if (Dbi) {
331 if (auto EC = Dbi->commit(Layout, MsfBuffer: Buffer))
332 return EC;
333 }
334
335 if (Tpi) {
336 if (auto EC = Tpi->commit(Layout, Buffer))
337 return EC;
338 }
339
340 if (Ipi) {
341 if (auto EC = Ipi->commit(Layout, Buffer))
342 return EC;
343 }
344
345 if (Gsi) {
346 if (auto EC = Gsi->commit(Layout, Buffer))
347 return EC;
348 }
349
350 if (Dxc) {
351 llvm::TimeTraceScope timeScope("DXContainer stream");
352 auto DxcS = WritableMappedBlockStream::createIndexedStream(
353 Layout, MsfData: Buffer, StreamIndex: StreamDXContainer, Allocator);
354 BinaryStreamWriter Writer(*DxcS);
355 llvm::ArrayRef<uint8_t> DataRef(reinterpret_cast<uint8_t *>(Dxc->data()),
356 Dxc->size());
357 if (auto EC = Writer.writeBytes(Buffer: DataRef))
358 return EC;
359 }
360
361 auto InfoStreamBlocks = Layout.StreamMap[StreamPDB];
362 assert(!InfoStreamBlocks.empty());
363 uint64_t InfoStreamFileOffset =
364 blockToOffset(BlockNumber: InfoStreamBlocks.front(), BlockSize: Layout.SB->BlockSize);
365 InfoStreamHeader *H = reinterpret_cast<InfoStreamHeader *>(
366 Buffer.getBufferStart() + InfoStreamFileOffset);
367
368 commitInjectedSources(MsfBuffer&: Buffer, Layout);
369
370 // Set the build id at the very end, after every other byte of the PDB
371 // has been written.
372 if (Info->hashPDBContentsToGUID()) {
373 llvm::TimeTraceScope timeScope("Compute build ID");
374
375 // Compute a hash of all sections of the output file.
376 uint64_t Digest =
377 xxh3_64bits(data: {Buffer.getBufferStart(), Buffer.getBufferEnd()});
378
379 H->Age = 1;
380
381 memcpy(dest: H->Guid.Guid, src: &Digest, n: 8);
382 // xxhash only gives us 8 bytes, so put some fixed data in the other half.
383 memcpy(dest: H->Guid.Guid + 8, src: "LLD PDB.", n: 8);
384
385 // Put the hash in the Signature field too.
386 H->Signature = static_cast<uint32_t>(Digest);
387
388 // Return GUID to caller.
389 memcpy(dest: Guid, src: H->Guid.Guid, n: 16);
390 } else {
391 H->Age = Info->getAge();
392 H->Guid = Info->getGuid();
393 std::optional<uint32_t> Sig = Info->getSignature();
394 H->Signature = Sig ? *Sig : time(timer: nullptr);
395 }
396
397 return Buffer.commit();
398}
399