1//===- OffloadBundle.cpp - Utilities for offload bundles---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------===//
8
9#include "llvm/Object/OffloadBundle.h"
10#include "llvm/BinaryFormat/Magic.h"
11#include "llvm/IR/Module.h"
12#include "llvm/IRReader/IRReader.h"
13#include "llvm/MC/StringTableBuilder.h"
14#include "llvm/Object/Archive.h"
15#include "llvm/Object/Binary.h"
16#include "llvm/Object/COFF.h"
17#include "llvm/Object/ELFObjectFile.h"
18#include "llvm/Object/Error.h"
19#include "llvm/Object/IRObjectFile.h"
20#include "llvm/Object/ObjectFile.h"
21#include "llvm/Support/BinaryStreamReader.h"
22#include "llvm/Support/SourceMgr.h"
23#include "llvm/Support/Timer.h"
24
25using namespace llvm;
26using namespace llvm::object;
27
28static TimerGroup OffloadBundlerTimerGroup("Offload Bundler Timer Group",
29 "Timer group for offload bundler");
30
31// Returns the on-disk size recorded in the compressed offload bundle header at
32// the start of \p Blob, or std::nullopt if the header carries no size field.
33static std::optional<size_t> getCompressedBundleSize(StringRef Blob) {
34 Expected<CompressedOffloadBundle::CompressedBundleHeader> HeaderOrErr =
35 CompressedOffloadBundle::CompressedBundleHeader::tryParse(Blob);
36 if (!HeaderOrErr) {
37 consumeError(Err: HeaderOrErr.takeError());
38 return std::nullopt;
39 }
40 return HeaderOrErr->FileSize;
41}
42
43// Extract an Offload bundle (usually a Offload Bundle) from a fat_bin
44// section.
45Error extractOffloadBundle(MemoryBufferRef Contents, uint64_t SectionOffset,
46 StringRef FileName,
47 SmallVectorImpl<OffloadBundleFatBin> &Bundles) {
48
49 size_t Offset = 0;
50 size_t NextbundleStart = 0;
51 StringRef Magic;
52 std::unique_ptr<MemoryBuffer> Buffer;
53
54 // There could be multiple offloading bundles stored at this section.
55 while ((NextbundleStart != StringRef::npos) &&
56 (Offset < Contents.getBuffer().size())) {
57 Buffer =
58 MemoryBuffer::getMemBuffer(InputData: Contents.getBuffer().drop_front(N: Offset), BufferName: "",
59 /*RequiresNullTerminator=*/false);
60
61 if (identify_magic(magic: (*Buffer).getBuffer()) ==
62 file_magic::offload_bundle_compressed) {
63 Magic = "CCOB";
64 // Locate this bundle's end and the next bundle from the header size.
65 size_t CurBundleEnd;
66 if (std::optional<size_t> Size =
67 getCompressedBundleSize(Blob: (*Buffer).getBuffer())) {
68 CurBundleEnd = *Size;
69 NextbundleStart = (*Buffer).getBuffer().find(Str: Magic, From: *Size);
70 } else {
71 // Legacy bundle without a recorded size: fall back to magic scanning.
72 NextbundleStart = (*Buffer).getBuffer().find(Str: Magic, From: Magic.size());
73 CurBundleEnd = NextbundleStart;
74 }
75 if (NextbundleStart == StringRef::npos) {
76 NextbundleStart = (*Buffer).getBuffer().size();
77 if (CurBundleEnd == StringRef::npos)
78 CurBundleEnd = (*Buffer).getBuffer().size();
79 }
80
81 ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr =
82 MemoryBuffer::getMemBuffer(
83 InputData: (*Buffer).getBuffer().take_front(N: CurBundleEnd), BufferName: FileName, RequiresNullTerminator: false);
84 if (std::error_code EC = CodeOrErr.getError())
85 return createFileError(F: FileName, EC);
86
87 Expected<std::unique_ptr<MemoryBuffer>> DecompressedBufferOrErr =
88 CompressedOffloadBundle::decompress(Input: **CodeOrErr, VerboseStream: nullptr);
89 if (!DecompressedBufferOrErr)
90 return createStringError(S: "failed to decompress input: " +
91 toString(E: DecompressedBufferOrErr.takeError()));
92
93 auto FatBundleOrErr = OffloadBundleFatBin::create(
94 **DecompressedBufferOrErr, SectionOffset: Offset, FileName, Decompress: true);
95 if (!FatBundleOrErr)
96 return FatBundleOrErr.takeError();
97
98 // Add current Bundle to list.
99 Bundles.emplace_back(Args: std::move(**FatBundleOrErr));
100
101 } else if (identify_magic(magic: (*Buffer).getBuffer()) ==
102 file_magic::offload_bundle) {
103 // Create the OffloadBundleFatBin object. This will also create the Bundle
104 // Entry list info.
105 auto FatBundleOrErr = OffloadBundleFatBin::create(
106 *Buffer, SectionOffset: SectionOffset + Offset, FileName);
107 if (!FatBundleOrErr)
108 return FatBundleOrErr.takeError();
109
110 // Add current Bundle to list.
111 Bundles.emplace_back(Args: std::move(**FatBundleOrErr));
112
113 Magic = "__CLANG_OFFLOAD_BUNDLE__";
114 NextbundleStart = (*Buffer).getBuffer().find(Str: Magic, From: Magic.size());
115 }
116
117 if (NextbundleStart != StringRef::npos)
118 Offset += NextbundleStart;
119 }
120
121 return Error::success();
122}
123
124Error OffloadBundleFatBin::readEntries(StringRef Buffer,
125 uint64_t SectionOffset) {
126 uint64_t NumOfEntries = 0;
127
128 BinaryStreamReader Reader(Buffer, llvm::endianness::little);
129
130 // Read the Magic String first.
131 StringRef Magic;
132 if (auto EC = Reader.readFixedString(Dest&: Magic, Length: 24))
133 return errorCodeToError(EC: object_error::parse_failed);
134
135 // Read the number of Code Objects (Entries) in the current Bundle.
136 if (auto EC = Reader.readInteger(Dest&: NumOfEntries))
137 return errorCodeToError(EC: object_error::parse_failed);
138
139 NumberOfEntries = NumOfEntries;
140
141 // For each Bundle Entry (code object).
142 for (uint64_t I = 0; I < NumOfEntries; I++) {
143 uint64_t EntrySize;
144 uint64_t EntryOffset;
145 uint64_t EntryIDSize;
146 StringRef EntryID;
147
148 if (Error Err = Reader.readInteger(Dest&: EntryOffset))
149 return Err;
150
151 if (Error Err = Reader.readInteger(Dest&: EntrySize))
152 return Err;
153
154 if (Error Err = Reader.readInteger(Dest&: EntryIDSize))
155 return Err;
156
157 if (Error Err = Reader.readFixedString(Dest&: EntryID, Length: EntryIDSize))
158 return Err;
159
160 auto Entry = std::make_unique<OffloadBundleEntry>(
161 args: EntryOffset + SectionOffset, args&: EntrySize, args&: EntryIDSize, args&: EntryID);
162
163 Entries.push_back(Elt: *Entry);
164 }
165
166 return Error::success();
167}
168
169Expected<std::unique_ptr<OffloadBundleFatBin>>
170OffloadBundleFatBin::create(MemoryBufferRef Buf, uint64_t SectionOffset,
171 StringRef FileName, bool Decompress) {
172 if (Buf.getBufferSize() < 24)
173 return errorCodeToError(EC: object_error::parse_failed);
174
175 // Check for magic bytes.
176 if ((identify_magic(magic: Buf.getBuffer()) != file_magic::offload_bundle) &&
177 (identify_magic(magic: Buf.getBuffer()) !=
178 file_magic::offload_bundle_compressed))
179 return errorCodeToError(EC: object_error::parse_failed);
180
181 std::unique_ptr<OffloadBundleFatBin> TheBundle(
182 new OffloadBundleFatBin(Buf, FileName, Decompress));
183
184 // Read the Bundle Entries.
185 Error Err =
186 TheBundle->readEntries(Buffer: Buf.getBuffer(), SectionOffset: Decompress ? 0 : SectionOffset);
187 if (Err)
188 return Err;
189
190 return std::move(TheBundle);
191}
192
193Error OffloadBundleFatBin::extractBundle(const ObjectFile &Source) {
194 // This will extract all entries in the Bundle.
195 for (OffloadBundleEntry &Entry : Entries) {
196
197 if (Entry.Size == 0)
198 continue;
199
200 // create output file name. Which should be
201 // <fileName>-offset<Offset>-size<Size>.co"
202 std::string Str = getFileName().str() + "-offset" + itostr(X: Entry.Offset) +
203 "-size" + itostr(X: Entry.Size) + ".co";
204 if (Error Err = object::extractCodeObject(Source, Offset: Entry.Offset, Size: Entry.Size,
205 OutputFileName: StringRef(Str)))
206 return Err;
207 }
208
209 return Error::success();
210}
211
212Error object::extractOffloadBundleFatBinary(
213 const ObjectFile &Obj, SmallVectorImpl<OffloadBundleFatBin> &Bundles) {
214 // Ignore unsupported object formats.
215 if (!Obj.isELF() && !Obj.isCOFF())
216 return Error::success();
217
218 // Iterate through Sections until we find an offload_bundle section.
219 for (SectionRef Sec : Obj.sections()) {
220 Expected<StringRef> Buffer = Sec.getContents();
221 if (!Buffer)
222 return Buffer.takeError();
223
224 // If it does not start with the reserved suffix, just skip this section.
225 if ((llvm::identify_magic(magic: *Buffer) == file_magic::offload_bundle) ||
226 (llvm::identify_magic(magic: *Buffer) ==
227 file_magic::offload_bundle_compressed)) {
228
229 uint64_t SectionOffset = 0;
230 if (Obj.isELF()) {
231 SectionOffset = ELFSectionRef(Sec).getOffset();
232 } else if (Obj.isCOFF()) {
233 const COFFObjectFile &COFF = cast<COFFObjectFile>(Val: Obj);
234 Expected<const coff_section *> SecOrErr =
235 COFF.getSection(index: COFF.getSectionID(Sec));
236 if (!SecOrErr)
237 return SecOrErr.takeError();
238 SectionOffset = (*SecOrErr)->PointerToRawData;
239 }
240
241 MemoryBufferRef Contents(*Buffer, Obj.getFileName());
242 if (Error Err = extractOffloadBundle(Contents, SectionOffset,
243 FileName: Obj.getFileName(), Bundles))
244 return Err;
245 }
246 }
247 return Error::success();
248}
249
250Error object::extractCodeObject(const ObjectFile &Source, size_t Offset,
251 size_t Size, StringRef OutputFileName) {
252 Expected<std::unique_ptr<FileOutputBuffer>> BufferOrErr =
253 FileOutputBuffer::create(FilePath: OutputFileName, Size);
254
255 if (!BufferOrErr)
256 return BufferOrErr.takeError();
257
258 Expected<MemoryBufferRef> InputBuffOrErr = Source.getMemoryBufferRef();
259 if (Error Err = InputBuffOrErr.takeError())
260 return createFileError(F: Source.getFileName(), E: std::move(Err));
261
262 if (Size > InputBuffOrErr->getBufferSize())
263 return createStringError(Fmt: "size in URI (%zu) is larger than source (%zu)",
264 Vals: Size, Vals: InputBuffOrErr->getBufferSize());
265
266 if (Offset > InputBuffOrErr->getBufferSize())
267 return createStringError(
268 Fmt: "offset in URI (%zu) is beyond the end of the source (%zu)", Vals: Offset,
269 Vals: InputBuffOrErr->getBufferSize());
270
271 if (Offset + Size > InputBuffOrErr->getBufferSize())
272 return createStringError(
273 Fmt: "offset + size (%zu) in URI is beyond the end of the source (%zu)",
274 Vals: Offset + Size, Vals: InputBuffOrErr->getBufferSize());
275
276 std::unique_ptr<FileOutputBuffer> Buf = std::move(*BufferOrErr);
277 std::copy(first: InputBuffOrErr->getBufferStart() + Offset,
278 last: InputBuffOrErr->getBufferStart() + Offset + Size,
279 result: Buf->getBufferStart());
280 if (Error E = Buf->commit())
281 return createFileError(F: OutputFileName, E: std::move(E));
282
283 return Error::success();
284}
285
286Error object::extractCodeObject(const MemoryBufferRef Buffer, int64_t Offset,
287 int64_t Size, StringRef OutputFileName) {
288 Expected<std::unique_ptr<FileOutputBuffer>> BufferOrErr =
289 FileOutputBuffer::create(FilePath: OutputFileName, Size);
290 if (!BufferOrErr)
291 return BufferOrErr.takeError();
292
293 std::unique_ptr<FileOutputBuffer> Buf = std::move(*BufferOrErr);
294 std::copy(first: Buffer.getBufferStart() + Offset,
295 last: Buffer.getBufferStart() + Offset + Size, result: Buf->getBufferStart());
296
297 return Buf->commit();
298}
299
300// given a file name, offset, and size, extract data into a code object file,
301// into file "<SourceFile>-offset<Offset>-size<Size>.co".
302Error object::extractOffloadBundleByURI(StringRef URIstr) {
303 // create a URI object
304 Expected<std::unique_ptr<OffloadBundleURI>> UriOrErr(
305 OffloadBundleURI::createOffloadBundleURI(Str: URIstr, Type: FILE_URI));
306
307 if (!UriOrErr)
308 return UriOrErr.takeError();
309
310 OffloadBundleURI &Uri = **UriOrErr;
311 std::string OutputFile = Uri.FileName.str();
312 OutputFile +=
313 "-offset" + itostr(X: Uri.Offset) + "-size" + itostr(X: Uri.Size) + ".co";
314
315 // Create an ObjectFile object from uri.file_uri.
316 auto ObjOrErr = ObjectFile::createObjectFile(ObjectPath: Uri.FileName);
317 if (!ObjOrErr)
318 return ObjOrErr.takeError();
319
320 auto Obj = ObjOrErr->getBinary();
321 if (Error Err =
322 object::extractCodeObject(Source: *Obj, Offset: Uri.Offset, Size: Uri.Size, OutputFileName: OutputFile))
323 return createFileError(F: Uri.FileName, E: std::move(Err));
324
325 return Error::success();
326}
327
328// Utility function to format numbers with commas.
329static std::string formatWithCommas(unsigned long long Value) {
330 std::string Num = std::to_string(val: Value);
331 int InsertPosition = Num.length() - 3;
332 while (InsertPosition > 0) {
333 Num.insert(pos: InsertPosition, s: ",");
334 InsertPosition -= 3;
335 }
336 return Num;
337}
338
339Expected<std::unique_ptr<MemoryBuffer>>
340CompressedOffloadBundle::compress(compression::Params P,
341 const MemoryBuffer &Input, uint16_t Version,
342 raw_ostream *VerboseStream) {
343 if (!compression::zstd::isAvailable() && !compression::zlib::isAvailable())
344 return createStringError(Fmt: "compression not supported.");
345 Timer HashTimer("Hash Calculation Timer", "Hash calculation time",
346 OffloadBundlerTimerGroup);
347 if (VerboseStream)
348 HashTimer.startTimer();
349 MD5 Hash;
350 MD5::MD5Result Result;
351 Hash.update(Str: Input.getBuffer());
352 Hash.final(Result);
353 uint64_t TruncatedHash = Result.low();
354 if (VerboseStream)
355 HashTimer.stopTimer();
356
357 SmallVector<uint8_t, 0> CompressedBuffer;
358 auto BufferUint8 = ArrayRef<uint8_t>(
359 reinterpret_cast<const uint8_t *>(Input.getBuffer().data()),
360 Input.getBuffer().size());
361 Timer CompressTimer("Compression Timer", "Compression time",
362 OffloadBundlerTimerGroup);
363 if (VerboseStream)
364 CompressTimer.startTimer();
365 compression::compress(P, Input: BufferUint8, Output&: CompressedBuffer);
366 if (VerboseStream)
367 CompressTimer.stopTimer();
368
369 uint16_t CompressionMethod = static_cast<uint16_t>(P.format);
370
371 // Store sizes in 64-bit variables first.
372 uint64_t UncompressedSize64 = Input.getBuffer().size();
373 uint64_t TotalFileSize64;
374
375 // Calculate total file size based on version.
376 if (Version == 2) {
377 // For V2, ensure the sizes don't exceed 32-bit limit.
378 if (UncompressedSize64 > std::numeric_limits<uint32_t>::max())
379 return createStringError(Fmt: "uncompressed size (%llu) exceeds version 2 "
380 "unsigned 32-bit integer limit",
381 Vals: UncompressedSize64);
382 TotalFileSize64 = MagicNumber.size() + sizeof(uint32_t) + sizeof(Version) +
383 sizeof(CompressionMethod) + sizeof(uint32_t) +
384 sizeof(TruncatedHash) + CompressedBuffer.size();
385 if (TotalFileSize64 > std::numeric_limits<uint32_t>::max())
386 return createStringError(Fmt: "total file size (%llu) exceeds version 2 "
387 "unsigned 32-bit integer limit",
388 Vals: TotalFileSize64);
389
390 } else { // Version 3.
391 TotalFileSize64 = MagicNumber.size() + sizeof(uint64_t) + sizeof(Version) +
392 sizeof(CompressionMethod) + sizeof(uint64_t) +
393 sizeof(TruncatedHash) + CompressedBuffer.size();
394 }
395
396 SmallVector<char, 0> FinalBuffer;
397 raw_svector_ostream OS(FinalBuffer);
398 OS << MagicNumber;
399 OS.write(Ptr: reinterpret_cast<const char *>(&Version), Size: sizeof(Version));
400 OS.write(Ptr: reinterpret_cast<const char *>(&CompressionMethod),
401 Size: sizeof(CompressionMethod));
402
403 // Write size fields according to version.
404 if (Version == 2) {
405 uint32_t TotalFileSize32 = static_cast<uint32_t>(TotalFileSize64);
406 uint32_t UncompressedSize32 = static_cast<uint32_t>(UncompressedSize64);
407 OS.write(Ptr: reinterpret_cast<const char *>(&TotalFileSize32),
408 Size: sizeof(TotalFileSize32));
409 OS.write(Ptr: reinterpret_cast<const char *>(&UncompressedSize32),
410 Size: sizeof(UncompressedSize32));
411 } else { // Version 3.
412 OS.write(Ptr: reinterpret_cast<const char *>(&TotalFileSize64),
413 Size: sizeof(TotalFileSize64));
414 OS.write(Ptr: reinterpret_cast<const char *>(&UncompressedSize64),
415 Size: sizeof(UncompressedSize64));
416 }
417
418 OS.write(Ptr: reinterpret_cast<const char *>(&TruncatedHash),
419 Size: sizeof(TruncatedHash));
420 OS.write(Ptr: reinterpret_cast<const char *>(CompressedBuffer.data()),
421 Size: CompressedBuffer.size());
422
423 if (VerboseStream) {
424 auto MethodUsed = P.format == compression::Format::Zstd ? "zstd" : "zlib";
425 double CompressionRate =
426 static_cast<double>(UncompressedSize64) / CompressedBuffer.size();
427 double CompressionTimeSeconds = CompressTimer.getTotalTime().getWallTime();
428 double CompressionSpeedMBs =
429 (UncompressedSize64 / (1024.0 * 1024.0)) / CompressionTimeSeconds;
430 *VerboseStream << "Compressed bundle format version: " << Version << "\n"
431 << "Total file size (including headers): "
432 << formatWithCommas(Value: TotalFileSize64) << " bytes\n"
433 << "Compression method used: " << MethodUsed << "\n"
434 << "Compression level: " << P.level << "\n"
435 << "Binary size before compression: "
436 << formatWithCommas(Value: UncompressedSize64) << " bytes\n"
437 << "Binary size after compression: "
438 << formatWithCommas(Value: CompressedBuffer.size()) << " bytes\n"
439 << "Compression rate: " << format(Fmt: "%.2lf", Vals: CompressionRate)
440 << "\n"
441 << "Compression ratio: "
442 << format(Fmt: "%.2lf%%", Vals: 100.0 / CompressionRate) << "\n"
443 << "Compression speed: "
444 << format(Fmt: "%.2lf MB/s", Vals: CompressionSpeedMBs) << "\n"
445 << "Truncated MD5 hash: " << format_hex(N: TruncatedHash, Width: 16)
446 << "\n";
447 }
448
449 return MemoryBuffer::getMemBufferCopy(
450 InputData: StringRef(FinalBuffer.data(), FinalBuffer.size()));
451}
452
453// Use packed structs to avoid padding, such that the structs map the serialized
454// format.
455LLVM_PACKED_START
456union RawCompressedBundleHeader {
457 struct CommonFields {
458 uint32_t Magic;
459 uint16_t Version;
460 uint16_t Method;
461 };
462
463 struct V1Header {
464 CommonFields Common;
465 uint32_t UncompressedFileSize;
466 uint64_t Hash;
467 };
468
469 struct V2Header {
470 CommonFields Common;
471 uint32_t FileSize;
472 uint32_t UncompressedFileSize;
473 uint64_t Hash;
474 };
475
476 struct V3Header {
477 CommonFields Common;
478 uint64_t FileSize;
479 uint64_t UncompressedFileSize;
480 uint64_t Hash;
481 };
482
483 CommonFields Common;
484 V1Header V1;
485 V2Header V2;
486 V3Header V3;
487};
488LLVM_PACKED_END
489
490// Helper method to get header size based on version.
491static size_t getHeaderSize(uint16_t Version) {
492 switch (Version) {
493 case 1:
494 return sizeof(RawCompressedBundleHeader::V1Header);
495 case 2:
496 return sizeof(RawCompressedBundleHeader::V2Header);
497 case 3:
498 return sizeof(RawCompressedBundleHeader::V3Header);
499 default:
500 llvm_unreachable("Unsupported version");
501 }
502}
503
504Expected<CompressedOffloadBundle::CompressedBundleHeader>
505CompressedOffloadBundle::CompressedBundleHeader::tryParse(StringRef Blob) {
506 assert(Blob.size() >= sizeof(RawCompressedBundleHeader::CommonFields));
507 assert(identify_magic(Blob) == file_magic::offload_bundle_compressed);
508
509 RawCompressedBundleHeader Header;
510 std::memcpy(dest: &Header, src: Blob.data(), n: std::min(a: Blob.size(), b: sizeof(Header)));
511
512 CompressedBundleHeader Normalized;
513 Normalized.Version = Header.Common.Version;
514
515 size_t RequiredSize = getHeaderSize(Version: Normalized.Version);
516
517 if (Blob.size() < RequiredSize)
518 return createStringError(Fmt: "compressed bundle header size too small");
519
520 switch (Normalized.Version) {
521 case 1:
522 Normalized.UncompressedFileSize = Header.V1.UncompressedFileSize;
523 Normalized.Hash = Header.V1.Hash;
524 break;
525 case 2:
526 Normalized.FileSize = Header.V2.FileSize;
527 Normalized.UncompressedFileSize = Header.V2.UncompressedFileSize;
528 Normalized.Hash = Header.V2.Hash;
529 break;
530 case 3:
531 Normalized.FileSize = Header.V3.FileSize;
532 Normalized.UncompressedFileSize = Header.V3.UncompressedFileSize;
533 Normalized.Hash = Header.V3.Hash;
534 break;
535 default:
536 return createStringError(Fmt: "unknown compressed bundle version");
537 }
538
539 // Determine compression format.
540 switch (Header.Common.Method) {
541 case static_cast<uint16_t>(compression::Format::Zlib):
542 case static_cast<uint16_t>(compression::Format::Zstd):
543 Normalized.CompressionFormat =
544 static_cast<compression::Format>(Header.Common.Method);
545 break;
546 default:
547 return createStringError(Fmt: "unknown compressing method");
548 }
549
550 return Normalized;
551}
552
553Expected<std::unique_ptr<MemoryBuffer>>
554CompressedOffloadBundle::decompress(const MemoryBuffer &Input,
555 raw_ostream *VerboseStream) {
556 StringRef Blob = Input.getBuffer();
557
558 // Check minimum header size (using V1 as it's the smallest).
559 if (Blob.size() < sizeof(RawCompressedBundleHeader::CommonFields))
560 return MemoryBuffer::getMemBufferCopy(InputData: Blob);
561
562 if (identify_magic(magic: Blob) != file_magic::offload_bundle_compressed) {
563 if (VerboseStream)
564 *VerboseStream << "Uncompressed bundle\n";
565 return MemoryBuffer::getMemBufferCopy(InputData: Blob);
566 }
567
568 Expected<CompressedBundleHeader> HeaderOrErr =
569 CompressedBundleHeader::tryParse(Blob);
570 if (!HeaderOrErr)
571 return HeaderOrErr.takeError();
572
573 const CompressedBundleHeader &Normalized = *HeaderOrErr;
574 unsigned ThisVersion = Normalized.Version;
575 size_t HeaderSize = getHeaderSize(Version: ThisVersion);
576
577 compression::Format CompressionFormat = Normalized.CompressionFormat;
578
579 size_t TotalFileSize = Normalized.FileSize.value_or(u: 0);
580 size_t UncompressedSize = Normalized.UncompressedFileSize;
581 auto StoredHash = Normalized.Hash;
582
583 Timer DecompressTimer("Decompression Timer", "Decompression time",
584 OffloadBundlerTimerGroup);
585 if (VerboseStream)
586 DecompressTimer.startTimer();
587
588 SmallVector<uint8_t, 0> DecompressedData;
589 StringRef CompressedData =
590 Blob.substr(Start: HeaderSize, N: TotalFileSize - HeaderSize);
591
592 if (Error DecompressionError = compression::decompress(
593 F: CompressionFormat, Input: arrayRefFromStringRef(Input: CompressedData),
594 Output&: DecompressedData, UncompressedSize))
595 return createStringError(S: "could not decompress embedded file contents: " +
596 toString(E: std::move(DecompressionError)));
597
598 if (VerboseStream) {
599 DecompressTimer.stopTimer();
600
601 double DecompressionTimeSeconds =
602 DecompressTimer.getTotalTime().getWallTime();
603
604 // Recalculate MD5 hash for integrity check.
605 Timer HashRecalcTimer("Hash Recalculation Timer", "Hash recalculation time",
606 OffloadBundlerTimerGroup);
607 HashRecalcTimer.startTimer();
608 MD5 Hash;
609 MD5::MD5Result Result;
610 Hash.update(Data: ArrayRef<uint8_t>(DecompressedData));
611 Hash.final(Result);
612 uint64_t RecalculatedHash = Result.low();
613 HashRecalcTimer.stopTimer();
614 bool HashMatch = (StoredHash == RecalculatedHash);
615
616 double CompressionRate =
617 static_cast<double>(UncompressedSize) / CompressedData.size();
618 double DecompressionSpeedMBs =
619 (UncompressedSize / (1024.0 * 1024.0)) / DecompressionTimeSeconds;
620
621 *VerboseStream << "Compressed bundle format version: " << ThisVersion
622 << "\n";
623 if (ThisVersion >= 2)
624 *VerboseStream << "Total file size (from header): "
625 << formatWithCommas(Value: TotalFileSize) << " bytes\n";
626 *VerboseStream
627 << "Decompression method: "
628 << (CompressionFormat == compression::Format::Zlib ? "zlib" : "zstd")
629 << "\n"
630 << "Size before decompression: "
631 << formatWithCommas(Value: CompressedData.size()) << " bytes\n"
632 << "Size after decompression: " << formatWithCommas(Value: UncompressedSize)
633 << " bytes\n"
634 << "Compression rate: " << format(Fmt: "%.2lf", Vals: CompressionRate) << "\n"
635 << "Compression ratio: " << format(Fmt: "%.2lf%%", Vals: 100.0 / CompressionRate)
636 << "\n"
637 << "Decompression speed: "
638 << format(Fmt: "%.2lf MB/s", Vals: DecompressionSpeedMBs) << "\n"
639 << "Stored hash: " << format_hex(N: StoredHash, Width: 16) << "\n"
640 << "Recalculated hash: " << format_hex(N: RecalculatedHash, Width: 16) << "\n"
641 << "Hashes match: " << (HashMatch ? "Yes" : "No") << "\n";
642 }
643
644 return MemoryBuffer::getMemBufferCopy(InputData: toStringRef(Input: DecompressedData));
645}
646