1//===- OffloadBundle.cpp - Utilities for offload bundles---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------===//
8
9#include "llvm/Object/OffloadBundle.h"
10#include "llvm/BinaryFormat/Magic.h"
11#include "llvm/IR/Module.h"
12#include "llvm/IRReader/IRReader.h"
13#include "llvm/MC/StringTableBuilder.h"
14#include "llvm/Object/Archive.h"
15#include "llvm/Object/Binary.h"
16#include "llvm/Object/COFF.h"
17#include "llvm/Object/ELFObjectFile.h"
18#include "llvm/Object/Error.h"
19#include "llvm/Object/IRObjectFile.h"
20#include "llvm/Object/ObjectFile.h"
21#include "llvm/Support/BinaryStreamReader.h"
22#include "llvm/Support/EndianStream.h"
23#include "llvm/Support/SourceMgr.h"
24#include "llvm/Support/Timer.h"
25
26using namespace llvm;
27using namespace llvm::object;
28
29static TimerGroup OffloadBundlerTimerGroup("Offload Bundler Timer Group",
30 "Timer group for offload bundler");
31
32// Returns the on-disk size recorded in the compressed offload bundle header at
33// the start of \p Blob, or std::nullopt if the header carries no size field.
34static std::optional<size_t> getCompressedBundleSize(StringRef Blob) {
35 Expected<CompressedOffloadBundle::CompressedBundleHeader> HeaderOrErr =
36 CompressedOffloadBundle::CompressedBundleHeader::tryParse(Blob);
37 if (!HeaderOrErr) {
38 consumeError(Err: HeaderOrErr.takeError());
39 return std::nullopt;
40 }
41 return HeaderOrErr->FileSize;
42}
43
44// Extract an Offload bundle (usually a Offload Bundle) from a fat_bin
45// section.
46Error extractOffloadBundle(MemoryBufferRef Contents, uint64_t SectionOffset,
47 StringRef FileName,
48 SmallVectorImpl<OffloadBundleFatBin> &Bundles) {
49
50 size_t Offset = 0;
51 size_t NextbundleStart = 0;
52 StringRef Magic;
53 std::unique_ptr<MemoryBuffer> Buffer;
54
55 // There could be multiple offloading bundles stored at this section.
56 while ((NextbundleStart != StringRef::npos) &&
57 (Offset < Contents.getBuffer().size())) {
58 Buffer =
59 MemoryBuffer::getMemBuffer(InputData: Contents.getBuffer().drop_front(N: Offset), BufferName: "",
60 /*RequiresNullTerminator=*/false);
61
62 if (identify_magic(magic: (*Buffer).getBuffer()) ==
63 file_magic::offload_bundle_compressed) {
64 Magic = "CCOB";
65 // Locate this bundle's end and the next bundle from the header size.
66 size_t CurBundleEnd;
67 if (std::optional<size_t> Size =
68 getCompressedBundleSize(Blob: (*Buffer).getBuffer())) {
69 CurBundleEnd = *Size;
70 NextbundleStart = (*Buffer).getBuffer().find(Str: Magic, From: *Size);
71 } else {
72 // Legacy bundle without a recorded size: fall back to magic scanning.
73 NextbundleStart = (*Buffer).getBuffer().find(Str: Magic, From: Magic.size());
74 CurBundleEnd = NextbundleStart;
75 }
76 if (NextbundleStart == StringRef::npos) {
77 NextbundleStart = (*Buffer).getBuffer().size();
78 if (CurBundleEnd == StringRef::npos)
79 CurBundleEnd = (*Buffer).getBuffer().size();
80 }
81
82 ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr =
83 MemoryBuffer::getMemBuffer(
84 InputData: (*Buffer).getBuffer().take_front(N: CurBundleEnd), BufferName: FileName, RequiresNullTerminator: false);
85 if (std::error_code EC = CodeOrErr.getError())
86 return createFileError(F: FileName, EC);
87
88 Expected<std::unique_ptr<MemoryBuffer>> DecompressedBufferOrErr =
89 CompressedOffloadBundle::decompress(Input: **CodeOrErr, VerboseStream: nullptr);
90 if (!DecompressedBufferOrErr)
91 return createStringError(S: "failed to decompress input: " +
92 toString(E: DecompressedBufferOrErr.takeError()));
93
94 auto FatBundleOrErr = OffloadBundleFatBin::create(
95 **DecompressedBufferOrErr, SectionOffset: Offset, FileName, Decompress: true);
96 if (!FatBundleOrErr)
97 return FatBundleOrErr.takeError();
98
99 // Add current Bundle to list.
100 Bundles.emplace_back(Args: std::move(**FatBundleOrErr));
101
102 } else if (identify_magic(magic: (*Buffer).getBuffer()) ==
103 file_magic::offload_bundle) {
104 // Create the OffloadBundleFatBin object. This will also create the Bundle
105 // Entry list info.
106 auto FatBundleOrErr = OffloadBundleFatBin::create(
107 *Buffer, SectionOffset: SectionOffset + Offset, FileName);
108 if (!FatBundleOrErr)
109 return FatBundleOrErr.takeError();
110
111 // Add current Bundle to list.
112 Bundles.emplace_back(Args: std::move(**FatBundleOrErr));
113
114 Magic = "__CLANG_OFFLOAD_BUNDLE__";
115 NextbundleStart = (*Buffer).getBuffer().find(Str: Magic, From: Magic.size());
116 }
117
118 if (NextbundleStart != StringRef::npos)
119 Offset += NextbundleStart;
120 }
121
122 return Error::success();
123}
124
125Error OffloadBundleFatBin::readEntries(StringRef Buffer,
126 uint64_t SectionOffset) {
127 uint64_t NumOfEntries = 0;
128
129 BinaryStreamReader Reader(Buffer, llvm::endianness::little);
130
131 // Read the Magic String first.
132 StringRef Magic;
133 if (auto EC = Reader.readFixedString(Dest&: Magic, Length: 24))
134 return errorCodeToError(EC: object_error::parse_failed);
135
136 // Read the number of Code Objects (Entries) in the current Bundle.
137 if (auto EC = Reader.readInteger(Dest&: NumOfEntries))
138 return errorCodeToError(EC: object_error::parse_failed);
139
140 NumberOfEntries = NumOfEntries;
141
142 // For each Bundle Entry (code object).
143 for (uint64_t I = 0; I < NumOfEntries; I++) {
144 uint64_t EntrySize;
145 uint64_t EntryOffset;
146 uint64_t EntryIDSize;
147 StringRef EntryID;
148
149 if (Error Err = Reader.readInteger(Dest&: EntryOffset))
150 return Err;
151
152 if (Error Err = Reader.readInteger(Dest&: EntrySize))
153 return Err;
154
155 if (Error Err = Reader.readInteger(Dest&: EntryIDSize))
156 return Err;
157
158 if (Error Err = Reader.readFixedString(Dest&: EntryID, Length: EntryIDSize))
159 return Err;
160
161 auto Entry = std::make_unique<OffloadBundleEntry>(
162 args: EntryOffset + SectionOffset, args&: EntrySize, args&: EntryIDSize, args&: EntryID);
163
164 Entries.push_back(Elt: *Entry);
165 }
166
167 return Error::success();
168}
169
170Expected<std::unique_ptr<OffloadBundleFatBin>>
171OffloadBundleFatBin::create(MemoryBufferRef Buf, uint64_t SectionOffset,
172 StringRef FileName, bool Decompress) {
173 if (Buf.getBufferSize() < 24)
174 return errorCodeToError(EC: object_error::parse_failed);
175
176 // Check for magic bytes.
177 if ((identify_magic(magic: Buf.getBuffer()) != file_magic::offload_bundle) &&
178 (identify_magic(magic: Buf.getBuffer()) !=
179 file_magic::offload_bundle_compressed))
180 return errorCodeToError(EC: object_error::parse_failed);
181
182 std::unique_ptr<OffloadBundleFatBin> TheBundle(
183 new OffloadBundleFatBin(Buf, FileName, Decompress));
184
185 // Read the Bundle Entries.
186 Error Err =
187 TheBundle->readEntries(Buffer: Buf.getBuffer(), SectionOffset: Decompress ? 0 : SectionOffset);
188 if (Err)
189 return Err;
190
191 return std::move(TheBundle);
192}
193
194Error OffloadBundleFatBin::extractBundle(const ObjectFile &Source) {
195 // This will extract all entries in the Bundle.
196 for (OffloadBundleEntry &Entry : Entries) {
197
198 if (Entry.Size == 0)
199 continue;
200
201 // create output file name. Which should be
202 // <fileName>-offset<Offset>-size<Size>.co"
203 std::string Str = getFileName().str() + "-offset" + itostr(X: Entry.Offset) +
204 "-size" + itostr(X: Entry.Size) + ".co";
205 if (Error Err = object::extractCodeObject(Source, Offset: Entry.Offset, Size: Entry.Size,
206 OutputFileName: StringRef(Str)))
207 return Err;
208 }
209
210 return Error::success();
211}
212
213Error object::extractOffloadBundleFatBinary(
214 const ObjectFile &Obj, SmallVectorImpl<OffloadBundleFatBin> &Bundles) {
215 // Ignore unsupported object formats.
216 if (!Obj.isELF() && !Obj.isCOFF())
217 return Error::success();
218
219 // Iterate through Sections until we find an offload_bundle section.
220 for (SectionRef Sec : Obj.sections()) {
221 Expected<StringRef> Buffer = Sec.getContents();
222 if (!Buffer)
223 return Buffer.takeError();
224
225 // If it does not start with the reserved suffix, just skip this section.
226 if ((llvm::identify_magic(magic: *Buffer) == file_magic::offload_bundle) ||
227 (llvm::identify_magic(magic: *Buffer) ==
228 file_magic::offload_bundle_compressed)) {
229
230 uint64_t SectionOffset = 0;
231 if (Obj.isELF()) {
232 SectionOffset = ELFSectionRef(Sec).getOffset();
233 } else if (Obj.isCOFF()) {
234 const COFFObjectFile &COFF = cast<COFFObjectFile>(Val: Obj);
235 Expected<const coff_section *> SecOrErr =
236 COFF.getSection(index: COFF.getSectionID(Sec));
237 if (!SecOrErr)
238 return SecOrErr.takeError();
239 SectionOffset = (*SecOrErr)->PointerToRawData;
240 }
241
242 MemoryBufferRef Contents(*Buffer, Obj.getFileName());
243 if (Error Err = extractOffloadBundle(Contents, SectionOffset,
244 FileName: Obj.getFileName(), Bundles))
245 return Err;
246 }
247 }
248 return Error::success();
249}
250
251Error object::extractCodeObject(const ObjectFile &Source, size_t Offset,
252 size_t Size, StringRef OutputFileName) {
253 Expected<std::unique_ptr<FileOutputBuffer>> BufferOrErr =
254 FileOutputBuffer::create(FilePath: OutputFileName, Size);
255
256 if (!BufferOrErr)
257 return BufferOrErr.takeError();
258
259 Expected<MemoryBufferRef> InputBuffOrErr = Source.getMemoryBufferRef();
260 if (Error Err = InputBuffOrErr.takeError())
261 return createFileError(F: Source.getFileName(), E: std::move(Err));
262
263 if (Size > InputBuffOrErr->getBufferSize())
264 return createStringError(Fmt: "size in URI (%zu) is larger than source (%zu)",
265 Vals: Size, Vals: InputBuffOrErr->getBufferSize());
266
267 if (Offset > InputBuffOrErr->getBufferSize())
268 return createStringError(
269 Fmt: "offset in URI (%zu) is beyond the end of the source (%zu)", Vals: Offset,
270 Vals: InputBuffOrErr->getBufferSize());
271
272 if (Offset + Size > InputBuffOrErr->getBufferSize())
273 return createStringError(
274 Fmt: "offset + size (%zu) in URI is beyond the end of the source (%zu)",
275 Vals: Offset + Size, Vals: InputBuffOrErr->getBufferSize());
276
277 std::unique_ptr<FileOutputBuffer> Buf = std::move(*BufferOrErr);
278 std::copy(first: InputBuffOrErr->getBufferStart() + Offset,
279 last: InputBuffOrErr->getBufferStart() + Offset + Size,
280 result: Buf->getBufferStart());
281 if (Error E = Buf->commit())
282 return createFileError(F: OutputFileName, E: std::move(E));
283
284 return Error::success();
285}
286
287Error object::extractCodeObject(const MemoryBufferRef Buffer, int64_t Offset,
288 int64_t Size, StringRef OutputFileName) {
289 Expected<std::unique_ptr<FileOutputBuffer>> BufferOrErr =
290 FileOutputBuffer::create(FilePath: OutputFileName, Size);
291 if (!BufferOrErr)
292 return BufferOrErr.takeError();
293
294 std::unique_ptr<FileOutputBuffer> Buf = std::move(*BufferOrErr);
295 std::copy(first: Buffer.getBufferStart() + Offset,
296 last: Buffer.getBufferStart() + Offset + Size, result: Buf->getBufferStart());
297
298 return Buf->commit();
299}
300
301// given a file name, offset, and size, extract data into a code object file,
302// into file "<SourceFile>-offset<Offset>-size<Size>.co".
303Error object::extractOffloadBundleByURI(StringRef URIstr) {
304 // create a URI object
305 Expected<std::unique_ptr<OffloadBundleURI>> UriOrErr(
306 OffloadBundleURI::createOffloadBundleURI(Str: URIstr, Type: FILE_URI));
307
308 if (!UriOrErr)
309 return UriOrErr.takeError();
310
311 OffloadBundleURI &Uri = **UriOrErr;
312 std::string OutputFile = Uri.FileName.str();
313 OutputFile +=
314 "-offset" + itostr(X: Uri.Offset) + "-size" + itostr(X: Uri.Size) + ".co";
315
316 // Create an ObjectFile object from uri.file_uri.
317 auto ObjOrErr = ObjectFile::createObjectFile(ObjectPath: Uri.FileName);
318 if (!ObjOrErr)
319 return ObjOrErr.takeError();
320
321 auto Obj = ObjOrErr->getBinary();
322 if (Error Err =
323 object::extractCodeObject(Source: *Obj, Offset: Uri.Offset, Size: Uri.Size, OutputFileName: OutputFile))
324 return createFileError(F: Uri.FileName, E: std::move(Err));
325
326 return Error::success();
327}
328
329// Utility function to format numbers with commas.
330static std::string formatWithCommas(unsigned long long Value) {
331 std::string Num = std::to_string(val: Value);
332 int InsertPosition = Num.length() - 3;
333 while (InsertPosition > 0) {
334 Num.insert(pos: InsertPosition, s: ",");
335 InsertPosition -= 3;
336 }
337 return Num;
338}
339
340Expected<std::unique_ptr<MemoryBuffer>>
341CompressedOffloadBundle::compress(compression::Params P,
342 const MemoryBuffer &Input, uint16_t Version,
343 raw_ostream *VerboseStream) {
344 if (!compression::zstd::isAvailable() && !compression::zlib::isAvailable())
345 return createStringError(Fmt: "compression not supported.");
346 Timer HashTimer("Hash Calculation Timer", "Hash calculation time",
347 OffloadBundlerTimerGroup);
348 if (VerboseStream)
349 HashTimer.startTimer();
350 MD5 Hash;
351 MD5::MD5Result Result;
352 Hash.update(Str: Input.getBuffer());
353 Hash.final(Result);
354 uint64_t TruncatedHash = Result.low();
355 if (VerboseStream)
356 HashTimer.stopTimer();
357
358 SmallVector<uint8_t, 0> CompressedBuffer;
359 auto BufferUint8 = ArrayRef<uint8_t>(
360 reinterpret_cast<const uint8_t *>(Input.getBuffer().data()),
361 Input.getBuffer().size());
362 Timer CompressTimer("Compression Timer", "Compression time",
363 OffloadBundlerTimerGroup);
364 if (VerboseStream)
365 CompressTimer.startTimer();
366 compression::compress(P, Input: BufferUint8, Output&: CompressedBuffer);
367 if (VerboseStream)
368 CompressTimer.stopTimer();
369
370 uint16_t CompressionMethod = static_cast<uint16_t>(P.format);
371
372 // Store sizes in 64-bit variables first.
373 uint64_t UncompressedSize64 = Input.getBuffer().size();
374 uint64_t TotalFileSize64;
375
376 // Calculate total file size based on version.
377 if (Version == 2) {
378 // For V2, ensure the sizes don't exceed 32-bit limit.
379 if (UncompressedSize64 > std::numeric_limits<uint32_t>::max())
380 return createStringError(Fmt: "uncompressed size (%llu) exceeds version 2 "
381 "unsigned 32-bit integer limit",
382 Vals: UncompressedSize64);
383 TotalFileSize64 = MagicNumber.size() + sizeof(uint32_t) + sizeof(Version) +
384 sizeof(CompressionMethod) + sizeof(uint32_t) +
385 sizeof(TruncatedHash) + CompressedBuffer.size();
386 if (TotalFileSize64 > std::numeric_limits<uint32_t>::max())
387 return createStringError(Fmt: "total file size (%llu) exceeds version 2 "
388 "unsigned 32-bit integer limit",
389 Vals: TotalFileSize64);
390
391 } else { // Version 3.
392 TotalFileSize64 = MagicNumber.size() + sizeof(uint64_t) + sizeof(Version) +
393 sizeof(CompressionMethod) + sizeof(uint64_t) +
394 sizeof(TruncatedHash) + CompressedBuffer.size();
395 }
396
397 SmallVector<char, 0> FinalBuffer;
398 raw_svector_ostream OS(FinalBuffer);
399 // The on-disk header is always little-endian, independent of the host.
400 support::endian::Writer Writer(OS, endianness::little);
401 OS << MagicNumber;
402 Writer.write(Val: Version);
403 Writer.write(Val: CompressionMethod);
404
405 // Write size fields according to version.
406 if (Version == 2) {
407 Writer.write(Val: static_cast<uint32_t>(TotalFileSize64));
408 Writer.write(Val: static_cast<uint32_t>(UncompressedSize64));
409 } else { // Version 3.
410 Writer.write(Val: TotalFileSize64);
411 Writer.write(Val: UncompressedSize64);
412 }
413
414 Writer.write(Val: TruncatedHash);
415 OS.write(Ptr: reinterpret_cast<const char *>(CompressedBuffer.data()),
416 Size: CompressedBuffer.size());
417
418 if (VerboseStream) {
419 auto MethodUsed = P.format == compression::Format::Zstd ? "zstd" : "zlib";
420 double CompressionRate =
421 static_cast<double>(UncompressedSize64) / CompressedBuffer.size();
422 double CompressionTimeSeconds = CompressTimer.getTotalTime().getWallTime();
423 double CompressionSpeedMBs =
424 (UncompressedSize64 / (1024.0 * 1024.0)) / CompressionTimeSeconds;
425 *VerboseStream << "Compressed bundle format version: " << Version << "\n"
426 << "Total file size (including headers): "
427 << formatWithCommas(Value: TotalFileSize64) << " bytes\n"
428 << "Compression method used: " << MethodUsed << "\n"
429 << "Compression level: " << P.level << "\n"
430 << "Binary size before compression: "
431 << formatWithCommas(Value: UncompressedSize64) << " bytes\n"
432 << "Binary size after compression: "
433 << formatWithCommas(Value: CompressedBuffer.size()) << " bytes\n"
434 << "Compression rate: " << format(Fmt: "%.2lf", Vals: CompressionRate)
435 << "\n"
436 << "Compression ratio: "
437 << format(Fmt: "%.2lf%%", Vals: 100.0 / CompressionRate) << "\n"
438 << "Compression speed: "
439 << format(Fmt: "%.2lf MB/s", Vals: CompressionSpeedMBs) << "\n"
440 << "Truncated MD5 hash: " << format_hex(N: TruncatedHash, Width: 16)
441 << "\n";
442 }
443
444 return MemoryBuffer::getMemBufferCopy(
445 InputData: StringRef(FinalBuffer.data(), FinalBuffer.size()));
446}
447
448// Use packed structs to avoid padding, such that the structs map the serialized
449// format.
450LLVM_PACKED_START
451union RawCompressedBundleHeader {
452 struct CommonFields {
453 support::ulittle32_t Magic;
454 support::ulittle16_t Version;
455 support::ulittle16_t Method;
456 };
457
458 struct V1Header {
459 CommonFields Common;
460 support::ulittle32_t UncompressedFileSize;
461 support::ulittle64_t Hash;
462 };
463
464 struct V2Header {
465 CommonFields Common;
466 support::ulittle32_t FileSize;
467 support::ulittle32_t UncompressedFileSize;
468 support::ulittle64_t Hash;
469 };
470
471 struct V3Header {
472 CommonFields Common;
473 support::ulittle64_t FileSize;
474 support::ulittle64_t UncompressedFileSize;
475 support::ulittle64_t Hash;
476 };
477
478 CommonFields Common;
479 V1Header V1;
480 V2Header V2;
481 V3Header V3;
482};
483LLVM_PACKED_END
484
485// Helper method to get header size based on version.
486static size_t getHeaderSize(uint16_t Version) {
487 switch (Version) {
488 case 1:
489 return sizeof(RawCompressedBundleHeader::V1Header);
490 case 2:
491 return sizeof(RawCompressedBundleHeader::V2Header);
492 case 3:
493 return sizeof(RawCompressedBundleHeader::V3Header);
494 default:
495 llvm_unreachable("Unsupported version");
496 }
497}
498
499Expected<CompressedOffloadBundle::CompressedBundleHeader>
500CompressedOffloadBundle::CompressedBundleHeader::tryParse(StringRef Blob) {
501 assert(Blob.size() >= sizeof(RawCompressedBundleHeader::CommonFields));
502 assert(identify_magic(Blob) == file_magic::offload_bundle_compressed);
503
504 RawCompressedBundleHeader Header;
505 std::memcpy(dest: &Header, src: Blob.data(), n: std::min(a: Blob.size(), b: sizeof(Header)));
506
507 CompressedBundleHeader Normalized;
508 Normalized.Version = Header.Common.Version;
509
510 size_t RequiredSize = getHeaderSize(Version: Normalized.Version);
511
512 if (Blob.size() < RequiredSize)
513 return createStringError(Fmt: "compressed bundle header size too small");
514
515 switch (Normalized.Version) {
516 case 1:
517 Normalized.UncompressedFileSize = Header.V1.UncompressedFileSize;
518 Normalized.Hash = Header.V1.Hash;
519 break;
520 case 2:
521 Normalized.FileSize = Header.V2.FileSize;
522 Normalized.UncompressedFileSize = Header.V2.UncompressedFileSize;
523 Normalized.Hash = Header.V2.Hash;
524 break;
525 case 3:
526 Normalized.FileSize = Header.V3.FileSize;
527 Normalized.UncompressedFileSize = Header.V3.UncompressedFileSize;
528 Normalized.Hash = Header.V3.Hash;
529 break;
530 default:
531 return createStringError(Fmt: "unknown compressed bundle version");
532 }
533
534 // Determine compression format.
535 switch (Header.Common.Method) {
536 case static_cast<uint16_t>(compression::Format::Zlib):
537 case static_cast<uint16_t>(compression::Format::Zstd):
538 Normalized.CompressionFormat =
539 static_cast<compression::Format>(Header.Common.Method.value());
540 break;
541 default:
542 return createStringError(Fmt: "unknown compressing method");
543 }
544
545 return Normalized;
546}
547
548Expected<std::unique_ptr<MemoryBuffer>>
549CompressedOffloadBundle::decompress(const MemoryBuffer &Input,
550 raw_ostream *VerboseStream) {
551 StringRef Blob = Input.getBuffer();
552
553 // Check minimum header size (using V1 as it's the smallest).
554 if (Blob.size() < sizeof(RawCompressedBundleHeader::CommonFields))
555 return MemoryBuffer::getMemBufferCopy(InputData: Blob);
556
557 if (identify_magic(magic: Blob) != file_magic::offload_bundle_compressed) {
558 if (VerboseStream)
559 *VerboseStream << "Uncompressed bundle\n";
560 return MemoryBuffer::getMemBufferCopy(InputData: Blob);
561 }
562
563 Expected<CompressedBundleHeader> HeaderOrErr =
564 CompressedBundleHeader::tryParse(Blob);
565 if (!HeaderOrErr)
566 return HeaderOrErr.takeError();
567
568 const CompressedBundleHeader &Normalized = *HeaderOrErr;
569 unsigned ThisVersion = Normalized.Version;
570 size_t HeaderSize = getHeaderSize(Version: ThisVersion);
571
572 compression::Format CompressionFormat = Normalized.CompressionFormat;
573
574 size_t TotalFileSize = Normalized.FileSize.value_or(u: 0);
575 size_t UncompressedSize = Normalized.UncompressedFileSize;
576 auto StoredHash = Normalized.Hash;
577
578 Timer DecompressTimer("Decompression Timer", "Decompression time",
579 OffloadBundlerTimerGroup);
580 if (VerboseStream)
581 DecompressTimer.startTimer();
582
583 SmallVector<uint8_t, 0> DecompressedData;
584 StringRef CompressedData =
585 Blob.substr(Start: HeaderSize, N: TotalFileSize - HeaderSize);
586
587 if (Error DecompressionError = compression::decompress(
588 F: CompressionFormat, Input: arrayRefFromStringRef(Input: CompressedData),
589 Output&: DecompressedData, UncompressedSize))
590 return createStringError(S: "could not decompress embedded file contents: " +
591 toString(E: std::move(DecompressionError)));
592
593 if (VerboseStream) {
594 DecompressTimer.stopTimer();
595
596 double DecompressionTimeSeconds =
597 DecompressTimer.getTotalTime().getWallTime();
598
599 // Recalculate MD5 hash for integrity check.
600 Timer HashRecalcTimer("Hash Recalculation Timer", "Hash recalculation time",
601 OffloadBundlerTimerGroup);
602 HashRecalcTimer.startTimer();
603 MD5 Hash;
604 MD5::MD5Result Result;
605 Hash.update(Data: ArrayRef<uint8_t>(DecompressedData));
606 Hash.final(Result);
607 uint64_t RecalculatedHash = Result.low();
608 HashRecalcTimer.stopTimer();
609 bool HashMatch = (StoredHash == RecalculatedHash);
610
611 double CompressionRate =
612 static_cast<double>(UncompressedSize) / CompressedData.size();
613 double DecompressionSpeedMBs =
614 (UncompressedSize / (1024.0 * 1024.0)) / DecompressionTimeSeconds;
615
616 *VerboseStream << "Compressed bundle format version: " << ThisVersion
617 << "\n";
618 if (ThisVersion >= 2)
619 *VerboseStream << "Total file size (from header): "
620 << formatWithCommas(Value: TotalFileSize) << " bytes\n";
621 *VerboseStream
622 << "Decompression method: "
623 << (CompressionFormat == compression::Format::Zlib ? "zlib" : "zstd")
624 << "\n"
625 << "Size before decompression: "
626 << formatWithCommas(Value: CompressedData.size()) << " bytes\n"
627 << "Size after decompression: " << formatWithCommas(Value: UncompressedSize)
628 << " bytes\n"
629 << "Compression rate: " << format(Fmt: "%.2lf", Vals: CompressionRate) << "\n"
630 << "Compression ratio: " << format(Fmt: "%.2lf%%", Vals: 100.0 / CompressionRate)
631 << "\n"
632 << "Decompression speed: "
633 << format(Fmt: "%.2lf MB/s", Vals: DecompressionSpeedMBs) << "\n"
634 << "Stored hash: " << format_hex(N: StoredHash, Width: 16) << "\n"
635 << "Recalculated hash: " << format_hex(N: RecalculatedHash, Width: 16) << "\n"
636 << "Hashes match: " << (HashMatch ? "Yes" : "No") << "\n";
637 }
638
639 return MemoryBuffer::getMemBufferCopy(InputData: toStringRef(Input: DecompressedData));
640}
641