1//===- BytecodeWriter.cpp - MLIR Bytecode Writer --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Bytecode/BytecodeWriter.h"
10#include "IRNumbering.h"
11#include "mlir/Bytecode/BytecodeImplementation.h"
12#include "mlir/Bytecode/BytecodeOpInterface.h"
13#include "mlir/Bytecode/Encoding.h"
14#include "mlir/IR/Attributes.h"
15#include "mlir/IR/Diagnostics.h"
16#include "mlir/IR/MLIRContext.h"
17#include "mlir/IR/OpImplementation.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/CachedHashString.h"
20#include "llvm/ADT/MapVector.h"
21#include "llvm/ADT/ScopeExit.h"
22#include "llvm/ADT/SmallVector.h"
23#include "llvm/Support/Debug.h"
24#include "llvm/Support/DebugLog.h"
25#include "llvm/Support/Endian.h"
26#include "llvm/Support/MathExtras.h"
27#include "llvm/Support/raw_ostream.h"
28#include <optional>
29
30#define DEBUG_TYPE "mlir-bytecode-writer"
31
32using namespace mlir;
33using namespace mlir::bytecode::detail;
34
35//===----------------------------------------------------------------------===//
36// BytecodeWriterConfig
37//===----------------------------------------------------------------------===//
38
39struct BytecodeWriterConfig::Impl {
40 Impl(StringRef producer) : producer(producer) {}
41
42 /// Version to use when writing.
43 /// Note: This only differs from kVersion if a specific version is set.
44 int64_t bytecodeVersion = bytecode::kVersion;
45
46 /// A flag specifying whether to elide emission of resources into the bytecode
47 /// file.
48 bool shouldElideResourceData = false;
49
50 /// A flag specifying whether to elide emission of locations.
51 bool shouldElideLocations = false;
52
53 /// A map containing dialect version information for each dialect to emit.
54 llvm::StringMap<std::unique_ptr<DialectVersion>> dialectVersionMap;
55
56 /// The producer of the bytecode.
57 StringRef producer;
58
59 /// Printer callbacks used to emit custom type and attribute encodings.
60 llvm::SmallVector<std::unique_ptr<AttrTypeBytecodeWriter<Attribute>>>
61 attributeWriterCallbacks;
62 llvm::SmallVector<std::unique_ptr<AttrTypeBytecodeWriter<Type>>>
63 typeWriterCallbacks;
64
65 /// A collection of non-dialect resource printers.
66 SmallVector<std::unique_ptr<AsmResourcePrinter>> externalResourcePrinters;
67};
68
69BytecodeWriterConfig::BytecodeWriterConfig(StringRef producer)
70 : impl(std::make_unique<Impl>(args&: producer)) {}
71BytecodeWriterConfig::BytecodeWriterConfig(FallbackAsmResourceMap &map,
72 StringRef producer)
73 : BytecodeWriterConfig(producer) {
74 attachFallbackResourcePrinter(map);
75}
76BytecodeWriterConfig::BytecodeWriterConfig(BytecodeWriterConfig &&config)
77 : impl(std::move(config.impl)) {}
78
79BytecodeWriterConfig::~BytecodeWriterConfig() = default;
80
81ArrayRef<std::unique_ptr<AttrTypeBytecodeWriter<Attribute>>>
82BytecodeWriterConfig::getAttributeWriterCallbacks() const {
83 return impl->attributeWriterCallbacks;
84}
85
86ArrayRef<std::unique_ptr<AttrTypeBytecodeWriter<Type>>>
87BytecodeWriterConfig::getTypeWriterCallbacks() const {
88 return impl->typeWriterCallbacks;
89}
90
91void BytecodeWriterConfig::attachAttributeCallback(
92 std::unique_ptr<AttrTypeBytecodeWriter<Attribute>> callback) {
93 impl->attributeWriterCallbacks.emplace_back(Args: std::move(callback));
94}
95
96void BytecodeWriterConfig::attachTypeCallback(
97 std::unique_ptr<AttrTypeBytecodeWriter<Type>> callback) {
98 impl->typeWriterCallbacks.emplace_back(Args: std::move(callback));
99}
100
101void BytecodeWriterConfig::attachResourcePrinter(
102 std::unique_ptr<AsmResourcePrinter> printer) {
103 impl->externalResourcePrinters.emplace_back(Args: std::move(printer));
104}
105
106void BytecodeWriterConfig::setElideResourceDataFlag(
107 bool shouldElideResourceData) {
108 impl->shouldElideResourceData = shouldElideResourceData;
109}
110
111void BytecodeWriterConfig::setElideLocations(bool shouldElideLocations) {
112 impl->shouldElideLocations = shouldElideLocations;
113}
114
115bool BytecodeWriterConfig::shouldElideLocations() const {
116 return impl->shouldElideLocations;
117}
118
119void BytecodeWriterConfig::setDesiredBytecodeVersion(int64_t bytecodeVersion) {
120 impl->bytecodeVersion = bytecodeVersion;
121}
122
123int64_t BytecodeWriterConfig::getDesiredBytecodeVersion() const {
124 return impl->bytecodeVersion;
125}
126
127llvm::StringMap<std::unique_ptr<DialectVersion>> &
128BytecodeWriterConfig::getDialectVersionMap() const {
129 return impl->dialectVersionMap;
130}
131
132void BytecodeWriterConfig::setDialectVersion(
133 llvm::StringRef dialectName,
134 std::unique_ptr<DialectVersion> dialectVersion) const {
135 assert(!impl->dialectVersionMap.contains(dialectName) &&
136 "cannot override a previously set dialect version");
137 impl->dialectVersionMap.insert(KV: {dialectName, std::move(dialectVersion)});
138}
139
140//===----------------------------------------------------------------------===//
141// EncodingEmitter
142//===----------------------------------------------------------------------===//
143
144namespace {
145/// This class functions as the underlying encoding emitter for the bytecode
146/// writer. This class is a bit different compared to other types of encoders;
147/// it does not use a single buffer, but instead may contain several buffers
148/// (some owned by the writer, and some not) that get concatted during the final
149/// emission.
150class EncodingEmitter {
151public:
152 EncodingEmitter() = default;
153 EncodingEmitter(const EncodingEmitter &) = delete;
154 EncodingEmitter &operator=(const EncodingEmitter &) = delete;
155
156 /// Write the current contents to the provided stream.
157 void writeTo(raw_ostream &os) const;
158
159 /// Return the current size of the encoded buffer.
160 size_t size() const { return prevResultSize + currentResult.size(); }
161
162 //===--------------------------------------------------------------------===//
163 // Emission
164 //===--------------------------------------------------------------------===//
165
166 /// Backpatch a byte in the result buffer at the given offset.
167 void patchByte(uint64_t offset, uint8_t value, StringLiteral desc) {
168 LDBG() << "patchByte(" << offset << ',' << uint64_t(value) << ")\t" << desc;
169 assert(offset < size() && offset >= prevResultSize &&
170 "cannot patch previously emitted data");
171 currentResult[offset - prevResultSize] = value;
172 }
173
174 /// Emit the provided blob of data, which is owned by the caller and is
175 /// guaranteed to not die before the end of the bytecode process.
176 void emitOwnedBlob(ArrayRef<uint8_t> data, StringLiteral desc) {
177 LDBG() << "emitOwnedBlob(" << data.size() << "b)\t" << desc;
178 // Push the current buffer before adding the provided data.
179 appendResult(result: std::move(currentResult));
180 appendOwnedResult(result: data);
181 }
182
183 /// Emit the provided blob of data that has the given alignment, which is
184 /// owned by the caller and is guaranteed to not die before the end of the
185 /// bytecode process. The alignment value is also encoded, making it available
186 /// on load.
187 void emitOwnedBlobAndAlignment(ArrayRef<uint8_t> data, uint32_t alignment,
188 StringLiteral desc) {
189 emitVarInt(value: alignment, desc);
190 emitVarInt(value: data.size(), desc);
191
192 alignTo(alignment);
193 emitOwnedBlob(data, desc);
194 }
195 void emitOwnedBlobAndAlignment(ArrayRef<char> data, uint32_t alignment,
196 StringLiteral desc) {
197 ArrayRef<uint8_t> castedData(reinterpret_cast<const uint8_t *>(data.data()),
198 data.size());
199 emitOwnedBlobAndAlignment(data: castedData, alignment, desc);
200 }
201
202 /// Align the emitter to the given alignment.
203 void alignTo(unsigned alignment) {
204 if (alignment < 2)
205 return;
206 assert(llvm::isPowerOf2_32(alignment) && "expected valid alignment");
207
208 // Check to see if we need to emit any padding bytes to meet the desired
209 // alignment.
210 size_t curOffset = size();
211 size_t paddingSize = llvm::alignTo(Value: curOffset, Align: alignment) - curOffset;
212 while (paddingSize--)
213 emitByte(byte: bytecode::kAlignmentByte, desc: "alignment byte");
214
215 // Keep track of the maximum required alignment.
216 requiredAlignment = std::max(a: requiredAlignment, b: alignment);
217 }
218
219 //===--------------------------------------------------------------------===//
220 // Integer Emission
221
222 /// Emit a single byte.
223 template <typename T>
224 void emitByte(T byte, StringLiteral desc) {
225 LDBG() << "emitByte(" << uint64_t(byte) << ")\t" << desc;
226 currentResult.push_back(x: static_cast<uint8_t>(byte));
227 }
228
229 /// Emit a range of bytes.
230 void emitBytes(ArrayRef<uint8_t> bytes, StringLiteral desc) {
231 LDBG() << "emitBytes(" << bytes.size() << "b)\t" << desc;
232 llvm::append_range(C&: currentResult, R&: bytes);
233 }
234
235 /// Emit a variable length integer. The first encoded byte contains a prefix
236 /// in the low bits indicating the encoded length of the value. This length
237 /// prefix is a bit sequence of '0's followed by a '1'. The number of '0' bits
238 /// indicate the number of _additional_ bytes (not including the prefix byte).
239 /// All remaining bits in the first byte, along with all of the bits in
240 /// additional bytes, provide the value of the integer encoded in
241 /// little-endian order.
242 void emitVarInt(uint64_t value, StringLiteral desc) {
243 LDBG() << "emitVarInt(" << value << ")\t" << desc;
244
245 // In the most common case, the value can be represented in a single byte.
246 // Given how hot this case is, explicitly handle that here.
247 if ((value >> 7) == 0)
248 return emitByte(byte: (value << 1) | 0x1, desc);
249 emitMultiByteVarInt(value, desc);
250 }
251
252 /// Emit a signed variable length integer. Signed varints are encoded using
253 /// a varint with zigzag encoding, meaning that we use the low bit of the
254 /// value to indicate the sign of the value. This allows for more efficient
255 /// encoding of negative values by limiting the number of active bits
256 void emitSignedVarInt(uint64_t value, StringLiteral desc) {
257 emitVarInt(value: (value << 1) ^ (uint64_t)((int64_t)value >> 63), desc);
258 }
259
260 /// Emit a variable length integer whose low bit is used to encode the
261 /// provided flag, i.e. encoded as: (value << 1) | (flag ? 1 : 0).
262 void emitVarIntWithFlag(uint64_t value, bool flag, StringLiteral desc) {
263 emitVarInt(value: (value << 1) | (flag ? 1 : 0), desc);
264 }
265
266 //===--------------------------------------------------------------------===//
267 // String Emission
268
269 /// Emit the given string as a nul terminated string.
270 void emitNulTerminatedString(StringRef str, StringLiteral desc) {
271 emitString(str, desc);
272 emitByte(byte: 0, desc: "null terminator");
273 }
274
275 /// Emit the given string without a nul terminator.
276 void emitString(StringRef str, StringLiteral desc) {
277 emitBytes(bytes: {reinterpret_cast<const uint8_t *>(str.data()), str.size()},
278 desc);
279 }
280
281 //===--------------------------------------------------------------------===//
282 // Section Emission
283
284 /// Emit a nested section of the given code, whose contents are encoded in the
285 /// provided emitter.
286 void emitSection(bytecode::Section::ID code, EncodingEmitter &&emitter) {
287 // Emit the section code and length. The high bit of the code is used to
288 // indicate whether the section alignment is present, so save an offset to
289 // it.
290 uint64_t codeOffset = currentResult.size();
291 emitByte(byte: code, desc: "section code");
292 emitVarInt(value: emitter.size(), desc: "section size");
293
294 // Integrate the alignment of the section into this emitter if necessary.
295 unsigned emitterAlign = emitter.requiredAlignment;
296 if (emitterAlign > 1) {
297 if (size() & (emitterAlign - 1)) {
298 emitVarInt(value: emitterAlign, desc: "section alignment");
299 alignTo(alignment: emitterAlign);
300
301 // Indicate that we needed to align the section, the high bit of the
302 // code field is used for this.
303 currentResult[codeOffset] |= 0b10000000;
304 } else {
305 // Otherwise, if we happen to be at a compatible offset, we just
306 // remember that we need this alignment.
307 requiredAlignment = std::max(a: requiredAlignment, b: emitterAlign);
308 }
309 }
310
311 // Push our current buffer and then merge the provided section body into
312 // ours.
313 appendResult(result: std::move(currentResult));
314 for (std::vector<uint8_t> &result : emitter.prevResultStorage)
315 prevResultStorage.push_back(x: std::move(result));
316 llvm::append_range(C&: prevResultList, R&: emitter.prevResultList);
317 prevResultSize += emitter.prevResultSize;
318 appendResult(result: std::move(emitter.currentResult));
319 }
320
321private:
322 /// Emit the given value using a variable width encoding. This method is a
323 /// fallback when the number of bytes needed to encode the value is greater
324 /// than 1. We mark it noinline here so that the single byte hot path isn't
325 /// pessimized.
326 LLVM_ATTRIBUTE_NOINLINE void emitMultiByteVarInt(uint64_t value,
327 StringLiteral desc);
328
329 /// Append a new result buffer to the current contents.
330 void appendResult(std::vector<uint8_t> &&result) {
331 if (result.empty())
332 return;
333 prevResultStorage.emplace_back(args: std::move(result));
334 appendOwnedResult(result: prevResultStorage.back());
335 }
336 void appendOwnedResult(ArrayRef<uint8_t> result) {
337 if (result.empty())
338 return;
339 prevResultSize += result.size();
340 prevResultList.emplace_back(args&: result);
341 }
342
343 /// The result of the emitter currently being built. We refrain from building
344 /// a single buffer to simplify emitting sections, large data, and more. The
345 /// result is thus represented using multiple distinct buffers, some of which
346 /// we own (via prevResultStorage), and some of which are just pointers into
347 /// externally owned buffers.
348 std::vector<uint8_t> currentResult;
349 std::vector<ArrayRef<uint8_t>> prevResultList;
350 std::vector<std::vector<uint8_t>> prevResultStorage;
351
352 /// An up-to-date total size of all of the buffers within `prevResultList`.
353 /// This enables O(1) size checks of the current encoding.
354 size_t prevResultSize = 0;
355
356 /// The highest required alignment for the start of this section.
357 unsigned requiredAlignment = 1;
358};
359
360//===----------------------------------------------------------------------===//
361// StringSectionBuilder
362//===----------------------------------------------------------------------===//
363
364namespace {
365/// This class is used to simplify the process of emitting the string section.
366class StringSectionBuilder {
367public:
368 /// Add the given string to the string section, and return the index of the
369 /// string within the section.
370 size_t insert(StringRef str) {
371 auto it = strings.insert(KV: {llvm::CachedHashStringRef(str), strings.size()});
372 return it.first->second;
373 }
374
375 /// Write the current set of strings to the given emitter.
376 void write(EncodingEmitter &emitter) {
377 emitter.emitVarInt(value: strings.size(), desc: "string section size");
378
379 // Emit the sizes in reverse order, so that we don't need to backpatch an
380 // offset to the string data or have a separate section.
381 for (const auto &it : llvm::reverse(C&: strings))
382 emitter.emitVarInt(value: it.first.size() + 1, desc: "string size");
383 // Emit the string data itself.
384 for (const auto &it : strings)
385 emitter.emitNulTerminatedString(str: it.first.val(), desc: "string");
386 }
387
388private:
389 /// A set of strings referenced within the bytecode. The value of the map is
390 /// unused.
391 llvm::MapVector<llvm::CachedHashStringRef, size_t> strings;
392};
393} // namespace
394
395class DialectWriter : public DialectBytecodeWriter {
396 using DialectVersionMapT = llvm::StringMap<std::unique_ptr<DialectVersion>>;
397
398public:
399 DialectWriter(int64_t bytecodeVersion, EncodingEmitter &emitter,
400 IRNumberingState &numberingState,
401 StringSectionBuilder &stringSection,
402 const DialectVersionMapT &dialectVersionMap)
403 : bytecodeVersion(bytecodeVersion), emitter(emitter),
404 numberingState(numberingState), stringSection(stringSection),
405 dialectVersionMap(dialectVersionMap) {}
406
407 //===--------------------------------------------------------------------===//
408 // IR
409 //===--------------------------------------------------------------------===//
410
411 void writeAttribute(Attribute attr) override {
412 emitter.emitVarInt(value: numberingState.getNumber(attr), desc: "dialect attr");
413 }
414 void writeOptionalAttribute(Attribute attr) override {
415 if (!attr) {
416 emitter.emitVarInt(value: 0, desc: "dialect optional attr none");
417 return;
418 }
419 emitter.emitVarIntWithFlag(value: numberingState.getNumber(attr), flag: true,
420 desc: "dialect optional attr");
421 }
422
423 void writeType(Type type) override {
424 emitter.emitVarInt(value: numberingState.getNumber(type), desc: "dialect type");
425 }
426
427 void writeResourceHandle(const AsmDialectResourceHandle &resource) override {
428 emitter.emitVarInt(value: numberingState.getNumber(resource), desc: "dialect resource");
429 }
430
431 //===--------------------------------------------------------------------===//
432 // Primitives
433 //===--------------------------------------------------------------------===//
434
435 void writeVarInt(uint64_t value) override {
436 emitter.emitVarInt(value, desc: "dialect writer");
437 }
438
439 void writeSignedVarInt(int64_t value) override {
440 emitter.emitSignedVarInt(value, desc: "dialect writer");
441 }
442
443 void writeAPIntWithKnownWidth(const APInt &value) override {
444 size_t bitWidth = value.getBitWidth();
445
446 // If the value is a single byte, just emit it directly without going
447 // through a varint.
448 if (bitWidth <= 8)
449 return emitter.emitByte(byte: value.getLimitedValue(), desc: "dialect APInt");
450
451 // If the value fits within a single varint, emit it directly.
452 if (bitWidth <= 64)
453 return emitter.emitSignedVarInt(value: value.getLimitedValue(), desc: "dialect APInt");
454
455 // Otherwise, we need to encode a variable number of active words. We use
456 // active words instead of the number of total words under the observation
457 // that smaller values will be more common.
458 unsigned numActiveWords = value.getActiveWords();
459 emitter.emitVarInt(value: numActiveWords, desc: "dialect APInt word count");
460
461 const uint64_t *rawValueData = value.getRawData();
462 for (unsigned i = 0; i < numActiveWords; ++i)
463 emitter.emitSignedVarInt(value: rawValueData[i], desc: "dialect APInt word");
464 }
465
466 void writeAPFloatWithKnownSemantics(const APFloat &value) override {
467 writeAPIntWithKnownWidth(value: value.bitcastToAPInt());
468 }
469
470 void writeOwnedString(StringRef str) override {
471 emitter.emitVarInt(value: stringSection.insert(str), desc: "dialect string");
472 }
473
474 void writeOwnedBlob(ArrayRef<char> blob) override {
475 emitter.emitVarInt(value: blob.size(), desc: "dialect blob");
476 emitter.emitOwnedBlob(
477 data: ArrayRef<uint8_t>(reinterpret_cast<const uint8_t *>(blob.data()),
478 blob.size()),
479 desc: "dialect blob");
480 }
481
482 void writeUnownedBlob(ArrayRef<char> blob) override {
483 emitter.emitVarInt(value: blob.size(), desc: "dialect blob");
484 emitter.emitBytes(
485 bytes: ArrayRef<uint8_t>(reinterpret_cast<const uint8_t *>(blob.data()),
486 blob.size()),
487 desc: "dialect blob");
488 }
489
490 void writeOwnedBool(bool value) override {
491 emitter.emitByte(byte: value, desc: "dialect bool");
492 }
493
494 int64_t getBytecodeVersion() const override { return bytecodeVersion; }
495
496 FailureOr<const DialectVersion *>
497 getDialectVersion(StringRef dialectName) const override {
498 auto dialectEntry = dialectVersionMap.find(Key: dialectName);
499 if (dialectEntry == dialectVersionMap.end())
500 return failure();
501 return dialectEntry->getValue().get();
502 }
503
504private:
505 int64_t bytecodeVersion;
506 EncodingEmitter &emitter;
507 IRNumberingState &numberingState;
508 StringSectionBuilder &stringSection;
509 const DialectVersionMapT &dialectVersionMap;
510};
511
512namespace {
513class PropertiesSectionBuilder {
514public:
515 PropertiesSectionBuilder(IRNumberingState &numberingState,
516 StringSectionBuilder &stringSection,
517 const BytecodeWriterConfig::Impl &config)
518 : numberingState(numberingState), stringSection(stringSection),
519 config(config) {}
520
521 /// Emit the op properties in the properties section and return the index of
522 /// the properties within the section. Return -1 if no properties was emitted.
523 std::optional<ssize_t> emit(Operation *op) {
524 EncodingEmitter propertiesEmitter;
525 if (!op->getPropertiesStorageSize())
526 return std::nullopt;
527 if (!op->isRegistered()) {
528 // Unregistered op are storing properties as an optional attribute.
529 Attribute prop = *op->getPropertiesStorage().as<Attribute *>();
530 if (!prop)
531 return std::nullopt;
532 EncodingEmitter sizeEmitter;
533 sizeEmitter.emitVarInt(value: numberingState.getNumber(attr: prop), desc: "properties size");
534 scratch.clear();
535 llvm::raw_svector_ostream os(scratch);
536 sizeEmitter.writeTo(os);
537 return emit(rawProperties: scratch);
538 }
539
540 EncodingEmitter emitter;
541 DialectWriter propertiesWriter(config.bytecodeVersion, emitter,
542 numberingState, stringSection,
543 config.dialectVersionMap);
544 auto iface = cast<BytecodeOpInterface>(Val: op);
545 iface.writeProperties(writer&: propertiesWriter);
546 scratch.clear();
547 llvm::raw_svector_ostream os(scratch);
548 emitter.writeTo(os);
549 return emit(rawProperties: scratch);
550 }
551
552 /// Write the current set of properties to the given emitter.
553 void write(EncodingEmitter &emitter) {
554 emitter.emitVarInt(value: propertiesStorage.size(), desc: "properties size");
555 if (propertiesStorage.empty())
556 return;
557 for (const auto &storage : propertiesStorage) {
558 if (storage.empty()) {
559 emitter.emitBytes(bytes: ArrayRef<uint8_t>(), desc: "empty properties");
560 continue;
561 }
562 emitter.emitBytes(bytes: ArrayRef(reinterpret_cast<const uint8_t *>(&storage[0]),
563 storage.size()),
564 desc: "property");
565 }
566 }
567
568 /// Returns true if the section is empty.
569 bool empty() { return propertiesStorage.empty(); }
570
571private:
572 /// Emit raw data and returns the offset in the internal buffer.
573 /// Data are deduplicated and will be copied in the internal buffer only if
574 /// they don't exist there already.
575 ssize_t emit(ArrayRef<char> rawProperties) {
576 // Populate a scratch buffer with the properties size.
577 SmallVector<char> sizeScratch;
578 {
579 EncodingEmitter sizeEmitter;
580 sizeEmitter.emitVarInt(value: rawProperties.size(), desc: "properties");
581 llvm::raw_svector_ostream os(sizeScratch);
582 sizeEmitter.writeTo(os);
583 }
584 // Append a new storage to the table now.
585 size_t index = propertiesStorage.size();
586 propertiesStorage.emplace_back();
587 std::vector<char> &newStorage = propertiesStorage.back();
588 size_t propertiesSize = sizeScratch.size() + rawProperties.size();
589 newStorage.reserve(n: propertiesSize);
590 llvm::append_range(C&: newStorage, R&: sizeScratch);
591 llvm::append_range(C&: newStorage, R&: rawProperties);
592
593 // Try to de-duplicate the new serialized properties.
594 // If the properties is a duplicate, pop it back from the storage.
595 auto inserted = propertiesUniquing.insert(
596 KV: std::make_pair(x: ArrayRef<char>(newStorage), y&: index));
597 if (!inserted.second)
598 propertiesStorage.pop_back();
599 return inserted.first->getSecond();
600 }
601
602 /// Storage for properties.
603 std::vector<std::vector<char>> propertiesStorage;
604 SmallVector<char> scratch;
605 DenseMap<ArrayRef<char>, int64_t> propertiesUniquing;
606 IRNumberingState &numberingState;
607 StringSectionBuilder &stringSection;
608 const BytecodeWriterConfig::Impl &config;
609};
610} // namespace
611
612/// A simple raw_ostream wrapper around a EncodingEmitter. This removes the need
613/// to go through an intermediate buffer when interacting with code that wants a
614/// raw_ostream.
615class RawEmitterOstream : public raw_ostream {
616public:
617 explicit RawEmitterOstream(EncodingEmitter &emitter) : emitter(emitter) {
618 SetUnbuffered();
619 }
620
621private:
622 void write_impl(const char *ptr, size_t size) override {
623 emitter.emitBytes(bytes: {reinterpret_cast<const uint8_t *>(ptr), size},
624 desc: "raw emitter");
625 }
626 uint64_t current_pos() const override { return emitter.size(); }
627
628 /// The section being emitted to.
629 EncodingEmitter &emitter;
630};
631} // namespace
632
633void EncodingEmitter::writeTo(raw_ostream &os) const {
634 // Reserve space in the ostream for the encoded contents.
635 os.reserveExtraSpace(ExtraSize: size());
636
637 for (auto &prevResult : prevResultList)
638 os.write(Ptr: (const char *)prevResult.data(), Size: prevResult.size());
639 os.write(Ptr: (const char *)currentResult.data(), Size: currentResult.size());
640}
641
642void EncodingEmitter::emitMultiByteVarInt(uint64_t value, StringLiteral desc) {
643 // Compute the number of bytes needed to encode the value. Each byte can hold
644 // up to 7-bits of data. We only check up to the number of bits we can encode
645 // in the first byte (8).
646 uint64_t it = value >> 7;
647 for (size_t numBytes = 2; numBytes < 9; ++numBytes) {
648 if (LLVM_LIKELY(it >>= 7) == 0) {
649 uint64_t encodedValue = (value << 1) | 0x1;
650 encodedValue <<= (numBytes - 1);
651 llvm::support::ulittle64_t encodedValueLE(encodedValue);
652 emitBytes(bytes: {reinterpret_cast<uint8_t *>(&encodedValueLE), numBytes}, desc);
653 return;
654 }
655 }
656
657 // If the value is too large to encode in a single byte, emit a special all
658 // zero marker byte and splat the value directly.
659 emitByte(byte: 0, desc);
660 llvm::support::ulittle64_t valueLE(value);
661 emitBytes(bytes: {reinterpret_cast<uint8_t *>(&valueLE), sizeof(valueLE)}, desc);
662}
663
664//===----------------------------------------------------------------------===//
665// Bytecode Writer
666//===----------------------------------------------------------------------===//
667
668namespace {
669class BytecodeWriter {
670public:
671 BytecodeWriter(Operation *op, const BytecodeWriterConfig &config)
672 : numberingState(op, config), config(config.getImpl()),
673 propertiesSection(numberingState, stringSection, config.getImpl()) {}
674
675 /// Write the bytecode for the given root operation.
676 LogicalResult write(Operation *rootOp, raw_ostream &os);
677
678private:
679 //===--------------------------------------------------------------------===//
680 // Dialects
681
682 void writeDialectSection(EncodingEmitter &emitter);
683
684 //===--------------------------------------------------------------------===//
685 // Attributes and Types
686
687 void writeAttrTypeSection(EncodingEmitter &emitter);
688
689 //===--------------------------------------------------------------------===//
690 // Operations
691
692 LogicalResult writeBlock(EncodingEmitter &emitter, Block *block);
693 LogicalResult writeOp(EncodingEmitter &emitter, Operation *op);
694 LogicalResult writeRegion(EncodingEmitter &emitter, Region *region);
695 LogicalResult writeIRSection(EncodingEmitter &emitter, Operation *op);
696
697 LogicalResult writeRegions(EncodingEmitter &emitter,
698 MutableArrayRef<Region> regions) {
699 return success(IsSuccess: llvm::all_of(Range&: regions, P: [&](Region &region) {
700 return succeeded(Result: writeRegion(emitter, region: &region));
701 }));
702 }
703
704 //===--------------------------------------------------------------------===//
705 // Resources
706
707 void writeResourceSection(Operation *op, EncodingEmitter &emitter);
708
709 //===--------------------------------------------------------------------===//
710 // Strings
711
712 void writeStringSection(EncodingEmitter &emitter);
713
714 //===--------------------------------------------------------------------===//
715 // Properties
716
717 void writePropertiesSection(EncodingEmitter &emitter);
718
719 //===--------------------------------------------------------------------===//
720 // Helpers
721
722 void writeUseListOrders(EncodingEmitter &emitter, uint8_t &opEncodingMask,
723 ValueRange range);
724
725 //===--------------------------------------------------------------------===//
726 // Fields
727
728 /// The builder used for the string section.
729 StringSectionBuilder stringSection;
730
731 /// The IR numbering state generated for the root operation.
732 IRNumberingState numberingState;
733
734 /// Configuration dictating bytecode emission.
735 const BytecodeWriterConfig::Impl &config;
736
737 /// Storage for the properties section
738 PropertiesSectionBuilder propertiesSection;
739};
740} // namespace
741
742LogicalResult BytecodeWriter::write(Operation *rootOp, raw_ostream &os) {
743 EncodingEmitter emitter;
744
745 // Emit the bytecode file header. This is how we identify the output as a
746 // bytecode file.
747 emitter.emitString(str: "ML\xefR", desc: "bytecode header");
748
749 // Emit the bytecode version.
750 if (config.bytecodeVersion < bytecode::kMinSupportedVersion ||
751 config.bytecodeVersion > bytecode::kVersion)
752 return rootOp->emitError()
753 << "unsupported version requested " << config.bytecodeVersion
754 << ", must be in range ["
755 << static_cast<int64_t>(bytecode::kMinSupportedVersion) << ", "
756 << static_cast<int64_t>(bytecode::kVersion) << ']';
757 emitter.emitVarInt(value: config.bytecodeVersion, desc: "bytecode version");
758
759 // Emit the producer.
760 emitter.emitNulTerminatedString(str: config.producer, desc: "bytecode producer");
761
762 // Emit the dialect section.
763 writeDialectSection(emitter);
764
765 // Emit the attributes and types section.
766 writeAttrTypeSection(emitter);
767
768 // Emit the IR section.
769 if (failed(Result: writeIRSection(emitter, op: rootOp)))
770 return failure();
771
772 // Emit the resources section.
773 writeResourceSection(op: rootOp, emitter);
774
775 // Emit the string section.
776 writeStringSection(emitter);
777
778 // Emit the properties section.
779 if (config.bytecodeVersion >= bytecode::kNativePropertiesEncoding)
780 writePropertiesSection(emitter);
781 else if (!propertiesSection.empty())
782 return rootOp->emitError(
783 message: "unexpected properties emitted incompatible with bytecode <5");
784
785 // Write the generated bytecode to the provided output stream.
786 emitter.writeTo(os);
787
788 return success();
789}
790
791//===----------------------------------------------------------------------===//
792// Dialects
793//===----------------------------------------------------------------------===//
794
795/// Write the given entries in contiguous groups with the same parent dialect.
796/// Each dialect sub-group is encoded with the parent dialect and number of
797/// elements, followed by the encoding for the entries. The given callback is
798/// invoked to encode each individual entry.
799template <typename EntriesT, typename EntryCallbackT>
800static void writeDialectGrouping(EncodingEmitter &emitter, EntriesT &&entries,
801 EntryCallbackT &&callback) {
802 for (auto it = entries.begin(), e = entries.end(); it != e;) {
803 auto groupStart = it++;
804
805 // Find the end of the group that shares the same parent dialect.
806 DialectNumbering *currentDialect = groupStart->dialect;
807 it = std::find_if(it, e, [&](const auto &entry) {
808 return entry.dialect != currentDialect;
809 });
810
811 // Emit the dialect and number of elements.
812 emitter.emitVarInt(value: currentDialect->number, desc: "dialect number");
813 emitter.emitVarInt(value: std::distance(groupStart, it), desc: "dialect offset");
814
815 // Emit the entries within the group.
816 for (auto &entry : llvm::make_range(groupStart, it))
817 callback(entry);
818 }
819}
820
821void BytecodeWriter::writeDialectSection(EncodingEmitter &emitter) {
822 EncodingEmitter dialectEmitter;
823
824 // Emit the referenced dialects.
825 auto dialects = numberingState.getDialects();
826 dialectEmitter.emitVarInt(value: llvm::size(Range&: dialects), desc: "dialects count");
827 for (DialectNumbering &dialect : dialects) {
828 // Write the string section and get the ID.
829 size_t nameID = stringSection.insert(str: dialect.name);
830
831 if (config.bytecodeVersion < bytecode::kDialectVersioning) {
832 dialectEmitter.emitVarInt(value: nameID, desc: "dialect name ID");
833 continue;
834 }
835
836 // Try writing the version to the versionEmitter.
837 EncodingEmitter versionEmitter;
838 if (dialect.interface) {
839 // The writer used when emitting using a custom bytecode encoding.
840 DialectWriter versionWriter(config.bytecodeVersion, versionEmitter,
841 numberingState, stringSection,
842 config.dialectVersionMap);
843 dialect.interface->writeVersion(writer&: versionWriter);
844 }
845
846 // If the version emitter is empty, version is not available. We can encode
847 // this in the dialect ID, so if there is no version, we don't write the
848 // section.
849 size_t versionAvailable = versionEmitter.size() > 0;
850 dialectEmitter.emitVarIntWithFlag(value: nameID, flag: versionAvailable,
851 desc: "dialect version");
852 if (versionAvailable)
853 dialectEmitter.emitSection(code: bytecode::Section::kDialectVersions,
854 emitter: std::move(versionEmitter));
855 }
856
857 if (config.bytecodeVersion >= bytecode::kElideUnknownBlockArgLocation)
858 dialectEmitter.emitVarInt(value: size(Range: numberingState.getOpNames()),
859 desc: "op names count");
860
861 // Emit the referenced operation names grouped by dialect.
862 auto emitOpName = [&](OpNameNumbering &name) {
863 size_t stringId = stringSection.insert(str: name.name.stripDialect());
864 if (config.bytecodeVersion < bytecode::kNativePropertiesEncoding)
865 dialectEmitter.emitVarInt(value: stringId, desc: "dialect op name");
866 else
867 dialectEmitter.emitVarIntWithFlag(value: stringId, flag: name.name.isRegistered(),
868 desc: "dialect op name");
869 };
870 writeDialectGrouping(emitter&: dialectEmitter, entries: numberingState.getOpNames(), callback&: emitOpName);
871
872 emitter.emitSection(code: bytecode::Section::kDialect, emitter: std::move(dialectEmitter));
873}
874
875//===----------------------------------------------------------------------===//
876// Attributes and Types
877//===----------------------------------------------------------------------===//
878
879void BytecodeWriter::writeAttrTypeSection(EncodingEmitter &emitter) {
880 EncodingEmitter attrTypeEmitter;
881 EncodingEmitter offsetEmitter;
882 offsetEmitter.emitVarInt(value: llvm::size(Range: numberingState.getAttributes()),
883 desc: "attributes count");
884 offsetEmitter.emitVarInt(value: llvm::size(Range: numberingState.getTypes()),
885 desc: "types count");
886
887 // A functor used to emit an attribute or type entry.
888 uint64_t prevOffset = 0;
889 auto emitAttrOrType = [&](auto &entry) {
890 auto entryValue = entry.getValue();
891
892 auto emitAttrOrTypeRawImpl = [&]() -> void {
893 RawEmitterOstream(attrTypeEmitter) << entryValue;
894 attrTypeEmitter.emitByte(byte: 0, desc: "attr/type separator");
895 };
896 auto emitAttrOrTypeImpl = [&]() -> bool {
897 // Mutable types use a custom encoding when the dialect provides one; a
898 // self-reference resolves through
899 // DialectBytecodeReader::tryStartCyclicRead instead of recursing.
900 // Mutable attributes still use the fallback. Note the kind test:
901 // AttributeTrait::IsMutable also matches mutable types.
902 if constexpr (std::is_same_v<std::decay_t<decltype(entryValue)>,
903 Attribute>) {
904 if (entryValue.template hasTrait<AttributeTrait::IsMutable>()) {
905 emitAttrOrTypeRawImpl();
906 return false;
907 }
908 }
909
910 DialectWriter dialectWriter(config.bytecodeVersion, attrTypeEmitter,
911 numberingState, stringSection,
912 config.dialectVersionMap);
913 if constexpr (std::is_same_v<std::decay_t<decltype(entryValue)>, Type>) {
914 for (const auto &callback : config.typeWriterCallbacks) {
915 if (succeeded(callback->write(entryValue, dialectWriter)))
916 return true;
917 }
918 if (const BytecodeDialectInterface *interface =
919 entry.dialect->interface) {
920 if (succeeded(interface->writeType(type: entryValue, writer&: dialectWriter)))
921 return true;
922 }
923 } else {
924 for (const auto &callback : config.attributeWriterCallbacks) {
925 if (succeeded(callback->write(entryValue, dialectWriter)))
926 return true;
927 }
928 if (const BytecodeDialectInterface *interface =
929 entry.dialect->interface) {
930 if (succeeded(interface->writeAttribute(attribute: entryValue, writer&: dialectWriter)))
931 return true;
932 }
933 }
934
935 // If the entry was not emitted using a callback or a dialect interface,
936 // emit it using the textual format.
937 emitAttrOrTypeRawImpl();
938 return false;
939 };
940
941 bool hasCustomEncoding = emitAttrOrTypeImpl();
942
943 // Record the offset of this entry.
944 uint64_t curOffset = attrTypeEmitter.size();
945 offsetEmitter.emitVarIntWithFlag(value: curOffset - prevOffset, flag: hasCustomEncoding,
946 desc: "attr/type offset");
947 prevOffset = curOffset;
948 };
949
950 // Emit the attribute and type entries for each dialect.
951 writeDialectGrouping(emitter&: offsetEmitter, entries: numberingState.getAttributes(),
952 callback&: emitAttrOrType);
953 writeDialectGrouping(emitter&: offsetEmitter, entries: numberingState.getTypes(),
954 callback&: emitAttrOrType);
955
956 // Emit the sections to the stream.
957 emitter.emitSection(code: bytecode::Section::kAttrTypeOffset,
958 emitter: std::move(offsetEmitter));
959 emitter.emitSection(code: bytecode::Section::kAttrType, emitter: std::move(attrTypeEmitter));
960}
961
962//===----------------------------------------------------------------------===//
963// Operations
964//===----------------------------------------------------------------------===//
965
966LogicalResult BytecodeWriter::writeBlock(EncodingEmitter &emitter,
967 Block *block) {
968 ArrayRef<BlockArgument> args = block->getArguments();
969 bool hasArgs = !args.empty();
970
971 // Emit the number of operations in this block, and if it has arguments. We
972 // use the low bit of the operation count to indicate if the block has
973 // arguments.
974 unsigned numOps = numberingState.getOperationCount(block);
975 emitter.emitVarIntWithFlag(value: numOps, flag: hasArgs, desc: "block num ops");
976
977 // Emit the arguments of the block.
978 if (hasArgs) {
979 emitter.emitVarInt(value: args.size(), desc: "block args count");
980 for (BlockArgument arg : args) {
981 Location argLoc = arg.getLoc();
982 if (config.bytecodeVersion >= bytecode::kElideUnknownBlockArgLocation) {
983 emitter.emitVarIntWithFlag(value: numberingState.getNumber(type: arg.getType()),
984 flag: !isa<UnknownLoc>(Val: argLoc), desc: "block arg type");
985 if (!isa<UnknownLoc>(Val: argLoc))
986 emitter.emitVarInt(value: numberingState.getNumber(loc: argLoc),
987 desc: "block arg location");
988 } else {
989 emitter.emitVarInt(value: numberingState.getNumber(type: arg.getType()),
990 desc: "block arg type");
991 emitter.emitVarInt(value: numberingState.getNumber(loc: argLoc),
992 desc: "block arg location");
993 }
994 }
995 if (config.bytecodeVersion >= bytecode::kUseListOrdering) {
996 uint64_t maskOffset = emitter.size();
997 uint8_t encodingMask = 0;
998 emitter.emitByte(byte: 0, desc: "use-list separator");
999 writeUseListOrders(emitter, opEncodingMask&: encodingMask, range: args);
1000 if (encodingMask)
1001 emitter.patchByte(offset: maskOffset, value: encodingMask, desc: "block patch encoding");
1002 }
1003 }
1004
1005 // Emit the operations within the block.
1006 for (Operation &op : *block)
1007 if (failed(Result: writeOp(emitter, op: &op)))
1008 return failure();
1009 return success();
1010}
1011
1012/// Check legacy property reconstruction using scratch storage. Only inherent
1013/// attributes may restore properties; discardable attributes with native
1014/// property names must remain discardable.
1015static bool preservesLegacyProperties(Operation *op, DictionaryAttr attrs) {
1016 OperationName name = op->getName();
1017 // Operation property storage is aligned to eight bytes.
1018 SmallVector<uint64_t, 8> storage(
1019 llvm::divideCeil(Numerator: op->getPropertiesStorageSize(), Denominator: sizeof(uint64_t)));
1020 PropertyRef reconstructed(name.getOpPropertiesTypeID(), storage.data());
1021 name.initOpProperties(storage: reconstructed, init: {});
1022 auto cleanup =
1023 llvm::scope_exit([&] { name.destroyOpProperties(properties: reconstructed); });
1024
1025 // Unregistered operations cannot restore their property attribute from the
1026 // legacy dictionary: their initialized property attribute is null.
1027 if (name.isRegistered()) {
1028 NamedAttrList legacyAttrs(attrs);
1029 name.populateDefaultAttrs(attrs&: legacyAttrs);
1030 NamedAttrList inherentAttrs;
1031 for (NamedAttribute attr : legacyAttrs)
1032 if (name.getInherentAttr(op, name: attr.getName()))
1033 inherentAttrs.append(attr);
1034 if (!inherentAttrs.empty() &&
1035 failed(Result: name.setOpPropertiesFromAttribute(
1036 opName: name, properties: reconstructed, attr: inherentAttrs.getDictionary(context: op->getContext()),
1037 emitError: [&] {
1038 auto diag = mlir::emitError(loc: op->getLoc());
1039 diag.abandon();
1040 return diag;
1041 })))
1042 return false;
1043 }
1044 return name.compareOpProperties(lhs: op->getPropertiesStorage(), rhs: reconstructed);
1045}
1046
1047LogicalResult BytecodeWriter::writeOp(EncodingEmitter &emitter, Operation *op) {
1048 emitter.emitVarInt(value: numberingState.getNumber(opName: op->getName()), desc: "op name ID");
1049
1050 // Emit a mask for the operation components. We need to fill this in later
1051 // (when we actually know what needs to be emitted), so emit a placeholder for
1052 // now.
1053 uint64_t maskOffset = emitter.size();
1054 uint8_t opEncodingMask = 0;
1055 emitter.emitByte(byte: 0, desc: "op separator");
1056
1057 // Emit the location for this operation.
1058 emitter.emitVarInt(value: numberingState.getNumber(loc: op->getLoc()), desc: "op location");
1059
1060 // Emit the attributes of this operation.
1061 DictionaryAttr attrs = op->getRawDictionaryAttrs();
1062 // Allow deployment to version <kNativePropertiesEncoding by merging inherent
1063 // attribute with the discardable ones. We should fail if there are any
1064 // conflicts. When properties are not used by the op, also store everything as
1065 // attributes.
1066 if (config.bytecodeVersion < bytecode::kNativePropertiesEncoding &&
1067 op->getPropertiesStorage()) {
1068 NamedAttrList allAttrs;
1069 op->getName().walkInherentAttrs(op, visitor: [&](StringRef name, Attribute &attr) {
1070 allAttrs.append(name, attr);
1071 });
1072 allAttrs.append(newAttributes: op->getDiscardableAttrDictionary().getValue());
1073 attrs = allAttrs.getDictionary(context: op->getContext());
1074
1075 // Versions before kNativePropertiesEncoding can only reconstruct properties
1076 // from these attributes. Check that native properties would not silently
1077 // revert to their defaults. Later versions encode properties directly and
1078 // do not need this legacy reconstruction check.
1079 if (!preservesLegacyProperties(op, attrs))
1080 return op->emitOpError()
1081 << "cannot emit bytecode before version "
1082 << static_cast<int64_t>(bytecode::kNativePropertiesEncoding)
1083 << " without losing operation properties";
1084 }
1085 if (!attrs.empty()) {
1086 opEncodingMask |= bytecode::OpEncodingMask::kHasAttrs;
1087 emitter.emitVarInt(value: numberingState.getNumber(attr: attrs), desc: "op attrs count");
1088 }
1089
1090 // Emit the properties of this operation, for now we still support deployment
1091 // to version <kNativePropertiesEncoding.
1092 if (config.bytecodeVersion >= bytecode::kNativePropertiesEncoding) {
1093 std::optional<ssize_t> propertiesId = propertiesSection.emit(op);
1094 if (propertiesId.has_value()) {
1095 opEncodingMask |= bytecode::OpEncodingMask::kHasProperties;
1096 emitter.emitVarInt(value: *propertiesId, desc: "op properties ID");
1097 }
1098 }
1099
1100 // Emit the result types of the operation.
1101 if (unsigned numResults = op->getNumResults()) {
1102 opEncodingMask |= bytecode::OpEncodingMask::kHasResults;
1103 emitter.emitVarInt(value: numResults, desc: "op results count");
1104 for (Type type : op->getResultTypes())
1105 emitter.emitVarInt(value: numberingState.getNumber(type), desc: "op result type");
1106 }
1107
1108 // Emit the operands of the operation.
1109 if (unsigned numOperands = op->getNumOperands()) {
1110 opEncodingMask |= bytecode::OpEncodingMask::kHasOperands;
1111 emitter.emitVarInt(value: numOperands, desc: "op operands count");
1112 for (Value operand : op->getOperands())
1113 emitter.emitVarInt(value: numberingState.getNumber(value: operand), desc: "op operand types");
1114 }
1115
1116 // Emit the successors of the operation.
1117 if (unsigned numSuccessors = op->getNumSuccessors()) {
1118 opEncodingMask |= bytecode::OpEncodingMask::kHasSuccessors;
1119 emitter.emitVarInt(value: numSuccessors, desc: "op successors count");
1120 for (Block *successor : op->getSuccessors())
1121 emitter.emitVarInt(value: numberingState.getNumber(block: successor), desc: "op successor");
1122 }
1123
1124 // Emit the use-list orders to bytecode, so we can reconstruct the same order
1125 // at parsing.
1126 if (config.bytecodeVersion >= bytecode::kUseListOrdering)
1127 writeUseListOrders(emitter, opEncodingMask, range: ValueRange(op->getResults()));
1128
1129 // Check for regions.
1130 unsigned numRegions = op->getNumRegions();
1131 if (numRegions)
1132 opEncodingMask |= bytecode::OpEncodingMask::kHasInlineRegions;
1133
1134 // Update the mask for the operation.
1135 emitter.patchByte(offset: maskOffset, value: opEncodingMask, desc: "op encoding mask");
1136
1137 // With the mask emitted, we can now emit the regions of the operation. We do
1138 // this after mask emission to avoid offset complications that may arise by
1139 // emitting the regions first (e.g. if the regions are huge, backpatching the
1140 // op encoding mask is more annoying).
1141 if (numRegions) {
1142 bool isIsolatedFromAbove = numberingState.isIsolatedFromAbove(op);
1143 emitter.emitVarIntWithFlag(value: numRegions, flag: isIsolatedFromAbove,
1144 desc: "op regions count");
1145
1146 // If the region is not isolated from above, or we are emitting bytecode
1147 // targeting version <kLazyLoading, we don't use a section.
1148 if (isIsolatedFromAbove &&
1149 config.bytecodeVersion >= bytecode::kLazyLoading) {
1150 EncodingEmitter regionEmitter;
1151 if (failed(Result: writeRegions(emitter&: regionEmitter, regions: op->getRegions())))
1152 return failure();
1153 emitter.emitSection(code: bytecode::Section::kIR, emitter: std::move(regionEmitter));
1154
1155 } else if (failed(Result: writeRegions(emitter, regions: op->getRegions()))) {
1156 return failure();
1157 }
1158 }
1159 return success();
1160}
1161
1162void BytecodeWriter::writeUseListOrders(EncodingEmitter &emitter,
1163 uint8_t &opEncodingMask,
1164 ValueRange range) {
1165 // Loop over the results and store the use-list order per result index.
1166 llvm::MapVector<unsigned, llvm::SmallVector<unsigned>> map;
1167 for (auto item : llvm::enumerate(First&: range)) {
1168 auto value = item.value();
1169 // No need to store a custom use-list order if the result does not have
1170 // multiple uses.
1171 if (value.use_empty() || value.hasOneUse())
1172 continue;
1173
1174 // For each result, assemble the list of pairs (use-list-index,
1175 // global-value-index). While doing so, detect if the global-value-index is
1176 // already ordered with respect to the use-list-index.
1177 bool alreadyOrdered = true;
1178 auto &firstUse = *value.use_begin();
1179 uint64_t prevID = bytecode::getUseID(
1180 val&: firstUse, ownerID: numberingState.getNumber(op: firstUse.getOwner()));
1181 llvm::SmallVector<std::pair<unsigned, uint64_t>> useListPairs(
1182 {{0, prevID}});
1183
1184 for (auto use : llvm::drop_begin(RangeOrContainer: llvm::enumerate(First: value.getUses()))) {
1185 uint64_t currentID = bytecode::getUseID(
1186 val&: use.value(), ownerID: numberingState.getNumber(op: use.value().getOwner()));
1187 // The use-list order achieved when building the IR at parsing always
1188 // pushes new uses on front. Hence, if the order by unique ID is
1189 // monotonically decreasing, a roundtrip to bytecode preserves such order.
1190 alreadyOrdered &= (prevID > currentID);
1191 useListPairs.push_back(Elt: {use.index(), currentID});
1192 prevID = currentID;
1193 }
1194
1195 // Do not emit if the order is already sorted.
1196 if (alreadyOrdered)
1197 continue;
1198
1199 // Sort the use indices by the unique ID indices in descending order.
1200 std::sort(
1201 first: useListPairs.begin(), last: useListPairs.end(),
1202 comp: [](auto elem1, auto elem2) { return elem1.second > elem2.second; });
1203
1204 map.try_emplace(Key: item.index(), Args: llvm::map_range(C&: useListPairs, F: [](auto elem) {
1205 return elem.first;
1206 }));
1207 }
1208
1209 if (map.empty())
1210 return;
1211
1212 opEncodingMask |= bytecode::OpEncodingMask::kHasUseListOrders;
1213 // Emit the number of results that have a custom use-list order if the number
1214 // of results is greater than one.
1215 if (range.size() != 1) {
1216 emitter.emitVarInt(value: map.size(), desc: "custom use-list size");
1217 }
1218
1219 for (const auto &[resultIdx, useListOrder] : map) {
1220 // Compute the number of uses that are actually shuffled. If those are less
1221 // than half of the total uses, encoding the index pair `(src, dst)` is more
1222 // space efficient.
1223 size_t shuffledElements =
1224 llvm::count_if(Range: llvm::enumerate(First: useListOrder),
1225 P: [](auto item) { return item.index() != item.value(); });
1226 bool indexPairEncoding = shuffledElements < (useListOrder.size() / 2);
1227
1228 // For single result, we don't need to store the result index.
1229 if (range.size() != 1)
1230 emitter.emitVarInt(value: resultIdx, desc: "use-list result index");
1231
1232 if (indexPairEncoding) {
1233 emitter.emitVarIntWithFlag(value: shuffledElements * 2, flag: indexPairEncoding,
1234 desc: "use-list index pair size");
1235 for (auto pair : llvm::enumerate(First: useListOrder)) {
1236 if (pair.index() != pair.value()) {
1237 emitter.emitVarInt(value: pair.value(), desc: "use-list index pair first");
1238 emitter.emitVarInt(value: pair.index(), desc: "use-list index pair second");
1239 }
1240 }
1241 } else {
1242 emitter.emitVarIntWithFlag(value: useListOrder.size(), flag: indexPairEncoding,
1243 desc: "use-list size");
1244 for (const auto &index : useListOrder)
1245 emitter.emitVarInt(value: index, desc: "use-list order");
1246 }
1247 }
1248}
1249
1250LogicalResult BytecodeWriter::writeRegion(EncodingEmitter &emitter,
1251 Region *region) {
1252 // If the region is empty, we only need to emit the number of blocks (which is
1253 // zero).
1254 if (region->empty()) {
1255 emitter.emitVarInt(/*numBlocks*/ value: 0, desc: "region block count empty");
1256 return success();
1257 }
1258
1259 // Emit the number of blocks and values within the region.
1260 unsigned numBlocks, numValues;
1261 std::tie(args&: numBlocks, args&: numValues) = numberingState.getBlockValueCount(region);
1262 emitter.emitVarInt(value: numBlocks, desc: "region block count");
1263 emitter.emitVarInt(value: numValues, desc: "region value count");
1264
1265 // Emit the blocks within the region.
1266 for (Block &block : *region)
1267 if (failed(Result: writeBlock(emitter, block: &block)))
1268 return failure();
1269 return success();
1270}
1271
1272LogicalResult BytecodeWriter::writeIRSection(EncodingEmitter &emitter,
1273 Operation *op) {
1274 EncodingEmitter irEmitter;
1275
1276 // Write the IR section the same way as a block with no arguments. Note that
1277 // the low-bit of the operation count for a block is used to indicate if the
1278 // block has arguments, which in this case is always false.
1279 irEmitter.emitVarIntWithFlag(/*numOps*/ value: 1, /*hasArgs*/ flag: false, desc: "ir section");
1280
1281 // Emit the operations.
1282 if (failed(Result: writeOp(emitter&: irEmitter, op)))
1283 return failure();
1284
1285 emitter.emitSection(code: bytecode::Section::kIR, emitter: std::move(irEmitter));
1286 return success();
1287}
1288
1289//===----------------------------------------------------------------------===//
1290// Resources
1291//===----------------------------------------------------------------------===//
1292
1293namespace {
1294/// This class represents a resource builder implementation for the MLIR
1295/// bytecode format.
1296class ResourceBuilder : public AsmResourceBuilder {
1297public:
1298 using PostProcessFn = function_ref<void(StringRef, AsmResourceEntryKind)>;
1299
1300 ResourceBuilder(EncodingEmitter &emitter, StringSectionBuilder &stringSection,
1301 PostProcessFn postProcessFn, bool shouldElideData)
1302 : emitter(emitter), stringSection(stringSection),
1303 postProcessFn(postProcessFn), shouldElideData(shouldElideData) {}
1304 ~ResourceBuilder() override = default;
1305
1306 void buildBlob(StringRef key, ArrayRef<char> data,
1307 uint32_t dataAlignment) final {
1308 if (!shouldElideData)
1309 emitter.emitOwnedBlobAndAlignment(data, alignment: dataAlignment, desc: "resource blob");
1310 postProcessFn(key, AsmResourceEntryKind::Blob);
1311 }
1312 void buildBool(StringRef key, bool data) final {
1313 if (!shouldElideData)
1314 emitter.emitByte(byte: data, desc: "resource bool");
1315 postProcessFn(key, AsmResourceEntryKind::Bool);
1316 }
1317 void buildString(StringRef key, StringRef data) final {
1318 if (!shouldElideData)
1319 emitter.emitVarInt(value: stringSection.insert(str: data), desc: "resource string");
1320 postProcessFn(key, AsmResourceEntryKind::String);
1321 }
1322
1323private:
1324 EncodingEmitter &emitter;
1325 StringSectionBuilder &stringSection;
1326 PostProcessFn postProcessFn;
1327 bool shouldElideData = false;
1328};
1329} // namespace
1330
1331void BytecodeWriter::writeResourceSection(Operation *op,
1332 EncodingEmitter &emitter) {
1333 EncodingEmitter resourceEmitter;
1334 EncodingEmitter resourceOffsetEmitter;
1335 uint64_t prevOffset = 0;
1336 SmallVector<std::tuple<StringRef, AsmResourceEntryKind, uint64_t>>
1337 curResourceEntries;
1338
1339 // Functor used to process the offset for a resource of `kind` defined by
1340 // 'key'.
1341 auto appendResourceOffset = [&](StringRef key, AsmResourceEntryKind kind) {
1342 uint64_t curOffset = resourceEmitter.size();
1343 curResourceEntries.emplace_back(Args&: key, Args&: kind, Args: curOffset - prevOffset);
1344 prevOffset = curOffset;
1345 };
1346
1347 // Functor used to emit a resource group defined by 'key'.
1348 auto emitResourceGroup = [&](uint64_t key) {
1349 resourceOffsetEmitter.emitVarInt(value: key, desc: "resource group key");
1350 resourceOffsetEmitter.emitVarInt(value: curResourceEntries.size(),
1351 desc: "resource group size");
1352 for (auto [key, kind, size] : curResourceEntries) {
1353 resourceOffsetEmitter.emitVarInt(value: stringSection.insert(str: key),
1354 desc: "resource key");
1355 resourceOffsetEmitter.emitVarInt(value: size, desc: "resource size");
1356 resourceOffsetEmitter.emitByte(byte: kind, desc: "resource kind");
1357 }
1358 };
1359
1360 // Builder used to emit resources.
1361 ResourceBuilder entryBuilder(resourceEmitter, stringSection,
1362 appendResourceOffset,
1363 config.shouldElideResourceData);
1364
1365 // Emit the external resource entries.
1366 resourceOffsetEmitter.emitVarInt(value: config.externalResourcePrinters.size(),
1367 desc: "external resource printer count");
1368 for (const auto &printer : config.externalResourcePrinters) {
1369 curResourceEntries.clear();
1370 printer->buildResources(op, builder&: entryBuilder);
1371 emitResourceGroup(stringSection.insert(str: printer->getName()));
1372 }
1373
1374 // Emit the dialect resource entries.
1375 for (DialectNumbering &dialect : numberingState.getDialects()) {
1376 if (!dialect.asmInterface)
1377 continue;
1378 curResourceEntries.clear();
1379 dialect.asmInterface->buildResources(op, referencedResources: dialect.resources, builder&: entryBuilder);
1380
1381 // Emit the declaration resources for this dialect, these didn't get emitted
1382 // by the interface. These resources don't have data attached, so just use a
1383 // "blob" kind as a placeholder.
1384 for (const auto &resource : dialect.resourceMap)
1385 if (resource.second->isDeclaration)
1386 appendResourceOffset(resource.first, AsmResourceEntryKind::Blob);
1387
1388 // Emit the resource group for this dialect.
1389 if (!curResourceEntries.empty())
1390 emitResourceGroup(dialect.number);
1391 }
1392
1393 // If we didn't emit any resource groups, elide the resource sections.
1394 if (resourceOffsetEmitter.size() == 0)
1395 return;
1396
1397 emitter.emitSection(code: bytecode::Section::kResourceOffset,
1398 emitter: std::move(resourceOffsetEmitter));
1399 emitter.emitSection(code: bytecode::Section::kResource, emitter: std::move(resourceEmitter));
1400}
1401
1402//===----------------------------------------------------------------------===//
1403// Strings
1404//===----------------------------------------------------------------------===//
1405
1406void BytecodeWriter::writeStringSection(EncodingEmitter &emitter) {
1407 EncodingEmitter stringEmitter;
1408 stringSection.write(emitter&: stringEmitter);
1409 emitter.emitSection(code: bytecode::Section::kString, emitter: std::move(stringEmitter));
1410}
1411
1412//===----------------------------------------------------------------------===//
1413// Properties
1414//===----------------------------------------------------------------------===//
1415
1416void BytecodeWriter::writePropertiesSection(EncodingEmitter &emitter) {
1417 EncodingEmitter propertiesEmitter;
1418 propertiesSection.write(emitter&: propertiesEmitter);
1419 emitter.emitSection(code: bytecode::Section::kProperties,
1420 emitter: std::move(propertiesEmitter));
1421}
1422
1423//===----------------------------------------------------------------------===//
1424// Entry Points
1425//===----------------------------------------------------------------------===//
1426
1427LogicalResult mlir::writeBytecodeToFile(Operation *op, raw_ostream &os,
1428 const BytecodeWriterConfig &config) {
1429 // Validate before numbering the IR, which may invoke dialect writers.
1430 for (const auto &entry : config.getDialectVersionMap()) {
1431 Dialect *dialect = op->getContext()->getLoadedDialect(name: entry.getKey());
1432 if (!dialect)
1433 continue;
1434 auto *interface =
1435 dialect->getRegisteredInterface<BytecodeDialectInterface>();
1436 if (interface && failed(Result: interface->verifyVersion(topLevelOp: op, version: *entry.getValue())))
1437 return failure();
1438 }
1439 BytecodeWriter writer(op, config);
1440 return writer.write(rootOp: op, os);
1441}
1442