1//===- BytecodeReader.cpp - MLIR Bytecode Reader --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Bytecode/BytecodeReader.h"
10#include "mlir/AsmParser/AsmParser.h"
11#include "mlir/Bytecode/BytecodeImplementation.h"
12#include "mlir/Bytecode/BytecodeOpInterface.h"
13#include "mlir/Bytecode/Encoding.h"
14#include "mlir/IR/BuiltinOps.h"
15#include "mlir/IR/Diagnostics.h"
16#include "mlir/IR/OpImplementation.h"
17#include "mlir/IR/Verifier.h"
18#include "mlir/IR/Visitors.h"
19#include "mlir/Support/LLVM.h"
20#include "llvm/ADT/ArrayRef.h"
21#include "llvm/ADT/MapVector.h"
22#include "llvm/ADT/ScopeExit.h"
23#include "llvm/ADT/StringExtras.h"
24#include "llvm/ADT/StringRef.h"
25#include "llvm/Support/Endian.h"
26#include "llvm/Support/MemoryBufferRef.h"
27#include "llvm/Support/SourceMgr.h"
28
29#include <cstddef>
30#include <cstdint>
31#include <deque>
32#include <list>
33#include <memory>
34#include <numeric>
35#include <optional>
36
37#define DEBUG_TYPE "mlir-bytecode-reader"
38
39using namespace mlir;
40
41/// Stringify the given section ID.
42static std::string toString(bytecode::Section::ID sectionID) {
43 switch (sectionID) {
44 case bytecode::Section::kString:
45 return "String (0)";
46 case bytecode::Section::kDialect:
47 return "Dialect (1)";
48 case bytecode::Section::kAttrType:
49 return "AttrType (2)";
50 case bytecode::Section::kAttrTypeOffset:
51 return "AttrTypeOffset (3)";
52 case bytecode::Section::kIR:
53 return "IR (4)";
54 case bytecode::Section::kResource:
55 return "Resource (5)";
56 case bytecode::Section::kResourceOffset:
57 return "ResourceOffset (6)";
58 case bytecode::Section::kDialectVersions:
59 return "DialectVersions (7)";
60 case bytecode::Section::kProperties:
61 return "Properties (8)";
62 default:
63 return ("Unknown (" + Twine(static_cast<unsigned>(sectionID)) + ")").str();
64 }
65}
66
67/// Returns true if the given top-level section ID is optional.
68static bool isSectionOptional(bytecode::Section::ID sectionID, int version) {
69 switch (sectionID) {
70 case bytecode::Section::kString:
71 case bytecode::Section::kDialect:
72 case bytecode::Section::kAttrType:
73 case bytecode::Section::kAttrTypeOffset:
74 case bytecode::Section::kIR:
75 return false;
76 case bytecode::Section::kResource:
77 case bytecode::Section::kResourceOffset:
78 case bytecode::Section::kDialectVersions:
79 return true;
80 case bytecode::Section::kProperties:
81 return version < bytecode::kNativePropertiesEncoding;
82 default:
83 llvm_unreachable("unknown section ID");
84 }
85}
86
87//===----------------------------------------------------------------------===//
88// EncodingReader
89//===----------------------------------------------------------------------===//
90
91namespace {
92class EncodingReader {
93public:
94 explicit EncodingReader(ArrayRef<uint8_t> contents, Location fileLoc)
95 : buffer(contents), dataIt(buffer.begin()), fileLoc(fileLoc) {}
96 explicit EncodingReader(StringRef contents, Location fileLoc)
97 : EncodingReader({reinterpret_cast<const uint8_t *>(contents.data()),
98 contents.size()},
99 fileLoc) {}
100
101 /// Returns true if the entire section has been read.
102 bool empty() const { return dataIt == buffer.end(); }
103
104 /// Returns the remaining size of the bytecode.
105 size_t size() const { return buffer.end() - dataIt; }
106
107 /// Align the current reader position to the specified alignment.
108 LogicalResult alignTo(unsigned alignment) {
109 if (!llvm::isPowerOf2_32(Value: alignment))
110 return emitError(args: "expected alignment to be a power-of-two");
111
112 auto isUnaligned = [&](const uint8_t *ptr) {
113 return ((uintptr_t)ptr & (alignment - 1)) != 0;
114 };
115
116 // Shift the reader position to the next alignment boundary.
117 // Note: this assumes the pointer alignment matches the alignment of the
118 // data from the start of the buffer. In other words, this code is only
119 // valid if `dataIt` is offsetting into an already aligned buffer.
120 while (isUnaligned(dataIt)) {
121 uint8_t padding;
122 if (failed(Result: parseByte(value&: padding)))
123 return failure();
124 if (padding != bytecode::kAlignmentByte) {
125 return emitError(args: "expected alignment byte (0xCB), but got: '0x" +
126 llvm::utohexstr(X: padding) + "'");
127 }
128 }
129
130 // Ensure the data iterator is now aligned. This case is unlikely because we
131 // *just* went through the effort to align the data iterator.
132 if (LLVM_UNLIKELY(isUnaligned(dataIt))) {
133 return emitError(args: "expected data iterator aligned to ", args&: alignment,
134 args: ", but got pointer: '0x" +
135 llvm::utohexstr(X: (uintptr_t)dataIt) + "'");
136 }
137
138 return success();
139 }
140
141 /// Emit an error using the given arguments.
142 template <typename... Args>
143 InFlightDiagnostic emitError(Args &&...args) const {
144 return ::emitError(loc: fileLoc).append(std::forward<Args>(args)...);
145 }
146 InFlightDiagnostic emitError() const { return ::emitError(loc: fileLoc); }
147
148 /// Emit a warning using the given arguments.
149 template <typename... Args>
150 InFlightDiagnostic emitWarning(Args &&...args) const {
151 return ::emitWarning(loc: fileLoc).append(std::forward<Args>(args)...);
152 }
153 InFlightDiagnostic emitWarning() const { return ::emitWarning(loc: fileLoc); }
154
155 /// Parse a single byte from the stream.
156 template <typename T>
157 LogicalResult parseByte(T &value) {
158 if (empty())
159 return emitError(args: "attempting to parse a byte at the end of the bytecode");
160 value = static_cast<T>(*dataIt++);
161 return success();
162 }
163 /// Parse a range of bytes of 'length' into the given result.
164 LogicalResult parseBytes(size_t length, ArrayRef<uint8_t> &result) {
165 if (length > size()) {
166 return emitError(args: "attempting to parse ", args&: length, args: " bytes when only ",
167 args: size(), args: " remain");
168 }
169 result = {dataIt, length};
170 dataIt += length;
171 return success();
172 }
173 /// Parse a range of bytes of 'length' into the given result, which can be
174 /// assumed to be large enough to hold `length`.
175 LogicalResult parseBytes(size_t length, uint8_t *result) {
176 if (length > size()) {
177 return emitError(args: "attempting to parse ", args&: length, args: " bytes when only ",
178 args: size(), args: " remain");
179 }
180 memcpy(dest: result, src: dataIt, n: length);
181 dataIt += length;
182 return success();
183 }
184
185 /// Parse an aligned blob of data, where the alignment was encoded alongside
186 /// the data.
187 LogicalResult parseBlobAndAlignment(ArrayRef<uint8_t> &data,
188 uint64_t &alignment) {
189 uint64_t dataSize;
190 if (failed(Result: parseVarInt(result&: alignment)) || failed(Result: parseVarInt(result&: dataSize)) ||
191 failed(Result: alignTo(alignment)))
192 return failure();
193 return parseBytes(length: dataSize, result&: data);
194 }
195
196 /// Parse a variable length encoded integer from the byte stream. The first
197 /// encoded byte contains a prefix in the low bits indicating the encoded
198 /// length of the value. This length prefix is a bit sequence of '0's followed
199 /// by a '1'. The number of '0' bits indicate the number of _additional_ bytes
200 /// (not including the prefix byte). All remaining bits in the first byte,
201 /// along with all of the bits in additional bytes, provide the value of the
202 /// integer encoded in little-endian order.
203 LogicalResult parseVarInt(uint64_t &result) {
204 // Parse the first byte of the encoding, which contains the length prefix.
205 if (failed(Result: parseByte(value&: result)))
206 return failure();
207
208 // Handle the overwhelmingly common case where the value is stored in a
209 // single byte. In this case, the first bit is the `1` marker bit.
210 if (LLVM_LIKELY(result & 1)) {
211 result >>= 1;
212 return success();
213 }
214
215 // Handle the overwhelming uncommon case where the value required all 8
216 // bytes (i.e. a really really big number). In this case, the marker byte is
217 // all zeros: `00000000`.
218 if (LLVM_UNLIKELY(result == 0)) {
219 llvm::support::ulittle64_t resultLE;
220 if (failed(Result: parseBytes(length: sizeof(resultLE),
221 result: reinterpret_cast<uint8_t *>(&resultLE))))
222 return failure();
223 result = resultLE;
224 return success();
225 }
226 return parseMultiByteVarInt(result);
227 }
228
229 /// Parse a signed variable length encoded integer from the byte stream. A
230 /// signed varint is encoded as a normal varint with zigzag encoding applied,
231 /// i.e. the low bit of the value is used to indicate the sign.
232 LogicalResult parseSignedVarInt(uint64_t &result) {
233 if (failed(Result: parseVarInt(result)))
234 return failure();
235 // Essentially (but using unsigned): (x >> 1) ^ -(x & 1)
236 result = (result >> 1) ^ (~(result & 1) + 1);
237 return success();
238 }
239
240 /// Parse a variable length encoded integer whose low bit is used to encode an
241 /// unrelated flag, i.e: `(integerValue << 1) | (flag ? 1 : 0)`.
242 LogicalResult parseVarIntWithFlag(uint64_t &result, bool &flag) {
243 if (failed(Result: parseVarInt(result)))
244 return failure();
245 flag = result & 1;
246 result >>= 1;
247 return success();
248 }
249
250 /// Skip the first `length` bytes within the reader.
251 LogicalResult skipBytes(size_t length) {
252 if (length > size()) {
253 return emitError(args: "attempting to skip ", args&: length, args: " bytes when only ",
254 args: size(), args: " remain");
255 }
256 dataIt += length;
257 return success();
258 }
259
260 /// Parse a null-terminated string into `result` (without including the NUL
261 /// terminator).
262 LogicalResult parseNullTerminatedString(StringRef &result) {
263 const char *startIt = (const char *)dataIt;
264 const char *nulIt = (const char *)memchr(s: startIt, c: 0, n: size());
265 if (!nulIt)
266 return emitError(
267 args: "malformed null-terminated string, no null character found");
268
269 result = StringRef(startIt, nulIt - startIt);
270 dataIt = (const uint8_t *)nulIt + 1;
271 return success();
272 }
273
274 /// Validate that the alignment requested in the section is valid.
275 using ValidateAlignmentFn = function_ref<LogicalResult(unsigned alignment)>;
276
277 /// Parse a section header, placing the kind of section in `sectionID` and the
278 /// contents of the section in `sectionData`.
279 LogicalResult parseSection(bytecode::Section::ID &sectionID,
280 ValidateAlignmentFn alignmentValidator,
281 ArrayRef<uint8_t> &sectionData) {
282 uint8_t sectionIDAndHasAlignment;
283 uint64_t length;
284 if (failed(Result: parseByte(value&: sectionIDAndHasAlignment)) ||
285 failed(Result: parseVarInt(result&: length)))
286 return failure();
287
288 // Extract the section ID and whether the section is aligned. The high bit
289 // of the ID is the alignment flag.
290 sectionID = static_cast<bytecode::Section::ID>(sectionIDAndHasAlignment &
291 0b01111111);
292 bool hasAlignment = sectionIDAndHasAlignment & 0b10000000;
293
294 // Check that the section is actually valid before trying to process its
295 // data.
296 if (sectionID >= bytecode::Section::kNumSections)
297 return emitError(args: "invalid section ID: ", args: unsigned(sectionID));
298
299 // Process the section alignment if present.
300 if (hasAlignment) {
301 // Read the requested alignment from the bytecode parser.
302 uint64_t alignment;
303 if (failed(Result: parseVarInt(result&: alignment)))
304 return failure();
305
306 // Check that the requested alignment must not exceed the alignment of
307 // the root buffer itself. Otherwise we cannot guarantee that pointers
308 // derived from this buffer will actually satisfy the requested alignment
309 // globally.
310 //
311 // Consider a bytecode buffer that is guaranteed to be 8k aligned, but not
312 // 16k aligned (e.g. absolute address 40960. If a section inside this
313 // buffer declares a 16k alignment requirement, two problems can arise:
314 //
315 // (a) If we "align forward" the current pointer to the next
316 // 16k boundary, the amount of padding we skip depends on the
317 // buffer's starting address. For example:
318 //
319 // buffer_start = 40960
320 // next 16k boundary = 49152
321 // bytes skipped = 49152 - 40960 = 8192
322 //
323 // This leaves behind variable padding that could be misinterpreted
324 // as part of the next section.
325 //
326 // (b) If we align relative to the buffer start, we may
327 // obtain addresses that are multiples of "buffer_start +
328 // section_alignment" rather than truly globally aligned
329 // addresses. For example:
330 //
331 // buffer_start = 40960 (5×8k, 8k aligned but not 16k)
332 // offset = 16384 (first multiple of 16k)
333 // section_ptr = 40960 + 16384 = 57344
334 //
335 // 57344 is 8k aligned but not 16k aligned.
336 // Any consumer expecting true 16k alignment would see this as a
337 // violation.
338 if (failed(Result: alignmentValidator(alignment)))
339 return emitError(args: "failed to align section ID: ", args: unsigned(sectionID));
340
341 // Align the buffer.
342 if (failed(Result: alignTo(alignment)))
343 return failure();
344 }
345
346 // Parse the actual section data.
347 return parseBytes(length: static_cast<size_t>(length), result&: sectionData);
348 }
349
350 Location getLoc() const { return fileLoc; }
351
352private:
353 /// Parse a variable length encoded integer from the byte stream. This method
354 /// is a fallback when the number of bytes used to encode the value is greater
355 /// than 1, but less than the max (9). The provided `result` value can be
356 /// assumed to already contain the first byte of the value.
357 /// NOTE: This method is marked noinline to avoid pessimizing the common case
358 /// of single byte encoding.
359 LLVM_ATTRIBUTE_NOINLINE LogicalResult parseMultiByteVarInt(uint64_t &result) {
360 // Count the number of trailing zeros in the marker byte, this indicates the
361 // number of trailing bytes that are part of the value. We use `uint32_t`
362 // here because we only care about the first byte, and so that be actually
363 // get ctz intrinsic calls when possible (the `uint8_t` overload uses a loop
364 // implementation).
365 uint32_t numBytes = llvm::countr_zero<uint32_t>(Val: result);
366 assert(numBytes > 0 && numBytes <= 7 &&
367 "unexpected number of trailing zeros in varint encoding");
368
369 // Parse in the remaining bytes of the value.
370 llvm::support::ulittle64_t resultLE(result);
371 if (failed(
372 Result: parseBytes(length: numBytes, result: reinterpret_cast<uint8_t *>(&resultLE) + 1)))
373 return failure();
374
375 // Shift out the low-order bits that were used to mark how the value was
376 // encoded.
377 result = resultLE >> (numBytes + 1);
378 return success();
379 }
380
381 /// The bytecode buffer.
382 ArrayRef<uint8_t> buffer;
383
384 /// The current iterator within the 'buffer'.
385 const uint8_t *dataIt;
386
387 /// A location for the bytecode used to report errors.
388 Location fileLoc;
389};
390} // namespace
391
392/// Resolve an index into the given entry list. `entry` may either be a
393/// reference, in which case it is assigned to the corresponding value in
394/// `entries`, or a pointer, in which case it is assigned to the address of the
395/// element in `entries`.
396template <typename RangeT, typename T>
397static LogicalResult resolveEntry(EncodingReader &reader, RangeT &entries,
398 uint64_t index, T &entry,
399 StringRef entryStr) {
400 if (index >= entries.size())
401 return reader.emitError(args: "invalid ", args&: entryStr, args: " index: ", args&: index);
402
403 // If the provided entry is a pointer, resolve to the address of the entry.
404 if constexpr (std::is_convertible_v<llvm::detail::ValueOfRange<RangeT>, T>)
405 entry = entries[index];
406 else
407 entry = &entries[index];
408 return success();
409}
410
411/// Parse and resolve an index into the given entry list.
412template <typename RangeT, typename T>
413static LogicalResult parseEntry(EncodingReader &reader, RangeT &entries,
414 T &entry, StringRef entryStr) {
415 uint64_t entryIdx;
416 if (failed(Result: reader.parseVarInt(result&: entryIdx)))
417 return failure();
418 return resolveEntry(reader, entries, entryIdx, entry, entryStr);
419}
420
421//===----------------------------------------------------------------------===//
422// StringSectionReader
423//===----------------------------------------------------------------------===//
424
425namespace {
426/// This class is used to read references to the string section from the
427/// bytecode.
428class StringSectionReader {
429public:
430 /// Initialize the string section reader with the given section data.
431 LogicalResult initialize(Location fileLoc, ArrayRef<uint8_t> sectionData);
432
433 /// Parse a shared string from the string section. The shared string is
434 /// encoded using an index to a corresponding string in the string section.
435 LogicalResult parseString(EncodingReader &reader, StringRef &result) const {
436 return parseEntry(reader, entries: strings, entry&: result, entryStr: "string");
437 }
438
439 /// Parse a shared string from the string section. The shared string is
440 /// encoded using an index to a corresponding string in the string section.
441 /// This variant parses a flag compressed with the index.
442 LogicalResult parseStringWithFlag(EncodingReader &reader, StringRef &result,
443 bool &flag) const {
444 uint64_t entryIdx;
445 if (failed(Result: reader.parseVarIntWithFlag(result&: entryIdx, flag)))
446 return failure();
447 return parseStringAtIndex(reader, index: entryIdx, result);
448 }
449
450 /// Parse a shared string from the string section. The shared string is
451 /// encoded using an index to a corresponding string in the string section.
452 LogicalResult parseStringAtIndex(EncodingReader &reader, uint64_t index,
453 StringRef &result) const {
454 return resolveEntry(reader, entries: strings, index, entry&: result, entryStr: "string");
455 }
456
457private:
458 /// The table of strings referenced within the bytecode file.
459 SmallVector<StringRef> strings;
460};
461} // namespace
462
463LogicalResult StringSectionReader::initialize(Location fileLoc,
464 ArrayRef<uint8_t> sectionData) {
465 EncodingReader stringReader(sectionData, fileLoc);
466
467 // Parse the number of strings in the section.
468 uint64_t numStrings;
469 if (failed(Result: stringReader.parseVarInt(result&: numStrings)))
470 return failure();
471 strings.resize(N: numStrings);
472
473 // Parse each of the strings. The sizes of the strings are encoded in reverse
474 // order, so that's the order we populate the table.
475 size_t stringDataEndOffset = sectionData.size();
476 for (StringRef &string : llvm::reverse(C&: strings)) {
477 uint64_t stringSize;
478 if (failed(Result: stringReader.parseVarInt(result&: stringSize)))
479 return failure();
480 if (stringDataEndOffset < stringSize) {
481 return stringReader.emitError(
482 args: "string size exceeds the available data size");
483 }
484
485 // Extract the string from the data, dropping the null character.
486 size_t stringOffset = stringDataEndOffset - stringSize;
487 string = StringRef(
488 reinterpret_cast<const char *>(sectionData.data() + stringOffset),
489 stringSize - 1);
490 stringDataEndOffset = stringOffset;
491 }
492
493 // Check that the only remaining data was for the strings, i.e. the reader
494 // should be at the same offset as the first string.
495 if ((sectionData.size() - stringReader.size()) != stringDataEndOffset) {
496 return stringReader.emitError(args: "unexpected trailing data between the "
497 "offsets for strings and their data");
498 }
499 return success();
500}
501
502//===----------------------------------------------------------------------===//
503// BytecodeDialect
504//===----------------------------------------------------------------------===//
505
506namespace {
507class DialectReader;
508
509/// This struct represents a dialect entry within the bytecode.
510struct BytecodeDialect {
511 /// Load the dialect into the provided context if it hasn't been loaded yet.
512 /// Returns failure if the dialect couldn't be loaded *and* the provided
513 /// context does not allow unregistered dialects. The provided reader is used
514 /// for error emission if necessary.
515 LogicalResult load(const DialectReader &reader, MLIRContext *ctx);
516
517 /// Return the loaded dialect, or nullptr if the dialect is unknown. This can
518 /// only be called after `load`.
519 Dialect *getLoadedDialect() const {
520 assert(dialect &&
521 "expected `load` to be invoked before `getLoadedDialect`");
522 return *dialect;
523 }
524
525 /// The loaded dialect entry. This field is std::nullopt if we haven't
526 /// attempted to load, nullptr if we failed to load, otherwise the loaded
527 /// dialect.
528 std::optional<Dialect *> dialect;
529
530 /// The bytecode interface of the dialect, or nullptr if the dialect does not
531 /// implement the bytecode interface. This field should only be checked if the
532 /// `dialect` field is not std::nullopt.
533 const BytecodeDialectInterface *interface = nullptr;
534
535 /// The name of the dialect.
536 StringRef name;
537
538 /// A buffer containing the encoding of the dialect version parsed.
539 ArrayRef<uint8_t> versionBuffer;
540
541 /// Lazy loaded dialect version from the handle above.
542 std::unique_ptr<DialectVersion> loadedVersion;
543};
544
545/// This struct represents an operation name entry within the bytecode.
546struct BytecodeOperationName {
547 BytecodeOperationName(BytecodeDialect *dialect, StringRef name,
548 std::optional<bool> wasRegistered)
549 : dialect(dialect), name(name), wasRegistered(wasRegistered) {}
550
551 /// The loaded operation name, or std::nullopt if it hasn't been processed
552 /// yet.
553 std::optional<OperationName> opName;
554
555 /// The dialect that owns this operation name.
556 BytecodeDialect *dialect;
557
558 /// The name of the operation, without the dialect prefix.
559 StringRef name;
560
561 /// Whether this operation was registered when the bytecode was produced.
562 /// This flag is populated when bytecode version >=kNativePropertiesEncoding.
563 std::optional<bool> wasRegistered;
564};
565} // namespace
566
567/// Parse a single dialect group encoded in the byte stream.
568static LogicalResult parseDialectGrouping(
569 EncodingReader &reader,
570 MutableArrayRef<std::unique_ptr<BytecodeDialect>> dialects,
571 function_ref<LogicalResult(BytecodeDialect *)> entryCallback) {
572 // Parse the dialect and the number of entries in the group.
573 std::unique_ptr<BytecodeDialect> *dialect;
574 if (failed(Result: parseEntry(reader, entries&: dialects, entry&: dialect, entryStr: "dialect")))
575 return failure();
576 uint64_t numEntries;
577 if (failed(Result: reader.parseVarInt(result&: numEntries)))
578 return failure();
579
580 for (uint64_t i = 0; i < numEntries; ++i)
581 if (failed(Result: entryCallback(dialect->get())))
582 return failure();
583 return success();
584}
585
586//===----------------------------------------------------------------------===//
587// ResourceSectionReader
588//===----------------------------------------------------------------------===//
589
590namespace {
591/// This class is used to read the resource section from the bytecode.
592class ResourceSectionReader {
593public:
594 /// Initialize the resource section reader with the given section data.
595 LogicalResult
596 initialize(Location fileLoc, const ParserConfig &config,
597 MutableArrayRef<std::unique_ptr<BytecodeDialect>> dialects,
598 StringSectionReader &stringReader, ArrayRef<uint8_t> sectionData,
599 ArrayRef<uint8_t> offsetSectionData, DialectReader &dialectReader,
600 const std::shared_ptr<llvm::SourceMgr> &bufferOwnerRef);
601
602 /// Parse a dialect resource handle from the resource section.
603 LogicalResult parseResourceHandle(EncodingReader &reader,
604 AsmDialectResourceHandle &result) const {
605 return parseEntry(reader, entries: dialectResources, entry&: result, entryStr: "resource handle");
606 }
607
608private:
609 /// The table of dialect resources within the bytecode file.
610 SmallVector<AsmDialectResourceHandle> dialectResources;
611 llvm::StringMap<std::string> dialectResourceHandleRenamingMap;
612};
613
614class ParsedResourceEntry : public AsmParsedResourceEntry {
615public:
616 ParsedResourceEntry(StringRef key, AsmResourceEntryKind kind,
617 EncodingReader &reader, StringSectionReader &stringReader,
618 const std::shared_ptr<llvm::SourceMgr> &bufferOwnerRef)
619 : key(key), kind(kind), reader(reader), stringReader(stringReader),
620 bufferOwnerRef(bufferOwnerRef) {}
621 ~ParsedResourceEntry() override = default;
622
623 StringRef getKey() const final { return key; }
624
625 InFlightDiagnostic emitError() const final { return reader.emitError(); }
626
627 AsmResourceEntryKind getKind() const final { return kind; }
628
629 FailureOr<bool> parseAsBool() const final {
630 if (kind != AsmResourceEntryKind::Bool)
631 return emitError() << "expected a bool resource entry, but found a "
632 << toString(kind) << " entry instead";
633
634 bool value;
635 if (failed(Result: reader.parseByte(value)))
636 return failure();
637 return value;
638 }
639 FailureOr<std::string> parseAsString() const final {
640 if (kind != AsmResourceEntryKind::String)
641 return emitError() << "expected a string resource entry, but found a "
642 << toString(kind) << " entry instead";
643
644 StringRef string;
645 if (failed(Result: stringReader.parseString(reader, result&: string)))
646 return failure();
647 return string.str();
648 }
649
650 FailureOr<AsmResourceBlob>
651 parseAsBlob(BlobAllocatorFn allocator) const final {
652 if (kind != AsmResourceEntryKind::Blob)
653 return emitError() << "expected a blob resource entry, but found a "
654 << toString(kind) << " entry instead";
655
656 ArrayRef<uint8_t> data;
657 uint64_t alignment;
658 if (failed(Result: reader.parseBlobAndAlignment(data, alignment)))
659 return failure();
660
661 // If we have an extendable reference to the buffer owner, we don't need to
662 // allocate a new buffer for the data, and can use the data directly.
663 if (bufferOwnerRef) {
664 ArrayRef<char> charData(reinterpret_cast<const char *>(data.data()),
665 data.size());
666
667 // Allocate an unmanager buffer which captures a reference to the owner.
668 // For now we just mark this as immutable, but in the future we should
669 // explore marking this as mutable when desired.
670 return UnmanagedAsmResourceBlob::allocateWithAlign(
671 data: charData, align: alignment,
672 deleter: [bufferOwnerRef = bufferOwnerRef](void *, size_t, size_t) {});
673 }
674
675 // Allocate memory for the blob using the provided allocator and copy the
676 // data into it.
677 AsmResourceBlob blob = allocator(data.size(), alignment);
678 assert(llvm::isAddrAligned(llvm::Align(alignment), blob.getData().data()) &&
679 blob.isMutable() &&
680 "blob allocator did not return a properly aligned address");
681 memcpy(dest: blob.getMutableData().data(), src: data.data(), n: data.size());
682 return blob;
683 }
684
685private:
686 StringRef key;
687 AsmResourceEntryKind kind;
688 EncodingReader &reader;
689 StringSectionReader &stringReader;
690 const std::shared_ptr<llvm::SourceMgr> &bufferOwnerRef;
691};
692} // namespace
693
694template <typename T>
695static LogicalResult
696parseResourceGroup(Location fileLoc, bool allowEmpty,
697 EncodingReader &offsetReader, EncodingReader &resourceReader,
698 StringSectionReader &stringReader, T *handler,
699 const std::shared_ptr<llvm::SourceMgr> &bufferOwnerRef,
700 function_ref<StringRef(StringRef)> remapKey = {},
701 function_ref<LogicalResult(StringRef)> processKeyFn = {}) {
702 uint64_t numResources;
703 if (failed(Result: offsetReader.parseVarInt(result&: numResources)))
704 return failure();
705
706 for (uint64_t i = 0; i < numResources; ++i) {
707 StringRef key;
708 AsmResourceEntryKind kind;
709 uint64_t resourceOffset;
710 ArrayRef<uint8_t> data;
711 if (failed(Result: stringReader.parseString(reader&: offsetReader, result&: key)) ||
712 failed(Result: offsetReader.parseVarInt(result&: resourceOffset)) ||
713 failed(Result: offsetReader.parseByte(value&: kind)) ||
714 failed(Result: resourceReader.parseBytes(length: resourceOffset, result&: data)))
715 return failure();
716
717 // Process the resource key.
718 if ((processKeyFn && failed(Result: processKeyFn(key))))
719 return failure();
720
721 // If the resource data is empty and we allow it, don't error out when
722 // parsing below, just skip it.
723 if (allowEmpty && data.empty())
724 continue;
725
726 // Ignore the entry if we don't have a valid handler.
727 if (!handler)
728 continue;
729
730 // Otherwise, parse the resource value.
731 EncodingReader entryReader(data, fileLoc);
732 key = remapKey(key);
733 ParsedResourceEntry entry(key, kind, entryReader, stringReader,
734 bufferOwnerRef);
735 if (failed(handler->parseResource(entry)))
736 return failure();
737 if (!entryReader.empty()) {
738 return entryReader.emitError(
739 args: "unexpected trailing bytes in resource entry '", args&: key, args: "'");
740 }
741 }
742 return success();
743}
744
745LogicalResult ResourceSectionReader::initialize(
746 Location fileLoc, const ParserConfig &config,
747 MutableArrayRef<std::unique_ptr<BytecodeDialect>> dialects,
748 StringSectionReader &stringReader, ArrayRef<uint8_t> sectionData,
749 ArrayRef<uint8_t> offsetSectionData, DialectReader &dialectReader,
750 const std::shared_ptr<llvm::SourceMgr> &bufferOwnerRef) {
751 EncodingReader resourceReader(sectionData, fileLoc);
752 EncodingReader offsetReader(offsetSectionData, fileLoc);
753
754 // Read the number of external resource providers.
755 uint64_t numExternalResourceGroups;
756 if (failed(Result: offsetReader.parseVarInt(result&: numExternalResourceGroups)))
757 return failure();
758
759 // Utility functor that dispatches to `parseResourceGroup`, but implicitly
760 // provides most of the arguments.
761 auto parseGroup = [&](auto *handler, bool allowEmpty = false,
762 function_ref<LogicalResult(StringRef)> keyFn = {}) {
763 auto resolveKey = [&](StringRef key) -> StringRef {
764 auto it = dialectResourceHandleRenamingMap.find(Key: key);
765 if (it == dialectResourceHandleRenamingMap.end())
766 return key;
767 return it->second;
768 };
769
770 return parseResourceGroup(fileLoc, allowEmpty, offsetReader, resourceReader,
771 stringReader, handler, bufferOwnerRef, resolveKey,
772 keyFn);
773 };
774
775 // Read the external resources from the bytecode.
776 for (uint64_t i = 0; i < numExternalResourceGroups; ++i) {
777 StringRef key;
778 if (failed(Result: stringReader.parseString(reader&: offsetReader, result&: key)))
779 return failure();
780
781 // Get the handler for these resources.
782 // TODO: Should we require handling external resources in some scenarios?
783 AsmResourceParser *handler = config.getResourceParser(name: key);
784 if (!handler) {
785 emitWarning(loc: fileLoc) << "ignoring unknown external resources for '" << key
786 << "'";
787 }
788
789 if (failed(Result: parseGroup(handler)))
790 return failure();
791 }
792
793 // Read the dialect resources from the bytecode.
794 MLIRContext *ctx = fileLoc->getContext();
795 while (!offsetReader.empty()) {
796 std::unique_ptr<BytecodeDialect> *dialect;
797 if (failed(Result: parseEntry(reader&: offsetReader, entries&: dialects, entry&: dialect, entryStr: "dialect")) ||
798 failed(Result: (*dialect)->load(reader: dialectReader, ctx)))
799 return failure();
800 Dialect *loadedDialect = (*dialect)->getLoadedDialect();
801 if (!loadedDialect) {
802 return resourceReader.emitError()
803 << "dialect '" << (*dialect)->name << "' is unknown";
804 }
805 const auto *handler = dyn_cast<OpAsmDialectInterface>(Val: loadedDialect);
806 if (!handler) {
807 return resourceReader.emitError()
808 << "unexpected resources for dialect '" << (*dialect)->name << "'";
809 }
810
811 // Ensure that each resource is declared before being processed.
812 auto processResourceKeyFn = [&](StringRef key) -> LogicalResult {
813 FailureOr<AsmDialectResourceHandle> handle =
814 handler->declareResource(key);
815 if (failed(Result: handle)) {
816 return resourceReader.emitError()
817 << "unknown 'resource' key '" << key << "' for dialect '"
818 << (*dialect)->name << "'";
819 }
820 dialectResourceHandleRenamingMap[key] = handler->getResourceKey(handle: *handle);
821 dialectResources.push_back(Elt: *handle);
822 return success();
823 };
824
825 // Parse the resources for this dialect. We allow empty resources because we
826 // just treat these as declarations.
827 if (failed(Result: parseGroup(handler, /*allowEmpty=*/true, processResourceKeyFn)))
828 return failure();
829 }
830
831 return success();
832}
833
834//===----------------------------------------------------------------------===//
835// Attribute/Type Reader
836//===----------------------------------------------------------------------===//
837
838namespace {
839/// This class provides support for reading attribute and type entries from the
840/// bytecode. Attribute and Type entries are read lazily on demand, so we use
841/// this reader to manage when to actually parse them from the bytecode.
842///
843/// The parsing of attributes & types are generally recursive, this can lead to
844/// stack overflows for deeply nested structures, so we track a few extra pieces
845/// of information to avoid this:
846///
847/// - `depth`: The current depth while parsing nested attributes. We defer on
848/// parsing deeply nested attributes to avoid potential stack overflows. The
849/// deferred parsing is achieved by reporting a failure when parsing a nested
850/// attribute/type and registering the index of the encountered attribute/type
851/// in the deferred parsing worklist. Hence, a failure with deffered entry
852/// does not constitute a failure, it also requires that folks return on
853/// first failure rather than attempting additional parses.
854/// - `deferredWorklist`: A list of attribute/type indices that we could not
855/// parse due to hitting the depth limit. The worklist is used to capture the
856/// indices of attributes/types that need to be parsed/reparsed when we hit
857/// the depth limit. This enables moving the tracking of what needs to be
858/// parsed to the heap.
859class AttrTypeReader {
860 /// This class represents a single attribute or type entry.
861 template <typename T>
862 struct Entry {
863 /// The entry, or null if it hasn't been resolved yet.
864 T entry = {};
865 /// The parent dialect of this entry.
866 BytecodeDialect *dialect = nullptr;
867 /// A flag indicating if the entry was encoded using a custom encoding,
868 /// instead of using the textual assembly format.
869 bool hasCustomEncoding = false;
870 /// The raw data of this entry in the bytecode.
871 ArrayRef<uint8_t> data;
872 };
873 using AttrEntry = Entry<Attribute>;
874 using TypeEntry = Entry<Type>;
875
876public:
877 AttrTypeReader(const StringSectionReader &stringReader,
878 const ResourceSectionReader &resourceReader,
879 const llvm::StringMap<BytecodeDialect *> &dialectsMap,
880 uint64_t &bytecodeVersion, Location fileLoc,
881 const ParserConfig &config)
882 : stringReader(stringReader), resourceReader(resourceReader),
883 dialectsMap(dialectsMap), fileLoc(fileLoc),
884 bytecodeVersion(bytecodeVersion), parserConfig(config) {}
885
886 /// Initialize the attribute and type information within the reader.
887 LogicalResult
888 initialize(MutableArrayRef<std::unique_ptr<BytecodeDialect>> dialects,
889 ArrayRef<uint8_t> sectionData,
890 ArrayRef<uint8_t> offsetSectionData);
891
892 LogicalResult readAttribute(uint64_t index, Attribute &result,
893 uint64_t depth = 0) {
894 return readEntry(entries&: attributes, index, result, entryType: "attribute", depth);
895 }
896
897 /// `allowPartial` controls whether an in-flight partial (published by a
898 /// mutable type to break a cycle) may satisfy this read. Nested reads want
899 /// that; the worklist does not, because a partial is not a resolved entry
900 /// and the entry still has to be parsed to completion.
901 LogicalResult readType(uint64_t index, Type &result, uint64_t depth = 0,
902 bool allowPartial = true) {
903 return readEntry(entries&: types, index, result, entryType: "type", depth, allowPartial);
904 }
905
906 /// Resolve the attribute or type at the given index. Returns nullptr on
907 /// failure.
908 Attribute resolveAttribute(size_t index, uint64_t depth = 0) {
909 return resolveEntry(entries&: attributes, index, entryType: "Attribute", depth);
910 }
911 Type resolveType(size_t index, uint64_t depth = 0) {
912 return resolveEntry(entries&: types, index, entryType: "Type", depth);
913 }
914
915 Attribute getAttributeOrSentinel(size_t index) {
916 if (index >= attributes.size())
917 return nullptr;
918 return attributes[index].entry;
919 }
920 Type getTypeOrSentinel(size_t index) {
921 if (index >= types.size())
922 return nullptr;
923 return types[index].entry;
924 }
925
926 /// Parse a reference to an attribute or type using the given reader.
927 LogicalResult parseAttribute(EncodingReader &reader, Attribute &result) {
928 uint64_t attrIdx;
929 if (failed(Result: reader.parseVarInt(result&: attrIdx)))
930 return failure();
931 result = resolveAttribute(index: attrIdx);
932 return success(IsSuccess: !!result);
933 }
934 LogicalResult parseOptionalAttribute(EncodingReader &reader,
935 Attribute &result) {
936 uint64_t attrIdx;
937 bool flag;
938 if (failed(Result: reader.parseVarIntWithFlag(result&: attrIdx, flag)))
939 return failure();
940 if (!flag)
941 return success();
942 result = resolveAttribute(index: attrIdx);
943 return success(IsSuccess: !!result);
944 }
945
946 LogicalResult parseType(EncodingReader &reader, Type &result) {
947 uint64_t typeIdx;
948 if (failed(Result: reader.parseVarInt(result&: typeIdx)))
949 return failure();
950 result = resolveType(index: typeIdx);
951 return success(IsSuccess: !!result);
952 }
953
954 template <typename T>
955 LogicalResult parseAttribute(EncodingReader &reader, T &result) {
956 Attribute baseResult;
957 if (failed(Result: parseAttribute(reader, result&: baseResult)))
958 return failure();
959 if ((result = dyn_cast<T>(baseResult)))
960 return success();
961 return reader.emitError("expected attribute of type: ",
962 llvm::getTypeName<T>(), ", but got: ", baseResult);
963 }
964
965 /// The kind of entry being parsed.
966 enum class EntryKind { Attribute, Type };
967
968 /// Add an index to the deferred worklist for re-parsing.
969 void addDeferredParsing(uint64_t index, EntryKind kind) {
970 deferredWorklist.emplace_back(args&: index, args&: kind);
971 }
972
973 /// Whether currently resolving.
974 bool isResolving() const { return resolving; }
975
976 /// Register `type` as the partial result of the type entry at `index`. The
977 /// partial stays visible to nested reads until the top-level resolve ends.
978 /// Re-publishing the same instance (deferred entries are retried, which
979 /// re-runs the dialect) succeeds; a different instance fails, since
980 /// consumers may already hold the first one. See
981 /// DialectBytecodeReader::tryStartCyclicRead.
982 LogicalResult setPartialType(uint64_t index, Type type) {
983 Type &slot = partialTypes[index];
984 if (slot && slot != type)
985 return failure();
986 slot = type;
987 return success();
988 }
989
990 /// A partial may answer any nested read. That is safe precisely because the
991 /// type is mutable with stable identity: completing it mutates the same
992 /// instance the consumer is holding, so a consumer that took the partial
993 /// ends up with the finished type. What must not happen is letting the
994 /// entry stay unparsed; resolveEntry completes every published partial.
995 Type getPartialType(uint64_t index) const {
996 return partialTypes.lookup(Key: index);
997 }
998
999 /// Entries that published a partial, in publish order. Entries still
1000 /// unparsed at the end of the slow path are seeded into the worklist.
1001 const llvm::MapVector<uint64_t, Type> &getPartialTypes() const {
1002 return partialTypes;
1003 }
1004
1005private:
1006 /// Resolve the given entry at `index`.
1007 template <typename T>
1008 T resolveEntry(SmallVectorImpl<Entry<T>> &entries, uint64_t index,
1009 StringRef entryType, uint64_t depth = 0);
1010
1011 /// Read the entry at the given index, returning failure if the entry is not
1012 /// yet resolved.
1013 template <typename T>
1014 LogicalResult readEntry(SmallVectorImpl<Entry<T>> &entries, uint64_t index,
1015 T &result, StringRef entryType, uint64_t depth,
1016 bool allowPartial = true);
1017
1018 /// Parse an entry using the given reader that was encoded using a custom
1019 /// bytecode format.
1020 template <typename T>
1021 LogicalResult parseCustomEntry(Entry<T> &entry, EncodingReader &reader,
1022 StringRef entryType, uint64_t index,
1023 uint64_t depth);
1024
1025 /// Drain the worklist, resolving entries and retrying deferred ones after
1026 /// their dependencies. Shared by the slow path and by the completion of
1027 /// entries that were only answered with a partial.
1028 LogicalResult
1029 drainWorklist(std::deque<std::pair<uint64_t, EntryKind>> &worklist,
1030 llvm::DenseSet<std::pair<uint64_t, EntryKind>> &inWorklist,
1031 uint64_t depth);
1032
1033 /// Parse an entry using the given reader that was encoded using the textual
1034 /// assembly format.
1035 template <typename T>
1036 LogicalResult parseAsmEntry(T &result, EncodingReader &reader,
1037 StringRef entryType);
1038
1039 /// The string section reader used to resolve string references when parsing
1040 /// custom encoded attribute/type entries.
1041 const StringSectionReader &stringReader;
1042
1043 /// The resource section reader used to resolve resource references when
1044 /// parsing custom encoded attribute/type entries.
1045 const ResourceSectionReader &resourceReader;
1046
1047 /// The map of the loaded dialects used to retrieve dialect information, such
1048 /// as the dialect version.
1049 const llvm::StringMap<BytecodeDialect *> &dialectsMap;
1050
1051 /// The set of attribute and type entries.
1052 SmallVector<AttrEntry> attributes;
1053 SmallVector<TypeEntry> types;
1054
1055 /// Type entries that have published an incomplete value so a self-reference
1056 /// can resolve. A partial is honoured by nested reads until the top-level
1057 /// resolve ends; every published entry is completed before that.
1058 llvm::MapVector<uint64_t, Type> partialTypes;
1059
1060 /// A location used for error emission.
1061 Location fileLoc;
1062
1063 /// Current bytecode version being used.
1064 uint64_t &bytecodeVersion;
1065
1066 /// Reference to the parser configuration.
1067 const ParserConfig &parserConfig;
1068
1069 /// Worklist for deferred attribute/type parsing. This is used to handle
1070 /// deeply nested structures like CallSiteLoc iteratively.
1071 /// - The first element is the index of the attribute/type to parse.
1072 /// - The second element is the kind of entry being parsed.
1073 std::vector<std::pair<uint64_t, EntryKind>> deferredWorklist;
1074
1075 /// Flag indicating if we are currently resolving an attribute or type.
1076 bool resolving = false;
1077};
1078
1079class DialectReader : public DialectBytecodeReader {
1080public:
1081 DialectReader(AttrTypeReader &attrTypeReader,
1082 const StringSectionReader &stringReader,
1083 const ResourceSectionReader &resourceReader,
1084 const llvm::StringMap<BytecodeDialect *> &dialectsMap,
1085 EncodingReader &reader, uint64_t &bytecodeVersion,
1086 uint64_t depth = 0,
1087 std::optional<uint64_t> cyclicTypeIndex = std::nullopt)
1088 : attrTypeReader(attrTypeReader), stringReader(stringReader),
1089 resourceReader(resourceReader), dialectsMap(dialectsMap),
1090 reader(reader), bytecodeVersion(bytecodeVersion), depth(depth),
1091 cyclicTypeIndex(cyclicTypeIndex) {}
1092
1093 /// See DialectBytecodeReader::tryStartCyclicRead.
1094 ///
1095 /// The partial is registered against the entry index, deliberately *not*
1096 /// stored into the entry itself. An attempt can be abandoned and retried
1097 /// by the deferred worklist, and an incomplete type left behind in the
1098 /// entry would be handed out as if it were fully resolved.
1099 LogicalResult tryStartCyclicRead(Type type) override;
1100
1101 InFlightDiagnostic emitError(const Twine &msg) const override {
1102 return reader.emitError(args: msg);
1103 }
1104
1105 InFlightDiagnostic emitWarning(const Twine &msg) const override {
1106 return reader.emitWarning(args: msg);
1107 }
1108
1109 FailureOr<const DialectVersion *>
1110 getDialectVersion(StringRef dialectName) const override {
1111 // First check if the dialect is available in the map.
1112 auto dialectEntry = dialectsMap.find(Key: dialectName);
1113 if (dialectEntry == dialectsMap.end())
1114 return failure();
1115 // If the dialect was found, try to load it. This will trigger reading the
1116 // bytecode version from the version buffer if it wasn't already processed.
1117 // Return failure if either of those two actions could not be completed.
1118 if (failed(Result: dialectEntry->getValue()->load(reader: *this, ctx: getLoc().getContext())) ||
1119 dialectEntry->getValue()->loadedVersion == nullptr)
1120 return failure();
1121 return dialectEntry->getValue()->loadedVersion.get();
1122 }
1123
1124 MLIRContext *getContext() const override { return getLoc().getContext(); }
1125
1126 uint64_t getBytecodeVersion() const override { return bytecodeVersion; }
1127
1128 DialectReader withEncodingReader(EncodingReader &encReader) const {
1129 return DialectReader(attrTypeReader, stringReader, resourceReader,
1130 dialectsMap, encReader, bytecodeVersion);
1131 }
1132
1133 Location getLoc() const { return reader.getLoc(); }
1134
1135 //===--------------------------------------------------------------------===//
1136 // IR
1137 //===--------------------------------------------------------------------===//
1138
1139 /// The maximum depth to eagerly parse nested attributes/types before
1140 /// deferring.
1141 static constexpr uint64_t maxAttrTypeDepth = 5;
1142
1143 LogicalResult readAttribute(Attribute &result) override {
1144 uint64_t index;
1145 if (failed(Result: reader.parseVarInt(result&: index)))
1146 return failure();
1147
1148 // If we aren't currently resolving an attribute/type, we resolve this
1149 // attribute eagerly. This is the case when we are parsing properties, which
1150 // aren't processed via the worklist.
1151 if (!attrTypeReader.isResolving()) {
1152 if (Attribute attr = attrTypeReader.resolveAttribute(index)) {
1153 result = attr;
1154 return success();
1155 }
1156 return failure();
1157 }
1158
1159 if (depth > maxAttrTypeDepth) {
1160 if (Attribute attr = attrTypeReader.getAttributeOrSentinel(index)) {
1161 result = attr;
1162 return success();
1163 }
1164 attrTypeReader.addDeferredParsing(index,
1165 kind: AttrTypeReader::EntryKind::Attribute);
1166 return failure();
1167 }
1168 return attrTypeReader.readAttribute(index, result, depth: depth + 1);
1169 }
1170 LogicalResult readOptionalAttribute(Attribute &result) override {
1171 return attrTypeReader.parseOptionalAttribute(reader, result);
1172 }
1173 LogicalResult readType(Type &result) override {
1174 uint64_t index;
1175 if (failed(Result: reader.parseVarInt(result&: index)))
1176 return failure();
1177
1178 // If we aren't currently resolving an attribute/type, we resolve this
1179 // type eagerly. This is the case when we are parsing properties, which
1180 // aren't processed via the worklist.
1181 if (!attrTypeReader.isResolving()) {
1182 if (Type type = attrTypeReader.resolveType(index)) {
1183 result = type;
1184 return success();
1185 }
1186 return failure();
1187 }
1188
1189 if (depth > maxAttrTypeDepth) {
1190 if (Type type = attrTypeReader.getTypeOrSentinel(index)) {
1191 result = type;
1192 return success();
1193 }
1194 if (Type partial = attrTypeReader.getPartialType(index)) {
1195 result = partial;
1196 return success();
1197 }
1198 attrTypeReader.addDeferredParsing(index, kind: AttrTypeReader::EntryKind::Type);
1199 return failure();
1200 }
1201 return attrTypeReader.readType(index, result, depth: depth + 1);
1202 }
1203
1204 FailureOr<AsmDialectResourceHandle> readResourceHandle() override {
1205 AsmDialectResourceHandle handle;
1206 if (failed(Result: resourceReader.parseResourceHandle(reader, result&: handle)))
1207 return failure();
1208 return handle;
1209 }
1210
1211 //===--------------------------------------------------------------------===//
1212 // Primitives
1213 //===--------------------------------------------------------------------===//
1214
1215 LogicalResult readVarInt(uint64_t &result) override {
1216 return reader.parseVarInt(result);
1217 }
1218
1219 LogicalResult readSignedVarInt(int64_t &result) override {
1220 uint64_t unsignedResult;
1221 if (failed(Result: reader.parseSignedVarInt(result&: unsignedResult)))
1222 return failure();
1223 result = static_cast<int64_t>(unsignedResult);
1224 return success();
1225 }
1226
1227 FailureOr<APInt> readAPIntWithKnownWidth(unsigned bitWidth) override {
1228 // Small values are encoded using a single byte.
1229 if (bitWidth <= 8) {
1230 uint8_t value;
1231 if (failed(Result: reader.parseByte(value)))
1232 return failure();
1233 return APInt(bitWidth, value);
1234 }
1235
1236 // Large values up to 64 bits are encoded using a single varint.
1237 if (bitWidth <= 64) {
1238 uint64_t value;
1239 if (failed(Result: reader.parseSignedVarInt(result&: value)))
1240 return failure();
1241 return APInt(bitWidth, value);
1242 }
1243
1244 // Otherwise, for really big values we encode the array of active words in
1245 // the value.
1246 uint64_t numActiveWords;
1247 if (failed(Result: reader.parseVarInt(result&: numActiveWords)))
1248 return failure();
1249 SmallVector<uint64_t, 4> words(numActiveWords);
1250 for (uint64_t i = 0; i < numActiveWords; ++i)
1251 if (failed(Result: reader.parseSignedVarInt(result&: words[i])))
1252 return failure();
1253 return APInt(bitWidth, words);
1254 }
1255
1256 FailureOr<APFloat>
1257 readAPFloatWithKnownSemantics(const llvm::fltSemantics &semantics) override {
1258 FailureOr<APInt> intVal =
1259 readAPIntWithKnownWidth(bitWidth: APFloat::getSizeInBits(Sem: semantics));
1260 if (failed(Result: intVal))
1261 return failure();
1262 return APFloat(semantics, *intVal);
1263 }
1264
1265 LogicalResult readString(StringRef &result) override {
1266 return stringReader.parseString(reader, result);
1267 }
1268
1269 LogicalResult readBlob(ArrayRef<char> &result) override {
1270 uint64_t dataSize;
1271 ArrayRef<uint8_t> data;
1272 if (failed(Result: reader.parseVarInt(result&: dataSize)) ||
1273 failed(Result: reader.parseBytes(length: dataSize, result&: data)))
1274 return failure();
1275 result = llvm::ArrayRef(reinterpret_cast<const char *>(data.data()),
1276 data.size());
1277 return success();
1278 }
1279
1280 LogicalResult readBool(bool &result) override {
1281 return reader.parseByte(value&: result);
1282 }
1283
1284private:
1285 AttrTypeReader &attrTypeReader;
1286 const StringSectionReader &stringReader;
1287 const ResourceSectionReader &resourceReader;
1288 const llvm::StringMap<BytecodeDialect *> &dialectsMap;
1289 EncodingReader &reader;
1290 uint64_t &bytecodeVersion;
1291 uint64_t depth;
1292
1293 /// Index of the type entry being parsed, when this reader is parsing one.
1294 /// Only then can a cyclic read be published.
1295 std::optional<uint64_t> cyclicTypeIndex;
1296};
1297
1298/// Wraps the properties section and handles reading properties out of it.
1299class PropertiesSectionReader {
1300public:
1301 /// Initialize the properties section reader with the given section data.
1302 LogicalResult initialize(Location fileLoc, ArrayRef<uint8_t> sectionData) {
1303 if (sectionData.empty())
1304 return success();
1305 EncodingReader propReader(sectionData, fileLoc);
1306 uint64_t count;
1307 if (failed(Result: propReader.parseVarInt(result&: count)))
1308 return failure();
1309 // Parse the raw properties buffer.
1310 if (failed(Result: propReader.parseBytes(length: propReader.size(), result&: propertiesBuffers)))
1311 return failure();
1312
1313 EncodingReader offsetsReader(propertiesBuffers, fileLoc);
1314 offsetTable.reserve(N: count);
1315 for (auto idx : llvm::seq<int64_t>(Begin: 0, End: count)) {
1316 (void)idx;
1317 offsetTable.push_back(Elt: propertiesBuffers.size() - offsetsReader.size());
1318 ArrayRef<uint8_t> rawProperties;
1319 uint64_t dataSize;
1320 if (failed(Result: offsetsReader.parseVarInt(result&: dataSize)) ||
1321 failed(Result: offsetsReader.parseBytes(length: dataSize, result&: rawProperties)))
1322 return failure();
1323 }
1324 if (!offsetsReader.empty())
1325 return offsetsReader.emitError()
1326 << "Broken properties section: didn't exhaust the offsets table";
1327 return success();
1328 }
1329
1330 LogicalResult read(Location fileLoc, DialectReader &dialectReader,
1331 OperationName *opName, OperationState &opState) const {
1332 uint64_t propertiesIdx;
1333 if (failed(Result: dialectReader.readVarInt(result&: propertiesIdx)))
1334 return failure();
1335 if (propertiesIdx >= offsetTable.size())
1336 return dialectReader.emitError(msg: "Properties idx out-of-bound for ")
1337 << opName->getStringRef();
1338 size_t propertiesOffset = offsetTable[propertiesIdx];
1339 if (propertiesIdx >= propertiesBuffers.size())
1340 return dialectReader.emitError(msg: "Properties offset out-of-bound for ")
1341 << opName->getStringRef();
1342
1343 // Acquire the sub-buffer that represent the requested properties.
1344 ArrayRef<char> rawProperties;
1345 {
1346 // "Seek" to the requested offset by getting a new reader with the right
1347 // sub-buffer.
1348 EncodingReader reader(propertiesBuffers.drop_front(N: propertiesOffset),
1349 fileLoc);
1350 // Properties are stored as a sequence of {size + raw_data}.
1351 if (failed(
1352 Result: dialectReader.withEncodingReader(encReader&: reader).readBlob(result&: rawProperties)))
1353 return failure();
1354 }
1355 // Setup a new reader to read from the `rawProperties` sub-buffer.
1356 EncodingReader reader(
1357 StringRef(rawProperties.begin(), rawProperties.size()), fileLoc);
1358 DialectReader propReader = dialectReader.withEncodingReader(encReader&: reader);
1359
1360 auto *iface = opName->getInterface<BytecodeOpInterface>();
1361 if (iface)
1362 return iface->readProperties(propReader, opState);
1363 if (opName->isRegistered())
1364 return propReader.emitError(
1365 msg: "has properties but missing BytecodeOpInterface for ")
1366 << opName->getStringRef();
1367 // Unregistered op are storing properties as an attribute.
1368 return propReader.readAttribute(result&: opState.propertiesAttr);
1369 }
1370
1371private:
1372 /// The properties buffer referenced within the bytecode file.
1373 ArrayRef<uint8_t> propertiesBuffers;
1374
1375 /// Table of offset in the buffer above.
1376 SmallVector<int64_t> offsetTable;
1377};
1378} // namespace
1379
1380LogicalResult AttrTypeReader::initialize(
1381 MutableArrayRef<std::unique_ptr<BytecodeDialect>> dialects,
1382 ArrayRef<uint8_t> sectionData, ArrayRef<uint8_t> offsetSectionData) {
1383 EncodingReader offsetReader(offsetSectionData, fileLoc);
1384
1385 // Parse the number of attribute and type entries.
1386 uint64_t numAttributes, numTypes;
1387 if (failed(Result: offsetReader.parseVarInt(result&: numAttributes)) ||
1388 failed(Result: offsetReader.parseVarInt(result&: numTypes)))
1389 return failure();
1390 attributes.resize(N: numAttributes);
1391 types.resize(N: numTypes);
1392
1393 // A functor used to accumulate the offsets for the entries in the given
1394 // range.
1395 uint64_t currentOffset = 0;
1396 auto parseEntries = [&](auto &&range) {
1397 size_t currentIndex = 0, endIndex = range.size();
1398
1399 // Parse an individual entry.
1400 auto parseEntryFn = [&](BytecodeDialect *dialect) -> LogicalResult {
1401 // Verify that the dialect groups don't contain more entries than were
1402 // declared for this range.
1403 if (currentIndex == endIndex) {
1404 return offsetReader.emitError(
1405 args: "Attribute or Type dialect group entries exceed the declared "
1406 "number of entries (",
1407 args&: endIndex, args: ")");
1408 }
1409 auto &entry = range[currentIndex++];
1410
1411 uint64_t entrySize;
1412 if (failed(offsetReader.parseVarIntWithFlag(result&: entrySize,
1413 flag&: entry.hasCustomEncoding)))
1414 return failure();
1415
1416 // Verify that the offset is actually valid.
1417 if (currentOffset + entrySize > sectionData.size()) {
1418 return offsetReader.emitError(
1419 args: "Attribute or Type entry offset points past the end of section");
1420 }
1421
1422 entry.data = sectionData.slice(N: currentOffset, M: entrySize);
1423 entry.dialect = dialect;
1424 currentOffset += entrySize;
1425 return success();
1426 };
1427 while (currentIndex != endIndex)
1428 if (failed(parseDialectGrouping(offsetReader, dialects, parseEntryFn)))
1429 return failure();
1430 return success();
1431 };
1432
1433 // Process each of the attributes, and then the types.
1434 if (failed(Result: parseEntries(attributes)) || failed(Result: parseEntries(types)))
1435 return failure();
1436
1437 // Ensure that we read everything from the section.
1438 if (!offsetReader.empty()) {
1439 return offsetReader.emitError(
1440 args: "unexpected trailing data in the Attribute/Type offset section");
1441 }
1442
1443 return success();
1444}
1445
1446template <typename T>
1447T AttrTypeReader::resolveEntry(SmallVectorImpl<Entry<T>> &entries,
1448 uint64_t index, StringRef entryType,
1449 uint64_t depth) {
1450 bool oldResolving = resolving;
1451 resolving = true;
1452 llvm::scope_exit restoreResolving([&]() {
1453 resolving = oldResolving;
1454 // Partials published to break cycles are scoped to one top-level resolve.
1455 // Past that point every entry is either fully resolved or the resolve
1456 // failed, so keeping them would risk handing out an incomplete type.
1457 if (!oldResolving)
1458 partialTypes.clear();
1459 });
1460
1461 if (index >= entries.size()) {
1462 emitError(loc: fileLoc) << "invalid " << entryType << " index: " << index;
1463 return {};
1464 }
1465
1466 // Fast path: Try direct parsing without worklist overhead. This handles the
1467 // common case where there are no deferred dependencies.
1468 assert(deferredWorklist.empty());
1469 T result;
1470 if (succeeded(readEntry(entries, index, result, entryType, depth,
1471 /*allowPartial=*/false))) {
1472 assert(deferredWorklist.empty());
1473 return result;
1474 }
1475 if (deferredWorklist.empty()) {
1476 // Failed with no deferred entries is error.
1477 return T();
1478 }
1479
1480 // Slow path: Use worklist to handle deferred dependencies. Use a deque to
1481 // iteratively resolve entries with dependencies.
1482 // - Pop from front to process
1483 // - Push new dependencies to front (depth-first)
1484 // - Move failed entries to back (retry after dependencies)
1485 std::deque<std::pair<uint64_t, EntryKind>> worklist;
1486 llvm::DenseSet<std::pair<uint64_t, EntryKind>> inWorklist;
1487
1488 EntryKind entryKind =
1489 std::is_same_v<T, Type> ? EntryKind::Type : EntryKind::Attribute;
1490
1491 static_assert((std::is_same_v<T, Type> || std::is_same_v<T, Attribute>) &&
1492 "Only support resolving Attributes and Types");
1493
1494 auto addToWorklistFront = [&](std::pair<uint64_t, EntryKind> entry) {
1495 if (inWorklist.insert(V: entry).second)
1496 worklist.push_front(x: entry);
1497 };
1498
1499 // Add the original index and any dependencies from the fast path attempt.
1500 worklist.emplace_back(args&: index, args&: entryKind);
1501 inWorklist.insert(V: {index, entryKind});
1502 for (auto entry : llvm::reverse(C&: deferredWorklist))
1503 addToWorklistFront(entry);
1504
1505 if (failed(Result: drainWorklist(worklist, inWorklist, depth)))
1506 return T();
1507
1508 // An entry that was answered with a partial has not necessarily been
1509 // parsed: a nested read inside the depth limit takes the partial and
1510 // returns without scheduling anything. Seed those entries and drain them
1511 // like the slow path, so deferrals below them are honoured too. Consumers
1512 // need no revisiting: they hold the same mutable instance that completing
1513 // mutates.
1514 for (;;) {
1515 bool seeded = false;
1516 for (const auto &partial : getPartialTypes()) {
1517 uint64_t partialIndex = partial.first;
1518 if (!types[partialIndex].entry &&
1519 inWorklist.insert(V: {partialIndex, EntryKind::Type}).second) {
1520 worklist.emplace_back(args&: partialIndex, args: EntryKind::Type);
1521 seeded = true;
1522 }
1523 }
1524 if (!seeded)
1525 break;
1526 if (failed(Result: drainWorklist(worklist, inWorklist, depth)))
1527 return T();
1528 }
1529
1530 return entries[index].entry;
1531}
1532
1533LogicalResult AttrTypeReader::drainWorklist(
1534 std::deque<std::pair<uint64_t, EntryKind>> &worklist,
1535 llvm::DenseSet<std::pair<uint64_t, EntryKind>> &inWorklist,
1536 uint64_t depth) {
1537 while (!worklist.empty()) {
1538 auto [currentIndex, entryKind] = worklist.front();
1539 worklist.pop_front();
1540
1541 // Clear the deferred worklist before parsing to capture any new entries.
1542 deferredWorklist.clear();
1543
1544 if (entryKind == EntryKind::Type) {
1545 Type result;
1546 if (succeeded(Result: readType(index: currentIndex, result, depth,
1547 /*allowPartial=*/false))) {
1548 inWorklist.erase(V: {currentIndex, entryKind});
1549 continue;
1550 }
1551 } else {
1552 assert(entryKind == EntryKind::Attribute && "Unexpected entry kind");
1553 Attribute result;
1554 if (succeeded(Result: readAttribute(index: currentIndex, result, depth))) {
1555 inWorklist.erase(V: {currentIndex, entryKind});
1556 continue;
1557 }
1558 }
1559
1560 if (deferredWorklist.empty()) {
1561 // Parsing failed with no deferred entries which implies an error.
1562 return failure();
1563 }
1564
1565 // Move this entry to the back to retry after dependencies.
1566 worklist.emplace_back(args&: currentIndex, args&: entryKind);
1567
1568 // Add dependencies to the front (in reverse so they maintain order).
1569 for (auto entry : llvm::reverse(C&: deferredWorklist)) {
1570 if (inWorklist.insert(V: entry).second)
1571 worklist.push_front(x: entry);
1572 }
1573
1574 deferredWorklist.clear();
1575 }
1576 return success();
1577}
1578
1579LogicalResult DialectReader::tryStartCyclicRead(Type type) {
1580 if (!cyclicTypeIndex || !type)
1581 return failure();
1582 return attrTypeReader.setPartialType(index: *cyclicTypeIndex, type);
1583}
1584
1585template <typename T>
1586LogicalResult AttrTypeReader::readEntry(SmallVectorImpl<Entry<T>> &entries,
1587 uint64_t index, T &result,
1588 StringRef entryType, uint64_t depth,
1589 bool allowPartial) {
1590 if (index >= entries.size())
1591 return emitError(loc: fileLoc) << "invalid " << entryType << " index: " << index;
1592
1593 // If the entry has already been resolved, return it.
1594 Entry<T> &entry = entries[index];
1595 if (entry.entry) {
1596 result = entry.entry;
1597 return success();
1598 }
1599
1600 // A mutable type that is mid-parse may have published an incomplete value so
1601 // that a reference back to it (which is how a self-referential record is
1602 // encoded) resolves instead of re-entering this entry.
1603 if constexpr (std::is_same_v<T, Type>) {
1604 if (allowPartial) {
1605 if (Type partial = getPartialType(index)) {
1606 result = partial;
1607 return success();
1608 }
1609 }
1610 }
1611
1612 // If the entry hasn't been resolved, try to parse it.
1613 EncodingReader reader(entry.data, fileLoc);
1614 LogicalResult parseResult =
1615 entry.hasCustomEncoding
1616 ? parseCustomEntry(entry, reader, entryType, index, depth)
1617 : parseAsmEntry(entry.entry, reader, entryType);
1618 if (failed(Result: parseResult))
1619 return failure();
1620
1621 if (!reader.empty())
1622 return reader.emitError(args: "unexpected trailing bytes after " + entryType +
1623 " entry");
1624
1625 result = entry.entry;
1626 return success();
1627}
1628
1629template <typename T>
1630LogicalResult AttrTypeReader::parseCustomEntry(Entry<T> &entry,
1631 EncodingReader &reader,
1632 StringRef entryType,
1633 uint64_t index, uint64_t depth) {
1634 std::optional<uint64_t> cyclicTypeIndex;
1635 if constexpr (std::is_same_v<T, Type>)
1636 cyclicTypeIndex = index;
1637 DialectReader dialectReader(*this, stringReader, resourceReader, dialectsMap,
1638 reader, bytecodeVersion, depth, cyclicTypeIndex);
1639 if (failed(entry.dialect->load(dialectReader, fileLoc.getContext())))
1640 return failure();
1641
1642 if constexpr (std::is_same_v<T, Type>) {
1643 // Try parsing with callbacks first if available.
1644 for (const auto &callback :
1645 parserConfig.getBytecodeReaderConfig().getTypeCallbacks()) {
1646 size_t savedWorklistSize = deferredWorklist.size();
1647 if (failed(
1648 callback->read(reader&: dialectReader, dialectName: entry.dialect->name, entry&: entry.entry)))
1649 return failure();
1650 // Early return if parsing was successful.
1651 if (!!entry.entry)
1652 return success();
1653
1654 // The callback fell through without consuming the encoding. Reset the
1655 // reader and restore the deferred worklist: any entries added during the
1656 // callback's partial read are stale and must not persist.
1657 deferredWorklist.resize(new_size: savedWorklistSize);
1658 reader = EncodingReader(entry.data, reader.getLoc());
1659 }
1660 } else {
1661 // Try parsing with callbacks first if available.
1662 for (const auto &callback :
1663 parserConfig.getBytecodeReaderConfig().getAttributeCallbacks()) {
1664 size_t savedWorklistSize = deferredWorklist.size();
1665 if (failed(
1666 callback->read(reader&: dialectReader, dialectName: entry.dialect->name, entry&: entry.entry)))
1667 return failure();
1668 // Early return if parsing was successful.
1669 if (!!entry.entry)
1670 return success();
1671
1672 // The callback fell through without consuming the encoding. Reset the
1673 // reader and restore the deferred worklist: any entries added during the
1674 // callback's partial read are stale and must not persist.
1675 deferredWorklist.resize(new_size: savedWorklistSize);
1676 reader = EncodingReader(entry.data, reader.getLoc());
1677 }
1678 }
1679
1680 // Ensure that the dialect implements the bytecode interface.
1681 if (!entry.dialect->interface) {
1682 return reader.emitError("dialect '", entry.dialect->name,
1683 "' does not implement the bytecode interface");
1684 }
1685
1686 if constexpr (std::is_same_v<T, Type>)
1687 entry.entry = entry.dialect->interface->readType(dialectReader);
1688 else
1689 entry.entry = entry.dialect->interface->readAttribute(dialectReader);
1690
1691 if constexpr (std::is_same_v<T, Type>) {
1692 // A dialect that published a partial must return that same instance:
1693 // consumers may already hold it. Anything else is a contract violation,
1694 // not a type that can complete in place.
1695 if (Type partial = getPartialType(index)) {
1696 if (entry.entry && entry.entry != partial) {
1697 return reader.emitError("dialect '", entry.dialect->name,
1698 "' returned a different type than the one "
1699 "published with tryStartCyclicRead");
1700 }
1701 }
1702 }
1703
1704 return success(!!entry.entry);
1705}
1706
1707template <typename T>
1708LogicalResult AttrTypeReader::parseAsmEntry(T &result, EncodingReader &reader,
1709 StringRef entryType) {
1710 StringRef asmStr;
1711 if (failed(Result: reader.parseNullTerminatedString(result&: asmStr)))
1712 return failure();
1713
1714 // Invoke the MLIR assembly parser to parse the entry text.
1715 size_t numRead = 0;
1716 MLIRContext *context = fileLoc->getContext();
1717 if constexpr (std::is_same_v<T, Type>)
1718 result =
1719 ::parseType(typeStr: asmStr, context, numRead: &numRead, /*isKnownNullTerminated=*/true);
1720 else
1721 result = ::parseAttribute(attrStr: asmStr, context, type: Type(), numRead: &numRead,
1722 /*isKnownNullTerminated=*/true);
1723 if (!result)
1724 return failure();
1725
1726 // Ensure there weren't dangling characters after the entry.
1727 if (numRead != asmStr.size()) {
1728 return reader.emitError(args: "trailing characters found after ", args&: entryType,
1729 args: " assembly format: ", args: asmStr.drop_front(N: numRead));
1730 }
1731 return success();
1732}
1733
1734//===----------------------------------------------------------------------===//
1735// Bytecode Reader
1736//===----------------------------------------------------------------------===//
1737
1738/// This class is used to read a bytecode buffer and translate it into MLIR.
1739class mlir::BytecodeReader::Impl {
1740 struct RegionReadState;
1741 using LazyLoadableOpsInfo =
1742 std::list<std::pair<Operation *, RegionReadState>>;
1743 using LazyLoadableOpsMap =
1744 DenseMap<Operation *, LazyLoadableOpsInfo::iterator>;
1745
1746public:
1747 Impl(Location fileLoc, const ParserConfig &config, bool lazyLoading,
1748 llvm::MemoryBufferRef buffer,
1749 const std::shared_ptr<llvm::SourceMgr> &bufferOwnerRef)
1750 : config(config), fileLoc(fileLoc), lazyLoading(lazyLoading),
1751 attrTypeReader(stringReader, resourceReader, dialectsMap, version,
1752 fileLoc, config),
1753 // Use the builtin unrealized conversion cast operation to represent
1754 // forward references to values that aren't yet defined.
1755 forwardRefOpState(UnknownLoc::get(context: config.getContext()),
1756 "builtin.unrealized_conversion_cast", ValueRange(),
1757 NoneType::get(context: config.getContext())),
1758 buffer(buffer), bufferOwnerRef(bufferOwnerRef) {}
1759
1760 /// Read the bytecode defined within `buffer` into the given block.
1761 LogicalResult read(Block *block,
1762 llvm::function_ref<bool(Operation *)> lazyOps);
1763
1764 /// Return the number of ops that haven't been materialized yet.
1765 int64_t getNumOpsToMaterialize() const { return lazyLoadableOpsMap.size(); }
1766
1767 bool isMaterializable(Operation *op) { return lazyLoadableOpsMap.count(Val: op); }
1768
1769 /// Materialize the provided operation, invoke the lazyOpsCallback on every
1770 /// newly found lazy operation.
1771 LogicalResult
1772 materialize(Operation *op,
1773 llvm::function_ref<bool(Operation *)> lazyOpsCallback) {
1774 this->lazyOpsCallback = lazyOpsCallback;
1775 llvm::scope_exit resetlazyOpsCallback(
1776 [&] { this->lazyOpsCallback = nullptr; });
1777 auto it = lazyLoadableOpsMap.find(Val: op);
1778 assert(it != lazyLoadableOpsMap.end() &&
1779 "materialize called on non-materializable op");
1780 return materialize(it);
1781 }
1782
1783 /// Materialize all operations.
1784 LogicalResult materializeAll() {
1785 while (!lazyLoadableOpsMap.empty()) {
1786 if (failed(Result: materialize(it: lazyLoadableOpsMap.begin())))
1787 return failure();
1788 }
1789 return success();
1790 }
1791
1792 /// Finalize the lazy-loading by calling back with every op that hasn't been
1793 /// materialized to let the client decide if the op should be deleted or
1794 /// materialized. The op is materialized if the callback returns true, deleted
1795 /// otherwise.
1796 LogicalResult finalize(function_ref<bool(Operation *)> shouldMaterialize) {
1797 while (!lazyLoadableOps.empty()) {
1798 Operation *op = lazyLoadableOps.begin()->first;
1799 if (shouldMaterialize(op)) {
1800 if (failed(Result: materialize(it: lazyLoadableOpsMap.find(Val: op))))
1801 return failure();
1802 continue;
1803 }
1804 op->dropAllReferences();
1805 op->erase();
1806 lazyLoadableOps.pop_front();
1807 lazyLoadableOpsMap.erase(Val: op);
1808 }
1809 return success();
1810 }
1811
1812private:
1813 LogicalResult materialize(LazyLoadableOpsMap::iterator it) {
1814 assert(it != lazyLoadableOpsMap.end() &&
1815 "materialize called on non-materializable op");
1816 valueScopes.emplace_back();
1817 std::vector<RegionReadState> regionStack;
1818 regionStack.push_back(x: std::move(it->getSecond()->second));
1819 lazyLoadableOps.erase(position: it->getSecond());
1820 lazyLoadableOpsMap.erase(I: it);
1821
1822 while (!regionStack.empty())
1823 if (failed(Result: parseRegions(regionStack, readState&: regionStack.back())))
1824 return failure();
1825 return success();
1826 }
1827
1828 LogicalResult checkSectionAlignment(
1829 unsigned alignment,
1830 function_ref<InFlightDiagnostic(const Twine &error)> emitError) {
1831 // Check that the bytecode buffer meets the requested section alignment.
1832 //
1833 // If it does not, the virtual address of the item in the section will
1834 // not be aligned to the requested alignment.
1835 //
1836 // The typical case where this is necessary is the resource blob
1837 // optimization in `parseAsBlob` where we reference the weights from the
1838 // provided buffer instead of copying them to a new allocation.
1839 const bool isGloballyAligned =
1840 ((uintptr_t)buffer.getBufferStart() & (alignment - 1)) == 0;
1841
1842 if (!isGloballyAligned)
1843 return emitError("expected section alignment ")
1844 << alignment << " but bytecode buffer 0x"
1845 << Twine::utohexstr(Val: (uint64_t)buffer.getBufferStart())
1846 << " is not aligned";
1847
1848 return success();
1849 };
1850
1851 /// Return the context for this config.
1852 MLIRContext *getContext() const { return config.getContext(); }
1853
1854 /// Parse the bytecode version.
1855 LogicalResult parseVersion(EncodingReader &reader);
1856
1857 //===--------------------------------------------------------------------===//
1858 // Dialect Section
1859
1860 LogicalResult parseDialectSection(ArrayRef<uint8_t> sectionData);
1861
1862 /// Parse an operation name reference using the given reader, and set the
1863 /// `wasRegistered` flag that indicates if the bytecode was produced by a
1864 /// context where opName was registered.
1865 FailureOr<OperationName> parseOpName(EncodingReader &reader,
1866 std::optional<bool> &wasRegistered);
1867
1868 //===--------------------------------------------------------------------===//
1869 // Attribute/Type Section
1870
1871 /// Parse an attribute or type using the given reader.
1872 template <typename T>
1873 LogicalResult parseAttribute(EncodingReader &reader, T &result) {
1874 return attrTypeReader.parseAttribute(reader, result);
1875 }
1876 LogicalResult parseType(EncodingReader &reader, Type &result) {
1877 return attrTypeReader.parseType(reader, result);
1878 }
1879
1880 //===--------------------------------------------------------------------===//
1881 // Resource Section
1882
1883 LogicalResult
1884 parseResourceSection(EncodingReader &reader,
1885 std::optional<ArrayRef<uint8_t>> resourceData,
1886 std::optional<ArrayRef<uint8_t>> resourceOffsetData);
1887
1888 //===--------------------------------------------------------------------===//
1889 // IR Section
1890
1891 /// This struct represents the current read state of a range of regions. This
1892 /// struct is used to enable iterative parsing of regions.
1893 struct RegionReadState {
1894 RegionReadState(Operation *op, EncodingReader *reader,
1895 bool isIsolatedFromAbove)
1896 : RegionReadState(op->getRegions(), reader, isIsolatedFromAbove) {}
1897 RegionReadState(MutableArrayRef<Region> regions, EncodingReader *reader,
1898 bool isIsolatedFromAbove)
1899 : curRegion(regions.begin()), endRegion(regions.end()), reader(reader),
1900 isIsolatedFromAbove(isIsolatedFromAbove) {}
1901
1902 /// The current regions being read.
1903 MutableArrayRef<Region>::iterator curRegion, endRegion;
1904 /// This is the reader to use for this region, this pointer is pointing to
1905 /// the parent region reader unless the current region is IsolatedFromAbove,
1906 /// in which case the pointer is pointing to the `owningReader` which is a
1907 /// section dedicated to the current region.
1908 EncodingReader *reader;
1909 std::unique_ptr<EncodingReader> owningReader;
1910
1911 /// The number of values defined immediately within this region.
1912 unsigned numValues = 0;
1913
1914 /// The current blocks of the region being read.
1915 SmallVector<Block *> curBlocks;
1916 Region::iterator curBlock = {};
1917
1918 /// The number of operations remaining to be read from the current block
1919 /// being read.
1920 uint64_t numOpsRemaining = 0;
1921
1922 /// A flag indicating if the regions being read are isolated from above.
1923 bool isIsolatedFromAbove = false;
1924 };
1925
1926 LogicalResult parseIRSection(ArrayRef<uint8_t> sectionData, Block *block);
1927 LogicalResult parseRegions(std::vector<RegionReadState> &regionStack,
1928 RegionReadState &readState);
1929 FailureOr<Operation *> parseOpWithoutRegions(EncodingReader &reader,
1930 RegionReadState &readState,
1931 bool &isIsolatedFromAbove);
1932
1933 LogicalResult parseRegion(RegionReadState &readState);
1934 LogicalResult parseBlockHeader(EncodingReader &reader,
1935 RegionReadState &readState);
1936 LogicalResult parseBlockArguments(EncodingReader &reader, Block *block);
1937
1938 //===--------------------------------------------------------------------===//
1939 // Value Processing
1940
1941 /// Parse an operand reference using the given reader. Returns nullptr in the
1942 /// case of failure.
1943 Value parseOperand(EncodingReader &reader);
1944
1945 /// Sequentially define the given value range.
1946 LogicalResult defineValues(EncodingReader &reader, ValueRange values);
1947
1948 /// Create a value to use for a forward reference.
1949 Value createForwardRef();
1950
1951 //===--------------------------------------------------------------------===//
1952 // Use-list order helpers
1953
1954 /// This struct is a simple storage that contains information required to
1955 /// reorder the use-list of a value with respect to the pre-order traversal
1956 /// ordering.
1957 struct UseListOrderStorage {
1958 UseListOrderStorage(bool isIndexPairEncoding,
1959 SmallVector<unsigned, 4> &&indices)
1960 : indices(std::move(indices)),
1961 isIndexPairEncoding(isIndexPairEncoding) {};
1962 /// The vector containing the information required to reorder the
1963 /// use-list of a value.
1964 SmallVector<unsigned, 4> indices;
1965
1966 /// Whether indices represent a pair of type `(src, dst)` or it is a direct
1967 /// indexing, such as `dst = order[src]`.
1968 bool isIndexPairEncoding;
1969 };
1970
1971 /// Parse use-list order from bytecode for a range of values if available. The
1972 /// range is expected to be either a block argument or an op result range. On
1973 /// success, return a map of the position in the range and the use-list order
1974 /// encoding. The function assumes to know the size of the range it is
1975 /// processing.
1976 using UseListMapT = DenseMap<unsigned, UseListOrderStorage>;
1977 FailureOr<UseListMapT> parseUseListOrderForRange(EncodingReader &reader,
1978 uint64_t rangeSize);
1979
1980 /// Shuffle the use-chain according to the order parsed.
1981 LogicalResult sortUseListOrder(Value value);
1982
1983 /// Recursively visit all the values defined within topLevelOp and sort the
1984 /// use-list orders according to the indices parsed.
1985 LogicalResult processUseLists(Operation *topLevelOp);
1986
1987 //===--------------------------------------------------------------------===//
1988 // Fields
1989
1990 /// This class represents a single value scope, in which a value scope is
1991 /// delimited by isolated from above regions.
1992 struct ValueScope {
1993 /// Push a new region state onto this scope, reserving enough values for
1994 /// those defined within the current region of the provided state.
1995 void push(RegionReadState &readState) {
1996 nextValueIDs.push_back(Elt: values.size());
1997 values.resize(new_size: values.size() + readState.numValues);
1998 }
1999
2000 /// Pop the values defined for the current region within the provided region
2001 /// state.
2002 void pop(RegionReadState &readState) {
2003 values.resize(new_size: values.size() - readState.numValues);
2004 nextValueIDs.pop_back();
2005 }
2006
2007 /// The set of values defined in this scope.
2008 std::vector<Value> values;
2009
2010 /// The ID for the next defined value for each region current being
2011 /// processed in this scope.
2012 SmallVector<unsigned, 4> nextValueIDs;
2013 };
2014
2015 /// The configuration of the parser.
2016 const ParserConfig &config;
2017
2018 /// A location to use when emitting errors.
2019 Location fileLoc;
2020
2021 /// Flag that indicates if lazyloading is enabled.
2022 bool lazyLoading;
2023
2024 /// Keep track of operations that have been lazy loaded (their regions haven't
2025 /// been materialized), along with the `RegionReadState` that allows to
2026 /// lazy-load the regions nested under the operation.
2027 LazyLoadableOpsInfo lazyLoadableOps;
2028 LazyLoadableOpsMap lazyLoadableOpsMap;
2029 llvm::function_ref<bool(Operation *)> lazyOpsCallback;
2030
2031 /// The reader used to process attribute and types within the bytecode.
2032 AttrTypeReader attrTypeReader;
2033
2034 /// The version of the bytecode being read.
2035 uint64_t version = 0;
2036
2037 /// The producer of the bytecode being read.
2038 StringRef producer;
2039
2040 /// The table of IR units referenced within the bytecode file.
2041 SmallVector<std::unique_ptr<BytecodeDialect>> dialects;
2042 llvm::StringMap<BytecodeDialect *> dialectsMap;
2043 SmallVector<BytecodeOperationName> opNames;
2044
2045 /// The reader used to process resources within the bytecode.
2046 ResourceSectionReader resourceReader;
2047
2048 /// Worklist of values with custom use-list orders to process before the end
2049 /// of the parsing.
2050 DenseMap<void *, UseListOrderStorage> valueToUseListMap;
2051
2052 /// The table of strings referenced within the bytecode file.
2053 StringSectionReader stringReader;
2054
2055 /// The table of properties referenced by the operation in the bytecode file.
2056 PropertiesSectionReader propertiesReader;
2057
2058 /// The current set of available IR value scopes.
2059 std::vector<ValueScope> valueScopes;
2060
2061 /// The global pre-order operation ordering.
2062 DenseMap<Operation *, unsigned> operationIDs;
2063
2064 /// A block containing the set of operations defined to create forward
2065 /// references.
2066 Block forwardRefOps;
2067
2068 /// A block containing previously created, and no longer used, forward
2069 /// reference operations.
2070 Block openForwardRefOps;
2071
2072 /// An operation state used when instantiating forward references.
2073 OperationState forwardRefOpState;
2074
2075 /// Reference to the input buffer.
2076 llvm::MemoryBufferRef buffer;
2077
2078 /// The optional owning source manager, which when present may be used to
2079 /// extend the lifetime of the input buffer.
2080 const std::shared_ptr<llvm::SourceMgr> bufferOwnerRef;
2081};
2082
2083LogicalResult BytecodeReader::Impl::read(
2084 Block *block, llvm::function_ref<bool(Operation *)> lazyOpsCallback) {
2085 EncodingReader reader(buffer.getBuffer(), fileLoc);
2086 this->lazyOpsCallback = lazyOpsCallback;
2087 llvm::scope_exit resetlazyOpsCallback(
2088 [&] { this->lazyOpsCallback = nullptr; });
2089
2090 // Skip over the bytecode header, this should have already been checked.
2091 if (failed(Result: reader.skipBytes(length: StringRef("ML\xefR").size())))
2092 return failure();
2093 // Parse the bytecode version and producer.
2094 if (failed(Result: parseVersion(reader)) ||
2095 failed(Result: reader.parseNullTerminatedString(result&: producer)))
2096 return failure();
2097
2098 // Add a diagnostic handler that attaches a note that includes the original
2099 // producer of the bytecode.
2100 ScopedDiagnosticHandler diagHandler(getContext(), [&](Diagnostic &diag) {
2101 diag.attachNote() << "in bytecode version " << version
2102 << " produced by: " << producer;
2103 return failure();
2104 });
2105
2106 const auto checkSectionAlignment = [&](unsigned alignment) {
2107 return this->checkSectionAlignment(
2108 alignment, emitError: [&](const auto &msg) { return reader.emitError(msg); });
2109 };
2110
2111 // Parse the raw data for each of the top-level sections of the bytecode.
2112 std::optional<ArrayRef<uint8_t>>
2113 sectionDatas[bytecode::Section::kNumSections];
2114 while (!reader.empty()) {
2115 // Read the next section from the bytecode.
2116 bytecode::Section::ID sectionID;
2117 ArrayRef<uint8_t> sectionData;
2118 if (failed(
2119 Result: reader.parseSection(sectionID, alignmentValidator: checkSectionAlignment, sectionData)))
2120 return failure();
2121
2122 // Check for duplicate sections, we only expect one instance of each.
2123 if (sectionDatas[sectionID]) {
2124 return reader.emitError(args: "duplicate top-level section: ",
2125 args: ::toString(sectionID));
2126 }
2127 sectionDatas[sectionID] = sectionData;
2128 }
2129 // Check that all of the required sections were found.
2130 for (int i = 0; i < bytecode::Section::kNumSections; ++i) {
2131 bytecode::Section::ID sectionID = static_cast<bytecode::Section::ID>(i);
2132 if (!sectionDatas[i] && !isSectionOptional(sectionID, version)) {
2133 return reader.emitError(args: "missing data for top-level section: ",
2134 args: ::toString(sectionID));
2135 }
2136 }
2137
2138 // Process the string section first.
2139 if (failed(Result: stringReader.initialize(
2140 fileLoc, sectionData: *sectionDatas[bytecode::Section::kString])))
2141 return failure();
2142
2143 // Process the properties section.
2144 if (sectionDatas[bytecode::Section::kProperties] &&
2145 failed(Result: propertiesReader.initialize(
2146 fileLoc, sectionData: *sectionDatas[bytecode::Section::kProperties])))
2147 return failure();
2148
2149 // Process the dialect section.
2150 if (failed(Result: parseDialectSection(sectionData: *sectionDatas[bytecode::Section::kDialect])))
2151 return failure();
2152
2153 // Process the resource section if present.
2154 if (failed(Result: parseResourceSection(
2155 reader, resourceData: sectionDatas[bytecode::Section::kResource],
2156 resourceOffsetData: sectionDatas[bytecode::Section::kResourceOffset])))
2157 return failure();
2158
2159 // Process the attribute and type section.
2160 if (failed(Result: attrTypeReader.initialize(
2161 dialects, sectionData: *sectionDatas[bytecode::Section::kAttrType],
2162 offsetSectionData: *sectionDatas[bytecode::Section::kAttrTypeOffset])))
2163 return failure();
2164
2165 // Finally, process the IR section.
2166 return parseIRSection(sectionData: *sectionDatas[bytecode::Section::kIR], block);
2167}
2168
2169LogicalResult BytecodeReader::Impl::parseVersion(EncodingReader &reader) {
2170 if (failed(Result: reader.parseVarInt(result&: version)))
2171 return failure();
2172
2173 // Validate the bytecode version.
2174 uint64_t currentVersion = bytecode::kVersion;
2175 uint64_t minSupportedVersion = bytecode::kMinSupportedVersion;
2176 if (version < minSupportedVersion) {
2177 return reader.emitError(args: "bytecode version ", args&: version,
2178 args: " is older than the current version of ",
2179 args&: currentVersion, args: ", and upgrade is not supported");
2180 }
2181 if (version > currentVersion) {
2182 return reader.emitError(args: "bytecode version ", args&: version,
2183 args: " is newer than the current version ",
2184 args&: currentVersion);
2185 }
2186 // Override any request to lazy-load if the bytecode version is too old.
2187 if (version < bytecode::kLazyLoading)
2188 lazyLoading = false;
2189 return success();
2190}
2191
2192//===----------------------------------------------------------------------===//
2193// Dialect Section
2194//===----------------------------------------------------------------------===//
2195
2196LogicalResult BytecodeDialect::load(const DialectReader &reader,
2197 MLIRContext *ctx) {
2198 if (dialect)
2199 return success();
2200 Dialect *loadedDialect = ctx->getOrLoadDialect(name);
2201 if (!loadedDialect && !ctx->allowsUnregisteredDialects()) {
2202 return reader.emitError(msg: "dialect '")
2203 << name
2204 << "' is unknown. If this is intended, please call "
2205 "allowUnregisteredDialects() on the MLIRContext, or use "
2206 "-allow-unregistered-dialect with the MLIR tool used.";
2207 }
2208 dialect = loadedDialect;
2209
2210 // If the dialect was actually loaded, check to see if it has a bytecode
2211 // interface.
2212 if (loadedDialect)
2213 interface = dyn_cast<BytecodeDialectInterface>(Val: loadedDialect);
2214 if (!versionBuffer.empty()) {
2215 if (!interface)
2216 return reader.emitError(msg: "dialect '")
2217 << name
2218 << "' does not implement the bytecode interface, "
2219 "but found a version entry";
2220 EncodingReader encReader(versionBuffer, reader.getLoc());
2221 DialectReader versionReader = reader.withEncodingReader(encReader);
2222 loadedVersion = interface->readVersion(reader&: versionReader);
2223 if (!loadedVersion)
2224 return failure();
2225 }
2226 return success();
2227}
2228
2229LogicalResult
2230BytecodeReader::Impl::parseDialectSection(ArrayRef<uint8_t> sectionData) {
2231 EncodingReader sectionReader(sectionData, fileLoc);
2232
2233 // Parse the number of dialects in the section.
2234 uint64_t numDialects;
2235 if (failed(Result: sectionReader.parseVarInt(result&: numDialects)))
2236 return failure();
2237 dialects.resize(N: numDialects);
2238
2239 const auto checkSectionAlignment = [&](unsigned alignment) {
2240 return this->checkSectionAlignment(alignment, emitError: [&](const auto &msg) {
2241 return sectionReader.emitError(msg);
2242 });
2243 };
2244
2245 // Parse each of the dialects.
2246 for (uint64_t i = 0; i < numDialects; ++i) {
2247 dialects[i] = std::make_unique<BytecodeDialect>();
2248 /// Before version kDialectVersioning, there wasn't any versioning available
2249 /// for dialects, and the entryIdx represent the string itself.
2250 if (version < bytecode::kDialectVersioning) {
2251 if (failed(Result: stringReader.parseString(reader&: sectionReader, result&: dialects[i]->name)))
2252 return failure();
2253 continue;
2254 }
2255
2256 // Parse ID representing dialect and version.
2257 uint64_t dialectNameIdx;
2258 bool versionAvailable;
2259 if (failed(Result: sectionReader.parseVarIntWithFlag(result&: dialectNameIdx,
2260 flag&: versionAvailable)))
2261 return failure();
2262 if (failed(Result: stringReader.parseStringAtIndex(reader&: sectionReader, index: dialectNameIdx,
2263 result&: dialects[i]->name)))
2264 return failure();
2265 if (versionAvailable) {
2266 bytecode::Section::ID sectionID;
2267 if (failed(Result: sectionReader.parseSection(sectionID, alignmentValidator: checkSectionAlignment,
2268 sectionData&: dialects[i]->versionBuffer)))
2269 return failure();
2270 if (sectionID != bytecode::Section::kDialectVersions) {
2271 emitError(loc: fileLoc, message: "expected dialect version section");
2272 return failure();
2273 }
2274 }
2275 dialectsMap[dialects[i]->name] = dialects[i].get();
2276 }
2277
2278 // Parse the operation names, which are grouped by dialect.
2279 auto parseOpName = [&](BytecodeDialect *dialect) {
2280 StringRef opName;
2281 std::optional<bool> wasRegistered;
2282 // Prior to version kNativePropertiesEncoding, the information about wheter
2283 // an op was registered or not wasn't encoded.
2284 if (version < bytecode::kNativePropertiesEncoding) {
2285 if (failed(Result: stringReader.parseString(reader&: sectionReader, result&: opName)))
2286 return failure();
2287 } else {
2288 bool wasRegisteredFlag;
2289 if (failed(Result: stringReader.parseStringWithFlag(reader&: sectionReader, result&: opName,
2290 flag&: wasRegisteredFlag)))
2291 return failure();
2292 wasRegistered = wasRegisteredFlag;
2293 }
2294 opNames.emplace_back(Args&: dialect, Args&: opName, Args&: wasRegistered);
2295 return success();
2296 };
2297 // Avoid re-allocation in bytecode version >=kElideUnknownBlockArgLocation
2298 // where the number of ops are known.
2299 if (version >= bytecode::kElideUnknownBlockArgLocation) {
2300 uint64_t numOps;
2301 if (failed(Result: sectionReader.parseVarInt(result&: numOps)))
2302 return failure();
2303 opNames.reserve(N: numOps);
2304 }
2305 while (!sectionReader.empty())
2306 if (failed(Result: parseDialectGrouping(reader&: sectionReader, dialects, entryCallback: parseOpName)))
2307 return failure();
2308 return success();
2309}
2310
2311FailureOr<OperationName>
2312BytecodeReader::Impl::parseOpName(EncodingReader &reader,
2313 std::optional<bool> &wasRegistered) {
2314 BytecodeOperationName *opName = nullptr;
2315 if (failed(Result: parseEntry(reader, entries&: opNames, entry&: opName, entryStr: "operation name")))
2316 return failure();
2317 wasRegistered = opName->wasRegistered;
2318 // Check to see if this operation name has already been resolved. If we
2319 // haven't, load the dialect and build the operation name.
2320 if (!opName->opName) {
2321 // If the opName is empty, this is because we use to accept names such as
2322 // `foo` without any `.` separator. We shouldn't tolerate this in textual
2323 // format anymore but for now we'll be backward compatible. This can only
2324 // happen with unregistered dialects.
2325 if (opName->name.empty()) {
2326 opName->opName.emplace(args&: opName->dialect->name, args: getContext());
2327 } else {
2328 // Load the dialect and its version.
2329 DialectReader dialectReader(attrTypeReader, stringReader, resourceReader,
2330 dialectsMap, reader, version);
2331 if (failed(Result: opName->dialect->load(reader: dialectReader, ctx: getContext())))
2332 return failure();
2333 opName->opName.emplace(args: (opName->dialect->name + "." + opName->name).str(),
2334 args: getContext());
2335 }
2336 }
2337 return *opName->opName;
2338}
2339
2340//===----------------------------------------------------------------------===//
2341// Resource Section
2342//===----------------------------------------------------------------------===//
2343
2344LogicalResult BytecodeReader::Impl::parseResourceSection(
2345 EncodingReader &reader, std::optional<ArrayRef<uint8_t>> resourceData,
2346 std::optional<ArrayRef<uint8_t>> resourceOffsetData) {
2347 // Ensure both sections are either present or not.
2348 if (resourceData.has_value() != resourceOffsetData.has_value()) {
2349 if (resourceOffsetData)
2350 return emitError(loc: fileLoc, message: "unexpected resource offset section when "
2351 "resource section is not present");
2352 return emitError(
2353 loc: fileLoc,
2354 message: "expected resource offset section when resource section is present");
2355 }
2356
2357 // If the resource sections are absent, there is nothing to do.
2358 if (!resourceData)
2359 return success();
2360
2361 // Initialize the resource reader with the resource sections.
2362 DialectReader dialectReader(attrTypeReader, stringReader, resourceReader,
2363 dialectsMap, reader, version);
2364 return resourceReader.initialize(fileLoc, config, dialects, stringReader,
2365 sectionData: *resourceData, offsetSectionData: *resourceOffsetData,
2366 dialectReader, bufferOwnerRef);
2367}
2368
2369//===----------------------------------------------------------------------===//
2370// UseListOrder Helpers
2371//===----------------------------------------------------------------------===//
2372
2373FailureOr<BytecodeReader::Impl::UseListMapT>
2374BytecodeReader::Impl::parseUseListOrderForRange(EncodingReader &reader,
2375 uint64_t numResults) {
2376 BytecodeReader::Impl::UseListMapT map;
2377 uint64_t numValuesToRead = 1;
2378 if (numResults > 1 && failed(Result: reader.parseVarInt(result&: numValuesToRead)))
2379 return failure();
2380
2381 for (size_t valueIdx = 0; valueIdx < numValuesToRead; valueIdx++) {
2382 uint64_t resultIdx = 0;
2383 if (numResults > 1 && failed(Result: reader.parseVarInt(result&: resultIdx)))
2384 return failure();
2385
2386 uint64_t numValues;
2387 bool indexPairEncoding;
2388 if (failed(Result: reader.parseVarIntWithFlag(result&: numValues, flag&: indexPairEncoding)))
2389 return failure();
2390
2391 SmallVector<unsigned, 4> useListOrders;
2392 for (size_t idx = 0; idx < numValues; idx++) {
2393 uint64_t index;
2394 if (failed(Result: reader.parseVarInt(result&: index)))
2395 return failure();
2396 useListOrders.push_back(Elt: index);
2397 }
2398
2399 // Store in a map the result index
2400 map.try_emplace(Key: resultIdx, Args: UseListOrderStorage(indexPairEncoding,
2401 std::move(useListOrders)));
2402 }
2403
2404 return map;
2405}
2406
2407/// Sorts each use according to the order specified in the use-list parsed. If
2408/// the custom use-list is not found, this means that the order needs to be
2409/// consistent with the reverse pre-order walk of the IR. If multiple uses lie
2410/// on the same operation, the order will follow the reverse operand number
2411/// ordering.
2412LogicalResult BytecodeReader::Impl::sortUseListOrder(Value value) {
2413 // Early return for trivial use-lists.
2414 if (value.use_empty() || value.hasOneUse())
2415 return success();
2416
2417 bool hasIncomingOrder =
2418 valueToUseListMap.contains(Val: value.getAsOpaquePointer());
2419
2420 // Compute the current order of the use-list with respect to the global
2421 // ordering. Detect if the order is already sorted while doing so.
2422 bool alreadySorted = true;
2423 auto &firstUse = *value.use_begin();
2424 uint64_t prevID =
2425 bytecode::getUseID(val&: firstUse, ownerID: operationIDs.at(Val: firstUse.getOwner()));
2426 llvm::SmallVector<std::pair<unsigned, uint64_t>> currentOrder = {{0, prevID}};
2427 for (auto item : llvm::drop_begin(RangeOrContainer: llvm::enumerate(First: value.getUses()))) {
2428 uint64_t currentID = bytecode::getUseID(
2429 val&: item.value(), ownerID: operationIDs.at(Val: item.value().getOwner()));
2430 alreadySorted &= prevID > currentID;
2431 currentOrder.push_back(Elt: {item.index(), currentID});
2432 prevID = currentID;
2433 }
2434
2435 // If the order is already sorted, and there wasn't a custom order to apply
2436 // from the bytecode file, we are done.
2437 if (alreadySorted && !hasIncomingOrder)
2438 return success();
2439
2440 // If not already sorted, sort the indices of the current order by descending
2441 // useIDs.
2442 if (!alreadySorted)
2443 std::sort(
2444 first: currentOrder.begin(), last: currentOrder.end(),
2445 comp: [](auto elem1, auto elem2) { return elem1.second > elem2.second; });
2446
2447 if (!hasIncomingOrder) {
2448 // If the bytecode file did not contain any custom use-list order, it means
2449 // that the order was descending useID. Hence, shuffle by the first index
2450 // of the `currentOrder` pair.
2451 SmallVector<unsigned> shuffle(llvm::make_first_range(c&: currentOrder));
2452 value.shuffleUseList(indices: shuffle);
2453 return success();
2454 }
2455
2456 // Pull the custom order info from the map.
2457 UseListOrderStorage customOrder =
2458 valueToUseListMap.at(Val: value.getAsOpaquePointer());
2459 SmallVector<unsigned, 4> shuffle = std::move(customOrder.indices);
2460 uint64_t numUses = value.getNumUses();
2461
2462 // If the encoding was a pair of indices `(src, dst)` for every permutation,
2463 // reconstruct the shuffle vector for every use. Initialize the shuffle vector
2464 // as identity, and then apply the mapping encoded in the indices.
2465 // This produces shuffle[oldIdx] = newPos (i.e., old_index -> new_position).
2466 if (customOrder.isIndexPairEncoding) {
2467 // Return failure if the number of indices was not representing pairs.
2468 if (shuffle.size() & 1)
2469 return failure();
2470
2471 SmallVector<unsigned, 4> newShuffle(numUses);
2472 size_t idx = 0;
2473 std::iota(first: newShuffle.begin(), last: newShuffle.end(), value: idx);
2474 for (idx = 0; idx < shuffle.size(); idx += 2)
2475 newShuffle[shuffle[idx]] = shuffle[idx + 1];
2476
2477 shuffle = std::move(newShuffle);
2478 }
2479
2480 // Make sure that the indices represent a valid mapping. That is, the sum of
2481 // all the values needs to be equal to (numUses - 1) * numUses / 2, and no
2482 // duplicates are allowed in the list.
2483 DenseSet<unsigned> set;
2484 uint64_t accumulator = 0;
2485 for (const auto &elem : shuffle) {
2486 if (!set.insert(V: elem).second)
2487 return failure();
2488 accumulator += elem;
2489 }
2490 if (numUses != shuffle.size() ||
2491 accumulator != (((numUses - 1) * numUses) >> 1))
2492 return failure();
2493
2494 // Compose the shuffle with the current memory layout to produce the final
2495 // indices for shuffleUseList. shuffleUseList(indices) places the use at
2496 // current position i into position indices[i], so we need to compute
2497 // finalShuffle[readerMemIdx] = writerMemIdx.
2498 //
2499 // The two encoding paths have different shuffle conventions:
2500 //
2501 // Index-pair encoding (already normalized above):
2502 // shuffle[writerMemIdx] = sortedPos (old_index -> new_position)
2503 //
2504 // Full-shuffle encoding (writer's native format):
2505 // shuffle[sortedPos] = writerMemIdx (new_position -> old_index)
2506 //
2507 // In both cases, currentOrder[sortedPos].first gives the readerMemIdx for
2508 // a given sorted position. We fold the permutation inversion for the
2509 // full-shuffle case directly into the composition to avoid an extra pass.
2510 SmallVector<unsigned, 4> finalShuffle(numUses);
2511 if (customOrder.isIndexPairEncoding) {
2512 for (size_t writerMemIdx = 0; writerMemIdx < numUses; ++writerMemIdx)
2513 finalShuffle[currentOrder[shuffle[writerMemIdx]].first] = writerMemIdx;
2514 } else {
2515 for (size_t sortedPos = 0; sortedPos < numUses; ++sortedPos)
2516 finalShuffle[currentOrder[sortedPos].first] = shuffle[sortedPos];
2517 }
2518 shuffle = std::move(finalShuffle);
2519
2520 value.shuffleUseList(indices: shuffle);
2521 return success();
2522}
2523
2524LogicalResult BytecodeReader::Impl::processUseLists(Operation *topLevelOp) {
2525 // Precompute operation IDs according to the pre-order walk of the IR. We
2526 // can't do this while parsing since parseRegions ordering is not strictly
2527 // equal to the pre-order walk.
2528 unsigned operationID = 0;
2529 topLevelOp->walk<mlir::WalkOrder::PreOrder>(
2530 callback: [&](Operation *op) { operationIDs.try_emplace(Key: op, Args: operationID++); });
2531
2532 auto blockWalk = topLevelOp->walk(callback: [this](Block *block) {
2533 for (auto arg : block->getArguments())
2534 if (failed(Result: sortUseListOrder(value: arg)))
2535 return WalkResult::interrupt();
2536 return WalkResult::advance();
2537 });
2538
2539 auto resultWalk = topLevelOp->walk(callback: [this](Operation *op) {
2540 for (auto result : op->getResults())
2541 if (failed(Result: sortUseListOrder(value: result)))
2542 return WalkResult::interrupt();
2543 return WalkResult::advance();
2544 });
2545
2546 return failure(IsFailure: blockWalk.wasInterrupted() || resultWalk.wasInterrupted());
2547}
2548
2549//===----------------------------------------------------------------------===//
2550// IR Section
2551//===----------------------------------------------------------------------===//
2552
2553LogicalResult
2554BytecodeReader::Impl::parseIRSection(ArrayRef<uint8_t> sectionData,
2555 Block *block) {
2556 EncodingReader reader(sectionData, fileLoc);
2557
2558 // A stack of operation regions currently being read from the bytecode.
2559 std::vector<RegionReadState> regionStack;
2560
2561 // Parse the top-level block using a temporary module operation.
2562 OwningOpRef<ModuleOp> moduleOp = ModuleOp::create(loc: fileLoc);
2563 regionStack.emplace_back(args: *moduleOp, args: &reader, /*isIsolatedFromAbove=*/args: true);
2564 regionStack.back().curBlocks.push_back(Elt: moduleOp->getBody());
2565 regionStack.back().curBlock = regionStack.back().curRegion->begin();
2566 if (failed(Result: parseBlockHeader(reader, readState&: regionStack.back())))
2567 return failure();
2568 valueScopes.emplace_back();
2569 valueScopes.back().push(readState&: regionStack.back());
2570
2571 // Iteratively parse regions until everything has been resolved.
2572 while (!regionStack.empty())
2573 if (failed(Result: parseRegions(regionStack, readState&: regionStack.back())))
2574 return failure();
2575 if (!forwardRefOps.empty()) {
2576 return reader.emitError(
2577 args: "not all forward unresolved forward operand references");
2578 }
2579
2580 // Sort use-lists according to what specified in bytecode.
2581 if (failed(Result: processUseLists(topLevelOp: *moduleOp)))
2582 return reader.emitError(
2583 args: "parsed use-list orders were invalid and could not be applied");
2584
2585 // Resolve dialect version.
2586 for (const std::unique_ptr<BytecodeDialect> &byteCodeDialect : dialects) {
2587 // Parsing is complete, give an opportunity to each dialect to visit the
2588 // IR and perform upgrades.
2589 if (!byteCodeDialect->loadedVersion)
2590 continue;
2591 if (byteCodeDialect->interface &&
2592 failed(Result: byteCodeDialect->interface->upgradeFromVersion(
2593 topLevelOp: *moduleOp, version: *byteCodeDialect->loadedVersion)))
2594 return failure();
2595 }
2596
2597 // Verify that the parsed operations are valid.
2598 if (config.shouldVerifyAfterParse() && failed(Result: verify(op: *moduleOp)))
2599 return failure();
2600
2601 // Splice the parsed operations over to the provided top-level block.
2602 auto &parsedOps = moduleOp->getBody()->getOperations();
2603 auto &destOps = block->getOperations();
2604 destOps.splice(where: destOps.end(), L2&: parsedOps, first: parsedOps.begin(), last: parsedOps.end());
2605 return success();
2606}
2607
2608LogicalResult
2609BytecodeReader::Impl::parseRegions(std::vector<RegionReadState> &regionStack,
2610 RegionReadState &readState) {
2611 const auto checkSectionAlignment = [&](unsigned alignment) {
2612 return this->checkSectionAlignment(
2613 alignment, emitError: [&](const auto &msg) { return emitError(fileLoc, msg); });
2614 };
2615
2616 // Process regions, blocks, and operations until the end or if a nested
2617 // region is encountered. In this case we push a new state in regionStack and
2618 // return, the processing of the current region will resume afterward.
2619 for (; readState.curRegion != readState.endRegion; ++readState.curRegion) {
2620 // If the current block hasn't been setup yet, parse the header for this
2621 // region. The current block is already setup when this function was
2622 // interrupted to recurse down in a nested region and we resume the current
2623 // block after processing the nested region.
2624 if (readState.curBlock == Region::iterator()) {
2625 if (failed(Result: parseRegion(readState)))
2626 return failure();
2627
2628 // If the region is empty, there is nothing to more to do.
2629 if (readState.curRegion->empty())
2630 continue;
2631 }
2632
2633 // Parse the blocks within the region.
2634 EncodingReader &reader = *readState.reader;
2635 do {
2636 while (readState.numOpsRemaining--) {
2637 // Read in the next operation. We don't read its regions directly, we
2638 // handle those afterwards as necessary.
2639 bool isIsolatedFromAbove = false;
2640 FailureOr<Operation *> op =
2641 parseOpWithoutRegions(reader, readState, isIsolatedFromAbove);
2642 if (failed(Result: op))
2643 return failure();
2644
2645 // If the op has regions, add it to the stack for processing and return:
2646 // we stop the processing of the current region and resume it after the
2647 // inner one is completed. Unless LazyLoading is activated in which case
2648 // nested region parsing is delayed.
2649 if ((*op)->getNumRegions()) {
2650 RegionReadState childState(*op, &reader, isIsolatedFromAbove);
2651
2652 // Isolated regions are encoded as a section in version 2 and above.
2653 if (version >= bytecode::kLazyLoading && isIsolatedFromAbove) {
2654 bytecode::Section::ID sectionID;
2655 ArrayRef<uint8_t> sectionData;
2656 if (failed(Result: reader.parseSection(sectionID, alignmentValidator: checkSectionAlignment,
2657 sectionData)))
2658 return failure();
2659 if (sectionID != bytecode::Section::kIR)
2660 return emitError(loc: fileLoc, message: "expected IR section for region");
2661 childState.owningReader =
2662 std::make_unique<EncodingReader>(args&: sectionData, args&: fileLoc);
2663 childState.reader = childState.owningReader.get();
2664
2665 // If the user has a callback set, they have the opportunity to
2666 // control lazyloading as we go.
2667 if (lazyLoading && (!lazyOpsCallback || !lazyOpsCallback(*op))) {
2668 lazyLoadableOps.emplace_back(args&: *op, args: std::move(childState));
2669 lazyLoadableOpsMap.try_emplace(Key: *op,
2670 Args: std::prev(x: lazyLoadableOps.end()));
2671 continue;
2672 }
2673 }
2674 regionStack.push_back(x: std::move(childState));
2675
2676 // If the op is isolated from above, push a new value scope.
2677 if (isIsolatedFromAbove)
2678 valueScopes.emplace_back();
2679 return success();
2680 }
2681 }
2682
2683 // Move to the next block of the region.
2684 if (++readState.curBlock == readState.curRegion->end())
2685 break;
2686 if (failed(Result: parseBlockHeader(reader, readState)))
2687 return failure();
2688 } while (true);
2689
2690 // Reset the current block and any values reserved for this region.
2691 readState.curBlock = {};
2692 valueScopes.back().pop(readState);
2693 }
2694
2695 // When the regions have been fully parsed, pop them off of the read stack. If
2696 // the regions were isolated from above, we also pop the last value scope.
2697 if (readState.isIsolatedFromAbove) {
2698 assert(!valueScopes.empty() && "Expect a valueScope after reading region");
2699 valueScopes.pop_back();
2700 }
2701 assert(!regionStack.empty() && "Expect a regionStack after reading region");
2702 regionStack.pop_back();
2703 return success();
2704}
2705
2706FailureOr<Operation *>
2707BytecodeReader::Impl::parseOpWithoutRegions(EncodingReader &reader,
2708 RegionReadState &readState,
2709 bool &isIsolatedFromAbove) {
2710 // Parse the name of the operation.
2711 std::optional<bool> wasRegistered;
2712 FailureOr<OperationName> opName = parseOpName(reader, wasRegistered);
2713 if (failed(Result: opName))
2714 return failure();
2715
2716 // Parse the operation mask, which indicates which components of the operation
2717 // are present.
2718 uint8_t opMask;
2719 if (failed(Result: reader.parseByte(value&: opMask)))
2720 return failure();
2721
2722 /// Parse the location.
2723 LocationAttr opLoc;
2724 if (failed(Result: parseAttribute(reader, result&: opLoc)))
2725 return failure();
2726
2727 // With the location and name resolved, we can start building the operation
2728 // state.
2729 OperationState opState(opLoc, *opName);
2730
2731 // Parse the attributes of the operation.
2732 if (opMask & bytecode::OpEncodingMask::kHasAttrs) {
2733 DictionaryAttr dictAttr;
2734 if (failed(Result: parseAttribute(reader, result&: dictAttr)))
2735 return failure();
2736 opState.attributes = dictAttr;
2737 }
2738
2739 if (opMask & bytecode::OpEncodingMask::kHasProperties) {
2740 // kHasProperties wasn't emitted in older bytecode, we should never get
2741 // there without also having the `wasRegistered` flag available.
2742 if (!wasRegistered)
2743 return emitError(loc: fileLoc,
2744 message: "Unexpected missing `wasRegistered` opname flag at "
2745 "bytecode version ")
2746 << version << " with properties.";
2747 // When an operation is emitted without being registered, the properties are
2748 // stored as an attribute. Otherwise the op must implement the bytecode
2749 // interface and control the serialization.
2750 if (wasRegistered) {
2751 DialectReader dialectReader(attrTypeReader, stringReader, resourceReader,
2752 dialectsMap, reader, version);
2753 if (failed(
2754 Result: propertiesReader.read(fileLoc, dialectReader, opName: &*opName, opState)))
2755 return failure();
2756 } else {
2757 // If the operation wasn't registered when it was emitted, the properties
2758 // was serialized as an attribute.
2759 if (failed(Result: parseAttribute(reader, result&: opState.propertiesAttr)))
2760 return failure();
2761 }
2762 }
2763
2764 /// Parse the results of the operation.
2765 if (opMask & bytecode::OpEncodingMask::kHasResults) {
2766 uint64_t numResults;
2767 if (failed(Result: reader.parseVarInt(result&: numResults)))
2768 return failure();
2769 opState.types.resize(N: numResults);
2770 for (int i = 0, e = numResults; i < e; ++i)
2771 if (failed(Result: parseType(reader, result&: opState.types[i])))
2772 return failure();
2773 }
2774
2775 /// Parse the operands of the operation.
2776 if (opMask & bytecode::OpEncodingMask::kHasOperands) {
2777 uint64_t numOperands;
2778 if (failed(Result: reader.parseVarInt(result&: numOperands)))
2779 return failure();
2780 opState.operands.resize(N: numOperands);
2781 for (int i = 0, e = numOperands; i < e; ++i)
2782 if (!(opState.operands[i] = parseOperand(reader)))
2783 return failure();
2784 }
2785
2786 /// Parse the successors of the operation.
2787 if (opMask & bytecode::OpEncodingMask::kHasSuccessors) {
2788 uint64_t numSuccs;
2789 if (failed(Result: reader.parseVarInt(result&: numSuccs)))
2790 return failure();
2791 opState.successors.resize(N: numSuccs);
2792 for (int i = 0, e = numSuccs; i < e; ++i) {
2793 if (failed(Result: parseEntry(reader, entries&: readState.curBlocks, entry&: opState.successors[i],
2794 entryStr: "successor")))
2795 return failure();
2796 }
2797 }
2798
2799 /// Parse the use-list orders for the results of the operation. Use-list
2800 /// orders are available since version 3 of the bytecode.
2801 std::optional<UseListMapT> resultIdxToUseListMap = std::nullopt;
2802 if (version >= bytecode::kUseListOrdering &&
2803 (opMask & bytecode::OpEncodingMask::kHasUseListOrders)) {
2804 size_t numResults = opState.types.size();
2805 auto parseResult = parseUseListOrderForRange(reader, numResults);
2806 if (failed(Result: parseResult))
2807 return failure();
2808 resultIdxToUseListMap = std::move(*parseResult);
2809 }
2810
2811 /// Parse the regions of the operation.
2812 if (opMask & bytecode::OpEncodingMask::kHasInlineRegions) {
2813 uint64_t numRegions;
2814 if (failed(Result: reader.parseVarIntWithFlag(result&: numRegions, flag&: isIsolatedFromAbove)))
2815 return failure();
2816
2817 opState.regions.reserve(N: numRegions);
2818 for (int i = 0, e = numRegions; i < e; ++i)
2819 opState.regions.push_back(Elt: std::make_unique<Region>());
2820 }
2821
2822 // Create the operation at the back of the current block.
2823 Operation *op = Operation::create(state: opState);
2824 readState.curBlock->push_back(op);
2825
2826 // If the operation had results, update the value references. We don't need to
2827 // do this if the current value scope is empty. That is, the op was not
2828 // encoded within a parent region.
2829 if (readState.numValues && op->getNumResults() &&
2830 failed(Result: defineValues(reader, values: op->getResults())))
2831 return failure();
2832
2833 /// Store a map for every value that received a custom use-list order from the
2834 /// bytecode file.
2835 if (resultIdxToUseListMap.has_value()) {
2836 for (size_t idx = 0; idx < op->getNumResults(); idx++) {
2837 if (resultIdxToUseListMap->contains(Val: idx)) {
2838 valueToUseListMap.try_emplace(Key: op->getResult(idx).getAsOpaquePointer(),
2839 Args&: resultIdxToUseListMap->at(Val: idx));
2840 }
2841 }
2842 }
2843 return op;
2844}
2845
2846LogicalResult BytecodeReader::Impl::parseRegion(RegionReadState &readState) {
2847 EncodingReader &reader = *readState.reader;
2848
2849 // Parse the number of blocks in the region.
2850 uint64_t numBlocks;
2851 if (failed(Result: reader.parseVarInt(result&: numBlocks)))
2852 return failure();
2853
2854 // If the region is empty, there is nothing else to do.
2855 if (numBlocks == 0)
2856 return success();
2857
2858 // Parse the number of values defined in this region.
2859 uint64_t numValues;
2860 if (failed(Result: reader.parseVarInt(result&: numValues)))
2861 return failure();
2862 readState.numValues = numValues;
2863
2864 // Create the blocks within this region. We do this before processing so that
2865 // we can rely on the blocks existing when creating operations.
2866 readState.curBlocks.clear();
2867 readState.curBlocks.reserve(N: numBlocks);
2868 for (uint64_t i = 0; i < numBlocks; ++i) {
2869 readState.curBlocks.push_back(Elt: new Block());
2870 readState.curRegion->push_back(block: readState.curBlocks.back());
2871 }
2872
2873 // Prepare the current value scope for this region.
2874 valueScopes.back().push(readState);
2875
2876 // Parse the entry block of the region.
2877 readState.curBlock = readState.curRegion->begin();
2878 return parseBlockHeader(reader, readState);
2879}
2880
2881LogicalResult
2882BytecodeReader::Impl::parseBlockHeader(EncodingReader &reader,
2883 RegionReadState &readState) {
2884 bool hasArgs;
2885 if (failed(Result: reader.parseVarIntWithFlag(result&: readState.numOpsRemaining, flag&: hasArgs)))
2886 return failure();
2887
2888 // Parse the arguments of the block.
2889 if (hasArgs && failed(Result: parseBlockArguments(reader, block: &*readState.curBlock)))
2890 return failure();
2891
2892 // Uselist orders are available since version 3 of the bytecode.
2893 if (version < bytecode::kUseListOrdering)
2894 return success();
2895
2896 uint8_t hasUseListOrders = 0;
2897 if (hasArgs && failed(Result: reader.parseByte(value&: hasUseListOrders)))
2898 return failure();
2899
2900 if (!hasUseListOrders)
2901 return success();
2902
2903 Block &blk = *readState.curBlock;
2904 auto argIdxToUseListMap =
2905 parseUseListOrderForRange(reader, numResults: blk.getNumArguments());
2906 if (failed(Result: argIdxToUseListMap) || argIdxToUseListMap->empty())
2907 return failure();
2908
2909 for (size_t idx = 0; idx < blk.getNumArguments(); idx++)
2910 if (argIdxToUseListMap->contains(Val: idx))
2911 valueToUseListMap.try_emplace(Key: blk.getArgument(i: idx).getAsOpaquePointer(),
2912 Args&: argIdxToUseListMap->at(Val: idx));
2913
2914 // We don't parse the operations of the block here, that's done elsewhere.
2915 return success();
2916}
2917
2918LogicalResult BytecodeReader::Impl::parseBlockArguments(EncodingReader &reader,
2919 Block *block) {
2920 // Parse the value ID for the first argument, and the number of arguments.
2921 uint64_t numArgs;
2922 if (failed(Result: reader.parseVarInt(result&: numArgs)))
2923 return failure();
2924
2925 SmallVector<Type> argTypes;
2926 SmallVector<Location> argLocs;
2927 argTypes.reserve(N: numArgs);
2928 argLocs.reserve(N: numArgs);
2929
2930 Location unknownLoc = UnknownLoc::get(context: config.getContext());
2931 while (numArgs--) {
2932 Type argType;
2933 LocationAttr argLoc = unknownLoc;
2934 if (version >= bytecode::kElideUnknownBlockArgLocation) {
2935 // Parse the type with hasLoc flag to determine if it has type.
2936 uint64_t typeIdx;
2937 bool hasLoc;
2938 if (failed(Result: reader.parseVarIntWithFlag(result&: typeIdx, flag&: hasLoc)) ||
2939 !(argType = attrTypeReader.resolveType(index: typeIdx)))
2940 return failure();
2941 if (hasLoc && failed(Result: parseAttribute(reader, result&: argLoc)))
2942 return failure();
2943 } else {
2944 // All args has type and location.
2945 if (failed(Result: parseType(reader, result&: argType)) ||
2946 failed(Result: parseAttribute(reader, result&: argLoc)))
2947 return failure();
2948 }
2949 argTypes.push_back(Elt: argType);
2950 argLocs.push_back(Elt: argLoc);
2951 }
2952 block->addArguments(types: argTypes, locs: argLocs);
2953 return defineValues(reader, values: block->getArguments());
2954}
2955
2956//===----------------------------------------------------------------------===//
2957// Value Processing
2958//===----------------------------------------------------------------------===//
2959
2960Value BytecodeReader::Impl::parseOperand(EncodingReader &reader) {
2961 std::vector<Value> &values = valueScopes.back().values;
2962 Value *value = nullptr;
2963 if (failed(Result: parseEntry(reader, entries&: values, entry&: value, entryStr: "value")))
2964 return Value();
2965
2966 // Create a new forward reference if necessary.
2967 if (!*value)
2968 *value = createForwardRef();
2969 return *value;
2970}
2971
2972LogicalResult BytecodeReader::Impl::defineValues(EncodingReader &reader,
2973 ValueRange newValues) {
2974 ValueScope &valueScope = valueScopes.back();
2975 std::vector<Value> &values = valueScope.values;
2976
2977 unsigned &valueID = valueScope.nextValueIDs.back();
2978 unsigned valueIDEnd = valueID + newValues.size();
2979 if (valueIDEnd > values.size()) {
2980 return reader.emitError(
2981 args: "value index range was outside of the expected range for "
2982 "the parent region, got [",
2983 args&: valueID, args: ", ", args&: valueIDEnd, args: "), but the maximum index was ",
2984 args: values.size() - 1);
2985 }
2986
2987 // Assign the values and update any forward references.
2988 for (unsigned i = 0, e = newValues.size(); i != e; ++i, ++valueID) {
2989 Value newValue = newValues[i];
2990
2991 // Check to see if a definition for this value already exists.
2992 if (Value oldValue = std::exchange(obj&: values[valueID], new_val&: newValue)) {
2993 Operation *forwardRefOp = oldValue.getDefiningOp();
2994
2995 // Assert that this is a forward reference operation. Given how we compute
2996 // definition ids (incrementally as we parse), it shouldn't be possible
2997 // for the value to be defined any other way.
2998 assert(forwardRefOp && forwardRefOp->getBlock() == &forwardRefOps &&
2999 "value index was already defined?");
3000
3001 oldValue.replaceAllUsesWith(newValue);
3002 forwardRefOp->moveBefore(block: &openForwardRefOps, iterator: openForwardRefOps.end());
3003 }
3004 }
3005 return success();
3006}
3007
3008Value BytecodeReader::Impl::createForwardRef() {
3009 // Check for an available existing operation to use. Otherwise, create a new
3010 // fake operation to use for the reference.
3011 if (!openForwardRefOps.empty()) {
3012 Operation *op = &openForwardRefOps.back();
3013 op->moveBefore(block: &forwardRefOps, iterator: forwardRefOps.end());
3014 } else {
3015 forwardRefOps.push_back(op: Operation::create(state: forwardRefOpState));
3016 }
3017 return forwardRefOps.back().getResult(idx: 0);
3018}
3019
3020//===----------------------------------------------------------------------===//
3021// Entry Points
3022//===----------------------------------------------------------------------===//
3023
3024BytecodeReader::~BytecodeReader() { assert(getNumOpsToMaterialize() == 0); }
3025
3026BytecodeReader::BytecodeReader(
3027 llvm::MemoryBufferRef buffer, const ParserConfig &config, bool lazyLoading,
3028 const std::shared_ptr<llvm::SourceMgr> &bufferOwnerRef) {
3029 Location sourceFileLoc =
3030 FileLineColLoc::get(context: config.getContext(), fileName: buffer.getBufferIdentifier(),
3031 /*line=*/0, /*column=*/0);
3032 impl = std::make_unique<Impl>(args&: sourceFileLoc, args: config, args&: lazyLoading, args&: buffer,
3033 args: bufferOwnerRef);
3034}
3035
3036LogicalResult BytecodeReader::readTopLevel(
3037 Block *block, llvm::function_ref<bool(Operation *)> lazyOpsCallback) {
3038 return impl->read(block, lazyOpsCallback);
3039}
3040
3041int64_t BytecodeReader::getNumOpsToMaterialize() const {
3042 return impl->getNumOpsToMaterialize();
3043}
3044
3045bool BytecodeReader::isMaterializable(Operation *op) {
3046 return impl->isMaterializable(op);
3047}
3048
3049LogicalResult BytecodeReader::materialize(
3050 Operation *op, llvm::function_ref<bool(Operation *)> lazyOpsCallback) {
3051 return impl->materialize(op, lazyOpsCallback);
3052}
3053
3054LogicalResult
3055BytecodeReader::finalize(function_ref<bool(Operation *)> shouldMaterialize) {
3056 return impl->finalize(shouldMaterialize);
3057}
3058
3059bool mlir::isBytecode(llvm::MemoryBufferRef buffer) {
3060 return buffer.getBuffer().starts_with(Prefix: "ML\xefR");
3061}
3062
3063/// Read the bytecode from the provided memory buffer reference.
3064/// `bufferOwnerRef` if provided is the owning source manager for the buffer,
3065/// and may be used to extend the lifetime of the buffer.
3066static LogicalResult
3067readBytecodeFileImpl(llvm::MemoryBufferRef buffer, Block *block,
3068 const ParserConfig &config,
3069 const std::shared_ptr<llvm::SourceMgr> &bufferOwnerRef) {
3070 Location sourceFileLoc =
3071 FileLineColLoc::get(context: config.getContext(), fileName: buffer.getBufferIdentifier(),
3072 /*line=*/0, /*column=*/0);
3073 if (!isBytecode(buffer)) {
3074 return emitError(loc: sourceFileLoc,
3075 message: "input buffer is not an MLIR bytecode file");
3076 }
3077
3078 BytecodeReader::Impl reader(sourceFileLoc, config, /*lazyLoading=*/false,
3079 buffer, bufferOwnerRef);
3080 return reader.read(block, /*lazyOpsCallback=*/nullptr);
3081}
3082
3083LogicalResult mlir::readBytecodeFile(llvm::MemoryBufferRef buffer, Block *block,
3084 const ParserConfig &config) {
3085 return readBytecodeFileImpl(buffer, block, config, /*bufferOwnerRef=*/{});
3086}
3087LogicalResult
3088mlir::readBytecodeFile(const std::shared_ptr<llvm::SourceMgr> &sourceMgr,
3089 Block *block, const ParserConfig &config) {
3090 return readBytecodeFileImpl(
3091 buffer: *sourceMgr->getMemoryBuffer(i: sourceMgr->getMainFileID()), block, config,
3092 bufferOwnerRef: sourceMgr);
3093}
3094