1//===- TestBytecodeCallbacks.cpp - Pass to test bytecode callback hooks --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "TestOpsCore.h"
10#include "mlir/Bytecode/BytecodeReader.h"
11#include "mlir/Bytecode/BytecodeWriter.h"
12#include "mlir/IR/BuiltinOps.h"
13#include "mlir/IR/OperationSupport.h"
14#include "mlir/Parser/Parser.h"
15#include "mlir/Pass/Pass.h"
16#include "llvm/Support/CommandLine.h"
17#include "llvm/Support/MemoryBufferRef.h"
18#include "llvm/Support/raw_ostream.h"
19#include <list>
20
21using namespace mlir;
22using namespace llvm;
23
24namespace {
25class TestDialectVersionParser : public cl::parser<test::TestDialectVersion> {
26public:
27 TestDialectVersionParser(cl::Option &o)
28 : cl::parser<test::TestDialectVersion>(o) {}
29
30 bool parse(cl::Option &o, StringRef /*argName*/, StringRef arg,
31 test::TestDialectVersion &v) {
32 long long major, minor;
33 if (getAsSignedInteger(Str: arg.split(Separator: ".").first, Radix: 10, Result&: major))
34 return o.error(Message: "Invalid argument '" + arg);
35 if (getAsSignedInteger(Str: arg.split(Separator: ".").second, Radix: 10, Result&: minor))
36 return o.error(Message: "Invalid argument '" + arg);
37 v = test::TestDialectVersion(major, minor);
38 // Returns true on error.
39 return false;
40 }
41 static void print(raw_ostream &os, const test::TestDialectVersion &v) {
42 os << v.major_ << "." << v.minor_;
43 };
44};
45
46/// This is a test pass which uses callbacks to encode attributes and types in a
47/// custom fashion.
48struct TestBytecodeRoundtripPass
49 : public PassWrapper<TestBytecodeRoundtripPass, OperationPass<ModuleOp>> {
50 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestBytecodeRoundtripPass)
51
52 StringRef getArgument() const final { return "test-bytecode-roundtrip"; }
53 StringRef getDescription() const final {
54 return "Test pass to implement bytecode roundtrip tests.";
55 }
56 void getDependentDialects(DialectRegistry &registry) const override {
57 registry.insert<test::TestDialect>();
58 }
59 TestBytecodeRoundtripPass() = default;
60 TestBytecodeRoundtripPass(const TestBytecodeRoundtripPass &) {}
61
62 LogicalResult initialize(MLIRContext *context) override {
63 testDialect = context->getOrLoadDialect<test::TestDialect>();
64 return success();
65 }
66
67 void runOnOperation() override {
68 switch (testKind) {
69 // Tests 0-5 implement a custom roundtrip with callbacks.
70 case (0):
71 return runTest0(op: getOperation());
72 case (1):
73 return runTest1(op: getOperation());
74 case (2):
75 return runTest2(op: getOperation());
76 case (3):
77 return runTest3(op: getOperation());
78 case (4):
79 return runTest4(op: getOperation());
80 case (5):
81 return runTest5(op: getOperation());
82 case (6):
83 // test-kind 6 is a plain roundtrip with downgrade/upgrade to/from
84 // `targetVersion`.
85 return runTest6(op: getOperation());
86 case (7):
87 return runTest7(op: getOperation());
88 default:
89 llvm_unreachable("unhandled test kind for TestBytecodeCallbacks pass");
90 }
91 }
92
93 mlir::Pass::Option<test::TestDialectVersion, TestDialectVersionParser>
94 targetVersion{*this, "test-dialect-version",
95 llvm::cl::desc(
96 "Specifies the test dialect version to emit and parse"),
97 cl::init(Val: test::TestDialectVersion())};
98
99 mlir::Pass::Option<int> testKind{
100 *this, "test-kind", llvm::cl::desc("Specifies the test kind to execute"),
101 cl::init(Val: 0)};
102
103private:
104 void doRoundtripWithConfigs(Operation *op,
105 const BytecodeWriterConfig &writeConfig,
106 const ParserConfig &parseConfig) {
107 std::string bytecode;
108 llvm::raw_string_ostream os(bytecode);
109 if (failed(Result: writeBytecodeToFile(op, os, config: writeConfig))) {
110 op->emitError() << "failed to write bytecode\n";
111 signalPassFailure();
112 return;
113 }
114 auto newModuleOp = parseSourceString(sourceStr: StringRef(bytecode), config: parseConfig);
115 if (!newModuleOp.get()) {
116 op->emitError() << "failed to read bytecode\n";
117 signalPassFailure();
118 return;
119 }
120 // Print the module to the output stream, so that we can filecheck the
121 // result.
122 newModuleOp->print(os&: llvm::outs());
123 }
124
125 // Test0: let's assume that versions older than 2.0 were relying on a special
126 // integer attribute of a deprecated dialect called "funky". Assume that its
127 // encoding was made by two varInts, the first was the ID (999) and the second
128 // contained width and signedness info. We can emit it using a callback
129 // writing a custom encoding for the "funky" dialect group, and parse it back
130 // with a custom parser reading the same encoding in the same dialect group.
131 // Note that the ID 999 does not correspond to a valid integer type in the
132 // current encodings of builtin types.
133 void runTest0(Operation *op) {
134 auto newCtx = std::make_shared<MLIRContext>();
135 test::TestDialectVersion targetEmissionVersion = targetVersion;
136 BytecodeWriterConfig writeConfig;
137 // Set the emission version for the test dialect.
138 writeConfig.setDialectVersion<test::TestDialect>(
139 std::make_unique<test::TestDialectVersion>(args&: targetEmissionVersion));
140 writeConfig.attachTypeCallback(
141 emitFn: [&](Type entryValue, std::optional<StringRef> &dialectGroupName,
142 DialectBytecodeWriter &writer) -> LogicalResult {
143 // Do not override anything if version greater than 2.0.
144 auto versionOr = writer.getDialectVersion<test::TestDialect>();
145 assert(succeeded(versionOr) && "expected reader to be able to access "
146 "the version for test dialect");
147 const auto *version =
148 reinterpret_cast<const test::TestDialectVersion *>(*versionOr);
149 if (version->major_ >= 2)
150 return failure();
151
152 // For version less than 2.0, override the encoding of IntegerType.
153 if (auto type = llvm::dyn_cast<IntegerType>(Val&: entryValue)) {
154 llvm::outs() << "Overriding IntegerType encoding...\n";
155 dialectGroupName = StringLiteral("funky");
156 writer.writeVarInt(/* IntegerType */ value: 999);
157 writer.writeVarInt(value: type.getWidth() << 2 | type.getSignedness());
158 return success();
159 }
160 return failure();
161 });
162 newCtx->appendDialectRegistry(registry: op->getContext()->getDialectRegistry());
163 newCtx->allowUnregisteredDialects();
164 ParserConfig parseConfig(newCtx.get(), /*verifyAfterParse=*/true);
165 parseConfig.getBytecodeReaderConfig().attachTypeCallback(
166 parserFn: [&](DialectBytecodeReader &reader, StringRef dialectName,
167 Type &entry) -> LogicalResult {
168 // Get test dialect version from the version map. If the test dialect
169 // is not present in the bytecode (e.g., the module contains no test
170 // dialect types), version info will be absent -- just skip.
171 auto versionOr = reader.getDialectVersion<test::TestDialect>();
172 if (failed(Result: versionOr))
173 return success();
174 const auto *version =
175 reinterpret_cast<const test::TestDialectVersion *>(*versionOr);
176 if (version->major_ >= 2)
177 return success();
178
179 // `dialectName` is the name of the group we have the opportunity to
180 // override. In this case, override only the dialect group "funky",
181 // for which does not exist in memory.
182 if (dialectName != StringLiteral("funky"))
183 return success();
184
185 uint64_t encoding;
186 if (failed(Result: reader.readVarInt(result&: encoding)) || encoding != 999)
187 return success();
188 llvm::outs() << "Overriding parsing of IntegerType encoding...\n";
189 uint64_t widthAndSignedness, width;
190 IntegerType::SignednessSemantics signedness;
191 if (succeeded(Result: reader.readVarInt(result&: widthAndSignedness)) &&
192 ((width = widthAndSignedness >> 2), true) &&
193 ((signedness = static_cast<IntegerType::SignednessSemantics>(
194 widthAndSignedness & 0x3)),
195 true))
196 entry = IntegerType::get(context: reader.getContext(), width, signedness);
197 // Return nullopt to fall through the rest of the parsing code path.
198 return success();
199 });
200 doRoundtripWithConfigs(op, writeConfig, parseConfig);
201 }
202
203 // Test1: When writing bytecode, we override the encoding of TestI32Type with
204 // the encoding of builtin IntegerType. We can natively parse this without
205 // the use of a callback, relying on the existing builtin reader mechanism.
206 void runTest1(Operation *op) {
207 auto *builtin = op->getContext()->getLoadedDialect<mlir::BuiltinDialect>();
208 BytecodeDialectInterface *iface =
209 builtin->getRegisteredInterface<BytecodeDialectInterface>();
210 BytecodeWriterConfig writeConfig;
211 writeConfig.attachTypeCallback(
212 emitFn: [&](Type entryValue, std::optional<StringRef> &dialectGroupName,
213 DialectBytecodeWriter &writer) -> LogicalResult {
214 // Emit TestIntegerType using the builtin dialect encoding.
215 if (llvm::isa<test::TestI32Type>(entryValue)) {
216 llvm::outs() << "Overriding TestI32Type encoding...\n";
217 auto builtinI32Type =
218 IntegerType::get(context: op->getContext(), width: 32,
219 signedness: IntegerType::SignednessSemantics::Signless);
220 // Specify that this type will need to be written as part of the
221 // builtin group. This will override the default dialect group of
222 // the attribute (test).
223 dialectGroupName = StringLiteral("builtin");
224 if (succeeded(Result: iface->writeType(type: builtinI32Type, writer)))
225 return success();
226 }
227 return failure();
228 });
229 // We natively parse the attribute as a builtin, so no callback needed.
230 ParserConfig parseConfig(op->getContext(), /*verifyAfterParse=*/true);
231 doRoundtripWithConfigs(op, writeConfig, parseConfig);
232 }
233
234 // Test2: When writing bytecode, we write standard builtin IntegerTypes. At
235 // parsing, we use the encoding of IntegerType to intercept all i32. Then,
236 // instead of creating i32s, we assemble TestI32Type and return it.
237 void runTest2(Operation *op) {
238 auto *builtin = op->getContext()->getLoadedDialect<mlir::BuiltinDialect>();
239 BytecodeDialectInterface *iface =
240 builtin->getRegisteredInterface<BytecodeDialectInterface>();
241 BytecodeWriterConfig writeConfig;
242 ParserConfig parseConfig(op->getContext(), /*verifyAfterParse=*/true);
243 parseConfig.getBytecodeReaderConfig().attachTypeCallback(
244 parserFn: [&](DialectBytecodeReader &reader, StringRef dialectName,
245 Type &entry) -> LogicalResult {
246 if (dialectName != StringLiteral("builtin"))
247 return success();
248 Type builtinAttr = iface->readType(reader);
249 if (auto integerType =
250 llvm::dyn_cast_or_null<IntegerType>(Val&: builtinAttr)) {
251 if (integerType.getWidth() == 32 && integerType.isSignless()) {
252 llvm::outs() << "Overriding parsing of TestI32Type encoding...\n";
253 entry = test::TestI32Type::get(reader.getContext());
254 }
255 }
256 return success();
257 });
258 doRoundtripWithConfigs(op, writeConfig, parseConfig);
259 }
260
261 // Test3: When writing bytecode, we override the encoding of
262 // TestAttrParamsAttr with the encoding of builtin DenseIntElementsAttr. We
263 // can natively parse this without the use of a callback, relying on the
264 // existing builtin reader mechanism.
265 void runTest3(Operation *op) {
266 auto *builtin = op->getContext()->getLoadedDialect<mlir::BuiltinDialect>();
267 BytecodeDialectInterface *iface =
268 builtin->getRegisteredInterface<BytecodeDialectInterface>();
269 auto i32Type = IntegerType::get(context: op->getContext(), width: 32,
270 signedness: IntegerType::SignednessSemantics::Signless);
271 BytecodeWriterConfig writeConfig;
272 writeConfig.attachAttributeCallback(
273 emitFn: [&](Attribute entryValue, std::optional<StringRef> &dialectGroupName,
274 DialectBytecodeWriter &writer) -> LogicalResult {
275 // Emit TestIntegerType using the builtin dialect encoding.
276 if (auto testParamAttrs =
277 llvm::dyn_cast<test::TestAttrParamsAttr>(entryValue)) {
278 llvm::outs() << "Overriding TestAttrParamsAttr encoding...\n";
279 // Specify that this attribute will need to be written as part of
280 // the builtin group. This will override the default dialect group
281 // of the attribute (test).
282 dialectGroupName = StringLiteral("builtin");
283 auto denseAttr = DenseIntElementsAttr::get(
284 RankedTensorType::get(shape: {2}, elementType: i32Type),
285 {testParamAttrs.getV0(), testParamAttrs.getV1()});
286 if (succeeded(iface->writeAttribute(attribute: denseAttr, writer)))
287 return success();
288 }
289 return failure();
290 });
291 // We natively parse the attribute as a builtin, so no callback needed.
292 ParserConfig parseConfig(op->getContext(), /*verifyAfterParse=*/false);
293 doRoundtripWithConfigs(op, writeConfig, parseConfig);
294 }
295
296 // Test4: When writing bytecode, we write standard builtin
297 // DenseIntElementsAttr. At parsing, we use the encoding of
298 // DenseIntElementsAttr to intercept all ElementsAttr that have shaped type of
299 // <2xi32>. Instead of assembling a DenseIntElementsAttr, we assemble
300 // TestAttrParamsAttr and return it.
301 void runTest4(Operation *op) {
302 auto *builtin = op->getContext()->getLoadedDialect<mlir::BuiltinDialect>();
303 BytecodeDialectInterface *iface =
304 builtin->getRegisteredInterface<BytecodeDialectInterface>();
305 auto i32Type = IntegerType::get(context: op->getContext(), width: 32,
306 signedness: IntegerType::SignednessSemantics::Signless);
307 BytecodeWriterConfig writeConfig;
308 ParserConfig parseConfig(op->getContext(), /*verifyAfterParse=*/false);
309 parseConfig.getBytecodeReaderConfig().attachAttributeCallback(
310 parserFn: [&](DialectBytecodeReader &reader, StringRef dialectName,
311 Attribute &entry) -> LogicalResult {
312 // Override only the case where the return type of the builtin reader
313 // is an i32 and fall through on all the other cases, since we want to
314 // still use TestDialect normal codepath to parse the other types.
315 Attribute builtinAttr = iface->readAttribute(reader);
316 if (auto denseAttr =
317 llvm::dyn_cast_or_null<DenseIntElementsAttr>(Val&: builtinAttr)) {
318 if (denseAttr.getType().getShape() == ArrayRef<int64_t>(2) &&
319 denseAttr.getElementType() == i32Type) {
320 llvm::outs()
321 << "Overriding parsing of TestAttrParamsAttr encoding...\n";
322 int v0 = denseAttr.getValues<IntegerAttr>()[0].getInt();
323 int v1 = denseAttr.getValues<IntegerAttr>()[1].getInt();
324 entry =
325 test::TestAttrParamsAttr::get(reader.getContext(), v0, v1);
326 }
327 }
328 return success();
329 });
330 doRoundtripWithConfigs(op, writeConfig, parseConfig);
331 }
332
333 // Test5: When writing bytecode, we want TestDialect to use nothing else than
334 // the builtin types and attributes and take full control of the encoding,
335 // returning failure if any type or attribute is not part of builtin.
336 void runTest5(Operation *op) {
337 auto *builtin = op->getContext()->getLoadedDialect<mlir::BuiltinDialect>();
338 BytecodeDialectInterface *iface =
339 builtin->getRegisteredInterface<BytecodeDialectInterface>();
340 BytecodeWriterConfig writeConfig;
341 writeConfig.attachAttributeCallback(
342 emitFn: [&](Attribute attr, std::optional<StringRef> &dialectGroupName,
343 DialectBytecodeWriter &writer) -> LogicalResult {
344 return iface->writeAttribute(attribute: attr, writer);
345 });
346 writeConfig.attachTypeCallback(
347 emitFn: [&](Type type, std::optional<StringRef> &dialectGroupName,
348 DialectBytecodeWriter &writer) -> LogicalResult {
349 return iface->writeType(type, writer);
350 });
351 ParserConfig parseConfig(op->getContext(), /*verifyAfterParse=*/false);
352 parseConfig.getBytecodeReaderConfig().attachAttributeCallback(
353 parserFn: [&](DialectBytecodeReader &reader, StringRef dialectName,
354 Attribute &entry) -> LogicalResult {
355 Attribute builtinAttr = iface->readAttribute(reader);
356 if (!builtinAttr)
357 return failure();
358 entry = builtinAttr;
359 return success();
360 });
361 parseConfig.getBytecodeReaderConfig().attachTypeCallback(
362 parserFn: [&](DialectBytecodeReader &reader, StringRef dialectName,
363 Type &entry) -> LogicalResult {
364 Type builtinType = iface->readType(reader);
365 if (!builtinType) {
366 return failure();
367 }
368 entry = builtinType;
369 return success();
370 });
371 doRoundtripWithConfigs(op, writeConfig, parseConfig);
372 }
373
374 LogicalResult downgradeToVersion(Operation *op,
375 const test::TestDialectVersion &version) {
376 if ((version.major_ == 2) && (version.minor_ == 0))
377 return success();
378 if (version.major_ > 2 || (version.major_ == 2 && version.minor_ > 0)) {
379 return op->emitError() << "current test dialect version is 2.0, "
380 "can't downgrade to version: "
381 << version.major_ << "." << version.minor_;
382 }
383 // Prior version 2.0, the old op supported only a single attribute called
384 // "dimensions". We need to check that the modifier is false, otherwise we
385 // can't do the downgrade.
386 auto status = op->walk(callback: [&](test::TestVersionedOpA op) {
387 auto &prop = op.getProperties();
388 if (prop.modifier.getValue()) {
389 op->emitOpError() << "cannot downgrade to version " << version.major_
390 << "." << version.minor_
391 << " since the modifier is not compatible";
392 return WalkResult::interrupt();
393 }
394 llvm::outs() << "downgrading op...\n";
395 return WalkResult::advance();
396 });
397 return failure(IsFailure: status.wasInterrupted());
398 }
399
400 // Test6: Downgrade IR to `targetVersion`, write to bytecode. Then, read and
401 // upgrade IR when back in memory. The module is expected to be unmodified at
402 // the end of the function.
403 void runTest6(Operation *op) {
404 test::TestDialectVersion targetEmissionVersion = targetVersion;
405
406 // Downgrade IR constructs before writing the IR to bytecode.
407 auto status = downgradeToVersion(op, version: targetEmissionVersion);
408 assert(succeeded(status) && "expected the downgrade to succeed");
409 (void)status;
410
411 BytecodeWriterConfig writeConfig;
412 writeConfig.setDialectVersion<test::TestDialect>(
413 std::make_unique<test::TestDialectVersion>(args&: targetEmissionVersion));
414 ParserConfig parseConfig(op->getContext(), /*verifyAfterParse=*/true);
415 doRoundtripWithConfigs(op, writeConfig, parseConfig);
416 }
417
418 // Test7: When writing bytecode, we override the encoding of TestI32Type with
419 // the encoding of builtin IntegerType, but we also write an unowned blob.
420 // We can natively parse this without the use of a callback, relying on the
421 // existing builtin reader mechanism.
422 void runTest7(Operation *op) {
423 auto *builtin = op->getContext()->getLoadedDialect<mlir::BuiltinDialect>();
424 BytecodeDialectInterface *iface =
425 builtin->getRegisteredInterface<BytecodeDialectInterface>();
426 BytecodeWriterConfig writeConfig;
427 writeConfig.attachTypeCallback(
428 emitFn: [&](Type entryValue, std::optional<StringRef> &dialectGroupName,
429 DialectBytecodeWriter &writer) -> LogicalResult {
430 // Emit TestIntegerType using the builtin dialect encoding.
431 if (llvm::isa<test::TestI32Type>(entryValue)) {
432 auto builtinI32Type =
433 IntegerType::get(context: op->getContext(), width: 32,
434 signedness: IntegerType::SignednessSemantics::Signless);
435 // Specify that this type will need to be written as part of the
436 // builtin group. This will override the default dialect group of
437 // the attribute (test).
438 dialectGroupName = StringLiteral("builtin");
439 if (succeeded(Result: iface->writeType(type: builtinI32Type, writer))) {
440 char dummyBlob[] = "test_blob";
441 llvm::outs() << "Writing unowned blob...\n";
442 writer.writeUnownedBlob(blob: ArrayRef<char>(dummyBlob, 9));
443 return success();
444 }
445 }
446 return failure();
447 });
448 ParserConfig parseConfig(op->getContext(), /*verifyAfterParse=*/true);
449 parseConfig.getBytecodeReaderConfig().attachTypeCallback(
450 parserFn: [&](DialectBytecodeReader &reader, StringRef dialectName,
451 Type &entry) -> LogicalResult {
452 if (dialectName != StringLiteral("builtin"))
453 return failure();
454 Type builtinAttr = iface->readType(reader);
455 if (auto integerType =
456 llvm::dyn_cast_or_null<IntegerType>(Val&: builtinAttr)) {
457 if (integerType.getWidth() == 32 && integerType.isSignless()) {
458 ArrayRef<char> blob;
459 if (succeeded(Result: reader.readBlob(result&: blob)) &&
460 blob == ArrayRef<char>("test_blob", 9)) {
461 llvm::outs() << "Successfully read the unowned blob.\n";
462 entry = test::TestI32Type::get(reader.getContext());
463 }
464 }
465 }
466 return success();
467 });
468 doRoundtripWithConfigs(op, writeConfig, parseConfig);
469 }
470
471 test::TestDialect *testDialect;
472};
473} // namespace
474
475namespace mlir {
476void registerTestBytecodeRoundtripPasses() {
477 PassRegistration<TestBytecodeRoundtripPass>();
478}
479} // namespace mlir
480