1//===- TestEmulateNarrowType.cpp - Test Narrow Type Emulation ------*- c++
2//-*-===//
3//
4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5// See https://llvm.org/LICENSE.txt for license information.
6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7//
8//===----------------------------------------------------------------------===//
9
10#include "mlir/Dialect/Affine/IR/AffineDialect.h"
11#include "mlir/Dialect/Arith/IR/Arith.h"
12#include "mlir/Dialect/Arith/Transforms/NarrowTypeEmulationConverter.h"
13#include "mlir/Dialect/Arith/Transforms/Passes.h"
14#include "mlir/Dialect/ControlFlow/IR/ControlFlow.h"
15#include "mlir/Dialect/ControlFlow/Transforms/StructuralTypeConversions.h"
16#include "mlir/Dialect/Func/IR/FuncOps.h"
17#include "mlir/Dialect/MemRef/IR/MemRef.h"
18#include "mlir/Dialect/MemRef/Transforms/Transforms.h"
19#include "mlir/Dialect/Vector/IR/VectorOps.h"
20#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
21#include "mlir/Pass/Pass.h"
22#include "mlir/Transforms/DialectConversion.h"
23#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
24
25using namespace mlir;
26
27namespace {
28
29struct TestEmulateNarrowTypePass
30 : public PassWrapper<TestEmulateNarrowTypePass,
31 OperationPass<func::FuncOp>> {
32 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestEmulateNarrowTypePass)
33
34 TestEmulateNarrowTypePass() = default;
35 TestEmulateNarrowTypePass(const TestEmulateNarrowTypePass &pass)
36 : PassWrapper(pass) {}
37
38 void getDependentDialects(DialectRegistry &registry) const override {
39 registry.insert<arith::ArithDialect, cf::ControlFlowDialect,
40 func::FuncDialect, memref::MemRefDialect,
41 vector::VectorDialect, affine::AffineDialect>();
42 }
43 StringRef getArgument() const final { return "test-emulate-narrow-int"; }
44 StringRef getDescription() const final {
45 return "Function pass to test Narrow Integer Emulation";
46 }
47
48 void runOnOperation() override {
49 if (!llvm::isPowerOf2_32(Value: loadStoreEmulateBitwidth) ||
50 loadStoreEmulateBitwidth < 8) {
51 signalPassFailure();
52 return;
53 }
54
55 Operation *op = getOperation();
56 MLIRContext *ctx = op->getContext();
57
58 arith::NarrowTypeEmulationConverter typeConverter(loadStoreEmulateBitwidth);
59
60 // Convert scalar type.
61 typeConverter.addConversion(callback: [this](IntegerType ty) -> std::optional<Type> {
62 unsigned width = ty.getWidth();
63 if (width >= arithComputeBitwidth)
64 return ty;
65
66 return IntegerType::get(context: ty.getContext(), width: arithComputeBitwidth);
67 });
68
69 // Convert vector type.
70 typeConverter.addConversion(callback: [this](VectorType ty) -> std::optional<Type> {
71 auto intTy = dyn_cast<IntegerType>(Val: ty.getElementType());
72 if (!intTy)
73 return ty;
74
75 unsigned width = intTy.getWidth();
76 if (width >= arithComputeBitwidth)
77 return ty;
78
79 return VectorType::get(
80 shape: to_vector(Range: ty.getShape()),
81 elementType: IntegerType::get(context: ty.getContext(), width: arithComputeBitwidth));
82 });
83
84 // With the type converter enabled, we are effectively unable to write
85 // negative tests. This is a workaround specifically for negative tests.
86 if (!disableMemrefTypeConversion)
87 memref::populateMemRefNarrowTypeEmulationConversions(typeConverter);
88
89 ConversionTarget target(*ctx);
90 target.addDynamicallyLegalOp<func::FuncOp>(callback: [&typeConverter](Operation *op) {
91 return typeConverter.isLegal(type: cast<func::FuncOp>(Val: op).getFunctionType());
92 });
93 auto opLegalCallback = [&typeConverter](Operation *op) {
94 return typeConverter.isLegal(op);
95 };
96 target.addDynamicallyLegalOp<func::CallOp, func::ReturnOp>(callback: opLegalCallback);
97 target.addDynamicallyLegalDialect<
98 arith::ArithDialect, vector::VectorDialect, memref::MemRefDialect,
99 affine::AffineDialect>(callback: opLegalCallback);
100
101 RewritePatternSet patterns(ctx);
102
103 arith::populateArithNarrowTypeEmulationPatterns(typeConverter, patterns);
104 memref::populateMemRefNarrowTypeEmulationPatterns(
105 typeConverter, patterns, disableAtomicRMW, assumeAligned);
106 vector::populateVectorNarrowTypeEmulationPatterns(
107 typeConverter, patterns, disableAtomicRMW, assumeAligned);
108
109 cf::populateCFStructuralTypeConversionsAndLegality(typeConverter, patterns,
110 target);
111
112 if (failed(Result: applyPartialConversion(op, target, patterns: std::move(patterns))))
113 signalPassFailure();
114 }
115
116 Option<unsigned> loadStoreEmulateBitwidth{
117 *this, "memref-load-bitwidth",
118 llvm::cl::desc("memref load/store emulation bit width"),
119 llvm::cl::init(Val: 8)};
120
121 Option<unsigned> arithComputeBitwidth{
122 *this, "arith-compute-bitwidth",
123 llvm::cl::desc("arith computation bit width"), llvm::cl::init(Val: 4)};
124
125 Option<bool> disableMemrefTypeConversion{
126 *this, "skip-memref-type-conversion",
127 llvm::cl::desc("disable memref type conversion (to test failures)"),
128 llvm::cl::init(Val: false)};
129
130 Option<bool> disableAtomicRMW{
131 *this, "disable-atomic-rmw",
132 llvm::cl::desc("disable atomic read-modify-write and prefer generating "
133 "normal sequence"),
134 llvm::cl::init(Val: false)};
135
136 Option<bool> assumeAligned{
137 *this, "assume-aligned",
138 llvm::cl::desc("assume store offsets are aligned to container element "
139 "boundaries"),
140 llvm::cl::init(Val: false)};
141};
142
143struct TestMemRefFlattenAndVectorNarrowTypeEmulationPass
144 : public PassWrapper<TestMemRefFlattenAndVectorNarrowTypeEmulationPass,
145 OperationPass<func::FuncOp>> {
146 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(
147 TestMemRefFlattenAndVectorNarrowTypeEmulationPass)
148
149 TestMemRefFlattenAndVectorNarrowTypeEmulationPass() = default;
150 TestMemRefFlattenAndVectorNarrowTypeEmulationPass(
151 const TestMemRefFlattenAndVectorNarrowTypeEmulationPass &pass)
152 : PassWrapper(pass) {}
153
154 void getDependentDialects(DialectRegistry &registry) const override {
155 registry
156 .insert<arith::ArithDialect, func::FuncDialect, memref::MemRefDialect,
157 vector::VectorDialect, affine::AffineDialect>();
158 }
159
160 StringRef getArgument() const final {
161 return "test-memref-flatten-and-vector-narrow-type-emulation";
162 }
163
164 StringRef getDescription() const final {
165 return "Test MemRef flattening and vector narrow type emulation patterns";
166 }
167
168 void runOnOperation() override {
169 Operation *op = getOperation();
170 MLIRContext *ctx = &getContext();
171
172 // Create a type converter for narrow type emulation (8-bit)
173 arith::NarrowTypeEmulationConverter typeConverter(8);
174
175 // Add conversions for memref types with i4 elements
176 memref::populateMemRefNarrowTypeEmulationConversions(typeConverter);
177
178 ConversionTarget target(*ctx);
179 target.addDynamicallyLegalOp<func::FuncOp>(callback: [&typeConverter](Operation *op) {
180 return typeConverter.isLegal(type: cast<func::FuncOp>(Val: op).getFunctionType());
181 });
182 auto opLegalCallback = [&typeConverter](Operation *op) {
183 return typeConverter.isLegal(op);
184 };
185 target.addDynamicallyLegalOp<func::CallOp, func::ReturnOp>(callback: opLegalCallback);
186 target.addDynamicallyLegalDialect<
187 arith::ArithDialect, vector::VectorDialect, memref::MemRefDialect,
188 affine::AffineDialect>(callback: opLegalCallback);
189
190 RewritePatternSet patterns(ctx);
191
192 // This is necessary for the purpose of emulating `memref.alloc` and
193 // function boundaries.
194 memref::populateMemRefNarrowTypeEmulationPatterns(typeConverter, patterns);
195
196 vector::populateMemRefFlattenAndVectorNarrowTypeEmulationPatterns(
197 typeConverter, patterns);
198
199 // Apply partial conversion
200 if (failed(Result: applyPartialConversion(op, target, patterns: std::move(patterns))))
201 signalPassFailure();
202 }
203};
204} // namespace
205
206namespace mlir::test {
207void registerTestEmulateNarrowTypePass() {
208 PassRegistration<TestEmulateNarrowTypePass>();
209 PassRegistration<TestMemRefFlattenAndVectorNarrowTypeEmulationPass>();
210}
211} // namespace mlir::test
212