| 1 | //===- TestEmulateNarrowType.cpp - Test Narrow Type Emulation ------*- c++ |
| 2 | //-*-===// |
| 3 | // |
| 4 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 5 | // See https://llvm.org/LICENSE.txt for license information. |
| 6 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | |
| 10 | #include "mlir/Dialect/Affine/IR/AffineDialect.h" |
| 11 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 12 | #include "mlir/Dialect/Arith/Transforms/NarrowTypeEmulationConverter.h" |
| 13 | #include "mlir/Dialect/Arith/Transforms/Passes.h" |
| 14 | #include "mlir/Dialect/ControlFlow/IR/ControlFlow.h" |
| 15 | #include "mlir/Dialect/ControlFlow/Transforms/StructuralTypeConversions.h" |
| 16 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 17 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 18 | #include "mlir/Dialect/MemRef/Transforms/Transforms.h" |
| 19 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 20 | #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h" |
| 21 | #include "mlir/Pass/Pass.h" |
| 22 | #include "mlir/Transforms/DialectConversion.h" |
| 23 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 24 | |
| 25 | using namespace mlir; |
| 26 | |
| 27 | namespace { |
| 28 | |
| 29 | struct TestEmulateNarrowTypePass |
| 30 | : public PassWrapper<TestEmulateNarrowTypePass, |
| 31 | OperationPass<func::FuncOp>> { |
| 32 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestEmulateNarrowTypePass) |
| 33 | |
| 34 | TestEmulateNarrowTypePass() = default; |
| 35 | TestEmulateNarrowTypePass(const TestEmulateNarrowTypePass &pass) |
| 36 | : PassWrapper(pass) {} |
| 37 | |
| 38 | void getDependentDialects(DialectRegistry ®istry) const override { |
| 39 | registry.insert<arith::ArithDialect, cf::ControlFlowDialect, |
| 40 | func::FuncDialect, memref::MemRefDialect, |
| 41 | vector::VectorDialect, affine::AffineDialect>(); |
| 42 | } |
| 43 | StringRef getArgument() const final { return "test-emulate-narrow-int" ; } |
| 44 | StringRef getDescription() const final { |
| 45 | return "Function pass to test Narrow Integer Emulation" ; |
| 46 | } |
| 47 | |
| 48 | void runOnOperation() override { |
| 49 | if (!llvm::isPowerOf2_32(Value: loadStoreEmulateBitwidth) || |
| 50 | loadStoreEmulateBitwidth < 8) { |
| 51 | signalPassFailure(); |
| 52 | return; |
| 53 | } |
| 54 | |
| 55 | Operation *op = getOperation(); |
| 56 | MLIRContext *ctx = op->getContext(); |
| 57 | |
| 58 | arith::NarrowTypeEmulationConverter typeConverter(loadStoreEmulateBitwidth); |
| 59 | |
| 60 | // Convert scalar type. |
| 61 | typeConverter.addConversion(callback: [this](IntegerType ty) -> std::optional<Type> { |
| 62 | unsigned width = ty.getWidth(); |
| 63 | if (width >= arithComputeBitwidth) |
| 64 | return ty; |
| 65 | |
| 66 | return IntegerType::get(context: ty.getContext(), width: arithComputeBitwidth); |
| 67 | }); |
| 68 | |
| 69 | // Convert vector type. |
| 70 | typeConverter.addConversion(callback: [this](VectorType ty) -> std::optional<Type> { |
| 71 | auto intTy = dyn_cast<IntegerType>(Val: ty.getElementType()); |
| 72 | if (!intTy) |
| 73 | return ty; |
| 74 | |
| 75 | unsigned width = intTy.getWidth(); |
| 76 | if (width >= arithComputeBitwidth) |
| 77 | return ty; |
| 78 | |
| 79 | return VectorType::get( |
| 80 | shape: to_vector(Range: ty.getShape()), |
| 81 | elementType: IntegerType::get(context: ty.getContext(), width: arithComputeBitwidth)); |
| 82 | }); |
| 83 | |
| 84 | // With the type converter enabled, we are effectively unable to write |
| 85 | // negative tests. This is a workaround specifically for negative tests. |
| 86 | if (!disableMemrefTypeConversion) |
| 87 | memref::populateMemRefNarrowTypeEmulationConversions(typeConverter); |
| 88 | |
| 89 | ConversionTarget target(*ctx); |
| 90 | target.addDynamicallyLegalOp<func::FuncOp>(callback: [&typeConverter](Operation *op) { |
| 91 | return typeConverter.isLegal(type: cast<func::FuncOp>(Val: op).getFunctionType()); |
| 92 | }); |
| 93 | auto opLegalCallback = [&typeConverter](Operation *op) { |
| 94 | return typeConverter.isLegal(op); |
| 95 | }; |
| 96 | target.addDynamicallyLegalOp<func::CallOp, func::ReturnOp>(callback: opLegalCallback); |
| 97 | target.addDynamicallyLegalDialect< |
| 98 | arith::ArithDialect, vector::VectorDialect, memref::MemRefDialect, |
| 99 | affine::AffineDialect>(callback: opLegalCallback); |
| 100 | |
| 101 | RewritePatternSet patterns(ctx); |
| 102 | |
| 103 | arith::populateArithNarrowTypeEmulationPatterns(typeConverter, patterns); |
| 104 | memref::populateMemRefNarrowTypeEmulationPatterns( |
| 105 | typeConverter, patterns, disableAtomicRMW, assumeAligned); |
| 106 | vector::populateVectorNarrowTypeEmulationPatterns( |
| 107 | typeConverter, patterns, disableAtomicRMW, assumeAligned); |
| 108 | |
| 109 | cf::populateCFStructuralTypeConversionsAndLegality(typeConverter, patterns, |
| 110 | target); |
| 111 | |
| 112 | if (failed(Result: applyPartialConversion(op, target, patterns: std::move(patterns)))) |
| 113 | signalPassFailure(); |
| 114 | } |
| 115 | |
| 116 | Option<unsigned> loadStoreEmulateBitwidth{ |
| 117 | *this, "memref-load-bitwidth" , |
| 118 | llvm::cl::desc("memref load/store emulation bit width" ), |
| 119 | llvm::cl::init(Val: 8)}; |
| 120 | |
| 121 | Option<unsigned> arithComputeBitwidth{ |
| 122 | *this, "arith-compute-bitwidth" , |
| 123 | llvm::cl::desc("arith computation bit width" ), llvm::cl::init(Val: 4)}; |
| 124 | |
| 125 | Option<bool> disableMemrefTypeConversion{ |
| 126 | *this, "skip-memref-type-conversion" , |
| 127 | llvm::cl::desc("disable memref type conversion (to test failures)" ), |
| 128 | llvm::cl::init(Val: false)}; |
| 129 | |
| 130 | Option<bool> disableAtomicRMW{ |
| 131 | *this, "disable-atomic-rmw" , |
| 132 | llvm::cl::desc("disable atomic read-modify-write and prefer generating " |
| 133 | "normal sequence" ), |
| 134 | llvm::cl::init(Val: false)}; |
| 135 | |
| 136 | Option<bool> assumeAligned{ |
| 137 | *this, "assume-aligned" , |
| 138 | llvm::cl::desc("assume store offsets are aligned to container element " |
| 139 | "boundaries" ), |
| 140 | llvm::cl::init(Val: false)}; |
| 141 | }; |
| 142 | |
| 143 | struct TestMemRefFlattenAndVectorNarrowTypeEmulationPass |
| 144 | : public PassWrapper<TestMemRefFlattenAndVectorNarrowTypeEmulationPass, |
| 145 | OperationPass<func::FuncOp>> { |
| 146 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID( |
| 147 | TestMemRefFlattenAndVectorNarrowTypeEmulationPass) |
| 148 | |
| 149 | TestMemRefFlattenAndVectorNarrowTypeEmulationPass() = default; |
| 150 | TestMemRefFlattenAndVectorNarrowTypeEmulationPass( |
| 151 | const TestMemRefFlattenAndVectorNarrowTypeEmulationPass &pass) |
| 152 | : PassWrapper(pass) {} |
| 153 | |
| 154 | void getDependentDialects(DialectRegistry ®istry) const override { |
| 155 | registry |
| 156 | .insert<arith::ArithDialect, func::FuncDialect, memref::MemRefDialect, |
| 157 | vector::VectorDialect, affine::AffineDialect>(); |
| 158 | } |
| 159 | |
| 160 | StringRef getArgument() const final { |
| 161 | return "test-memref-flatten-and-vector-narrow-type-emulation" ; |
| 162 | } |
| 163 | |
| 164 | StringRef getDescription() const final { |
| 165 | return "Test MemRef flattening and vector narrow type emulation patterns" ; |
| 166 | } |
| 167 | |
| 168 | void runOnOperation() override { |
| 169 | Operation *op = getOperation(); |
| 170 | MLIRContext *ctx = &getContext(); |
| 171 | |
| 172 | // Create a type converter for narrow type emulation (8-bit) |
| 173 | arith::NarrowTypeEmulationConverter typeConverter(8); |
| 174 | |
| 175 | // Add conversions for memref types with i4 elements |
| 176 | memref::populateMemRefNarrowTypeEmulationConversions(typeConverter); |
| 177 | |
| 178 | ConversionTarget target(*ctx); |
| 179 | target.addDynamicallyLegalOp<func::FuncOp>(callback: [&typeConverter](Operation *op) { |
| 180 | return typeConverter.isLegal(type: cast<func::FuncOp>(Val: op).getFunctionType()); |
| 181 | }); |
| 182 | auto opLegalCallback = [&typeConverter](Operation *op) { |
| 183 | return typeConverter.isLegal(op); |
| 184 | }; |
| 185 | target.addDynamicallyLegalOp<func::CallOp, func::ReturnOp>(callback: opLegalCallback); |
| 186 | target.addDynamicallyLegalDialect< |
| 187 | arith::ArithDialect, vector::VectorDialect, memref::MemRefDialect, |
| 188 | affine::AffineDialect>(callback: opLegalCallback); |
| 189 | |
| 190 | RewritePatternSet patterns(ctx); |
| 191 | |
| 192 | // This is necessary for the purpose of emulating `memref.alloc` and |
| 193 | // function boundaries. |
| 194 | memref::populateMemRefNarrowTypeEmulationPatterns(typeConverter, patterns); |
| 195 | |
| 196 | vector::populateMemRefFlattenAndVectorNarrowTypeEmulationPatterns( |
| 197 | typeConverter, patterns); |
| 198 | |
| 199 | // Apply partial conversion |
| 200 | if (failed(Result: applyPartialConversion(op, target, patterns: std::move(patterns)))) |
| 201 | signalPassFailure(); |
| 202 | } |
| 203 | }; |
| 204 | } // namespace |
| 205 | |
| 206 | namespace mlir::test { |
| 207 | void registerTestEmulateNarrowTypePass() { |
| 208 | PassRegistration<TestEmulateNarrowTypePass>(); |
| 209 | PassRegistration<TestMemRefFlattenAndVectorNarrowTypeEmulationPass>(); |
| 210 | } |
| 211 | } // namespace mlir::test |
| 212 | |