| 1 | //===- ArithToLLVM.cpp - Arithmetic to LLVM dialect conversion -------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h" |
| 10 | |
| 11 | #include "mlir/Analysis/DataLayoutAnalysis.h" |
| 12 | #include "mlir/Conversion/ArithAttrToLLVM/AttrToLLVMConverter.h" |
| 13 | #include "mlir/Conversion/ArithCommon/CeilFloorDivExpansion.h" |
| 14 | #include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h" |
| 15 | #include "mlir/Conversion/LLVMCommon/ConversionTarget.h" |
| 16 | #include "mlir/Conversion/LLVMCommon/VectorPattern.h" |
| 17 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 18 | #include "mlir/Dialect/LLVMIR/FunctionCallUtils.h" |
| 19 | #include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h" |
| 20 | #include "mlir/Dialect/LLVMIR/LLVMArithOps.h" |
| 21 | #include "mlir/Dialect/LLVMIR/LLVMAttrs.h" |
| 22 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 23 | #include "mlir/Dialect/LLVMIR/LLVMConstrainedIntrinsicOps.h" |
| 24 | #include "mlir/Dialect/LLVMIR/LLVMConversionOps.h" |
| 25 | #include "mlir/Dialect/LLVMIR/LLVMFloatIntrinsicOps.h" |
| 26 | #include "mlir/Dialect/LLVMIR/LLVMIntegerIntrinsicOps.h" |
| 27 | #include "mlir/IR/DialectResourceBlobManager.h" |
| 28 | #include "mlir/IR/TypeUtilities.h" |
| 29 | #include "mlir/Pass/Pass.h" |
| 30 | #include <type_traits> |
| 31 | |
| 32 | namespace mlir { |
| 33 | #define GEN_PASS_DEF_ARITHTOLLVMCONVERSIONPASS |
| 34 | #include "mlir/Conversion/Passes.h.inc" |
| 35 | } // namespace mlir |
| 36 | |
| 37 | using namespace mlir; |
| 38 | |
| 39 | namespace { |
| 40 | |
| 41 | /// Lowering pattern that matches only when the source op's rounding mode |
| 42 | /// presence agrees with `HasRoundingMode`. This allows registering two |
| 43 | /// instances of the same pattern for one source op: one that handles the |
| 44 | /// unconstrained case (no rounding mode, lowering to a regular LLVM op) and |
| 45 | /// one that handles the constrained case (rounding mode present, lowering to |
| 46 | /// a constrained LLVM intrinsic). |
| 47 | /// |
| 48 | /// * `HasRoundingMode`: the pattern matches if and only if the source op has |
| 49 | /// a rounding mode attribute. |
| 50 | /// * `AttrConvert`: attribute converter to translate source attributes to |
| 51 | /// target attributes. |
| 52 | /// * `FailOnUnsupportedFP`: whether to fail if the source op has unsupported |
| 53 | /// floating point types. |
| 54 | template <typename SourceOp, typename TargetOp, bool HasRoundingMode, |
| 55 | template <typename, typename> typename AttrConvert = |
| 56 | AttrConvertPassThrough, |
| 57 | bool FailOnUnsupportedFP = false> |
| 58 | struct ConstrainedVectorConvertToLLVMPattern |
| 59 | : public VectorConvertToLLVMPattern<SourceOp, TargetOp, AttrConvert, |
| 60 | FailOnUnsupportedFP> { |
| 61 | using VectorConvertToLLVMPattern< |
| 62 | SourceOp, TargetOp, AttrConvert, |
| 63 | FailOnUnsupportedFP>::VectorConvertToLLVMPattern; |
| 64 | |
| 65 | LogicalResult |
| 66 | matchAndRewrite(SourceOp op, typename SourceOp::Adaptor adaptor, |
| 67 | ConversionPatternRewriter &rewriter) const override { |
| 68 | if (HasRoundingMode != static_cast<bool>(op.getRoundingModeAttr())) |
| 69 | return failure(); |
| 70 | return VectorConvertToLLVMPattern< |
| 71 | SourceOp, TargetOp, AttrConvert, |
| 72 | FailOnUnsupportedFP>::matchAndRewrite(op, adaptor, rewriter); |
| 73 | } |
| 74 | }; |
| 75 | |
| 76 | /// No-op bitcast. Propagate type input arg if converted source and dest types |
| 77 | /// are the same. |
| 78 | struct IdentityBitcastLowering final |
| 79 | : public OpConversionPattern<arith::BitcastOp> { |
| 80 | using Base::Base; |
| 81 | |
| 82 | LogicalResult |
| 83 | matchAndRewrite(arith::BitcastOp op, OpAdaptor adaptor, |
| 84 | ConversionPatternRewriter &rewriter) const final { |
| 85 | Value src = adaptor.getIn(); |
| 86 | Type resultType = getTypeConverter()->convertType(t: op.getType()); |
| 87 | if (src.getType() != resultType) |
| 88 | return rewriter.notifyMatchFailure(arg&: op, msg: "Types are different" ); |
| 89 | |
| 90 | rewriter.replaceOp(op, newValues: src); |
| 91 | return success(); |
| 92 | } |
| 93 | }; |
| 94 | |
| 95 | //===----------------------------------------------------------------------===// |
| 96 | // Straightforward Op Lowerings |
| 97 | //===----------------------------------------------------------------------===// |
| 98 | |
| 99 | using AddFOpLowering = |
| 100 | ConstrainedVectorConvertToLLVMPattern<arith::AddFOp, LLVM::FAddOp, |
| 101 | /*HasRoundingMode=*/false, |
| 102 | arith::AttrConvertFastMathToLLVM, |
| 103 | /*FailOnUnsupportedFP=*/true>; |
| 104 | using ConstrainedAddFOpLowering = ConstrainedVectorConvertToLLVMPattern< |
| 105 | arith::AddFOp, LLVM::ConstrainedFAddIntr, /*HasRoundingMode=*/true, |
| 106 | arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>; |
| 107 | using AddIOpLowering = |
| 108 | VectorConvertToLLVMPattern<arith::AddIOp, LLVM::AddOp, |
| 109 | arith::AttrConvertOverflowToLLVM>; |
| 110 | using AndIOpLowering = VectorConvertToLLVMPattern<arith::AndIOp, LLVM::AndOp>; |
| 111 | using BitcastOpLowering = |
| 112 | VectorConvertToLLVMPattern<arith::BitcastOp, LLVM::BitcastOp>; |
| 113 | using DivFOpLowering = |
| 114 | ConstrainedVectorConvertToLLVMPattern<arith::DivFOp, LLVM::FDivOp, |
| 115 | /*HasRoundingMode=*/false, |
| 116 | arith::AttrConvertFastMathToLLVM, |
| 117 | /*FailOnUnsupportedFP=*/true>; |
| 118 | using ConstrainedDivFOpLowering = ConstrainedVectorConvertToLLVMPattern< |
| 119 | arith::DivFOp, LLVM::ConstrainedFDivIntr, /*HasRoundingMode=*/true, |
| 120 | arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>; |
| 121 | using DivSIOpLowering = |
| 122 | VectorConvertToLLVMPattern<arith::DivSIOp, LLVM::SDivOp>; |
| 123 | using DivUIOpLowering = |
| 124 | VectorConvertToLLVMPattern<arith::DivUIOp, LLVM::UDivOp>; |
| 125 | using ExtFOpLowering = |
| 126 | VectorConvertToLLVMPattern<arith::ExtFOp, LLVM::FPExtOp, |
| 127 | arith::AttrConvertFastMathToLLVM, |
| 128 | /*FailOnUnsupportedFP=*/true>; |
| 129 | using ExtSIOpLowering = |
| 130 | VectorConvertToLLVMPattern<arith::ExtSIOp, LLVM::SExtOp>; |
| 131 | using ExtUIOpLowering = |
| 132 | VectorConvertToLLVMPattern<arith::ExtUIOp, LLVM::ZExtOp, |
| 133 | arith::AttrConvertNonNegToLLVM>; |
| 134 | using FPToSIOpLowering = |
| 135 | VectorConvertToLLVMPattern<arith::FPToSIOp, LLVM::FPToSIOp, |
| 136 | AttrConvertPassThrough, |
| 137 | /*FailOnUnsupportedFP=*/true>; |
| 138 | using FPToUIOpLowering = |
| 139 | VectorConvertToLLVMPattern<arith::FPToUIOp, LLVM::FPToUIOp, |
| 140 | AttrConvertPassThrough, |
| 141 | /*FailOnUnsupportedFP=*/true>; |
| 142 | using MaximumFOpLowering = |
| 143 | VectorConvertToLLVMPattern<arith::MaximumFOp, LLVM::MaximumOp, |
| 144 | arith::AttrConvertFastMathToLLVM, |
| 145 | /*FailOnUnsupportedFP=*/true>; |
| 146 | using MaxNumFOpLowering = |
| 147 | VectorConvertToLLVMPattern<arith::MaxNumFOp, LLVM::MaxNumOp, |
| 148 | arith::AttrConvertFastMathToLLVM, |
| 149 | /*FailOnUnsupportedFP=*/true>; |
| 150 | using MaximumNumFOpLowering = |
| 151 | VectorConvertToLLVMPattern<arith::MaximumNumFOp, LLVM::MaximumNumOp, |
| 152 | arith::AttrConvertFastMathToLLVM, |
| 153 | /*FailOnUnsupportedFP=*/true>; |
| 154 | using MaxSIOpLowering = |
| 155 | VectorConvertToLLVMPattern<arith::MaxSIOp, LLVM::SMaxOp>; |
| 156 | using MaxUIOpLowering = |
| 157 | VectorConvertToLLVMPattern<arith::MaxUIOp, LLVM::UMaxOp>; |
| 158 | using MinimumFOpLowering = |
| 159 | VectorConvertToLLVMPattern<arith::MinimumFOp, LLVM::MinimumOp, |
| 160 | arith::AttrConvertFastMathToLLVM, |
| 161 | /*FailOnUnsupportedFP=*/true>; |
| 162 | using MinNumFOpLowering = |
| 163 | VectorConvertToLLVMPattern<arith::MinNumFOp, LLVM::MinNumOp, |
| 164 | arith::AttrConvertFastMathToLLVM, |
| 165 | /*FailOnUnsupportedFP=*/true>; |
| 166 | using MinimumNumFOpLowering = |
| 167 | VectorConvertToLLVMPattern<arith::MinimumNumFOp, LLVM::MinimumNumOp, |
| 168 | arith::AttrConvertFastMathToLLVM, |
| 169 | /*FailOnUnsupportedFP=*/true>; |
| 170 | using MinSIOpLowering = |
| 171 | VectorConvertToLLVMPattern<arith::MinSIOp, LLVM::SMinOp>; |
| 172 | using MinUIOpLowering = |
| 173 | VectorConvertToLLVMPattern<arith::MinUIOp, LLVM::UMinOp>; |
| 174 | using MulFOpLowering = |
| 175 | ConstrainedVectorConvertToLLVMPattern<arith::MulFOp, LLVM::FMulOp, |
| 176 | /*HasRoundingMode=*/false, |
| 177 | arith::AttrConvertFastMathToLLVM, |
| 178 | /*FailOnUnsupportedFP=*/true>; |
| 179 | using ConstrainedMulFOpLowering = ConstrainedVectorConvertToLLVMPattern< |
| 180 | arith::MulFOp, LLVM::ConstrainedFMulIntr, /*HasRoundingMode=*/true, |
| 181 | arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>; |
| 182 | using MulIOpLowering = |
| 183 | VectorConvertToLLVMPattern<arith::MulIOp, LLVM::MulOp, |
| 184 | arith::AttrConvertOverflowToLLVM>; |
| 185 | using NegFOpLowering = |
| 186 | VectorConvertToLLVMPattern<arith::NegFOp, LLVM::FNegOp, |
| 187 | arith::AttrConvertFastMathToLLVM, |
| 188 | /*FailOnUnsupportedFP=*/true>; |
| 189 | using OrIOpLowering = VectorConvertToLLVMPattern<arith::OrIOp, LLVM::OrOp>; |
| 190 | using RemFOpLowering = |
| 191 | VectorConvertToLLVMPattern<arith::RemFOp, LLVM::FRemOp, |
| 192 | arith::AttrConvertFastMathToLLVM, |
| 193 | /*FailOnUnsupportedFP=*/true>; |
| 194 | using RemSIOpLowering = |
| 195 | VectorConvertToLLVMPattern<arith::RemSIOp, LLVM::SRemOp>; |
| 196 | using RemUIOpLowering = |
| 197 | VectorConvertToLLVMPattern<arith::RemUIOp, LLVM::URemOp>; |
| 198 | using SelectOpLowering = |
| 199 | VectorConvertToLLVMPattern<arith::SelectOp, LLVM::SelectOp>; |
| 200 | using ShLIOpLowering = |
| 201 | VectorConvertToLLVMPattern<arith::ShLIOp, LLVM::ShlOp, |
| 202 | arith::AttrConvertOverflowToLLVM>; |
| 203 | using ShRSIOpLowering = |
| 204 | VectorConvertToLLVMPattern<arith::ShRSIOp, LLVM::AShrOp>; |
| 205 | using ShRUIOpLowering = |
| 206 | VectorConvertToLLVMPattern<arith::ShRUIOp, LLVM::LShrOp>; |
| 207 | using SIToFPOpLowering = |
| 208 | VectorConvertToLLVMPattern<arith::SIToFPOp, LLVM::SIToFPOp, |
| 209 | AttrConvertPassThrough, |
| 210 | /*FailOnUnsupportedFP=*/true>; |
| 211 | using SubFOpLowering = |
| 212 | ConstrainedVectorConvertToLLVMPattern<arith::SubFOp, LLVM::FSubOp, |
| 213 | /*HasRoundingMode=*/false, |
| 214 | arith::AttrConvertFastMathToLLVM, |
| 215 | /*FailOnUnsupportedFP=*/true>; |
| 216 | using ConstrainedSubFOpLowering = ConstrainedVectorConvertToLLVMPattern< |
| 217 | arith::SubFOp, LLVM::ConstrainedFSubIntr, /*HasRoundingMode=*/true, |
| 218 | arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>; |
| 219 | using SubIOpLowering = |
| 220 | VectorConvertToLLVMPattern<arith::SubIOp, LLVM::SubOp, |
| 221 | arith::AttrConvertOverflowToLLVM>; |
| 222 | using TruncFOpLowering = |
| 223 | ConstrainedVectorConvertToLLVMPattern<arith::TruncFOp, LLVM::FPTruncOp, |
| 224 | /*HasRoundingMode=*/false, |
| 225 | arith::AttrConvertFastMathToLLVM, |
| 226 | /*FailOnUnsupportedFP=*/true>; |
| 227 | using ConstrainedTruncFOpLowering = ConstrainedVectorConvertToLLVMPattern< |
| 228 | arith::TruncFOp, LLVM::ConstrainedFPTruncIntr, /*HasRoundingMode=*/true, |
| 229 | arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>; |
| 230 | using TruncIOpLowering = |
| 231 | VectorConvertToLLVMPattern<arith::TruncIOp, LLVM::TruncOp, |
| 232 | arith::AttrConvertOverflowToLLVM>; |
| 233 | using UIToFPOpLowering = |
| 234 | VectorConvertToLLVMPattern<arith::UIToFPOp, LLVM::UIToFPOp, |
| 235 | arith::AttrConvertNonNegToLLVM, |
| 236 | /*FailOnUnsupportedFP=*/true>; |
| 237 | using XOrIOpLowering = VectorConvertToLLVMPattern<arith::XOrIOp, LLVM::XOrOp>; |
| 238 | |
| 239 | //===----------------------------------------------------------------------===// |
| 240 | // Op Lowering Patterns |
| 241 | //===----------------------------------------------------------------------===// |
| 242 | |
| 243 | /// Directly lower to LLVM op. |
| 244 | struct ConstantOpLowering : public ConvertOpToLLVMPattern<arith::ConstantOp> { |
| 245 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 246 | |
| 247 | LogicalResult |
| 248 | matchAndRewrite(arith::ConstantOp op, OpAdaptor adaptor, |
| 249 | ConversionPatternRewriter &rewriter) const override; |
| 250 | }; |
| 251 | |
| 252 | /// The lowering of index_cast becomes an integer conversion since index |
| 253 | /// becomes an integer. If the bit width of the source and target integer |
| 254 | /// types is the same, just erase the cast. If the target type is wider, |
| 255 | /// sign-extend the value, otherwise truncate it. |
| 256 | template <typename OpTy, typename ExtCastTy> |
| 257 | struct IndexCastOpLowering : public ConvertOpToLLVMPattern<OpTy> { |
| 258 | using ConvertOpToLLVMPattern<OpTy>::ConvertOpToLLVMPattern; |
| 259 | |
| 260 | LogicalResult |
| 261 | matchAndRewrite(OpTy op, typename OpTy::Adaptor adaptor, |
| 262 | ConversionPatternRewriter &rewriter) const override; |
| 263 | }; |
| 264 | |
| 265 | using IndexCastOpSILowering = |
| 266 | IndexCastOpLowering<arith::IndexCastOp, LLVM::SExtOp>; |
| 267 | using IndexCastOpUILowering = |
| 268 | IndexCastOpLowering<arith::IndexCastUIOp, LLVM::ZExtOp>; |
| 269 | |
| 270 | struct AddUIExtendedOpLowering |
| 271 | : public ConvertOpToLLVMPattern<arith::AddUIExtendedOp> { |
| 272 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 273 | |
| 274 | LogicalResult |
| 275 | matchAndRewrite(arith::AddUIExtendedOp op, OpAdaptor adaptor, |
| 276 | ConversionPatternRewriter &rewriter) const override; |
| 277 | }; |
| 278 | |
| 279 | struct SubUIExtendedOpLowering |
| 280 | : public ConvertOpToLLVMPattern<arith::SubUIExtendedOp> { |
| 281 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 282 | |
| 283 | LogicalResult |
| 284 | matchAndRewrite(arith::SubUIExtendedOp op, OpAdaptor adaptor, |
| 285 | ConversionPatternRewriter &rewriter) const override; |
| 286 | }; |
| 287 | |
| 288 | template <typename ArithMulOp, bool IsSigned> |
| 289 | struct MulIExtendedOpLowering : public ConvertOpToLLVMPattern<ArithMulOp> { |
| 290 | using ConvertOpToLLVMPattern<ArithMulOp>::ConvertOpToLLVMPattern; |
| 291 | |
| 292 | LogicalResult |
| 293 | matchAndRewrite(ArithMulOp op, typename ArithMulOp::Adaptor adaptor, |
| 294 | ConversionPatternRewriter &rewriter) const override; |
| 295 | }; |
| 296 | |
| 297 | using MulSIExtendedOpLowering = |
| 298 | MulIExtendedOpLowering<arith::MulSIExtendedOp, true>; |
| 299 | using MulUIExtendedOpLowering = |
| 300 | MulIExtendedOpLowering<arith::MulUIExtendedOp, false>; |
| 301 | |
| 302 | struct CmpIOpLowering : public ConvertOpToLLVMPattern<arith::CmpIOp> { |
| 303 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 304 | |
| 305 | LogicalResult |
| 306 | matchAndRewrite(arith::CmpIOp op, OpAdaptor adaptor, |
| 307 | ConversionPatternRewriter &rewriter) const override; |
| 308 | }; |
| 309 | |
| 310 | struct CmpFOpLowering : public ConvertOpToLLVMPattern<arith::CmpFOp> { |
| 311 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 312 | |
| 313 | LogicalResult |
| 314 | matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor, |
| 315 | ConversionPatternRewriter &rewriter) const override; |
| 316 | }; |
| 317 | |
| 318 | /// Lower arith.convertf (same-bitwidth FP cast) to LLVM. |
| 319 | /// |
| 320 | /// Extends to f32 via llvm.fpext, then truncates to the target type via |
| 321 | /// llvm.fptrunc. This handles bf16 <-> f16, which is the only same-bitwidth |
| 322 | /// pair of LLVM-supported FP types. |
| 323 | struct ConvertFOpLowering : public ConvertOpToLLVMPattern<arith::ConvertFOp> { |
| 324 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 325 | |
| 326 | LogicalResult |
| 327 | matchAndRewrite(arith::ConvertFOp op, OpAdaptor adaptor, |
| 328 | ConversionPatternRewriter &rewriter) const override { |
| 329 | if (LLVM::detail::opHasUnsupportedFloatingPointTypes(op, |
| 330 | typeConverter: *getTypeConverter())) |
| 331 | return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported floating point type" ); |
| 332 | |
| 333 | // Only bf16 <-> f16 conversions are supported. There is currently no other |
| 334 | // pair of FP types that are valid LLVM types. |
| 335 | [[maybe_unused]] auto srcType = getElementTypeOrSelf(type: op.getIn().getType()); |
| 336 | [[maybe_unused]] auto dstType = getElementTypeOrSelf(type: op.getType()); |
| 337 | assert(((srcType.isBF16() && dstType.isF16()) || |
| 338 | (srcType.isF16() && dstType.isBF16())) && |
| 339 | "only bf16 <-> f16 conversions are supported" ); |
| 340 | |
| 341 | Type convertedType = getTypeConverter()->convertType(t: op.getType()); |
| 342 | if (!convertedType) |
| 343 | return rewriter.notifyMatchFailure(arg&: op, msg: "failed to convert result type" ); |
| 344 | |
| 345 | Value input = adaptor.getIn(); |
| 346 | Location loc = op.getLoc(); |
| 347 | |
| 348 | if (!isa<LLVM::LLVMArrayType>(Val: input.getType())) { |
| 349 | rewriter.replaceOp(op, |
| 350 | newValues: emitConversion(rewriter, loc, input, targetType: convertedType)); |
| 351 | return success(); |
| 352 | } |
| 353 | |
| 354 | if (!isa<VectorType>(Val: op.getType())) |
| 355 | return rewriter.notifyMatchFailure(arg&: op, msg: "expected vector result type" ); |
| 356 | |
| 357 | return LLVM::detail::handleMultidimensionalVectors( |
| 358 | op: op.getOperation(), operands: adaptor.getOperands(), typeConverter: *getTypeConverter(), |
| 359 | createOperand: [&](Type llvm1DVectorTy, ValueRange operands) -> Value { |
| 360 | return emitConversion(rewriter, loc, input: operands.front(), |
| 361 | targetType: llvm1DVectorTy); |
| 362 | }, |
| 363 | rewriter); |
| 364 | } |
| 365 | |
| 366 | private: |
| 367 | static Value emitConversion(ConversionPatternRewriter &rewriter, Location loc, |
| 368 | Value input, Type targetType) { |
| 369 | Type f32Scalar = Float32Type::get(context: rewriter.getContext()); |
| 370 | Type f32Ty = f32Scalar; |
| 371 | if (auto vecTy = dyn_cast<VectorType>(Val&: targetType)) |
| 372 | f32Ty = VectorType::get(shape: vecTy.getShape(), elementType: f32Scalar); |
| 373 | |
| 374 | Value ext = LLVM::FPExtOp::create(builder&: rewriter, location: loc, res: f32Ty, arg: input); |
| 375 | return LLVM::FPTruncOp::create(builder&: rewriter, location: loc, res: targetType, arg: ext); |
| 376 | } |
| 377 | }; |
| 378 | |
| 379 | struct SelectOpOneToNLowering : public ConvertOpToLLVMPattern<arith::SelectOp> { |
| 380 | using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; |
| 381 | using Adaptor = ConvertOpToLLVMPattern<arith::SelectOp>::OneToNOpAdaptor; |
| 382 | |
| 383 | LogicalResult |
| 384 | matchAndRewrite(arith::SelectOp op, Adaptor adaptor, |
| 385 | ConversionPatternRewriter &rewriter) const override; |
| 386 | }; |
| 387 | |
| 388 | } // namespace |
| 389 | |
| 390 | //===----------------------------------------------------------------------===// |
| 391 | // ConstantOpLowering |
| 392 | //===----------------------------------------------------------------------===// |
| 393 | |
| 394 | /// Retypes `attr` for a `llvm.mlir.constant` of `resultType`. `arith.constant` |
| 395 | /// requires the value attribute and the result to have the same type, but the |
| 396 | /// type converter may map the element type to a different one, e.g. `index` to |
| 397 | /// `i32` or `i64` depending on the configured index bitwidth. Build the |
| 398 | /// attribute from the converted type so that the two agree wherever that is |
| 399 | /// representable. Returns a null attribute if `attr` cannot be used for |
| 400 | /// `resultType`. |
| 401 | static TypedAttr convertConstantValue(TypedAttr attr, Type resultType) { |
| 402 | // Compare the element types, but explicitly ignore the non-scalar portions of |
| 403 | // types. This relaxation is required to support multi-dimensional vector |
| 404 | // constant that result in nested LLVM array values. |
| 405 | Type sourceElementType = LLVM::getConstantElementType(type: attr.getType()); |
| 406 | Type targetElementType = LLVM::getConstantElementType(type: resultType); |
| 407 | if (sourceElementType == targetElementType) |
| 408 | return attr; |
| 409 | |
| 410 | auto targetIntType = dyn_cast<IntegerType>(Val&: targetElementType); |
| 411 | if (!targetIntType) |
| 412 | return {}; |
| 413 | |
| 414 | // The converter maps the low-precision float types that have no LLVM |
| 415 | // equivalent to an integer of the same width. The attribute stays a float |
| 416 | // attribute in that case. |
| 417 | if (auto sourceFloatType = dyn_cast<FloatType>(Val&: sourceElementType)) { |
| 418 | if (sourceFloatType.getWidth() != targetIntType.getWidth()) |
| 419 | return {}; |
| 420 | return attr; |
| 421 | } |
| 422 | |
| 423 | // Apart from those floats, `index` is the only element type the converter |
| 424 | // rewrites, so anything else is a malformed `arith.constant`. Bail out rather |
| 425 | // than reinterpret its value, which would silently change the constant (e.g. |
| 426 | // sign-extending an `i1` `true` to -1). |
| 427 | if (!isa<IndexType>(Val: sourceElementType)) |
| 428 | return {}; |
| 429 | // `index` is signless but holds signed values, so narrowing to a smaller |
| 430 | // index bitwidth truncates and widening sign-extends. |
| 431 | unsigned width = targetIntType.getWidth(); |
| 432 | |
| 433 | if (auto intAttr = dyn_cast<IntegerAttr>(Val&: attr)) |
| 434 | return IntegerAttr::get(type: targetIntType, |
| 435 | value: intAttr.getValue().sextOrTrunc(width)); |
| 436 | |
| 437 | auto retypeValues = [&](DenseIntElementsAttr values) { |
| 438 | return values.mapValues(newElementType: targetIntType, mapping: [&](const APInt &value) { |
| 439 | return value.sextOrTrunc(width); |
| 440 | }); |
| 441 | }; |
| 442 | |
| 443 | if (auto denseAttr = dyn_cast<DenseIntElementsAttr>(Val&: attr)) |
| 444 | return retypeValues(denseAttr); |
| 445 | |
| 446 | if (auto sparseAttr = dyn_cast<SparseElementsAttr>(Val&: attr)) |
| 447 | return SparseElementsAttr::get( |
| 448 | type: cast<ShapedType>(Val: attr.getType()).clone(elementType: targetIntType), |
| 449 | indices: sparseAttr.getIndices(), |
| 450 | values: retypeValues(cast<DenseIntElementsAttr>(Val: sparseAttr.getValues()))); |
| 451 | |
| 452 | // A resource-backed elements attribute refers to a blob laid out for its own |
| 453 | // element type. The blob cannot be rewritten here, only reinterpreted, which |
| 454 | // is correct exactly when the target type has the same width as the storage |
| 455 | // `index` uses in a blob. |
| 456 | if (auto resourceAttr = dyn_cast<DenseResourceElementsAttr>(Val&: attr)) { |
| 457 | if (width != IndexType::kInternalStorageBitWidth) |
| 458 | return {}; |
| 459 | return DenseResourceElementsAttr::get( |
| 460 | type: cast<ShapedType>(Val: attr.getType()).clone(elementType: targetIntType), |
| 461 | handle: resourceAttr.getRawHandle()); |
| 462 | } |
| 463 | |
| 464 | return {}; |
| 465 | } |
| 466 | |
| 467 | LogicalResult |
| 468 | ConstantOpLowering::matchAndRewrite(arith::ConstantOp op, OpAdaptor adaptor, |
| 469 | ConversionPatternRewriter &rewriter) const { |
| 470 | Type resultType = getTypeConverter()->convertType(t: op.getType()); |
| 471 | if (!resultType) |
| 472 | return rewriter.notifyMatchFailure(arg&: op, msg: "failed to convert result type" ); |
| 473 | |
| 474 | TypedAttr value = convertConstantValue(attr: op.getValue(), resultType); |
| 475 | if (!value) |
| 476 | return rewriter.notifyMatchFailure( |
| 477 | arg&: op, msg: "failed to convert value attribute to the converted result type" ); |
| 478 | |
| 479 | // `arith.constant` has no operands and a single result, so there is nothing |
| 480 | // for `oneToOneRewrite` to do here beyond converting `resultType` a second |
| 481 | // time. |
| 482 | DictionaryAttr discardableAttrs = op->getDiscardableAttrDictionary(); |
| 483 | auto constantOp = |
| 484 | LLVM::ConstantOp::create(builder&: rewriter, location: op.getLoc(), res: resultType, value); |
| 485 | constantOp->setDiscardableAttrs(discardableAttrs); |
| 486 | rewriter.replaceOp(op, newOp: constantOp); |
| 487 | return success(); |
| 488 | } |
| 489 | |
| 490 | //===----------------------------------------------------------------------===// |
| 491 | // IndexCastOpLowering |
| 492 | //===----------------------------------------------------------------------===// |
| 493 | |
| 494 | template <typename OpTy, typename ExtCastTy> |
| 495 | LogicalResult IndexCastOpLowering<OpTy, ExtCastTy>::matchAndRewrite( |
| 496 | OpTy op, typename OpTy::Adaptor adaptor, |
| 497 | ConversionPatternRewriter &rewriter) const { |
| 498 | Type resultType = op.getResult().getType(); |
| 499 | Type targetElementType = |
| 500 | this->typeConverter->convertType(getElementTypeOrSelf(type: resultType)); |
| 501 | Type sourceElementType = |
| 502 | this->typeConverter->convertType(getElementTypeOrSelf(op.getIn())); |
| 503 | unsigned targetBits = targetElementType.getIntOrFloatBitWidth(); |
| 504 | unsigned sourceBits = sourceElementType.getIntOrFloatBitWidth(); |
| 505 | |
| 506 | if (targetBits == sourceBits) { |
| 507 | rewriter.replaceOp(op, adaptor.getIn()); |
| 508 | return success(); |
| 509 | } |
| 510 | |
| 511 | // Memref index_cast is a no-op at the LLVM level since LLVM uses opaque |
| 512 | // pointers and memrefs of different integer/index element types all convert |
| 513 | // to the same LLVM struct type. |
| 514 | if (isa<MemRefType>(op.getIn().getType())) { |
| 515 | rewriter.replaceOp(op, adaptor.getIn()); |
| 516 | return success(); |
| 517 | } |
| 518 | |
| 519 | bool isNonNeg = false; |
| 520 | if constexpr (std::is_same_v<ExtCastTy, LLVM::ZExtOp>) |
| 521 | isNonNeg = op.getNonNeg(); |
| 522 | |
| 523 | // Handle the scalar and 1D vector cases. |
| 524 | Type operandType = adaptor.getIn().getType(); |
| 525 | if (!isa<LLVM::LLVMArrayType>(Val: operandType)) { |
| 526 | Type targetType = this->typeConverter->convertType(resultType); |
| 527 | if (targetBits < sourceBits) { |
| 528 | rewriter.replaceOpWithNewOp<LLVM::TruncOp>(op, targetType, |
| 529 | adaptor.getIn()); |
| 530 | } else { |
| 531 | auto extOp = rewriter.replaceOpWithNewOp<ExtCastTy>(op, targetType, |
| 532 | adaptor.getIn()); |
| 533 | if constexpr (std::is_same_v<ExtCastTy, LLVM::ZExtOp>) |
| 534 | extOp.setNonNeg(isNonNeg); |
| 535 | } |
| 536 | return success(); |
| 537 | } |
| 538 | |
| 539 | if (!isa<VectorType>(Val: resultType)) |
| 540 | return rewriter.notifyMatchFailure(op, "expected vector result type" ); |
| 541 | |
| 542 | return LLVM::detail::handleMultidimensionalVectors( |
| 543 | op: op.getOperation(), operands: adaptor.getOperands(), typeConverter: *(this->getTypeConverter()), |
| 544 | createOperand: [&](Type llvm1DVectorTy, ValueRange operands) -> Value { |
| 545 | typename OpTy::Adaptor adaptor(operands); |
| 546 | if (targetBits < sourceBits) { |
| 547 | return LLVM::TruncOp::create(rewriter, op.getLoc(), llvm1DVectorTy, |
| 548 | adaptor.getIn()); |
| 549 | } |
| 550 | auto extOp = ExtCastTy::create(rewriter, op.getLoc(), llvm1DVectorTy, |
| 551 | adaptor.getIn()); |
| 552 | if constexpr (std::is_same_v<ExtCastTy, LLVM::ZExtOp>) { |
| 553 | if (isNonNeg) |
| 554 | extOp.setNonNeg(true); |
| 555 | } |
| 556 | return extOp; |
| 557 | }, |
| 558 | rewriter); |
| 559 | } |
| 560 | |
| 561 | //===----------------------------------------------------------------------===// |
| 562 | // AddUIExtendedOpLowering |
| 563 | //===----------------------------------------------------------------------===// |
| 564 | |
| 565 | LogicalResult AddUIExtendedOpLowering::matchAndRewrite( |
| 566 | arith::AddUIExtendedOp op, OpAdaptor adaptor, |
| 567 | ConversionPatternRewriter &rewriter) const { |
| 568 | Type operandType = adaptor.getLhs().getType(); |
| 569 | Type sumResultType = op.getSum().getType(); |
| 570 | Type overflowResultType = op.getOverflow().getType(); |
| 571 | |
| 572 | if (!LLVM::isCompatibleType(type: operandType)) |
| 573 | return failure(); |
| 574 | |
| 575 | MLIRContext *ctx = rewriter.getContext(); |
| 576 | Location loc = op.getLoc(); |
| 577 | |
| 578 | // Handle the scalar and 1D vector cases. |
| 579 | if (!isa<LLVM::LLVMArrayType>(Val: operandType)) { |
| 580 | Type newSumType = typeConverter->convertType(t: sumResultType); |
| 581 | Type newOverflowType = typeConverter->convertType(t: overflowResultType); |
| 582 | Type structType = |
| 583 | LLVM::LLVMStructType::getLiteral(context: ctx, types: {newSumType, newOverflowType}); |
| 584 | Value addOverflow = LLVM::UAddWithOverflowOp::create( |
| 585 | builder&: rewriter, location: loc, res: structType, odsArg_0: adaptor.getLhs(), odsArg_1: adaptor.getRhs()); |
| 586 | Value = |
| 587 | LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: addOverflow, position: 0); |
| 588 | Value = |
| 589 | LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: addOverflow, position: 1); |
| 590 | rewriter.replaceOp(op, newValues: {sumExtracted, overflowExtracted}); |
| 591 | return success(); |
| 592 | } |
| 593 | |
| 594 | if (!isa<VectorType>(Val: sumResultType)) |
| 595 | return rewriter.notifyMatchFailure(arg&: loc, msg: "expected vector result types" ); |
| 596 | |
| 597 | return rewriter.notifyMatchFailure(arg&: loc, |
| 598 | msg: "ND vector types are not supported yet" ); |
| 599 | } |
| 600 | |
| 601 | //===----------------------------------------------------------------------===// |
| 602 | // SubUIExtendedOpLowering |
| 603 | //===----------------------------------------------------------------------===// |
| 604 | |
| 605 | LogicalResult SubUIExtendedOpLowering::matchAndRewrite( |
| 606 | arith::SubUIExtendedOp op, OpAdaptor adaptor, |
| 607 | ConversionPatternRewriter &rewriter) const { |
| 608 | Type operandType = adaptor.getLhs().getType(); |
| 609 | Type diffResultType = op.getDiff().getType(); |
| 610 | Type borrowResultType = op.getBorrow().getType(); |
| 611 | |
| 612 | if (!LLVM::isCompatibleType(type: operandType)) |
| 613 | return failure(); |
| 614 | |
| 615 | MLIRContext *ctx = rewriter.getContext(); |
| 616 | Location loc = op.getLoc(); |
| 617 | |
| 618 | // Handle the scalar and 1D vector cases. |
| 619 | if (!isa<LLVM::LLVMArrayType>(Val: operandType)) { |
| 620 | Type newBorrowType = typeConverter->convertType(t: borrowResultType); |
| 621 | Type structType = |
| 622 | LLVM::LLVMStructType::getLiteral(context: ctx, types: {diffResultType, newBorrowType}); |
| 623 | Value subOverflow = LLVM::USubWithOverflowOp::create( |
| 624 | builder&: rewriter, location: loc, res: structType, odsArg_0: adaptor.getLhs(), odsArg_1: adaptor.getRhs()); |
| 625 | Value = |
| 626 | LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: subOverflow, position: 0); |
| 627 | Value = |
| 628 | LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: subOverflow, position: 1); |
| 629 | rewriter.replaceOp(op, newValues: {diffExtracted, borrowExtracted}); |
| 630 | return success(); |
| 631 | } |
| 632 | |
| 633 | if (!isa<VectorType>(Val: diffResultType)) |
| 634 | return rewriter.notifyMatchFailure(arg&: loc, msg: "expected vector result types" ); |
| 635 | |
| 636 | return rewriter.notifyMatchFailure(arg&: loc, |
| 637 | msg: "ND vector types are not supported yet" ); |
| 638 | } |
| 639 | |
| 640 | //===----------------------------------------------------------------------===// |
| 641 | // MulIExtendedOpLowering |
| 642 | //===----------------------------------------------------------------------===// |
| 643 | |
| 644 | template <typename ArithMulOp, bool IsSigned> |
| 645 | LogicalResult MulIExtendedOpLowering<ArithMulOp, IsSigned>::matchAndRewrite( |
| 646 | ArithMulOp op, typename ArithMulOp::Adaptor adaptor, |
| 647 | ConversionPatternRewriter &rewriter) const { |
| 648 | Type resultType = adaptor.getLhs().getType(); |
| 649 | |
| 650 | if (!LLVM::isCompatibleType(type: resultType)) |
| 651 | return failure(); |
| 652 | |
| 653 | Location loc = op.getLoc(); |
| 654 | |
| 655 | // Handle the scalar and 1D vector cases. Because LLVM does not have a |
| 656 | // matching extended multiplication intrinsic, perform regular multiplication |
| 657 | // on operands zero-extended to i(2*N) bits, and truncate the results back to |
| 658 | // iN types. |
| 659 | if (!isa<LLVM::LLVMArrayType>(Val: resultType)) { |
| 660 | // Shift amount necessary to extract the high bits from widened result. |
| 661 | TypedAttr shiftValAttr; |
| 662 | |
| 663 | if (auto intTy = dyn_cast<IntegerType>(Val&: resultType)) { |
| 664 | unsigned resultBitwidth = intTy.getWidth(); |
| 665 | auto attrTy = rewriter.getIntegerType(width: resultBitwidth * 2); |
| 666 | shiftValAttr = rewriter.getIntegerAttr(type: attrTy, value: resultBitwidth); |
| 667 | } else { |
| 668 | auto vecTy = cast<VectorType>(Val&: resultType); |
| 669 | unsigned resultBitwidth = vecTy.getElementTypeBitWidth(); |
| 670 | auto attrTy = VectorType::get( |
| 671 | shape: vecTy.getShape(), elementType: rewriter.getIntegerType(width: resultBitwidth * 2)); |
| 672 | shiftValAttr = SplatElementsAttr::get( |
| 673 | type: attrTy, values: APInt(resultBitwidth * 2, resultBitwidth)); |
| 674 | } |
| 675 | Type wideType = shiftValAttr.getType(); |
| 676 | assert(LLVM::isCompatibleType(wideType) && |
| 677 | "LLVM dialect should support all signless integer types" ); |
| 678 | |
| 679 | using LLVMExtOp = std::conditional_t<IsSigned, LLVM::SExtOp, LLVM::ZExtOp>; |
| 680 | Value lhsExt = LLVMExtOp::create(rewriter, loc, wideType, adaptor.getLhs()); |
| 681 | Value rhsExt = LLVMExtOp::create(rewriter, loc, wideType, adaptor.getRhs()); |
| 682 | Value mulExt = LLVM::MulOp::create(builder&: rewriter, location: loc, res: wideType, lhs: lhsExt, rhs: rhsExt); |
| 683 | |
| 684 | // Split the 2*N-bit wide result into two N-bit values. |
| 685 | Value low = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: resultType, arg: mulExt); |
| 686 | Value shiftVal = LLVM::ConstantOp::create(builder&: rewriter, location: loc, value: shiftValAttr); |
| 687 | Value highExt = LLVM::LShrOp::create(builder&: rewriter, location: loc, lhs: mulExt, rhs: shiftVal); |
| 688 | Value high = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: resultType, arg: highExt); |
| 689 | |
| 690 | rewriter.replaceOp(op, {low, high}); |
| 691 | return success(); |
| 692 | } |
| 693 | |
| 694 | if (!isa<VectorType>(Val: resultType)) |
| 695 | return rewriter.notifyMatchFailure(op, "expected vector result type" ); |
| 696 | |
| 697 | return rewriter.notifyMatchFailure(op, |
| 698 | "ND vector types are not supported yet" ); |
| 699 | } |
| 700 | |
| 701 | //===----------------------------------------------------------------------===// |
| 702 | // CmpIOpLowering |
| 703 | //===----------------------------------------------------------------------===// |
| 704 | |
| 705 | // Convert arith.cmp predicate into the LLVM dialect CmpPredicate. The two enums |
| 706 | // share numerical values so just cast. |
| 707 | template <typename LLVMPredType, typename PredType> |
| 708 | static LLVMPredType convertCmpPredicate(PredType pred) { |
| 709 | return static_cast<LLVMPredType>(pred); |
| 710 | } |
| 711 | |
| 712 | LogicalResult |
| 713 | CmpIOpLowering::matchAndRewrite(arith::CmpIOp op, OpAdaptor adaptor, |
| 714 | ConversionPatternRewriter &rewriter) const { |
| 715 | Type operandType = adaptor.getLhs().getType(); |
| 716 | Type resultType = op.getResult().getType(); |
| 717 | |
| 718 | // Handle the scalar and 1D vector cases. |
| 719 | if (!isa<LLVM::LLVMArrayType>(Val: operandType)) { |
| 720 | rewriter.replaceOpWithNewOp<LLVM::ICmpOp>( |
| 721 | op, args: typeConverter->convertType(t: resultType), |
| 722 | args: convertCmpPredicate<LLVM::ICmpPredicate>(pred: op.getPredicate()), |
| 723 | args: adaptor.getLhs(), args: adaptor.getRhs()); |
| 724 | return success(); |
| 725 | } |
| 726 | |
| 727 | if (!isa<VectorType>(Val: resultType)) |
| 728 | return rewriter.notifyMatchFailure(arg&: op, msg: "expected vector result type" ); |
| 729 | |
| 730 | return LLVM::detail::handleMultidimensionalVectors( |
| 731 | op: op.getOperation(), operands: adaptor.getOperands(), typeConverter: *getTypeConverter(), |
| 732 | createOperand: [&](Type llvm1DVectorTy, ValueRange operands) { |
| 733 | OpAdaptor adaptor(operands); |
| 734 | return LLVM::ICmpOp::create( |
| 735 | builder&: rewriter, location: op.getLoc(), res: llvm1DVectorTy, |
| 736 | predicate: convertCmpPredicate<LLVM::ICmpPredicate>(pred: op.getPredicate()), |
| 737 | lhs: adaptor.getLhs(), rhs: adaptor.getRhs()); |
| 738 | }, |
| 739 | rewriter); |
| 740 | } |
| 741 | |
| 742 | //===----------------------------------------------------------------------===// |
| 743 | // CmpFOpLowering |
| 744 | //===----------------------------------------------------------------------===// |
| 745 | |
| 746 | LogicalResult |
| 747 | CmpFOpLowering::matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor, |
| 748 | ConversionPatternRewriter &rewriter) const { |
| 749 | if (LLVM::detail::isUnsupportedFloatingPointType(typeConverter: *this->getTypeConverter(), |
| 750 | type: op.getLhs().getType())) |
| 751 | return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported floating point type" ); |
| 752 | |
| 753 | Type operandType = adaptor.getLhs().getType(); |
| 754 | Type resultType = op.getResult().getType(); |
| 755 | LLVM::FastmathFlags fmf = |
| 756 | arith::convertArithFastMathFlagsToLLVM(arithFMF: op.getFastmath()); |
| 757 | |
| 758 | // Handle the scalar and 1D vector cases. |
| 759 | if (!isa<LLVM::LLVMArrayType>(Val: operandType)) { |
| 760 | rewriter.replaceOpWithNewOp<LLVM::FCmpOp>( |
| 761 | op, args: typeConverter->convertType(t: resultType), |
| 762 | args: convertCmpPredicate<LLVM::FCmpPredicate>(pred: op.getPredicate()), |
| 763 | args: adaptor.getLhs(), args: adaptor.getRhs(), args&: fmf); |
| 764 | return success(); |
| 765 | } |
| 766 | |
| 767 | if (!isa<VectorType>(Val: resultType)) |
| 768 | return rewriter.notifyMatchFailure(arg&: op, msg: "expected vector result type" ); |
| 769 | |
| 770 | return LLVM::detail::handleMultidimensionalVectors( |
| 771 | op: op.getOperation(), operands: adaptor.getOperands(), typeConverter: *getTypeConverter(), |
| 772 | createOperand: [&](Type llvm1DVectorTy, ValueRange operands) { |
| 773 | OpAdaptor adaptor(operands); |
| 774 | return LLVM::FCmpOp::create( |
| 775 | builder&: rewriter, location: op.getLoc(), res: llvm1DVectorTy, |
| 776 | predicate: convertCmpPredicate<LLVM::FCmpPredicate>(pred: op.getPredicate()), |
| 777 | lhs: adaptor.getLhs(), rhs: adaptor.getRhs(), fastmathFlags: fmf); |
| 778 | }, |
| 779 | rewriter); |
| 780 | } |
| 781 | |
| 782 | //===----------------------------------------------------------------------===// |
| 783 | // SelectOpOneToNLowering |
| 784 | //===----------------------------------------------------------------------===// |
| 785 | |
| 786 | /// Pattern for arith.select where the true/false values lower to multiple |
| 787 | /// SSA values (1:N conversion). This pattern generates multiple arith.select |
| 788 | /// than can be lowered by the 1:1 arith.select pattern. |
| 789 | LogicalResult SelectOpOneToNLowering::matchAndRewrite( |
| 790 | arith::SelectOp op, Adaptor adaptor, |
| 791 | ConversionPatternRewriter &rewriter) const { |
| 792 | // In case of a 1:1 conversion, the 1:1 pattern will match. |
| 793 | if (llvm::hasSingleElement(C: adaptor.getTrueValue())) |
| 794 | return rewriter.notifyMatchFailure( |
| 795 | arg&: op, msg: "not a 1:N conversion, 1:1 pattern will match" ); |
| 796 | if (!op.getCondition().getType().isInteger(width: 1)) |
| 797 | return rewriter.notifyMatchFailure(arg&: op, |
| 798 | msg: "non-i1 conditions are not supported" ); |
| 799 | SmallVector<Value> results; |
| 800 | for (auto [trueValue, falseValue] : |
| 801 | llvm::zip_equal(t: adaptor.getTrueValue(), u: adaptor.getFalseValue())) |
| 802 | results.push_back(Elt: arith::SelectOp::create( |
| 803 | builder&: rewriter, location: op.getLoc(), condition: op.getCondition(), true_value: trueValue, false_value: falseValue)); |
| 804 | rewriter.replaceOpWithMultiple(op, newValues: {results}); |
| 805 | return success(); |
| 806 | } |
| 807 | |
| 808 | //===----------------------------------------------------------------------===// |
| 809 | // Pass Definition |
| 810 | //===----------------------------------------------------------------------===// |
| 811 | |
| 812 | namespace { |
| 813 | struct ArithToLLVMConversionPass |
| 814 | : public impl::ArithToLLVMConversionPassBase<ArithToLLVMConversionPass> { |
| 815 | using Base::Base; |
| 816 | |
| 817 | void runOnOperation() override { |
| 818 | LLVMConversionTarget target(getContext()); |
| 819 | RewritePatternSet patterns(&getContext()); |
| 820 | |
| 821 | const auto &dataLayoutAnalysis = getAnalysis<DataLayoutAnalysis>(); |
| 822 | LowerToLLVMOptions options(&getContext(), |
| 823 | dataLayoutAnalysis.getAtOrAbove(operation: getOperation())); |
| 824 | if (indexBitwidth != kDeriveIndexBitwidthFromDataLayout) |
| 825 | options.overrideIndexBitwidth(bitwidth: indexBitwidth); |
| 826 | |
| 827 | LLVMTypeConverter converter(&getContext(), options, &dataLayoutAnalysis); |
| 828 | arith::populateCeilFloorDivExpandOpsPatterns(patterns); |
| 829 | arith::populateArithToLLVMConversionPatterns(converter, patterns); |
| 830 | |
| 831 | if (failed(Result: applyPartialConversion(op: getOperation(), target, |
| 832 | patterns: std::move(patterns)))) |
| 833 | signalPassFailure(); |
| 834 | } |
| 835 | }; |
| 836 | } // namespace |
| 837 | |
| 838 | //===----------------------------------------------------------------------===// |
| 839 | // ConvertToLLVMPatternInterface implementation |
| 840 | //===----------------------------------------------------------------------===// |
| 841 | |
| 842 | namespace { |
| 843 | /// Implement the interface to convert MemRef to LLVM. |
| 844 | struct ArithToLLVMDialectInterface : public ConvertToLLVMPatternInterface { |
| 845 | ArithToLLVMDialectInterface(Dialect *dialect) |
| 846 | : ConvertToLLVMPatternInterface(dialect) {} |
| 847 | |
| 848 | void loadDependentDialects(MLIRContext *context) const final { |
| 849 | context->loadDialect<LLVM::LLVMDialect>(); |
| 850 | } |
| 851 | |
| 852 | /// Hook for derived dialect interface to provide conversion patterns |
| 853 | /// and mark dialect legal for the conversion target. |
| 854 | void populateConvertToLLVMConversionPatterns( |
| 855 | ConversionTarget &target, LLVMTypeConverter &typeConverter, |
| 856 | RewritePatternSet &patterns) const final { |
| 857 | arith::populateCeilFloorDivExpandOpsPatterns(patterns); |
| 858 | arith::populateArithToLLVMConversionPatterns(converter: typeConverter, patterns); |
| 859 | } |
| 860 | }; |
| 861 | } // namespace |
| 862 | |
| 863 | void mlir::arith::registerConvertArithToLLVMInterface( |
| 864 | DialectRegistry ®istry) { |
| 865 | registry.addExtension(extensionFn: +[](MLIRContext *ctx, arith::ArithDialect *dialect) { |
| 866 | dialect->addInterfaces<ArithToLLVMDialectInterface>(); |
| 867 | }); |
| 868 | } |
| 869 | |
| 870 | //===----------------------------------------------------------------------===// |
| 871 | // Pattern Population |
| 872 | //===----------------------------------------------------------------------===// |
| 873 | |
| 874 | void mlir::arith::populateArithToLLVMConversionPatterns( |
| 875 | const LLVMTypeConverter &converter, RewritePatternSet &patterns) { |
| 876 | |
| 877 | // Set a higher pattern benefit for IdentityBitcastLowering so it will run |
| 878 | // before BitcastOpLowering. |
| 879 | patterns.add<IdentityBitcastLowering>(arg: converter, args: patterns.getContext(), |
| 880 | /*patternBenefit*/ args: 10); |
| 881 | |
| 882 | // clang-format off |
| 883 | patterns.add< |
| 884 | AddFOpLowering, |
| 885 | ConstrainedAddFOpLowering, |
| 886 | AddIOpLowering, |
| 887 | AndIOpLowering, |
| 888 | AddUIExtendedOpLowering, |
| 889 | SubUIExtendedOpLowering, |
| 890 | BitcastOpLowering, |
| 891 | ConstantOpLowering, |
| 892 | CmpFOpLowering, |
| 893 | CmpIOpLowering, |
| 894 | DivFOpLowering, |
| 895 | ConstrainedDivFOpLowering, |
| 896 | DivSIOpLowering, |
| 897 | DivUIOpLowering, |
| 898 | ExtFOpLowering, |
| 899 | ExtSIOpLowering, |
| 900 | ExtUIOpLowering, |
| 901 | ConvertFOpLowering, |
| 902 | FPToSIOpLowering, |
| 903 | FPToUIOpLowering, |
| 904 | IndexCastOpSILowering, |
| 905 | IndexCastOpUILowering, |
| 906 | MaximumFOpLowering, |
| 907 | MaxNumFOpLowering, |
| 908 | MaximumNumFOpLowering, |
| 909 | MaxSIOpLowering, |
| 910 | MaxUIOpLowering, |
| 911 | MinimumFOpLowering, |
| 912 | MinNumFOpLowering, |
| 913 | MinimumNumFOpLowering, |
| 914 | MinSIOpLowering, |
| 915 | MinUIOpLowering, |
| 916 | MulFOpLowering, |
| 917 | ConstrainedMulFOpLowering, |
| 918 | MulIOpLowering, |
| 919 | MulSIExtendedOpLowering, |
| 920 | MulUIExtendedOpLowering, |
| 921 | NegFOpLowering, |
| 922 | OrIOpLowering, |
| 923 | RemFOpLowering, |
| 924 | RemSIOpLowering, |
| 925 | RemUIOpLowering, |
| 926 | SelectOpLowering, |
| 927 | SelectOpOneToNLowering, |
| 928 | ShLIOpLowering, |
| 929 | ShRSIOpLowering, |
| 930 | ShRUIOpLowering, |
| 931 | SIToFPOpLowering, |
| 932 | SubFOpLowering, |
| 933 | ConstrainedSubFOpLowering, |
| 934 | SubIOpLowering, |
| 935 | TruncFOpLowering, |
| 936 | ConstrainedTruncFOpLowering, |
| 937 | TruncIOpLowering, |
| 938 | UIToFPOpLowering, |
| 939 | XOrIOpLowering |
| 940 | >(arg: converter); |
| 941 | // clang-format on |
| 942 | } |
| 943 | |