1//===- ArithToLLVM.cpp - Arithmetic to LLVM dialect conversion -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
10
11#include "mlir/Analysis/DataLayoutAnalysis.h"
12#include "mlir/Conversion/ArithAttrToLLVM/AttrToLLVMConverter.h"
13#include "mlir/Conversion/ArithCommon/CeilFloorDivExpansion.h"
14#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
15#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
16#include "mlir/Conversion/LLVMCommon/VectorPattern.h"
17#include "mlir/Dialect/Arith/IR/Arith.h"
18#include "mlir/Dialect/LLVMIR/FunctionCallUtils.h"
19#include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h"
20#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
21#include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
22#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
23#include "mlir/Dialect/LLVMIR/LLVMConstrainedIntrinsicOps.h"
24#include "mlir/Dialect/LLVMIR/LLVMConversionOps.h"
25#include "mlir/Dialect/LLVMIR/LLVMFloatIntrinsicOps.h"
26#include "mlir/Dialect/LLVMIR/LLVMIntegerIntrinsicOps.h"
27#include "mlir/IR/DialectResourceBlobManager.h"
28#include "mlir/IR/TypeUtilities.h"
29#include "mlir/Pass/Pass.h"
30#include <type_traits>
31
32namespace mlir {
33#define GEN_PASS_DEF_ARITHTOLLVMCONVERSIONPASS
34#include "mlir/Conversion/Passes.h.inc"
35} // namespace mlir
36
37using namespace mlir;
38
39namespace {
40
41/// Lowering pattern that matches only when the source op's rounding mode
42/// presence agrees with `HasRoundingMode`. This allows registering two
43/// instances of the same pattern for one source op: one that handles the
44/// unconstrained case (no rounding mode, lowering to a regular LLVM op) and
45/// one that handles the constrained case (rounding mode present, lowering to
46/// a constrained LLVM intrinsic).
47///
48/// * `HasRoundingMode`: the pattern matches if and only if the source op has
49/// a rounding mode attribute.
50/// * `AttrConvert`: attribute converter to translate source attributes to
51/// target attributes.
52/// * `FailOnUnsupportedFP`: whether to fail if the source op has unsupported
53/// floating point types.
54template <typename SourceOp, typename TargetOp, bool HasRoundingMode,
55 template <typename, typename> typename AttrConvert =
56 AttrConvertPassThrough,
57 bool FailOnUnsupportedFP = false>
58struct ConstrainedVectorConvertToLLVMPattern
59 : public VectorConvertToLLVMPattern<SourceOp, TargetOp, AttrConvert,
60 FailOnUnsupportedFP> {
61 using VectorConvertToLLVMPattern<
62 SourceOp, TargetOp, AttrConvert,
63 FailOnUnsupportedFP>::VectorConvertToLLVMPattern;
64
65 LogicalResult
66 matchAndRewrite(SourceOp op, typename SourceOp::Adaptor adaptor,
67 ConversionPatternRewriter &rewriter) const override {
68 if (HasRoundingMode != static_cast<bool>(op.getRoundingModeAttr()))
69 return failure();
70 return VectorConvertToLLVMPattern<
71 SourceOp, TargetOp, AttrConvert,
72 FailOnUnsupportedFP>::matchAndRewrite(op, adaptor, rewriter);
73 }
74};
75
76/// No-op bitcast. Propagate type input arg if converted source and dest types
77/// are the same.
78struct IdentityBitcastLowering final
79 : public OpConversionPattern<arith::BitcastOp> {
80 using Base::Base;
81
82 LogicalResult
83 matchAndRewrite(arith::BitcastOp op, OpAdaptor adaptor,
84 ConversionPatternRewriter &rewriter) const final {
85 Value src = adaptor.getIn();
86 Type resultType = getTypeConverter()->convertType(t: op.getType());
87 if (src.getType() != resultType)
88 return rewriter.notifyMatchFailure(arg&: op, msg: "Types are different");
89
90 rewriter.replaceOp(op, newValues: src);
91 return success();
92 }
93};
94
95//===----------------------------------------------------------------------===//
96// Straightforward Op Lowerings
97//===----------------------------------------------------------------------===//
98
99using AddFOpLowering =
100 ConstrainedVectorConvertToLLVMPattern<arith::AddFOp, LLVM::FAddOp,
101 /*HasRoundingMode=*/false,
102 arith::AttrConvertFastMathToLLVM,
103 /*FailOnUnsupportedFP=*/true>;
104using ConstrainedAddFOpLowering = ConstrainedVectorConvertToLLVMPattern<
105 arith::AddFOp, LLVM::ConstrainedFAddIntr, /*HasRoundingMode=*/true,
106 arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
107using AddIOpLowering =
108 VectorConvertToLLVMPattern<arith::AddIOp, LLVM::AddOp,
109 arith::AttrConvertOverflowToLLVM>;
110using AndIOpLowering = VectorConvertToLLVMPattern<arith::AndIOp, LLVM::AndOp>;
111using BitcastOpLowering =
112 VectorConvertToLLVMPattern<arith::BitcastOp, LLVM::BitcastOp>;
113using DivFOpLowering =
114 ConstrainedVectorConvertToLLVMPattern<arith::DivFOp, LLVM::FDivOp,
115 /*HasRoundingMode=*/false,
116 arith::AttrConvertFastMathToLLVM,
117 /*FailOnUnsupportedFP=*/true>;
118using ConstrainedDivFOpLowering = ConstrainedVectorConvertToLLVMPattern<
119 arith::DivFOp, LLVM::ConstrainedFDivIntr, /*HasRoundingMode=*/true,
120 arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
121using DivSIOpLowering =
122 VectorConvertToLLVMPattern<arith::DivSIOp, LLVM::SDivOp>;
123using DivUIOpLowering =
124 VectorConvertToLLVMPattern<arith::DivUIOp, LLVM::UDivOp>;
125using ExtFOpLowering =
126 VectorConvertToLLVMPattern<arith::ExtFOp, LLVM::FPExtOp,
127 arith::AttrConvertFastMathToLLVM,
128 /*FailOnUnsupportedFP=*/true>;
129using ExtSIOpLowering =
130 VectorConvertToLLVMPattern<arith::ExtSIOp, LLVM::SExtOp>;
131using ExtUIOpLowering =
132 VectorConvertToLLVMPattern<arith::ExtUIOp, LLVM::ZExtOp,
133 arith::AttrConvertNonNegToLLVM>;
134using FPToSIOpLowering =
135 VectorConvertToLLVMPattern<arith::FPToSIOp, LLVM::FPToSIOp,
136 AttrConvertPassThrough,
137 /*FailOnUnsupportedFP=*/true>;
138using FPToUIOpLowering =
139 VectorConvertToLLVMPattern<arith::FPToUIOp, LLVM::FPToUIOp,
140 AttrConvertPassThrough,
141 /*FailOnUnsupportedFP=*/true>;
142using MaximumFOpLowering =
143 VectorConvertToLLVMPattern<arith::MaximumFOp, LLVM::MaximumOp,
144 arith::AttrConvertFastMathToLLVM,
145 /*FailOnUnsupportedFP=*/true>;
146using MaxNumFOpLowering =
147 VectorConvertToLLVMPattern<arith::MaxNumFOp, LLVM::MaxNumOp,
148 arith::AttrConvertFastMathToLLVM,
149 /*FailOnUnsupportedFP=*/true>;
150using MaximumNumFOpLowering =
151 VectorConvertToLLVMPattern<arith::MaximumNumFOp, LLVM::MaximumNumOp,
152 arith::AttrConvertFastMathToLLVM,
153 /*FailOnUnsupportedFP=*/true>;
154using MaxSIOpLowering =
155 VectorConvertToLLVMPattern<arith::MaxSIOp, LLVM::SMaxOp>;
156using MaxUIOpLowering =
157 VectorConvertToLLVMPattern<arith::MaxUIOp, LLVM::UMaxOp>;
158using MinimumFOpLowering =
159 VectorConvertToLLVMPattern<arith::MinimumFOp, LLVM::MinimumOp,
160 arith::AttrConvertFastMathToLLVM,
161 /*FailOnUnsupportedFP=*/true>;
162using MinNumFOpLowering =
163 VectorConvertToLLVMPattern<arith::MinNumFOp, LLVM::MinNumOp,
164 arith::AttrConvertFastMathToLLVM,
165 /*FailOnUnsupportedFP=*/true>;
166using MinimumNumFOpLowering =
167 VectorConvertToLLVMPattern<arith::MinimumNumFOp, LLVM::MinimumNumOp,
168 arith::AttrConvertFastMathToLLVM,
169 /*FailOnUnsupportedFP=*/true>;
170using MinSIOpLowering =
171 VectorConvertToLLVMPattern<arith::MinSIOp, LLVM::SMinOp>;
172using MinUIOpLowering =
173 VectorConvertToLLVMPattern<arith::MinUIOp, LLVM::UMinOp>;
174using MulFOpLowering =
175 ConstrainedVectorConvertToLLVMPattern<arith::MulFOp, LLVM::FMulOp,
176 /*HasRoundingMode=*/false,
177 arith::AttrConvertFastMathToLLVM,
178 /*FailOnUnsupportedFP=*/true>;
179using ConstrainedMulFOpLowering = ConstrainedVectorConvertToLLVMPattern<
180 arith::MulFOp, LLVM::ConstrainedFMulIntr, /*HasRoundingMode=*/true,
181 arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
182using MulIOpLowering =
183 VectorConvertToLLVMPattern<arith::MulIOp, LLVM::MulOp,
184 arith::AttrConvertOverflowToLLVM>;
185using NegFOpLowering =
186 VectorConvertToLLVMPattern<arith::NegFOp, LLVM::FNegOp,
187 arith::AttrConvertFastMathToLLVM,
188 /*FailOnUnsupportedFP=*/true>;
189using OrIOpLowering = VectorConvertToLLVMPattern<arith::OrIOp, LLVM::OrOp>;
190using RemFOpLowering =
191 VectorConvertToLLVMPattern<arith::RemFOp, LLVM::FRemOp,
192 arith::AttrConvertFastMathToLLVM,
193 /*FailOnUnsupportedFP=*/true>;
194using RemSIOpLowering =
195 VectorConvertToLLVMPattern<arith::RemSIOp, LLVM::SRemOp>;
196using RemUIOpLowering =
197 VectorConvertToLLVMPattern<arith::RemUIOp, LLVM::URemOp>;
198using SelectOpLowering =
199 VectorConvertToLLVMPattern<arith::SelectOp, LLVM::SelectOp>;
200using ShLIOpLowering =
201 VectorConvertToLLVMPattern<arith::ShLIOp, LLVM::ShlOp,
202 arith::AttrConvertOverflowToLLVM>;
203using ShRSIOpLowering =
204 VectorConvertToLLVMPattern<arith::ShRSIOp, LLVM::AShrOp>;
205using ShRUIOpLowering =
206 VectorConvertToLLVMPattern<arith::ShRUIOp, LLVM::LShrOp>;
207using SIToFPOpLowering =
208 VectorConvertToLLVMPattern<arith::SIToFPOp, LLVM::SIToFPOp,
209 AttrConvertPassThrough,
210 /*FailOnUnsupportedFP=*/true>;
211using SubFOpLowering =
212 ConstrainedVectorConvertToLLVMPattern<arith::SubFOp, LLVM::FSubOp,
213 /*HasRoundingMode=*/false,
214 arith::AttrConvertFastMathToLLVM,
215 /*FailOnUnsupportedFP=*/true>;
216using ConstrainedSubFOpLowering = ConstrainedVectorConvertToLLVMPattern<
217 arith::SubFOp, LLVM::ConstrainedFSubIntr, /*HasRoundingMode=*/true,
218 arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
219using SubIOpLowering =
220 VectorConvertToLLVMPattern<arith::SubIOp, LLVM::SubOp,
221 arith::AttrConvertOverflowToLLVM>;
222using TruncFOpLowering =
223 ConstrainedVectorConvertToLLVMPattern<arith::TruncFOp, LLVM::FPTruncOp,
224 /*HasRoundingMode=*/false,
225 arith::AttrConvertFastMathToLLVM,
226 /*FailOnUnsupportedFP=*/true>;
227using ConstrainedTruncFOpLowering = ConstrainedVectorConvertToLLVMPattern<
228 arith::TruncFOp, LLVM::ConstrainedFPTruncIntr, /*HasRoundingMode=*/true,
229 arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
230using TruncIOpLowering =
231 VectorConvertToLLVMPattern<arith::TruncIOp, LLVM::TruncOp,
232 arith::AttrConvertOverflowToLLVM>;
233using UIToFPOpLowering =
234 VectorConvertToLLVMPattern<arith::UIToFPOp, LLVM::UIToFPOp,
235 arith::AttrConvertNonNegToLLVM,
236 /*FailOnUnsupportedFP=*/true>;
237using XOrIOpLowering = VectorConvertToLLVMPattern<arith::XOrIOp, LLVM::XOrOp>;
238
239//===----------------------------------------------------------------------===//
240// Op Lowering Patterns
241//===----------------------------------------------------------------------===//
242
243/// Directly lower to LLVM op.
244struct ConstantOpLowering : public ConvertOpToLLVMPattern<arith::ConstantOp> {
245 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
246
247 LogicalResult
248 matchAndRewrite(arith::ConstantOp op, OpAdaptor adaptor,
249 ConversionPatternRewriter &rewriter) const override;
250};
251
252/// The lowering of index_cast becomes an integer conversion since index
253/// becomes an integer. If the bit width of the source and target integer
254/// types is the same, just erase the cast. If the target type is wider,
255/// sign-extend the value, otherwise truncate it.
256template <typename OpTy, typename ExtCastTy>
257struct IndexCastOpLowering : public ConvertOpToLLVMPattern<OpTy> {
258 using ConvertOpToLLVMPattern<OpTy>::ConvertOpToLLVMPattern;
259
260 LogicalResult
261 matchAndRewrite(OpTy op, typename OpTy::Adaptor adaptor,
262 ConversionPatternRewriter &rewriter) const override;
263};
264
265using IndexCastOpSILowering =
266 IndexCastOpLowering<arith::IndexCastOp, LLVM::SExtOp>;
267using IndexCastOpUILowering =
268 IndexCastOpLowering<arith::IndexCastUIOp, LLVM::ZExtOp>;
269
270struct AddUIExtendedOpLowering
271 : public ConvertOpToLLVMPattern<arith::AddUIExtendedOp> {
272 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
273
274 LogicalResult
275 matchAndRewrite(arith::AddUIExtendedOp op, OpAdaptor adaptor,
276 ConversionPatternRewriter &rewriter) const override;
277};
278
279struct SubUIExtendedOpLowering
280 : public ConvertOpToLLVMPattern<arith::SubUIExtendedOp> {
281 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
282
283 LogicalResult
284 matchAndRewrite(arith::SubUIExtendedOp op, OpAdaptor adaptor,
285 ConversionPatternRewriter &rewriter) const override;
286};
287
288template <typename ArithMulOp, bool IsSigned>
289struct MulIExtendedOpLowering : public ConvertOpToLLVMPattern<ArithMulOp> {
290 using ConvertOpToLLVMPattern<ArithMulOp>::ConvertOpToLLVMPattern;
291
292 LogicalResult
293 matchAndRewrite(ArithMulOp op, typename ArithMulOp::Adaptor adaptor,
294 ConversionPatternRewriter &rewriter) const override;
295};
296
297using MulSIExtendedOpLowering =
298 MulIExtendedOpLowering<arith::MulSIExtendedOp, true>;
299using MulUIExtendedOpLowering =
300 MulIExtendedOpLowering<arith::MulUIExtendedOp, false>;
301
302struct CmpIOpLowering : public ConvertOpToLLVMPattern<arith::CmpIOp> {
303 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
304
305 LogicalResult
306 matchAndRewrite(arith::CmpIOp op, OpAdaptor adaptor,
307 ConversionPatternRewriter &rewriter) const override;
308};
309
310struct CmpFOpLowering : public ConvertOpToLLVMPattern<arith::CmpFOp> {
311 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
312
313 LogicalResult
314 matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor,
315 ConversionPatternRewriter &rewriter) const override;
316};
317
318/// Lower arith.convertf (same-bitwidth FP cast) to LLVM.
319///
320/// Extends to f32 via llvm.fpext, then truncates to the target type via
321/// llvm.fptrunc. This handles bf16 <-> f16, which is the only same-bitwidth
322/// pair of LLVM-supported FP types.
323struct ConvertFOpLowering : public ConvertOpToLLVMPattern<arith::ConvertFOp> {
324 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
325
326 LogicalResult
327 matchAndRewrite(arith::ConvertFOp op, OpAdaptor adaptor,
328 ConversionPatternRewriter &rewriter) const override {
329 if (LLVM::detail::opHasUnsupportedFloatingPointTypes(op,
330 typeConverter: *getTypeConverter()))
331 return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported floating point type");
332
333 // Only bf16 <-> f16 conversions are supported. There is currently no other
334 // pair of FP types that are valid LLVM types.
335 [[maybe_unused]] auto srcType = getElementTypeOrSelf(type: op.getIn().getType());
336 [[maybe_unused]] auto dstType = getElementTypeOrSelf(type: op.getType());
337 assert(((srcType.isBF16() && dstType.isF16()) ||
338 (srcType.isF16() && dstType.isBF16())) &&
339 "only bf16 <-> f16 conversions are supported");
340
341 Type convertedType = getTypeConverter()->convertType(t: op.getType());
342 if (!convertedType)
343 return rewriter.notifyMatchFailure(arg&: op, msg: "failed to convert result type");
344
345 Value input = adaptor.getIn();
346 Location loc = op.getLoc();
347
348 if (!isa<LLVM::LLVMArrayType>(Val: input.getType())) {
349 rewriter.replaceOp(op,
350 newValues: emitConversion(rewriter, loc, input, targetType: convertedType));
351 return success();
352 }
353
354 if (!isa<VectorType>(Val: op.getType()))
355 return rewriter.notifyMatchFailure(arg&: op, msg: "expected vector result type");
356
357 return LLVM::detail::handleMultidimensionalVectors(
358 op: op.getOperation(), operands: adaptor.getOperands(), typeConverter: *getTypeConverter(),
359 createOperand: [&](Type llvm1DVectorTy, ValueRange operands) -> Value {
360 return emitConversion(rewriter, loc, input: operands.front(),
361 targetType: llvm1DVectorTy);
362 },
363 rewriter);
364 }
365
366private:
367 static Value emitConversion(ConversionPatternRewriter &rewriter, Location loc,
368 Value input, Type targetType) {
369 Type f32Scalar = Float32Type::get(context: rewriter.getContext());
370 Type f32Ty = f32Scalar;
371 if (auto vecTy = dyn_cast<VectorType>(Val&: targetType))
372 f32Ty = VectorType::get(shape: vecTy.getShape(), elementType: f32Scalar);
373
374 Value ext = LLVM::FPExtOp::create(builder&: rewriter, location: loc, res: f32Ty, arg: input);
375 return LLVM::FPTruncOp::create(builder&: rewriter, location: loc, res: targetType, arg: ext);
376 }
377};
378
379struct SelectOpOneToNLowering : public ConvertOpToLLVMPattern<arith::SelectOp> {
380 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
381 using Adaptor = ConvertOpToLLVMPattern<arith::SelectOp>::OneToNOpAdaptor;
382
383 LogicalResult
384 matchAndRewrite(arith::SelectOp op, Adaptor adaptor,
385 ConversionPatternRewriter &rewriter) const override;
386};
387
388} // namespace
389
390//===----------------------------------------------------------------------===//
391// ConstantOpLowering
392//===----------------------------------------------------------------------===//
393
394/// Retypes `attr` for a `llvm.mlir.constant` of `resultType`. `arith.constant`
395/// requires the value attribute and the result to have the same type, but the
396/// type converter may map the element type to a different one, e.g. `index` to
397/// `i32` or `i64` depending on the configured index bitwidth. Build the
398/// attribute from the converted type so that the two agree wherever that is
399/// representable. Returns a null attribute if `attr` cannot be used for
400/// `resultType`.
401static TypedAttr convertConstantValue(TypedAttr attr, Type resultType) {
402 // Compare the element types, but explicitly ignore the non-scalar portions of
403 // types. This relaxation is required to support multi-dimensional vector
404 // constant that result in nested LLVM array values.
405 Type sourceElementType = LLVM::getConstantElementType(type: attr.getType());
406 Type targetElementType = LLVM::getConstantElementType(type: resultType);
407 if (sourceElementType == targetElementType)
408 return attr;
409
410 auto targetIntType = dyn_cast<IntegerType>(Val&: targetElementType);
411 if (!targetIntType)
412 return {};
413
414 // The converter maps the low-precision float types that have no LLVM
415 // equivalent to an integer of the same width. The attribute stays a float
416 // attribute in that case.
417 if (auto sourceFloatType = dyn_cast<FloatType>(Val&: sourceElementType)) {
418 if (sourceFloatType.getWidth() != targetIntType.getWidth())
419 return {};
420 return attr;
421 }
422
423 // Apart from those floats, `index` is the only element type the converter
424 // rewrites, so anything else is a malformed `arith.constant`. Bail out rather
425 // than reinterpret its value, which would silently change the constant (e.g.
426 // sign-extending an `i1` `true` to -1).
427 if (!isa<IndexType>(Val: sourceElementType))
428 return {};
429 // `index` is signless but holds signed values, so narrowing to a smaller
430 // index bitwidth truncates and widening sign-extends.
431 unsigned width = targetIntType.getWidth();
432
433 if (auto intAttr = dyn_cast<IntegerAttr>(Val&: attr))
434 return IntegerAttr::get(type: targetIntType,
435 value: intAttr.getValue().sextOrTrunc(width));
436
437 auto retypeValues = [&](DenseIntElementsAttr values) {
438 return values.mapValues(newElementType: targetIntType, mapping: [&](const APInt &value) {
439 return value.sextOrTrunc(width);
440 });
441 };
442
443 if (auto denseAttr = dyn_cast<DenseIntElementsAttr>(Val&: attr))
444 return retypeValues(denseAttr);
445
446 if (auto sparseAttr = dyn_cast<SparseElementsAttr>(Val&: attr))
447 return SparseElementsAttr::get(
448 type: cast<ShapedType>(Val: attr.getType()).clone(elementType: targetIntType),
449 indices: sparseAttr.getIndices(),
450 values: retypeValues(cast<DenseIntElementsAttr>(Val: sparseAttr.getValues())));
451
452 // A resource-backed elements attribute refers to a blob laid out for its own
453 // element type. The blob cannot be rewritten here, only reinterpreted, which
454 // is correct exactly when the target type has the same width as the storage
455 // `index` uses in a blob.
456 if (auto resourceAttr = dyn_cast<DenseResourceElementsAttr>(Val&: attr)) {
457 if (width != IndexType::kInternalStorageBitWidth)
458 return {};
459 return DenseResourceElementsAttr::get(
460 type: cast<ShapedType>(Val: attr.getType()).clone(elementType: targetIntType),
461 handle: resourceAttr.getRawHandle());
462 }
463
464 return {};
465}
466
467LogicalResult
468ConstantOpLowering::matchAndRewrite(arith::ConstantOp op, OpAdaptor adaptor,
469 ConversionPatternRewriter &rewriter) const {
470 Type resultType = getTypeConverter()->convertType(t: op.getType());
471 if (!resultType)
472 return rewriter.notifyMatchFailure(arg&: op, msg: "failed to convert result type");
473
474 TypedAttr value = convertConstantValue(attr: op.getValue(), resultType);
475 if (!value)
476 return rewriter.notifyMatchFailure(
477 arg&: op, msg: "failed to convert value attribute to the converted result type");
478
479 // `arith.constant` has no operands and a single result, so there is nothing
480 // for `oneToOneRewrite` to do here beyond converting `resultType` a second
481 // time.
482 DictionaryAttr discardableAttrs = op->getDiscardableAttrDictionary();
483 auto constantOp =
484 LLVM::ConstantOp::create(builder&: rewriter, location: op.getLoc(), res: resultType, value);
485 constantOp->setDiscardableAttrs(discardableAttrs);
486 rewriter.replaceOp(op, newOp: constantOp);
487 return success();
488}
489
490//===----------------------------------------------------------------------===//
491// IndexCastOpLowering
492//===----------------------------------------------------------------------===//
493
494template <typename OpTy, typename ExtCastTy>
495LogicalResult IndexCastOpLowering<OpTy, ExtCastTy>::matchAndRewrite(
496 OpTy op, typename OpTy::Adaptor adaptor,
497 ConversionPatternRewriter &rewriter) const {
498 Type resultType = op.getResult().getType();
499 Type targetElementType =
500 this->typeConverter->convertType(getElementTypeOrSelf(type: resultType));
501 Type sourceElementType =
502 this->typeConverter->convertType(getElementTypeOrSelf(op.getIn()));
503 unsigned targetBits = targetElementType.getIntOrFloatBitWidth();
504 unsigned sourceBits = sourceElementType.getIntOrFloatBitWidth();
505
506 if (targetBits == sourceBits) {
507 rewriter.replaceOp(op, adaptor.getIn());
508 return success();
509 }
510
511 // Memref index_cast is a no-op at the LLVM level since LLVM uses opaque
512 // pointers and memrefs of different integer/index element types all convert
513 // to the same LLVM struct type.
514 if (isa<MemRefType>(op.getIn().getType())) {
515 rewriter.replaceOp(op, adaptor.getIn());
516 return success();
517 }
518
519 bool isNonNeg = false;
520 if constexpr (std::is_same_v<ExtCastTy, LLVM::ZExtOp>)
521 isNonNeg = op.getNonNeg();
522
523 // Handle the scalar and 1D vector cases.
524 Type operandType = adaptor.getIn().getType();
525 if (!isa<LLVM::LLVMArrayType>(Val: operandType)) {
526 Type targetType = this->typeConverter->convertType(resultType);
527 if (targetBits < sourceBits) {
528 rewriter.replaceOpWithNewOp<LLVM::TruncOp>(op, targetType,
529 adaptor.getIn());
530 } else {
531 auto extOp = rewriter.replaceOpWithNewOp<ExtCastTy>(op, targetType,
532 adaptor.getIn());
533 if constexpr (std::is_same_v<ExtCastTy, LLVM::ZExtOp>)
534 extOp.setNonNeg(isNonNeg);
535 }
536 return success();
537 }
538
539 if (!isa<VectorType>(Val: resultType))
540 return rewriter.notifyMatchFailure(op, "expected vector result type");
541
542 return LLVM::detail::handleMultidimensionalVectors(
543 op: op.getOperation(), operands: adaptor.getOperands(), typeConverter: *(this->getTypeConverter()),
544 createOperand: [&](Type llvm1DVectorTy, ValueRange operands) -> Value {
545 typename OpTy::Adaptor adaptor(operands);
546 if (targetBits < sourceBits) {
547 return LLVM::TruncOp::create(rewriter, op.getLoc(), llvm1DVectorTy,
548 adaptor.getIn());
549 }
550 auto extOp = ExtCastTy::create(rewriter, op.getLoc(), llvm1DVectorTy,
551 adaptor.getIn());
552 if constexpr (std::is_same_v<ExtCastTy, LLVM::ZExtOp>) {
553 if (isNonNeg)
554 extOp.setNonNeg(true);
555 }
556 return extOp;
557 },
558 rewriter);
559}
560
561//===----------------------------------------------------------------------===//
562// AddUIExtendedOpLowering
563//===----------------------------------------------------------------------===//
564
565LogicalResult AddUIExtendedOpLowering::matchAndRewrite(
566 arith::AddUIExtendedOp op, OpAdaptor adaptor,
567 ConversionPatternRewriter &rewriter) const {
568 Type operandType = adaptor.getLhs().getType();
569 Type sumResultType = op.getSum().getType();
570 Type overflowResultType = op.getOverflow().getType();
571
572 if (!LLVM::isCompatibleType(type: operandType))
573 return failure();
574
575 MLIRContext *ctx = rewriter.getContext();
576 Location loc = op.getLoc();
577
578 // Handle the scalar and 1D vector cases.
579 if (!isa<LLVM::LLVMArrayType>(Val: operandType)) {
580 Type newSumType = typeConverter->convertType(t: sumResultType);
581 Type newOverflowType = typeConverter->convertType(t: overflowResultType);
582 Type structType =
583 LLVM::LLVMStructType::getLiteral(context: ctx, types: {newSumType, newOverflowType});
584 Value addOverflow = LLVM::UAddWithOverflowOp::create(
585 builder&: rewriter, location: loc, res: structType, odsArg_0: adaptor.getLhs(), odsArg_1: adaptor.getRhs());
586 Value sumExtracted =
587 LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: addOverflow, position: 0);
588 Value overflowExtracted =
589 LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: addOverflow, position: 1);
590 rewriter.replaceOp(op, newValues: {sumExtracted, overflowExtracted});
591 return success();
592 }
593
594 if (!isa<VectorType>(Val: sumResultType))
595 return rewriter.notifyMatchFailure(arg&: loc, msg: "expected vector result types");
596
597 return rewriter.notifyMatchFailure(arg&: loc,
598 msg: "ND vector types are not supported yet");
599}
600
601//===----------------------------------------------------------------------===//
602// SubUIExtendedOpLowering
603//===----------------------------------------------------------------------===//
604
605LogicalResult SubUIExtendedOpLowering::matchAndRewrite(
606 arith::SubUIExtendedOp op, OpAdaptor adaptor,
607 ConversionPatternRewriter &rewriter) const {
608 Type operandType = adaptor.getLhs().getType();
609 Type diffResultType = op.getDiff().getType();
610 Type borrowResultType = op.getBorrow().getType();
611
612 if (!LLVM::isCompatibleType(type: operandType))
613 return failure();
614
615 MLIRContext *ctx = rewriter.getContext();
616 Location loc = op.getLoc();
617
618 // Handle the scalar and 1D vector cases.
619 if (!isa<LLVM::LLVMArrayType>(Val: operandType)) {
620 Type newBorrowType = typeConverter->convertType(t: borrowResultType);
621 Type structType =
622 LLVM::LLVMStructType::getLiteral(context: ctx, types: {diffResultType, newBorrowType});
623 Value subOverflow = LLVM::USubWithOverflowOp::create(
624 builder&: rewriter, location: loc, res: structType, odsArg_0: adaptor.getLhs(), odsArg_1: adaptor.getRhs());
625 Value diffExtracted =
626 LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: subOverflow, position: 0);
627 Value borrowExtracted =
628 LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: subOverflow, position: 1);
629 rewriter.replaceOp(op, newValues: {diffExtracted, borrowExtracted});
630 return success();
631 }
632
633 if (!isa<VectorType>(Val: diffResultType))
634 return rewriter.notifyMatchFailure(arg&: loc, msg: "expected vector result types");
635
636 return rewriter.notifyMatchFailure(arg&: loc,
637 msg: "ND vector types are not supported yet");
638}
639
640//===----------------------------------------------------------------------===//
641// MulIExtendedOpLowering
642//===----------------------------------------------------------------------===//
643
644template <typename ArithMulOp, bool IsSigned>
645LogicalResult MulIExtendedOpLowering<ArithMulOp, IsSigned>::matchAndRewrite(
646 ArithMulOp op, typename ArithMulOp::Adaptor adaptor,
647 ConversionPatternRewriter &rewriter) const {
648 Type resultType = adaptor.getLhs().getType();
649
650 if (!LLVM::isCompatibleType(type: resultType))
651 return failure();
652
653 Location loc = op.getLoc();
654
655 // Handle the scalar and 1D vector cases. Because LLVM does not have a
656 // matching extended multiplication intrinsic, perform regular multiplication
657 // on operands zero-extended to i(2*N) bits, and truncate the results back to
658 // iN types.
659 if (!isa<LLVM::LLVMArrayType>(Val: resultType)) {
660 // Shift amount necessary to extract the high bits from widened result.
661 TypedAttr shiftValAttr;
662
663 if (auto intTy = dyn_cast<IntegerType>(Val&: resultType)) {
664 unsigned resultBitwidth = intTy.getWidth();
665 auto attrTy = rewriter.getIntegerType(width: resultBitwidth * 2);
666 shiftValAttr = rewriter.getIntegerAttr(type: attrTy, value: resultBitwidth);
667 } else {
668 auto vecTy = cast<VectorType>(Val&: resultType);
669 unsigned resultBitwidth = vecTy.getElementTypeBitWidth();
670 auto attrTy = VectorType::get(
671 shape: vecTy.getShape(), elementType: rewriter.getIntegerType(width: resultBitwidth * 2));
672 shiftValAttr = SplatElementsAttr::get(
673 type: attrTy, values: APInt(resultBitwidth * 2, resultBitwidth));
674 }
675 Type wideType = shiftValAttr.getType();
676 assert(LLVM::isCompatibleType(wideType) &&
677 "LLVM dialect should support all signless integer types");
678
679 using LLVMExtOp = std::conditional_t<IsSigned, LLVM::SExtOp, LLVM::ZExtOp>;
680 Value lhsExt = LLVMExtOp::create(rewriter, loc, wideType, adaptor.getLhs());
681 Value rhsExt = LLVMExtOp::create(rewriter, loc, wideType, adaptor.getRhs());
682 Value mulExt = LLVM::MulOp::create(builder&: rewriter, location: loc, res: wideType, lhs: lhsExt, rhs: rhsExt);
683
684 // Split the 2*N-bit wide result into two N-bit values.
685 Value low = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: resultType, arg: mulExt);
686 Value shiftVal = LLVM::ConstantOp::create(builder&: rewriter, location: loc, value: shiftValAttr);
687 Value highExt = LLVM::LShrOp::create(builder&: rewriter, location: loc, lhs: mulExt, rhs: shiftVal);
688 Value high = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: resultType, arg: highExt);
689
690 rewriter.replaceOp(op, {low, high});
691 return success();
692 }
693
694 if (!isa<VectorType>(Val: resultType))
695 return rewriter.notifyMatchFailure(op, "expected vector result type");
696
697 return rewriter.notifyMatchFailure(op,
698 "ND vector types are not supported yet");
699}
700
701//===----------------------------------------------------------------------===//
702// CmpIOpLowering
703//===----------------------------------------------------------------------===//
704
705// Convert arith.cmp predicate into the LLVM dialect CmpPredicate. The two enums
706// share numerical values so just cast.
707template <typename LLVMPredType, typename PredType>
708static LLVMPredType convertCmpPredicate(PredType pred) {
709 return static_cast<LLVMPredType>(pred);
710}
711
712LogicalResult
713CmpIOpLowering::matchAndRewrite(arith::CmpIOp op, OpAdaptor adaptor,
714 ConversionPatternRewriter &rewriter) const {
715 Type operandType = adaptor.getLhs().getType();
716 Type resultType = op.getResult().getType();
717
718 // Handle the scalar and 1D vector cases.
719 if (!isa<LLVM::LLVMArrayType>(Val: operandType)) {
720 rewriter.replaceOpWithNewOp<LLVM::ICmpOp>(
721 op, args: typeConverter->convertType(t: resultType),
722 args: convertCmpPredicate<LLVM::ICmpPredicate>(pred: op.getPredicate()),
723 args: adaptor.getLhs(), args: adaptor.getRhs());
724 return success();
725 }
726
727 if (!isa<VectorType>(Val: resultType))
728 return rewriter.notifyMatchFailure(arg&: op, msg: "expected vector result type");
729
730 return LLVM::detail::handleMultidimensionalVectors(
731 op: op.getOperation(), operands: adaptor.getOperands(), typeConverter: *getTypeConverter(),
732 createOperand: [&](Type llvm1DVectorTy, ValueRange operands) {
733 OpAdaptor adaptor(operands);
734 return LLVM::ICmpOp::create(
735 builder&: rewriter, location: op.getLoc(), res: llvm1DVectorTy,
736 predicate: convertCmpPredicate<LLVM::ICmpPredicate>(pred: op.getPredicate()),
737 lhs: adaptor.getLhs(), rhs: adaptor.getRhs());
738 },
739 rewriter);
740}
741
742//===----------------------------------------------------------------------===//
743// CmpFOpLowering
744//===----------------------------------------------------------------------===//
745
746LogicalResult
747CmpFOpLowering::matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor,
748 ConversionPatternRewriter &rewriter) const {
749 if (LLVM::detail::isUnsupportedFloatingPointType(typeConverter: *this->getTypeConverter(),
750 type: op.getLhs().getType()))
751 return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported floating point type");
752
753 Type operandType = adaptor.getLhs().getType();
754 Type resultType = op.getResult().getType();
755 LLVM::FastmathFlags fmf =
756 arith::convertArithFastMathFlagsToLLVM(arithFMF: op.getFastmath());
757
758 // Handle the scalar and 1D vector cases.
759 if (!isa<LLVM::LLVMArrayType>(Val: operandType)) {
760 rewriter.replaceOpWithNewOp<LLVM::FCmpOp>(
761 op, args: typeConverter->convertType(t: resultType),
762 args: convertCmpPredicate<LLVM::FCmpPredicate>(pred: op.getPredicate()),
763 args: adaptor.getLhs(), args: adaptor.getRhs(), args&: fmf);
764 return success();
765 }
766
767 if (!isa<VectorType>(Val: resultType))
768 return rewriter.notifyMatchFailure(arg&: op, msg: "expected vector result type");
769
770 return LLVM::detail::handleMultidimensionalVectors(
771 op: op.getOperation(), operands: adaptor.getOperands(), typeConverter: *getTypeConverter(),
772 createOperand: [&](Type llvm1DVectorTy, ValueRange operands) {
773 OpAdaptor adaptor(operands);
774 return LLVM::FCmpOp::create(
775 builder&: rewriter, location: op.getLoc(), res: llvm1DVectorTy,
776 predicate: convertCmpPredicate<LLVM::FCmpPredicate>(pred: op.getPredicate()),
777 lhs: adaptor.getLhs(), rhs: adaptor.getRhs(), fastmathFlags: fmf);
778 },
779 rewriter);
780}
781
782//===----------------------------------------------------------------------===//
783// SelectOpOneToNLowering
784//===----------------------------------------------------------------------===//
785
786/// Pattern for arith.select where the true/false values lower to multiple
787/// SSA values (1:N conversion). This pattern generates multiple arith.select
788/// than can be lowered by the 1:1 arith.select pattern.
789LogicalResult SelectOpOneToNLowering::matchAndRewrite(
790 arith::SelectOp op, Adaptor adaptor,
791 ConversionPatternRewriter &rewriter) const {
792 // In case of a 1:1 conversion, the 1:1 pattern will match.
793 if (llvm::hasSingleElement(C: adaptor.getTrueValue()))
794 return rewriter.notifyMatchFailure(
795 arg&: op, msg: "not a 1:N conversion, 1:1 pattern will match");
796 if (!op.getCondition().getType().isInteger(width: 1))
797 return rewriter.notifyMatchFailure(arg&: op,
798 msg: "non-i1 conditions are not supported");
799 SmallVector<Value> results;
800 for (auto [trueValue, falseValue] :
801 llvm::zip_equal(t: adaptor.getTrueValue(), u: adaptor.getFalseValue()))
802 results.push_back(Elt: arith::SelectOp::create(
803 builder&: rewriter, location: op.getLoc(), condition: op.getCondition(), true_value: trueValue, false_value: falseValue));
804 rewriter.replaceOpWithMultiple(op, newValues: {results});
805 return success();
806}
807
808//===----------------------------------------------------------------------===//
809// Pass Definition
810//===----------------------------------------------------------------------===//
811
812namespace {
813struct ArithToLLVMConversionPass
814 : public impl::ArithToLLVMConversionPassBase<ArithToLLVMConversionPass> {
815 using Base::Base;
816
817 void runOnOperation() override {
818 LLVMConversionTarget target(getContext());
819 RewritePatternSet patterns(&getContext());
820
821 const auto &dataLayoutAnalysis = getAnalysis<DataLayoutAnalysis>();
822 LowerToLLVMOptions options(&getContext(),
823 dataLayoutAnalysis.getAtOrAbove(operation: getOperation()));
824 if (indexBitwidth != kDeriveIndexBitwidthFromDataLayout)
825 options.overrideIndexBitwidth(bitwidth: indexBitwidth);
826
827 LLVMTypeConverter converter(&getContext(), options, &dataLayoutAnalysis);
828 arith::populateCeilFloorDivExpandOpsPatterns(patterns);
829 arith::populateArithToLLVMConversionPatterns(converter, patterns);
830
831 if (failed(Result: applyPartialConversion(op: getOperation(), target,
832 patterns: std::move(patterns))))
833 signalPassFailure();
834 }
835};
836} // namespace
837
838//===----------------------------------------------------------------------===//
839// ConvertToLLVMPatternInterface implementation
840//===----------------------------------------------------------------------===//
841
842namespace {
843/// Implement the interface to convert MemRef to LLVM.
844struct ArithToLLVMDialectInterface : public ConvertToLLVMPatternInterface {
845 ArithToLLVMDialectInterface(Dialect *dialect)
846 : ConvertToLLVMPatternInterface(dialect) {}
847
848 void loadDependentDialects(MLIRContext *context) const final {
849 context->loadDialect<LLVM::LLVMDialect>();
850 }
851
852 /// Hook for derived dialect interface to provide conversion patterns
853 /// and mark dialect legal for the conversion target.
854 void populateConvertToLLVMConversionPatterns(
855 ConversionTarget &target, LLVMTypeConverter &typeConverter,
856 RewritePatternSet &patterns) const final {
857 arith::populateCeilFloorDivExpandOpsPatterns(patterns);
858 arith::populateArithToLLVMConversionPatterns(converter: typeConverter, patterns);
859 }
860};
861} // namespace
862
863void mlir::arith::registerConvertArithToLLVMInterface(
864 DialectRegistry &registry) {
865 registry.addExtension(extensionFn: +[](MLIRContext *ctx, arith::ArithDialect *dialect) {
866 dialect->addInterfaces<ArithToLLVMDialectInterface>();
867 });
868}
869
870//===----------------------------------------------------------------------===//
871// Pattern Population
872//===----------------------------------------------------------------------===//
873
874void mlir::arith::populateArithToLLVMConversionPatterns(
875 const LLVMTypeConverter &converter, RewritePatternSet &patterns) {
876
877 // Set a higher pattern benefit for IdentityBitcastLowering so it will run
878 // before BitcastOpLowering.
879 patterns.add<IdentityBitcastLowering>(arg: converter, args: patterns.getContext(),
880 /*patternBenefit*/ args: 10);
881
882 // clang-format off
883 patterns.add<
884 AddFOpLowering,
885 ConstrainedAddFOpLowering,
886 AddIOpLowering,
887 AndIOpLowering,
888 AddUIExtendedOpLowering,
889 SubUIExtendedOpLowering,
890 BitcastOpLowering,
891 ConstantOpLowering,
892 CmpFOpLowering,
893 CmpIOpLowering,
894 DivFOpLowering,
895 ConstrainedDivFOpLowering,
896 DivSIOpLowering,
897 DivUIOpLowering,
898 ExtFOpLowering,
899 ExtSIOpLowering,
900 ExtUIOpLowering,
901 ConvertFOpLowering,
902 FPToSIOpLowering,
903 FPToUIOpLowering,
904 IndexCastOpSILowering,
905 IndexCastOpUILowering,
906 MaximumFOpLowering,
907 MaxNumFOpLowering,
908 MaximumNumFOpLowering,
909 MaxSIOpLowering,
910 MaxUIOpLowering,
911 MinimumFOpLowering,
912 MinNumFOpLowering,
913 MinimumNumFOpLowering,
914 MinSIOpLowering,
915 MinUIOpLowering,
916 MulFOpLowering,
917 ConstrainedMulFOpLowering,
918 MulIOpLowering,
919 MulSIExtendedOpLowering,
920 MulUIExtendedOpLowering,
921 NegFOpLowering,
922 OrIOpLowering,
923 RemFOpLowering,
924 RemSIOpLowering,
925 RemUIOpLowering,
926 SelectOpLowering,
927 SelectOpOneToNLowering,
928 ShLIOpLowering,
929 ShRSIOpLowering,
930 ShRUIOpLowering,
931 SIToFPOpLowering,
932 SubFOpLowering,
933 ConstrainedSubFOpLowering,
934 SubIOpLowering,
935 TruncFOpLowering,
936 ConstrainedTruncFOpLowering,
937 TruncIOpLowering,
938 UIToFPOpLowering,
939 XOrIOpLowering
940 >(arg: converter);
941 // clang-format on
942}
943