| 1 | //===- ComplexOps.cpp - MLIR Complex Operations ---------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 10 | #include "mlir/Dialect/Complex/IR/Complex.h" |
| 11 | #include "mlir/IR/BuiltinTypes.h" |
| 12 | #include "mlir/IR/PatternMatch.h" |
| 13 | |
| 14 | using namespace mlir; |
| 15 | using namespace mlir::complex; |
| 16 | |
| 17 | //===----------------------------------------------------------------------===// |
| 18 | // ConstantOp |
| 19 | //===----------------------------------------------------------------------===// |
| 20 | |
| 21 | OpFoldResult ConstantOp::fold(FoldAdaptor adaptor) { return getValue(); } |
| 22 | |
| 23 | void ConstantOp::getAsmResultNames( |
| 24 | function_ref<void(Value, StringRef)> setNameFn) { |
| 25 | setNameFn(getResult(), "cst" ); |
| 26 | } |
| 27 | |
| 28 | bool ConstantOp::isBuildableWith(Attribute value, Type type) { |
| 29 | if (auto arrAttr = llvm::dyn_cast<ArrayAttr>(Val&: value)) { |
| 30 | auto complexTy = llvm::dyn_cast<ComplexType>(Val&: type); |
| 31 | if (!complexTy || arrAttr.size() != 2) |
| 32 | return false; |
| 33 | auto complexEltTy = complexTy.getElementType(); |
| 34 | if (auto fre = llvm::dyn_cast<FloatAttr>(Val: arrAttr[0])) { |
| 35 | auto im = llvm::dyn_cast<FloatAttr>(Val: arrAttr[1]); |
| 36 | return im && fre.getType() == complexEltTy && |
| 37 | im.getType() == complexEltTy; |
| 38 | } |
| 39 | if (auto ire = llvm::dyn_cast<IntegerAttr>(Val: arrAttr[0])) { |
| 40 | auto im = llvm::dyn_cast<IntegerAttr>(Val: arrAttr[1]); |
| 41 | return im && ire.getType() == complexEltTy && |
| 42 | im.getType() == complexEltTy; |
| 43 | } |
| 44 | } |
| 45 | return false; |
| 46 | } |
| 47 | |
| 48 | LogicalResult ConstantOp::verify() { |
| 49 | ArrayAttr arrayAttr = getValue(); |
| 50 | if (arrayAttr.size() != 2) { |
| 51 | return emitOpError( |
| 52 | message: "requires 'value' to be a complex constant, represented as array of " |
| 53 | "two values" ); |
| 54 | } |
| 55 | |
| 56 | auto complexEltTy = getType().getElementType(); |
| 57 | if (!isa<FloatAttr, IntegerAttr>(Val: arrayAttr[0]) || |
| 58 | !isa<FloatAttr, IntegerAttr>(Val: arrayAttr[1])) |
| 59 | return emitOpError( |
| 60 | message: "requires attribute's elements to be float or integer attributes" ); |
| 61 | auto re = llvm::dyn_cast<TypedAttr>(Val: arrayAttr[0]); |
| 62 | auto im = llvm::dyn_cast<TypedAttr>(Val: arrayAttr[1]); |
| 63 | if (complexEltTy != re.getType() || complexEltTy != im.getType()) { |
| 64 | return emitOpError() |
| 65 | << "requires attribute's element types (" << re.getType() << ", " |
| 66 | << im.getType() |
| 67 | << ") to match the element type of the op's return type (" |
| 68 | << complexEltTy << ")" ; |
| 69 | } |
| 70 | return success(); |
| 71 | } |
| 72 | |
| 73 | //===----------------------------------------------------------------------===// |
| 74 | // BitcastOp |
| 75 | //===----------------------------------------------------------------------===// |
| 76 | |
| 77 | OpFoldResult BitcastOp::fold(FoldAdaptor bitcast) { |
| 78 | if (getOperand().getType() == getType()) |
| 79 | return getOperand(); |
| 80 | |
| 81 | return {}; |
| 82 | } |
| 83 | |
| 84 | LogicalResult BitcastOp::verify() { |
| 85 | auto operandType = getOperand().getType(); |
| 86 | auto resultType = getType(); |
| 87 | |
| 88 | // We allow this to be legal as it can be folded away. |
| 89 | if (operandType == resultType) |
| 90 | return success(); |
| 91 | |
| 92 | if (!operandType.isIntOrFloat() && !isa<ComplexType>(Val: operandType)) { |
| 93 | return emitOpError(message: "operand must be int/float/complex" ); |
| 94 | } |
| 95 | |
| 96 | if (!resultType.isIntOrFloat() && !isa<ComplexType>(Val: resultType)) { |
| 97 | return emitOpError(message: "result must be int/float/complex" ); |
| 98 | } |
| 99 | |
| 100 | if (isa<ComplexType>(Val: operandType) == isa<ComplexType>(Val: resultType)) { |
| 101 | return emitOpError( |
| 102 | message: "requires that either input or output has a complex type" ); |
| 103 | } |
| 104 | |
| 105 | if (isa<ComplexType>(Val: resultType)) |
| 106 | std::swap(a&: operandType, b&: resultType); |
| 107 | |
| 108 | int32_t operandBitwidth = dyn_cast<ComplexType>(Val&: operandType) |
| 109 | .getElementType() |
| 110 | .getIntOrFloatBitWidth() * |
| 111 | 2; |
| 112 | int32_t resultBitwidth = resultType.getIntOrFloatBitWidth(); |
| 113 | |
| 114 | if (operandBitwidth != resultBitwidth) { |
| 115 | return emitOpError(message: "casting bitwidths do not match" ); |
| 116 | } |
| 117 | |
| 118 | return success(); |
| 119 | } |
| 120 | |
| 121 | struct MergeComplexBitcast final : OpRewritePattern<BitcastOp> { |
| 122 | using OpRewritePattern<BitcastOp>::OpRewritePattern; |
| 123 | |
| 124 | LogicalResult matchAndRewrite(BitcastOp op, |
| 125 | PatternRewriter &rewriter) const override { |
| 126 | if (auto defining = op.getOperand().getDefiningOp<BitcastOp>()) { |
| 127 | if (isa<ComplexType>(Val: op.getType()) || |
| 128 | isa<ComplexType>(Val: defining.getOperand().getType())) { |
| 129 | // complex.bitcast requires that input or output is complex. |
| 130 | rewriter.replaceOpWithNewOp<BitcastOp>(op, args: op.getType(), |
| 131 | args: defining.getOperand()); |
| 132 | } else { |
| 133 | rewriter.replaceOpWithNewOp<arith::BitcastOp>(op, args: op.getType(), |
| 134 | args: defining.getOperand()); |
| 135 | } |
| 136 | return success(); |
| 137 | } |
| 138 | |
| 139 | if (auto defining = op.getOperand().getDefiningOp<arith::BitcastOp>()) { |
| 140 | rewriter.replaceOpWithNewOp<BitcastOp>(op, args: op.getType(), |
| 141 | args: defining.getOperand()); |
| 142 | return success(); |
| 143 | } |
| 144 | |
| 145 | return failure(); |
| 146 | } |
| 147 | }; |
| 148 | |
| 149 | struct MergeArithBitcast final : OpRewritePattern<arith::BitcastOp> { |
| 150 | using OpRewritePattern<arith::BitcastOp>::OpRewritePattern; |
| 151 | |
| 152 | LogicalResult matchAndRewrite(arith::BitcastOp op, |
| 153 | PatternRewriter &rewriter) const override { |
| 154 | if (auto defining = op.getOperand().getDefiningOp<complex::BitcastOp>()) { |
| 155 | rewriter.replaceOpWithNewOp<complex::BitcastOp>(op, args: op.getType(), |
| 156 | args: defining.getOperand()); |
| 157 | return success(); |
| 158 | } |
| 159 | |
| 160 | return failure(); |
| 161 | } |
| 162 | }; |
| 163 | |
| 164 | void BitcastOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 165 | MLIRContext *context) { |
| 166 | results.add<MergeComplexBitcast, MergeArithBitcast>(arg&: context); |
| 167 | } |
| 168 | |
| 169 | //===----------------------------------------------------------------------===// |
| 170 | // CreateOp |
| 171 | //===----------------------------------------------------------------------===// |
| 172 | |
| 173 | OpFoldResult CreateOp::fold(FoldAdaptor adaptor) { |
| 174 | // Fold complex.create(complex.re(op), complex.im(op)). |
| 175 | if (auto reOp = getOperand(i: 0).getDefiningOp<ReOp>()) { |
| 176 | if (auto imOp = getOperand(i: 1).getDefiningOp<ImOp>()) { |
| 177 | if (reOp.getOperand() == imOp.getOperand()) { |
| 178 | return reOp.getOperand(); |
| 179 | } |
| 180 | } |
| 181 | } |
| 182 | return {}; |
| 183 | } |
| 184 | |
| 185 | //===----------------------------------------------------------------------===// |
| 186 | // ImOp |
| 187 | //===----------------------------------------------------------------------===// |
| 188 | |
| 189 | OpFoldResult ImOp::fold(FoldAdaptor adaptor) { |
| 190 | ArrayAttr arrayAttr = |
| 191 | llvm::dyn_cast_if_present<ArrayAttr>(Val: adaptor.getComplex()); |
| 192 | if (arrayAttr && arrayAttr.size() == 2) |
| 193 | return arrayAttr[1]; |
| 194 | if (auto createOp = getOperand().getDefiningOp<CreateOp>()) |
| 195 | return createOp.getOperand(i: 1); |
| 196 | return {}; |
| 197 | } |
| 198 | |
| 199 | namespace { |
| 200 | template <typename OpKind, int ComponentIndex> |
| 201 | struct FoldComponentNeg final : OpRewritePattern<OpKind> { |
| 202 | using OpRewritePattern<OpKind>::OpRewritePattern; |
| 203 | |
| 204 | LogicalResult matchAndRewrite(OpKind op, |
| 205 | PatternRewriter &rewriter) const override { |
| 206 | auto negOp = op.getOperand().template getDefiningOp<NegOp>(); |
| 207 | if (!negOp) |
| 208 | return failure(); |
| 209 | |
| 210 | auto createOp = negOp.getComplex().template getDefiningOp<CreateOp>(); |
| 211 | if (!createOp) |
| 212 | return failure(); |
| 213 | |
| 214 | Type elementType = createOp.getType().getElementType(); |
| 215 | assert(isa<FloatType>(elementType)); |
| 216 | |
| 217 | rewriter.replaceOpWithNewOp<arith::NegFOp>( |
| 218 | op, elementType, createOp.getOperand(ComponentIndex)); |
| 219 | return success(); |
| 220 | } |
| 221 | }; |
| 222 | } // namespace |
| 223 | |
| 224 | void ImOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 225 | MLIRContext *context) { |
| 226 | results.add<FoldComponentNeg<ImOp, 1>>(arg&: context); |
| 227 | } |
| 228 | |
| 229 | //===----------------------------------------------------------------------===// |
| 230 | // ReOp |
| 231 | //===----------------------------------------------------------------------===// |
| 232 | |
| 233 | OpFoldResult ReOp::fold(FoldAdaptor adaptor) { |
| 234 | ArrayAttr arrayAttr = |
| 235 | llvm::dyn_cast_if_present<ArrayAttr>(Val: adaptor.getComplex()); |
| 236 | if (arrayAttr && arrayAttr.size() == 2) |
| 237 | return arrayAttr[0]; |
| 238 | if (auto createOp = getOperand().getDefiningOp<CreateOp>()) |
| 239 | return createOp.getOperand(i: 0); |
| 240 | return {}; |
| 241 | } |
| 242 | |
| 243 | void ReOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 244 | MLIRContext *context) { |
| 245 | results.add<FoldComponentNeg<ReOp, 0>>(arg&: context); |
| 246 | } |
| 247 | |
| 248 | //===----------------------------------------------------------------------===// |
| 249 | // AddOp |
| 250 | //===----------------------------------------------------------------------===// |
| 251 | |
| 252 | OpFoldResult AddOp::fold(FoldAdaptor adaptor) { |
| 253 | // complex.add(complex.sub(a, b), b) -> a |
| 254 | // complex.add(b, complex.sub(a, b)) -> a |
| 255 | // The intermediate result is rounded, so both ops need `reassoc`, and |
| 256 | // (-0.0 - b) + b is +0.0, so the add also needs `nsz`. |
| 257 | if (arith::bitEnumContainsAll(bits: getFastmath(), bit: arith::FastMathFlags::reassoc | |
| 258 | arith::FastMathFlags::nsz)) { |
| 259 | if (auto sub = getLhs().getDefiningOp<SubOp>()) |
| 260 | if (getRhs() == sub.getRhs() && |
| 261 | arith::bitEnumContainsAll(bits: sub.getFastmath(), |
| 262 | bit: arith::FastMathFlags::reassoc)) |
| 263 | return sub.getLhs(); |
| 264 | |
| 265 | if (auto sub = getRhs().getDefiningOp<SubOp>()) |
| 266 | if (getLhs() == sub.getRhs() && |
| 267 | arith::bitEnumContainsAll(bits: sub.getFastmath(), |
| 268 | bit: arith::FastMathFlags::reassoc)) |
| 269 | return sub.getLhs(); |
| 270 | } |
| 271 | |
| 272 | // complex.add(a, complex.constant<-0.0, -0.0>) -> a |
| 273 | if (auto constantOp = getRhs().getDefiningOp<ConstantOp>()) { |
| 274 | auto arrayAttr = constantOp.getValue(); |
| 275 | if (llvm::cast<FloatAttr>(Val: arrayAttr[0]).getValue().isNegZero() && |
| 276 | llvm::cast<FloatAttr>(Val: arrayAttr[1]).getValue().isNegZero()) { |
| 277 | return getLhs(); |
| 278 | } |
| 279 | } |
| 280 | |
| 281 | return {}; |
| 282 | } |
| 283 | |
| 284 | //===----------------------------------------------------------------------===// |
| 285 | // SubOp |
| 286 | //===----------------------------------------------------------------------===// |
| 287 | |
| 288 | OpFoldResult SubOp::fold(FoldAdaptor adaptor) { |
| 289 | // complex.sub(complex.add(a, b), b) -> a |
| 290 | // The intermediate result is rounded, so both ops need `reassoc`, and |
| 291 | // (-0.0 + b) - b is +0.0, so the sub also needs `nsz`. |
| 292 | if (arith::bitEnumContainsAll(bits: getFastmath(), bit: arith::FastMathFlags::reassoc | |
| 293 | arith::FastMathFlags::nsz)) |
| 294 | if (auto add = getLhs().getDefiningOp<AddOp>()) |
| 295 | if (getRhs() == add.getRhs() && |
| 296 | arith::bitEnumContainsAll(bits: add.getFastmath(), |
| 297 | bit: arith::FastMathFlags::reassoc)) |
| 298 | return add.getLhs(); |
| 299 | |
| 300 | // complex.sub(a, complex.constant<0.0, 0.0>) -> a |
| 301 | if (auto constantOp = getRhs().getDefiningOp<ConstantOp>()) { |
| 302 | auto arrayAttr = constantOp.getValue(); |
| 303 | if (llvm::cast<FloatAttr>(Val: arrayAttr[0]).getValue().isZero() && |
| 304 | llvm::cast<FloatAttr>(Val: arrayAttr[1]).getValue().isZero()) { |
| 305 | return getLhs(); |
| 306 | } |
| 307 | } |
| 308 | |
| 309 | return {}; |
| 310 | } |
| 311 | |
| 312 | //===----------------------------------------------------------------------===// |
| 313 | // NegOp |
| 314 | //===----------------------------------------------------------------------===// |
| 315 | |
| 316 | OpFoldResult NegOp::fold(FoldAdaptor adaptor) { |
| 317 | // complex.neg(complex.neg(a)) -> a |
| 318 | if (auto negOp = getOperand().getDefiningOp<NegOp>()) |
| 319 | return negOp.getOperand(); |
| 320 | |
| 321 | return {}; |
| 322 | } |
| 323 | |
| 324 | //===----------------------------------------------------------------------===// |
| 325 | // ExpOp |
| 326 | //===----------------------------------------------------------------------===// |
| 327 | |
| 328 | OpFoldResult ExpOp::fold(FoldAdaptor adaptor) { |
| 329 | // complex.exp(complex.log(a)) -> a |
| 330 | // exp(log(a)) is only an approximation of a, so both ops need `reassoc`. |
| 331 | // log(a) is not finite for a zero or non-finite a, so the log also needs |
| 332 | // `nnan` and `ninf`. |
| 333 | if (arith::bitEnumContainsAll(bits: getFastmath(), bit: arith::FastMathFlags::reassoc)) |
| 334 | if (auto logOp = getOperand().getDefiningOp<LogOp>()) |
| 335 | if (arith::bitEnumContainsAll(bits: logOp.getFastmath(), |
| 336 | bit: arith::FastMathFlags::reassoc | |
| 337 | arith::FastMathFlags::nnan | |
| 338 | arith::FastMathFlags::ninf)) |
| 339 | return logOp.getOperand(); |
| 340 | |
| 341 | return {}; |
| 342 | } |
| 343 | |
| 344 | //===----------------------------------------------------------------------===// |
| 345 | // ConjOp |
| 346 | //===----------------------------------------------------------------------===// |
| 347 | |
| 348 | OpFoldResult ConjOp::fold(FoldAdaptor adaptor) { |
| 349 | // complex.conj(complex.conj(a)) -> a |
| 350 | if (auto conjOp = getOperand().getDefiningOp<ConjOp>()) |
| 351 | return conjOp.getOperand(); |
| 352 | |
| 353 | return {}; |
| 354 | } |
| 355 | |
| 356 | //===----------------------------------------------------------------------===// |
| 357 | // MulOp |
| 358 | //===----------------------------------------------------------------------===// |
| 359 | |
| 360 | OpFoldResult MulOp::fold(FoldAdaptor adaptor) { |
| 361 | auto constant = getRhs().getDefiningOp<ConstantOp>(); |
| 362 | if (!constant) |
| 363 | return {}; |
| 364 | |
| 365 | ArrayAttr arrayAttr = constant.getValue(); |
| 366 | APFloat real = cast<FloatAttr>(Val: arrayAttr[0]).getValue(); |
| 367 | APFloat imag = cast<FloatAttr>(Val: arrayAttr[1]).getValue(); |
| 368 | |
| 369 | if (!imag.isZero()) |
| 370 | return {}; |
| 371 | |
| 372 | // complex.mul(a, complex.constant<1.0, 0.0>) -> a |
| 373 | if (real == APFloat(real.getSemantics(), 1)) |
| 374 | return getLhs(); |
| 375 | |
| 376 | return {}; |
| 377 | } |
| 378 | |
| 379 | //===----------------------------------------------------------------------===// |
| 380 | // DivOp |
| 381 | //===----------------------------------------------------------------------===// |
| 382 | |
| 383 | OpFoldResult DivOp::fold(FoldAdaptor adaptor) { |
| 384 | Attribute rhs = adaptor.getRhs(); |
| 385 | Attribute lhs = adaptor.getLhs(); |
| 386 | |
| 387 | // complex.div(complex.constant<NaN, NaN>, a) -> complex.constant<NaN, NaN> |
| 388 | // complex.div(complex.constant<NaN, a>, b) -> complex.constant<NaN, NaN> |
| 389 | // complex.div(complex.constant<a, NaN>, b) -> complex.constant<NaN, NaN> |
| 390 | bool isLhsComplexHasNan = false; |
| 391 | ArrayAttr lhsArrayAttr = dyn_cast_if_present<ArrayAttr>(Val&: lhs); |
| 392 | if (lhsArrayAttr && lhsArrayAttr.size() == 2) { |
| 393 | APFloat lhsReal = cast<FloatAttr>(Val: lhsArrayAttr[0]).getValue(); |
| 394 | APFloat lhsImag = cast<FloatAttr>(Val: lhsArrayAttr[1]).getValue(); |
| 395 | isLhsComplexHasNan = lhsReal.isNaN() || lhsImag.isNaN(); |
| 396 | if (isLhsComplexHasNan) { |
| 397 | Attribute nanValue = lhsReal.isNaN() ? lhsArrayAttr[0] : lhsArrayAttr[1]; |
| 398 | return ArrayAttr::get(context: getContext(), value: {nanValue, nanValue}); |
| 399 | } |
| 400 | } |
| 401 | |
| 402 | ArrayAttr rhsArrayAttr = dyn_cast_if_present<ArrayAttr>(Val&: rhs); |
| 403 | if (!rhsArrayAttr || rhsArrayAttr.size() != 2) |
| 404 | return {}; |
| 405 | |
| 406 | // Fold only if RHS is complex.constant<1.0, 0.0> |
| 407 | APFloat rhsImag = cast<FloatAttr>(Val: rhsArrayAttr[1]).getValue(); |
| 408 | APFloat rhsReal = cast<FloatAttr>(Val: rhsArrayAttr[0]).getValue(); |
| 409 | if (!rhsImag.isZero() || rhsReal != APFloat(rhsReal.getSemantics(), 1)) |
| 410 | return {}; |
| 411 | |
| 412 | // Fold to LHS if it doesn't contains NaNs or fast math flag nan is set |
| 413 | // complex.div(a, complex.constant<1.0, 0.0>) fastmath<nnan> -> a |
| 414 | if ((lhsArrayAttr && !isLhsComplexHasNan) || |
| 415 | arith::bitEnumContainsAll(bits: getFastmath(), bit: arith::FastMathFlags::nnan)) |
| 416 | return getLhs(); |
| 417 | |
| 418 | return {}; |
| 419 | } |
| 420 | |
| 421 | //===----------------------------------------------------------------------===// |
| 422 | // TableGen'd op method definitions |
| 423 | //===----------------------------------------------------------------------===// |
| 424 | |
| 425 | #define GET_OP_CLASSES |
| 426 | #include "mlir/Dialect/Complex/IR/ComplexOps.cpp.inc" |
| 427 | |