| 1 | //===- Utils.cpp - Utilities to support the Linalg dialect ----------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements utilities for the Linalg dialect. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "mlir/Dialect/Arith/Utils/Utils.h" |
| 14 | #include "mlir/AsmParser/AsmParser.h" |
| 15 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 16 | #include "mlir/Dialect/Complex/IR/Complex.h" |
| 17 | #include "mlir/Dialect/Utils/StaticValueUtils.h" |
| 18 | #include "mlir/IR/Diagnostics.h" |
| 19 | #include "llvm/ADT/SmallBitVector.h" |
| 20 | #include "llvm/ADT/SmallVectorExtras.h" |
| 21 | #include <numeric> |
| 22 | |
| 23 | using namespace mlir; |
| 24 | |
| 25 | std::optional<SmallVector<OpFoldResult>> |
| 26 | mlir::inferExpandShapeOutputShape(OpBuilder &b, Location loc, |
| 27 | ShapedType expandedType, |
| 28 | ArrayRef<ReassociationIndices> reassociation, |
| 29 | ArrayRef<OpFoldResult> inputShape) { |
| 30 | |
| 31 | SmallVector<Value> outputShapeValues; |
| 32 | SmallVector<int64_t> outputShapeInts; |
| 33 | // For zero-rank inputs, all dims in result shape are unit extent. |
| 34 | if (inputShape.empty()) { |
| 35 | outputShapeInts.resize(N: expandedType.getRank(), NV: 1); |
| 36 | return getMixedValues(staticValues: outputShapeInts, dynamicValues: outputShapeValues, b); |
| 37 | } |
| 38 | |
| 39 | // Check for all static shapes. |
| 40 | if (expandedType.hasStaticShape()) { |
| 41 | ArrayRef<int64_t> staticShape = expandedType.getShape(); |
| 42 | outputShapeInts.assign(in_start: staticShape.begin(), in_end: staticShape.end()); |
| 43 | return getMixedValues(staticValues: outputShapeInts, dynamicValues: outputShapeValues, b); |
| 44 | } |
| 45 | |
| 46 | outputShapeInts.resize(N: expandedType.getRank(), NV: ShapedType::kDynamic); |
| 47 | for (const auto &it : llvm::enumerate(First&: reassociation)) { |
| 48 | ReassociationIndices indexGroup = it.value(); |
| 49 | |
| 50 | int64_t indexGroupStaticSizesProductInt = 1; |
| 51 | bool foundDynamicShape = false; |
| 52 | for (int64_t index : indexGroup) { |
| 53 | int64_t outputDimSize = expandedType.getDimSize(idx: index); |
| 54 | // Cannot infer expanded shape with multiple dynamic dims in the |
| 55 | // same reassociation group! |
| 56 | if (ShapedType::isDynamic(dValue: outputDimSize)) { |
| 57 | if (foundDynamicShape) |
| 58 | return std::nullopt; |
| 59 | foundDynamicShape = true; |
| 60 | } else { |
| 61 | outputShapeInts[index] = outputDimSize; |
| 62 | indexGroupStaticSizesProductInt *= outputDimSize; |
| 63 | } |
| 64 | } |
| 65 | if (!foundDynamicShape) |
| 66 | continue; |
| 67 | |
| 68 | int64_t inputIndex = it.index(); |
| 69 | // Call get<Value>() under the assumption that we're not casting |
| 70 | // dynamism. |
| 71 | Value indexGroupSize = cast<Value>(Val: inputShape[inputIndex]); |
| 72 | Value indexGroupStaticSizesProduct = |
| 73 | arith::ConstantIndexOp::create(builder&: b, location: loc, value: indexGroupStaticSizesProductInt); |
| 74 | Value dynamicDimSize = b.createOrFold<arith::DivSIOp>( |
| 75 | location: loc, args&: indexGroupSize, args&: indexGroupStaticSizesProduct); |
| 76 | outputShapeValues.push_back(Elt: dynamicDimSize); |
| 77 | } |
| 78 | |
| 79 | if ((int64_t)outputShapeValues.size() != |
| 80 | llvm::count(Range&: outputShapeInts, Element: ShapedType::kDynamic)) |
| 81 | return std::nullopt; |
| 82 | |
| 83 | return getMixedValues(staticValues: outputShapeInts, dynamicValues: outputShapeValues, b); |
| 84 | } |
| 85 | |
| 86 | /// Matches a ConstantIndexOp. |
| 87 | /// TODO: This should probably just be a general matcher that uses matchConstant |
| 88 | /// and checks the operation for an index type. |
| 89 | detail::op_matcher<arith::ConstantIndexOp> mlir::matchConstantIndex() { |
| 90 | return detail::op_matcher<arith::ConstantIndexOp>(); |
| 91 | } |
| 92 | |
| 93 | llvm::SmallBitVector mlir::getPositionsOfShapeOne(unsigned rank, |
| 94 | ArrayRef<int64_t> shape) { |
| 95 | llvm::SmallBitVector dimsToProject(shape.size()); |
| 96 | for (unsigned pos = 0, e = shape.size(); pos < e && rank > 0; ++pos) { |
| 97 | if (shape[pos] == 1) { |
| 98 | dimsToProject.set(pos); |
| 99 | --rank; |
| 100 | } |
| 101 | } |
| 102 | return dimsToProject; |
| 103 | } |
| 104 | |
| 105 | Value mlir::getValueOrCreateConstantIntOp(OpBuilder &b, Location loc, |
| 106 | OpFoldResult ofr) { |
| 107 | if (auto value = dyn_cast_if_present<Value>(Val&: ofr)) |
| 108 | return value; |
| 109 | auto attr = cast<IntegerAttr>(Val: cast<Attribute>(Val&: ofr)); |
| 110 | return arith::ConstantOp::create( |
| 111 | builder&: b, location: loc, value: b.getIntegerAttr(type: attr.getType(), value: attr.getValue().getSExtValue())); |
| 112 | } |
| 113 | |
| 114 | Value mlir::getValueOrCreateConstantIndexOp(OpBuilder &b, Location loc, |
| 115 | OpFoldResult ofr) { |
| 116 | if (auto value = dyn_cast_if_present<Value>(Val&: ofr)) |
| 117 | return value; |
| 118 | auto attr = cast<IntegerAttr>(Val: cast<Attribute>(Val&: ofr)); |
| 119 | return arith::ConstantIndexOp::create(builder&: b, location: loc, value: attr.getValue().getSExtValue()); |
| 120 | } |
| 121 | |
| 122 | Value mlir::getValueOrCreateCastToIndexLike(OpBuilder &b, Location loc, |
| 123 | Type targetType, Value value) { |
| 124 | if (targetType == value.getType()) |
| 125 | return value; |
| 126 | |
| 127 | bool targetIsIndex = targetType.isIndex(); |
| 128 | bool valueIsIndex = value.getType().isIndex(); |
| 129 | if (targetIsIndex ^ valueIsIndex) |
| 130 | return arith::IndexCastOp::create(builder&: b, location: loc, out: targetType, in: value); |
| 131 | |
| 132 | auto targetIntegerType = dyn_cast<IntegerType>(Val&: targetType); |
| 133 | auto valueIntegerType = dyn_cast<IntegerType>(Val: value.getType()); |
| 134 | assert(targetIntegerType && valueIntegerType && |
| 135 | "unexpected cast between types other than integers and index" ); |
| 136 | assert(targetIntegerType.getSignedness() == valueIntegerType.getSignedness()); |
| 137 | |
| 138 | if (targetIntegerType.getWidth() > valueIntegerType.getWidth()) |
| 139 | return arith::ExtSIOp::create(builder&: b, location: loc, out: targetIntegerType, in: value); |
| 140 | return arith::TruncIOp::create(builder&: b, location: loc, out: targetIntegerType, in: value); |
| 141 | } |
| 142 | |
| 143 | static Value convertScalarToIntDtype(ImplicitLocOpBuilder &b, Value operand, |
| 144 | IntegerType toType, bool isUnsigned) { |
| 145 | // If operand is floating point, cast directly to the int type. |
| 146 | if (isa<FloatType>(Val: operand.getType())) { |
| 147 | if (isUnsigned) |
| 148 | return arith::FPToUIOp::create(builder&: b, out: toType, in: operand); |
| 149 | return arith::FPToSIOp::create(builder&: b, out: toType, in: operand); |
| 150 | } |
| 151 | // Cast index operands directly to the int type. |
| 152 | if (operand.getType().isIndex()) |
| 153 | return arith::IndexCastOp::create(builder&: b, out: toType, in: operand); |
| 154 | if (auto fromIntType = dyn_cast<IntegerType>(Val: operand.getType())) { |
| 155 | // Either extend or truncate. |
| 156 | if (toType.getWidth() > fromIntType.getWidth()) { |
| 157 | if (isUnsigned) |
| 158 | return arith::ExtUIOp::create(builder&: b, out: toType, in: operand); |
| 159 | return arith::ExtSIOp::create(builder&: b, out: toType, in: operand); |
| 160 | } |
| 161 | if (toType.getWidth() < fromIntType.getWidth()) |
| 162 | return arith::TruncIOp::create(builder&: b, out: toType, in: operand); |
| 163 | return operand; |
| 164 | } |
| 165 | |
| 166 | return {}; |
| 167 | } |
| 168 | |
| 169 | static Value convertScalarToFpDtype(ImplicitLocOpBuilder &b, Value operand, |
| 170 | FloatType toType, bool isUnsigned) { |
| 171 | // If operand is integer, cast directly to the float type. |
| 172 | if (isa<IntegerType>(Val: operand.getType())) { |
| 173 | if (isUnsigned) |
| 174 | return arith::UIToFPOp::create(builder&: b, out: toType, in: operand); |
| 175 | return arith::SIToFPOp::create(builder&: b, out: toType, in: operand); |
| 176 | } |
| 177 | if (auto fromFpTy = dyn_cast<FloatType>(Val: operand.getType())) { |
| 178 | if (toType.getWidth() > fromFpTy.getWidth()) |
| 179 | return arith::ExtFOp::create(builder&: b, out: toType, in: operand, |
| 180 | fastmath: arith::FastMathFlagsAttr{}); |
| 181 | if (toType.getWidth() < fromFpTy.getWidth()) |
| 182 | return arith::TruncFOp::create(builder&: b, out: toType, in: operand); |
| 183 | |
| 184 | return arith::ConvertFOp::create(builder&: b, resultTypes: toType, operands: operand, |
| 185 | properties: arith::ConvertFOp::Properties{}, discardableAttributes: {}); |
| 186 | } |
| 187 | |
| 188 | return {}; |
| 189 | } |
| 190 | |
| 191 | static Value convertScalarToComplexDtype(ImplicitLocOpBuilder &b, Value operand, |
| 192 | ComplexType targetType, |
| 193 | bool isUnsigned) { |
| 194 | if (auto fromComplexType = dyn_cast<ComplexType>(Val: operand.getType())) { |
| 195 | if (isa<FloatType>(Val: targetType.getElementType()) && |
| 196 | isa<FloatType>(Val: fromComplexType.getElementType())) { |
| 197 | Value real = complex::ReOp::create(builder&: b, complex: operand); |
| 198 | Value imag = complex::ImOp::create(builder&: b, complex: operand); |
| 199 | Type targetETy = targetType.getElementType(); |
| 200 | if (targetType.getElementType().getIntOrFloatBitWidth() < |
| 201 | fromComplexType.getElementType().getIntOrFloatBitWidth()) { |
| 202 | real = arith::TruncFOp::create(builder&: b, out: targetETy, in: real); |
| 203 | imag = arith::TruncFOp::create(builder&: b, out: targetETy, in: imag); |
| 204 | } else { |
| 205 | real = arith::ExtFOp::create(builder&: b, out: targetETy, in: real, |
| 206 | fastmath: arith::FastMathFlagsAttr{}); |
| 207 | imag = arith::ExtFOp::create(builder&: b, out: targetETy, in: imag, |
| 208 | fastmath: arith::FastMathFlagsAttr{}); |
| 209 | } |
| 210 | return complex::CreateOp::create(builder&: b, complex: targetType, real, imaginary: imag); |
| 211 | } |
| 212 | } |
| 213 | |
| 214 | if (isa<FloatType>(Val: operand.getType())) { |
| 215 | FloatType toFpTy = cast<FloatType>(Val: targetType.getElementType()); |
| 216 | auto toBitwidth = toFpTy.getIntOrFloatBitWidth(); |
| 217 | Value from = operand; |
| 218 | if (from.getType().getIntOrFloatBitWidth() < toBitwidth) { |
| 219 | from = arith::ExtFOp::create(builder&: b, out: toFpTy, in: from, fastmath: arith::FastMathFlagsAttr{}); |
| 220 | } |
| 221 | if (from.getType().getIntOrFloatBitWidth() > toBitwidth) { |
| 222 | from = arith::TruncFOp::create(builder&: b, out: toFpTy, in: from); |
| 223 | } |
| 224 | Value zero = mlir::arith::ConstantFloatOp::create( |
| 225 | builder&: b, type: toFpTy, value: mlir::APFloat(toFpTy.getFloatSemantics(), 0)); |
| 226 | return complex::CreateOp::create(builder&: b, complex: targetType, real: from, imaginary: zero); |
| 227 | } |
| 228 | |
| 229 | if (isa<IntegerType>(Val: operand.getType())) { |
| 230 | FloatType toFpTy = cast<FloatType>(Val: targetType.getElementType()); |
| 231 | Value from = operand; |
| 232 | if (isUnsigned) { |
| 233 | from = arith::UIToFPOp::create(builder&: b, out: toFpTy, in: from); |
| 234 | } else { |
| 235 | from = arith::SIToFPOp::create(builder&: b, out: toFpTy, in: from); |
| 236 | } |
| 237 | Value zero = mlir::arith::ConstantFloatOp::create( |
| 238 | builder&: b, type: toFpTy, value: mlir::APFloat(toFpTy.getFloatSemantics(), 0)); |
| 239 | return complex::CreateOp::create(builder&: b, complex: targetType, real: from, imaginary: zero); |
| 240 | } |
| 241 | |
| 242 | return {}; |
| 243 | } |
| 244 | |
| 245 | Value mlir::convertScalarToDtype(OpBuilder &b, Location loc, Value operand, |
| 246 | Type toType, bool isUnsignedCast) { |
| 247 | if (operand.getType() == toType) |
| 248 | return operand; |
| 249 | ImplicitLocOpBuilder ib(loc, b); |
| 250 | Value result; |
| 251 | if (auto intTy = dyn_cast<IntegerType>(Val&: toType)) { |
| 252 | result = convertScalarToIntDtype(b&: ib, operand, toType: intTy, isUnsigned: isUnsignedCast); |
| 253 | } else if (auto floatTy = dyn_cast<FloatType>(Val&: toType)) { |
| 254 | result = convertScalarToFpDtype(b&: ib, operand, toType: floatTy, isUnsigned: isUnsignedCast); |
| 255 | } else if (auto complexTy = dyn_cast<ComplexType>(Val&: toType)) { |
| 256 | result = |
| 257 | convertScalarToComplexDtype(b&: ib, operand, targetType: complexTy, isUnsigned: isUnsignedCast); |
| 258 | } |
| 259 | |
| 260 | if (result) |
| 261 | return result; |
| 262 | |
| 263 | emitWarning(loc) << "could not cast operand of type " << operand.getType() |
| 264 | << " to " << toType; |
| 265 | return operand; |
| 266 | } |
| 267 | |
| 268 | SmallVector<Value> |
| 269 | mlir::getValueOrCreateConstantIndexOp(OpBuilder &b, Location loc, |
| 270 | ArrayRef<OpFoldResult> valueOrAttrVec) { |
| 271 | return llvm::map_to_vector<4>( |
| 272 | C&: valueOrAttrVec, F: [&](OpFoldResult value) -> Value { |
| 273 | return getValueOrCreateConstantIndexOp(b, loc, ofr: value); |
| 274 | }); |
| 275 | } |
| 276 | |
| 277 | Value mlir::createScalarOrSplatConstant(OpBuilder &builder, Location loc, |
| 278 | Type type, const APInt &value) { |
| 279 | TypedAttr attr; |
| 280 | if (isa<IntegerType>(Val: type)) { |
| 281 | attr = builder.getIntegerAttr(type, value); |
| 282 | } else { |
| 283 | auto vecTy = cast<ShapedType>(Val&: type); |
| 284 | attr = SplatElementsAttr::get(type: vecTy, values: value); |
| 285 | } |
| 286 | |
| 287 | return arith::ConstantOp::create(builder, location: loc, value: attr); |
| 288 | } |
| 289 | |
| 290 | Value mlir::createScalarOrSplatConstant(OpBuilder &builder, Location loc, |
| 291 | Type type, int64_t value) { |
| 292 | unsigned elementBitWidth = 0; |
| 293 | if (auto intTy = dyn_cast<IntegerType>(Val&: type)) |
| 294 | elementBitWidth = intTy.getWidth(); |
| 295 | else |
| 296 | elementBitWidth = cast<ShapedType>(Val&: type).getElementTypeBitWidth(); |
| 297 | |
| 298 | return createScalarOrSplatConstant(builder, loc, type, |
| 299 | value: APInt(elementBitWidth, value)); |
| 300 | } |
| 301 | |
| 302 | Value mlir::createScalarOrSplatConstant(OpBuilder &builder, Location loc, |
| 303 | Type type, const APFloat &value) { |
| 304 | if (isa<FloatType>(Val: type)) |
| 305 | return builder.createOrFold<arith::ConstantOp>( |
| 306 | location: loc, args&: type, args: builder.getFloatAttr(type, value)); |
| 307 | TypedAttr splat = SplatElementsAttr::get(type: cast<ShapedType>(Val&: type), values: value); |
| 308 | return builder.createOrFold<arith::ConstantOp>(location: loc, args&: type, args&: splat); |
| 309 | } |
| 310 | |
| 311 | Type mlir::getType(OpFoldResult ofr) { |
| 312 | if (auto value = dyn_cast_if_present<Value>(Val&: ofr)) |
| 313 | return value.getType(); |
| 314 | auto attr = cast<IntegerAttr>(Val: cast<Attribute>(Val&: ofr)); |
| 315 | return attr.getType(); |
| 316 | } |
| 317 | |
| 318 | Value ArithBuilder::_and(Value lhs, Value rhs) { |
| 319 | return arith::AndIOp::create(builder&: b, location: loc, lhs, rhs); |
| 320 | } |
| 321 | Value ArithBuilder::add(Value lhs, Value rhs) { |
| 322 | if (isa<FloatType>(Val: lhs.getType())) |
| 323 | return arith::AddFOp::create(builder&: b, location: loc, lhs, rhs); |
| 324 | return arith::AddIOp::create(builder&: b, location: loc, lhs, rhs, overflowFlags: ovf); |
| 325 | } |
| 326 | Value ArithBuilder::sub(Value lhs, Value rhs) { |
| 327 | if (isa<FloatType>(Val: lhs.getType())) |
| 328 | return arith::SubFOp::create(builder&: b, location: loc, lhs, rhs); |
| 329 | return arith::SubIOp::create(builder&: b, location: loc, lhs, rhs, overflowFlags: ovf); |
| 330 | } |
| 331 | Value ArithBuilder::mul(Value lhs, Value rhs) { |
| 332 | if (isa<FloatType>(Val: lhs.getType())) |
| 333 | return arith::MulFOp::create(builder&: b, location: loc, lhs, rhs); |
| 334 | return arith::MulIOp::create(builder&: b, location: loc, lhs, rhs, overflowFlags: ovf); |
| 335 | } |
| 336 | Value ArithBuilder::sgt(Value lhs, Value rhs) { |
| 337 | if (isa<FloatType>(Val: lhs.getType())) |
| 338 | return arith::CmpFOp::create(builder&: b, location: loc, predicate: arith::CmpFPredicate::OGT, lhs, rhs); |
| 339 | return arith::CmpIOp::create(builder&: b, location: loc, predicate: arith::CmpIPredicate::sgt, lhs, rhs); |
| 340 | } |
| 341 | Value ArithBuilder::slt(Value lhs, Value rhs) { |
| 342 | if (isa<FloatType>(Val: lhs.getType())) |
| 343 | return arith::CmpFOp::create(builder&: b, location: loc, predicate: arith::CmpFPredicate::OLT, lhs, rhs); |
| 344 | return arith::CmpIOp::create(builder&: b, location: loc, predicate: arith::CmpIPredicate::slt, lhs, rhs); |
| 345 | } |
| 346 | Value ArithBuilder::select(Value cmp, Value lhs, Value rhs) { |
| 347 | return arith::SelectOp::create(builder&: b, location: loc, condition: cmp, true_value: lhs, false_value: rhs); |
| 348 | } |
| 349 | |
| 350 | namespace mlir::arith { |
| 351 | |
| 352 | Value createProduct(OpBuilder &builder, Location loc, ArrayRef<Value> values) { |
| 353 | return createProduct(builder, loc, values, resultType: values.front().getType()); |
| 354 | } |
| 355 | |
| 356 | Value createProduct(OpBuilder &builder, Location loc, ArrayRef<Value> values, |
| 357 | Type resultType) { |
| 358 | Value one = ConstantOp::create(builder, location: loc, result: resultType, |
| 359 | value: builder.getOneAttr(type: resultType)); |
| 360 | ArithBuilder arithBuilder(builder, loc); |
| 361 | return llvm::accumulate(Range&: values, Init&: one, Op: [&arithBuilder](Value acc, Value v) { |
| 362 | return arithBuilder.mul(lhs: acc, rhs: v); |
| 363 | }); |
| 364 | } |
| 365 | |
| 366 | FloatType parseFloatType(MLIRContext *ctx, StringRef name) { |
| 367 | // Parsing non-builtin types is unsafe because the respective dialect may not |
| 368 | // have been loaded. |
| 369 | if (!name.empty() && name.front() == '!') |
| 370 | return FloatType(); |
| 371 | |
| 372 | // Suppress diagnostics: callers handle invalid type strings themselves. |
| 373 | ScopedDiagnosticHandler handler(ctx, [](Diagnostic &) {}); |
| 374 | return dyn_cast_or_null<FloatType>(Val: mlir::parseType(typeStr: name, context: ctx)); |
| 375 | } |
| 376 | |
| 377 | } // namespace mlir::arith |
| 378 | |