1//===- Utils.cpp - Utilities to support the Linalg dialect ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements utilities for the Linalg dialect.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/Arith/Utils/Utils.h"
14#include "mlir/AsmParser/AsmParser.h"
15#include "mlir/Dialect/Arith/IR/Arith.h"
16#include "mlir/Dialect/Complex/IR/Complex.h"
17#include "mlir/Dialect/Utils/StaticValueUtils.h"
18#include "mlir/IR/Diagnostics.h"
19#include "llvm/ADT/SmallBitVector.h"
20#include "llvm/ADT/SmallVectorExtras.h"
21#include <numeric>
22
23using namespace mlir;
24
25std::optional<SmallVector<OpFoldResult>>
26mlir::inferExpandShapeOutputShape(OpBuilder &b, Location loc,
27 ShapedType expandedType,
28 ArrayRef<ReassociationIndices> reassociation,
29 ArrayRef<OpFoldResult> inputShape) {
30
31 SmallVector<Value> outputShapeValues;
32 SmallVector<int64_t> outputShapeInts;
33 // For zero-rank inputs, all dims in result shape are unit extent.
34 if (inputShape.empty()) {
35 outputShapeInts.resize(N: expandedType.getRank(), NV: 1);
36 return getMixedValues(staticValues: outputShapeInts, dynamicValues: outputShapeValues, b);
37 }
38
39 // Check for all static shapes.
40 if (expandedType.hasStaticShape()) {
41 ArrayRef<int64_t> staticShape = expandedType.getShape();
42 outputShapeInts.assign(in_start: staticShape.begin(), in_end: staticShape.end());
43 return getMixedValues(staticValues: outputShapeInts, dynamicValues: outputShapeValues, b);
44 }
45
46 outputShapeInts.resize(N: expandedType.getRank(), NV: ShapedType::kDynamic);
47 for (const auto &it : llvm::enumerate(First&: reassociation)) {
48 ReassociationIndices indexGroup = it.value();
49
50 int64_t indexGroupStaticSizesProductInt = 1;
51 bool foundDynamicShape = false;
52 for (int64_t index : indexGroup) {
53 int64_t outputDimSize = expandedType.getDimSize(idx: index);
54 // Cannot infer expanded shape with multiple dynamic dims in the
55 // same reassociation group!
56 if (ShapedType::isDynamic(dValue: outputDimSize)) {
57 if (foundDynamicShape)
58 return std::nullopt;
59 foundDynamicShape = true;
60 } else {
61 outputShapeInts[index] = outputDimSize;
62 indexGroupStaticSizesProductInt *= outputDimSize;
63 }
64 }
65 if (!foundDynamicShape)
66 continue;
67
68 int64_t inputIndex = it.index();
69 // Call get<Value>() under the assumption that we're not casting
70 // dynamism.
71 Value indexGroupSize = cast<Value>(Val: inputShape[inputIndex]);
72 Value indexGroupStaticSizesProduct =
73 arith::ConstantIndexOp::create(builder&: b, location: loc, value: indexGroupStaticSizesProductInt);
74 Value dynamicDimSize = b.createOrFold<arith::DivSIOp>(
75 location: loc, args&: indexGroupSize, args&: indexGroupStaticSizesProduct);
76 outputShapeValues.push_back(Elt: dynamicDimSize);
77 }
78
79 if ((int64_t)outputShapeValues.size() !=
80 llvm::count(Range&: outputShapeInts, Element: ShapedType::kDynamic))
81 return std::nullopt;
82
83 return getMixedValues(staticValues: outputShapeInts, dynamicValues: outputShapeValues, b);
84}
85
86/// Matches a ConstantIndexOp.
87/// TODO: This should probably just be a general matcher that uses matchConstant
88/// and checks the operation for an index type.
89detail::op_matcher<arith::ConstantIndexOp> mlir::matchConstantIndex() {
90 return detail::op_matcher<arith::ConstantIndexOp>();
91}
92
93llvm::SmallBitVector mlir::getPositionsOfShapeOne(unsigned rank,
94 ArrayRef<int64_t> shape) {
95 llvm::SmallBitVector dimsToProject(shape.size());
96 for (unsigned pos = 0, e = shape.size(); pos < e && rank > 0; ++pos) {
97 if (shape[pos] == 1) {
98 dimsToProject.set(pos);
99 --rank;
100 }
101 }
102 return dimsToProject;
103}
104
105Value mlir::getValueOrCreateConstantIntOp(OpBuilder &b, Location loc,
106 OpFoldResult ofr) {
107 if (auto value = dyn_cast_if_present<Value>(Val&: ofr))
108 return value;
109 auto attr = cast<IntegerAttr>(Val: cast<Attribute>(Val&: ofr));
110 return arith::ConstantOp::create(
111 builder&: b, location: loc, value: b.getIntegerAttr(type: attr.getType(), value: attr.getValue().getSExtValue()));
112}
113
114Value mlir::getValueOrCreateConstantIndexOp(OpBuilder &b, Location loc,
115 OpFoldResult ofr) {
116 if (auto value = dyn_cast_if_present<Value>(Val&: ofr))
117 return value;
118 auto attr = cast<IntegerAttr>(Val: cast<Attribute>(Val&: ofr));
119 return arith::ConstantIndexOp::create(builder&: b, location: loc, value: attr.getValue().getSExtValue());
120}
121
122Value mlir::getValueOrCreateCastToIndexLike(OpBuilder &b, Location loc,
123 Type targetType, Value value) {
124 if (targetType == value.getType())
125 return value;
126
127 bool targetIsIndex = targetType.isIndex();
128 bool valueIsIndex = value.getType().isIndex();
129 if (targetIsIndex ^ valueIsIndex)
130 return arith::IndexCastOp::create(builder&: b, location: loc, out: targetType, in: value);
131
132 auto targetIntegerType = dyn_cast<IntegerType>(Val&: targetType);
133 auto valueIntegerType = dyn_cast<IntegerType>(Val: value.getType());
134 assert(targetIntegerType && valueIntegerType &&
135 "unexpected cast between types other than integers and index");
136 assert(targetIntegerType.getSignedness() == valueIntegerType.getSignedness());
137
138 if (targetIntegerType.getWidth() > valueIntegerType.getWidth())
139 return arith::ExtSIOp::create(builder&: b, location: loc, out: targetIntegerType, in: value);
140 return arith::TruncIOp::create(builder&: b, location: loc, out: targetIntegerType, in: value);
141}
142
143static Value convertScalarToIntDtype(ImplicitLocOpBuilder &b, Value operand,
144 IntegerType toType, bool isUnsigned) {
145 // If operand is floating point, cast directly to the int type.
146 if (isa<FloatType>(Val: operand.getType())) {
147 if (isUnsigned)
148 return arith::FPToUIOp::create(builder&: b, out: toType, in: operand);
149 return arith::FPToSIOp::create(builder&: b, out: toType, in: operand);
150 }
151 // Cast index operands directly to the int type.
152 if (operand.getType().isIndex())
153 return arith::IndexCastOp::create(builder&: b, out: toType, in: operand);
154 if (auto fromIntType = dyn_cast<IntegerType>(Val: operand.getType())) {
155 // Either extend or truncate.
156 if (toType.getWidth() > fromIntType.getWidth()) {
157 if (isUnsigned)
158 return arith::ExtUIOp::create(builder&: b, out: toType, in: operand);
159 return arith::ExtSIOp::create(builder&: b, out: toType, in: operand);
160 }
161 if (toType.getWidth() < fromIntType.getWidth())
162 return arith::TruncIOp::create(builder&: b, out: toType, in: operand);
163 return operand;
164 }
165
166 return {};
167}
168
169static Value convertScalarToFpDtype(ImplicitLocOpBuilder &b, Value operand,
170 FloatType toType, bool isUnsigned) {
171 // If operand is integer, cast directly to the float type.
172 if (isa<IntegerType>(Val: operand.getType())) {
173 if (isUnsigned)
174 return arith::UIToFPOp::create(builder&: b, out: toType, in: operand);
175 return arith::SIToFPOp::create(builder&: b, out: toType, in: operand);
176 }
177 if (auto fromFpTy = dyn_cast<FloatType>(Val: operand.getType())) {
178 if (toType.getWidth() > fromFpTy.getWidth())
179 return arith::ExtFOp::create(builder&: b, out: toType, in: operand,
180 fastmath: arith::FastMathFlagsAttr{});
181 if (toType.getWidth() < fromFpTy.getWidth())
182 return arith::TruncFOp::create(builder&: b, out: toType, in: operand);
183
184 return arith::ConvertFOp::create(builder&: b, resultTypes: toType, operands: operand,
185 properties: arith::ConvertFOp::Properties{}, discardableAttributes: {});
186 }
187
188 return {};
189}
190
191static Value convertScalarToComplexDtype(ImplicitLocOpBuilder &b, Value operand,
192 ComplexType targetType,
193 bool isUnsigned) {
194 if (auto fromComplexType = dyn_cast<ComplexType>(Val: operand.getType())) {
195 if (isa<FloatType>(Val: targetType.getElementType()) &&
196 isa<FloatType>(Val: fromComplexType.getElementType())) {
197 Value real = complex::ReOp::create(builder&: b, complex: operand);
198 Value imag = complex::ImOp::create(builder&: b, complex: operand);
199 Type targetETy = targetType.getElementType();
200 if (targetType.getElementType().getIntOrFloatBitWidth() <
201 fromComplexType.getElementType().getIntOrFloatBitWidth()) {
202 real = arith::TruncFOp::create(builder&: b, out: targetETy, in: real);
203 imag = arith::TruncFOp::create(builder&: b, out: targetETy, in: imag);
204 } else {
205 real = arith::ExtFOp::create(builder&: b, out: targetETy, in: real,
206 fastmath: arith::FastMathFlagsAttr{});
207 imag = arith::ExtFOp::create(builder&: b, out: targetETy, in: imag,
208 fastmath: arith::FastMathFlagsAttr{});
209 }
210 return complex::CreateOp::create(builder&: b, complex: targetType, real, imaginary: imag);
211 }
212 }
213
214 if (isa<FloatType>(Val: operand.getType())) {
215 FloatType toFpTy = cast<FloatType>(Val: targetType.getElementType());
216 auto toBitwidth = toFpTy.getIntOrFloatBitWidth();
217 Value from = operand;
218 if (from.getType().getIntOrFloatBitWidth() < toBitwidth) {
219 from = arith::ExtFOp::create(builder&: b, out: toFpTy, in: from, fastmath: arith::FastMathFlagsAttr{});
220 }
221 if (from.getType().getIntOrFloatBitWidth() > toBitwidth) {
222 from = arith::TruncFOp::create(builder&: b, out: toFpTy, in: from);
223 }
224 Value zero = mlir::arith::ConstantFloatOp::create(
225 builder&: b, type: toFpTy, value: mlir::APFloat(toFpTy.getFloatSemantics(), 0));
226 return complex::CreateOp::create(builder&: b, complex: targetType, real: from, imaginary: zero);
227 }
228
229 if (isa<IntegerType>(Val: operand.getType())) {
230 FloatType toFpTy = cast<FloatType>(Val: targetType.getElementType());
231 Value from = operand;
232 if (isUnsigned) {
233 from = arith::UIToFPOp::create(builder&: b, out: toFpTy, in: from);
234 } else {
235 from = arith::SIToFPOp::create(builder&: b, out: toFpTy, in: from);
236 }
237 Value zero = mlir::arith::ConstantFloatOp::create(
238 builder&: b, type: toFpTy, value: mlir::APFloat(toFpTy.getFloatSemantics(), 0));
239 return complex::CreateOp::create(builder&: b, complex: targetType, real: from, imaginary: zero);
240 }
241
242 return {};
243}
244
245Value mlir::convertScalarToDtype(OpBuilder &b, Location loc, Value operand,
246 Type toType, bool isUnsignedCast) {
247 if (operand.getType() == toType)
248 return operand;
249 ImplicitLocOpBuilder ib(loc, b);
250 Value result;
251 if (auto intTy = dyn_cast<IntegerType>(Val&: toType)) {
252 result = convertScalarToIntDtype(b&: ib, operand, toType: intTy, isUnsigned: isUnsignedCast);
253 } else if (auto floatTy = dyn_cast<FloatType>(Val&: toType)) {
254 result = convertScalarToFpDtype(b&: ib, operand, toType: floatTy, isUnsigned: isUnsignedCast);
255 } else if (auto complexTy = dyn_cast<ComplexType>(Val&: toType)) {
256 result =
257 convertScalarToComplexDtype(b&: ib, operand, targetType: complexTy, isUnsigned: isUnsignedCast);
258 }
259
260 if (result)
261 return result;
262
263 emitWarning(loc) << "could not cast operand of type " << operand.getType()
264 << " to " << toType;
265 return operand;
266}
267
268SmallVector<Value>
269mlir::getValueOrCreateConstantIndexOp(OpBuilder &b, Location loc,
270 ArrayRef<OpFoldResult> valueOrAttrVec) {
271 return llvm::map_to_vector<4>(
272 C&: valueOrAttrVec, F: [&](OpFoldResult value) -> Value {
273 return getValueOrCreateConstantIndexOp(b, loc, ofr: value);
274 });
275}
276
277Value mlir::createScalarOrSplatConstant(OpBuilder &builder, Location loc,
278 Type type, const APInt &value) {
279 TypedAttr attr;
280 if (isa<IntegerType>(Val: type)) {
281 attr = builder.getIntegerAttr(type, value);
282 } else {
283 auto vecTy = cast<ShapedType>(Val&: type);
284 attr = SplatElementsAttr::get(type: vecTy, values: value);
285 }
286
287 return arith::ConstantOp::create(builder, location: loc, value: attr);
288}
289
290Value mlir::createScalarOrSplatConstant(OpBuilder &builder, Location loc,
291 Type type, int64_t value) {
292 unsigned elementBitWidth = 0;
293 if (auto intTy = dyn_cast<IntegerType>(Val&: type))
294 elementBitWidth = intTy.getWidth();
295 else
296 elementBitWidth = cast<ShapedType>(Val&: type).getElementTypeBitWidth();
297
298 return createScalarOrSplatConstant(builder, loc, type,
299 value: APInt(elementBitWidth, value));
300}
301
302Value mlir::createScalarOrSplatConstant(OpBuilder &builder, Location loc,
303 Type type, const APFloat &value) {
304 if (isa<FloatType>(Val: type))
305 return builder.createOrFold<arith::ConstantOp>(
306 location: loc, args&: type, args: builder.getFloatAttr(type, value));
307 TypedAttr splat = SplatElementsAttr::get(type: cast<ShapedType>(Val&: type), values: value);
308 return builder.createOrFold<arith::ConstantOp>(location: loc, args&: type, args&: splat);
309}
310
311Type mlir::getType(OpFoldResult ofr) {
312 if (auto value = dyn_cast_if_present<Value>(Val&: ofr))
313 return value.getType();
314 auto attr = cast<IntegerAttr>(Val: cast<Attribute>(Val&: ofr));
315 return attr.getType();
316}
317
318Value ArithBuilder::_and(Value lhs, Value rhs) {
319 return arith::AndIOp::create(builder&: b, location: loc, lhs, rhs);
320}
321Value ArithBuilder::add(Value lhs, Value rhs) {
322 if (isa<FloatType>(Val: lhs.getType()))
323 return arith::AddFOp::create(builder&: b, location: loc, lhs, rhs);
324 return arith::AddIOp::create(builder&: b, location: loc, lhs, rhs, overflowFlags: ovf);
325}
326Value ArithBuilder::sub(Value lhs, Value rhs) {
327 if (isa<FloatType>(Val: lhs.getType()))
328 return arith::SubFOp::create(builder&: b, location: loc, lhs, rhs);
329 return arith::SubIOp::create(builder&: b, location: loc, lhs, rhs, overflowFlags: ovf);
330}
331Value ArithBuilder::mul(Value lhs, Value rhs) {
332 if (isa<FloatType>(Val: lhs.getType()))
333 return arith::MulFOp::create(builder&: b, location: loc, lhs, rhs);
334 return arith::MulIOp::create(builder&: b, location: loc, lhs, rhs, overflowFlags: ovf);
335}
336Value ArithBuilder::sgt(Value lhs, Value rhs) {
337 if (isa<FloatType>(Val: lhs.getType()))
338 return arith::CmpFOp::create(builder&: b, location: loc, predicate: arith::CmpFPredicate::OGT, lhs, rhs);
339 return arith::CmpIOp::create(builder&: b, location: loc, predicate: arith::CmpIPredicate::sgt, lhs, rhs);
340}
341Value ArithBuilder::slt(Value lhs, Value rhs) {
342 if (isa<FloatType>(Val: lhs.getType()))
343 return arith::CmpFOp::create(builder&: b, location: loc, predicate: arith::CmpFPredicate::OLT, lhs, rhs);
344 return arith::CmpIOp::create(builder&: b, location: loc, predicate: arith::CmpIPredicate::slt, lhs, rhs);
345}
346Value ArithBuilder::select(Value cmp, Value lhs, Value rhs) {
347 return arith::SelectOp::create(builder&: b, location: loc, condition: cmp, true_value: lhs, false_value: rhs);
348}
349
350namespace mlir::arith {
351
352Value createProduct(OpBuilder &builder, Location loc, ArrayRef<Value> values) {
353 return createProduct(builder, loc, values, resultType: values.front().getType());
354}
355
356Value createProduct(OpBuilder &builder, Location loc, ArrayRef<Value> values,
357 Type resultType) {
358 Value one = ConstantOp::create(builder, location: loc, result: resultType,
359 value: builder.getOneAttr(type: resultType));
360 ArithBuilder arithBuilder(builder, loc);
361 return llvm::accumulate(Range&: values, Init&: one, Op: [&arithBuilder](Value acc, Value v) {
362 return arithBuilder.mul(lhs: acc, rhs: v);
363 });
364}
365
366FloatType parseFloatType(MLIRContext *ctx, StringRef name) {
367 // Parsing non-builtin types is unsafe because the respective dialect may not
368 // have been loaded.
369 if (!name.empty() && name.front() == '!')
370 return FloatType();
371
372 // Suppress diagnostics: callers handle invalid type strings themselves.
373 ScopedDiagnosticHandler handler(ctx, [](Diagnostic &) {});
374 return dyn_cast_or_null<FloatType>(Val: mlir::parseType(typeStr: name, context: ctx));
375}
376
377} // namespace mlir::arith
378