1//===- LinalgOps.cpp - Implementation of the linalg operations ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Linalg operations.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/Linalg/IR/Linalg.h"
14
15#include "mlir/AsmParser/AsmParser.h"
16#include "mlir/Dialect/Affine/IR/AffineOps.h"
17#include "mlir/Dialect/Arith/IR/Arith.h"
18#include "mlir/Dialect/Arith/Utils/Utils.h"
19#include "mlir/Dialect/Complex/IR/Complex.h"
20#include "mlir/Dialect/Math/IR/Math.h"
21#include "mlir/Dialect/MemRef/IR/MemRef.h"
22#include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
23#include "mlir/Dialect/Tensor/IR/Tensor.h"
24#include "mlir/Dialect/Utils/IndexingUtils.h"
25#include "mlir/Dialect/Utils/ReshapeOpsUtils.h"
26#include "mlir/Dialect/Utils/StaticValueUtils.h"
27#include "mlir/Dialect/Utils/VerificationUtils.h"
28#include "mlir/IR/AffineMap.h"
29#include "mlir/IR/Attributes.h"
30#include "mlir/IR/Builders.h"
31#include "mlir/IR/BuiltinAttributes.h"
32#include "mlir/IR/BuiltinTypeInterfaces.h"
33#include "mlir/IR/Matchers.h"
34#include "mlir/IR/OpImplementation.h"
35#include "mlir/IR/OperationSupport.h"
36#include "mlir/IR/PatternMatch.h"
37#include "mlir/IR/TypeUtilities.h"
38#include "mlir/Interfaces/InferTypeOpInterface.h"
39#include "mlir/Interfaces/SideEffectInterfaces.h"
40
41#include "llvm/ADT/DenseMap.h"
42#include "llvm/ADT/STLExtras.h"
43#include "llvm/ADT/SetOperations.h"
44#include "llvm/ADT/SmallVector.h"
45#include "llvm/ADT/SmallVectorExtras.h"
46#include "llvm/ADT/StringSet.h"
47#include "llvm/ADT/TypeSwitch.h"
48#include "llvm/Support/FormatVariadic.h"
49#include "llvm/Support/InterleavedRange.h"
50#include "llvm/Support/LogicalResult.h"
51#include "llvm/Support/MathExtras.h"
52#include "llvm/Support/raw_ostream.h"
53#include <cassert>
54#include <optional>
55
56using namespace mlir;
57using namespace mlir::linalg;
58
59/// Return a `memref.dim` or `tensor.dim` for the shape of `v` at `dim`.
60static OpFoldResult getDimValue(OpBuilder &builder, Location loc, Value v,
61 int64_t dim) {
62 auto type = cast<ShapedType>(Val: v.getType());
63 if (!type.isDynamicDim(idx: dim))
64 return builder.getIndexAttr(value: type.getDimSize(idx: dim));
65
66 return getAsOpFoldResult(
67 val: TypeSwitch<Type, Value>(v.getType())
68 .Case(caseFn: [&](RankedTensorType t) -> Value {
69 return tensor::DimOp::create(builder, location: loc, source: v, index: dim);
70 })
71 .Case(caseFn: [&](MemRefType t) -> Value {
72 return memref::DimOp::create(builder, location: loc, source: v, index: dim);
73 }));
74}
75
76/// Returns a memref.subview or a tensor.extract_slice based on the type of the
77/// `source`.
78static Operation *getSlice(OpBuilder &b, Location loc, Value source,
79 ArrayRef<OpFoldResult> offsets,
80 ArrayRef<OpFoldResult> sizes,
81 ArrayRef<OpFoldResult> strides) {
82 return TypeSwitch<Type, Operation *>(source.getType())
83 .Case(caseFn: [&](RankedTensorType t) -> Operation * {
84 return tensor::ExtractSliceOp::create(builder&: b, location: loc, source, offsets, sizes,
85 strides);
86 })
87 .Case(caseFn: [&](MemRefType type) -> Operation * {
88 return memref::SubViewOp::create(builder&: b, location: loc, source, offsets, sizes,
89 strides);
90 })
91 .Default(defaultFn: [&](Type t) -> Operation * { return nullptr; });
92}
93
94static std::optional<TypedAttr>
95getScalarConstantAttrFromDenseSplat(Value input) {
96 DenseElementsAttr splatAttr;
97 matchPattern(value: input, pattern: m_Constant<DenseElementsAttr>(bind_value: &splatAttr));
98 if (!splatAttr || !splatAttr.isSplat())
99 return std::nullopt;
100
101 // Not every element type has a TypedAttr splat value: a complex splat, for
102 // one, is an ArrayAttr. Decline the fold instead of asserting in the cast.
103 auto splatValue = dyn_cast<TypedAttr>(Val: splatAttr.getSplatValue<Attribute>());
104 if (!splatValue)
105 return std::nullopt;
106
107 return splatValue;
108}
109
110//===----------------------------------------------------------------------===//
111// Helper functions
112//===----------------------------------------------------------------------===//
113
114Value linalg::createOrFoldDimOp(OpBuilder &b, Location loc, Value source,
115 int64_t dim) {
116 if (llvm::isa<UnrankedMemRefType, MemRefType>(Val: source.getType()))
117 return b.createOrFold<memref::DimOp>(location: loc, args&: source, args&: dim);
118 if (llvm::isa<UnrankedTensorType, RankedTensorType>(Val: source.getType()))
119 return b.createOrFold<tensor::DimOp>(location: loc, args&: source, args&: dim);
120 llvm_unreachable("Expected MemRefType or TensorType");
121}
122
123OpFoldResult linalg::createFoldedDimOp(OpBuilder &b, Location loc, Value source,
124 int64_t dim) {
125 auto shapedType = llvm::cast<ShapedType>(Val: source.getType());
126 if (!shapedType.hasRank() || shapedType.isDynamicDim(idx: dim))
127 return createOrFoldDimOp(b, loc, source, dim);
128 return b.getIndexAttr(value: shapedType.getDimSize(idx: dim));
129}
130
131//===----------------------------------------------------------------------===//
132// Support for named Linalg ops defined in ods-gen.
133//===----------------------------------------------------------------------===//
134
135using RegionBuilderFn = llvm::function_ref<void(
136 ImplicitLocOpBuilder &, Block &, ArrayRef<NamedAttribute>,
137 function_ref<InFlightDiagnostic()>)>;
138
139/// Fills the region of a structured operation using the provided
140/// `regionBuilder`. The method is used by both named structured ops created by
141/// ods-gen and by manually defined C++ ops. It is called by both builders and
142/// parsers and creates a block with arguments corresponding to the elemental
143/// types of `inputTypes` and `outputTypes`.
144static void fillStructuredOpRegion(OpBuilder &opBuilder, Region &region,
145 TypeRange inputTypes, TypeRange outputTypes,
146 ArrayRef<NamedAttribute> attrs,
147 function_ref<InFlightDiagnostic()> emitError,
148 RegionBuilderFn regionBuilder) {
149 SmallVector<Type, 8> argTypes;
150 SmallVector<Location, 8> argLocs;
151 for (auto containers : {inputTypes, outputTypes}) {
152 for (auto t : containers) {
153 argTypes.push_back(
154 Elt: isa<MemRefType, RankedTensorType>(Val: t) ? getElementTypeOrSelf(type: t) : t);
155
156 // TODO: Pass in a proper location here.
157 argLocs.push_back(Elt: opBuilder.getUnknownLoc());
158 }
159 }
160
161 // RAII.
162 OpBuilder::InsertionGuard guard(opBuilder);
163 Block *body =
164 opBuilder.createBlock(parent: &region, /*insertPt=*/{}, argTypes, locs: argLocs);
165
166 opBuilder.setInsertionPointToStart(body);
167 ImplicitLocOpBuilder b(opBuilder.getUnknownLoc(), opBuilder);
168 regionBuilder(b, *body, attrs, emitError);
169
170 // indexing_maps is an auto-generated method.
171
172 // iterator_types is an auto-generated method.
173}
174
175/// Creates a structured operation given `inputs`, `outputs`, and `attributes`.
176/// The result types are derived automatically if `resultTensorTypes` is none.
177/// The body of the operation is filled using `regionBuilder`. All ods-gen
178/// created structured operations use the method to implement their builders.
179static void buildStructuredOp(OpBuilder &b, OperationState &state,
180 std::optional<TypeRange> resultTensorTypes,
181 ValueRange inputs, ValueRange outputs,
182 ArrayRef<NamedAttribute> attributes,
183 RegionBuilderFn regionBuilder) {
184 // Derive the result types if needed.
185 SmallVector<Type> derivedResultTypes =
186 resultTensorTypes.value_or(u: TypeRange());
187 if (!resultTensorTypes)
188 copy_if(Range: outputs.getTypes(), Out: std::back_inserter(x&: derivedResultTypes),
189 P: llvm::IsaPred<RankedTensorType>);
190
191 state.addOperands(newOperands: inputs);
192 state.addOperands(newOperands: outputs);
193 state.addTypes(newTypes: derivedResultTypes);
194
195 state.addAttributes(newAttributes: attributes);
196 state.addAttribute(
197 name: "operandSegmentSizes",
198 attr: b.getDenseI32ArrayAttr(values: {static_cast<int32_t>(inputs.size()),
199 static_cast<int32_t>(outputs.size())}));
200
201 // Create and fill the region of the structured operation.
202 Region &region = *state.addRegion();
203 fillStructuredOpRegion(opBuilder&: b, region, inputTypes: TypeRange(inputs), outputTypes: TypeRange(outputs),
204 attrs: state.attributes.getAttrs(), /*emitError=*/{},
205 regionBuilder);
206}
207
208static void buildMatmulOp(OpBuilder &b, OperationState &state,
209 std::optional<TypeRange> resultTensorTypes,
210 ValueRange inputs, ValueRange outputs,
211 ArrayRef<NamedAttribute> attributes,
212 RegionBuilderFn regionBuilder,
213 ArrayRef<AffineMap> defaultIndexingMaps) {
214 // If indexing maps are not provided, apply the default ones.
215 if (none_of(Range&: attributes, P: [](NamedAttribute attr) {
216 return attr.getName() == "indexing_maps";
217 })) {
218 SmallVector<Attribute, 3> indexingMapsAttrVal;
219 indexingMapsAttrVal = llvm::map_to_vector(
220 C&: defaultIndexingMaps,
221 F: [](AffineMap map) -> Attribute { return AffineMapAttr::get(value: map); });
222 state.addAttribute(name: "indexing_maps", attr: b.getArrayAttr(value: indexingMapsAttrVal));
223 }
224 return buildStructuredOp(b, state, resultTensorTypes, inputs, outputs,
225 attributes, regionBuilder);
226}
227
228static void buildBatchMatmulOp(OpBuilder &b, OperationState &state,
229 std::optional<TypeRange> resultTensorTypes,
230 ValueRange inputs, ValueRange outputs,
231 ArrayRef<NamedAttribute> attributes,
232 RegionBuilderFn regionBuilder,
233 ArrayRef<AffineMap> defaultIndexingMaps) {
234 // If indexing maps are not provided, apply the default ones.
235 if (none_of(Range&: attributes, P: [](NamedAttribute attr) {
236 return attr.getName() == "indexing_maps";
237 })) {
238 SmallVector<Attribute, 4> indexingMapsAttrVal;
239 indexingMapsAttrVal = llvm::map_to_vector(
240 C&: defaultIndexingMaps,
241 F: [](AffineMap map) -> Attribute { return AffineMapAttr::get(value: map); });
242 state.addAttribute(name: "indexing_maps", attr: b.getArrayAttr(value: indexingMapsAttrVal));
243 }
244 return buildStructuredOp(b, state, resultTensorTypes, inputs, outputs,
245 attributes, regionBuilder);
246}
247
248static void buildBatchReduceMatmulOp(OpBuilder &b, OperationState &state,
249 std::optional<TypeRange> resultTensorTypes,
250 ValueRange inputs, ValueRange outputs,
251 ArrayRef<NamedAttribute> attributes,
252 RegionBuilderFn regionBuilder,
253 ArrayRef<AffineMap> indexingMaps) {
254 // Initialize indexingMaps attribute, for BatchReduceMatmulOp.
255 SmallVector<Attribute, 4> indexingMapsAttrVal;
256 indexingMapsAttrVal =
257 llvm::map_to_vector(C&: indexingMaps, F: [](AffineMap map) -> Attribute {
258 return AffineMapAttr::get(value: map);
259 });
260 state.addAttribute(name: "indexing_maps", attr: b.getArrayAttr(value: indexingMapsAttrVal));
261 return buildStructuredOp(b, state, resultTensorTypes, inputs, outputs,
262 attributes, regionBuilder);
263}
264
265/// Common parsing used for both named structured ops created by ods-gen and by
266/// manually defined C++ ops. Does not handle regions.
267static ParseResult
268parseCommonStructuredOpParts(OpAsmParser &parser, OperationState &result,
269 SmallVectorImpl<Type> &inputTypes,
270 SmallVectorImpl<Type> &outputTypes,
271 bool addOperandSegmentSizes = true) {
272 SMLoc attrsLoc, inputsOperandsLoc, outputsOperandsLoc;
273 SmallVector<OpAsmParser::UnresolvedOperand, 4> inputsOperands,
274 outputsOperands;
275
276 if (succeeded(Result: parser.parseOptionalLess())) {
277 if (parser.parseAttribute(result&: result.propertiesAttr) || parser.parseGreater())
278 return failure();
279 }
280 attrsLoc = parser.getCurrentLocation();
281 if (parser.parseOptionalAttrDict(result&: result.attributes))
282 return failure();
283
284 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "ins"))) {
285 if (parser.parseLParen())
286 return failure();
287
288 inputsOperandsLoc = parser.getCurrentLocation();
289 if (parser.parseOperandList(result&: inputsOperands) ||
290 parser.parseColonTypeList(result&: inputTypes) || parser.parseRParen())
291 return failure();
292 }
293
294 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "outs"))) {
295 outputsOperandsLoc = parser.getCurrentLocation();
296 if (parser.parseLParen() || parser.parseOperandList(result&: outputsOperands) ||
297 parser.parseColonTypeList(result&: outputTypes) || parser.parseRParen())
298 return failure();
299 }
300
301 if (parser.resolveOperands(operands&: inputsOperands, types&: inputTypes, loc: inputsOperandsLoc,
302 result&: result.operands) ||
303 parser.resolveOperands(operands&: outputsOperands, types&: outputTypes, loc: outputsOperandsLoc,
304 result&: result.operands))
305 return failure();
306
307 if (addOperandSegmentSizes) {
308 // This is a bit complex because we're trying to be backward compatible with
309 // operation syntax that mix the inherent attributes and the discardable
310 // ones in the same dictionary. If the properties are used, we append the
311 // operandSegmentSizes there directly. Otherwise we append it to the
312 // discardable attributes dictionary where it is handled by the generic
313 // Operation::create(...) method.
314 if (result.propertiesAttr) {
315 NamedAttrList attrs = llvm::cast<DictionaryAttr>(Val&: result.propertiesAttr);
316 attrs.append(name: "operandSegmentSizes",
317 attr: parser.getBuilder().getDenseI32ArrayAttr(
318 values: {static_cast<int32_t>(inputsOperands.size()),
319 static_cast<int32_t>(outputsOperands.size())}));
320 result.propertiesAttr = attrs.getDictionary(context: parser.getContext());
321 } else {
322 result.addAttribute(name: "operandSegmentSizes",
323 attr: parser.getBuilder().getDenseI32ArrayAttr(
324 values: {static_cast<int32_t>(inputsOperands.size()),
325 static_cast<int32_t>(outputsOperands.size())}));
326 }
327 }
328 if (!result.propertiesAttr) {
329 std::optional<RegisteredOperationName> info =
330 result.name.getRegisteredInfo();
331 if (info) {
332 if (failed(Result: info->verifyInherentAttrs(attributes&: result.attributes, emitError: [&]() {
333 return parser.emitError(loc: attrsLoc)
334 << "'" << result.name.getStringRef() << "' op ";
335 })))
336 return failure();
337 }
338 }
339 return success();
340}
341
342static void printCommonStructuredOpParts(OpAsmPrinter &p, ValueRange inputs,
343 ValueRange outputs) {
344 if (!inputs.empty())
345 p << " ins(" << inputs << " : " << inputs.getTypes() << ")";
346 if (!outputs.empty())
347 p << " outs(" << outputs << " : " << outputs.getTypes() << ")";
348}
349
350//===----------------------------------------------------------------------===//
351// Specific parsing and printing for named structured ops created by ods-gen.
352//===----------------------------------------------------------------------===//
353
354static ParseResult parseNamedStructuredOpRegion(
355 OpAsmParser &parser, Region &region, unsigned numRegionArgs,
356 TypeRange inputTypes, TypeRange outputTypes, ArrayRef<NamedAttribute> attrs,
357 RegionBuilderFn regionBuilder, SMLoc loc) {
358 if (numRegionArgs != inputTypes.size() + outputTypes.size()) {
359 return parser.emitError(
360 loc: parser.getCurrentLocation(),
361 message: llvm::formatv(Fmt: "[parseNamedStructuredOpRegion] ods-gen generated "
362 "region expects {0} args, got {1}",
363 Vals&: numRegionArgs, Vals: inputTypes.size() + outputTypes.size()));
364 }
365
366 OpBuilder opBuilder(parser.getContext());
367 ParseResult result = success();
368 fillStructuredOpRegion(
369 opBuilder, region, inputTypes, outputTypes, attrs,
370 emitError: [&]() {
371 result = failure();
372 return parser.emitError(loc);
373 },
374 regionBuilder);
375 return result;
376}
377
378static ParseResult
379parseNamedStructuredOpResults(OpAsmParser &parser,
380 SmallVectorImpl<Type> &resultTypes) {
381 if (parser.parseOptionalArrowTypeList(result&: resultTypes))
382 return failure();
383 return success();
384}
385
386static ParseResult parseNamedStructuredOp(OpAsmParser &parser,
387 OperationState &result,
388 unsigned numRegionArgs,
389 RegionBuilderFn regionBuilder) {
390 // TODO: Enable when ods-gen supports captures.
391 SmallVector<Type, 1> inputTypes, outputTypes;
392 SMLoc loc = parser.getCurrentLocation();
393 if (parseCommonStructuredOpParts(parser, result, inputTypes, outputTypes))
394 return failure();
395
396 // Parse optional attributes.
397 if (parser.parseOptionalAttrDict(result&: result.attributes))
398 return failure();
399
400 // TODO: consider merging results parsing into region parsing.
401 // Need to wait for declarative assembly resolution to decide.
402 SmallVector<Type, 1> outputTensorsTypes;
403 if (parseNamedStructuredOpResults(parser, resultTypes&: outputTensorsTypes))
404 return failure();
405 result.addTypes(newTypes: outputTensorsTypes);
406
407 std::unique_ptr<Region> region = std::make_unique<Region>();
408 if (parseNamedStructuredOpRegion(parser, region&: *region, numRegionArgs, inputTypes,
409 outputTypes, attrs: result.attributes.getAttrs(),
410 regionBuilder, loc))
411 return failure();
412 result.addRegion(region: std::move(region));
413
414 return success();
415}
416
417static void printNamedStructuredOpResults(OpAsmPrinter &p,
418 TypeRange resultTypes) {
419 if (resultTypes.empty())
420 return;
421 p.printOptionalArrowTypeList(types&: resultTypes);
422}
423
424static void printNamedStructuredOp(OpAsmPrinter &p, Operation *op,
425 ValueRange inputs, ValueRange outputs,
426 ArrayRef<StringRef> elidedAttrs = {}) {
427 NamedAttrList attrs(op->getDiscardableAttrDictionary());
428 op->getName().walkInherentAttrs(
429 op, visitor: [&](StringRef name, Attribute &attr) { attrs.append(name, attr); });
430 p.printOptionalAttrDict(attrs, elidedAttrs);
431
432 // Printing is shared with generic ops, except for the region and
433 // attributes.
434 printCommonStructuredOpParts(p, inputs, outputs);
435
436 // Results printing.
437 printNamedStructuredOpResults(p, resultTypes: op->getResultTypes());
438
439 // Region is elided.
440}
441
442//===----------------------------------------------------------------------===//
443// Region builder helper.
444// TODO: Move this to a utility library.
445// The public methods on this class are referenced directly from generated code.
446// Helper build the unary, binary, and type conversion functions defined by the
447// DSL. See LinalgNamedStructuredOps.yamlgen.cpp.inc for the code that uses this
448// class.
449//
450// Implementations of the math functions must be polymorphic over numeric types,
451// internally performing necessary casts. If the function application makes no
452// sense, then the only recourse is to assert and return nullptr. This can be
453// extended later if it becomes possible to fail construction of the region. The
454// invariant should be enforced at a higher level.
455//
456// TODO: These helpers are currently type polymorphic over the class of integer
457// and floating point types, but they will not internally cast within bit
458// widths of a class (mixed precision such as i8->i32) or across classes
459// (i.e. mixed float and integer). Many such combinations are ambiguous or need
460// to be handled with care and work is being considered to extend the op
461// language to make such cases explicit. In the mean-time, violating this will
462// fail verification, which is deemed acceptable.
463//===----------------------------------------------------------------------===//
464
465namespace {
466
467class RegionBuilderHelper {
468public:
469 RegionBuilderHelper(OpBuilder &builder, Block &block)
470 : builder(builder), block(block) {}
471
472 // Build the unary functions defined by OpDSL.
473 Value buildUnaryFn(UnaryFn unaryFn, Value arg,
474 function_ref<InFlightDiagnostic()> emitError = {}) {
475 if (!isFloatingPoint(value: arg)) {
476 if (emitError) {
477 emitError() << "unsupported non numeric type";
478 return nullptr;
479 }
480 llvm_unreachable("unsupported non numeric type");
481 }
482 OpBuilder::InsertionGuard g(builder);
483 builder.setInsertionPointToEnd(&block);
484 switch (unaryFn) {
485 case UnaryFn::exp:
486 return math::ExpOp::create(builder, location: arg.getLoc(), operand: arg);
487 case UnaryFn::log:
488 return math::LogOp::create(builder, location: arg.getLoc(), operand: arg);
489 case UnaryFn::abs:
490 return math::AbsFOp::create(builder, location: arg.getLoc(), operand: arg);
491 case UnaryFn::ceil:
492 return math::CeilOp::create(builder, location: arg.getLoc(), operand: arg);
493 case UnaryFn::floor:
494 return math::FloorOp::create(builder, location: arg.getLoc(), operand: arg);
495 case UnaryFn::negf:
496 return arith::NegFOp::create(builder, location: arg.getLoc(), operand: arg);
497 case UnaryFn::reciprocal: {
498 Attribute oneAttr = builder.getOneAttr(type: arg.getType());
499 auto one = arith::ConstantOp::create(builder, location: arg.getLoc(),
500 value: ::cast<TypedAttr>(Val&: oneAttr));
501 return arith::DivFOp::create(builder, location: arg.getLoc(), lhs: one, rhs: arg);
502 }
503 case UnaryFn::round:
504 return math::RoundOp::create(builder, location: arg.getLoc(), operand: arg);
505 case UnaryFn::sqrt:
506 return math::SqrtOp::create(builder, location: arg.getLoc(), operand: arg);
507 case UnaryFn::rsqrt:
508 return math::RsqrtOp::create(builder, location: arg.getLoc(), operand: arg);
509 case UnaryFn::square:
510 return arith::MulFOp::create(builder, location: arg.getLoc(), lhs: arg, rhs: arg);
511 case UnaryFn::tanh:
512 return math::TanhOp::create(builder, location: arg.getLoc(), operand: arg);
513 case UnaryFn::erf:
514 return math::ErfOp::create(builder, location: arg.getLoc(), operand: arg);
515 case UnaryFn::sin:
516 return math::SinOp::create(builder, location: arg.getLoc(), operand: arg);
517 case UnaryFn::cos:
518 return math::CosOp::create(builder, location: arg.getLoc(), operand: arg);
519 case UnaryFn::tan:
520 return math::TanOp::create(builder, location: arg.getLoc(), operand: arg);
521 case UnaryFn::acos:
522 return math::AcosOp::create(builder, location: arg.getLoc(), operand: arg);
523 case UnaryFn::acosh:
524 return math::AcoshOp::create(builder, location: arg.getLoc(), operand: arg);
525 case UnaryFn::asin:
526 return math::AsinOp::create(builder, location: arg.getLoc(), operand: arg);
527 case UnaryFn::asinh:
528 return math::AsinhOp::create(builder, location: arg.getLoc(), operand: arg);
529 case UnaryFn::atan:
530 return math::AtanOp::create(builder, location: arg.getLoc(), operand: arg);
531 case UnaryFn::atanh:
532 return math::AtanhOp::create(builder, location: arg.getLoc(), operand: arg);
533 case UnaryFn::log10:
534 return math::Log10Op::create(builder, location: arg.getLoc(), operand: arg);
535 case UnaryFn::log1p:
536 return math::Log1pOp::create(builder, location: arg.getLoc(), operand: arg);
537 case UnaryFn::log2:
538 return math::Log2Op::create(builder, location: arg.getLoc(), operand: arg);
539 }
540 if (emitError) {
541 emitError() << "unsupported unary function";
542 return nullptr;
543 }
544 llvm_unreachable("unsupported unary function");
545 }
546
547 // Build the binary functions defined by OpDSL.
548 // If emitError is provided, an error will be emitted if the operation is not
549 // supported and a nullptr will be returned, otherwise an assertion will be
550 // raised.
551 Value buildBinaryFn(BinaryFn binaryFn, Value arg0, Value arg1,
552 function_ref<InFlightDiagnostic()> emitError = {}) {
553 bool allComplex = isComplex(value: arg0) && isComplex(value: arg1);
554 bool allFloatingPoint = isFloatingPoint(value: arg0) && isFloatingPoint(value: arg1);
555 bool allInteger = isInteger(value: arg0) && isInteger(value: arg1);
556 bool allBool = allInteger && arg0.getType().getIntOrFloatBitWidth() == 1 &&
557 arg1.getType().getIntOrFloatBitWidth() == 1;
558 if (!allComplex && !allFloatingPoint && !allInteger) {
559 if (emitError) {
560 emitError()
561 << "Cannot build binary Linalg operation: expects allComplex, "
562 "allFloatingPoint, or allInteger, got "
563 << arg0.getType() << " and " << arg1.getType();
564 return nullptr;
565 }
566 llvm_unreachable("unsupported non numeric type");
567 }
568 OpBuilder::InsertionGuard g(builder);
569 builder.setInsertionPointToEnd(&block);
570 switch (binaryFn) {
571 case BinaryFn::add:
572 if (allComplex)
573 return complex::AddOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
574 if (allFloatingPoint)
575 return arith::AddFOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
576 if (allBool)
577 return arith::OrIOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
578 return arith::AddIOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
579 case BinaryFn::sub:
580 if (allComplex)
581 return complex::SubOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
582 if (allFloatingPoint)
583 return arith::SubFOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
584 if (allBool) {
585 if (emitError) {
586 emitError() << "unsupported operation: sub with bools";
587 return nullptr;
588 }
589 llvm_unreachable("unsupported operation: sub with bools");
590 }
591 return arith::SubIOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
592 case BinaryFn::mul:
593 if (allComplex)
594 return complex::MulOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
595 if (allFloatingPoint)
596 return arith::MulFOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
597 if (allBool)
598 return arith::AndIOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
599 return arith::MulIOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
600 case BinaryFn::div:
601 if (allComplex)
602 return complex::DivOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
603 if (allFloatingPoint)
604 return arith::DivFOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
605 if (allBool) {
606 if (emitError) {
607 emitError() << "unsupported operation: div with bools";
608 return nullptr;
609 }
610 llvm_unreachable("unsupported operation: div with bools");
611 }
612 return arith::DivSIOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
613 case BinaryFn::div_unsigned:
614 if (!allInteger || allBool) {
615 if (emitError) {
616 emitError() << "unsupported operation: unsigned div not on uint";
617 return nullptr;
618 }
619 llvm_unreachable("unsupported operation: unsigned div not on uint");
620 }
621 return arith::DivUIOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
622 case BinaryFn::max_signed:
623 assert(!allComplex);
624 if (allFloatingPoint)
625 return arith::MaximumFOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
626 return arith::MaxSIOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
627 case BinaryFn::min_signed:
628 assert(!allComplex);
629 if (allFloatingPoint)
630 return arith::MinimumFOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
631 return arith::MinSIOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
632 case BinaryFn::max_unsigned:
633 assert(!allComplex);
634 if (!allInteger || allBool) {
635 if (emitError) {
636 emitError() << "unsupported operation: unsigned max not on uint";
637 return nullptr;
638 }
639 llvm_unreachable("unsupported operation: unsigned max not on uint");
640 }
641 return arith::MaxUIOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
642 case BinaryFn::min_unsigned:
643 assert(!allComplex);
644 if (!allInteger || allBool) {
645 if (emitError) {
646 emitError() << "unsupported operation: unsigned min not on uint";
647 return nullptr;
648 }
649 llvm_unreachable("unsupported operation: unsigned min not on uint");
650 }
651 return arith::MinUIOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
652 case BinaryFn::powf:
653 assert(allFloatingPoint);
654 return math::PowFOp::create(builder, location: arg0.getLoc(), lhs: arg0, rhs: arg1);
655 }
656 if (emitError) {
657 emitError() << "unsupported binary function";
658 return nullptr;
659 }
660 llvm_unreachable("unsupported binary function");
661 }
662
663 // Build the ternary functions defined by OpDSL.
664 Value buildTernaryFn(TernaryFn ternaryFn, Value arg0, Value arg1, Value arg2,
665 function_ref<InFlightDiagnostic()> emitError = {}) {
666 OpBuilder::InsertionGuard g(builder);
667 builder.setInsertionPointToEnd(&block);
668 switch (ternaryFn) {
669 case TernaryFn::select:
670 return arith::SelectOp::create(builder, location: arg0.getLoc(), condition: arg0, true_value: arg1, false_value: arg2);
671 }
672 if (emitError) {
673 emitError() << "unsupported ternary function";
674 return nullptr;
675 }
676 llvm_unreachable("unsupported ternary function");
677 }
678
679 // Build the type functions defined by OpDSL.
680 Value buildTypeFn(TypeFn typeFn, Type toType, Value operand,
681 function_ref<InFlightDiagnostic()> emitError = {}) {
682 switch (typeFn) {
683 case TypeFn::cast_signed:
684 return cast(toType, operand, isUnsignedCast: false);
685 case TypeFn::cast_unsigned:
686 return cast(toType, operand, isUnsignedCast: true);
687 }
688 if (emitError) {
689 emitError() << "unsupported type conversion function";
690 return nullptr;
691 }
692 llvm_unreachable("unsupported type conversion function");
693 }
694
695 void yieldOutputs(ValueRange values) {
696 OpBuilder::InsertionGuard g(builder);
697 builder.setInsertionPointToEnd(&block);
698 Location loc = builder.getUnknownLoc();
699 YieldOp::create(builder, location: loc, values);
700 }
701
702 Value constant(const std::string &value) {
703 OpBuilder::InsertionGuard g(builder);
704 builder.setInsertionPointToEnd(&block);
705 Location loc = builder.getUnknownLoc();
706 Attribute valueAttr = parseAttribute(attrStr: value, context: builder.getContext());
707 return arith::ConstantOp::create(builder, location: loc,
708 value: ::cast<TypedAttr>(Val&: valueAttr));
709 }
710
711 Value index(int64_t dim) {
712 OpBuilder::InsertionGuard g(builder);
713 builder.setInsertionPointToEnd(&block);
714 return IndexOp::create(builder, location: builder.getUnknownLoc(), dim);
715 }
716
717 Type getIntegerType(unsigned width) {
718 return IntegerType::get(context: builder.getContext(), width);
719 }
720
721 Type getFloat32Type() { return Float32Type::get(context: builder.getContext()); }
722 Type getFloat64Type() { return Float64Type::get(context: builder.getContext()); }
723
724private:
725 // Generates operations to cast the given operand to a specified type.
726 // If the cast cannot be performed, a warning will be issued and the
727 // operand returned as-is (which will presumably yield a verification
728 // issue downstream).
729 Value cast(Type toType, Value operand, bool isUnsignedCast) {
730 OpBuilder::InsertionGuard g(builder);
731 builder.setInsertionPointToEnd(&block);
732 auto loc = operand.getLoc();
733 if (isa<UnknownLoc>(Val: loc)) {
734 if (operand.getDefiningOp())
735 loc = operand.getDefiningOp()->getLoc();
736 else if (operand.getParentBlock() &&
737 operand.getParentBlock()->getParentOp())
738 loc = operand.getParentBlock()->getParentOp()->getLoc();
739 }
740 return convertScalarToDtype(b&: builder, loc, operand, toType, isUnsignedCast);
741 }
742
743 bool isComplex(Value value) {
744 return llvm::isa<ComplexType>(Val: value.getType());
745 }
746 bool isFloatingPoint(Value value) {
747 return llvm::isa<FloatType>(Val: value.getType());
748 }
749 bool isInteger(Value value) {
750 return llvm::isa<IntegerType>(Val: value.getType());
751 }
752
753 OpBuilder &builder;
754 Block &block;
755};
756
757} // namespace
758
759//===----------------------------------------------------------------------===//
760// CopyOp
761//===----------------------------------------------------------------------===//
762
763namespace {
764
765struct EraseSelfCopy : OpRewritePattern<CopyOp> {
766 using OpRewritePattern<CopyOp>::OpRewritePattern;
767 LogicalResult matchAndRewrite(CopyOp copyOp,
768 PatternRewriter &rewriter) const override {
769 if (copyOp.getInputs() != copyOp.getOutputs())
770 return rewriter.notifyMatchFailure(arg&: copyOp, msg: "not a self copy");
771 if (copyOp.hasPureBufferSemantics())
772 rewriter.eraseOp(op: copyOp);
773 else
774 rewriter.replaceOp(op: copyOp, newValues: copyOp.getInputs());
775
776 return success();
777 }
778};
779
780} // namespace
781
782void CopyOp::getCanonicalizationPatterns(RewritePatternSet &results,
783 MLIRContext *context) {
784 results.add<EraseSelfCopy>(arg&: context);
785}
786
787//===----------------------------------------------------------------------===//
788// FillOp
789//===----------------------------------------------------------------------===//
790
791namespace {
792
793/// Fold linalg.fill -> tensor.expand/collapse_shape chain.
794///
795/// For such op chains, we can create new linalg.fill ops with the result
796/// type of the tensor.expand/collapse_shape op.
797template <typename TensorReshapeOp>
798struct FoldFillWithTensorReshape : OpRewritePattern<TensorReshapeOp> {
799 using OpRewritePattern<TensorReshapeOp>::OpRewritePattern;
800 LogicalResult matchAndRewrite(TensorReshapeOp reshapeOp,
801 PatternRewriter &rewriter) const override {
802 auto oldFill = reshapeOp.getSrc().template getDefiningOp<FillOp>();
803 if (!oldFill)
804 return failure();
805
806 Location loc = oldFill.getLoc();
807 TensorReshapeOp newInit;
808 if constexpr (std::is_same<TensorReshapeOp, tensor::ExpandShapeOp>::value) {
809
810 newInit = TensorReshapeOp::create(
811 rewriter, loc, reshapeOp.getResultType(), oldFill.output(),
812 reshapeOp.getReassociation(), reshapeOp.getOutputShape(),
813 reshapeOp.getStaticOutputShape());
814 } else {
815 newInit = TensorReshapeOp::create(
816 rewriter, loc, reshapeOp.getResultType(), oldFill.output(),
817 reshapeOp.getReassociation());
818 }
819 rewriter.replaceOpWithNewOp<FillOp>(reshapeOp, ValueRange{oldFill.value()},
820 ValueRange{newInit});
821 return success();
822 }
823};
824
825/// Fold tensor.pad(linalg.fill) into linalg.fill if the padding value and the
826/// filling value are the same.
827struct FoldFillWithPad final : public OpRewritePattern<tensor::PadOp> {
828 using Base::Base;
829
830 LogicalResult matchAndRewrite(tensor::PadOp padOp,
831 PatternRewriter &rewriter) const override {
832 auto fillOp = padOp.getSource().getDefiningOp<linalg::FillOp>();
833 if (!fillOp)
834 return failure();
835
836 // We can only fold if the padding value is the same as the original
837 // filling value.
838 Value padValue = padOp.getConstantPaddingValue();
839 if (!padValue || fillOp.value() != padValue)
840 return failure();
841
842 ReifiedRankedShapedTypeDims reifiedShape;
843 if (failed(Result: reifyResultShapes(b&: rewriter, op: padOp, reifiedReturnShapes&: reifiedShape)))
844 return rewriter.notifyMatchFailure(
845 arg&: padOp, msg: "failed to reify tensor.pad op result shape");
846
847 auto emptyTensor =
848 tensor::EmptyOp::create(builder&: rewriter, location: padOp.getLoc(), sizes: reifiedShape.front(),
849 elementType: padOp.getResultType().getElementType());
850 Value replacement =
851 FillOp::create(builder&: rewriter, location: fillOp.getLoc(), inputs: ValueRange{padValue},
852 outputs: ValueRange{emptyTensor})
853 .getResult(i: 0);
854 if (replacement.getType() != padOp.getResultType()) {
855 replacement = tensor::CastOp::create(builder&: rewriter, location: fillOp.getLoc(),
856 dest: padOp.getResultType(), source: replacement);
857 }
858 rewriter.replaceOp(op: padOp, newValues: replacement);
859 return success();
860 }
861};
862
863/// Fold tensor.insert_slice(tensor.pad(<input>), linalg.fill) into
864/// tensor.insert_slice(<input>, linalg.fill) if the padding value and the
865/// filling value are the same.
866struct FoldInsertPadIntoFill : public OpRewritePattern<tensor::InsertSliceOp> {
867 using Base::Base;
868
869 LogicalResult matchAndRewrite(tensor::InsertSliceOp insertOp,
870 PatternRewriter &rewriter) const override {
871 auto srcPadOp = insertOp.getSource().getDefiningOp<tensor::PadOp>();
872 if (!srcPadOp)
873 return failure();
874
875 if (insertOp.getType().getRank() != insertOp.getSourceType().getRank())
876 return failure();
877
878 // Walk back the tensor.insert_slice chain and find the first destination
879 // value at the start of the chain.
880 Value firstDest = insertOp.getDest();
881 while (auto prevOp = firstDest.getDefiningOp<tensor::InsertSliceOp>()) {
882 if (prevOp.getType().getRank() != prevOp.getSourceType().getRank())
883 return failure();
884
885 // Make sure the range of values accessed are disjoint. Without this, we
886 // cannot fold tensor.pad away.
887 bool disjoint = false;
888 for (int i = 0, e = prevOp.getType().getRank(); i < e; ++i) {
889 // If the dimension has dynamic offset/size, we cannot guarantee
890 // disjoint. So just skip it.
891 if (insertOp.isDynamicOffset(idx: i) || insertOp.isDynamicSize(idx: i) ||
892 insertOp.isDynamicStride(idx: i) || prevOp.isDynamicOffset(idx: i) ||
893 prevOp.isDynamicSize(idx: i) || prevOp.isDynamicStride(idx: i))
894 continue;
895
896 // Get the range start and end, inclusively for both.
897 int64_t prevStart = prevOp.getStaticOffset(idx: i);
898 int64_t prevEnd = prevStart + (prevOp.getStaticSize(idx: i) - 1) *
899 prevOp.getStaticStride(idx: i);
900 int64_t nextStart = insertOp.getStaticOffset(idx: i);
901 int64_t nextEnd = nextStart + (insertOp.getStaticSize(idx: i) - 1) *
902 insertOp.getStaticStride(idx: i);
903 if (prevEnd < nextStart || nextEnd < prevStart) {
904 disjoint = true;
905 break;
906 }
907 }
908
909 if (!disjoint)
910 break;
911 firstDest = prevOp.getDest();
912 }
913
914 // Check whether the first destination is a fill op. For overlapped cases,
915 // this also cannot be true.
916 auto dstFillOp = firstDest.getDefiningOp<linalg::FillOp>();
917 if (!dstFillOp)
918 return failure();
919
920 // We can only fold if the padding value is the same as the original
921 // filling value.
922 Value padValue = srcPadOp.getConstantPaddingValue();
923 if (!padValue || dstFillOp.value() != padValue)
924 return failure();
925
926 SmallVector<OpFoldResult> lowPads = srcPadOp.getMixedLowPad();
927 SmallVector<OpFoldResult> oldOffsets = insertOp.getMixedOffsets();
928
929 Location loc = insertOp.getLoc();
930 MLIRContext *context = getContext();
931
932 AffineExpr sym0, sym1;
933 bindSymbols(ctx: context, exprs&: sym0, exprs&: sym1);
934 auto addMap = AffineMap::get(dimCount: 0, symbolCount: 2, results: {sym0 + sym1}, context);
935
936 // Calculate the new offsets for the insert. It should be the old offsets
937 // plus low padding sizes.
938 SmallVector<OpFoldResult, 4> newOffsets;
939 for (const auto &p : llvm::zip(t&: lowPads, u&: oldOffsets)) {
940 newOffsets.push_back(Elt: affine::makeComposedFoldedAffineApply(
941 b&: rewriter, loc, map: addMap, operands: {std::get<0>(t: p), std::get<1>(t: p)}));
942 }
943
944 RankedTensorType srcPadType = srcPadOp.getSourceType();
945 SmallVector<OpFoldResult, 4> newSizes;
946 for (int i = 0, e = srcPadType.getRank(); i < e; ++i) {
947 if (srcPadType.isDynamicDim(idx: i)) {
948 newSizes.push_back(
949 Elt: tensor::DimOp::create(builder&: rewriter, location: loc, source: srcPadOp.getSource(), index: i)
950 .getResult());
951 } else {
952 newSizes.push_back(Elt: rewriter.getIndexAttr(value: srcPadType.getDimSize(idx: i)));
953 }
954 }
955
956 rewriter.replaceOpWithNewOp<tensor::InsertSliceOp>(
957 op: insertOp, args: srcPadOp.getSource(), args: insertOp.getDest(), args&: newOffsets,
958 args&: newSizes, args: insertOp.getMixedStrides());
959 return success();
960 }
961};
962
963/// Fold tensor.extract(linalg.fill(<input>)) into <input>
964struct FoldFillWithTensorExtract : public OpRewritePattern<tensor::ExtractOp> {
965public:
966 using OpRewritePattern<tensor::ExtractOp>::OpRewritePattern;
967
968 LogicalResult matchAndRewrite(tensor::ExtractOp extractOp,
969 PatternRewriter &rewriter) const override {
970 // See if tensor input of tensor.extract op is the result of a linalg.fill
971 // op.
972 auto fillOp = extractOp.getTensor().getDefiningOp<linalg::FillOp>();
973 if (!fillOp)
974 return failure();
975
976 // Get scalar input operand of linalg.fill op.
977 Value extractedScalar = fillOp.getInputs()[0];
978
979 // Replace tensor.extract op with scalar value used to fill the tensor.
980 rewriter.replaceOp(op: extractOp, newValues: extractedScalar);
981 return success();
982 }
983};
984
985/// Folds pack(fill) into a single fill op if
986/// 1. The pack op does not have padding value, or
987/// 2. The filled value and padding value are the same.
988static FailureOr<FillOp> foldFillPackIntoFillOp(RewriterBase &rewriter,
989 linalg::PackOp packOp) {
990 auto fillOp = packOp.getSource().getDefiningOp<FillOp>();
991 if (!fillOp)
992 return failure();
993
994 if (auto paddingValue = packOp.getPaddingValue())
995 if (!isEqualConstantIntOrValue(ofr1: paddingValue, ofr2: fillOp.value()))
996 return failure();
997
998 Value packOpDest = packOp.getDest();
999 if (!packOpDest.hasOneUse())
1000 return failure();
1001
1002 return linalg::FillOp::create(builder&: rewriter, location: packOp.getLoc(), inputs: fillOp.getInputs(),
1003 outputs: packOp.getDest());
1004}
1005
1006/// Wrapper pattern that applies foldFillPackIntoFillOp method.
1007struct FoldFillWithPack : public OpRewritePattern<linalg::PackOp> {
1008public:
1009 FoldFillWithPack(MLIRContext *context)
1010 : OpRewritePattern<linalg::PackOp>(context) {}
1011
1012 LogicalResult matchAndRewrite(linalg::PackOp packOp,
1013 PatternRewriter &rewriter) const override {
1014 auto fillOp = foldFillPackIntoFillOp(rewriter, packOp);
1015 if (failed(Result: fillOp))
1016 return failure();
1017 rewriter.replaceOp(op: packOp, newValues: fillOp.value().result());
1018 return success();
1019 }
1020};
1021
1022/// Fold fill with copy.
1023struct FoldFillWithCopy : OpRewritePattern<linalg::CopyOp> {
1024 using OpRewritePattern<linalg::CopyOp>::OpRewritePattern;
1025
1026 LogicalResult matchAndRewrite(linalg::CopyOp copyOp,
1027 PatternRewriter &rewriter) const override {
1028 if (auto fillOp = copyOp.getInputs().front().getDefiningOp<FillOp>()) {
1029 rewriter.replaceOpWithNewOp<FillOp>(op: copyOp, args: copyOp.getResultTypes(),
1030 args: fillOp.getInputs(),
1031 args: copyOp.getOutputs());
1032 return success();
1033 }
1034 if (auto fillOp = copyOp.getOutputs().front().getDefiningOp<FillOp>()) {
1035 rewriter.replaceOpWithNewOp<linalg::CopyOp>(op: copyOp, args: copyOp.getInputs(),
1036 args: fillOp.getOutputs());
1037 return success();
1038 }
1039 return failure();
1040 }
1041};
1042
1043/// Fold fill with transpose.
1044struct FoldFillWithTranspose : OpRewritePattern<linalg::TransposeOp> {
1045 using OpRewritePattern<linalg::TransposeOp>::OpRewritePattern;
1046
1047 LogicalResult matchAndRewrite(linalg::TransposeOp transposeOp,
1048 PatternRewriter &rewriter) const override {
1049 if (auto fillOp = transposeOp.getInput().getDefiningOp<FillOp>()) {
1050 rewriter.replaceOpWithNewOp<FillOp>(
1051 op: transposeOp, args: transposeOp.getResultTypes(), args: fillOp.getInputs(),
1052 args: transposeOp.getDpsInitOperand(i: 0)->get());
1053 return success();
1054 }
1055 return failure();
1056 }
1057};
1058
1059/// Fold a concat with all elements being fills of the same value
1060/// into a fill of the concat result shape.
1061struct FoldConcatsOfFill : public OpRewritePattern<tensor::ConcatOp> {
1062 using Base::Base;
1063
1064 LogicalResult matchAndRewrite(tensor::ConcatOp concatOp,
1065 PatternRewriter &rewriter) const override {
1066 auto concatOperands = concatOp.getInputs();
1067 if (concatOperands.empty()) {
1068 return failure();
1069 }
1070
1071 auto firstFillOp = concatOperands.front().getDefiningOp<linalg::FillOp>();
1072 if (!firstFillOp) {
1073 return failure();
1074 }
1075 // Prefetch the fill value.
1076 OpFoldResult firstFillVal =
1077 getAsOpFoldResult(val: firstFillOp.getDpsInputOperand(i: 0)->get());
1078 // Collect all the outs values for the fill operations.
1079 SmallVector<Value> allOuts;
1080 allOuts.push_back(Elt: firstFillOp.getDpsInitOperand(i: 0)->get());
1081
1082 auto isDefinedByCompatibleFillOp = [&](Value v) -> bool {
1083 auto fillOp = v.getDefiningOp<linalg::FillOp>();
1084 if (!fillOp) {
1085 return false;
1086 }
1087
1088 OpFoldResult fillVal =
1089 getAsOpFoldResult(val: fillOp.getDpsInputOperand(i: 0)->get());
1090 if (fillVal != firstFillVal)
1091 return false;
1092
1093 allOuts.push_back(Elt: fillOp.getDpsInitOperand(i: 0)->get());
1094 return true;
1095 };
1096 if (!llvm::all_of(Range: concatOperands.drop_front(),
1097 P: isDefinedByCompatibleFillOp)) {
1098 return rewriter.notifyMatchFailure(
1099 arg&: concatOp, msg: "not all operands are defined by a compatible fill op");
1100 }
1101
1102 Value outsConcat = tensor::ConcatOp::create(builder&: rewriter, location: concatOp.getLoc(),
1103 dim: concatOp.getDim(), inputs: allOuts);
1104 rewriter.replaceOpWithNewOp<linalg::FillOp>(
1105 op: concatOp, args: firstFillOp.getDpsInputOperand(i: 0)->get(), args&: outsConcat);
1106 return success();
1107 }
1108};
1109
1110} // namespace
1111
1112void FillOp::getCanonicalizationPatterns(RewritePatternSet &results,
1113 MLIRContext *context) {
1114 results.add<FoldConcatsOfFill, FoldFillWithCopy, FoldFillWithTensorExtract,
1115 FoldFillWithPack, FoldFillWithPad,
1116 FoldFillWithTensorReshape<tensor::CollapseShapeOp>,
1117 FoldFillWithTensorReshape<tensor::ExpandShapeOp>,
1118 FoldInsertPadIntoFill, FoldFillWithTranspose>(arg&: context);
1119}
1120
1121//===----------------------------------------------------------------------===//
1122// GenericOp
1123//===----------------------------------------------------------------------===//
1124
1125static void buildGenericRegion(
1126 OpBuilder &builder, Location loc, Region &region, ValueRange inputs,
1127 ValueRange outputs,
1128 function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuild) {
1129 SmallVector<Type, 4> blockArgTypes;
1130 SmallVector<Location, 4> blockArgLocs;
1131 for (ValueRange container : {inputs, outputs}) {
1132 for (Value v : container) {
1133 Type t = v.getType();
1134 blockArgTypes.push_back(
1135 Elt: isa<MemRefType, RankedTensorType>(Val: t) ? getElementTypeOrSelf(type: t) : t);
1136 blockArgLocs.push_back(Elt: v.getLoc());
1137 }
1138 }
1139
1140 OpBuilder::InsertionGuard guard(builder);
1141 Block *bodyBlock =
1142 builder.createBlock(parent: &region, insertPt: region.end(), argTypes: blockArgTypes, locs: blockArgLocs);
1143 bodyBuild(builder, loc, bodyBlock->getArguments());
1144}
1145
1146void GenericOp::getAsmBlockArgumentNames(Region &region,
1147 OpAsmSetValueNameFn setNameFn) {
1148 for (Value v : getRegionInputArgs())
1149 setNameFn(v, "in");
1150 for (Value v : getRegionOutputArgs())
1151 setNameFn(v, "out");
1152}
1153
1154void GenericOp::build(
1155 OpBuilder &builder, OperationState &result, TypeRange resultTensorTypes,
1156 ValueRange inputs, ValueRange outputs, ArrayAttr indexingMaps,
1157 ArrayAttr iteratorTypes, StringAttr doc, StringAttr libraryCall,
1158 function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuild,
1159 ArrayRef<NamedAttribute> attributes) {
1160 build(odsBuilder&: builder, odsState&: result, result_tensors: resultTensorTypes, inputs, outputs, indexing_maps: indexingMaps,
1161 iterator_types: iteratorTypes, doc, library_call: libraryCall);
1162 result.addAttributes(newAttributes: attributes);
1163 if (bodyBuild)
1164 buildGenericRegion(builder, loc: result.location, region&: *result.regions.front(),
1165 inputs, outputs, bodyBuild);
1166}
1167
1168void GenericOp::build(
1169 OpBuilder &builder, OperationState &result, TypeRange resultTensorTypes,
1170 ValueRange inputs, ValueRange outputs, ArrayRef<AffineMap> indexingMaps,
1171 ArrayRef<utils::IteratorType> iteratorTypes, StringRef doc,
1172 StringRef libraryCall,
1173 function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuild,
1174 ArrayRef<NamedAttribute> attributes) {
1175 build(builder, result, resultTensorTypes, inputs, outputs,
1176 indexingMaps: builder.getAffineMapArrayAttr(values: indexingMaps),
1177 iteratorTypes: builder.getArrayAttr(value: llvm::map_to_vector(
1178 C&: iteratorTypes,
1179 F: [&](utils::IteratorType iter) -> mlir::Attribute {
1180 return IteratorTypeAttr::get(context: builder.getContext(), value: iter);
1181 })),
1182 doc: doc.empty() ? StringAttr() : builder.getStringAttr(bytes: doc),
1183 libraryCall: libraryCall.empty() ? StringAttr() : builder.getStringAttr(bytes: libraryCall),
1184 bodyBuild, attributes);
1185}
1186
1187void GenericOp::build(
1188 OpBuilder &builder, OperationState &result, ValueRange inputs,
1189 ValueRange outputs, ArrayRef<AffineMap> indexingMaps,
1190 ArrayRef<utils::IteratorType> iteratorTypes, StringRef doc,
1191 StringRef libraryCall,
1192 function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuild,
1193 ArrayRef<NamedAttribute> attributes) {
1194 build(builder, result, resultTensorTypes: TypeRange{}, inputs, outputs, indexingMaps,
1195 iteratorTypes, doc, libraryCall, bodyBuild, attributes);
1196}
1197
1198void GenericOp::build(
1199 OpBuilder &builder, OperationState &result, ValueRange inputs,
1200 ValueRange outputs, ArrayRef<AffineMap> indexingMaps,
1201 ArrayRef<utils::IteratorType> iteratorTypes,
1202 function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuild,
1203 ArrayRef<NamedAttribute> attributes) {
1204 build(builder, result, inputs, outputs, indexingMaps, iteratorTypes,
1205 /*doc=*/"",
1206 /*libraryCall=*/"", bodyBuild, attributes);
1207}
1208
1209void GenericOp::build(
1210 OpBuilder &builder, OperationState &result, TypeRange resultTensorTypes,
1211 ValueRange inputs, ValueRange outputs, ArrayRef<AffineMap> indexingMaps,
1212 ArrayRef<utils::IteratorType> iteratorTypes,
1213 function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuild,
1214 ArrayRef<NamedAttribute> attributes) {
1215 build(builder, result, resultTensorTypes, inputs, outputs, indexingMaps,
1216 iteratorTypes,
1217 /*doc=*/"",
1218 /*libraryCall=*/"", bodyBuild, attributes);
1219}
1220
1221void GenericOp::print(OpAsmPrinter &p) {
1222 p << " ";
1223
1224 // Print extra attributes.
1225 auto genericAttrNames = linalgTraitAttrNames();
1226
1227 llvm::StringSet<> genericAttrNamesSet;
1228 genericAttrNamesSet.insert_range(R&: genericAttrNames);
1229 SmallVector<NamedAttribute, 8> genericAttrs;
1230 for (StringRef attrName : genericAttrNames) {
1231 std::optional<Attribute> value = (*this)->getInherentAttr(name: attrName);
1232 if (!value || !*value)
1233 continue;
1234 NamedAttribute attr{StringAttr::get(context: getContext(), bytes: attrName), *value};
1235 if (attr.getName() == getIteratorTypesAttrName()) {
1236 auto iteratorTypes =
1237 llvm::cast<ArrayAttr>(Val: attr.getValue())
1238 .getAsValueRange<IteratorTypeAttr, utils::IteratorType>();
1239 // Convert IteratorType enums into the string representation. This is
1240 // needed, because tests still use the old format when 'iterator_types'
1241 // attribute is represented as an array of strings.
1242 // TODO: Remove this conversion once tests are fixed.
1243 SmallVector<Attribute> iteratorTypeNames = llvm::map_to_vector(
1244 C&: iteratorTypes, F: [&](utils::IteratorType t) -> Attribute {
1245 return StringAttr::get(context: getContext(), bytes: stringifyIteratorType(t));
1246 });
1247
1248 genericAttrs.emplace_back(
1249 Args: getIteratorTypesAttrName(),
1250 Args: ArrayAttr::get(context: getContext(), value: iteratorTypeNames));
1251 } else if (genericAttrNamesSet.count(Key: attr.getName().strref()) > 0) {
1252 genericAttrs.push_back(Elt: attr);
1253 }
1254 }
1255 if (!genericAttrs.empty()) {
1256 auto genericDictAttr = DictionaryAttr::get(context: getContext(), value: genericAttrs);
1257 p << genericDictAttr;
1258 }
1259
1260 // Printing is shared with named ops, except for the region and attributes
1261 printCommonStructuredOpParts(p, inputs: getDpsInputs(), outputs: getDpsInits());
1262
1263 genericAttrNames.push_back(Elt: "operandSegmentSizes");
1264 genericAttrNamesSet.insert(key: genericAttrNames.back());
1265
1266 bool hasExtraAttrs = false;
1267 for (NamedAttribute n : (*this)->getDiscardableAttrDictionary()) {
1268 if ((hasExtraAttrs = !genericAttrNamesSet.contains(key: n.getName().strref())))
1269 break;
1270 }
1271 if (hasExtraAttrs) {
1272 p << " attrs = ";
1273 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary(),
1274 /*elidedAttrs=*/genericAttrNames);
1275 }
1276
1277 // Print region.
1278 if (!getRegion().empty()) {
1279 p << ' ';
1280 p.printRegion(blocks&: getRegion());
1281 }
1282
1283 // Print results.
1284 printNamedStructuredOpResults(p, resultTypes: getResultTensors().getTypes());
1285}
1286
1287ParseResult GenericOp::parse(OpAsmParser &parser, OperationState &result) {
1288 DictionaryAttr dictAttr;
1289 // Parse the core linalg traits that must check into a dictAttr.
1290 // The name is unimportant as we will overwrite result.attributes.
1291 // The core linalg traits must contain the information necessary to pass the
1292 // verifier.
1293 llvm::SMLoc attributeLocation = parser.getCurrentLocation();
1294 if (parser.parseAttribute(result&: dictAttr, attrName: "_", attrs&: result.attributes))
1295 return failure();
1296 result.attributes.assign(inStart: dictAttr.getValue().begin(),
1297 inEnd: dictAttr.getValue().end());
1298
1299 // Convert array of string into an array of IteratorType enums. This is
1300 // needed, because tests still use the old format when 'iterator_types'
1301 // attribute is represented as an array of strings.
1302 // TODO: Remove this conversion once tests are fixed.
1303 auto iteratorTypes = dyn_cast_or_null<ArrayAttr>(
1304 Val: result.attributes.get(name: getIteratorTypesAttrName(name: result.name)));
1305 if (!iteratorTypes) {
1306 return parser.emitError(loc: attributeLocation)
1307 << "expected " << getIteratorTypesAttrName(name: result.name)
1308 << " array attribute";
1309 }
1310
1311 SmallVector<Attribute> iteratorTypeAttrs;
1312
1313 for (StringRef s : iteratorTypes.getAsValueRange<StringAttr>()) {
1314 auto maybeIteratorType = utils::symbolizeIteratorType(s);
1315 if (!maybeIteratorType.has_value())
1316 return parser.emitError(loc: parser.getCurrentLocation())
1317 << "unexpected iterator_type (" << s << ")";
1318
1319 iteratorTypeAttrs.push_back(
1320 Elt: IteratorTypeAttr::get(context: parser.getContext(), value: maybeIteratorType.value()));
1321 }
1322 result.attributes.set(name: getIteratorTypesAttrName(name: result.name),
1323 value: parser.getBuilder().getArrayAttr(value: iteratorTypeAttrs));
1324
1325 // Parsing is shared with named ops, except for the region.
1326 SmallVector<Type, 1> inputTypes, outputTypes;
1327 if (parseCommonStructuredOpParts(parser, result, inputTypes, outputTypes))
1328 return failure();
1329
1330 // Optional attributes may be added.
1331 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "attrs")))
1332 if (failed(Result: parser.parseEqual()) ||
1333 failed(Result: parser.parseOptionalAttrDict(result&: result.attributes)))
1334 return failure();
1335
1336 std::unique_ptr<Region> region = std::make_unique<Region>();
1337 if (parser.parseRegion(region&: *region, arguments: {}))
1338 return failure();
1339 result.addRegion(region: std::move(region));
1340
1341 // Generic ops may specify that a subset of its outputs are tensors. Such
1342 // outputs are specified in the result type.
1343 // TODO: may need to move output parsing before region parsing.
1344 // Need to wait for declarative assembly resolution to decide.
1345 SmallVector<Type, 1> outputTensorsTypes;
1346 if (parseNamedStructuredOpResults(parser, resultTypes&: outputTensorsTypes))
1347 return failure();
1348 result.addTypes(newTypes: outputTensorsTypes);
1349
1350 return success();
1351}
1352
1353static void getGenericEffectsImpl(
1354 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1355 &effects,
1356 LinalgOp linalgOp) {
1357 for (auto [index, operand] : llvm::enumerate(First: linalgOp.getDpsInputs())) {
1358 if (!llvm::isa<MemRefType>(Val: operand.getType()))
1359 continue;
1360 effects.emplace_back(
1361 Args: MemoryEffects::Read::get(), Args: &linalgOp->getOpOperand(idx: index), /*stage=*/Args: 0,
1362 /*effectOnFullRegion=*/Args: true, Args: SideEffects::DefaultResource::get());
1363 }
1364
1365 for (OpOperand &operand : linalgOp.getDpsInitsMutable()) {
1366 if (!llvm::isa<MemRefType>(Val: operand.get().getType()))
1367 continue;
1368 if (linalgOp.payloadUsesValueFromOperand(opOperand: &operand)) {
1369 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: &operand, /*stage=*/Args: 0,
1370 /*effectOnFullRegion=*/Args: true,
1371 Args: SideEffects::DefaultResource::get());
1372 }
1373 effects.emplace_back(Args: MemoryEffects::Write::get(), Args: &operand, /*stage=*/Args: 0,
1374 /*effectOnFullRegion=*/Args: true,
1375 Args: SideEffects::DefaultResource::get());
1376 }
1377}
1378
1379void GenericOp::getEffects(
1380 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1381 &effects) {
1382 getGenericEffectsImpl(effects, linalgOp: cast<LinalgOp>(Val: getOperation()));
1383}
1384
1385static Speculation::Speculatability
1386getGenericSpeculatabilityImpl(LinalgOp linalgOp) {
1387 // Operands with value semantics are speculatable, while operands with memory
1388 // semantics are not.
1389 if (!linalgOp.hasPureTensorSemantics())
1390 return Speculation::NotSpeculatable;
1391 // The body of the op can still have speculation in its region.
1392 return Speculation::RecursivelySpeculatable;
1393}
1394
1395Speculation::Speculatability GenericOp::getSpeculatability() {
1396 return getGenericSpeculatabilityImpl(linalgOp: cast<LinalgOp>(Val: getOperation()));
1397}
1398
1399namespace {
1400
1401/// Remove linalg operations that are just copying the values from inputs to
1402/// results. In the memref case, the operation must be copying to and from the
1403/// same value. Requirements are:
1404/// 1) All iterator types are parallel
1405/// 2) The body contains just a yield operation with the yielded values being
1406/// the arguments corresponding to the operands.
1407template <typename OpTy>
1408struct EraseIdentityLinalgOp : public OpRewritePattern<OpTy> {
1409 using OpRewritePattern<OpTy>::OpRewritePattern;
1410
1411 LogicalResult matchAndRewrite(OpTy linalgOp,
1412 PatternRewriter &rewriter) const override {
1413 // All indexing maps must be equal. It follows that they are permutations.
1414 if (!llvm::all_equal(linalgOp.getIndexingMapsArray()))
1415 return failure();
1416
1417 // Check that the body of the linalg operation is just a linalg.yield
1418 // operation.
1419 Block &body = linalgOp->getRegion(0).front();
1420 if (!llvm::hasSingleElement(C&: body))
1421 return failure();
1422 auto yieldOp = dyn_cast<linalg::YieldOp>(Val: body.getTerminator());
1423 if (!yieldOp)
1424 return failure();
1425
1426 // In the buffer case, we need to check exact buffer equality.
1427 if (linalgOp.hasPureBufferSemantics()) {
1428 if (linalgOp.getNumDpsInputs() != 1 || linalgOp.getNumDpsInits() != 1 ||
1429 linalgOp.getDpsInputOperand(0)->get() !=
1430 linalgOp.getDpsInitOperand(0)->get()) {
1431 return rewriter.notifyMatchFailure(
1432 linalgOp, "expected single input and output to be the same value");
1433 }
1434
1435 auto yieldArg = dyn_cast<BlockArgument>(Val: yieldOp.getOperand(i: 0));
1436 if (!yieldArg || yieldArg.getOwner() != &body) {
1437 return rewriter.notifyMatchFailure(linalgOp,
1438 "cannot fold fill-like op");
1439 }
1440
1441 rewriter.eraseOp(op: linalgOp);
1442 return success();
1443 }
1444
1445 if (!linalgOp.hasPureTensorSemantics()) {
1446 return rewriter.notifyMatchFailure(
1447 linalgOp, "mixed semantics is not supported yet");
1448 }
1449
1450 // Get the argument number of the returned values. That is the operand
1451 // number to use for replacing uses of this operation.
1452 SmallVector<Value> returnedArgs;
1453 for (const auto &yieldVal : llvm::enumerate(First: yieldOp.getValues())) {
1454 auto yieldArg = llvm::dyn_cast<BlockArgument>(Val&: yieldVal.value());
1455 if (!yieldArg || yieldArg.getOwner() != &body)
1456 return failure();
1457 unsigned argumentNumber = yieldArg.getArgNumber();
1458 Value returnedArg = linalgOp->getOperand(argumentNumber);
1459 Type resultType = linalgOp->getResult(yieldVal.index()).getType();
1460 // The input can have a different type than the result, e.g. a dynamic
1461 // input dimension can be turned into a static output dimension.
1462 Type returnType = returnedArg.getType();
1463 if (returnType != resultType) {
1464 // Distinguish between sparse conversion or dense tensor casting.
1465 // TODO: unify the two ops?
1466 if (sparse_tensor::getSparseTensorEncoding(type: returnType) ||
1467 sparse_tensor::getSparseTensorEncoding(type: resultType))
1468 returnedArg = sparse_tensor::ConvertOp::create(
1469 rewriter, linalgOp.getLoc(), resultType, returnedArg);
1470 else {
1471 if (!tensor::CastOp::areCastCompatible(inputs: returnedArg.getType(),
1472 outputs: resultType))
1473 return failure();
1474 returnedArg = tensor::CastOp::create(rewriter, linalgOp.getLoc(),
1475 resultType, returnedArg);
1476 }
1477 }
1478 returnedArgs.push_back(Elt: returnedArg);
1479 }
1480
1481 if (returnedArgs.size() != linalgOp->getNumResults())
1482 return failure();
1483 rewriter.replaceOp(linalgOp, returnedArgs);
1484 return success();
1485 }
1486};
1487
1488} // namespace
1489
1490void GenericOp::getCanonicalizationPatterns(RewritePatternSet &results,
1491 MLIRContext *context) {
1492 results.add<EraseIdentityLinalgOp<GenericOp>>(arg&: context);
1493}
1494
1495LogicalResult GenericOp::fold(FoldAdaptor, SmallVectorImpl<OpFoldResult> &) {
1496 return memref::foldMemRefCast(op: *this);
1497}
1498
1499//===----------------------------------------------------------------------===//
1500// MapOp
1501//===----------------------------------------------------------------------===//
1502
1503static ParseResult parseDstStyleOp(
1504 OpAsmParser &parser, OperationState &result,
1505 function_ref<ParseResult(OpAsmParser &, NamedAttrList &)> parseAttrsFn =
1506 nullptr) {
1507 // Parse `ins` and `outs`.
1508 SmallVector<Type, 4> inputTypes, outputTypes;
1509 if (parseCommonStructuredOpParts(parser, result, inputTypes, outputTypes,
1510 /*addOperandSegmentSizes=*/false))
1511 return failure();
1512
1513 // Add result types.
1514 for (Type outputType : outputTypes) {
1515 if (llvm::isa<RankedTensorType>(Val: outputType))
1516 result.addTypes(newTypes: outputType);
1517 }
1518
1519 // Parse required attributes.
1520 if (parseAttrsFn && failed(Result: parseAttrsFn(parser, result.attributes)))
1521 return failure();
1522
1523 // Parse optional attributes.
1524 if (parser.parseOptionalAttrDict(result&: result.attributes))
1525 return failure();
1526 return success();
1527}
1528
1529void MapOp::getAsmBlockArgumentNames(Region &region,
1530 OpAsmSetValueNameFn setNameFn) {
1531 for (Value v : getRegionInputArgs())
1532 setNameFn(v, "in");
1533 for (Value v : getRegionOutputArgs())
1534 setNameFn(v, "init");
1535}
1536
1537void MapOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) {
1538 if (!getResults().empty())
1539 setNameFn(getResults().front(), "mapped");
1540}
1541
1542void MapOp::build(
1543 OpBuilder &builder, OperationState &result, ValueRange inputs, Value init,
1544 function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuild,
1545 ArrayRef<NamedAttribute> attributes) {
1546 build(odsBuilder&: builder, odsState&: result, result: TypeRange{}, inputs, init);
1547 result.addAttributes(newAttributes: attributes);
1548
1549 // Add output types for `RankedTensorType` output arguments.
1550 Type initType = init.getType();
1551 if (llvm::isa<RankedTensorType>(Val: initType))
1552 result.addTypes(newTypes: initType);
1553
1554 if (bodyBuild)
1555 buildGenericRegion(builder, loc: result.location, region&: *result.regions.front(),
1556 inputs, /*outputs=*/{init}, bodyBuild);
1557}
1558
1559static void addBodyWithPayloadOp(OpAsmParser &parser, OperationState &result,
1560 const OperationName &payloadOpName,
1561 const NamedAttrList &payloadOpAttrs,
1562 ArrayRef<Value> operands,
1563 bool initFirst = false, bool mapInit = true) {
1564 OpBuilder b(parser.getContext());
1565 Region *body = result.addRegion();
1566 Block &block = body->emplaceBlock();
1567 b.setInsertionPointToStart(&block);
1568 for (auto &operand : operands) {
1569 block.addArgument(
1570 type: llvm::cast<ShapedType>(Val: operand.getType()).getElementType(),
1571 loc: b.getUnknownLoc());
1572 }
1573 SmallVector<Value> payloadOpOperands;
1574 // If initFirst flag is enabled, we consider init as the first position of
1575 // payload operands.
1576 if (initFirst) {
1577 if (mapInit)
1578 payloadOpOperands.push_back(Elt: block.getArguments().back());
1579 for (const auto &arg : block.getArguments().drop_back())
1580 payloadOpOperands.push_back(Elt: arg);
1581 } else {
1582 payloadOpOperands = {block.getArguments().begin(),
1583 block.getArguments().end() - int(!mapInit)};
1584 }
1585
1586 Operation *payloadOp = b.create(
1587 loc: result.location, opName: b.getStringAttr(bytes: payloadOpName.getStringRef()),
1588 operands: payloadOpOperands,
1589 types: TypeRange{llvm::cast<ShapedType>(Val: result.operands.back().getType())
1590 .getElementType()},
1591 attributes: payloadOpAttrs);
1592 YieldOp::create(builder&: b, location: result.location, values: payloadOp->getResults());
1593}
1594
1595ParseResult MapOp::parse(OpAsmParser &parser, OperationState &result) {
1596 std::optional<OperationName> payloadOpName;
1597 NamedAttrList payloadOpAttrs;
1598 if (succeeded(Result: parser.parseOptionalLBrace())) {
1599 FailureOr<OperationName> operationName = parser.parseCustomOperationName();
1600 if (failed(Result: operationName))
1601 return failure();
1602 if (parser.parseOptionalAttrDict(result&: payloadOpAttrs))
1603 return failure();
1604 payloadOpName = operationName.value();
1605 if (parser.parseRBrace())
1606 return failure();
1607 }
1608
1609 if (parseDstStyleOp(parser, result))
1610 return failure();
1611
1612 if (payloadOpName.has_value()) {
1613 if (!result.operands.empty())
1614 addBodyWithPayloadOp(parser, result, payloadOpName: payloadOpName.value(),
1615 payloadOpAttrs, operands: ArrayRef(result.operands), initFirst: false,
1616 mapInit: false);
1617 else
1618 result.addRegion();
1619 } else {
1620 SmallVector<OpAsmParser::Argument> regionArgs;
1621 if (parser.parseArgumentList(result&: regionArgs, delimiter: OpAsmParser::Delimiter::Paren,
1622 /*allowType=*/true, /*allowAttrs=*/true)) {
1623 return failure();
1624 }
1625 Region *body = result.addRegion();
1626 if (parser.parseRegion(region&: *body, arguments: regionArgs))
1627 return failure();
1628 }
1629 return success();
1630}
1631
1632static bool canUseShortForm(Block *body, bool initFirst = false,
1633 bool mapInit = true) {
1634 // `intFirst == true` implies that we want to map init arg
1635 if (initFirst && !mapInit)
1636 return false;
1637 // Check if the body can be printed in short form. The following 4 conditions
1638 // must be satisfied:
1639
1640 // 1) The body must contain exactly 2 operations: the payload op and a yield.
1641 if (body->getOperations().size() != 2)
1642 return false;
1643 Operation &payload = body->getOperations().front();
1644
1645 // 2) The payload op must have the same number of operands as the number of
1646 // block arguments.
1647 if (payload.getNumOperands() == 0 ||
1648 payload.getNumOperands() != body->getNumArguments() - int(!mapInit))
1649 return false;
1650
1651 // 3) If `initFirst` is true (e.g., for reduction ops), the init block
1652 // must be the first operand of the payload op, otherwise, the operands
1653 // must match the block arguments in order.
1654 if (initFirst) {
1655 // check init
1656 if (payload.getOperands().back() != body->getArgument(i: 0))
1657 return false;
1658 // check rest
1659 for (const auto &[operand, bbArg] :
1660 llvm::zip(t: payload.getOperands(), u: body->getArguments().drop_front())) {
1661 if (bbArg != operand)
1662 return false;
1663 }
1664 } else {
1665 for (const auto &[operand, bbArg] :
1666 llvm::zip(t: payload.getOperands(),
1667 u: body->getArguments().drop_back(N: int(!mapInit)))) {
1668 if (bbArg != operand)
1669 return false;
1670 }
1671 }
1672
1673 // 4) The `yield` operand must be the result of the payload op.
1674 auto yieldOp = cast<YieldOp>(Val: body->getTerminator());
1675 return yieldOp.getNumOperands() == 1 &&
1676 yieldOp.getOperand(i: 0).getDefiningOp() &&
1677 yieldOp.getOperand(i: 0).getDefiningOp() == &payload;
1678}
1679
1680static void printShortForm(OpAsmPrinter &p, Operation *payloadOp) {
1681 SmallVector<StringRef> elidedAttrs;
1682 std::string attrToElide;
1683 NamedAttrList attrs(payloadOp->getDiscardableAttrDictionary());
1684 payloadOp->getName().walkInherentAttrs(
1685 op: payloadOp,
1686 visitor: [&](StringRef name, Attribute &attr) { attrs.append(name, attr); });
1687 p << " { " << payloadOp->getName().getStringRef();
1688 for (const auto &attr : attrs) {
1689 auto fastAttr =
1690 llvm::dyn_cast<mlir::arith::FastMathFlagsAttr>(Val: attr.getValue());
1691 if (fastAttr && fastAttr.getValue() == mlir::arith::FastMathFlags::none) {
1692 attrToElide = attr.getName().str();
1693 elidedAttrs.push_back(Elt: attrToElide);
1694 break;
1695 }
1696 }
1697 p.printOptionalAttrDict(attrs, elidedAttrs);
1698 p << " }";
1699}
1700
1701void MapOp::print(OpAsmPrinter &p) {
1702 Block *mapper = getBody();
1703 bool useShortForm =
1704 canUseShortForm(body: mapper, /*initFirst=*/false, /*mapInit*/ false);
1705 if (useShortForm) {
1706 printShortForm(p, payloadOp: &mapper->getOperations().front());
1707 }
1708
1709 printCommonStructuredOpParts(p, inputs: getDpsInputs(), outputs: getDpsInits());
1710 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary());
1711
1712 if (!useShortForm) {
1713 // Print region if the payload op was not detected.
1714 p.increaseIndent();
1715 p.printNewline();
1716 p << "(";
1717 llvm::interleaveComma(c: mapper->getArguments(), os&: p,
1718 each_fn: [&](auto arg) { p.printRegionArgument(arg); });
1719 p << ") ";
1720
1721 p.printRegion(blocks&: getMapper(), /*printEntryBlockArgs=*/false);
1722 p.decreaseIndent();
1723 }
1724}
1725
1726LogicalResult MapOp::verify() {
1727 auto *bodyBlock = getBody();
1728 auto blockArgs = bodyBlock->getArguments();
1729
1730 // Checks if the number of `inputs` + `init` match the arity of the `mapper`
1731 // region.
1732 if (getInputs().size() + 1 != blockArgs.size())
1733 return emitOpError() << "expects number of operands to match the arity of "
1734 "mapper, but got: "
1735 << getInputs().size() + 1 << " and "
1736 << blockArgs.size();
1737
1738 // The parameters of mapper should all match the element type of inputs.
1739 for (const auto &[bbArgType, inputArg] :
1740 llvm::zip(t: bodyBlock->getArgumentTypes(), u: getInputs())) {
1741 auto inputElemType =
1742 llvm::cast<ShapedType>(Val: inputArg.getType()).getElementType();
1743 if (bbArgType != inputElemType) {
1744 return emitOpError() << "expected element type of input " << inputElemType
1745 << " to match bbArg type " << bbArgType;
1746 }
1747 }
1748
1749 // The shape of each input must match the shape of the output.
1750 auto outputShape = getInit().getType().getShape();
1751 for (Type inputArgType : TypeRange{getInputs()}) {
1752 auto inputElemShape = llvm::cast<ShapedType>(Val&: inputArgType).getShape();
1753 if (inputElemShape != outputShape) {
1754 return emitOpError() << "expected shape of input (" << inputElemShape
1755 << ") to match shape of output (" << outputShape
1756 << ")";
1757 }
1758 }
1759
1760 return success();
1761}
1762
1763SmallVector<utils::IteratorType> MapOp::getIteratorTypesArray() {
1764 int64_t rank = getInit().getType().getRank();
1765 return SmallVector<utils::IteratorType>(rank, utils::IteratorType::parallel);
1766}
1767
1768ArrayAttr MapOp::getIndexingMaps() {
1769 Builder builder(getContext());
1770 int64_t rank = getInit().getType().getRank();
1771 int64_t numIndexingMaps = getOperands().size();
1772 return builder.getAffineMapArrayAttr(values: SmallVector<AffineMap>(
1773 numIndexingMaps, builder.getMultiDimIdentityMap(rank)));
1774}
1775
1776void MapOp::getEffects(
1777 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1778 &effects) {
1779 getGenericEffectsImpl(effects, linalgOp: cast<LinalgOp>(Val: getOperation()));
1780}
1781
1782Speculation::Speculatability MapOp::getSpeculatability() {
1783 return getGenericSpeculatabilityImpl(linalgOp: cast<LinalgOp>(Val: getOperation()));
1784}
1785
1786//===----------------------------------------------------------------------===//
1787// ReduceOp
1788//===----------------------------------------------------------------------===//
1789
1790void ReduceOp::getAsmBlockArgumentNames(Region &region,
1791 OpAsmSetValueNameFn setNameFn) {
1792 for (Value v : getRegionInputArgs())
1793 setNameFn(v, "in");
1794 for (Value v : getRegionOutputArgs())
1795 setNameFn(v, "init");
1796}
1797
1798void ReduceOp::getAsmResultNames(
1799 function_ref<void(Value, StringRef)> setNameFn) {
1800 if (!getResults().empty())
1801 setNameFn(getResults().front(), "reduced");
1802}
1803
1804void ReduceOp::build(
1805 OpBuilder &builder, OperationState &result, ValueRange inputs,
1806 ValueRange inits, ArrayRef<int64_t> dimensions,
1807 function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuild,
1808 ArrayRef<NamedAttribute> attributes) {
1809 build(odsBuilder&: builder, odsState&: result, resultType0: TypeRange{}, inputs, inits, dimensions);
1810 result.addAttributes(newAttributes: attributes);
1811
1812 // Add output types for `RankedTensorType` output arguments.
1813 for (Value init : inits) {
1814 Type initType = init.getType();
1815 if (llvm::isa<RankedTensorType>(Val: initType))
1816 result.addTypes(newTypes: initType);
1817 }
1818
1819 if (bodyBuild)
1820 buildGenericRegion(builder, loc: result.location, region&: *result.regions.front(),
1821 inputs, outputs: inits, bodyBuild);
1822}
1823
1824SmallVector<utils::IteratorType> ReduceOp::getIteratorTypesArray() {
1825 int64_t inputRank =
1826 llvm::cast<ShapedType>(Val: getInputs()[0].getType()).getRank();
1827 SmallVector<utils::IteratorType> iteratorTypes(inputRank,
1828 utils::IteratorType::parallel);
1829 for (int64_t reductionDim : getDimensions())
1830 iteratorTypes[reductionDim] = utils::IteratorType::reduction;
1831 return iteratorTypes;
1832}
1833
1834ArrayAttr ReduceOp::getIndexingMaps() {
1835 int64_t inputRank =
1836 llvm::cast<ShapedType>(Val: getInputs()[0].getType()).getRank();
1837 SmallVector<AffineMap> affineMaps(
1838 getNumDpsInputs(),
1839 AffineMap::getMultiDimIdentityMap(numDims: inputRank, context: getContext()));
1840 AffineMap resultMap =
1841 AffineMap::getMultiDimIdentityMap(numDims: inputRank, context: getContext())
1842 .dropResults(positions: getDimensions());
1843 for (int64_t i = 0, e = getNumDpsInits(); i < e; ++i)
1844 affineMaps.push_back(Elt: resultMap);
1845 return Builder(getContext()).getAffineMapArrayAttr(values: affineMaps);
1846}
1847
1848void ReduceOp::getEffects(
1849 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
1850 &effects) {
1851 getGenericEffectsImpl(effects, linalgOp: cast<LinalgOp>(Val: getOperation()));
1852}
1853
1854Speculation::Speculatability ReduceOp::getSpeculatability() {
1855 return getGenericSpeculatabilityImpl(linalgOp: cast<LinalgOp>(Val: getOperation()));
1856}
1857
1858static ParseResult parseDenseI64ArrayAttr(OpAsmParser &parser,
1859 NamedAttrList &attributes,
1860 StringRef attributeName) {
1861 if (parser.parseKeyword(keyword: attributeName) || parser.parseEqual())
1862 return failure();
1863
1864 attributes.set(name: attributeName, value: DenseI64ArrayAttr::parse(parser, type: Type{}));
1865 return success();
1866}
1867
1868ParseResult ReduceOp::parse(OpAsmParser &parser, OperationState &result) {
1869 std::optional<OperationName> payloadOpName;
1870 NamedAttrList payloadOpAttrs;
1871 if (succeeded(Result: parser.parseOptionalLBrace())) {
1872 FailureOr<OperationName> operationName = parser.parseCustomOperationName();
1873 if (failed(Result: operationName))
1874 return failure();
1875 if (parser.parseOptionalAttrDict(result&: payloadOpAttrs))
1876 return failure();
1877 payloadOpName = operationName.value();
1878 if (parser.parseRBrace())
1879 return failure();
1880 }
1881
1882 if (parseDstStyleOp(
1883 parser, result, parseAttrsFn: [&](OpAsmParser &parser, NamedAttrList &attributes) {
1884 return parseDenseI64ArrayAttr(parser, attributes, attributeName: "dimensions");
1885 }))
1886 return failure();
1887
1888 if (payloadOpName.has_value()) {
1889 addBodyWithPayloadOp(parser, result, payloadOpName: payloadOpName.value(), payloadOpAttrs,
1890 operands: ArrayRef(result.operands), /*initFirst=*/true);
1891 } else {
1892 SmallVector<OpAsmParser::Argument> regionArgs;
1893 if (parser.parseArgumentList(result&: regionArgs, delimiter: OpAsmParser::Delimiter::Paren,
1894 /*allowType=*/true, /*allowAttrs=*/true)) {
1895 return failure();
1896 }
1897
1898 Region *body = result.addRegion();
1899 if (parser.parseRegion(region&: *body, arguments: regionArgs))
1900 return failure();
1901 }
1902
1903 return success();
1904}
1905
1906static void printDenseI64ArrayAttr(OpAsmPrinter &p, StringRef attributeName,
1907 ArrayRef<int64_t> attributeValue) {
1908 p << ' ' << attributeName << " = [" << attributeValue << "] ";
1909}
1910
1911void ReduceOp::print(OpAsmPrinter &p) {
1912 Block *mapper = getBody();
1913 bool useShortForm = canUseShortForm(body: mapper, /*initFirst=*/true);
1914 if (useShortForm) {
1915 printShortForm(p, payloadOp: &mapper->getOperations().front());
1916 }
1917
1918 printCommonStructuredOpParts(p, inputs: getDpsInputs(), outputs: getDpsInits());
1919 printDenseI64ArrayAttr(p, attributeName: getDimensionsAttrName(), attributeValue: getDimensions());
1920 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary(),
1921 elidedAttrs: {getDimensionsAttrName()});
1922 if (!useShortForm) {
1923 // Print region if the payload op was not detected.
1924 p.increaseIndent();
1925 p.printNewline();
1926 p << "(";
1927 llvm::interleaveComma(c: mapper->getArguments(), os&: p,
1928 each_fn: [&](auto arg) { p.printRegionArgument(arg); });
1929 p << ") ";
1930
1931 p.printRegion(blocks&: getCombiner(), /*printEntryBlockArgs=*/false);
1932 p.decreaseIndent();
1933 }
1934}
1935
1936LogicalResult ReduceOp::verify() {
1937 ArrayRef<int64_t> dimensionsRef = getDimensions();
1938
1939 // The ReduceOp uses `SameVariadicOperandSize`, which requires equal numbers
1940 // of inputs and inits. Detect a mismatch early: when they differ, the
1941 // ODS-generated getInputs()/getInits() accessors compute each group's size
1942 // via floordiv of the total operand count, producing incorrect slices that
1943 // would cause out-of-bounds accesses below.
1944 if (getInputs().size() != static_cast<size_t>(getNumDpsInputs()))
1945 return emitOpError()
1946 << "expected equal number of inputs and outputs (required by "
1947 "SameVariadicOperandSize), got "
1948 << getNumDpsInputs() << " input(s) and " << getNumDpsInits()
1949 << " output(s)";
1950
1951 if (getInputs().empty())
1952 return emitOpError() << "expected at least one input";
1953
1954 for (int64_t i = 1; i < getNumDpsInputs(); ++i) {
1955 if (llvm::cast<ShapedType>(Val: getInputs()[i].getType()).getShape() !=
1956 llvm::cast<ShapedType>(Val: getInputs()[0].getType()).getShape()) {
1957 return emitOpError() << "expects all inputs to have the same shapes. "
1958 "Shape at input-index "
1959 << i
1960 << " is not equal to the shape at input-index 0.";
1961 }
1962 }
1963 for (int64_t i = 1; i < getNumDpsInits(); ++i) {
1964 if (llvm::cast<ShapedType>(Val: getInits()[i].getType()).getShape() !=
1965 llvm::cast<ShapedType>(Val: getInits()[0].getType()).getShape()) {
1966 return emitOpError() << "expects all outputs to have the same shapes. "
1967 "Shape at output-index "
1968 << i
1969 << " is not equal to the shape at output-index 0.";
1970 }
1971 }
1972 auto inputType = llvm::cast<ShapedType>(Val: getInputs()[0].getType());
1973 auto initType = llvm::cast<ShapedType>(Val: getInits()[0].getType());
1974
1975 DenseSet<int64_t> dimensionsToReduce;
1976 for (int64_t dimension : dimensionsRef) {
1977 if (dimension < 0 || dimension >= inputType.getRank()) {
1978 return emitOpError()
1979 << "dimensions for reduction should be in the range [0, "
1980 << inputType.getRank() - 1 << "].";
1981 }
1982 dimensionsToReduce.insert(V: dimension);
1983 }
1984
1985 auto inputDims = inputType.getShape();
1986 auto initDims = initType.getShape();
1987
1988 // Input dimensions that will be left after the reduction.
1989 SmallVector<int64_t> reducedInputDims;
1990 for (const auto &en : llvm::enumerate(First&: inputDims)) {
1991 if (!dimensionsToReduce.count(V: en.index()))
1992 reducedInputDims.push_back(Elt: en.value());
1993 }
1994
1995 if (reducedInputDims.size() != static_cast<size_t>(initType.getRank())) {
1996 return emitOpError() << "number of dimensions after reduction "
1997 << reducedInputDims.size()
1998 << " doesn't match the init rank "
1999 << initType.getRank();
2000 }
2001
2002 if (reducedInputDims != initDims)
2003 return emitOpError() << "init dimensions [" << initDims
2004 << "] doesn't match input dimensions after reduction ["
2005 << reducedInputDims << "]";
2006
2007 Block *block = getBody();
2008 if (block->getNumArguments() != this->getNumOperands())
2009 return emitOpError()
2010 << "mismatching number of operands and block arguments";
2011
2012 // Check that the first block arguments match the element type of the inputs.
2013 for (auto [input, bbArg] : llvm::zip(t: getInputs(), u: block->getArguments())) {
2014 Type inputElementType =
2015 llvm::cast<ShapedType>(Val: input.getType()).getElementType();
2016 if (inputElementType != bbArg.getType())
2017 return emitOpError()
2018 << "input element type " << inputElementType
2019 << " does not match corresponding block argument type "
2020 << bbArg.getType();
2021 }
2022
2023 // Check that the last block arguments match the element type of the outputs.
2024 for (auto [output, bbArg] : llvm::zip(
2025 t: getDpsInits(), u: block->getArguments().take_back(N: getNumDpsInits()))) {
2026 auto outputElementType =
2027 llvm::cast<ShapedType>(Val: output.getType()).getElementType();
2028 if (outputElementType != bbArg.getType())
2029 return emitOpError()
2030 << "output element type " << outputElementType
2031 << " does not match corresponding block argument type "
2032 << bbArg.getType();
2033 }
2034 return success();
2035}
2036
2037namespace {
2038
2039/// Reduction kinds supported by the reduce-of-broadcast fold.
2040/// Only the simplest reduce op are supported now.
2041/// TODO: We can extend the list in the future.
2042enum class BroadcastReduceKind {
2043 MaxSI,
2044 MaxUI,
2045 MinSI,
2046 MinUI,
2047 AndI,
2048 OrI,
2049 MaximumF,
2050 MinimumF,
2051};
2052
2053/// Match a supported reduction body and return its reduction kind.
2054static std::optional<BroadcastReduceKind>
2055matchBroadcastReduceBody(ReduceOp reduceOp) {
2056 if (reduceOp.getNumDpsInputs() != 1 || reduceOp.getNumDpsInits() != 1 ||
2057 !reduceOp.getBody())
2058 return std::nullopt;
2059
2060 // Match the simplest linalg.reduce body. e.g.,
2061 //
2062 // ^bb0(%in: i32, %acc: i32):
2063 // %max = arith.maxsi %in, %acc : i32
2064 // linalg.yield %max : i32
2065 Block &block = *reduceOp.getBody();
2066 if (block.getNumArguments() != 2 ||
2067 !llvm::hasSingleElement(C: block.without_terminator()))
2068 return std::nullopt;
2069
2070 auto yieldOp = cast<YieldOp>(Val: block.getTerminator());
2071
2072 Operation *combineOp = yieldOp.getOperand(i: 0).getDefiningOp();
2073 if (!combineOp || combineOp->getNumOperands() != 2)
2074 return std::nullopt;
2075
2076 // Checks that the combine op **only** used the block arguments and
2077 // we allow the block arguments to exchange their orders.
2078 if (!((combineOp->getOperand(idx: 0) == block.getArgument(i: 0) &&
2079 combineOp->getOperand(idx: 1) == block.getArgument(i: 1)) ||
2080 (combineOp->getOperand(idx: 0) == block.getArgument(i: 1) &&
2081 combineOp->getOperand(idx: 1) == block.getArgument(i: 0))))
2082 return std::nullopt;
2083
2084 // TODO: We can extend the list here.
2085 return TypeSwitch<Operation *, std::optional<BroadcastReduceKind>>(combineOp)
2086 .Case<arith::MaxSIOp>(
2087 caseFn: [](arith::MaxSIOp) { return BroadcastReduceKind::MaxSI; })
2088 .Case<arith::MaxUIOp>(
2089 caseFn: [](arith::MaxUIOp) { return BroadcastReduceKind::MaxUI; })
2090 .Case<arith::MinSIOp>(
2091 caseFn: [](arith::MinSIOp) { return BroadcastReduceKind::MinSI; })
2092 .Case<arith::MinUIOp>(
2093 caseFn: [](arith::MinUIOp) { return BroadcastReduceKind::MinUI; })
2094 .Case<arith::AndIOp>(
2095 caseFn: [](arith::AndIOp) { return BroadcastReduceKind::AndI; })
2096 .Case<arith::OrIOp>(caseFn: [](arith::OrIOp) { return BroadcastReduceKind::OrI; })
2097 .Case<arith::MaximumFOp>(
2098 caseFn: [](arith::MaximumFOp) { return BroadcastReduceKind::MaximumF; })
2099 .Case<arith::MinimumFOp>(
2100 caseFn: [](arith::MinimumFOp) { return BroadcastReduceKind::MinimumF; })
2101 .Default(defaultFn: [](Operation *) -> std::optional<BroadcastReduceKind> {
2102 return std::nullopt;
2103 });
2104}
2105
2106/// Return whether `init` is the identity value for `kind`.
2107static bool hasBroadcastReduceIdentity(Value init, BroadcastReduceKind kind) {
2108 auto initAttr = getScalarConstantAttrFromDenseSplat(input: init);
2109 if (!initAttr)
2110 return false;
2111
2112 // Float kinds: identity is the corresponding infinity.
2113 if (kind == BroadcastReduceKind::MaximumF ||
2114 kind == BroadcastReduceKind::MinimumF) {
2115 auto floatAttr = dyn_cast<FloatAttr>(Val&: *initAttr);
2116 if (!floatAttr)
2117 return false;
2118 const APFloat &value = floatAttr.getValue();
2119 return kind == BroadcastReduceKind::MaximumF ? value.isNegInfinity()
2120 : value.isPosInfinity();
2121 }
2122
2123 auto integerAttr = dyn_cast<IntegerAttr>(Val&: *initAttr);
2124 if (!integerAttr)
2125 return false;
2126
2127 const APInt &value = integerAttr.getValue();
2128 switch (kind) {
2129 case BroadcastReduceKind::MaxSI:
2130 return value.isMinSignedValue();
2131 case BroadcastReduceKind::MaxUI:
2132 return value.isZero();
2133 case BroadcastReduceKind::MinSI:
2134 return value.isMaxSignedValue();
2135 case BroadcastReduceKind::MinUI:
2136 case BroadcastReduceKind::AndI:
2137 return value.isAllOnes();
2138 case BroadcastReduceKind::OrI:
2139 return value.isZero();
2140 case BroadcastReduceKind::MaximumF:
2141 case BroadcastReduceKind::MinimumF:
2142 llvm_unreachable("float kinds handled by the branch above");
2143 }
2144 llvm_unreachable("unknown broadcast reduction kind");
2145}
2146
2147/// Fold cases like:
2148///
2149/// maxsi(broadcast(x)) -> x
2150/// minsi(broadcast(y)) -> y
2151/// andi(broadcast(z)) -> z
2152/// ori(broadcast(w)) -> w
2153/// maximumf(broadcast(x)) -> x
2154/// minimumf(broadcast(y)) -> y
2155///
2156/// provided the reduction init is the identity value of the combiner.
2157struct FoldReduceBroadcast : public OpRewritePattern<linalg::ReduceOp> {
2158 using OpRewritePattern<linalg::ReduceOp>::OpRewritePattern;
2159
2160 LogicalResult matchAndRewrite(linalg::ReduceOp reduceOp,
2161 PatternRewriter &rewriter) const override {
2162 if (reduceOp.getNumResults() != 1 || !reduceOp.hasPureTensorSemantics())
2163 return failure();
2164
2165 assert(reduceOp.getInputs().size() == 1 &&
2166 "expected one input for a single-result tensor reduce");
2167
2168 auto broadcastOp =
2169 reduceOp.getInputs().front().getDefiningOp<linalg::BroadcastOp>();
2170 if (!broadcastOp || !broadcastOp.hasPureTensorSemantics())
2171 return failure();
2172
2173 auto sourceType = cast<RankedTensorType>(Val: broadcastOp.getInput().getType());
2174 auto broadcastType =
2175 cast<RankedTensorType>(Val: broadcastOp.getResult().front().getType());
2176 auto resultType = cast<RankedTensorType>(Val: reduceOp.getResult(i: 0).getType());
2177 if (!sourceType.hasStaticShape() || !broadcastType.hasStaticShape() ||
2178 !resultType.hasStaticShape() || sourceType != resultType)
2179 return failure();
2180
2181 ArrayRef<int64_t> broadcastDims = broadcastOp.getDimensions();
2182 ArrayRef<int64_t> reduceDims = reduceOp.getDimensions();
2183 if (broadcastDims != reduceDims)
2184 return failure();
2185
2186 // Reducing an empty broadcast dimension yields the init value, not the
2187 // broadcast input.
2188 for (int64_t dimension : broadcastDims)
2189 if (broadcastType.getDimSize(idx: dimension) == 0)
2190 return failure();
2191
2192 std::optional<BroadcastReduceKind> kind =
2193 matchBroadcastReduceBody(reduceOp);
2194 if (!kind ||
2195 !hasBroadcastReduceIdentity(init: reduceOp.getInits().front(), kind: *kind))
2196 return failure();
2197
2198 rewriter.replaceOp(op: reduceOp, newValues: broadcastOp.getInput());
2199 return success();
2200 }
2201};
2202
2203} // namespace
2204
2205void ReduceOp::getCanonicalizationPatterns(RewritePatternSet &results,
2206 MLIRContext *context) {
2207 results.add<FoldReduceBroadcast>(arg&: context);
2208}
2209
2210//===----------------------------------------------------------------------===//
2211// TransposeOp
2212//===----------------------------------------------------------------------===//
2213
2214static void buildIdentityRegion(OpBuilder &builder, Location loc,
2215 Region &region, ValueRange inputs,
2216 ValueRange outputs) {
2217 buildGenericRegion(builder, loc, region, inputs, outputs,
2218 bodyBuild: [](OpBuilder &b, Location loc, ValueRange args) {
2219 if (!args.empty())
2220 linalg::YieldOp::create(builder&: b, location: loc, values: args[0]);
2221 });
2222}
2223
2224void TransposeOp::build(::mlir::OpBuilder &builder,
2225 ::mlir::OperationState &result, Value input, Value init,
2226 DenseI64ArrayAttr permutation,
2227 ArrayRef<NamedAttribute> attributes) {
2228 result.addOperands(newOperands: input);
2229 result.addOperands(newOperands: init);
2230 result.addAttribute(name: getPermutationAttrName(name: result.name), attr: permutation);
2231 result.addAttributes(newAttributes: attributes);
2232
2233 // Add output types for `RankedTensorType` output arguments.
2234 Type initType = init.getType();
2235 if (llvm::isa<RankedTensorType>(Val: initType))
2236 result.addTypes(newTypes: initType);
2237
2238 buildIdentityRegion(builder, loc: result.location, region&: *result.addRegion(), inputs: input,
2239 outputs: init);
2240}
2241
2242void TransposeOp::build(::mlir::OpBuilder &builder,
2243 ::mlir::OperationState &result, Value input, Value init,
2244 ArrayRef<int64_t> permutation,
2245 ArrayRef<NamedAttribute> attributes) {
2246 build(builder, result, input, init, permutation: builder.getDenseI64ArrayAttr(values: permutation),
2247 attributes);
2248}
2249
2250ParseResult TransposeOp::parse(OpAsmParser &parser, OperationState &result) {
2251 if (failed(Result: parseDstStyleOp(
2252 parser, result, parseAttrsFn: [&](OpAsmParser &parser, NamedAttrList &attributes) {
2253 return parseDenseI64ArrayAttr(parser, attributes, attributeName: "permutation");
2254 })))
2255 return failure();
2256
2257 OpBuilder builder(parser.getContext());
2258 buildIdentityRegion(builder, loc: result.location, region&: *result.addRegion(),
2259 /*inputs=*/result.operands,
2260 /*outputs=*/{});
2261 return success();
2262}
2263
2264void TransposeOp::getAsmResultNames(
2265 function_ref<void(Value, StringRef)> setNameFn) {
2266 if (!getResults().empty())
2267 setNameFn(getResults().front(), "transposed");
2268}
2269
2270void TransposeOp::print(OpAsmPrinter &p) {
2271 printCommonStructuredOpParts(p, inputs: getDpsInputs(), outputs: getDpsInits());
2272 printDenseI64ArrayAttr(p, attributeName: getPermutationAttrName(), attributeValue: getPermutation());
2273 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary(),
2274 elidedAttrs: {getPermutationAttrName()});
2275}
2276
2277LogicalResult TransposeOp::verify() {
2278 ArrayRef<int64_t> permutationRef = getPermutation();
2279
2280 if (!isPermutationVector(interchange: permutationRef))
2281 return emitOpError(message: "permutation is not valid");
2282
2283 auto inputType = getInput().getType();
2284 auto initType = getInit().getType();
2285
2286 int64_t rank = inputType.getRank();
2287
2288 if (failed(Result: verifyRanksMatch(op: getOperation(), lhs: inputType, rhs: initType, lhsName: "input",
2289 rhsName: "init")))
2290 return failure();
2291
2292 if (rank != static_cast<int64_t>(permutationRef.size()))
2293 return emitOpError() << "size of permutation " << permutationRef.size()
2294 << " does not match the argument rank " << rank;
2295
2296 auto inputDims = inputType.getShape();
2297 auto initDims = initType.getShape();
2298
2299 for (int64_t i = 0; i < rank; ++i) {
2300 int64_t inputDim = inputDims[permutationRef[i]];
2301 int64_t initDim = initDims[i];
2302
2303 if (inputDim != initDim) {
2304 return emitOpError() << "dim(result, " << i << ") = " << initDim
2305 << " doesn't match dim(input, permutation[" << i
2306 << "]) = " << inputDim;
2307 }
2308 }
2309
2310 return success();
2311}
2312
2313SmallVector<utils::IteratorType> TransposeOp::getIteratorTypesArray() {
2314 int64_t rank = getInit().getType().getRank();
2315 return SmallVector<utils::IteratorType>(rank, utils::IteratorType::parallel);
2316}
2317
2318ArrayAttr TransposeOp::getIndexingMaps() {
2319 Builder builder(getContext());
2320 int64_t rank = getInit().getType().getRank();
2321 return builder.getAffineMapArrayAttr(
2322 values: {inversePermutation(map: AffineMap::getPermutationMap(
2323 permutation: llvm::to_vector_of<unsigned>(Range: getPermutation()), context: getContext())),
2324 builder.getMultiDimIdentityMap(rank)});
2325}
2326
2327void TransposeOp::getEffects(
2328 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
2329 &effects) {
2330 getGenericEffectsImpl(effects, linalgOp: cast<LinalgOp>(Val: getOperation()));
2331}
2332
2333Speculation::Speculatability TransposeOp::getSpeculatability() {
2334 return getGenericSpeculatabilityImpl(linalgOp: cast<LinalgOp>(Val: getOperation()));
2335}
2336
2337LogicalResult TransposeOp::fold(FoldAdaptor adaptor,
2338 SmallVectorImpl<OpFoldResult> &result) {
2339 // Only the tensor type is supported.
2340 if (!isa<TensorType>(Val: getInput().getType()))
2341 return failure();
2342
2343 // Single dimension transpose.
2344 if (getPermutation().empty()) {
2345 result.push_back(Elt: getInput());
2346 return success();
2347 }
2348 // Identity permutation.
2349 if (isIdentityPermutation(permutation: getPermutation())) {
2350 result.push_back(Elt: getInput());
2351 return success();
2352 }
2353
2354 return failure();
2355}
2356
2357/// Fold transpose with transpose.
2358struct FoldTransposeWithTranspose : OpRewritePattern<linalg::TransposeOp> {
2359 using OpRewritePattern<linalg::TransposeOp>::OpRewritePattern;
2360
2361 LogicalResult matchAndRewrite(linalg::TransposeOp transposeOp,
2362 PatternRewriter &rewriter) const override {
2363 auto defTransposeOp = transposeOp.getInput().getDefiningOp<TransposeOp>();
2364 if (!defTransposeOp)
2365 return failure();
2366 ArrayRef<int64_t> defPerms = defTransposeOp.getPermutation();
2367 ArrayRef<int64_t> perms = transposeOp.getPermutation();
2368 SmallVector<int64_t> foldedPerms;
2369 foldedPerms.reserve(N: perms.size());
2370 for (int64_t perm : perms)
2371 foldedPerms.push_back(Elt: defPerms[perm]);
2372
2373 rewriter.replaceOpWithNewOp<TransposeOp>(
2374 op: transposeOp, args: defTransposeOp.getInput(), args: transposeOp.getInit(),
2375 args&: foldedPerms);
2376 return success();
2377 }
2378};
2379
2380/// Rewrite a transpose of a dense splat constant into a dense splat constant of
2381/// the transposed output shape.
2382struct FoldTransposeSplatConstant : OpRewritePattern<linalg::TransposeOp> {
2383 using OpRewritePattern<linalg::TransposeOp>::OpRewritePattern;
2384
2385 LogicalResult matchAndRewrite(linalg::TransposeOp transposeOp,
2386 PatternRewriter &rewriter) const override {
2387 if (!transposeOp.hasPureTensorSemantics())
2388 return failure();
2389
2390 auto splatValue =
2391 getScalarConstantAttrFromDenseSplat(input: transposeOp.getInput());
2392 if (!splatValue.has_value())
2393 return failure();
2394
2395 auto resultType =
2396 cast<RankedTensorType>(Val: transposeOp.getResult()[0].getType());
2397
2398 auto resultAttr = DenseElementsAttr::get(type: resultType, values: splatValue.value());
2399 rewriter.replaceOpWithNewOp<arith::ConstantOp>(op: transposeOp, args&: resultType,
2400 args&: resultAttr);
2401 return success();
2402 }
2403};
2404
2405/// This pattern canonicalize transpose by swapping the order of
2406/// broadcast and transpose:
2407/// transpose(broadcast(input)) -> broadcast(transpose(input))
2408struct SwapTransposeWithBroadcast : OpRewritePattern<linalg::TransposeOp> {
2409 using OpRewritePattern<linalg::TransposeOp>::OpRewritePattern;
2410
2411 LogicalResult matchAndRewrite(linalg::TransposeOp transposeOp,
2412 PatternRewriter &rewriter) const override {
2413 Value input = transposeOp.getInput();
2414 BroadcastOp broadcastOp = input.getDefiningOp<BroadcastOp>();
2415 if (!input.hasOneUse() || !broadcastOp)
2416 return failure();
2417
2418 ArrayRef<int64_t> dimensions = broadcastOp.getDimensions();
2419 ArrayRef<int64_t> perms = transposeOp.getPermutation();
2420
2421 // Get new perms and new dimensions.
2422 SmallVector<int64_t> resultPerms = dropDims(inputPerm: perms, dropPositions: dimensions);
2423 SmallVector<int64_t> invertPerm = invertPermutationVector(permutation: perms);
2424 SmallVector<int64_t> resultDimensions;
2425 unsigned dimensionSize = dimensions.size();
2426 for (unsigned i = 0; i < dimensionSize; ++i)
2427 resultDimensions.push_back(Elt: invertPerm[dimensions[i]]);
2428
2429 // Create transpose result.
2430 Value broadcastInput = broadcastOp.getInput();
2431 Location loc = transposeOp.getLoc();
2432 MLIRContext *ctx = transposeOp.getContext();
2433 SmallVector<OpFoldResult> dims;
2434 auto broadcastInputTy =
2435 mlir::cast<RankedTensorType>(Val: broadcastInput.getType());
2436 unsigned inputRank = broadcastInputTy.getRank();
2437 for (unsigned i = 0; i < inputRank; ++i) {
2438 if (broadcastInputTy.isDynamicDim(idx: i)) {
2439 dims.push_back(Elt: tensor::DimOp::create(builder&: rewriter, location: loc, source: broadcastInput, index: i)
2440 ->getResult(idx: 0));
2441 } else {
2442 dims.push_back(Elt: IntegerAttr::get(type: IndexType::get(context: ctx),
2443 value: broadcastInputTy.getDimSize(idx: i)));
2444 }
2445 }
2446 SmallVector<OpFoldResult> transposeResultShapes =
2447 applyPermutation(input: dims, permutation: resultPerms);
2448 Value transposeInit = tensor::EmptyOp::create(
2449 builder&: rewriter, location: transposeOp.getLoc(), sizes: transposeResultShapes,
2450 elementType: broadcastInputTy.getElementType());
2451
2452 // Create broadcast(transpose(input)).
2453 Value transposeResult =
2454 TransposeOp::create(builder&: rewriter, location: loc, input: broadcastOp.getInput(),
2455 init: transposeInit, permutation: resultPerms)
2456 ->getResult(idx: 0);
2457 rewriter.replaceOpWithNewOp<BroadcastOp>(
2458 op: transposeOp, args&: transposeResult, args: transposeOp.getInit(), args&: resultDimensions);
2459 return success();
2460 }
2461};
2462
2463void TransposeOp::getCanonicalizationPatterns(RewritePatternSet &results,
2464 MLIRContext *context) {
2465 results.add<FoldTransposeWithTranspose, FoldTransposeSplatConstant,
2466 SwapTransposeWithBroadcast>(arg&: context);
2467}
2468
2469//===----------------------------------------------------------------------===//
2470// BroadcastOp
2471//===----------------------------------------------------------------------===//
2472
2473void BroadcastOp::build(::mlir::OpBuilder &builder,
2474 ::mlir::OperationState &result, Value input, Value init,
2475 DenseI64ArrayAttr dimensions,
2476 ArrayRef<NamedAttribute> attributes) {
2477 result.addOperands(newOperands: input);
2478 result.addOperands(newOperands: init);
2479 result.addAttribute(name: getDimensionsAttrName(name: result.name), attr: dimensions);
2480 result.addAttributes(newAttributes: attributes);
2481
2482 // Add output types for `RankedTensorType` output arguments.
2483 Type initType = init.getType();
2484 if (llvm::isa<RankedTensorType>(Val: initType))
2485 result.addTypes(newTypes: initType);
2486
2487 buildIdentityRegion(builder, loc: result.location, region&: *result.addRegion(), inputs: input,
2488 outputs: init);
2489}
2490
2491void BroadcastOp::build(::mlir::OpBuilder &builder,
2492 ::mlir::OperationState &result, Value input, Value init,
2493 ArrayRef<int64_t> dimensions,
2494 ArrayRef<NamedAttribute> attributes) {
2495 build(builder, result, input, init, dimensions: builder.getDenseI64ArrayAttr(values: dimensions),
2496 attributes);
2497}
2498
2499ParseResult BroadcastOp::parse(OpAsmParser &parser, OperationState &result) {
2500 if (failed(Result: parseDstStyleOp(
2501 parser, result, parseAttrsFn: [&](OpAsmParser &parser, NamedAttrList &attributes) {
2502 return parseDenseI64ArrayAttr(parser, attributes, attributeName: "dimensions");
2503 })))
2504 return failure();
2505
2506 OpBuilder builder(parser.getContext());
2507 buildIdentityRegion(builder, loc: result.location, region&: *result.addRegion(),
2508 /*inputs=*/result.operands,
2509 /*outputs=*/{});
2510 return success();
2511}
2512
2513void BroadcastOp::getAsmResultNames(
2514 function_ref<void(Value, StringRef)> setNameFn) {
2515 if (!getResults().empty())
2516 setNameFn(getResults().front(), "broadcasted");
2517}
2518
2519void BroadcastOp::print(OpAsmPrinter &p) {
2520 printCommonStructuredOpParts(p, inputs: getDpsInputs(), outputs: getDpsInits());
2521 printDenseI64ArrayAttr(p, attributeName: getDimensionsAttrName(), attributeValue: getDimensions());
2522 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary(),
2523 elidedAttrs: {getDimensionsAttrName()});
2524}
2525
2526LogicalResult BroadcastOp::verify() {
2527 ArrayRef<int64_t> dimensionsRef = getDimensions();
2528
2529 auto inputType = getInput().getType();
2530 auto initType = getInit().getType();
2531
2532 int64_t inputRank = inputType.getRank();
2533 int64_t initRank = initType.getRank();
2534
2535 auto inputShape = inputType.getShape();
2536 auto initShape = initType.getShape();
2537
2538 if ((size_t)inputRank + dimensionsRef.size() != (size_t)initRank)
2539 return emitOpError() << "input rank plus added dimensions does not "
2540 "match init rank. input rank: "
2541 << inputRank
2542 << ", dimensions size: " << dimensionsRef.size()
2543 << ", init rank: " << initRank;
2544
2545 for (const auto &[idx, dim] : llvm::enumerate(First&: dimensionsRef)) {
2546 if (dim < 0 || dim >= initRank)
2547 return emitOpError() << "dimension " << idx
2548 << " is out of range. expected range: [0, "
2549 << initRank - 1 << "], got: " << dim;
2550 }
2551
2552 DenseSet<int64_t> uniquedDims(llvm::from_range, dimensionsRef);
2553 if (uniquedDims.size() != dimensionsRef.size())
2554 return emitOpError() << "dimensions should not contain duplicates";
2555
2556 // Mapping from input dims to init dims.
2557 SmallVector<int64_t> dimMap;
2558 for (auto dim : llvm::seq<int64_t>(Begin: 0, End: initRank)) {
2559 if (!llvm::is_contained(Range&: dimensionsRef, Element: dim))
2560 dimMap.push_back(Elt: dim);
2561 }
2562
2563 for (const auto &[inputDimIdx, initDimIdx] : llvm::enumerate(First&: dimMap)) {
2564 // This dimensions is mapped from the input. Init and input dims should
2565 // match.
2566 if (inputShape[inputDimIdx] != initShape[initDimIdx])
2567 return emitOpError() << "input dim " << inputDimIdx
2568 << " should match init dim " << initDimIdx
2569 << ". input: " << inputShape[inputDimIdx]
2570 << ", init: " << initShape[initDimIdx];
2571 }
2572
2573 return success();
2574}
2575
2576SmallVector<utils::IteratorType> BroadcastOp::getIteratorTypesArray() {
2577 int64_t rank = getInit().getType().getRank();
2578 return SmallVector<utils::IteratorType>(rank, utils::IteratorType::parallel);
2579}
2580
2581ArrayAttr BroadcastOp::getIndexingMaps() {
2582 Builder builder(getContext());
2583 int64_t rank = getInit().getType().getRank();
2584 return builder.getAffineMapArrayAttr(
2585 values: {builder.getMultiDimIdentityMap(rank).dropResults(positions: getDimensions()),
2586 builder.getMultiDimIdentityMap(rank)});
2587}
2588
2589void BroadcastOp::getEffects(
2590 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
2591 &effects) {
2592 getGenericEffectsImpl(effects, linalgOp: cast<LinalgOp>(Val: getOperation()));
2593}
2594
2595Speculation::Speculatability BroadcastOp::getSpeculatability() {
2596 return getGenericSpeculatabilityImpl(linalgOp: cast<LinalgOp>(Val: getOperation()));
2597}
2598
2599/// Fold back-to-back broadcasts together.
2600struct FoldBroadcasts : OpRewritePattern<linalg::BroadcastOp> {
2601 using OpRewritePattern<linalg::BroadcastOp>::OpRewritePattern;
2602
2603 LogicalResult matchAndRewrite(linalg::BroadcastOp broadcastOp,
2604 PatternRewriter &rewriter) const override {
2605 auto defBroadcastOp = broadcastOp.getInput().getDefiningOp<BroadcastOp>();
2606 if (!defBroadcastOp)
2607 return failure();
2608 ArrayRef<int64_t> defDimensions = defBroadcastOp.getDimensions();
2609 ArrayRef<int64_t> dimensions = broadcastOp.getDimensions();
2610 SmallVector<int64_t> foldedDims(dimensions);
2611 Value init = broadcastOp.getInit();
2612 int64_t initRank = cast<ShapedType>(Val: init.getType()).getRank();
2613 // Mapping from input dims to init dims.
2614 SmallVector<int64_t> dimMap;
2615 for (auto dim : llvm::seq<int64_t>(Begin: 0, End: initRank)) {
2616 if (!llvm::is_contained(Range&: dimensions, Element: dim))
2617 dimMap.push_back(Elt: dim);
2618 }
2619 for (auto dim : defDimensions)
2620 foldedDims.push_back(Elt: dimMap[dim]);
2621
2622 llvm::sort(C&: foldedDims);
2623 rewriter.replaceOpWithNewOp<BroadcastOp>(
2624 op: broadcastOp, args: defBroadcastOp.getInput(), args&: init, args&: foldedDims);
2625 return success();
2626 }
2627};
2628
2629/// Rewrite a broadcast of a dense splat constant into a dense splat constant of
2630/// the broadcast output shape.
2631struct FoldBroadcastSplatConstant : OpRewritePattern<linalg::BroadcastOp> {
2632 using OpRewritePattern<linalg::BroadcastOp>::OpRewritePattern;
2633
2634 LogicalResult matchAndRewrite(linalg::BroadcastOp broadcastOp,
2635 PatternRewriter &rewriter) const override {
2636 if (!broadcastOp.hasPureTensorSemantics())
2637 return failure();
2638
2639 auto splatValue =
2640 getScalarConstantAttrFromDenseSplat(input: broadcastOp.getInput());
2641
2642 if (!splatValue.has_value())
2643 return failure();
2644
2645 auto resultType =
2646 cast<RankedTensorType>(Val: broadcastOp.getResult()[0].getType());
2647 if (!resultType.hasStaticShape())
2648 return rewriter.notifyMatchFailure(arg&: broadcastOp,
2649 msg: "result type has dynamic shape");
2650
2651 auto resultAttr = DenseElementsAttr::get(type: resultType, values: splatValue.value());
2652 rewriter.replaceOpWithNewOp<arith::ConstantOp>(op: broadcastOp, args&: resultType,
2653 args&: resultAttr);
2654 return success();
2655 }
2656};
2657
2658void BroadcastOp::getCanonicalizationPatterns(RewritePatternSet &results,
2659 MLIRContext *context) {
2660 results.add<EraseIdentityLinalgOp<BroadcastOp>, FoldBroadcasts,
2661 FoldBroadcastSplatConstant>(arg&: context);
2662}
2663
2664//===----------------------------------------------------------------------===//
2665// YieldOp
2666//===----------------------------------------------------------------------===//
2667
2668void linalg::YieldOp::print(OpAsmPrinter &p) {
2669 if (getNumOperands() > 0)
2670 p << ' ' << getOperands();
2671 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary());
2672 if (getNumOperands() > 0)
2673 p << " : " << getOperandTypes();
2674}
2675
2676ParseResult YieldOp::parse(OpAsmParser &parser, OperationState &result) {
2677 SmallVector<OpAsmParser::UnresolvedOperand, 2> opInfo;
2678 SmallVector<Type, 2> types;
2679 SMLoc loc = parser.getCurrentLocation();
2680 return failure(IsFailure: parser.parseOperandList(result&: opInfo) ||
2681 parser.parseOptionalAttrDict(result&: result.attributes) ||
2682 (!opInfo.empty() && parser.parseColonTypeList(result&: types)) ||
2683 parser.resolveOperands(operands&: opInfo, types, loc, result&: result.operands));
2684}
2685
2686// Check the operand number and types must match the element types of the
2687// LinalgOp interface's shaped operands.
2688static LogicalResult verifyYield(linalg::YieldOp op, LinalgOp linalgOp) {
2689 if (op.getNumOperands() != linalgOp.getNumDpsInits())
2690 return op.emitOpError(message: "expected number of yield values (")
2691 << op.getNumOperands()
2692 << ") to match the number of inits / outs operands of the enclosing "
2693 << "LinalgOp (" << linalgOp.getNumDpsInits() << ")";
2694
2695 for (OpOperand &opOperand : op->getOpOperands()) {
2696 OpOperand *outputOperand =
2697 linalgOp.getDpsInitOperand(i: opOperand.getOperandNumber());
2698 Type elementType = outputOperand->get().getType();
2699 if (isa<MemRefType, RankedTensorType>(Val: elementType))
2700 elementType = getElementTypeOrSelf(type: outputOperand->get().getType());
2701 if (opOperand.get().getType() != elementType)
2702 return op.emitOpError(message: "type of yield operand ")
2703 << (opOperand.getOperandNumber() + 1) << " ("
2704 << opOperand.get().getType() << ") doesn't match "
2705 << "the element type of the enclosing linalg.generic op ("
2706 << elementType << ")";
2707 }
2708 return success();
2709}
2710
2711LogicalResult linalg::YieldOp::verify() {
2712 auto *parentOp = (*this)->getParentOp();
2713 if (parentOp->getNumRegions() != 1 || parentOp->getRegion(index: 0).empty())
2714 return emitOpError(message: "expected single non-empty parent region");
2715
2716 if (auto linalgOp = dyn_cast<LinalgOp>(Val: parentOp))
2717 return verifyYield(op: *this, linalgOp);
2718
2719 return emitOpError(message: "expected parent op with LinalgOp interface");
2720}
2721
2722//===----------------------------------------------------------------------===//
2723// IndexOp
2724//===----------------------------------------------------------------------===//
2725
2726LogicalResult IndexOp::verify() {
2727 auto linalgOp = dyn_cast<LinalgOp>(Val: (*this)->getParentOp());
2728 if (!linalgOp)
2729 return emitOpError(message: "expected parent op with LinalgOp interface");
2730 if (linalgOp.getNumLoops() <= getDim())
2731 return emitOpError(message: "expected dim (")
2732 << getDim() << ") to be lower than the number of loops ("
2733 << linalgOp.getNumLoops() << ") of the enclosing LinalgOp";
2734 return success();
2735}
2736
2737OpFoldResult IndexOp::fold(FoldAdaptor adaptor) {
2738 auto linalgOp = dyn_cast_or_null<LinalgOp>(Val: (*this)->getParentOp());
2739 // Bail out if `linalg.index` does not have a proper parent yet at this
2740 // point, e.g., when calling `createOrFold` during IR construction in
2741 // `genericOp::build`.
2742 if (!linalgOp)
2743 return OpFoldResult{};
2744
2745 // Index of unit dims is always 0.
2746 SmallVector<int64_t, 4> loopBounds = linalgOp.getStaticLoopRanges();
2747 uint64_t dim = getDim();
2748 assert(dim < loopBounds.size() && "Dim is out of bounds");
2749 if (loopBounds[dim] == 1)
2750 return IntegerAttr::get(type: IndexType::get(context: getContext()), value: 0);
2751
2752 return OpFoldResult{};
2753}
2754
2755/////// Operations corresponding to library calls defined with Tablegen ////////
2756
2757#include "mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yamlgen.cpp.inc"
2758
2759#define GET_OP_CLASSES
2760#include "mlir/Dialect/Linalg/IR/LinalgOps.cpp.inc"
2761
2762#define GET_OP_CLASSES
2763#include "mlir/Dialect/Linalg/IR/LinalgStructuredOps.cpp.inc"
2764#define GET_OP_CLASSES
2765#include "mlir/Dialect/Linalg/IR/LinalgRelayoutOps.cpp.inc"
2766
2767AffineMap mlir::linalg::extractOrIdentityMap(std::optional<AffineMap> maybeMap,
2768 unsigned rank,
2769 MLIRContext *context) {
2770 if (maybeMap)
2771 return *maybeMap;
2772 if (rank == 0)
2773 return AffineMap::get(context);
2774 return AffineMap::getMultiDimIdentityMap(numDims: rank, context);
2775}
2776
2777SmallVector<AffineExpr, 4>
2778mlir::linalg::makeAffineDimExprs(unsigned num, unsigned &startIdx,
2779 MLIRContext *context) {
2780 SmallVector<AffineExpr, 4> res;
2781 res.reserve(N: num);
2782 for (unsigned i = 0; i < num; ++i)
2783 res.push_back(Elt: getAffineDimExpr(position: startIdx++, context));
2784 return res;
2785}
2786
2787SmallVector<AffineExpr, 4> mlir::linalg::concat(ArrayRef<AffineExpr> a,
2788 ArrayRef<AffineExpr> b) {
2789 auto rangeA = llvm::make_range(x: a.begin(), y: a.end());
2790 auto rangeB = llvm::make_range(x: b.begin(), y: b.end());
2791 auto concatRanges = llvm::concat<const AffineExpr>(Ranges&: rangeA, Ranges&: rangeB);
2792 return llvm::to_vector<4>(Range&: concatRanges);
2793}
2794
2795static LogicalResult appendMangledType(llvm::raw_string_ostream &ss, Type t) {
2796 if (auto memref = llvm::dyn_cast<MemRefType>(Val&: t)) {
2797 ss << "view";
2798 for (auto size : memref.getShape())
2799 if (size < 0)
2800 ss << "sx";
2801 else
2802 ss << size << "x";
2803 if (failed(Result: appendMangledType(ss, t: memref.getElementType())))
2804 return failure();
2805 if (auto as = memref.getMemorySpace()) {
2806 if (auto attr = llvm::dyn_cast<IntegerAttr>(Val&: as))
2807 ss << "as" << attr.getInt();
2808 else
2809 return failure();
2810 }
2811 return success();
2812 }
2813 if (auto vec = llvm::dyn_cast<VectorType>(Val&: t)) {
2814 ss << "vector";
2815 llvm::interleave(
2816 c: vec.getShape(), each_fn: [&](int64_t i) { ss << i; }, between_fn: [&]() { ss << "x"; });
2817 if (failed(Result: appendMangledType(ss, t: vec.getElementType())))
2818 return failure();
2819 return success();
2820 }
2821 if (t.isSignlessIntOrIndexOrFloat()) {
2822 ss << t;
2823 return success();
2824 }
2825 return failure();
2826}
2827
2828std::string mlir::linalg::generateLibraryCallName(Operation *op) {
2829 assert(isa<LinalgOp>(op));
2830 std::string name(op->getName().getStringRef().str());
2831 std::string fun = "";
2832 op->getName().walkInherentAttrs(op, visitor: [&](StringRef, Attribute &attr) {
2833 if (UnaryFnAttr ufa = llvm::dyn_cast<UnaryFnAttr>(Val&: attr)) {
2834 fun = stringifyEnum(enumValue: ufa.getValue()).str() + "_";
2835 } else if (BinaryFnAttr bfa = llvm::dyn_cast<BinaryFnAttr>(Val&: attr)) {
2836 fun = stringifyEnum(enumValue: bfa.getValue()).str() + "_";
2837 }
2838 });
2839 name.reserve(res_arg: 128);
2840 llvm::replace(Range&: name, OldValue: '.', NewValue: '_');
2841 llvm::raw_string_ostream ss(name);
2842 ss << "_" << fun;
2843 for (Type t : op->getOperandTypes()) {
2844 if (failed(Result: appendMangledType(ss, t)))
2845 return std::string();
2846 ss << "_";
2847 }
2848 name.pop_back();
2849 return name;
2850}
2851
2852//===----------------------------------------------------------------------===//
2853// Canonicalizers and Folders.
2854//===----------------------------------------------------------------------===//
2855
2856namespace {
2857struct EraseDeadLinalgOp : public OpInterfaceRewritePattern<LinalgOp> {
2858 using OpInterfaceRewritePattern<LinalgOp>::OpInterfaceRewritePattern;
2859
2860 LogicalResult matchAndRewrite(LinalgOp op,
2861 PatternRewriter &rewriter) const override {
2862 for (OpOperand &opOperand : op->getOpOperands()) {
2863 // Linalg "inputs" may be either tensor or memref type.
2864 // tensor<0xelt_type> is a convention that may not always mean
2865 // "0 iterations". Only erase in cases we see memref<...x0x...>.
2866 auto mt = llvm::dyn_cast<MemRefType>(Val: opOperand.get().getType());
2867 if (!mt)
2868 continue;
2869 if (llvm::is_contained(Range: op.getShape(opOperand: &opOperand), Element: 0)) {
2870 rewriter.eraseOp(op);
2871 return success();
2872 }
2873 }
2874 return failure();
2875 }
2876};
2877
2878/// Fold LinalgOps with `tensor.cast` consumer if the `tensor.cast` has
2879/// result that is more static than the linalg op.
2880struct FoldTensorCastConsumerOp : public OpRewritePattern<tensor::CastOp> {
2881 using OpRewritePattern<tensor::CastOp>::OpRewritePattern;
2882
2883 LogicalResult matchAndRewrite(tensor::CastOp castOp,
2884 PatternRewriter &rewriter) const override {
2885 if (!tensor::canFoldIntoProducerOp(castOp))
2886 return failure();
2887
2888 auto linalgOp = castOp.getSource().getDefiningOp<LinalgOp>();
2889 if (!linalgOp)
2890 return failure();
2891
2892 // Cast can be in conditionally reachable region, if which case folding will
2893 // generate invalid code. Only conservatively fold ops in same block for
2894 // now.
2895 if (castOp->getBlock() != linalgOp->getBlock())
2896 return failure();
2897
2898 OpBuilder::InsertionGuard guard(rewriter);
2899 rewriter.setInsertionPoint(linalgOp);
2900
2901 Location loc = linalgOp.getLoc();
2902 OpResult resultValue = llvm::cast<OpResult>(Val: castOp.getSource());
2903 unsigned resultNumber = resultValue.getResultNumber();
2904 auto resultType =
2905 llvm::cast<RankedTensorType>(Val: castOp->getResult(idx: 0).getType());
2906 // Replace the `outs` for the result with a `tensor.cast`. This cast is now
2907 // going from a more dynamic shape to a less dynamic shape. If the producer
2908 // for this cast, i.e. producer of the out operand, is also an operation
2909 // that folds with tensor.cast consumer (like this pattern), the cast will
2910 // continue to propagate as far up the stack as it can go.
2911 OpOperand *outOperand = linalgOp.getDpsInitOperand(i: resultNumber);
2912 Value newOperand =
2913 tensor::CastOp::create(builder&: rewriter, location: loc, dest: resultType, source: outOperand->get());
2914 SmallVector<Value> newOperands = linalgOp.getDpsInputs();
2915 SmallVector<Value> outputOperands(linalgOp.getDpsInits().begin(),
2916 linalgOp.getDpsInits().end());
2917 outputOperands[resultNumber] = newOperand;
2918 newOperands.append(in_start: outputOperands.begin(), in_end: outputOperands.end());
2919
2920 SmallVector<Type> resultTypes(linalgOp->result_type_begin(),
2921 linalgOp->result_type_end());
2922 resultTypes[resultNumber] = resultType;
2923 Operation *newOp = clone(b&: rewriter, op: linalgOp, newResultTypes: resultTypes, newOperands);
2924
2925 // Create a tensor.cast operation back to the original type.
2926 Value castBack = tensor::CastOp::create(
2927 builder&: rewriter, location: loc, dest: resultValue.getType(), source: newOp->getResult(idx: resultNumber));
2928
2929 SmallVector<Value> results(newOp->result_begin(), newOp->result_end());
2930 results[resultNumber] = castBack;
2931 rewriter.replaceOp(op: linalgOp, newValues: results);
2932 rewriter.replaceOp(op: castOp, newValues: newOp->getResult(idx: resultNumber));
2933 return success();
2934 }
2935};
2936
2937/// For each of the operand in `operands` this function maps the static sizes of
2938/// dimensions to their affine dim expressions.
2939static void populateMap(LinalgOp linalgOp, MutableArrayRef<OpOperand> operands,
2940 llvm::DenseMap<AffineExpr, int64_t> &affineExprToSize) {
2941 for (OpOperand &opOperand : operands) {
2942 if (linalgOp.isScalar(opOperand: &opOperand))
2943 continue;
2944 Value src = opOperand.get();
2945 auto sourceType = llvm::cast<RankedTensorType>(Val: src.getType());
2946 auto sourceMap = linalgOp.getMatchingIndexingMap(opOperand: &opOperand);
2947
2948 // Get the `sourceShape` of the `sourceType`. If the operand is a result of
2949 // `tensor.cast` operation and source of the cast operation has a static
2950 // shape, then assign it to the `sourceShape`.
2951 auto *parentOp = src.getDefiningOp();
2952 ArrayRef<int64_t> sourceShape = sourceType.getShape();
2953 if (parentOp) {
2954 if (auto castOp = dyn_cast<tensor::CastOp>(Val: parentOp)) {
2955 Value castSource = castOp.getSource();
2956 auto castSourceType =
2957 llvm::dyn_cast<RankedTensorType>(Val: castSource.getType());
2958 if (castSourceType && castSourceType.hasStaticShape())
2959 sourceShape = castSourceType.getShape();
2960 }
2961 }
2962
2963 // If the source shape's dimension has a static shape, map the affine dim
2964 // expression to the known static size.
2965 for (unsigned i = 0; i < sourceShape.size(); i++) {
2966 if (sourceType.isDynamicDim(idx: i))
2967 continue;
2968 if (auto affineDimExpr = dyn_cast<AffineDimExpr>(Val: sourceMap.getResult(idx: i)))
2969 affineExprToSize.try_emplace(Key: affineDimExpr, Args: sourceShape[i]);
2970 }
2971 }
2972}
2973
2974/// Creates new operand w.r.t 'opOperand' of `linalgOp` with static sizes
2975/// mapped in `affineExprToSize`. New operands are created in `newOperands` and
2976/// their result types is stored in `resultTypes`. If `opOperand` requires no
2977/// change then `changeNeeded` is false and same operand is added in the
2978/// `newOperands` list.
2979static void createNewOperandWithStaticSizes(
2980 Location loc, PatternRewriter &rewriter, OpOperand *opOperand,
2981 llvm::DenseMap<AffineExpr, int64_t> &affineExprToSize, LinalgOp linalgOp,
2982 SmallVector<Value> &newOperands, SmallVector<Type> &resultTypes,
2983 bool &changeNeeded) {
2984 Value src = opOperand->get();
2985 newOperands.push_back(Elt: src);
2986 if (linalgOp.isScalar(opOperand))
2987 return;
2988 auto sourceType = llvm::cast<RankedTensorType>(Val: src.getType());
2989 Type resultType = sourceType;
2990 if (sourceType.hasStaticShape() && linalgOp.isDpsInit(opOperand)) {
2991 resultTypes.push_back(Elt: resultType);
2992 return;
2993 }
2994 ArrayRef<int64_t> sourceShape = sourceType.getShape();
2995 AffineMap sourceMap = linalgOp.getMatchingIndexingMap(opOperand);
2996 SmallVector<int64_t> newShape;
2997 // If operand is updated with new shape, `newOperandNeeded` will be
2998 // true.
2999 bool newOperandNeeded = false;
3000 for (unsigned i = 0; i < sourceShape.size(); i++) {
3001 int64_t dimShape = sourceShape[i];
3002 AffineExpr dimExpr = sourceMap.getResult(idx: i);
3003 if (!affineExprToSize.contains(Val: dimExpr) || !sourceType.isDynamicDim(idx: i)) {
3004 newShape.push_back(Elt: dimShape);
3005 continue;
3006 }
3007 // Dimension has a dynamic shape and corresponding affine dim
3008 // expression is present in the map. So assign the size for the
3009 // given affine dim expression to the dimension.
3010 newShape.push_back(Elt: affineExprToSize[dimExpr]);
3011 newOperandNeeded = true;
3012 }
3013 resultType = RankedTensorType::get(shape: newShape, elementType: sourceType.getElementType(),
3014 encoding: sourceType.getEncoding());
3015 if (newOperandNeeded) {
3016 changeNeeded = true;
3017 // Get the new operand value given its size and element type by
3018 // casting it.
3019 Value newOperand = tensor::CastOp::create(builder&: rewriter, location: loc, dest: resultType, source: src);
3020 unsigned index = opOperand->getOperandNumber();
3021 newOperands[index] = newOperand;
3022 }
3023 if (linalgOp.isDpsInit(opOperand))
3024 resultTypes.push_back(Elt: resultType);
3025}
3026
3027/// Static shapes for the operands can be inferred if any one of the operands
3028/// have a static shape. This can be done by referring to the affine dim
3029/// expressions for the operand.
3030struct InferStaticShapeOfOperands : public OpInterfaceRewritePattern<LinalgOp> {
3031 using OpInterfaceRewritePattern<LinalgOp>::OpInterfaceRewritePattern;
3032
3033 LogicalResult matchAndRewrite(LinalgOp linalgOp,
3034 PatternRewriter &rewriter) const override {
3035 if (!linalgOp.hasPureTensorSemantics())
3036 return failure();
3037
3038 // Maps must be projected permutations.
3039 if (llvm::any_of(Range: linalgOp.getIndexingMapsArray(), P: [](AffineMap map) {
3040 return !map.isProjectedPermutation();
3041 }))
3042 return failure();
3043
3044 // Maps affine dim expressions to the static size of that dimension.
3045 llvm::DenseMap<AffineExpr, int64_t> affineExprToSize;
3046 Location loc = linalgOp.getLoc();
3047
3048 // For each of the affine dim expression, check if the size is known. If
3049 // known add that in the map.
3050 populateMap(linalgOp, operands: linalgOp->getOpOperands(), affineExprToSize);
3051
3052 SmallVector<Value> newOperands;
3053 SmallVector<Type> resultTypes;
3054
3055 // `changeNeeded` is `false` if the operands of `linalgOp` require no
3056 // change in their types.
3057 bool changeNeeded = false;
3058 newOperands.reserve(N: linalgOp->getNumOperands());
3059 resultTypes.reserve(N: linalgOp.getNumDpsInits());
3060
3061 // Iterate over all the operands and update the static sizes.
3062 for (OpOperand &opOperand : linalgOp->getOpOperands()) {
3063 createNewOperandWithStaticSizes(loc, rewriter, opOperand: &opOperand,
3064 affineExprToSize, linalgOp, newOperands,
3065 resultTypes, changeNeeded);
3066 }
3067
3068 // If the generic op has all the required static information, no
3069 // canonicalization needed.
3070 if (!changeNeeded)
3071 return failure();
3072
3073 // Clone op.
3074 Operation *newOp = clone(b&: rewriter, op: linalgOp, newResultTypes: resultTypes, newOperands);
3075 SmallVector<Value> replacements;
3076 replacements.reserve(N: newOp->getNumResults());
3077 for (auto it : llvm::zip(t: linalgOp->getResults(), u: newOp->getResults())) {
3078 Value newResult = std::get<1>(t&: it);
3079 Value oldResult = std::get<0>(t&: it);
3080 Type newType = newResult.getType();
3081 Type oldType = oldResult.getType();
3082 replacements.push_back(
3083 Elt: (newType != oldType)
3084 ? tensor::CastOp::create(builder&: rewriter, location: loc, dest: oldType, source: newResult)
3085 : newResult);
3086 }
3087 rewriter.replaceOp(op: linalgOp, newValues: replacements);
3088 return success();
3089 }
3090};
3091
3092} // namespace
3093
3094// All named ops canonicalizers and folders are auto-generated in the
3095// .cpp.inc.
3096
3097//===----------------------------------------------------------------------===//
3098// SoftmaxOp
3099//===----------------------------------------------------------------------===//
3100
3101LogicalResult SoftmaxOp::verify() {
3102 ShapedType inputType = getInputOperandType();
3103 ShapedType outputType = getOutputOperandType();
3104
3105 ArrayRef<int64_t> inputShape = inputType.getShape();
3106 ArrayRef<int64_t> outputShape = outputType.getShape();
3107 if (failed(Result: verifyCompatibleShape(shape1: inputShape, shape2: outputShape)))
3108 return emitOpError(message: "incompatible output shape");
3109
3110 int64_t inputRank = getInputOperandRank();
3111 int64_t dimension = getDimension();
3112 if ((dimension < 0) || (dimension >= inputRank))
3113 return emitOpError(message: "incorrect dimension specified");
3114
3115 return success();
3116}
3117
3118SmallVector<Range> SoftmaxOp::getIterationDomain(OpBuilder &builder) {
3119 int64_t operandRank = getInputOperandRank();
3120 SmallVector<Range> loopBounds(operandRank);
3121 Location loc = getLoc();
3122 Value zero = arith::ConstantIndexOp::create(builder, location: loc, value: 0);
3123 Value one = arith::ConstantIndexOp::create(builder, location: loc, value: 1);
3124 Value source = getInput();
3125 for (auto dim : llvm::seq<int64_t>(Begin: 0, End: operandRank)) {
3126 loopBounds[dim].offset = zero;
3127 loopBounds[dim].size = getDimValue(builder, loc, v: source, dim);
3128 loopBounds[dim].stride = one;
3129 }
3130 return loopBounds;
3131}
3132
3133SmallVector<utils::IteratorType> SoftmaxOp::getLoopIteratorTypes() {
3134 SmallVector<utils::IteratorType> iteratorTypes(getInputOperandRank(),
3135 utils::IteratorType::parallel);
3136 iteratorTypes[getDimension()] = utils::IteratorType::reduction;
3137 return iteratorTypes;
3138}
3139
3140/// The inner tile alignment hint is only used by `linalg.pack` and
3141/// `linalg.unpack` operations. Therefore, this is forwarded to the hint-less
3142/// overload.
3143FailureOr<TilingResult> SoftmaxOp::getTiledImplementation(
3144 OpBuilder &builder, ArrayRef<OpFoldResult> offsets,
3145 ArrayRef<OpFoldResult> sizes, ArrayRef<InnerTileAlignment>) {
3146 return getTiledImplementation(b&: builder, offsets, sizes);
3147}
3148
3149FailureOr<TilingResult>
3150SoftmaxOp::getTiledImplementation(OpBuilder &builder,
3151 ArrayRef<OpFoldResult> offsets,
3152 ArrayRef<OpFoldResult> sizes) {
3153 int64_t rank = getInputOperandRank();
3154 auto oneAttr = builder.getI64IntegerAttr(value: 1);
3155 SmallVector<OpFoldResult> strides(rank, oneAttr);
3156 SmallVector<Value> tiledOperands;
3157 Operation *inputSlice =
3158 getSlice(b&: builder, loc: getLoc(), source: getInput(), offsets, sizes, strides);
3159 if (!inputSlice) {
3160 return emitOpError(message: "failed to compute input slice");
3161 }
3162 tiledOperands.emplace_back(Args: inputSlice->getResult(idx: 0));
3163 Operation *outputSlice =
3164 getSlice(b&: builder, loc: getLoc(), source: getOutput(), offsets, sizes, strides);
3165 if (!outputSlice) {
3166 return emitOpError(message: "failed to compute output slice");
3167 }
3168 tiledOperands.emplace_back(Args: outputSlice->getResult(idx: 0));
3169
3170 SmallVector<Type, 4> resultTypes;
3171 if (hasPureTensorSemantics())
3172 resultTypes.push_back(Elt: tiledOperands[1].getType());
3173 Operation *tiledOp =
3174 mlir::clone(b&: builder, op: getOperation(), newResultTypes: resultTypes, newOperands: tiledOperands);
3175
3176 return TilingResult{
3177 .tiledOps: {tiledOp},
3178 .tiledValues: SmallVector<Value>(tiledOp->getResults()),
3179 .generatedSlices: llvm::to_vector(Range: ArrayRef<Operation *>{inputSlice, outputSlice})};
3180}
3181
3182LogicalResult SoftmaxOp::getResultTilePosition(
3183 OpBuilder &builder, unsigned resultNumber, ArrayRef<OpFoldResult> offsets,
3184 ArrayRef<OpFoldResult> sizes, SmallVector<OpFoldResult> &resultOffsets,
3185 SmallVector<OpFoldResult> &resultSizes) {
3186 if (resultNumber == 0) {
3187 resultOffsets.assign(in_start: offsets.begin(), in_end: offsets.end());
3188 resultSizes.assign(in_start: sizes.begin(), in_end: sizes.end());
3189 return success();
3190 }
3191 return failure();
3192}
3193
3194// cast(dynamic) -> static.
3195LogicalResult SoftmaxOp::fold(FoldAdaptor, SmallVectorImpl<OpFoldResult> &) {
3196 return memref::foldMemRefCast(op: *this);
3197}
3198
3199LogicalResult
3200SoftmaxOp::reifyResultShapes(OpBuilder &b,
3201 ReifiedRankedShapedTypeDims &reifiedReturnShapes) {
3202 SmallVector<OpFoldResult> shapes;
3203 Location loc = getOperation()->getLoc();
3204 IRRewriter rewriter(b);
3205 auto inputShapedType = llvm::cast<ShapedType>(Val: getInputOperandType());
3206 auto outputShapedType = llvm::cast<ShapedType>(Val: getOutputOperandType());
3207 for (int64_t dim : llvm::seq<int64_t>(Begin: 0, End: getOutputOperandRank())) {
3208 if (!outputShapedType.isDynamicDim(idx: dim)) {
3209 // Static dim: Return IntegerAttr.
3210 shapes.push_back(Elt: b.getIndexAttr(value: inputShapedType.getDimSize(idx: dim)));
3211 } else {
3212 // Dynamic dim: Return Value.
3213 OpFoldResult ofr = createOrFoldDimOp(b, loc, source: getInput(), dim);
3214 shapes.push_back(Elt: getValueOrCreateConstantIndexOp(b, loc, ofr));
3215 }
3216 }
3217 reifiedReturnShapes.emplace_back(Args: std::move(shapes));
3218 return success();
3219}
3220
3221void SoftmaxOp::getEffects(
3222 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
3223 &effects) {
3224 for (auto [index, operand] : llvm::enumerate(First: getDpsInputs())) {
3225 if (!llvm::isa<MemRefType>(Val: operand.getType()))
3226 continue;
3227 effects.emplace_back(Args: MemoryEffects::Read::get(),
3228 Args: &getOperation()->getOpOperand(idx: index), /*stage=*/Args: 0,
3229 /*effectOnFullRegion=*/Args: true,
3230 Args: SideEffects::DefaultResource::get());
3231 }
3232
3233 for (OpOperand &operand : getDpsInitsMutable()) {
3234 if (!llvm::isa<MemRefType>(Val: operand.get().getType()))
3235 continue;
3236 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: &operand, /*stage=*/Args: 0,
3237 /*effectOnFullRegion=*/Args: true,
3238 Args: SideEffects::DefaultResource::get());
3239 effects.emplace_back(Args: MemoryEffects::Write::get(), Args: &operand, /*stage=*/Args: 0,
3240 /*effectOnFullRegion=*/Args: true,
3241 Args: SideEffects::DefaultResource::get());
3242 }
3243}
3244
3245// Helper functions for softmax decomposition.
3246// @{
3247
3248// Helper function to produce the iterator types (reduction or parallel) and
3249// affine maps for the iterators used in the decomposition of softmax.
3250// This method creates:
3251// If allParallel == true:
3252// - iterator type: {parallel, ..., parallel}
3253// - affine maps:
3254// -- identity with inputRank dimensions.
3255// -- (d0, ..., dN) -> (d0, ..., d_dim-1, d_dim+1, ..., dN),
3256// where N == inputRank.
3257//
3258// If allParallel == false:
3259// - iterator type at dim(i) == parallel for i != \p dim and
3260// dim(dim) == reduction.
3261// - affine map:
3262// -- identity with inputRank dimensions.
3263// -- (d0, ..., dN) -> (d0, ..., d_dim-1, d_dim+1, ..., dN),
3264// where N == inputRank.
3265static std::tuple<SmallVector<utils::IteratorType>, SmallVector<AffineMap>>
3266computeIteratorTypesAndIndexingMaps(OpBuilder &builder, int64_t inputRank,
3267 int64_t dim, bool allParallel = false) {
3268 SmallVector<utils::IteratorType> iteratorTypes(inputRank,
3269 utils::IteratorType::parallel);
3270 if (!allParallel)
3271 iteratorTypes[dim] = utils::IteratorType::reduction;
3272 MLIRContext *ctxt = builder.getContext();
3273 auto identityMap = AffineMap::getMultiDimIdentityMap(numDims: inputRank, context: ctxt);
3274 SmallVector<AffineExpr, 2> affineExprs;
3275 for (int i = 0; i < inputRank; i++) {
3276 if (i != dim)
3277 affineExprs.push_back(Elt: mlir::getAffineDimExpr(position: i, context: ctxt));
3278 }
3279 auto reductionMap =
3280 AffineMap::get(dimCount: inputRank, /*symbols=*/symbolCount: 0, results: affineExprs, context: ctxt);
3281 SmallVector<AffineMap> indexingMaps{identityMap, reductionMap};
3282 return std::make_tuple(args&: iteratorTypes, args&: indexingMaps);
3283}
3284
3285// Helper function to produce a linalg.generic that computes a reduction on
3286// dimension \p dim with the operation type \p T.
3287template <typename T>
3288static Value reduce(OpBuilder &builder, Location loc, Value input, Value output,
3289 int64_t dim) {
3290 auto inputType = cast<ShapedType>(Val: input.getType());
3291 ArrayRef<int64_t> inputShape = inputType.getShape();
3292 int64_t inputRank = inputShape.size();
3293 auto [iteratorTypes, indexingMaps] =
3294 computeIteratorTypesAndIndexingMaps(builder, inputRank, dim);
3295 assert(indexingMaps.size() == 2 &&
3296 "We should have two maps: 1 for the input, 1 for the output");
3297 assert(indexingMaps[0].isIdentity() && "input map should be identity");
3298
3299 auto genericOp = linalg::GenericOp::create(
3300 builder, loc, output.getType(), input, output, indexingMaps,
3301 iteratorTypes, [&](OpBuilder &b, Location loc, ValueRange args) {
3302 Value result = T::create(b, loc, args[0], args[1]);
3303 linalg::YieldOp::create(builder&: b, location: loc, values: result);
3304 });
3305 return genericOp.getResult(0);
3306}
3307
3308/// Produce a linalg generic that computes the second step of the softmax
3309/// decomposition: res = exp(input - max), where \p max is the max of \p input
3310/// on dimension \p dim.
3311static Value buildSubAndExpOp(OpBuilder &builder, Location loc, Value input,
3312 Value max, Value output, int64_t dim) {
3313 auto inputType = cast<ShapedType>(Val: input.getType());
3314 ArrayRef<int64_t> inputShape = inputType.getShape();
3315 int64_t inputRank = inputShape.size();
3316 auto [iteratorTypes, indexingMaps] = computeIteratorTypesAndIndexingMaps(
3317 builder, inputRank, dim, /*allParallel=*/true);
3318 assert(indexingMaps.size() == 2 && "We should have one map for each input");
3319 assert(indexingMaps[0].isIdentity() && "input map should be identity");
3320 // Add the affine map for the output argument.
3321 indexingMaps.push_back(Elt: indexingMaps[0]);
3322 auto genericOp = linalg::GenericOp::create(
3323 builder, location: loc, resultTensorTypes: input.getType(), inputs: ValueRange{input, max}, outputs: output,
3324 indexingMaps, iteratorTypes,
3325 odsArg5: [&](OpBuilder &b, Location loc, ValueRange args) {
3326 Value diff = arith::SubFOp::create(builder&: b, location: loc, lhs: args[0], rhs: args[1]);
3327 Value result = math::ExpOp::create(builder&: b, location: loc, operand: diff);
3328 linalg::YieldOp::create(builder&: b, location: loc, values: result);
3329 });
3330 return genericOp.getResult(i: 0);
3331}
3332
3333/// Produce a linalg generic that computes the final step of the softmax
3334/// decomposition.
3335/// \returns linalg.generic ins(\p numerator, \p denominator) outs(\p output) {
3336/// yield n / d
3337/// }
3338static Value buildDivOp(OpBuilder &builder, Location loc, Value numerator,
3339 Value denominator, Value output, int64_t dim) {
3340 auto inputType = cast<ShapedType>(Val: numerator.getType());
3341 ArrayRef<int64_t> inputShape = inputType.getShape();
3342 int64_t inputRank = inputShape.size();
3343 auto [iteratorTypes, indexingMaps] = computeIteratorTypesAndIndexingMaps(
3344 builder, inputRank, dim, /*allParallel=*/true);
3345 assert(indexingMaps.size() == 2 &&
3346 "We should have one map for each input (2)");
3347 assert(indexingMaps[0].isIdentity() && "Numerator map should be identity");
3348 // Add the affine map for the output tensor.
3349 indexingMaps.push_back(Elt: indexingMaps[0]);
3350 auto genericOp = linalg::GenericOp::create(
3351 builder, location: loc, resultTensorTypes: numerator.getType(), inputs: ValueRange{numerator, denominator},
3352 outputs: output, indexingMaps, iteratorTypes,
3353 odsArg5: [&](OpBuilder &b, Location loc, ValueRange args) {
3354 Value result = arith::DivFOp::create(builder&: b, location: loc, lhs: args[0], rhs: args[1]);
3355 linalg::YieldOp::create(builder&: b, location: loc, values: result);
3356 });
3357 return genericOp.getResult(i: 0);
3358}
3359// @} End helper functions for softmax decomposition.
3360
3361/// Given an N-dimensional tensor x, this method converts
3362/// softmax(x) to the following sequence of operations:
3363///
3364/// 1. Compute the max of x along dimension d. This results
3365/// in a N-1 dimensional tensor m.
3366/// m = max(x, dim = d)
3367///
3368/// 2. Subtract a broadcasted m from x and exponentiate. This results in
3369/// a N dimensional tensor z.
3370/// z = exp(x - m)
3371///
3372/// 3. Compute the sum of z along dimension d. This results in
3373/// a N-1 dimensional tensor l.
3374/// l = sum(z, dim = d)
3375///
3376/// 4. Divide z and l. This gives the N-dimensional softmax.
3377/// softmax = z / l
3378///
3379FailureOr<SmallVector<Value>> SoftmaxOp::decomposeOperation(OpBuilder &b) {
3380 OpBuilder::InsertionGuard guard(b);
3381 b.setInsertionPoint(*this);
3382 Location loc = getLoc();
3383 Value input = getInput();
3384 ShapedType inputType = getInputOperandType();
3385 Type elementType = inputType.getElementType();
3386 int64_t reductionDim = getDimension();
3387 SmallVector<OpFoldResult> dims = tensor::getMixedSizes(builder&: b, loc, value: input);
3388 Value output = getOutput();
3389 dims.erase(CI: dims.begin() + reductionDim);
3390 // Step 1: Compute max along dim.
3391 Value outputReduce = tensor::EmptyOp::create(builder&: b, location: loc, sizes: dims, elementType);
3392 Value neutralForMaxF = arith::getIdentityValue(op: arith::AtomicRMWKind::maxnumf,
3393 resultType: elementType, builder&: b, loc,
3394 /*useOnlyFiniteValue=*/true);
3395 Value neutralForMaxFInit =
3396 linalg::FillOp::create(builder&: b, location: loc, inputs: Value{neutralForMaxF}, outputs: outputReduce)
3397 .result();
3398 Value max =
3399 reduce<arith::MaxNumFOp>(builder&: b, loc, input, output: neutralForMaxFInit, dim: reductionDim);
3400
3401 // Step 2: Subtract max from input and exponentiate.
3402 Value numerator = buildSubAndExpOp(builder&: b, loc, input, max, output, dim: reductionDim);
3403
3404 // Step 3: Compute sum along dim.
3405 Value zero = arith::getIdentityValue(op: arith::AtomicRMWKind::addf, resultType: elementType,
3406 builder&: b, loc, /*useOnlyFiniteValue=*/true);
3407 Value zeroInit =
3408 linalg::FillOp::create(builder&: b, location: loc, inputs: Value{zero}, outputs: outputReduce).result();
3409 Value denominator =
3410 reduce<arith::AddFOp>(builder&: b, loc, input: numerator, output: zeroInit, dim: reductionDim);
3411
3412 // Step 4: Compute softmax.
3413 Value result =
3414 buildDivOp(builder&: b, loc, numerator, denominator, output, dim: reductionDim);
3415 return SmallVector<Value>{result};
3416}
3417
3418//===----------------------------------------------------------------------===//
3419// WinogradFilterTransformOp
3420//===----------------------------------------------------------------------===//
3421
3422LogicalResult WinogradFilterTransformOp::verify() {
3423 auto filterType = cast<ShapedType>(Val: getFilter().getType());
3424 ArrayRef<int64_t> filterShape = filterType.getShape();
3425 int64_t filterH = filterShape[getFilterHDim()];
3426 int64_t filterW = filterShape[getFilterWDim()];
3427 WinogradConv2DFmr fmr = getFmr();
3428 int64_t m, r;
3429 std::tie(args&: m, args&: r) = getFmrFromWinogradConv2DFmr(fmr);
3430
3431 if (filterH != r && filterH != 1)
3432 return emitOpError(message: "expect filter height either equals to r or 1");
3433 if (filterW != r && filterW != 1)
3434 return emitOpError(message: "expect filter width either equals to r or 1");
3435 if (filterH == 1 && filterW == 1)
3436 return emitOpError(message: "expect either filter height or width equals to r");
3437
3438 SmallVector<int64_t> expectedOutputShape;
3439 expectedOutputShape.push_back(Elt: filterH == r ? m + r - 1 : 1);
3440 expectedOutputShape.push_back(Elt: filterW == r ? m + r - 1 : 1);
3441 expectedOutputShape.push_back(Elt: filterShape[getFilterCDim()]);
3442 expectedOutputShape.push_back(Elt: filterShape[getFilterFDim()]);
3443
3444 auto outputType = cast<ShapedType>(Val: getOutput().getType());
3445 ArrayRef<int64_t> outputShape = outputType.getShape();
3446 if (failed(Result: verifyCompatibleShape(shape1: expectedOutputShape, shape2: outputShape))) {
3447 return emitOpError(message: "the output shape is not expected");
3448 }
3449 return success();
3450}
3451
3452SmallVector<Range>
3453WinogradFilterTransformOp::getIterationDomain(OpBuilder &builder) {
3454 Location loc = getLoc();
3455 IntegerAttr zeroAttr = builder.getIndexAttr(value: 0);
3456 IntegerAttr oneAttr = builder.getIndexAttr(value: 1);
3457 Value filter = getFilter();
3458 int64_t filterRank = getFilterOperandRank();
3459 SmallVector<Range> loopBounds(filterRank);
3460 for (unsigned dim = 0; dim < filterRank; ++dim) {
3461 loopBounds[dim].offset = zeroAttr;
3462 loopBounds[dim].size = getDimValue(builder, loc, v: filter, dim);
3463 loopBounds[dim].stride = oneAttr;
3464 }
3465 return loopBounds;
3466}
3467
3468SmallVector<utils::IteratorType>
3469WinogradFilterTransformOp::getLoopIteratorTypes() {
3470 int64_t filterRank = getFilterOperandRank();
3471 SmallVector<utils::IteratorType> iteratorTypes(filterRank,
3472 utils::IteratorType::parallel);
3473 return iteratorTypes;
3474}
3475
3476LogicalResult WinogradFilterTransformOp::getResultTilePosition(
3477 OpBuilder &builder, unsigned resultNumber, ArrayRef<OpFoldResult> offsets,
3478 ArrayRef<OpFoldResult> sizes, SmallVector<OpFoldResult> &resultOffsets,
3479 SmallVector<OpFoldResult> &resultSizes) {
3480 IntegerAttr zeroAttr = builder.getI64IntegerAttr(value: 0);
3481 ShapedType filterType = getFilterOperandType();
3482 ArrayRef<int64_t> filterShape = filterType.getShape();
3483 int64_t filterH = filterShape[getFilterHDim()];
3484 int64_t filterW = filterShape[getFilterWDim()];
3485 WinogradConv2DFmr fmr = getFmr();
3486 int64_t m, r;
3487 std::tie(args&: m, args&: r) = getFmrFromWinogradConv2DFmr(fmr);
3488 int64_t alpha = m + r - 1;
3489 int64_t alphaH = filterH != 1 ? alpha : 1;
3490 int64_t alphaW = filterW != 1 ? alpha : 1;
3491 IntegerAttr alphaHAttr = builder.getI64IntegerAttr(value: alphaH);
3492 IntegerAttr alphaWAttr = builder.getI64IntegerAttr(value: alphaW);
3493
3494 resultOffsets.append(
3495 IL: {zeroAttr, zeroAttr, offsets[getFilterCDim()], offsets[getFilterFDim()]});
3496 resultSizes.append(
3497 IL: {alphaHAttr, alphaWAttr, sizes[getFilterCDim()], sizes[getFilterFDim()]});
3498
3499 return success();
3500}
3501
3502/// The inner tile alignment hint is only used by `linalg.pack` and
3503/// `linalg.unpack` operations. Therefore, this is forwarded to the hint-less
3504/// overload.
3505FailureOr<TilingResult> WinogradFilterTransformOp::getTiledImplementation(
3506 OpBuilder &builder, ArrayRef<OpFoldResult> offsets,
3507 ArrayRef<OpFoldResult> sizes, ArrayRef<InnerTileAlignment>) {
3508 return getTiledImplementation(b&: builder, offsets, sizes);
3509}
3510
3511/// Implement tiling for winograd_filter_transform
3512/// The input of winograd_filter_transform is (F, KH, KW, C).
3513/// The output of winograd_filter_transform is (alphaH, alphaW, C, F)
3514/// Users can specify the tile sizes of F and C.
3515/// `offsets` are the values for the offsets of F, KH, KW, C for one tile.
3516/// `sizes` are the values for the sizes of F, KH, KW, C for one tile.
3517FailureOr<TilingResult> WinogradFilterTransformOp::getTiledImplementation(
3518 OpBuilder &builder, ArrayRef<OpFoldResult> offsets,
3519 ArrayRef<OpFoldResult> sizes) {
3520 IntegerAttr oneAttr = builder.getI64IntegerAttr(value: 1);
3521 IntegerAttr zeroAttr = builder.getI64IntegerAttr(value: 0);
3522 ShapedType filterType = getFilterOperandType();
3523 ArrayRef<int64_t> filterShape = filterType.getShape();
3524 int64_t filterH = filterShape[getFilterHDim()];
3525 int64_t filterW = filterShape[getFilterWDim()];
3526 IntegerAttr filterHAttr = builder.getI64IntegerAttr(value: filterH);
3527 IntegerAttr filterWAttr = builder.getI64IntegerAttr(value: filterW);
3528 SmallVector<Value> tiledOperands;
3529 SmallVector<OpFoldResult> sliceOffsets, sliceSizes;
3530
3531 sliceOffsets.append(
3532 IL: {offsets[getFilterFDim()], zeroAttr, zeroAttr, offsets[getFilterCDim()]});
3533 sliceSizes.append(IL: {sizes[getFilterFDim()], filterHAttr, filterWAttr,
3534 sizes[getFilterCDim()]});
3535 int64_t filterRank = getFilterOperandRank();
3536 SmallVector<OpFoldResult> filterStrides(filterRank, oneAttr);
3537 Location loc = getLoc();
3538 auto filterSlice = tensor::ExtractSliceOp::create(
3539 builder, location: loc, source: getFilter(), offsets: sliceOffsets, sizes: sliceSizes, strides: filterStrides);
3540 tiledOperands.emplace_back(Args&: filterSlice);
3541
3542 SmallVector<OpFoldResult> resultOffsets, resultSizes;
3543 if (failed(Result: getResultTilePosition(builder, resultNumber: 1, offsets, sizes, resultOffsets,
3544 resultSizes)))
3545 return failure();
3546
3547 int64_t outputRank = getOutputOperandRank();
3548 SmallVector<OpFoldResult> outputStrides(outputRank, oneAttr);
3549 auto outputSlice = tensor::ExtractSliceOp::create(
3550 builder, location: loc, source: getOutput(), offsets: resultOffsets, sizes: resultSizes, strides: outputStrides);
3551 tiledOperands.emplace_back(Args&: outputSlice);
3552
3553 SmallVector<Type> resultTypes;
3554 resultTypes.push_back(Elt: tiledOperands[1].getType());
3555 Operation *tiledOp =
3556 mlir::clone(b&: builder, op: getOperation(), newResultTypes: resultTypes, newOperands: tiledOperands);
3557
3558 return TilingResult{
3559 .tiledOps: {tiledOp},
3560 .tiledValues: SmallVector<Value>(tiledOp->getResults()),
3561 .generatedSlices: llvm::to_vector(Range: ArrayRef<Operation *>{filterSlice, outputSlice})};
3562}
3563
3564//===----------------------------------------------------------------------===//
3565// WinogradInputTransformOp
3566//===----------------------------------------------------------------------===//
3567
3568LogicalResult WinogradInputTransformOp::verify() {
3569 auto inputType = cast<ShapedType>(Val: getInput().getType());
3570 ArrayRef<int64_t> inputShape = inputType.getShape();
3571 int64_t inputH = inputShape[getInputHDim()];
3572 int64_t inputW = inputShape[getInputWDim()];
3573 WinogradConv2DFmr fmr = getFmr();
3574 int64_t m, r;
3575 std::tie(args&: m, args&: r) = getFmrFromWinogradConv2DFmr(fmr);
3576 int64_t tileSize = m + r - 1;
3577
3578 auto outputType = cast<ShapedType>(Val: getOutput().getType());
3579 ArrayRef<int64_t> outputShape = outputType.getShape();
3580 bool leftTransform = outputShape[getOutputAlphaHDim()] != 1;
3581 bool rightTransform = outputShape[getOutputAlphaWDim()] != 1;
3582
3583 SmallVector<int64_t> expectedOutputShape(6, inputH);
3584 if (ShapedType::isDynamic(dValue: inputH)) {
3585 expectedOutputShape[getOutputAlphaHDim()] = tileSize;
3586 expectedOutputShape[getOutputTileHDim()] = ShapedType::kDynamic;
3587 } else {
3588 expectedOutputShape[getOutputAlphaHDim()] = leftTransform ? tileSize : 1;
3589 expectedOutputShape[getOutputTileHDim()] =
3590 leftTransform ? (inputH - (r - 1)) / m : inputH;
3591 }
3592 if (ShapedType::isDynamic(dValue: inputW)) {
3593 expectedOutputShape[getOutputAlphaWDim()] = tileSize;
3594 expectedOutputShape[getOutputTileWDim()] = ShapedType::kDynamic;
3595 } else {
3596 expectedOutputShape[getOutputAlphaWDim()] = rightTransform ? tileSize : 1;
3597 expectedOutputShape[getOutputTileWDim()] =
3598 rightTransform ? (inputW - (r - 1)) / m : inputW;
3599 }
3600 expectedOutputShape[getOutputNDim()] = inputShape[getInputNDim()];
3601 expectedOutputShape[getOutputCDim()] = inputShape[getInputCDim()];
3602
3603 if (failed(Result: verifyCompatibleShape(shape1: expectedOutputShape, shape2: outputShape))) {
3604 return emitOpError(message: "the output shape is not expected");
3605 }
3606 return success();
3607}
3608
3609SmallVector<Range>
3610WinogradInputTransformOp::getIterationDomain(OpBuilder &builder) {
3611 Location loc = getLoc();
3612 IntegerAttr zeroAttr = builder.getIndexAttr(value: 0);
3613 IntegerAttr oneAttr = builder.getIndexAttr(value: 1);
3614 Value output = getOutput();
3615 int64_t outputRank = getOutputOperandRank();
3616 SmallVector<Range> loopBounds(outputRank);
3617 for (unsigned dim = 0; dim < outputRank; ++dim) {
3618 loopBounds[dim].offset = zeroAttr;
3619 // alphaH, alphaW, tileH, tileW, N, C
3620 loopBounds[dim].size = getDimValue(builder, loc, v: output, dim);
3621 loopBounds[dim].stride = oneAttr;
3622 }
3623 return loopBounds;
3624}
3625
3626SmallVector<utils::IteratorType>
3627WinogradInputTransformOp::getLoopIteratorTypes() {
3628 int64_t outputRank = getOutputOperandRank();
3629 SmallVector<utils::IteratorType> iteratorTypes(outputRank,
3630 utils::IteratorType::parallel);
3631 return iteratorTypes;
3632}
3633
3634LogicalResult WinogradInputTransformOp::getResultTilePosition(
3635 OpBuilder &builder, unsigned resultNumber, ArrayRef<OpFoldResult> offsets,
3636 ArrayRef<OpFoldResult> sizes, SmallVector<OpFoldResult> &resultOffsets,
3637 SmallVector<OpFoldResult> &resultSizes) {
3638 IntegerAttr zeroAttr = builder.getI64IntegerAttr(value: 0);
3639 ShapedType outputType = getOutputOperandType();
3640 ArrayRef<int64_t> outputShape = outputType.getShape();
3641 int64_t outputAlphaH = outputShape[getOutputAlphaHDim()];
3642 int64_t outputAlphaW = outputShape[getOutputAlphaWDim()];
3643
3644 WinogradConv2DFmr fmr = getFmr();
3645 int64_t m, r;
3646 std::tie(args&: m, args&: r) = getFmrFromWinogradConv2DFmr(fmr);
3647 int64_t alpha = m + r - 1;
3648 int64_t alphaH = outputAlphaH != 1 ? alpha : 1;
3649 int64_t alphaW = outputAlphaW != 1 ? alpha : 1;
3650
3651 IntegerAttr alphaHAttr = builder.getI64IntegerAttr(value: alphaH);
3652 IntegerAttr alphaWAttr = builder.getI64IntegerAttr(value: alphaW);
3653
3654 resultOffsets.append(IL: {zeroAttr, zeroAttr, offsets[getOutputTileHDim()],
3655 offsets[getOutputTileWDim()], offsets[getOutputNDim()],
3656 offsets[getOutputCDim()]});
3657 resultSizes.append(IL: {alphaHAttr, alphaWAttr, sizes[getOutputTileHDim()],
3658 sizes[getOutputTileWDim()], sizes[getOutputNDim()],
3659 sizes[getOutputCDim()]});
3660
3661 return success();
3662}
3663
3664/// The inner tile alignment hint is only used by `linalg.pack` and
3665/// `linalg.unpack` operations. Therefore, this is forwarded to the hint-less
3666/// overload.
3667FailureOr<TilingResult> WinogradInputTransformOp::getTiledImplementation(
3668 OpBuilder &builder, ArrayRef<OpFoldResult> offsets,
3669 ArrayRef<OpFoldResult> sizes, ArrayRef<InnerTileAlignment>) {
3670 return getTiledImplementation(b&: builder, offsets, sizes);
3671}
3672
3673/// Implement tiling for winograd_input_transform
3674/// The input of winograd_input_transform is (N, H, W, C).
3675/// The output of winograd_input_transform is (alphaH, alphaW, tileH, tileW, N,
3676/// C) Users can specify the tile sizes of tileH, tileW, N, and C. `offsets` are
3677/// the values for the offsets of tileH, tileW, N, C for one tile. `sizes` are
3678/// the values for the sizes of tileH, tileW, N, C for one tile.
3679FailureOr<TilingResult>
3680WinogradInputTransformOp::getTiledImplementation(OpBuilder &builder,
3681 ArrayRef<OpFoldResult> offsets,
3682 ArrayRef<OpFoldResult> sizes) {
3683 IntegerAttr oneAttr = builder.getI64IntegerAttr(value: 1);
3684 WinogradConv2DFmr fmr = getFmr();
3685 int64_t m, r;
3686 std::tie(args&: m, args&: r) = getFmrFromWinogradConv2DFmr(fmr);
3687
3688 ShapedType outputType = getOutputOperandType();
3689 ArrayRef<int64_t> outputShape = outputType.getShape();
3690 int64_t alphaH = outputShape[getOutputAlphaHDim()];
3691 int64_t alphaW = outputShape[getOutputAlphaWDim()];
3692
3693 Location loc = getLoc();
3694 MLIRContext *context = builder.getContext();
3695 auto identityAffineMap =
3696 AffineMap::get(dimCount: 1, symbolCount: 0, results: {builder.getAffineDimExpr(position: 0)}, context);
3697 auto offsetAffineMap =
3698 AffineMap::get(dimCount: 1, symbolCount: 0, results: {builder.getAffineDimExpr(position: 0) * m}, context);
3699 Value mappedOffsetH = affine::makeComposedAffineApply(
3700 b&: builder, loc, map: (alphaH != 1 ? offsetAffineMap : identityAffineMap),
3701 operands: offsets[getOutputTileHDim()]);
3702 Value mappedOffsetW = affine::makeComposedAffineApply(
3703 b&: builder, loc, map: (alphaW != 1 ? offsetAffineMap : identityAffineMap),
3704 operands: offsets[getOutputTileWDim()]);
3705 auto sizeAffineMap = AffineMap::get(
3706 dimCount: 1, symbolCount: 0, results: {builder.getAffineDimExpr(position: 0) * m + (r - 1)}, context);
3707 Value mappedSizeH = affine::makeComposedAffineApply(
3708 b&: builder, loc, map: sizeAffineMap, operands: sizes[getOutputTileHDim()]);
3709 Value mappedSizeW = affine::makeComposedAffineApply(
3710 b&: builder, loc, map: sizeAffineMap, operands: sizes[getOutputTileWDim()]);
3711
3712 SmallVector<Value> tiledOperands;
3713 SmallVector<OpFoldResult> sliceOffsets, sliceSizes;
3714
3715 OpFoldResult offsetH = OpFoldResult(mappedOffsetH);
3716 OpFoldResult offsetW = OpFoldResult(mappedOffsetW);
3717 sliceOffsets.append(
3718 IL: {offsets[getOutputNDim()], offsetH, offsetW, offsets[getOutputCDim()]});
3719 OpFoldResult sizeH =
3720 alphaH != 1 ? OpFoldResult(mappedSizeH) : OpFoldResult(oneAttr);
3721 OpFoldResult sizeW =
3722 alphaW != 1 ? OpFoldResult(mappedSizeW) : OpFoldResult(oneAttr);
3723 sliceSizes.append(
3724 IL: {sizes[getOutputNDim()], sizeH, sizeW, sizes[getOutputCDim()]});
3725 int64_t inputRank = getInputOperandRank();
3726 SmallVector<OpFoldResult> inputStrides(inputRank, oneAttr);
3727 auto inputSlice = tensor::ExtractSliceOp::create(
3728 builder, location: loc, source: getInput(), offsets: sliceOffsets, sizes: sliceSizes, strides: inputStrides);
3729 tiledOperands.emplace_back(Args&: inputSlice);
3730
3731 SmallVector<OpFoldResult> resultOffsets, resultSizes;
3732 if (failed(Result: getResultTilePosition(builder, resultNumber: 1, offsets, sizes, resultOffsets,
3733 resultSizes)))
3734 return failure();
3735
3736 int64_t outputRank = getOutputOperandRank();
3737 SmallVector<OpFoldResult> outputStrides(outputRank, oneAttr);
3738 auto outputSlice = tensor::ExtractSliceOp::create(
3739 builder, location: loc, source: getOutput(), offsets: resultOffsets, sizes: resultSizes, strides: outputStrides);
3740 tiledOperands.emplace_back(Args&: outputSlice);
3741
3742 SmallVector<Type> resultTypes;
3743 resultTypes.push_back(Elt: tiledOperands[1].getType());
3744 Operation *tiledOp =
3745 mlir::clone(b&: builder, op: getOperation(), newResultTypes: resultTypes, newOperands: tiledOperands);
3746
3747 return TilingResult{
3748 .tiledOps: {tiledOp},
3749 .tiledValues: SmallVector<Value>(tiledOp->getResults()),
3750 .generatedSlices: llvm::to_vector(Range: ArrayRef<Operation *>{inputSlice, outputSlice})};
3751}
3752
3753//===----------------------------------------------------------------------===//
3754// WinogradOutputTransformOp
3755//===----------------------------------------------------------------------===//
3756
3757LogicalResult WinogradOutputTransformOp::verify() {
3758 auto valueType = cast<ShapedType>(Val: getValue().getType());
3759 ArrayRef<int64_t> valueShape = valueType.getShape();
3760 int64_t valueH = valueShape[getValueAlphaHDim()];
3761 int64_t valueW = valueShape[getValueAlphaWDim()];
3762 int64_t valueTileH = valueShape[getValueTileHDim()];
3763 int64_t valueTileW = valueShape[getValueTileWDim()];
3764 WinogradConv2DFmr fmr = getFmr();
3765 int64_t m, r;
3766 std::tie(args&: m, args&: r) = getFmrFromWinogradConv2DFmr(fmr);
3767 bool leftTransform = valueH != 1;
3768 bool rightTransform = valueW != 1;
3769
3770 int64_t outputRank = getOutputOperandRank();
3771 SmallVector<int64_t> expectedOutputShape(outputRank, valueH);
3772 if (ShapedType::isDynamic(dValue: valueH) || ShapedType::isDynamic(dValue: valueTileH)) {
3773 expectedOutputShape[getOutputHDim()] = ShapedType::kDynamic;
3774 } else {
3775 if (valueH != (leftTransform ? m + r - 1 : 1))
3776 return emitOpError(message: "expect input height equals to input tile size");
3777 expectedOutputShape[getOutputHDim()] = (leftTransform ? m : 1) * valueTileH;
3778 }
3779 if (ShapedType::isDynamic(dValue: valueW) || ShapedType::isDynamic(dValue: valueTileW)) {
3780 expectedOutputShape[getOutputWDim()] = ShapedType::kDynamic;
3781 } else {
3782 if (valueW != (rightTransform ? m + r - 1 : 1))
3783 return emitOpError(message: "expect input width equals to input tile size");
3784 expectedOutputShape[getOutputWDim()] =
3785 (rightTransform ? m : 1) * valueTileW;
3786 }
3787 expectedOutputShape[getOutputNDim()] = valueShape[getValueNDim()];
3788 expectedOutputShape[getOutputFDim()] = valueShape[getValueFDim()];
3789
3790 auto outputType = cast<ShapedType>(Val: getOutput().getType());
3791 ArrayRef<int64_t> outputShape = outputType.getShape();
3792 if (failed(Result: verifyCompatibleShape(shape1: expectedOutputShape, shape2: outputShape))) {
3793 return emitOpError(message: "the output shape is not expected");
3794 }
3795 return success();
3796}
3797
3798SmallVector<Range>
3799WinogradOutputTransformOp::getIterationDomain(OpBuilder &builder) {
3800 Location loc = getLoc();
3801 IntegerAttr zeroAttr = builder.getIndexAttr(value: 0);
3802 IntegerAttr oneAttr = builder.getIndexAttr(value: 1);
3803 Value value = getValue();
3804 int64_t valueRank = getValueOperandRank();
3805 SmallVector<Range> loopBounds(valueRank);
3806 for (unsigned dim = 0; dim < valueRank; ++dim) {
3807 loopBounds[dim].offset = zeroAttr;
3808 // alphaH, alphaW, tileH, tileW, N, F
3809 loopBounds[dim].size = getDimValue(builder, loc, v: value, dim);
3810 loopBounds[dim].stride = oneAttr;
3811 }
3812 return loopBounds;
3813}
3814
3815SmallVector<utils::IteratorType>
3816WinogradOutputTransformOp::getLoopIteratorTypes() {
3817 int64_t valueRank = getValueOperandRank();
3818 SmallVector<utils::IteratorType> iteratorTypes(valueRank,
3819 utils::IteratorType::parallel);
3820 return iteratorTypes;
3821}
3822
3823LogicalResult WinogradOutputTransformOp::getResultTilePosition(
3824 OpBuilder &builder, unsigned resultNumber, ArrayRef<OpFoldResult> offsets,
3825 ArrayRef<OpFoldResult> sizes, SmallVector<OpFoldResult> &resultOffsets,
3826 SmallVector<OpFoldResult> &resultSizes) {
3827 WinogradConv2DFmr fmr = getFmr();
3828 int64_t m, r;
3829 std::tie(args&: m, args&: r) = getFmrFromWinogradConv2DFmr(fmr);
3830
3831 Location loc = getLoc();
3832 MLIRContext *context = builder.getContext();
3833 auto identityAffineMap =
3834 AffineMap::get(dimCount: 1, symbolCount: 0, results: {builder.getAffineDimExpr(position: 0)}, context);
3835 auto affineMap =
3836 AffineMap::get(dimCount: 1, symbolCount: 0, results: {builder.getAffineDimExpr(position: 0) * m}, context);
3837
3838 ShapedType valueType = getValueOperandType();
3839 ArrayRef<int64_t> valueShape = valueType.getShape();
3840 int64_t valueH = valueShape[0];
3841 int64_t valueW = valueShape[1];
3842 Value mappedOffsetH = affine::makeComposedAffineApply(
3843 b&: builder, loc, map: (valueH != 1 ? affineMap : identityAffineMap),
3844 operands: offsets[getValueTileHDim()]);
3845 Value mappedOffsetW = affine::makeComposedAffineApply(
3846 b&: builder, loc, map: (valueW != 1 ? affineMap : identityAffineMap),
3847 operands: offsets[getValueTileWDim()]);
3848 Value mappedSizeH = affine::makeComposedAffineApply(
3849 b&: builder, loc, map: affineMap, operands: sizes[getValueTileHDim()]);
3850 Value mappedSizeW = affine::makeComposedAffineApply(
3851 b&: builder, loc, map: affineMap, operands: sizes[getValueTileWDim()]);
3852
3853 IntegerAttr oneAttr = builder.getI64IntegerAttr(value: 1);
3854 OpFoldResult offsetH = OpFoldResult(mappedOffsetH);
3855 OpFoldResult offsetW = OpFoldResult(mappedOffsetW);
3856 OpFoldResult sizeH =
3857 valueH != 1 ? OpFoldResult(mappedSizeH) : OpFoldResult(oneAttr);
3858 OpFoldResult sizeW =
3859 valueW != 1 ? OpFoldResult(mappedSizeW) : OpFoldResult(oneAttr);
3860
3861 resultOffsets.append(
3862 IL: {offsets[getValueNDim()], offsetH, offsetW, offsets[getValueFDim()]});
3863 resultSizes.append(
3864 IL: {sizes[getValueNDim()], sizeH, sizeW, sizes[getValueFDim()]});
3865 return success();
3866}
3867
3868/// The inner tile alignment hint is only used by `linalg.pack` and
3869/// `linalg.unpack` operations. Therefore, this is forwarded to the hint-less
3870/// overload.
3871FailureOr<TilingResult> WinogradOutputTransformOp::getTiledImplementation(
3872 OpBuilder &builder, ArrayRef<OpFoldResult> offsets,
3873 ArrayRef<OpFoldResult> sizes, ArrayRef<InnerTileAlignment>) {
3874 return getTiledImplementation(b&: builder, offsets, sizes);
3875}
3876
3877/// Implement tiling for winograd_output_transform
3878/// The input of winograd_output_transform is (alphaH, alphaW, tileH, tileW, N,
3879/// F). The output of winograd_output_transform is (N, H, W, F) Users can
3880/// specify the tile sizes of tileH, tileW, N, and F. `offsets` are the values
3881/// for the offsets of tileH, tileW, N, F for one tile. `sizes` are the values
3882/// for the sizes of tileH, tileW, N, F for one tile.
3883FailureOr<TilingResult> WinogradOutputTransformOp::getTiledImplementation(
3884 OpBuilder &builder, ArrayRef<OpFoldResult> offsets,
3885 ArrayRef<OpFoldResult> sizes) {
3886 IntegerAttr oneAttr = builder.getI64IntegerAttr(value: 1);
3887 IntegerAttr zeroAttr = builder.getI64IntegerAttr(value: 0);
3888 Location loc = getLoc();
3889 SmallVector<Value> tiledOperands;
3890 SmallVector<OpFoldResult> sliceOffsets, sliceSizes;
3891
3892 ShapedType valueType = getValueOperandType();
3893 ArrayRef<int64_t> valueShape = valueType.getShape();
3894 int64_t alphaH = valueShape[getValueAlphaHDim()];
3895 int64_t alphaW = valueShape[getValueAlphaWDim()];
3896 IntegerAttr alphaHAttr = builder.getI64IntegerAttr(value: alphaH);
3897 IntegerAttr alphaWAttr = builder.getI64IntegerAttr(value: alphaW);
3898
3899 sliceOffsets.append(IL: {zeroAttr, zeroAttr, offsets[getValueTileHDim()],
3900 offsets[getValueTileWDim()], offsets[getValueNDim()],
3901 offsets[getValueFDim()]});
3902 sliceSizes.append(IL: {alphaHAttr, alphaWAttr, sizes[getValueTileHDim()],
3903 sizes[getValueTileWDim()], sizes[getValueNDim()],
3904 sizes[getValueFDim()]});
3905 int64_t valueRank = getValueOperandRank();
3906 SmallVector<OpFoldResult> sliceStrides(valueRank, oneAttr);
3907 auto valueSlice = tensor::ExtractSliceOp::create(
3908 builder, location: loc, source: getValue(), offsets: sliceOffsets, sizes: sliceSizes, strides: sliceStrides);
3909 tiledOperands.emplace_back(Args&: valueSlice);
3910
3911 SmallVector<OpFoldResult> resultOffsets, resultSizes;
3912 if (failed(Result: getResultTilePosition(builder, resultNumber: 1, offsets, sizes, resultOffsets,
3913 resultSizes)))
3914 return failure();
3915
3916 int64_t outputRank = getOutputOperandRank();
3917 SmallVector<OpFoldResult> strides(outputRank, oneAttr);
3918 auto outputSlice = tensor::ExtractSliceOp::create(
3919 builder, location: loc, source: getOutput(), offsets: resultOffsets, sizes: resultSizes, strides);
3920 tiledOperands.emplace_back(Args&: outputSlice);
3921
3922 SmallVector<Type> resultTypes;
3923 resultTypes.push_back(Elt: tiledOperands[1].getType());
3924 Operation *tiledOp =
3925 mlir::clone(b&: builder, op: getOperation(), newResultTypes: resultTypes, newOperands: tiledOperands);
3926
3927 return TilingResult{
3928 .tiledOps: {tiledOp},
3929 .tiledValues: SmallVector<Value>(tiledOp->getResults()),
3930 .generatedSlices: llvm::to_vector(Range: ArrayRef<Operation *>{valueSlice, outputSlice})};
3931}
3932
3933//===----------------------------------------------------------------------===//
3934// LinalgDialect
3935// TODO: Merge with the LinalgDialect block at the bottom
3936//===----------------------------------------------------------------------===//
3937
3938// Returns true if the result expression of `subMap` are a subset of `fullMap`.
3939static bool areResultExprsSubsetOf(AffineMap subMap, AffineMap fullMap) {
3940 auto explicitRange = subMap.getResults();
3941 auto defaultRange = fullMap.getResults();
3942 DenseSet<AffineExpr> explicitSet(explicitRange.begin(), explicitRange.end());
3943 DenseSet<AffineExpr> defaultSet(defaultRange.begin(), defaultRange.end());
3944 llvm::set_union(S1&: explicitSet, S2: defaultSet);
3945 return explicitSet == defaultSet;
3946}
3947
3948/// Check if the user defined map is valid broadcast map. Here broadcast
3949/// indexing maps are defined in context of corresponding default indexing maps
3950/// for the given Op. This way the check becomes very simple i.e just check the
3951/// number of result dims.
3952/// Returns true if the explictMap is broadcasted with respect to the
3953/// defaultMap.
3954static bool isBroadcasted(AffineMap explictMap, AffineMap defaultMap) {
3955 return explictMap.getNumResults() < defaultMap.getNumResults();
3956}
3957
3958/// Verifies the broadcast and transpose semantic sepecified by the explicit
3959/// indexing map for the MatmulOp \p op for each operand specified by \p
3960/// opIndex.
3961static LogicalResult verifyExtendedMatmulSemantic(MatmulOp matmulOp,
3962 unsigned opIndex) {
3963 SmallVector<AffineMap, 3> opIndexingMaps = matmulOp.getIndexingMapsArray();
3964 SmallVector<AffineMap, 3> defaultIndexingMaps =
3965 matmulOp.getDefaultIndexingMaps(context: matmulOp->getContext());
3966
3967 auto opIndexingMap = opIndexingMaps[opIndex];
3968 auto defaultIndexingMap = defaultIndexingMaps[opIndex];
3969 // Check general validity of indexing map results.
3970 if (!areResultExprsSubsetOf(subMap: opIndexingMap, fullMap: defaultIndexingMap))
3971 return matmulOp->emitOpError()
3972 << "Unexpected dim expression in map result.";
3973
3974 if (isBroadcasted(explictMap: opIndexingMap, defaultMap: defaultIndexingMap)) {
3975 if (!matmulOp.isValidLhsRhsBroadcastMap(bcastMap: opIndexingMap)) {
3976 return matmulOp->emitOpError()
3977 << "Invalid broadcast requested, should be (d2).";
3978 }
3979 return success();
3980 }
3981 return success();
3982}
3983
3984// Check general validity of input indexing map of
3985// BatchMatmulOp/BatchReduceMatmulOp.
3986template <typename OpTy>
3987static LogicalResult verifyInputMaps(OpTy batchVariantMatmulOp,
3988 AffineMap opIndexingMap,
3989 AffineMap defaultIndexingMap, bool isLHS) {
3990 assert((isa<BatchMatmulOp>(batchVariantMatmulOp) ||
3991 isa<BatchReduceMatmulOp>(batchVariantMatmulOp)) &&
3992 "Expected BatchMatmulOp or BatchReduceMatmulOp");
3993 // Check the result dims are valid.
3994 if (!areResultExprsSubsetOf(subMap: opIndexingMap, fullMap: defaultIndexingMap))
3995 return batchVariantMatmulOp->emitOpError()
3996 << "Unexpected result dim expression (outside the set of default "
3997 "result dims).";
3998
3999 // Check for valid number of result dims of input maps.
4000 if (opIndexingMap.getNumResults() > 3)
4001 return batchVariantMatmulOp->emitOpError()
4002 << "no. of result dim expressions exceeds 3.";
4003
4004 auto hasValidBatchDim = [](AffineMap map) {
4005 AffineExpr batchDim = map.getResult(idx: 0);
4006 return batchDim.isFunctionOfDim(position: 0);
4007 };
4008
4009 // Check if the requested broadcast is valid.
4010 if (isBroadcasted(explictMap: opIndexingMap, defaultMap: defaultIndexingMap)) {
4011 if (!batchVariantMatmulOp.isValidLhsRhsBroadcastMap(opIndexingMap, isLHS))
4012 return batchVariantMatmulOp->emitOpError()
4013 << "Invalid broadcast requested.";
4014 } else if (!hasValidBatchDim(opIndexingMap)) {
4015 return batchVariantMatmulOp->emitOpError()
4016 << "Invalid batch dimension expression.";
4017 }
4018 return success();
4019}
4020
4021/// This function checks if the given AffineMap for the output of a
4022/// BatchMatmulOp/BatchReduceMatmulOp has exactly the desired number of result
4023/// dimensions and if the output map result dimensions are valid.
4024template <typename OpTy>
4025static LogicalResult verifyOutputMap(OpTy batchVariantMatmulOp,
4026 AffineMap opIndexingMap) {
4027 assert((isa<BatchMatmulOp>(batchVariantMatmulOp) ||
4028 isa<BatchReduceMatmulOp>(batchVariantMatmulOp)) &&
4029 "Expected BatchMatmulOp or BatchReduceMatmulOp");
4030 if (isa<BatchMatmulOp>(batchVariantMatmulOp) &&
4031 opIndexingMap.getNumResults() != 3) {
4032
4033 return batchVariantMatmulOp->emitOpError()
4034 << "expects 3 dims, but got (" << opIndexingMap.getNumResults()
4035 << ").";
4036 }
4037 if (isa<BatchReduceMatmulOp>(batchVariantMatmulOp) &&
4038 opIndexingMap.getNumResults() != 2) {
4039 return batchVariantMatmulOp->emitOpError()
4040 << "expects 2 dims, but got (" << opIndexingMap.getNumResults()
4041 << ").";
4042 }
4043
4044 auto areValidOutputResultDim = [&](AffineMap outputMap) {
4045 return isa<BatchMatmulOp>(batchVariantMatmulOp)
4046 ? outputMap.getResult(idx: 0).isFunctionOfDim(position: 0) &&
4047 outputMap.getResult(idx: 1).isFunctionOfDim(position: 1) &&
4048 outputMap.getResult(idx: 2).isFunctionOfDim(position: 2)
4049 : outputMap.getResult(idx: 0).isFunctionOfDim(position: 1) &&
4050 outputMap.getResult(idx: 1).isFunctionOfDim(position: 2);
4051 };
4052
4053 if (!areValidOutputResultDim(opIndexingMap)) {
4054 return batchVariantMatmulOp->emitOpError()
4055 << "Invalid output map result dimension.";
4056 }
4057
4058 return success();
4059}
4060
4061/// Verifies the broadcast and transpose semantic specified by the explicit
4062/// indexing map for the BatchMatmulOp/BatchReduceMatmulOp op for each operand
4063/// specified by opIndex.
4064template <typename OpTy>
4065static LogicalResult
4066verifyExtendedBatchVariantMatmulSemantic(OpTy batchVariantMatmulOp,
4067 unsigned opIndex) {
4068 SmallVector<AffineMap, 3> opIndexingMaps =
4069 batchVariantMatmulOp.getIndexingMapsArray();
4070 SmallVector<AffineMap, 3> defaultIndexingMaps =
4071 batchVariantMatmulOp.getDefaultIndexingMaps(
4072 batchVariantMatmulOp->getContext());
4073
4074 if (opIndexingMaps.size() != 3)
4075 return batchVariantMatmulOp->emitOpError()
4076 << "Indexing_map attribute must have 3 affine maps.";
4077
4078 auto opIndexingMap = opIndexingMaps[opIndex];
4079 auto defaultIndexingMap = defaultIndexingMaps[opIndex];
4080
4081 if (opIndex == 2 &&
4082 failed(verifyOutputMap(batchVariantMatmulOp, opIndexingMap)))
4083 return failure();
4084
4085 if (opIndex != 2 &&
4086 failed(verifyInputMaps(batchVariantMatmulOp, opIndexingMap,
4087 defaultIndexingMap, opIndex == 0)))
4088 return failure();
4089
4090 return success();
4091}
4092
4093namespace mlir {
4094namespace linalg {
4095
4096std::optional<WinogradConv2DFmr> getWinogradConv2DFmr(int64_t m, int64_t r) {
4097 if (m == 2 && r == 3)
4098 return WinogradConv2DFmr::F_2_3;
4099 if (m == 4 && r == 3)
4100 return WinogradConv2DFmr::F_4_3;
4101 if (m == 2 && r == 5)
4102 return WinogradConv2DFmr::F_2_5;
4103 return std::nullopt;
4104}
4105
4106std::pair<int64_t, int64_t> getFmrFromWinogradConv2DFmr(WinogradConv2DFmr fmr) {
4107 switch (fmr) {
4108 case WinogradConv2DFmr::F_2_3:
4109 return {2, 3};
4110 case WinogradConv2DFmr::F_4_3:
4111 return {4, 3};
4112 case WinogradConv2DFmr::F_2_5:
4113 return {2, 5};
4114 }
4115 llvm_unreachable("Unkown WinogradConv2DFmr");
4116}
4117
4118//===----------------------------------------------------------------------===//
4119// MatMulOp
4120//===----------------------------------------------------------------------===//
4121
4122static FailureOr<SmallVector<SmallVector<int64_t>>>
4123getAffineResultPositions(ArrayAttr maps) {
4124 SmallVector<SmallVector<int64_t>> positions;
4125 for (auto map : maps) {
4126 AffineMapAttr attr = dyn_cast<AffineMapAttr>(Val&: map);
4127 if (!attr)
4128 return failure();
4129 SmallVector<int64_t> pos;
4130 for (auto result : attr.getAffineMap().getResults()) {
4131 auto dim = dyn_cast<AffineDimExpr>(Val&: result);
4132 if (!dim)
4133 return failure();
4134 pos.push_back(Elt: dim.getPosition());
4135 }
4136 positions.push_back(Elt: pos);
4137 }
4138 return positions;
4139}
4140
4141/// Returns a list of AffineMap with the typical matmul indexing charactristic.
4142SmallVector<AffineMap> MatmulOp::getDefaultIndexingMaps(MLIRContext *context) {
4143 AffineExpr d0, d1, d2;
4144 SmallVector<AffineMap> indexingMaps;
4145 bindDims(ctx: context, exprs&: d0, exprs&: d1, exprs&: d2);
4146 indexingMaps.push_back(Elt: AffineMap::get(dimCount: 3, symbolCount: 0, results: {d0, d2}, context));
4147 indexingMaps.push_back(Elt: AffineMap::get(dimCount: 3, symbolCount: 0, results: {d2, d1}, context));
4148 indexingMaps.push_back(Elt: AffineMap::get(dimCount: 3, symbolCount: 0, results: {d0, d1}, context));
4149 return indexingMaps;
4150}
4151
4152bool MatmulOp::isDefaultIndexingMaps(Attribute attr) {
4153 ArrayAttr maps = dyn_cast<ArrayAttr>(Val&: attr);
4154 if (!maps)
4155 return false;
4156 if (maps.size() != 3)
4157 return false;
4158 auto positions = getAffineResultPositions(maps);
4159 if (failed(Result: positions))
4160 return false;
4161 return (*positions)[0] == SmallVector<int64_t>{0, 2} &&
4162 (*positions)[1] == SmallVector<int64_t>{2, 1} &&
4163 (*positions)[2] == SmallVector<int64_t>{0, 1};
4164}
4165
4166SmallVector<utils::IteratorType> MatmulOp::getIteratorTypesArray() {
4167 return SmallVector<utils::IteratorType>{utils::IteratorType::parallel,
4168 utils::IteratorType::parallel,
4169 utils::IteratorType::reduction};
4170}
4171
4172unsigned MatmulOp::getNumRegionArgs() { return 3; }
4173
4174std::string MatmulOp::getLibraryCallName() {
4175 return generateLibraryCallName(op: getOperation());
4176}
4177
4178bool MatmulOp::hasDynamicIndexingMaps() { return true; }
4179
4180/// Check if the op has broadcast and/or transpose semantic. Returns true if
4181/// the user defined indexing maps are not equal to default map.
4182bool MatmulOp::hasUserDefinedMaps() {
4183 SmallVector<AffineMap, 3> defaultMaps =
4184 getDefaultIndexingMaps(context: this->getContext());
4185 SmallVector<AffineMap, 3> explicitMaps = getIndexingMapsArray();
4186 return defaultMaps != explicitMaps;
4187}
4188
4189/// Implements the block region builder for the MatmulOp. This is called by
4190/// 'fillStructuredOpRegion'.
4191void MatmulOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block,
4192 ArrayRef<NamedAttribute> attrs,
4193 function_ref<InFlightDiagnostic()> emitError) {
4194 if (emitError && block.getNumArguments() != 3) {
4195 emitError() << "MatmulOp regionBuilder expects 3 args, got "
4196 << block.getNumArguments();
4197 return;
4198 }
4199 assert(block.getNumArguments() == 3 &&
4200 "MatmulOp regionBuilder expects 3 args");
4201 RegionBuilderHelper helper(b, block);
4202 SmallVector<Value> yields;
4203
4204 TypeFn castVal = TypeFn::cast_signed;
4205 const auto *castIter = llvm::find_if(Range&: attrs, P: [&](const NamedAttribute &attr) {
4206 return attr.getName() == "cast";
4207 });
4208 if (castIter != attrs.end()) {
4209 if (auto attr = llvm::dyn_cast<TypeFnAttr>(Val: castIter->getValue()))
4210 castVal = attr.getValue();
4211 }
4212
4213 Value value1 = helper.buildTypeFn(typeFn: castVal, toType: block.getArgument(i: 2).getType(),
4214 operand: block.getArgument(i: 0));
4215 Value value2 = helper.buildTypeFn(typeFn: castVal, toType: block.getArgument(i: 2).getType(),
4216 operand: block.getArgument(i: 1));
4217 Value value3 = helper.buildBinaryFn(binaryFn: BinaryFn::mul, arg0: value1, arg1: value2, emitError);
4218 if (!value1 || !value2 || !value3)
4219 return;
4220 Value value4 = helper.buildBinaryFn(binaryFn: BinaryFn::add, arg0: block.getArgument(i: 2),
4221 arg1: value3, emitError);
4222 if (!value4)
4223 return;
4224 yields.push_back(Elt: value4);
4225 helper.yieldOutputs(values: yields);
4226}
4227
4228/// Returns true if the given bcastMap map is a valid broadcast map. A valid
4229/// broadcast map must include K dimension.
4230/// TODO: Strict inclusion of K dimension in the broadcast map is not
4231/// necessary for both input matrices simultaneously. We can relax this
4232/// condition to have K dimension for one input matrix map and infer the K
4233/// dimension for other input matrix map from the one already having K
4234/// dimension.
4235bool MatmulOp::isValidLhsRhsBroadcastMap(AffineMap bcastMap) {
4236 assert(bcastMap.getNumResults() == 1 && "Expected single result dim expr.");
4237 AffineExpr expr = bcastMap.getResult(idx: 0);
4238 // Invalid map if the common dimension of matmul not found.
4239 return expr.isFunctionOfDim(position: bcastMap.getNumDims() - 1);
4240}
4241
4242static FailureOr<ArrayAttr> parseIndexingMapsAttr(OpAsmParser &parser) {
4243 if (parser.parseOptionalKeyword(keyword: "indexing_maps"))
4244 return ArrayAttr{
4245 nullptr}; // Success in case indexing_maps was not provided.
4246
4247 ArrayAttr arrayAttr;
4248 if (parser.parseEqual() || parser.parseAttribute(result&: arrayAttr))
4249 return failure();
4250
4251 if (llvm::any_of(Range&: arrayAttr,
4252 P: [](auto elt) { return !dyn_cast<AffineMapAttr>(elt); }))
4253 return parser.emitError(loc: parser.getCurrentLocation())
4254 << "element of indexing_maps array is not an affine_map";
4255
4256 return arrayAttr;
4257}
4258
4259ParseResult MatmulOp::parse(OpAsmParser &parser, OperationState &result) {
4260 FailureOr<ArrayAttr> indexingMapsAttr = parseIndexingMapsAttr(parser);
4261 if (failed(Result: indexingMapsAttr))
4262 return failure();
4263
4264 if (*indexingMapsAttr == nullptr) {
4265 auto indexingMapAttrs = llvm::map_to_vector(
4266 C: MatmulOp::getDefaultIndexingMaps(context: parser.getContext()),
4267 F: [](AffineMap map) -> Attribute { return AffineMapAttr::get(value: map); });
4268 indexingMapsAttr = parser.getBuilder().getArrayAttr(value: indexingMapAttrs);
4269 }
4270
4271 result.addAttribute(name: "indexing_maps", attr: *indexingMapsAttr);
4272 return parseNamedStructuredOp(parser, result, numRegionArgs: MatmulOp::getNumRegionArgs(),
4273 regionBuilder: MatmulOp::getRegionBuilder());
4274}
4275
4276void MatmulOp::print(OpAsmPrinter &p) {
4277 SmallVector<Attribute, 3> indexingMaps = llvm::map_to_vector<3>(
4278 C: MatmulOp::getDefaultIndexingMaps(context: getContext()),
4279 F: [](AffineMap map) -> Attribute { return AffineMapAttr::get(value: map); });
4280 if (!llvm::equal(LRange: getIndexingMaps(), RRange&: indexingMaps))
4281 p << " indexing_maps = " << llvm::interleaved_array(R: getIndexingMaps());
4282
4283 std::array<StringRef, 3> elidedAttrs = {
4284 "operandSegmentSizes", "linalg.memoized_indexing_maps", "indexing_maps"};
4285 printNamedStructuredOp(p, op: getOperation(), inputs: getInputs(), outputs: getOutputs(),
4286 elidedAttrs);
4287}
4288
4289/// Verify the user defined indexing maps.
4290LogicalResult MatmulOp::verify() {
4291 // Verification of pure matmul is handled by verifyStructuredOpInterface().
4292 if (!hasUserDefinedMaps())
4293 return success();
4294
4295 for (unsigned opIndex = 0; opIndex < 2; opIndex++) {
4296 if (failed(Result: verifyExtendedMatmulSemantic(matmulOp: *this, opIndex)))
4297 return failure();
4298 }
4299 return success();
4300}
4301
4302LogicalResult MatmulOp::fold(FoldAdaptor, SmallVectorImpl<OpFoldResult> &) {
4303 return memref::foldMemRefCast(op: *this);
4304}
4305
4306void MatmulOp::getEffects(
4307 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
4308 &effects) {
4309 if (hasPureTensorSemantics())
4310 return;
4311 getGenericEffectsImpl(effects, linalgOp: cast<LinalgOp>(Val: getOperation()));
4312}
4313
4314Speculation::Speculatability MatmulOp::getSpeculatability() {
4315 return getGenericSpeculatabilityImpl(linalgOp: cast<LinalgOp>(Val: getOperation()));
4316}
4317
4318SmallVector<AffineMap>
4319MatmulTransposeAOp::getDefaultIndexingMaps(OpBuilder &builder) {
4320 AffineExpr d0, d1, d2;
4321 MLIRContext *context = builder.getContext();
4322 bindDims(ctx: context, exprs&: d0, exprs&: d1, exprs&: d2);
4323 AffineMap mapLHS = AffineMap::get(dimCount: 3, symbolCount: 0, results: {d2, d0}, context);
4324 AffineMap mapRHS = AffineMap::get(dimCount: 3, symbolCount: 0, results: {d2, d1}, context);
4325 AffineMap mapOut = AffineMap::get(dimCount: 3, symbolCount: 0, results: {d0, d1}, context);
4326 return {mapLHS, mapRHS, mapOut};
4327}
4328
4329bool MatmulTransposeAOp::isDefaultIndexingMaps(Attribute attr) {
4330 ArrayAttr maps = dyn_cast<ArrayAttr>(Val&: attr);
4331 if (!maps)
4332 return false;
4333 if (maps.size() != 3)
4334 return false;
4335 auto positions = getAffineResultPositions(maps);
4336 if (failed(Result: positions))
4337 return false;
4338 return (*positions)[0] == SmallVector<int64_t>{2, 0} &&
4339 (*positions)[1] == SmallVector<int64_t>{2, 1} &&
4340 (*positions)[2] == SmallVector<int64_t>{0, 1};
4341}
4342
4343void linalg::MatmulTransposeAOp::build(OpBuilder &builder,
4344 OperationState &result,
4345 ValueRange inputs, ValueRange outputs,
4346 ArrayRef<NamedAttribute> attributes) {
4347 buildMatmulOp(b&: builder, state&: result, resultTensorTypes: std::nullopt, inputs, outputs, attributes,
4348 regionBuilder: MatmulOp::getRegionBuilder(), defaultIndexingMaps: getDefaultIndexingMaps(builder));
4349}
4350
4351MatmulTransposeAOp
4352MatmulTransposeAOp::create(OpBuilder &builder, Location location,
4353 ValueRange inputs, ValueRange outputs,
4354 ArrayRef<NamedAttribute> attributes) {
4355 OperationState state(location, getOperationName());
4356 build(builder, result&: state, inputs, outputs, attributes);
4357 auto res = dyn_cast<MatmulTransposeAOp>(Val: builder.create(state));
4358 assert(res && "builder didn't return the right type");
4359 return res;
4360}
4361
4362void linalg::MatmulTransposeAOp::build(OpBuilder &builder,
4363 OperationState &result,
4364 TypeRange resultTensorTypes,
4365 ValueRange inputs, ValueRange outputs,
4366 ArrayRef<NamedAttribute> attributes) {
4367 buildMatmulOp(b&: builder, state&: result, resultTensorTypes, inputs, outputs, attributes,
4368 regionBuilder: MatmulOp::getRegionBuilder(), defaultIndexingMaps: getDefaultIndexingMaps(builder));
4369}
4370
4371MatmulTransposeAOp
4372MatmulTransposeAOp::create(OpBuilder &builder, Location location,
4373 TypeRange resultTensorTypes, ValueRange inputs,
4374 ValueRange outputs,
4375 ArrayRef<NamedAttribute> attributes) {
4376 OperationState state(location, getOperationName());
4377 build(builder, result&: state, resultTensorTypes, inputs, outputs, attributes);
4378 auto res = dyn_cast<MatmulTransposeAOp>(Val: builder.create(state));
4379 assert(res && "builder didn't return the right type");
4380 return res;
4381}
4382
4383void linalg::MatmulTransposeAOp::build(OpBuilder &builder,
4384 OperationState &result,
4385 TypeRange resultTensorTypes,
4386 ValueRange inputs, ValueRange outputs,
4387 Attribute cast,
4388 ArrayRef<NamedAttribute> attributes) {
4389 result.addAttribute(name: "cast", attr: cast);
4390 buildMatmulOp(b&: builder, state&: result, resultTensorTypes, inputs, outputs, attributes,
4391 regionBuilder: MatmulOp::getRegionBuilder(), defaultIndexingMaps: getDefaultIndexingMaps(builder));
4392}
4393
4394MatmulTransposeAOp
4395MatmulTransposeAOp::create(OpBuilder &builder, Location location,
4396 TypeRange resultTensorTypes, ValueRange inputs,
4397 ValueRange outputs, Attribute cast,
4398 ArrayRef<NamedAttribute> attributes) {
4399 OperationState state(location, getOperationName());
4400 build(builder, result&: state, resultTensorTypes, inputs, outputs, cast, attributes);
4401 auto res = dyn_cast<MatmulTransposeAOp>(Val: builder.create(state));
4402 assert(res && "builder didn't return the right type");
4403 return res;
4404}
4405
4406bool MatmulTransposeAOp::classof(Operation *op) {
4407 auto matmulOp = dyn_cast_or_null<linalg::MatmulOp>(Val: op);
4408 return matmulOp && MatmulTransposeAOp::isDefaultIndexingMaps(
4409 attr: matmulOp.getIndexingMapsAttr());
4410}
4411
4412SmallVector<AffineMap>
4413MatmulTransposeBOp::getDefaultIndexingMaps(OpBuilder &builder) {
4414 AffineExpr d0, d1, d2;
4415 MLIRContext *context = builder.getContext();
4416 bindDims(ctx: context, exprs&: d0, exprs&: d1, exprs&: d2);
4417 AffineMap mapLHS = AffineMap::get(dimCount: 3, symbolCount: 0, results: {d0, d2}, context);
4418 AffineMap mapRHS = AffineMap::get(dimCount: 3, symbolCount: 0, results: {d1, d2}, context);
4419 AffineMap mapOut = AffineMap::get(dimCount: 3, symbolCount: 0, results: {d0, d1}, context);
4420 return {mapLHS, mapRHS, mapOut};
4421}
4422
4423bool MatmulTransposeBOp::isDefaultIndexingMaps(Attribute attr) {
4424 ArrayAttr maps = dyn_cast<ArrayAttr>(Val&: attr);
4425 if (!maps)
4426 return false;
4427 if (maps.size() != 3)
4428 return false;
4429 auto positions = getAffineResultPositions(maps);
4430 if (failed(Result: positions))
4431 return false;
4432 return (*positions)[0] == SmallVector<int64_t>{0, 2} &&
4433 (*positions)[1] == SmallVector<int64_t>{1, 2} &&
4434 (*positions)[2] == SmallVector<int64_t>{0, 1};
4435}
4436
4437void linalg::MatmulTransposeBOp::build(OpBuilder &builder,
4438 OperationState &result,
4439 ValueRange inputs, ValueRange outputs,
4440 ArrayRef<NamedAttribute> attributes) {
4441 buildMatmulOp(b&: builder, state&: result, resultTensorTypes: std::nullopt, inputs, outputs, attributes,
4442 regionBuilder: MatmulOp::getRegionBuilder(), defaultIndexingMaps: getDefaultIndexingMaps(builder));
4443}
4444
4445MatmulTransposeBOp
4446MatmulTransposeBOp::create(OpBuilder &builder, Location location,
4447 ValueRange inputs, ValueRange outputs,
4448 ArrayRef<NamedAttribute> attributes) {
4449 OperationState state(location, getOperationName());
4450 build(builder, result&: state, inputs, outputs, attributes);
4451 auto res = dyn_cast<MatmulTransposeBOp>(Val: builder.create(state));
4452 assert(res && "builder didn't return the right type");
4453 return res;
4454}
4455
4456void linalg::MatmulTransposeBOp::build(OpBuilder &builder,
4457 OperationState &result,
4458 TypeRange resultTensorTypes,
4459 ValueRange inputs, ValueRange outputs,
4460 ArrayRef<NamedAttribute> attributes) {
4461 buildMatmulOp(b&: builder, state&: result, resultTensorTypes, inputs, outputs, attributes,
4462 regionBuilder: MatmulOp::getRegionBuilder(), defaultIndexingMaps: getDefaultIndexingMaps(builder));
4463}
4464
4465MatmulTransposeBOp
4466MatmulTransposeBOp::create(OpBuilder &builder, Location location,
4467 TypeRange resultTensorTypes, ValueRange inputs,
4468 ValueRange outputs,
4469 ArrayRef<NamedAttribute> attributes) {
4470 OperationState state(location, getOperationName());
4471 build(builder, result&: state, resultTensorTypes, inputs, outputs, attributes);
4472 auto res = dyn_cast<MatmulTransposeBOp>(Val: builder.create(state));
4473 assert(res && "builder didn't return the right type");
4474 return res;
4475}
4476
4477void linalg::MatmulTransposeBOp::build(OpBuilder &builder,
4478 OperationState &result,
4479 TypeRange resultTensorTypes,
4480 ValueRange inputs, ValueRange outputs,
4481 Attribute cast,
4482 ArrayRef<NamedAttribute> attributes) {
4483 result.addAttribute(name: "cast", attr: cast);
4484 buildMatmulOp(b&: builder, state&: result, resultTensorTypes, inputs, outputs, attributes,
4485 regionBuilder: MatmulOp::getRegionBuilder(), defaultIndexingMaps: getDefaultIndexingMaps(builder));
4486}
4487
4488MatmulTransposeBOp
4489MatmulTransposeBOp::create(OpBuilder &builder, Location location,
4490 TypeRange resultTensorTypes, ValueRange inputs,
4491 ValueRange outputs, Attribute cast,
4492 ArrayRef<NamedAttribute> attributes) {
4493 OperationState state(location, getOperationName());
4494 build(builder, result&: state, resultTensorTypes, inputs, outputs, cast, attributes);
4495 auto res = dyn_cast<MatmulTransposeBOp>(Val: builder.create(state));
4496 assert(res && "builder didn't return the right type");
4497 return res;
4498}
4499
4500bool MatmulTransposeBOp::classof(Operation *op) {
4501 auto matmulOp = dyn_cast_or_null<linalg::MatmulOp>(Val: op);
4502 return matmulOp && MatmulTransposeBOp::isDefaultIndexingMaps(
4503 attr: matmulOp.getIndexingMapsAttr());
4504}
4505
4506SmallVector<AffineMap>
4507BatchMatmulTransposeAOp::getDefaultIndexingMaps(OpBuilder &builder) {
4508 AffineExpr d0, d1, d2, d3;
4509 MLIRContext *context = builder.getContext();
4510 bindDims(ctx: context, exprs&: d0, exprs&: d1, exprs&: d2, exprs&: d3);
4511 AffineMap mapLHS = AffineMap::get(dimCount: 4, symbolCount: 0, results: {d0, d3, d1}, context);
4512 AffineMap mapRHS = AffineMap::get(dimCount: 4, symbolCount: 0, results: {d0, d3, d2}, context);
4513 AffineMap mapOut = AffineMap::get(dimCount: 4, symbolCount: 0, results: {d0, d1, d2}, context);
4514 return {mapLHS, mapRHS, mapOut};
4515}
4516
4517bool BatchMatmulTransposeAOp::isDefaultIndexingMaps(Attribute attr) {
4518 ArrayAttr maps = dyn_cast<ArrayAttr>(Val&: attr);
4519 if (!maps)
4520 return false;
4521 if (maps.size() != 3)
4522 return false;
4523 auto positions = getAffineResultPositions(maps);
4524 if (failed(Result: positions))
4525 return false;
4526 return (*positions)[0] == SmallVector<int64_t>{0, 3, 1} &&
4527 (*positions)[1] == SmallVector<int64_t>{0, 3, 2} &&
4528 (*positions)[2] == SmallVector<int64_t>{0, 1, 2};
4529}
4530
4531void linalg::BatchMatmulTransposeAOp::build(
4532 OpBuilder &builder, OperationState &result, ValueRange inputs,
4533 ValueRange outputs, ArrayRef<NamedAttribute> attributes) {
4534 buildMatmulOp(b&: builder, state&: result, resultTensorTypes: std::nullopt, inputs, outputs, attributes,
4535 regionBuilder: BatchMatmulOp::getRegionBuilder(),
4536 defaultIndexingMaps: getDefaultIndexingMaps(builder));
4537}
4538
4539BatchMatmulTransposeAOp
4540BatchMatmulTransposeAOp::create(OpBuilder &builder, Location location,
4541 ValueRange inputs, ValueRange outputs,
4542 ArrayRef<NamedAttribute> attributes) {
4543 OperationState state(location, getOperationName());
4544 build(builder, result&: state, inputs, outputs, attributes);
4545 auto res = dyn_cast<BatchMatmulTransposeAOp>(Val: builder.create(state));
4546 assert(res && "builder didn't return the right type");
4547 return res;
4548}
4549
4550void linalg::BatchMatmulTransposeAOp::build(
4551 OpBuilder &builder, OperationState &result, TypeRange resultTensorTypes,
4552 ValueRange inputs, ValueRange outputs,
4553 ArrayRef<NamedAttribute> attributes) {
4554 buildMatmulOp(b&: builder, state&: result, resultTensorTypes, inputs, outputs, attributes,
4555 regionBuilder: BatchMatmulOp::getRegionBuilder(),
4556 defaultIndexingMaps: getDefaultIndexingMaps(builder));
4557}
4558
4559BatchMatmulTransposeAOp
4560BatchMatmulTransposeAOp::create(OpBuilder &builder, Location location,
4561 TypeRange resultTensorTypes, ValueRange inputs,
4562 ValueRange outputs,
4563 ArrayRef<NamedAttribute> attributes) {
4564 OperationState state(location, getOperationName());
4565 build(builder, result&: state, resultTensorTypes, inputs, outputs, attributes);
4566 auto res = dyn_cast<BatchMatmulTransposeAOp>(Val: builder.create(state));
4567 assert(res && "builder didn't return the right type");
4568 return res;
4569}
4570
4571void linalg::BatchMatmulTransposeAOp::build(
4572 OpBuilder &builder, OperationState &result, TypeRange resultTensorTypes,
4573 ValueRange inputs, ValueRange outputs, Attribute cast,
4574 ArrayRef<NamedAttribute> attributes) {
4575 result.addAttribute(name: "cast", attr: cast);
4576 buildMatmulOp(b&: builder, state&: result, resultTensorTypes, inputs, outputs, attributes,
4577 regionBuilder: BatchMatmulOp::getRegionBuilder(),
4578 defaultIndexingMaps: getDefaultIndexingMaps(builder));
4579}
4580
4581BatchMatmulTransposeAOp
4582BatchMatmulTransposeAOp::create(OpBuilder &builder, Location location,
4583 TypeRange resultTensorTypes, ValueRange inputs,
4584 ValueRange outputs, Attribute cast,
4585 ArrayRef<NamedAttribute> attributes) {
4586 OperationState state(location, getOperationName());
4587 build(builder, result&: state, resultTensorTypes, inputs, outputs, cast, attributes);
4588 auto res = dyn_cast<BatchMatmulTransposeAOp>(Val: builder.create(state));
4589 assert(res && "builder didn't return the right type");
4590 return res;
4591}
4592
4593bool BatchMatmulTransposeAOp::classof(Operation *op) {
4594 auto matmulOp = dyn_cast_or_null<linalg::BatchMatmulOp>(Val: op);
4595 return matmulOp && BatchMatmulTransposeAOp::isDefaultIndexingMaps(
4596 attr: matmulOp.getIndexingMapsAttr());
4597}
4598
4599SmallVector<AffineMap>
4600BatchMatmulTransposeBOp::getDefaultIndexingMaps(OpBuilder &builder) {
4601 AffineExpr d0, d1, d2, d3;
4602 MLIRContext *context = builder.getContext();
4603 bindDims(ctx: context, exprs&: d0, exprs&: d1, exprs&: d2, exprs&: d3);
4604 AffineMap mapLHS = AffineMap::get(dimCount: 4, symbolCount: 0, results: {d0, d1, d3}, context);
4605 AffineMap mapRHS = AffineMap::get(dimCount: 4, symbolCount: 0, results: {d0, d2, d3}, context);
4606 AffineMap mapOut = AffineMap::get(dimCount: 4, symbolCount: 0, results: {d0, d1, d2}, context);
4607 return {mapLHS, mapRHS, mapOut};
4608}
4609
4610bool BatchMatmulTransposeBOp::isDefaultIndexingMaps(Attribute attr) {
4611 ArrayAttr maps = dyn_cast<ArrayAttr>(Val&: attr);
4612 if (!maps)
4613 return false;
4614 if (maps.size() != 3)
4615 return false;
4616 auto positions = getAffineResultPositions(maps);
4617 if (failed(Result: positions))
4618 return false;
4619 return (*positions)[0] == SmallVector<int64_t>{0, 1, 3} &&
4620 (*positions)[1] == SmallVector<int64_t>{0, 2, 3} &&
4621 (*positions)[2] == SmallVector<int64_t>{0, 1, 2};
4622}
4623
4624void linalg::BatchMatmulTransposeBOp::build(
4625 OpBuilder &builder, OperationState &result, ValueRange inputs,
4626 ValueRange outputs, ArrayRef<NamedAttribute> attributes) {
4627 buildMatmulOp(b&: builder, state&: result, resultTensorTypes: std::nullopt, inputs, outputs, attributes,
4628 regionBuilder: BatchMatmulOp::getRegionBuilder(),
4629 defaultIndexingMaps: getDefaultIndexingMaps(builder));
4630}
4631
4632BatchMatmulTransposeBOp
4633BatchMatmulTransposeBOp::create(OpBuilder &builder, Location location,
4634 ValueRange inputs, ValueRange outputs,
4635 ArrayRef<NamedAttribute> attributes) {
4636 OperationState state(location, getOperationName());
4637 build(builder, result&: state, inputs, outputs, attributes);
4638 auto res = dyn_cast<BatchMatmulTransposeBOp>(Val: builder.create(state));
4639 assert(res && "builder didn't return the right type");
4640 return res;
4641}
4642
4643void linalg::BatchMatmulTransposeBOp::build(
4644 OpBuilder &builder, OperationState &result, TypeRange resultTensorTypes,
4645 ValueRange inputs, ValueRange outputs,
4646 ArrayRef<NamedAttribute> attributes) {
4647 buildMatmulOp(b&: builder, state&: result, resultTensorTypes, inputs, outputs, attributes,
4648 regionBuilder: BatchMatmulOp::getRegionBuilder(),
4649 defaultIndexingMaps: getDefaultIndexingMaps(builder));
4650}
4651
4652BatchMatmulTransposeBOp
4653BatchMatmulTransposeBOp::create(OpBuilder &builder, Location location,
4654 TypeRange resultTensorTypes, ValueRange inputs,
4655 ValueRange outputs,
4656 ArrayRef<NamedAttribute> attributes) {
4657 OperationState state(location, getOperationName());
4658 build(builder, result&: state, resultTensorTypes, inputs, outputs, attributes);
4659 auto res = dyn_cast<BatchMatmulTransposeBOp>(Val: builder.create(state));
4660 assert(res && "builder didn't return the right type");
4661 return res;
4662}
4663
4664void linalg::BatchMatmulTransposeBOp::build(
4665 OpBuilder &builder, OperationState &result, TypeRange resultTensorTypes,
4666 ValueRange inputs, ValueRange outputs, Attribute cast,
4667 ArrayRef<NamedAttribute> attributes) {
4668 result.addAttribute(name: "cast", attr: cast);
4669 buildMatmulOp(b&: builder, state&: result, resultTensorTypes, inputs, outputs, attributes,
4670 regionBuilder: BatchMatmulOp::getRegionBuilder(),
4671 defaultIndexingMaps: getDefaultIndexingMaps(builder));
4672}
4673
4674BatchMatmulTransposeBOp
4675BatchMatmulTransposeBOp::create(OpBuilder &builder, Location location,
4676 TypeRange resultTensorTypes, ValueRange inputs,
4677 ValueRange outputs, Attribute cast,
4678 ArrayRef<NamedAttribute> attributes) {
4679 OperationState state(location, getOperationName());
4680 build(builder, result&: state, resultTensorTypes, inputs, outputs, cast, attributes);
4681 auto res = dyn_cast<BatchMatmulTransposeBOp>(Val: builder.create(state));
4682 assert(res && "builder didn't return the right type");
4683 return res;
4684}
4685
4686bool BatchMatmulTransposeBOp::classof(Operation *op) {
4687 auto matmulOp = dyn_cast_or_null<linalg::BatchMatmulOp>(Val: op);
4688 return matmulOp && BatchMatmulTransposeBOp::isDefaultIndexingMaps(
4689 attr: matmulOp.getIndexingMapsAttr());
4690}
4691
4692//===----------------------------------------------------------------------===//
4693// ContractOp
4694//===----------------------------------------------------------------------===//
4695
4696SmallVector<utils::IteratorType> ContractOp::getIteratorTypesArray() {
4697 AffineMap outAffineMap = getIndexingMapsArray().pop_back_val();
4698 // On well-formed IR, indexing_maps is non-empty, contained affine_maps'
4699 // domains are all the same, and each implements a projected permutation.
4700 // Each iteration space dim must occur for at least one operand and either
4701 // takes part in a contraction/reduction or else has parallel iteration type.
4702 // We have that a dim is a contraction/reduction dim if and only if the dim
4703 // occurs for the output operand. We use this fact for fast inference:
4704 // NB: In case we allow dims to occur solely for one input, the above still
4705 // holds: per the einsum semantics, these are reduction dims as well.
4706 SmallVector<bool> dimsInOutput(outAffineMap.getNumDims(), false);
4707 for (auto result : outAffineMap.getResults()) {
4708 auto dimExpr = dyn_cast<AffineDimExpr>(Val&: result);
4709 assert(dimExpr && "affine_map is a projected permutation");
4710 dimsInOutput[dimExpr.getPosition()] = true;
4711 }
4712
4713 SmallVector<utils::IteratorType> iteratorTypes;
4714 for (auto dimOccursInOutput : dimsInOutput)
4715 iteratorTypes.push_back(Elt: dimOccursInOutput ? utils::IteratorType::parallel
4716 : utils::IteratorType::reduction);
4717
4718 return iteratorTypes;
4719}
4720
4721unsigned ContractOp::getNumRegionArgs() { return 3; }
4722
4723/// Implement block region builder, which is called by 'fillStructuredOpRegion'.
4724void ContractOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block,
4725 ArrayRef<NamedAttribute> attrs,
4726 function_ref<InFlightDiagnostic()> emitError) {
4727 if (emitError && block.getNumArguments() != 3) {
4728 emitError() << "ContractOp regionBuilder expects 3 args, got "
4729 << block.getNumArguments();
4730 return;
4731 }
4732 assert(block.getNumArguments() == 3 &&
4733 "ContractOp regionBuilder expects 3 args");
4734 RegionBuilderHelper helper(b, block);
4735
4736 TypeFn castSignedness = TypeFn::cast_signed;
4737 auto castIter = llvm::find_if(Range&: attrs, P: [&](const NamedAttribute &attr) {
4738 return attr.getName() == "cast";
4739 });
4740 if (castIter != attrs.end()) {
4741 if (auto attr = llvm::dyn_cast<TypeFnAttr>(Val: castIter->getValue()))
4742 castSignedness = attr.getValue();
4743 }
4744
4745 // TODO: Support fields with operators besides mult & add.
4746 Type outType = block.getArgument(i: 2).getType();
4747 Value lhsAtOutType =
4748 helper.buildTypeFn(typeFn: castSignedness, toType: outType, operand: block.getArgument(i: 0));
4749 Value rhsAtOutType =
4750 helper.buildTypeFn(typeFn: castSignedness, toType: outType, operand: block.getArgument(i: 1));
4751 Value productAtOutType = helper.buildBinaryFn(binaryFn: BinaryFn::mul, arg0: lhsAtOutType,
4752 arg1: rhsAtOutType, emitError);
4753 if (!productAtOutType)
4754 return;
4755 Value result = helper.buildBinaryFn(binaryFn: BinaryFn::add, arg0: block.getArgument(i: 2),
4756 arg1: productAtOutType, emitError);
4757 if (!result)
4758 return;
4759 helper.yieldOutputs(values: {result});
4760}
4761
4762ParseResult ContractOp::parse(OpAsmParser &parser, OperationState &result) {
4763 FailureOr<ArrayAttr> indexingMapsAttr = parseIndexingMapsAttr(parser);
4764 if (failed(Result: indexingMapsAttr) || *indexingMapsAttr == nullptr)
4765 return parser.emitError(loc: parser.getCurrentLocation(),
4766 message: "expected 'indexing_maps' attribute");
4767 result.addAttribute(name: "indexing_maps", attr: *indexingMapsAttr);
4768
4769 return parseNamedStructuredOp(parser, result, numRegionArgs: getNumRegionArgs(),
4770 regionBuilder);
4771}
4772
4773void ContractOp::print(OpAsmPrinter &p) {
4774 p << " indexing_maps = " << llvm::interleaved_array(R: getIndexingMaps());
4775 printNamedStructuredOp(
4776 p, op: getOperation(), inputs: getInputs(), outputs: getOutputs(),
4777 /*elidedAttrs=*/{"indexing_maps", "operandSegmentSizes"});
4778}
4779
4780/// Validate contraction operands indexing maps and shapes.
4781/// For a given affine_map and type, checks that:
4782/// - the affine_map is a projected permutation;
4783/// - the rank of the affine_map's results and the corresponding type match;
4784/// - the rank of the affine_map's domain is consistent with prior maps.
4785/// Also updates the per-dim input/output occurrence counts.
4786static LogicalResult
4787checkContractionAffineMapAndType(AffineMap affineMap, Type operandType,
4788 bool isInput, int &iterationSpaceDims,
4789 SmallVector<size_t> &inOccurrences,
4790 SmallVector<size_t> &outOccurrences,
4791 function_ref<InFlightDiagnostic()> emitError) {
4792 if (!affineMap.isProjectedPermutation())
4793 return emitError() << "provided affine_map is not a projected permutation";
4794
4795 if (auto shapedType = dyn_cast<ShapedType>(Val&: operandType)) {
4796 if (affineMap.getNumResults() != shapedType.getRank())
4797 return emitError()
4798 << "ranks of shaped operand and results of corresponding "
4799 "affine_map differ";
4800 } else if (affineMap.getNumResults() != 0) {
4801 return emitError()
4802 << "affine_map specifies shaped access while operand has "
4803 "non-shaped type";
4804 }
4805
4806 if (iterationSpaceDims == -1) {
4807 iterationSpaceDims = affineMap.getNumDims();
4808 inOccurrences = SmallVector<size_t>(iterationSpaceDims, 0);
4809 outOccurrences = SmallVector<size_t>(iterationSpaceDims, 0);
4810 } else if (iterationSpaceDims != (int)affineMap.getNumDims()) {
4811 return emitError() << "iteration spaces of provided affine_maps differ";
4812 }
4813
4814 // Update counts of dims used to access either an input or the output.
4815 for (AffineExpr affineExpr : affineMap.getResults()) {
4816 auto affineDimExpr = dyn_cast<AffineDimExpr>(Val&: affineExpr);
4817 if (!affineDimExpr)
4818 llvm_unreachable("affine_map is a projected permutation");
4819
4820 if (isInput)
4821 inOccurrences[affineDimExpr.getPosition()] += 1;
4822 else
4823 outOccurrences[affineDimExpr.getPosition()] += 1;
4824 }
4825
4826 return success();
4827}
4828
4829/// Validates the contracting dimension constraints given the per-dim
4830/// occurrence counts. Checks that:
4831/// - every iteration-space dimension is used by at least one operand;
4832/// - every dimension is either contracting (appears in both inputs, not in
4833/// output) or parallel (appears in exactly one input and in the output);
4834/// - at least one contracting dimension exists.
4835static LogicalResult
4836verifyContractionDims(size_t iterationSpaceDims, ArrayRef<size_t> inOccurrences,
4837 ArrayRef<size_t> outOccurrences,
4838 function_ref<InFlightDiagnostic()> emitError) {
4839 bool hasContractingDim = false;
4840 for (size_t dimIndex = 0; dimIndex < iterationSpaceDims; dimIndex++) {
4841 size_t inOccCount = inOccurrences[dimIndex];
4842 size_t outOccCount = outOccurrences[dimIndex];
4843
4844 // We have a contracting dim if and only if ...
4845 hasContractingDim |= inOccCount == 2 && outOccCount == 0;
4846
4847 if (inOccCount == 0 && outOccCount == 0)
4848 return emitError() << "iteration space dim at index " << dimIndex
4849 << " not used to access any operand";
4850
4851 // NB: We disallow a dim which occurs for only one input operand and not
4852 // for the output. In terms of einsum semantics such dims have a
4853 // sensible meaning - namely an additional reduction per each such dim.
4854 // By contrast, the ContractionOpInterface does not know about this
4855 // iter type - cf. inferContractionDims' supported dim kinds.
4856 // TODO: Remove following once we have comprehensive support for input-only
4857 // reduction dims, at both the linalg- and vector-dialect levels.
4858 if (inOccCount == 1 && outOccCount != 1)
4859 return emitError() << "reduction dim at index " << dimIndex
4860 << " must appear in both LHS and RHS";
4861 }
4862
4863 if (!hasContractingDim)
4864 return emitError()
4865 << "'indexing_maps' do not specify a contracting dimension";
4866
4867 return success();
4868}
4869
4870LogicalResult ContractOp::verify() {
4871 int iterationSpaceDims = -1;
4872 // Map iter space dims to #occurrences in inputs' and output's affine_maps:
4873 // e.g., inOccurrences[0] will hold #times that dim (with index) 0 is used to
4874 // access an input operand (so occurrence count can be at most 2) and
4875 // outOccurrences[1] will indicate whether dim 1 occurred in the output, etc.
4876 SmallVector<size_t> inOccurrences;
4877 SmallVector<size_t> outOccurrences;
4878
4879 for (auto &&[affineMap, operandType, isInput] :
4880 llvm::zip(t: getIndexingMapsArray(), u: getOperandTypes(),
4881 args: SmallVector<bool>{true, true, false})) {
4882 if (failed(Result: checkContractionAffineMapAndType(
4883 affineMap, operandType, isInput, iterationSpaceDims, inOccurrences,
4884 outOccurrences, emitError: [&]() { return emitError(); })))
4885 return failure(); // NB: Validation helper emits relevant error.
4886 }
4887
4888 return verifyContractionDims(iterationSpaceDims: static_cast<size_t>(iterationSpaceDims),
4889 inOccurrences, outOccurrences,
4890 emitError: [&]() { return emitError(); });
4891}
4892
4893LogicalResult ContractOp::fold(FoldAdaptor, SmallVectorImpl<OpFoldResult> &) {
4894 return memref::foldMemRefCast(op: *this);
4895}
4896
4897void ContractOp::getEffects(
4898 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
4899 &effects) {
4900 if (hasPureTensorSemantics())
4901 return;
4902 getGenericEffectsImpl(effects, linalgOp: cast<LinalgOp>(Val: getOperation()));
4903}
4904
4905Speculation::Speculatability ContractOp::getSpeculatability() {
4906 return getGenericSpeculatabilityImpl(linalgOp: cast<LinalgOp>(Val: getOperation()));
4907}
4908
4909//===----------------------------------------------------------------------===//
4910// Implementation of BatchMatmulOp
4911//===----------------------------------------------------------------------===//
4912SmallVector<AffineMap>
4913BatchMatmulOp::getDefaultIndexingMaps(MLIRContext *context) {
4914 AffineExpr d0, d1, d2, d3;
4915 SmallVector<AffineMap> indexingMaps;
4916 bindDims(ctx: context, exprs&: d0, exprs&: d1, exprs&: d2, exprs&: d3);
4917 indexingMaps.push_back(Elt: AffineMap::get(dimCount: 4, symbolCount: 0, results: {d0, d1, d3}, context));
4918 indexingMaps.push_back(Elt: AffineMap::get(dimCount: 4, symbolCount: 0, results: {d0, d3, d2}, context));
4919 indexingMaps.push_back(Elt: AffineMap::get(dimCount: 4, symbolCount: 0, results: {d0, d1, d2}, context));
4920 return indexingMaps;
4921}
4922
4923bool BatchMatmulOp::isDefaultIndexingMaps(Attribute attr) {
4924 ArrayAttr maps = dyn_cast<ArrayAttr>(Val&: attr);
4925 if (!maps)
4926 return false;
4927 if (maps.size() != 3)
4928 return false;
4929 auto positions = getAffineResultPositions(maps);
4930 if (failed(Result: positions))
4931 return false;
4932 return (*positions)[0] == SmallVector<int64_t>{0, 1, 3} &&
4933 (*positions)[1] == SmallVector<int64_t>{0, 3, 2} &&
4934 (*positions)[2] == SmallVector<int64_t>{0, 1, 2};
4935}
4936
4937SmallVector<utils::IteratorType> BatchMatmulOp::getIteratorTypesArray() {
4938 return SmallVector<utils::IteratorType>{
4939 utils::IteratorType::parallel, utils::IteratorType::parallel,
4940 utils::IteratorType::parallel, utils::IteratorType::reduction};
4941}
4942
4943unsigned BatchMatmulOp::getNumRegionArgs() { return 3; }
4944
4945std::string BatchMatmulOp::getLibraryCallName() {
4946 return generateLibraryCallName(op: getOperation());
4947}
4948
4949/// Check if the op has broadcast and/or transpose semantic. Returns true if
4950/// the user defined indexing maps are not equal to default map.
4951bool BatchMatmulOp::hasUserDefinedMaps() {
4952 SmallVector<AffineMap, 3> defaultMaps =
4953 getDefaultIndexingMaps(context: this->getContext());
4954 SmallVector<AffineMap, 3> explicitMaps = getIndexingMapsArray();
4955 return defaultMaps != explicitMaps;
4956}
4957
4958/// Returns true if the given bcastMap map is a valid broadcast map. A valid
4959/// broadcast map must include K dimension.
4960/// TODO: Strict inclusion of K dimension in the broadcast map is not
4961/// necessary for both input matrices simultaneously. We can relax this
4962/// condition to have K dimension for one input matrix map and infer the K
4963/// dimension for other input matrix map from the one already having K
4964/// dimension.
4965bool BatchMatmulOp::isValidLhsRhsBroadcastMap(AffineMap bcastMap, bool isLHS) {
4966 assert(bcastMap.getNumResults() < 3 &&
4967 "Expected less than 3 result dim expr.");
4968 bool isValid = false;
4969 enum Indices { batchPos, mPos, nPos, kPos };
4970 if (bcastMap.getNumResults() == 1) {
4971 AffineExpr expr = bcastMap.getResult(idx: 0);
4972 isValid = expr.isFunctionOfDim(position: kPos);
4973 } else if (bcastMap.getNumResults() == 2) {
4974 AffineExpr expr0 = bcastMap.getResult(idx: 0);
4975 AffineExpr expr1 = bcastMap.getResult(idx: 1);
4976 isValid =
4977 isLHS ? ((expr0.isFunctionOfDim(position: batchPos) ||
4978 expr0.isFunctionOfDim(position: mPos)) &&
4979 expr1.isFunctionOfDim(position: kPos))
4980 : ((expr0.isFunctionOfDim(position: batchPos) &&
4981 expr1.isFunctionOfDim(position: kPos)) ||
4982 (expr0.isFunctionOfDim(position: kPos) && expr1.isFunctionOfDim(position: nPos)));
4983 }
4984 return isValid;
4985}
4986
4987void BatchMatmulOp::regionBuilder(
4988 ImplicitLocOpBuilder &b, Block &block, ArrayRef<NamedAttribute> attrs,
4989 function_ref<InFlightDiagnostic()> emitError) {
4990 if (emitError && block.getNumArguments() != 3) {
4991 emitError() << "BatchMatmulOp regionBuilder expects 3 args, got "
4992 << block.getNumArguments();
4993 return;
4994 }
4995 assert(block.getNumArguments() == 3 &&
4996 "BatchMatmulOp regionBuilder expects 3 args");
4997 RegionBuilderHelper helper(b, block);
4998 SmallVector<Value> yields;
4999
5000 TypeFn castVal = TypeFn::cast_signed;
5001 auto castIter = llvm::find_if(Range&: attrs, P: [&](const NamedAttribute &attr) {
5002 return attr.getName() == "cast";
5003 });
5004 if (castIter != attrs.end()) {
5005 if (auto attr = llvm::dyn_cast<TypeFnAttr>(Val: castIter->getValue()))
5006 castVal = attr.getValue();
5007 }
5008
5009 auto toType = block.getArgument(i: 2).getType();
5010 Value castValA = helper.buildTypeFn(typeFn: castVal, toType, operand: block.getArgument(i: 0));
5011 Value castValB = helper.buildTypeFn(typeFn: castVal, toType, operand: block.getArgument(i: 1));
5012 Value mulVal =
5013 helper.buildBinaryFn(binaryFn: BinaryFn::mul, arg0: castValA, arg1: castValB, emitError);
5014 if (!castValA || !castValB || !mulVal)
5015 return;
5016 Value addVal = helper.buildBinaryFn(binaryFn: BinaryFn::add, arg0: block.getArgument(i: 2),
5017 arg1: mulVal, emitError);
5018 if (!addVal)
5019 return;
5020 yields.push_back(Elt: addVal);
5021 helper.yieldOutputs(values: yields);
5022}
5023
5024ParseResult BatchMatmulOp::parse(OpAsmParser &parser, OperationState &result) {
5025 SmallVector<Attribute, 3> indexingMapsAttr;
5026 Attribute mapAttr;
5027 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "indexing_maps"))) {
5028 if (parser.parseEqual())
5029 return failure();
5030
5031 if (parser.parseLSquare())
5032 return failure();
5033
5034 do {
5035 if (parser.parseAttribute(result&: mapAttr))
5036 return failure();
5037 if (!isa<AffineMapAttr>(Val: mapAttr)) {
5038 return parser.emitError(loc: parser.getCurrentLocation(),
5039 message: "expected affine map attribute");
5040 }
5041 indexingMapsAttr.push_back(Elt: mapAttr);
5042
5043 if (parser.parseOptionalComma())
5044 break;
5045 } while (true);
5046
5047 if (parser.parseRSquare())
5048 return failure();
5049 }
5050 // Initialize indexingMaps, if not supplied explicitly.
5051 if (indexingMapsAttr.empty()) {
5052 indexingMapsAttr = llvm::map_to_vector(
5053 C: BatchMatmulOp::getDefaultIndexingMaps(context: parser.getContext()),
5054 F: [](AffineMap map) -> Attribute { return AffineMapAttr::get(value: map); });
5055 }
5056 result.addAttribute(name: "indexing_maps",
5057 attr: parser.getBuilder().getArrayAttr(value: indexingMapsAttr));
5058
5059 return ::parseNamedStructuredOp(parser, result,
5060 numRegionArgs: BatchMatmulOp::getNumRegionArgs(),
5061 regionBuilder: BatchMatmulOp::getRegionBuilder());
5062}
5063
5064void BatchMatmulOp::print(OpAsmPrinter &p) {
5065 SmallVector<Attribute, 3> indexingMaps = llvm::map_to_vector<3>(
5066 C: BatchMatmulOp::getDefaultIndexingMaps(context: getContext()),
5067 F: [](AffineMap map) -> Attribute { return AffineMapAttr::get(value: map); });
5068 if (!llvm::equal(LRange: getIndexingMaps(), RRange&: indexingMaps))
5069 p << " indexing_maps = " << llvm::interleaved_array(R: getIndexingMaps());
5070
5071 std::array<StringRef, 3> elidedAttrs = {
5072 "operandSegmentSizes", "linalg.memoized_indexing_maps", "indexing_maps"};
5073 ::printNamedStructuredOp(p, op: getOperation(), inputs: getInputs(), outputs: getOutputs(),
5074 elidedAttrs);
5075}
5076
5077/// Verify the user defined indexing maps.
5078LogicalResult BatchMatmulOp::verify() {
5079 // Verification of pure batch_matmul is handled by
5080 // verifyStructuredOpInterface().
5081 if (!hasUserDefinedMaps())
5082 return success();
5083
5084 for (unsigned opIndex = 0; opIndex < 3; opIndex++) {
5085 if (failed(Result: verifyExtendedBatchVariantMatmulSemantic(batchVariantMatmulOp: *this, opIndex)))
5086 return failure();
5087 }
5088 return success();
5089}
5090
5091LogicalResult BatchMatmulOp::fold(FoldAdaptor,
5092 SmallVectorImpl<OpFoldResult> &) {
5093 return memref::foldMemRefCast(op: *this);
5094}
5095
5096void BatchMatmulOp::getEffects(
5097 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
5098 &effects) {
5099 if (hasPureTensorSemantics())
5100 return;
5101 getGenericEffectsImpl(effects, linalgOp: cast<LinalgOp>(Val: getOperation()));
5102}
5103
5104Speculation::Speculatability BatchMatmulOp::getSpeculatability() {
5105 return getGenericSpeculatabilityImpl(linalgOp: cast<LinalgOp>(Val: getOperation()));
5106}
5107
5108//===----------------------------------------------------------------------===//
5109// ElementwiseOp
5110//===----------------------------------------------------------------------===//
5111//
5112namespace {
5113unsigned getArityGroupAsUInt(ElementwiseArityGroup arityGroup) {
5114 return static_cast<unsigned>(arityGroup);
5115}
5116} // namespace
5117
5118ArityGroupAndKind getArityGroupAndKind(ElementwiseKind kind) {
5119 constexpr int lastUnary = static_cast<int>(ElementwiseCaseLimits::LastUnary);
5120 constexpr int lastBinary =
5121 static_cast<int>(ElementwiseCaseLimits::LastBinary);
5122 constexpr int lastTernary =
5123 static_cast<int>(ElementwiseCaseLimits::LastTernary);
5124
5125 int val = static_cast<int>(kind);
5126 ArityGroupAndKind result;
5127
5128 if (val < lastUnary) {
5129 result.arityGroup = ElementwiseArityGroup::Unary;
5130 result.kind.unaryFn = static_cast<UnaryFn>(val);
5131 return result;
5132 }
5133 if (val < lastBinary) {
5134 result.arityGroup = ElementwiseArityGroup::Binary;
5135 result.kind.binaryFn = static_cast<BinaryFn>(val - lastUnary);
5136 return result;
5137 }
5138 if (val >= lastTernary) {
5139 llvm_unreachable("unhandled ElementwiseFn");
5140 }
5141 result.arityGroup = ElementwiseArityGroup::Ternary;
5142 result.kind.ternaryFn = static_cast<TernaryFn>(val - lastBinary);
5143 return result;
5144}
5145
5146SmallVector<utils::IteratorType> ElementwiseOp::getIteratorTypesArray() {
5147 auto rank = getResultRank();
5148 return SmallVector<utils::IteratorType>(rank, utils::IteratorType::parallel);
5149}
5150
5151SmallVector<AffineMap>
5152ElementwiseOp::getDefaultIndexingMaps(unsigned numMaps, unsigned numDims,
5153 MLIRContext *context) {
5154 auto map = AffineMap::getMultiDimIdentityMap(numDims, context);
5155 return SmallVector<AffineMap>(numMaps, map);
5156}
5157
5158ParseResult ElementwiseOp::parse(OpAsmParser &parser, OperationState &result) {
5159 // Expect e.g. `<add>` (also accepts the full
5160 // `#linalg.elementwise_kind<add>`).
5161 ElementwiseKindAttr kindAttr;
5162 mlir::linalg::ElementwiseKind elemwiseKindVal;
5163 if (parser.parseCustomAttributeWithFallback(result&: kindAttr))
5164 return failure();
5165 elemwiseKindVal = kindAttr.getValue();
5166 result.addAttribute(name: "kind", attr: kindAttr);
5167
5168 // Parse optional `indexing_maps`
5169 SmallVector<Attribute, 3> indexingMapsAttr;
5170 Attribute mapAttr;
5171 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "indexing_maps"))) {
5172 if (parser.parseEqual())
5173 return failure();
5174 if (parser.parseLSquare())
5175 return failure();
5176 do {
5177 if (parser.parseAttribute(result&: mapAttr))
5178 return failure();
5179 if (!isa<AffineMapAttr>(Val: mapAttr))
5180 return parser.emitError(loc: parser.getCurrentLocation(),
5181 message: "expected affine map attribute");
5182 indexingMapsAttr.push_back(Elt: mapAttr);
5183 if (parser.parseOptionalComma())
5184 break;
5185 } while (true);
5186 if (parser.parseRSquare())
5187 return failure();
5188 }
5189 // At this stage of parsing the only way to infer number of region
5190 // args is through op kind, as input output tensors are not parsed yet.
5191 auto arityGroupAndKind = getArityGroupAndKind(kind: elemwiseKindVal);
5192 int numRegionArgs =
5193 getArityGroupAsUInt(arityGroup: arityGroupAndKind.arityGroup) + 1 /*output*/;
5194 if (parseNamedStructuredOp(parser, result, numRegionArgs,
5195 regionBuilder: ElementwiseOp::getRegionBuilder())) {
5196 return parser.emitError(loc: parser.getCurrentLocation(),
5197 message: "unable to parse elemwise op");
5198 }
5199
5200 // Initialize indexingMaps, if not supplied explicitly.
5201 if (indexingMapsAttr.empty()) {
5202 // We need to infer the numDims of the indexing maps from the output
5203 // type which is already parsed by now.
5204 auto resultType = result.operands[result.operands.size() - 1].getType();
5205 auto shapedType = llvm::dyn_cast<ShapedType>(Val&: resultType);
5206 if (!shapedType)
5207 return parser.emitError(loc: parser.getCurrentLocation(),
5208 message: "return type needs to be shaped type");
5209 auto numDims = shapedType.getRank();
5210 indexingMapsAttr = llvm::map_to_vector(
5211 C: ElementwiseOp::getDefaultIndexingMaps(numMaps: numRegionArgs, numDims,
5212 context: parser.getContext()),
5213 F: [](AffineMap map) -> Attribute { return AffineMapAttr::get(value: map); });
5214 }
5215
5216 result.addAttribute(name: "indexing_maps",
5217 attr: parser.getBuilder().getArrayAttr(value: indexingMapsAttr));
5218 return success();
5219}
5220
5221void ElementwiseOp::print(OpAsmPrinter &p) {
5222 p << " ";
5223 p.printStrippedAttrOrType(attrOrType: getKindAttr());
5224 SmallVector<StringRef, 3> elidedAttrs = {"operandSegmentSizes", "kind",
5225 "indexing_maps"};
5226 unsigned arity =
5227 getArityGroupAsUInt(arityGroup: getArityGroupAndKind(kind: getKind()).arityGroup);
5228 unsigned numDims = getResultRank();
5229
5230 SmallVector<Attribute, 3> indexingMaps = llvm::map_to_vector<3>(
5231 C: ElementwiseOp::getDefaultIndexingMaps(numMaps: arity + 1 /*output*/, numDims,
5232 context: getContext()),
5233 F: [](AffineMap map) -> Attribute { return AffineMapAttr::get(value: map); });
5234
5235 if (!llvm::equal(LRange: getIndexingMaps(), RRange&: indexingMaps))
5236 p << " indexing_maps = " << llvm::interleaved_array(R: getIndexingMaps());
5237
5238 printNamedStructuredOp(p, op: getOperation(), inputs: getInputs(), outputs: getOutputs(),
5239 elidedAttrs);
5240}
5241
5242/// Implements the block region builder for the ElementwiseOp. This is called by
5243/// 'fillStructuredOpRegion'.
5244void ElementwiseOp::regionBuilder(
5245 ImplicitLocOpBuilder &b, Block &block, ArrayRef<NamedAttribute> attrs,
5246 function_ref<InFlightDiagnostic()> emitError) {
5247 std::optional<ElementwiseKind> elemwiseKind;
5248 for (auto attr : attrs) {
5249 if (attr.getName() == "kind") {
5250 auto kindAttr = dyn_cast<ElementwiseKindAttr>(Val: attr.getValue());
5251 if (!kindAttr) {
5252 if (emitError)
5253 emitError() << "'kind' must be an ElementwiseKindAttr";
5254 return;
5255 }
5256 elemwiseKind = kindAttr.getValue();
5257 break;
5258 }
5259 }
5260
5261 if (!elemwiseKind) {
5262 if (emitError)
5263 emitError() << "missing required 'kind' attribute";
5264 return;
5265 }
5266
5267 ArityGroupAndKind groupAndKind = getArityGroupAndKind(kind: *elemwiseKind);
5268 auto arityGroup = groupAndKind.arityGroup;
5269 auto kind = groupAndKind.kind;
5270 if (emitError && block.getNumArguments() !=
5271 getArityGroupAsUInt(arityGroup) + 1 /*output*/) {
5272 emitError() << "Elementwise regionBuilder expects "
5273 << (getArityGroupAsUInt(arityGroup) + 1) << " args, got "
5274 << block.getNumArguments();
5275 return;
5276 }
5277 assert(block.getNumArguments() ==
5278 getArityGroupAsUInt(arityGroup) + 1 /*output*/
5279 && "Elementwise regionBuilder number of block args mismatch");
5280
5281 RegionBuilderHelper helper(b, block);
5282 SmallVector<Value> yields;
5283 Value result;
5284
5285 if (arityGroup == ElementwiseArityGroup::Unary) {
5286 result = helper.buildUnaryFn(unaryFn: kind.unaryFn, arg: block.getArgument(i: 0));
5287
5288 } else if (arityGroup == ElementwiseArityGroup::Binary) {
5289 result = helper.buildBinaryFn(binaryFn: kind.binaryFn, arg0: block.getArgument(i: 0),
5290 arg1: block.getArgument(i: 1));
5291
5292 } else if (arityGroup == ElementwiseArityGroup::Ternary) {
5293 result = helper.buildTernaryFn(ternaryFn: kind.ternaryFn, arg0: block.getArgument(i: 0),
5294 arg1: block.getArgument(i: 1), arg2: block.getArgument(i: 2));
5295
5296 } else {
5297 assert(false && "found unhandled category in elemwise");
5298 }
5299
5300 yields.push_back(Elt: result);
5301 helper.yieldOutputs(values: yields);
5302}
5303
5304LogicalResult ElementwiseOp::fold(FoldAdaptor,
5305 SmallVectorImpl<OpFoldResult> &) {
5306 return memref::foldMemRefCast(op: *this);
5307}
5308
5309void ElementwiseOp::getEffects(
5310 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
5311 &effects) {
5312 if (hasPureTensorSemantics())
5313 return;
5314 getGenericEffectsImpl(effects, linalgOp: cast<LinalgOp>(Val: getOperation()));
5315}
5316
5317Speculation::Speculatability ElementwiseOp::getSpeculatability() {
5318 return getGenericSpeculatabilityImpl(linalgOp: cast<LinalgOp>(Val: getOperation()));
5319}
5320
5321//===----------------------------------------------------------------------===//
5322// PackOp/UnPackOp Common
5323//===----------------------------------------------------------------------===//
5324
5325template <typename OpTy, typename>
5326SmallVector<int64_t>
5327getPackedOuterShapeWithoutTransposition(OpTy packOrUnPack) {
5328 ShapedType packedType = (std::is_same<OpTy, PackOp>::value)
5329 ? packOrUnPack.getDestType()
5330 : packOrUnPack.getSourceType();
5331 ShapedType unpackedType = (std::is_same<OpTy, PackOp>::value)
5332 ? packOrUnPack.getSourceType()
5333 : packOrUnPack.getDestType();
5334 SmallVector<int64_t> result(
5335 packedType.getShape().take_front(N: unpackedType.getRank()));
5336 if (!packOrUnPack.getOuterDimsPerm().empty()) {
5337 applyPermutationToVector(
5338 result, invertPermutationVector(packOrUnPack.getOuterDimsPerm()));
5339 }
5340 return result;
5341}
5342template SmallVector<int64_t>
5343 getPackedOuterShapeWithoutTransposition<PackOp>(PackOp);
5344template SmallVector<int64_t>
5345 getPackedOuterShapeWithoutTransposition<UnPackOp>(UnPackOp);
5346
5347// Given the (potentially) updated packed type, `newPackedTy`, generates an
5348// updated mixed-tile-sizes list. For each inner packed dimension that is static
5349// in `newPackedTy`, the tile is set to that static size (replacing SSA values
5350// or mismatched constants). Dynamic packed dimensions preserve the original
5351// tile. The folded tensor type is treated as authoritative for static extents.
5352// Note - packed-type-dim and mixed-tile-size should always match!
5353static SmallVector<OpFoldResult>
5354getNewMixedTileSizes(PatternRewriter &rewriter, Type newPackedTy,
5355 ArrayRef<OpFoldResult> mixedTiles) {
5356 SmallVector<OpFoldResult> newMixedTileSizes;
5357 for (auto it : llvm::zip(t: cast<ShapedType>(Val&: newPackedTy)
5358 .getShape()
5359 .take_back(N: mixedTiles.size()),
5360 u&: mixedTiles)) {
5361 int64_t dimSize = std::get<0>(t&: it);
5362 if (dimSize == ShapedType::kDynamic) {
5363 newMixedTileSizes.push_back(Elt: std::get<1>(t&: it));
5364 continue;
5365 }
5366 newMixedTileSizes.push_back(Elt: rewriter.getIndexAttr(value: dimSize));
5367 }
5368
5369 return newMixedTileSizes;
5370}
5371
5372template <typename OpTy>
5373static LogicalResult
5374reifyResultShapesImpl(OpTy op, OpBuilder &builder,
5375 ReifiedRankedShapedTypeDims &reifiedReturnShapes) {
5376 static_assert(llvm::is_one_of<OpTy, PackOp, UnPackOp>::value,
5377 "applies to only pack or unpack operations");
5378 int64_t destRank = op.getDestRank();
5379 reifiedReturnShapes.resize(N: 1, NV: SmallVector<OpFoldResult>(destRank));
5380 for (auto dim : llvm::seq<int64_t>(Begin: 0, End: destRank))
5381 reifiedReturnShapes[0][dim] =
5382 createFoldedDimOp(builder, op.getLoc(), op.getDest(), dim);
5383 return success();
5384}
5385
5386template <typename OpTy>
5387static DenseMap<int64_t, OpFoldResult> getDimAndTileMappingImpl(OpTy op) {
5388 static_assert(llvm::is_one_of<OpTy, PackOp, UnPackOp>::value,
5389 "applies to only pack or unpack operations");
5390 DenseMap<int64_t, OpFoldResult> dimAndTileMapping;
5391 ArrayRef<int64_t> dimsToTile = op.getInnerDimsPos();
5392 SmallVector<OpFoldResult> tiles = op.getMixedTiles();
5393 assert(tiles.size() == dimsToTile.size() &&
5394 "tiles must match indices of dimension to block");
5395 // bind the dimension `i` with the tile factor.
5396 for (auto i : llvm::seq<int64_t>(Begin: 0, End: dimsToTile.size()))
5397 dimAndTileMapping[dimsToTile[i]] = tiles[i];
5398 return dimAndTileMapping;
5399}
5400
5401template <typename OpTy>
5402static SmallVector<OpFoldResult> getMixedTilesImpl(OpTy op) {
5403 static_assert(llvm::is_one_of<OpTy, PackOp, UnPackOp>::value,
5404 "applies to only pack or unpack operations");
5405 Builder builder(op);
5406 SmallVector<OpFoldResult> mixedInnerTiles;
5407 unsigned dynamicValIndex = 0;
5408 for (int64_t staticTile : op.getStaticInnerTiles()) {
5409 if (ShapedType::isStatic(dValue: staticTile))
5410 mixedInnerTiles.push_back(Elt: builder.getI64IntegerAttr(value: staticTile));
5411 else
5412 mixedInnerTiles.push_back(Elt: op.getInnerTiles()[dynamicValIndex++]);
5413 }
5414 return mixedInnerTiles;
5415}
5416
5417template <typename OpTy>
5418static SmallVector<int64_t> getStaticTilesImpl(OpTy op) {
5419 static_assert(llvm::is_one_of<OpTy, PackOp, UnPackOp>::value,
5420 "applies to only pack or unpack operations");
5421 SmallVector<Value> dynamicTiles;
5422 SmallVector<int64_t> staticTiles;
5423 dispatchIndexOpFoldResults(op.getMixedTiles(), dynamicTiles, staticTiles);
5424 return staticTiles;
5425}
5426
5427/// Returns true if `dimsPos` is invalid. It is invalid when:
5428/// a) It contains duplicate.
5429/// b) At least one dimension is out of bound (`dimPos` is >= 0 and < rank).
5430/// c) The number of elements in `dimsPos` is > than `rank`.
5431static bool isInvalidPackingPosSpecification(ArrayRef<int64_t> dimsPos,
5432 size_t rank) {
5433 size_t dimsPosSize = dimsPos.size();
5434 if (dimsPosSize > rank)
5435 return true;
5436 DenseSet<int64_t> uniqued(llvm::from_range, dimsPos);
5437 if (dimsPosSize != uniqued.size())
5438 return true;
5439 return llvm::any_of(Range&: dimsPos, P: [rank](int64_t dimPos) {
5440 return dimPos < 0 || dimPos >= static_cast<int64_t>(rank);
5441 });
5442}
5443
5444template <typename OpTy>
5445static LogicalResult commonVerifierPackAndUnPackOp(OpTy packOrUnPack) {
5446 static_assert(llvm::is_one_of<OpTy, PackOp, UnPackOp>::value,
5447 "applies to only pack or unpack operations");
5448 Operation *op = packOrUnPack.getOperation();
5449
5450 // Return true if we have a zero-value tile.
5451 auto hasZeros = [&](ArrayRef<OpFoldResult> tiles) {
5452 return llvm::any_of(tiles, [](OpFoldResult tile) {
5453 return isa<Attribute>(Val: tile) && isZeroInteger(v: tile);
5454 });
5455 };
5456
5457 // Verify that the source and destination are ranked types.
5458 if (!packOrUnPack.getSourceType().hasRank() ||
5459 !packOrUnPack.getDestType().hasRank())
5460 return op->emitError(message: "expected both source and destination to have rank");
5461
5462 // Verify that the Operation does not have mixed tensor/buffer semantics.
5463 if (!packOrUnPack.hasPureBufferSemantics() &&
5464 !packOrUnPack.hasPureTensorSemantics())
5465 return op->emitError(message: "mixing tensor and buffer semantics is not allowed");
5466 const unsigned numResults = packOrUnPack.getNumResults();
5467 if (packOrUnPack.hasPureTensorSemantics() && numResults != 1)
5468 return op->emitError(message: "expected 1 result, got ") << numResults;
5469 if (packOrUnPack.hasPureBufferSemantics() && numResults != 0)
5470 return op->emitError(message: "expected 0 results, got ") << numResults;
5471
5472 // Verify tiles. Do not allow zero tiles.
5473 SmallVector<OpFoldResult> mixedTiles = packOrUnPack.getMixedTiles();
5474 if (hasZeros(mixedTiles))
5475 return op->emitError(message: "invalid zero tile factor");
5476
5477 // Verify inner_dims_pos and outer_dims_perm.
5478 ShapedType unpackedType = (std::is_same<OpTy, PackOp>::value)
5479 ? packOrUnPack.getSourceType()
5480 : packOrUnPack.getDestType();
5481 size_t unpackedRank = unpackedType.getRank();
5482 ArrayRef<int64_t> innerDimsPos = packOrUnPack.getInnerDimsPos();
5483 ArrayRef<int64_t> outerDimPerm = packOrUnPack.getOuterDimsPerm();
5484 if (isInvalidPackingPosSpecification(dimsPos: innerDimsPos, rank: unpackedRank))
5485 return op->emitError(message: "invalid inner_dims_pos vector");
5486 if (isInvalidPackingPosSpecification(dimsPos: outerDimPerm, rank: unpackedRank))
5487 return op->emitError(message: "invalid outer_dims_perm vector");
5488 if (!outerDimPerm.empty() && outerDimPerm.size() != unpackedRank)
5489 return op->emitError(message: "outer_dims_perm must be a permutation or empty");
5490
5491 // Tiling factors must be less than or equal to the input rank for pack (or
5492 // output rank for unpack), and must match the number of `inner_dims_pos`.
5493 if (mixedTiles.size() > unpackedRank) {
5494 return op->emitError(message: "tiling factors must be less than or equal to the "
5495 "input rank for pack or output rank for unpack");
5496 }
5497 if (mixedTiles.size() != innerDimsPos.size()) {
5498 return op->emitError(
5499 message: "tiling factors must equal the number of dimensions to tile");
5500 }
5501
5502 ShapedType packedType = (std::is_same<OpTy, PackOp>::value)
5503 ? packOrUnPack.getDestType()
5504 : packOrUnPack.getSourceType();
5505 size_t packedRank = packedType.getRank();
5506 // Require output rank to match input rank + number of blocking factors.
5507 size_t expectedPackedRank = unpackedRank + mixedTiles.size();
5508 if (expectedPackedRank != packedRank) {
5509 return op->emitError(
5510 message: "packed rank != (unpacked rank + num tiling factors), got ")
5511 << packedRank << " != " << expectedPackedRank;
5512 }
5513
5514 // Verify result shape is greater than the minimum expected
5515 // by the pack operation, and that the output shape
5516 // represents full tiles.
5517 SmallVector<int64_t> expectedPackedShape = PackOp::inferPackedShape(
5518 inputShape: unpackedType.getShape(), innerTileSizes: packOrUnPack.getStaticTiles(),
5519 innerDimsPos: packOrUnPack.getInnerDimsPos(), outerDimsPerm: packOrUnPack.getOuterDimsPerm());
5520 for (auto it : llvm::enumerate(First: llvm::zip(
5521 t: packedType.getShape().take_back(N: mixedTiles.size()), u&: mixedTiles))) {
5522 int64_t dimSize = std::get<0>(t&: it.value());
5523 if (Attribute attr =
5524 llvm::dyn_cast_if_present<Attribute>(Val&: std::get<1>(t&: it.value()))) {
5525 IntegerAttr intAttr = dyn_cast_or_null<IntegerAttr>(Val&: attr);
5526 int64_t staticTileSize = intAttr.getValue().getSExtValue();
5527 if (dimSize != staticTileSize)
5528 return op->emitError(
5529 message: "mismatch in inner tile sizes specified and shaped of "
5530 "tiled dimension in the packed type at index ")
5531 << it.index() << ": got " << dimSize << " != " << staticTileSize;
5532 } else if (!ShapedType::isDynamic(dValue: dimSize)) {
5533 return op->emitError(message: "mismatch in inner tile sizes specified at index ")
5534 << it.index() << ": got static shape " << dimSize
5535 << " but dynamic tile size";
5536 }
5537 }
5538 if (failed(
5539 Result: verifyCompatibleShape(shape1: expectedPackedShape, shape2: packedType.getShape()))) {
5540 auto elementType = unpackedType.getElementType();
5541 Type expectedType, actualType;
5542 if (packOrUnPack.hasPureTensorSemantics()) {
5543 expectedType = RankedTensorType::get(shape: expectedPackedShape, elementType);
5544 actualType = RankedTensorType::get(shape: packedType.getShape(), elementType);
5545 } else {
5546 expectedType = MemRefType::get(shape: expectedPackedShape, elementType);
5547 actualType = MemRefType::get(shape: packedType.getShape(), elementType);
5548 }
5549 return op->emitError(message: "expected ")
5550 << expectedType << " for the packed domain value, got "
5551 << actualType;
5552 }
5553 return success();
5554}
5555
5556namespace {
5557/// Subset of PackOp/UnPackOp fields used to compute the result of applying
5558/// various permutations to the op.
5559// TODO: Add linalg.transpose + pack/unpack folding patterns that just reuse
5560// these. These may or may not become true foldings / canonicalizations
5561// depending on how aggressive we want to be in automatically folding
5562// transposes.
5563struct PackOrUnPackTransposeResult {
5564 SmallVector<int64_t> innerDimsPos;
5565 SmallVector<OpFoldResult> innerTiles;
5566 SmallVector<int64_t> outerDimsPerm;
5567};
5568} // namespace
5569
5570template <typename OpTy>
5571static PackOrUnPackTransposeResult
5572commonPermutationOfPackAndUnPackOp(OpTy packOrUnPackOp,
5573 ArrayRef<int64_t> innerPermutation,
5574 ArrayRef<int64_t> outerPermutation) {
5575 static_assert(llvm::is_one_of<OpTy, PackOp, UnPackOp>::value,
5576 "applies to only pack or unpack operations");
5577 assert((!innerPermutation.empty() || !outerPermutation.empty()) &&
5578 "some permutation must be non-empty");
5579 PackOrUnPackTransposeResult metadata;
5580 metadata.innerDimsPos =
5581 SmallVector<int64_t>(packOrUnPackOp.getInnerDimsPos());
5582 metadata.innerTiles =
5583 SmallVector<OpFoldResult>(packOrUnPackOp.getMixedTiles());
5584 int64_t numOuterDims = std::is_same<OpTy, PackOp>::value
5585 ? packOrUnPackOp.getSourceRank()
5586 : packOrUnPackOp.getDestRank();
5587 metadata.outerDimsPerm =
5588 packOrUnPackOp.getOuterDimsPerm().empty()
5589 ? llvm::to_vector(Range: llvm::seq<int64_t>(Begin: 0, End: numOuterDims))
5590 : SmallVector<int64_t>(packOrUnPackOp.getOuterDimsPerm());
5591 if (!innerPermutation.empty()) {
5592 assert(innerPermutation.size() == metadata.innerDimsPos.size() &&
5593 isPermutationVector(innerPermutation) &&
5594 "invalid inner permutation");
5595 applyPermutationToVector(inVec&: metadata.innerDimsPos, permutation: innerPermutation);
5596 applyPermutationToVector(inVec&: metadata.innerTiles, permutation: innerPermutation);
5597 }
5598 if (!outerPermutation.empty()) {
5599 assert(outerPermutation.size() == metadata.outerDimsPerm.size() &&
5600 isPermutationVector(outerPermutation) &&
5601 "invalid outer permutation");
5602 applyPermutationToVector(inVec&: metadata.outerDimsPerm, permutation: outerPermutation);
5603 }
5604 return metadata;
5605}
5606
5607//===----------------------------------------------------------------------===//
5608// PackOp
5609//===----------------------------------------------------------------------===//
5610
5611void PackOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) {
5612 if (!getResults().empty())
5613 setNameFn(getResult(), "pack");
5614}
5615
5616ParseResult PackOp::parse(OpAsmParser &parser, OperationState &result) {
5617 OpAsmParser::UnresolvedOperand source, dest;
5618 SmallVector<OpAsmParser::UnresolvedOperand> dynamicTiles;
5619 SmallVector<OpAsmParser::UnresolvedOperand> paddingValue;
5620 SmallVector<Type> paddingValueType;
5621 SmallVector<int64_t> staticTiles;
5622 DenseI64ArrayAttr innerDimsPos, outerDimsPerm;
5623 Type sourceType, destType, resultType;
5624
5625 if (parser.parseOperand(result&: source))
5626 return failure();
5627
5628 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "padding_value"))) {
5629 if (parser.parseLParen() ||
5630 parser.parseOperandList(result&: paddingValue, /*requiredOperandCount=*/1) ||
5631 parser.parseColon() || parser.parseTypeList(result&: paddingValueType) ||
5632 parser.parseRParen())
5633 return failure();
5634 }
5635
5636 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "outer_dims_perm"))) {
5637 if (parser.parseEqual())
5638 return failure();
5639
5640 SmallVector<int64_t> outerDimsPermVec;
5641 if (parser.parseCommaSeparatedList(delimiter: AsmParser::Delimiter::Square, parseElementFn: [&]() {
5642 int64_t value;
5643 if (parser.parseInteger(result&: value))
5644 return failure();
5645 outerDimsPermVec.push_back(Elt: value);
5646 return success();
5647 }))
5648 return failure();
5649 outerDimsPerm = parser.getBuilder().getDenseI64ArrayAttr(values: outerDimsPermVec);
5650 }
5651
5652 if (parser.parseKeyword(keyword: "inner_dims_pos") || parser.parseEqual())
5653 return failure();
5654
5655 SmallVector<int64_t> innerDimsPosVec;
5656 if (parser.parseCommaSeparatedList(delimiter: AsmParser::Delimiter::Square, parseElementFn: [&]() {
5657 int64_t value;
5658 if (parser.parseInteger(result&: value))
5659 return failure();
5660 innerDimsPosVec.push_back(Elt: value);
5661 return success();
5662 }))
5663 return failure();
5664 innerDimsPos = parser.getBuilder().getDenseI64ArrayAttr(values: innerDimsPosVec);
5665
5666 if (parser.parseKeyword(keyword: "inner_tiles") || parser.parseEqual())
5667 return failure();
5668
5669 DenseI64ArrayAttr staticTilesAttr;
5670 if (parseDynamicIndexList(parser, values&: dynamicTiles, integers&: staticTilesAttr))
5671 return failure();
5672 for (auto val : staticTilesAttr.asArrayRef())
5673 staticTiles.push_back(Elt: val);
5674
5675 if (parser.parseKeyword(keyword: "into") || parser.parseOperand(result&: dest))
5676 return failure();
5677
5678 if (parser.parseOptionalAttrDict(result&: result.attributes))
5679 return failure();
5680
5681 if (parser.parseColon() || parser.parseType(result&: sourceType))
5682 return failure();
5683
5684 bool hasArrow = succeeded(Result: parser.parseOptionalArrow());
5685 if (hasArrow) {
5686 if (parser.parseType(result&: destType))
5687 return failure();
5688 }
5689
5690 bool isMemRef = llvm::isa<MemRefType>(Val: sourceType);
5691 if (!hasArrow) {
5692 return parser.emitError(loc: parser.getCurrentLocation(),
5693 message: "pack/unpack requires '->' and destination type");
5694 }
5695
5696 if (!isMemRef)
5697 resultType = destType;
5698
5699 if (parser.resolveOperand(operand: source, type: sourceType, result&: result.operands) ||
5700 parser.resolveOperand(operand: dest, type: destType, result&: result.operands))
5701 return failure();
5702
5703 if (!paddingValue.empty() &&
5704 parser.resolveOperands(operands&: paddingValue, type: paddingValueType[0],
5705 result&: result.operands))
5706 return failure();
5707
5708 if (!dynamicTiles.empty() &&
5709 parser.resolveOperands(operands&: dynamicTiles, type: parser.getBuilder().getIndexType(),
5710 result&: result.operands))
5711 return failure();
5712
5713 result.addAttribute(name: "static_inner_tiles",
5714 attr: parser.getBuilder().getDenseI64ArrayAttr(values: staticTiles));
5715 result.addAttribute(name: "inner_dims_pos", attr: innerDimsPos);
5716 if (outerDimsPerm)
5717 result.addAttribute(name: "outer_dims_perm", attr: outerDimsPerm);
5718
5719 SmallVector<int32_t> segmentSizes = {
5720 1, 1, static_cast<int32_t>(paddingValue.size()),
5721 static_cast<int32_t>(dynamicTiles.size())};
5722 result.addAttribute(name: "operandSegmentSizes",
5723 attr: parser.getBuilder().getDenseI32ArrayAttr(values: segmentSizes));
5724
5725 if (!isMemRef)
5726 result.addTypes(newTypes: resultType);
5727
5728 return success();
5729}
5730
5731void PackOp::print(OpAsmPrinter &p) {
5732 p << " " << getSource();
5733
5734 if (getPaddingValue()) {
5735 p << " padding_value(" << getPaddingValue() << " : "
5736 << getPaddingValue().getType() << ")";
5737 }
5738
5739 if (!getOuterDimsPerm().empty()) {
5740 p << " outer_dims_perm = [";
5741 llvm::interleaveComma(c: getOuterDimsPerm(), os&: p);
5742 p << "]";
5743 }
5744
5745 p << " inner_dims_pos = [";
5746 llvm::interleaveComma(c: getInnerDimsPos(), os&: p);
5747 p << "]";
5748
5749 p << " inner_tiles = ";
5750 printDynamicIndexList(printer&: p, op: *this, values: getInnerTiles(), integers: getStaticInnerTilesAttr());
5751
5752 p << " into " << getDest();
5753
5754 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary(),
5755 elidedAttrs: {"static_inner_tiles", "inner_dims_pos",
5756 "outer_dims_perm", "operandSegmentSizes"});
5757
5758 p << " : " << getSource().getType();
5759 p << " -> " << getDest().getType();
5760}
5761
5762void PackOp::build(OpBuilder &builder, OperationState &state, Value source,
5763 Value dest, ArrayRef<int64_t> innerDimsPos,
5764 ArrayRef<OpFoldResult> innerTiles,
5765 std::optional<Value> paddingValue,
5766 ArrayRef<int64_t> outerDimsPerm) {
5767 assert(innerDimsPos.size() == innerTiles.size() &&
5768 "number of tile sizes specified must match the specified number of "
5769 "original dimensions to be tiled");
5770 SmallVector<int64_t> staticTileSizes;
5771 SmallVector<Value> dynamicTileSizes;
5772 dispatchIndexOpFoldResults(ofrs: innerTiles, dynamicVec&: dynamicTileSizes, staticVec&: staticTileSizes);
5773 build(odsBuilder&: builder, odsState&: state, result: dest.getType(), source, dest,
5774 padding_value: paddingValue ? *paddingValue : nullptr,
5775 outer_dims_perm: outerDimsPerm.empty() ? nullptr
5776 : builder.getDenseI64ArrayAttr(values: outerDimsPerm),
5777 inner_dims_pos: builder.getDenseI64ArrayAttr(values: innerDimsPos), inner_tiles: dynamicTileSizes,
5778 static_inner_tiles: builder.getDenseI64ArrayAttr(values: staticTileSizes));
5779}
5780
5781LogicalResult
5782PackOp::reifyResultShapes(OpBuilder &builder,
5783 ReifiedRankedShapedTypeDims &reifiedReturnShapes) {
5784 return reifyResultShapesImpl(op: *this, builder, reifiedReturnShapes);
5785}
5786
5787DenseMap<int64_t, OpFoldResult> PackOp::getDimAndTileMapping() {
5788 return getDimAndTileMappingImpl(op: *this);
5789}
5790
5791SmallVector<OpFoldResult> PackOp::getMixedTiles() {
5792 return getMixedTilesImpl(op: *this);
5793}
5794
5795SmallVector<int64_t> PackOp::getStaticTiles() {
5796 return getStaticTilesImpl(op: *this);
5797}
5798
5799ArrayRef<int64_t> PackOp::getAllOuterDims() {
5800 ShapedType inputType = getSourceType();
5801 int64_t inputRank = inputType.getRank();
5802 return getDestType().getShape().take_front(N: inputRank);
5803}
5804
5805SmallVector<int64_t> PackOp::getTiledOuterDims() {
5806 auto innerDimsPos = getInnerDimsPos();
5807 SmallVector<int64_t> outerDims(getAllOuterDims());
5808 SmallVector<int64_t> res;
5809
5810 // Recover the original order of the outer dims.
5811 SmallVector<int64_t> outerDimPermInv(getOuterDimsPerm());
5812 invertPermutationVector(permutation: outerDimPermInv);
5813 if (!outerDimPermInv.empty())
5814 applyPermutationToVector(inVec&: outerDims, permutation: outerDimPermInv);
5815
5816 // Collect the outer dims corresponding to the tilled inner dims.
5817 for (auto index : innerDimsPos)
5818 res.push_back(Elt: outerDims[index]);
5819
5820 return res;
5821}
5822
5823bool PackOp::requirePaddingValue(ArrayRef<int64_t> inputShape,
5824 ArrayRef<int64_t> innerDimsPos,
5825 ArrayRef<int64_t> outputShape,
5826 ArrayRef<int64_t> outerDimsPerm,
5827 ArrayRef<OpFoldResult> innerTiles) {
5828 SmallVector<int64_t> outputTileSizes(
5829 outputShape.take_front(N: inputShape.size()));
5830 if (!outerDimsPerm.empty()) {
5831 assert(outerDimsPerm.size() == outputTileSizes.size() &&
5832 "expected output and outer_dims_perm to have same size");
5833 applyPermutationToVector(inVec&: outputTileSizes,
5834 permutation: invertPermutationVector(permutation: outerDimsPerm));
5835 }
5836 for (auto [pos, tileSize] : llvm::zip_equal(t&: innerDimsPos, u&: innerTiles)) {
5837 if (ShapedType::isDynamic(dValue: inputShape[pos]))
5838 continue;
5839 std::optional<int64_t> constantTile = getConstantIntValue(ofr: tileSize);
5840 if (!constantTile) {
5841 if (ShapedType::isStatic(dValue: outputTileSizes[pos]) &&
5842 (inputShape[pos] % outputTileSizes[pos] != 0))
5843 return true;
5844 } else {
5845 assert(*constantTile != 0 && "static tile size can't be zero");
5846 if (inputShape[pos] % (*constantTile) != 0) {
5847 return true;
5848 }
5849 }
5850 }
5851 return false;
5852}
5853
5854bool PackOp::requirePaddingValueStrict(ArrayRef<int64_t> inputShape,
5855 ArrayRef<int64_t> innerDimsPos,
5856 ArrayRef<int64_t> outputShape,
5857 ArrayRef<int64_t> outerDimsPerm,
5858 ArrayRef<OpFoldResult> innerTiles) {
5859 SmallVector<int64_t> outputTileSizes(
5860 outputShape.take_front(N: inputShape.size()));
5861 if (!outerDimsPerm.empty()) {
5862 assert(outerDimsPerm.size() == outputTileSizes.size() &&
5863 "expected output and outer_dims_perm to have same size");
5864 applyPermutationToVector(inVec&: outputTileSizes,
5865 permutation: invertPermutationVector(permutation: outerDimsPerm));
5866 }
5867 for (auto [pos, tileSize] : llvm::zip_equal(t&: innerDimsPos, u&: innerTiles)) {
5868 if (ShapedType::isDynamic(dValue: inputShape[pos]) ||
5869 ShapedType::isDynamic(dValue: outputTileSizes[pos]))
5870 return true;
5871 std::optional<int64_t> constantTile = getConstantIntValue(ofr: tileSize);
5872 if (!constantTile)
5873 return true;
5874 assert(*constantTile != 0 && "static tile size can't be zero");
5875 if (inputShape[pos] % (*constantTile) != 0)
5876 return true;
5877 }
5878 return false;
5879}
5880
5881LogicalResult PackOp::verify() {
5882 if (failed(Result: commonVerifierPackAndUnPackOp(packOrUnPack: *this)))
5883 return failure();
5884
5885 // Verify padding value, and bail out if the tile does not divide the
5886 // dimension fully. In the case of dynamic tile factors or dimensions, having
5887 // a partial tile is undefined behavior.
5888 auto paddingValue = getPaddingValue();
5889 if (paddingValue &&
5890 paddingValue.getType() != getSourceType().getElementType()) {
5891 return emitOpError(message: "expected padding_value has ")
5892 << getSourceType().getElementType()
5893 << " but got: " << paddingValue.getType();
5894 }
5895
5896 if (!paddingValue &&
5897 requirePaddingValue(inputShape: getSourceType().getShape(), innerDimsPos: getInnerDimsPos(),
5898 outputShape: getDestType().getShape(), outerDimsPerm: getOuterDimsPerm(),
5899 innerTiles: getMixedTiles())) {
5900 return emitOpError(
5901 message: "invalid tile factor or output size provided. Only full tiles are "
5902 "supported when padding_value is not set");
5903 }
5904 return success();
5905}
5906
5907/// Converts OpFoldResults to int64_t shape entries, unconditionally mapping all
5908/// Value's to kDynamic, even if they are arith.constant values.
5909static SmallVector<int64_t>
5910asShapeWithAnyValueAsDynamic(ArrayRef<OpFoldResult> ofrs) {
5911 SmallVector<int64_t> result;
5912 for (auto o : ofrs) {
5913 // Have to do this first, as getConstantIntValue special-cases constants.
5914 if (llvm::dyn_cast_if_present<Value>(Val&: o))
5915 result.push_back(Elt: ShapedType::kDynamic);
5916 else
5917 result.push_back(Elt: getConstantIntValue(ofr: o).value_or(u: ShapedType::kDynamic));
5918 }
5919 return result;
5920}
5921
5922SmallVector<int64_t> PackOp::inferPackedShape(ArrayRef<int64_t> inputShape,
5923 ArrayRef<int64_t> innerTileSizes,
5924 ArrayRef<int64_t> innerDimsPos,
5925 ArrayRef<int64_t> outerDimsPerm) {
5926 SmallVector<int64_t> resultShape = llvm::to_vector(Range&: inputShape);
5927 for (auto tiledDim : llvm::enumerate(First: llvm::to_vector(Range&: innerDimsPos))) {
5928 if (ShapedType::isDynamic(dValue: resultShape[tiledDim.value()]))
5929 continue;
5930 if (ShapedType::isDynamic(dValue: innerTileSizes[tiledDim.index()])) {
5931 resultShape[tiledDim.value()] = ShapedType::kDynamic;
5932 continue;
5933 }
5934 resultShape[tiledDim.value()] = llvm::divideCeilSigned(
5935 Numerator: resultShape[tiledDim.value()], Denominator: innerTileSizes[tiledDim.index()]);
5936 }
5937
5938 // Swap tile loops if outer_dims_perm is available.
5939 if (!outerDimsPerm.empty())
5940 applyPermutationToVector(inVec&: resultShape, permutation: outerDimsPerm);
5941
5942 // Append the inner tile dimensions.
5943 resultShape.append(in_start: innerTileSizes.begin(), in_end: innerTileSizes.end());
5944 return resultShape;
5945}
5946
5947SmallVector<OpFoldResult> PackOp::getResultShape(
5948 OpBuilder &builder, Location loc, ArrayRef<OpFoldResult> sourceDims,
5949 ArrayRef<OpFoldResult> innerTileSizes, ArrayRef<int64_t> innerDimsPos,
5950 ArrayRef<int64_t> outerDimsPerm) {
5951 SmallVector<OpFoldResult> resultDims = llvm::to_vector(Range&: sourceDims);
5952
5953 AffineExpr s0, s1;
5954 bindSymbols(ctx: builder.getContext(), exprs&: s0, exprs&: s1);
5955 AffineExpr ceilDivExpr = s0.ceilDiv(other: s1);
5956 for (auto tiledDim : llvm::enumerate(First: llvm::to_vector(Range&: innerDimsPos))) {
5957 resultDims[tiledDim.value()] = affine::makeComposedFoldedAffineApply(
5958 b&: builder, loc, expr: ceilDivExpr,
5959 operands: {resultDims[tiledDim.value()], innerTileSizes[tiledDim.index()]});
5960 }
5961 if (!outerDimsPerm.empty())
5962 applyPermutationToVector(inVec&: resultDims, permutation: outerDimsPerm);
5963 resultDims.append(in_start: innerTileSizes.begin(), in_end: innerTileSizes.end());
5964
5965 SmallVector<int64_t> resultTypeShape =
5966 inferPackedShape(inputShape: asShapeWithAnyValueAsDynamic(ofrs: sourceDims),
5967 innerTileSizes: asShapeWithAnyValueAsDynamic(ofrs: innerTileSizes),
5968 innerDimsPos, outerDimsPerm);
5969
5970 // Fix-up `resultDims` to ensure that they are Value's if and only if the
5971 // result type shape says it's a dynamic dim. This is needed as callers may
5972 // use dispatchIndexOpFoldResults on the result, and rely on exact number of
5973 // dynamic dims returned by that.
5974 for (unsigned i = 0; i < resultDims.size(); ++i) {
5975 if (ShapedType::isStatic(dValue: resultTypeShape[i]))
5976 continue;
5977 resultDims[i] =
5978 getValueOrCreateConstantIndexOp(b&: builder, loc, ofr: resultDims[i]);
5979 }
5980
5981 return resultDims;
5982}
5983
5984RankedTensorType PackOp::inferPackedTensorType(
5985 RankedTensorType sourceType, ArrayRef<int64_t> innerTileSizes,
5986 ArrayRef<int64_t> innerDimsPos, ArrayRef<int64_t> outerDimsPerm) {
5987 SmallVector<int64_t> resultShape = inferPackedShape(
5988 inputShape: sourceType.getShape(), innerTileSizes, innerDimsPos, outerDimsPerm);
5989 return RankedTensorType::get(shape: resultShape, elementType: sourceType.getElementType());
5990}
5991
5992MemRefType PackOp::inferPackedMemRefType(MemRefType sourceType,
5993 ArrayRef<int64_t> innerTileSizes,
5994 ArrayRef<int64_t> innerDimsPos,
5995 ArrayRef<int64_t> outerDimsPerm) {
5996 SmallVector<int64_t> resultShape = inferPackedShape(
5997 inputShape: sourceType.getShape(), innerTileSizes, innerDimsPos, outerDimsPerm);
5998 return MemRefType::get(shape: resultShape, elementType: sourceType.getElementType());
5999}
6000
6001Value PackOp::createDestinationTensor(OpBuilder &b, Location loc, Value source,
6002 ArrayRef<OpFoldResult> innerTileSizes,
6003 ArrayRef<int64_t> innerDimsPos,
6004 ArrayRef<int64_t> outerDimsPerm) {
6005 AffineExpr dim0, dim1;
6006 bindDims(ctx: b.getContext(), exprs&: dim0, exprs&: dim1);
6007 auto ceilDiv = [&](OpFoldResult v1, OpFoldResult v2) -> OpFoldResult {
6008 return affine::makeComposedFoldedAffineApply(b, loc, expr: dim0.ceilDiv(other: dim1),
6009 operands: {v1, v2});
6010 };
6011
6012 SmallVector<OpFoldResult> mixedSizes;
6013 for (auto [index, value] : llvm::enumerate(
6014 First: llvm::cast<RankedTensorType>(Val: source.getType()).getShape())) {
6015 if (ShapedType::isDynamic(dValue: value))
6016 mixedSizes.push_back(
6017 Elt: tensor::DimOp::create(builder&: b, location: loc, source, index).getResult());
6018 else
6019 mixedSizes.push_back(Elt: b.getIndexAttr(value));
6020 }
6021 for (auto it : llvm::zip(t&: innerDimsPos, u&: innerTileSizes)) {
6022 int64_t dimPos = std::get<0>(t&: it);
6023 OpFoldResult tileSize = std::get<1>(t&: it);
6024 mixedSizes[dimPos] = ceilDiv(mixedSizes[dimPos], tileSize);
6025 }
6026 if (!outerDimsPerm.empty())
6027 applyPermutationToVector<OpFoldResult>(inVec&: mixedSizes, permutation: outerDimsPerm);
6028
6029 mixedSizes.append(in_start: innerTileSizes.begin(), in_end: innerTileSizes.end());
6030 auto elemType = llvm::cast<ShapedType>(Val: source.getType()).getElementType();
6031 return tensor::EmptyOp::create(builder&: b, location: loc, sizes: mixedSizes, elementType: elemType);
6032}
6033
6034PackOp PackOp::createTransposedClone(OpBuilder &b, Location loc,
6035 ArrayRef<int64_t> innerPermutation,
6036 ArrayRef<int64_t> outerPermutation) {
6037 PackOrUnPackTransposeResult metadata = commonPermutationOfPackAndUnPackOp(
6038 packOrUnPackOp: *this, innerPermutation, outerPermutation);
6039 Value transposedDest =
6040 createDestinationTensor(b, loc, source: getSource(), innerTileSizes: metadata.innerTiles,
6041 innerDimsPos: metadata.innerDimsPos, outerDimsPerm: metadata.outerDimsPerm);
6042 return PackOp::create(builder&: b, location: loc, source: getSource(), dest: transposedDest,
6043 innerDimsPos: metadata.innerDimsPos, innerTiles: metadata.innerTiles,
6044 paddingValue: getPaddingValue(), outerDimsPerm: metadata.outerDimsPerm);
6045}
6046
6047template <typename OpTy>
6048static void getPackUnPackEffectsImpl(
6049 OpTy op, SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
6050 &effects) {
6051 // No memory effects for pure tensor semantics
6052 if (op.hasPureTensorSemantics())
6053 return;
6054
6055 for (OpOperand &opOperand : op.getOperation()->getOpOperands()) {
6056 if (!llvm::isa<MemRefType>(Val: opOperand.get().getType()))
6057 continue;
6058
6059 if (&opOperand == &op.getSourceMutable()) {
6060 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: &opOperand, /*stage=*/Args: 0,
6061 /*effectOnFullRegion=*/Args: true,
6062 Args: SideEffects::DefaultResource::get());
6063 } else if (&opOperand == &op.getDestMutable()) {
6064 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: &opOperand, /*stage=*/Args: 0,
6065 /*effectOnFullRegion=*/Args: true,
6066 Args: SideEffects::DefaultResource::get());
6067 effects.emplace_back(Args: MemoryEffects::Write::get(), Args: &opOperand, /*stage=*/Args: 0,
6068 /*effectOnFullRegion=*/Args: true,
6069 Args: SideEffects::DefaultResource::get());
6070 }
6071 }
6072}
6073
6074void PackOp::getEffects(
6075 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
6076 &effects) {
6077 getPackUnPackEffectsImpl(op: *this, effects);
6078}
6079
6080void UnPackOp::getEffects(
6081 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
6082 &effects) {
6083 getPackUnPackEffectsImpl(op: *this, effects);
6084}
6085
6086/// Returns true if the tiles and the tiled dims are constant.
6087template <typename OpTy>
6088static bool areTilesAndTiledDimsAllConstant(OpTy op) {
6089 static_assert(llvm::is_one_of<OpTy, PackOp, UnPackOp>::value,
6090 "applies to only pack or unpack operations");
6091 ShapedType packedType = (std::is_same<OpTy, PackOp>::value)
6092 ? op.getDestType()
6093 : op.getSourceType();
6094 SmallVector<OpFoldResult> mixedTiles = op.getMixedTiles();
6095 for (auto [dimDest, tile] : llvm::zip(
6096 t: packedType.getShape().take_back(N: mixedTiles.size()), u&: mixedTiles)) {
6097 std::optional<int64_t> constTileSize = getConstantIntValue(ofr: tile);
6098 if (!constTileSize || ShapedType::isDynamic(dValue: dimDest))
6099 return false;
6100 }
6101 return true;
6102}
6103
6104Speculation::Speculatability PackOp::getSpeculatability() {
6105 if (!hasPureTensorSemantics())
6106 return Speculation::NotSpeculatable;
6107 if (getPaddingValue())
6108 return Speculation::Speculatable;
6109
6110 // The verifier rejects already operations if we can statically prove that the
6111 // sizes of the tiles do not divide perfectly the dimension; thus, check only
6112 // to have constant tiles and tiled inner dimensions.
6113 if (!areTilesAndTiledDimsAllConstant(op: *this))
6114 return Speculation::NotSpeculatable;
6115
6116 return Speculation::Speculatable;
6117}
6118
6119// Return true if `inner_dims_pos` and `outer_dims_perm` target the same
6120// dimensions for pack and unpack.
6121static bool hasSameInnerOuterAttribute(PackOp packOp, UnPackOp unPackOp) {
6122 if (packOp.getInnerDimsPos() != unPackOp.getInnerDimsPos())
6123 return false;
6124 if (packOp.getOuterDimsPerm() == unPackOp.getOuterDimsPerm())
6125 return true;
6126 // Outer dims permutation is optional.
6127 // To compare unbalanced pack-unpack pair, treat no permutation as equal to
6128 // identity permutation.
6129 return isIdentityPermutation(permutation: packOp.getOuterDimsPerm()) &&
6130 isIdentityPermutation(permutation: unPackOp.getOuterDimsPerm());
6131}
6132
6133// Return true if pack and unpack have the same tiles.
6134// Same SSA values or same integer constants.
6135static bool haveSameTiles(PackOp packOp, UnPackOp unPackOp) {
6136 auto packTiles = packOp.getMixedTiles();
6137 auto unPackTiles = unPackOp.getMixedTiles();
6138 if (packTiles.size() != unPackTiles.size())
6139 return false;
6140 for (size_t i = 0, e = packTiles.size(); i < e; i++) {
6141 if (!isEqualConstantIntOrValue(ofr1: packTiles[i], ofr2: unPackTiles[i]))
6142 return false;
6143 }
6144 return true;
6145}
6146
6147/// Returns true if the pack op does not need a padding value.
6148static bool paddingIsNotNeeded(PackOp op) {
6149 auto srcType = op.getSourceType();
6150 auto innerDimsPos = op.getInnerDimsPos();
6151 auto innerTiles = op.getStaticInnerTiles();
6152 if (ShapedType::isDynamicShape(dSizes: innerTiles))
6153 return false;
6154 for (auto [pos, tileSize] : llvm::zip_equal(t&: innerDimsPos, u&: innerTiles)) {
6155 if (srcType.isDynamicDim(idx: pos) && tileSize != 1)
6156 return false;
6157 }
6158 return !PackOp::requirePaddingValue(
6159 inputShape: srcType.getShape(), innerDimsPos: op.getInnerDimsPos(), outputShape: op.getDestType().getShape(),
6160 outerDimsPerm: op.getOuterDimsPerm(), innerTiles: op.getMixedTiles());
6161}
6162
6163/// Returns true if the `srcShape` or `destShape` is different from the one in
6164/// `packOp` and populates each with the inferred static shape.
6165static bool inferStaticShape(PackOp packOp, SmallVectorImpl<int64_t> &srcShape,
6166 SmallVectorImpl<int64_t> &destShape) {
6167 bool changeNeeded = false;
6168 srcShape.assign(in_start: packOp.getSourceType().getShape().begin(),
6169 in_end: packOp.getSourceType().getShape().end());
6170 destShape.assign(in_start: packOp.getDestType().getShape().begin(),
6171 in_end: packOp.getDestType().getShape().end());
6172 llvm::SmallSetVector<int64_t, 4> innerDims;
6173 innerDims.insert_range(R: packOp.getInnerDimsPos());
6174 SmallVector<int64_t> inverseOuterDimsPerm;
6175 if (!packOp.getOuterDimsPerm().empty())
6176 inverseOuterDimsPerm = invertPermutationVector(permutation: packOp.getOuterDimsPerm());
6177 int srcRank = packOp.getSourceRank();
6178 for (auto i : llvm::seq<int64_t>(Begin: 0, End: srcRank)) {
6179 if (innerDims.contains(key: i))
6180 continue;
6181 int64_t srcPos = i;
6182 int64_t destPos = i;
6183 if (!inverseOuterDimsPerm.empty())
6184 destPos = inverseOuterDimsPerm[srcPos];
6185 if (ShapedType::isDynamic(dValue: srcShape[srcPos]) ==
6186 ShapedType::isDynamic(dValue: destShape[destPos])) {
6187 continue;
6188 }
6189 int64_t size = srcShape[srcPos];
6190 if (ShapedType::isDynamic(dValue: size))
6191 size = destShape[destPos];
6192 srcShape[srcPos] = size;
6193 destShape[destPos] = size;
6194 changeNeeded = true;
6195 }
6196 return changeNeeded;
6197}
6198
6199LogicalResult PackOp::canonicalize(PackOp packOp, PatternRewriter &rewriter) {
6200 // Pack/unpack memref transformations are unsupported. The memref forms
6201 // are mainly for bufferization and scalar lowering. Other uses are not
6202 // recommended, see #225650 for details.
6203 if (!packOp.hasPureTensorSemantics())
6204 return failure();
6205
6206 // Fold an pack(unpack(x)) to x.
6207 if (auto unPackOp = packOp.getSource().getDefiningOp<UnPackOp>()) {
6208 if (unPackOp.getSourceType() == packOp.getDestType() &&
6209 !packOp.getPaddingValue() &&
6210 hasSameInnerOuterAttribute(packOp, unPackOp) &&
6211 haveSameTiles(packOp, unPackOp)) {
6212 rewriter.replaceOp(op: packOp, newValues: unPackOp.getSource());
6213 return success();
6214 }
6215 }
6216
6217 // Fold optional PaddingValue operand away if padding is not needed.
6218 if (packOp.getPaddingValue() && paddingIsNotNeeded(op: packOp)) {
6219 rewriter.startOpModification(op: packOp);
6220 packOp.getPaddingValueMutable().clear();
6221 rewriter.finalizeOpModification(op: packOp);
6222 return success();
6223 }
6224
6225 // Insert tensor.cast ops if static shape inference is available..
6226 SmallVector<int64_t> srcShape, destShape;
6227 if (inferStaticShape(packOp, srcShape, destShape)) {
6228 Location loc = packOp.getLoc();
6229 Value source = packOp.getSource();
6230 if (srcShape != packOp.getSourceType().getShape()) {
6231 auto newSrcType = packOp.getSourceType().clone(shape: srcShape);
6232 source =
6233 tensor::CastOp::create(builder&: rewriter, location: loc, dest: newSrcType, source: packOp.getSource());
6234 }
6235 Value dest = packOp.getDest();
6236 ShapedType originalResultType = packOp.getDestType();
6237 bool needUpdateDestType = (destShape != originalResultType.getShape());
6238 if (needUpdateDestType) {
6239 auto newDestType = packOp.getDestType().clone(shape: destShape);
6240 dest =
6241 tensor::CastOp::create(builder&: rewriter, location: loc, dest: newDestType, source: packOp.getDest());
6242 }
6243 rewriter.modifyOpInPlace(root: packOp, callable: [&] {
6244 packOp.getSourceMutable().assign(value: source);
6245 packOp.getDestMutable().assign(value: dest);
6246 packOp.getResult().setType(cast<RankedTensorType>(Val: dest.getType()));
6247 });
6248 // Insert a cast if needed
6249 if (needUpdateDestType) {
6250 rewriter.setInsertionPointAfter(packOp);
6251 auto castOp = tensor::CastOp::create(builder&: rewriter, location: loc, dest: originalResultType,
6252 source: packOp.getResult());
6253 rewriter.replaceAllUsesExcept(from: packOp.getResult(), to: castOp, exceptedUser: castOp);
6254 }
6255 return success();
6256 }
6257
6258 return failure();
6259}
6260
6261template <typename PackOrUnpackOp>
6262static bool isLikePadUnPad(PackOrUnpackOp packOp, ShapedType packedTensorType) {
6263 static_assert(std::is_same<PackOrUnpackOp, PackOp>::value ||
6264 std::is_same<PackOrUnpackOp, UnPackOp>::value,
6265 "Function meant for pack/unpack");
6266 // This is a pad if packing only adds ones and we don't transpose dimensions.
6267
6268 // Check that we are not transposing any dimensions.
6269 ArrayRef<int64_t> innerDimsPos = packOp.getInnerDimsPos();
6270 int64_t numPackedDims = innerDimsPos.size();
6271 auto orderedDims = llvm::to_vector<4>(Range: llvm::seq<int64_t>(Begin: 0, End: numPackedDims));
6272 if (orderedDims != innerDimsPos) {
6273 // Dimensions don't happen in order.
6274 return false;
6275 }
6276
6277 ArrayRef<int64_t> packedShape = packedTensorType.getShape();
6278 int64_t packedRank = packedTensorType.getRank();
6279 // At this point we know that we are taking numPackedDims outer
6280 // dimensions and pushing them all the way as the inner most dimensions.
6281 // What's left on the outer most dimensions is, in this order:
6282 // - the factor of the packed dimensions, then
6283 // - the untouched dimensions
6284 // This shifting inward of dimensions is a no-op (as opposed to a transpose)
6285 // if all the dimensions that bubble outerward are ones.
6286 // Therefore check that all the dimensions but the numPackedDims inner most
6287 // ones are ones.
6288 return llvm::all_of(
6289 llvm::seq<int64_t>(Begin: 0, End: packedRank - numPackedDims),
6290 [&packedShape](int64_t i) { return packedShape[i] == 1; });
6291}
6292
6293bool PackOp::isLikePad() {
6294 auto packedTensorType =
6295 llvm::cast<ShapedType>(Val: (*this)->getResultTypes().front());
6296 return isLikePadUnPad(packOp: *this, packedTensorType);
6297}
6298
6299::mlir::LogicalResult
6300PackOp::fold(FoldAdaptor adaptor,
6301 ::llvm::SmallVectorImpl<OpFoldResult> &results) {
6302 // Pack/unpack memref transformations are unsupported. The memref forms
6303 // are mainly for bufferization and scalar lowering. Other uses are not
6304 // recommended, see #225650 for details.
6305 if (!hasPureTensorSemantics())
6306 return failure();
6307 std::optional<Attribute> paddingValue;
6308 if (auto pad = adaptor.getPaddingValue())
6309 paddingValue = pad;
6310 if (OpFoldResult reshapedSource = reshapeConstantSource(
6311 source: llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getSource()),
6312 result: cast<TensorType>(Val: getDestType()), cst: paddingValue)) {
6313 results.push_back(Elt: reshapedSource);
6314 return success();
6315 }
6316 return failure();
6317}
6318
6319/// Folds a tensor.cast op into a consuming PackOp op if the
6320/// `tensor.cast` has source that is more static than the consuming op.
6321///
6322/// Example:
6323/// ```mlir
6324/// %1 = tensor.cast %0 : tensor<8x16xf32> to tensor<?x?xf32>
6325/// %2 = tensor.pack %1 ... : tensor<?x?xf32> ...
6326/// ```
6327///
6328/// folds into:
6329///
6330/// ```mlir
6331/// %2 = tensor.pack %0 ... : tensor<8x16xf32> ...
6332/// ```
6333struct FoldTensorCastPackOp : public OpRewritePattern<PackOp> {
6334 using OpRewritePattern<PackOp>::OpRewritePattern;
6335
6336 LogicalResult matchAndRewrite(PackOp op,
6337 PatternRewriter &rewriter) const override {
6338 // Pack/unpack memref transformations are unsupported. The memref forms
6339 // are mainly for bufferization and scalar lowering. Other uses are not
6340 // recommended, see #225650 for details.
6341 if (!op.hasPureTensorSemantics())
6342 return failure();
6343
6344 if (!tensor::hasFoldableTensorCastOperand(op))
6345 return failure();
6346
6347 SmallVector<Type> newResultTypes(op->getResultTypes());
6348 SmallVector<Value> newOperands =
6349 tensor::getUpdatedOperandsAfterCastOpFolding(op, newResTy&: newResultTypes);
6350
6351 // Get the updated mixed-tile-sizes attribute.
6352 SmallVector<OpFoldResult> newMixedTileSizes =
6353 getNewMixedTileSizes(rewriter, newPackedTy: newResultTypes[0], mixedTiles: op.getMixedTiles());
6354 if (llvm::any_of(Range&: newMixedTileSizes, P: isZeroInteger))
6355 return failure();
6356
6357 // Clone op.
6358 // TODO: Strictly speaking, discardable attributes should be _discarded_ at
6359 // this point. However, in practice, we use them for things that we'd like
6360 // to preserve. Implement a better abstraction.
6361 PackOp newOp =
6362 PackOp::create(builder&: rewriter, location: op.getLoc(), source: newOperands[0], dest: newOperands[1],
6363 innerDimsPos: op.getInnerDimsPos(), innerTiles: newMixedTileSizes,
6364 paddingValue: op.getPaddingValue(), outerDimsPerm: op.getOuterDimsPerm());
6365 newOp->setDiscardableAttrs(op->getDiscardableAttrDictionary());
6366
6367 // Replace op.
6368 Value oldResult = op.getResult();
6369 Value newResult = newOp.getResult();
6370 Value replacement =
6371 (newResult.getType() != oldResult.getType())
6372 ? tensor::CastOp::create(builder&: rewriter, location: op->getLoc(),
6373 dest: oldResult.getType(), source: newResult)
6374 : newResult;
6375
6376 rewriter.replaceOp(op, newValues: {replacement});
6377
6378 return success();
6379 }
6380};
6381
6382//===----------------------------------------------------------------------===//
6383// UnPackOp
6384//===----------------------------------------------------------------------===//
6385
6386void UnPackOp::getAsmResultNames(
6387 function_ref<void(Value, StringRef)> setNameFn) {
6388 if (!getResults().empty())
6389 setNameFn(getResult(), "unpack");
6390}
6391
6392// Custom parser for UnPackOp that handles the memref/tensor case distinction
6393ParseResult UnPackOp::parse(OpAsmParser &parser, OperationState &result) {
6394 OpAsmParser::UnresolvedOperand source, dest;
6395 SmallVector<OpAsmParser::UnresolvedOperand> dynamicTiles;
6396 SmallVector<int64_t> staticTiles;
6397 DenseI64ArrayAttr innerDimsPos, outerDimsPerm;
6398 Type sourceType, destType, resultType;
6399
6400 if (parser.parseOperand(result&: source))
6401 return failure();
6402
6403 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "outer_dims_perm"))) {
6404 if (parser.parseEqual())
6405 return failure();
6406
6407 SmallVector<int64_t> outerDimsPermVec;
6408 if (parser.parseCommaSeparatedList(delimiter: AsmParser::Delimiter::Square, parseElementFn: [&]() {
6409 int64_t value;
6410 if (parser.parseInteger(result&: value))
6411 return failure();
6412 outerDimsPermVec.push_back(Elt: value);
6413 return success();
6414 }))
6415 return failure();
6416 outerDimsPerm = parser.getBuilder().getDenseI64ArrayAttr(values: outerDimsPermVec);
6417 }
6418
6419 if (parser.parseKeyword(keyword: "inner_dims_pos") || parser.parseEqual())
6420 return failure();
6421
6422 SmallVector<int64_t> innerDimsPosVec;
6423 if (parser.parseCommaSeparatedList(delimiter: AsmParser::Delimiter::Square, parseElementFn: [&]() {
6424 int64_t value;
6425 if (parser.parseInteger(result&: value))
6426 return failure();
6427 innerDimsPosVec.push_back(Elt: value);
6428 return success();
6429 }))
6430 return failure();
6431 innerDimsPos = parser.getBuilder().getDenseI64ArrayAttr(values: innerDimsPosVec);
6432
6433 if (parser.parseKeyword(keyword: "inner_tiles") || parser.parseEqual())
6434 return failure();
6435
6436 DenseI64ArrayAttr staticTilesAttr;
6437 if (parseDynamicIndexList(parser, values&: dynamicTiles, integers&: staticTilesAttr))
6438 return failure();
6439 for (auto val : staticTilesAttr.asArrayRef())
6440 staticTiles.push_back(Elt: val);
6441
6442 if (parser.parseKeyword(keyword: "into") || parser.parseOperand(result&: dest))
6443 return failure();
6444
6445 if (parser.parseOptionalAttrDict(result&: result.attributes))
6446 return failure();
6447
6448 if (parser.parseColon() || parser.parseType(result&: sourceType))
6449 return failure();
6450
6451 bool hasArrow = succeeded(Result: parser.parseOptionalArrow());
6452 if (hasArrow) {
6453 if (parser.parseType(result&: destType))
6454 return failure();
6455 }
6456
6457 bool isMemRef = llvm::isa<MemRefType>(Val: sourceType);
6458 if (!hasArrow) {
6459 return parser.emitError(loc: parser.getCurrentLocation(),
6460 message: "pack/unpack requires '->' and destination type");
6461 }
6462
6463 if (!isMemRef)
6464 resultType = destType;
6465
6466 if (parser.resolveOperand(operand: source, type: sourceType, result&: result.operands) ||
6467 parser.resolveOperand(operand: dest, type: destType, result&: result.operands))
6468 return failure();
6469
6470 if (!dynamicTiles.empty() &&
6471 parser.resolveOperands(operands&: dynamicTiles, type: parser.getBuilder().getIndexType(),
6472 result&: result.operands))
6473 return failure();
6474
6475 result.addAttribute(name: "static_inner_tiles",
6476 attr: parser.getBuilder().getDenseI64ArrayAttr(values: staticTiles));
6477 result.addAttribute(name: "inner_dims_pos", attr: innerDimsPos);
6478 if (outerDimsPerm)
6479 result.addAttribute(name: "outer_dims_perm", attr: outerDimsPerm);
6480
6481 SmallVector<int32_t> segmentSizes = {
6482 1, 1, 0, static_cast<int32_t>(dynamicTiles.size())};
6483 result.addAttribute(name: "operandSegmentSizes",
6484 attr: parser.getBuilder().getDenseI32ArrayAttr(values: segmentSizes));
6485
6486 if (!isMemRef)
6487 result.addTypes(newTypes: resultType);
6488
6489 return success();
6490}
6491
6492void UnPackOp::print(OpAsmPrinter &p) {
6493 p << " " << getSource();
6494
6495 if (!getOuterDimsPerm().empty()) {
6496 p << " outer_dims_perm = [";
6497 llvm::interleaveComma(c: getOuterDimsPerm(), os&: p);
6498 p << "]";
6499 }
6500
6501 p << " inner_dims_pos = [";
6502 llvm::interleaveComma(c: getInnerDimsPos(), os&: p);
6503 p << "]";
6504
6505 p << " inner_tiles = ";
6506 printDynamicIndexList(printer&: p, op: *this, values: getInnerTiles(), integers: getStaticInnerTilesAttr());
6507
6508 p << " into " << getDest();
6509
6510 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary(),
6511 elidedAttrs: {"static_inner_tiles", "inner_dims_pos",
6512 "outer_dims_perm", "operandSegmentSizes"});
6513
6514 p << " : " << getSource().getType();
6515 p << " -> " << getDest().getType();
6516}
6517
6518LogicalResult
6519UnPackOp::reifyResultShapes(OpBuilder &builder,
6520 ReifiedRankedShapedTypeDims &reifiedReturnShapes) {
6521 return reifyResultShapesImpl(op: *this, builder, reifiedReturnShapes);
6522}
6523
6524DenseMap<int64_t, OpFoldResult> UnPackOp::getDimAndTileMapping() {
6525 return getDimAndTileMappingImpl(op: *this);
6526}
6527
6528SmallVector<OpFoldResult> UnPackOp::getMixedTiles() {
6529 return getMixedTilesImpl(op: *this);
6530}
6531
6532SmallVector<int64_t> UnPackOp::getStaticTiles() {
6533 return getStaticTilesImpl(op: *this);
6534}
6535
6536ArrayRef<int64_t> UnPackOp::getAllOuterDims() {
6537 ShapedType destType = getDestType();
6538 int64_t destRank = destType.getRank();
6539 return getSourceType().getShape().take_front(N: destRank);
6540}
6541
6542SmallVector<int64_t> UnPackOp::getTiledOuterDims() {
6543 auto innerDimsPos = getInnerDimsPos();
6544 SmallVector<int64_t> outerDims(getAllOuterDims());
6545 SmallVector<int64_t> res;
6546
6547 // Recover the original order of the outer dims.
6548 SmallVector<int64_t> outerDimPermInv(getOuterDimsPerm());
6549 invertPermutationVector(permutation: outerDimPermInv);
6550 if (!outerDimPermInv.empty())
6551 applyPermutationToVector(inVec&: outerDims, permutation: outerDimPermInv);
6552
6553 // Collect the outer dims corresponding to the tilled inner dims.
6554 for (auto index : innerDimsPos)
6555 res.push_back(Elt: outerDims[index]);
6556
6557 return res;
6558}
6559
6560LogicalResult UnPackOp::verify() {
6561 return commonVerifierPackAndUnPackOp(packOrUnPack: *this);
6562}
6563
6564Speculation::Speculatability UnPackOp::getSpeculatability() {
6565 if (!hasPureTensorSemantics())
6566 return Speculation::NotSpeculatable;
6567 // See PackOp::getSpeculatability.
6568 if (!areTilesAndTiledDimsAllConstant(op: *this))
6569 return Speculation::NotSpeculatable;
6570
6571 return Speculation::Speculatable;
6572}
6573
6574void UnPackOp::build(OpBuilder &builder, OperationState &state, Value source,
6575 Value dest, ArrayRef<int64_t> innerDimsPos,
6576 ArrayRef<OpFoldResult> innerTiles,
6577 ArrayRef<int64_t> outerDimsPerm) {
6578 assert(innerDimsPos.size() == innerTiles.size() &&
6579 "number of tile sizes specified must match the specified number of "
6580 "original dimensions to be tiled");
6581 SmallVector<int64_t> staticTileSizes;
6582 SmallVector<Value> dynamicTileSizes;
6583 dispatchIndexOpFoldResults(ofrs: innerTiles, dynamicVec&: dynamicTileSizes, staticVec&: staticTileSizes);
6584 build(odsBuilder&: builder, odsState&: state, result: dest.getType(), source, dest,
6585 outer_dims_perm: outerDimsPerm.empty() ? nullptr
6586 : builder.getDenseI64ArrayAttr(values: outerDimsPerm),
6587 inner_dims_pos: builder.getDenseI64ArrayAttr(values: innerDimsPos), inner_tiles: dynamicTileSizes,
6588 static_inner_tiles: builder.getDenseI64ArrayAttr(values: staticTileSizes));
6589}
6590
6591Value UnPackOp::createDestinationTensor(OpBuilder &b, Location loc,
6592 Value source,
6593 ArrayRef<OpFoldResult> innerTileSizes,
6594 ArrayRef<int64_t> innerDimsPos,
6595 ArrayRef<int64_t> outerDimsPerm) {
6596 AffineExpr sym0, sym1;
6597 bindSymbols(ctx: b.getContext(), exprs&: sym0, exprs&: sym1);
6598 auto dimMul = [&](OpFoldResult v1, OpFoldResult v2) -> OpFoldResult {
6599 return affine::makeComposedFoldedAffineApply(b, loc, expr: sym0 * sym1, operands: {v1, v2});
6600 };
6601
6602 SmallVector<OpFoldResult> mixedSizes;
6603 auto srcType = llvm::cast<RankedTensorType>(Val: source.getType());
6604 for (auto i :
6605 llvm::seq<unsigned>(Begin: 0, End: srcType.getRank() - innerTileSizes.size())) {
6606 if (srcType.isDynamicDim(idx: i))
6607 mixedSizes.push_back(
6608 Elt: tensor::DimOp::create(builder&: b, location: loc, source, index: i).getResult());
6609 else
6610 mixedSizes.push_back(Elt: b.getIndexAttr(value: srcType.getDimSize(idx: i)));
6611 }
6612 if (!outerDimsPerm.empty()) {
6613 applyPermutationToVector<OpFoldResult>(
6614 inVec&: mixedSizes, permutation: invertPermutationVector(permutation: outerDimsPerm));
6615 }
6616
6617 for (auto [dimPos, tileSize] : llvm::zip_equal(t&: innerDimsPos, u&: innerTileSizes))
6618 mixedSizes[dimPos] = dimMul(mixedSizes[dimPos], tileSize);
6619
6620 auto elemType = srcType.getElementType();
6621 return tensor::EmptyOp::create(builder&: b, location: loc, sizes: mixedSizes, elementType: elemType);
6622}
6623
6624UnPackOp UnPackOp::createTransposedClone(OpBuilder &b, Location loc,
6625 Value transposedSource,
6626 ArrayRef<int64_t> innerPermutation,
6627 ArrayRef<int64_t> outerPermutation) {
6628 PackOrUnPackTransposeResult metadata = commonPermutationOfPackAndUnPackOp(
6629 packOrUnPackOp: *this, innerPermutation, outerPermutation);
6630 return UnPackOp::create(builder&: b, location: loc, source: transposedSource, dest: getDest(),
6631 innerDimsPos: metadata.innerDimsPos, innerTiles: metadata.innerTiles,
6632 outerDimsPerm: metadata.outerDimsPerm);
6633}
6634
6635/// Returns true if the `srcShape` or `destShape` is different from the one in
6636/// `op` and populates each with the inferred static shape.
6637static bool inferStaticShape(UnPackOp op, SmallVectorImpl<int64_t> &srcShape,
6638 SmallVectorImpl<int64_t> &destShape) {
6639 bool changeNeeded = false;
6640 srcShape.assign(in_start: op.getSourceType().getShape().begin(),
6641 in_end: op.getSourceType().getShape().end());
6642 destShape.assign(in_start: op.getDestType().getShape().begin(),
6643 in_end: op.getDestType().getShape().end());
6644 llvm::SmallSetVector<int64_t, 4> innerDims;
6645 innerDims.insert_range(R: op.getInnerDimsPos());
6646 SmallVector<int64_t> inverseOuterDimsPerm;
6647 if (!op.getOuterDimsPerm().empty())
6648 inverseOuterDimsPerm = invertPermutationVector(permutation: op.getOuterDimsPerm());
6649 int destRank = op.getDestRank();
6650 for (auto i : llvm::seq<int64_t>(Begin: 0, End: destRank)) {
6651 if (innerDims.contains(key: i))
6652 continue;
6653 int64_t srcPos = i;
6654 int64_t destPos = i;
6655 if (!inverseOuterDimsPerm.empty())
6656 srcPos = inverseOuterDimsPerm[destPos];
6657 if (ShapedType::isDynamic(dValue: srcShape[srcPos]) ==
6658 ShapedType::isDynamic(dValue: destShape[destPos])) {
6659 continue;
6660 }
6661 int64_t size = srcShape[srcPos];
6662 if (ShapedType::isDynamic(dValue: size))
6663 size = destShape[destPos];
6664 srcShape[srcPos] = size;
6665 destShape[destPos] = size;
6666 changeNeeded = true;
6667 }
6668 return changeNeeded;
6669}
6670
6671LogicalResult UnPackOp::canonicalize(UnPackOp unPackOp,
6672 PatternRewriter &rewriter) {
6673 // Pack/unpack memref transformations are unsupported. The memref forms
6674 // are mainly for bufferization and scalar lowering. Other uses are not
6675 // recommended, see #225650 for details.
6676 if (!unPackOp.hasPureTensorSemantics())
6677 return failure();
6678
6679 /// unpack(pack(x)) -> x
6680 if (PackOp packOp = unPackOp.getSource().getDefiningOp<PackOp>()) {
6681 if (packOp.getSourceType() != unPackOp.getDestType())
6682 return failure();
6683 if (packOp.getPaddingValue() ||
6684 !hasSameInnerOuterAttribute(packOp, unPackOp) ||
6685 !haveSameTiles(packOp, unPackOp))
6686 return failure();
6687 rewriter.replaceOp(op: unPackOp, newValues: packOp.getSource());
6688 return success();
6689 }
6690 /// unpack(destinationStyleOp(x)) -> unpack(x)
6691 if (auto dstStyleOp =
6692 unPackOp.getDest().getDefiningOp<DestinationStyleOpInterface>()) {
6693 auto destValue = cast<OpResult>(Val: unPackOp.getDest());
6694 Value newDest = dstStyleOp.getDpsInits()[destValue.getResultNumber()];
6695 rewriter.modifyOpInPlace(root: unPackOp,
6696 callable: [&]() { unPackOp.setDpsInitOperand(i: 0, value: newDest); });
6697 return success();
6698 }
6699 /// extract_slice(unpack(x into y)) -> unpack(x into extract_slice(y))
6700 if (unPackOp->hasOneUse()) {
6701 auto extractSliceUser =
6702 dyn_cast<tensor::ExtractSliceOp>(Val: *unPackOp->getUsers().begin());
6703 if (extractSliceUser && unPackOp.canFoldSliceOp(sliceOp: extractSliceUser)) {
6704 OpBuilder::InsertionGuard g(rewriter);
6705 rewriter.setInsertionPoint(unPackOp);
6706 auto newDest = tensor::ExtractSliceOp::create(
6707 builder&: rewriter, location: unPackOp->getLoc(), source: unPackOp.getDest(),
6708 offsets: extractSliceUser.getMixedOffsets(), sizes: extractSliceUser.getMixedSizes(),
6709 strides: extractSliceUser.getMixedStrides());
6710 rewriter.modifyOpInPlace(root: unPackOp, callable: [&]() {
6711 unPackOp.setDpsInitOperand(i: 0, value: newDest);
6712 unPackOp.getResult().setType(newDest.getType());
6713 });
6714 rewriter.replaceOp(op: extractSliceUser, newOp: unPackOp);
6715 return success();
6716 }
6717 }
6718
6719 // Insert tensor.cast ops if static shape inference is available..
6720 SmallVector<int64_t> srcShape, destShape;
6721 if (inferStaticShape(op: unPackOp, srcShape, destShape)) {
6722 Location loc = unPackOp.getLoc();
6723 Value source = unPackOp.getSource();
6724 if (srcShape != unPackOp.getSourceType().getShape()) {
6725 auto newSrcType = unPackOp.getSourceType().clone(shape: srcShape);
6726 source = tensor::CastOp::create(builder&: rewriter, location: loc, dest: newSrcType,
6727 source: unPackOp.getSource());
6728 }
6729 Value dest = unPackOp.getDest();
6730 if (destShape != unPackOp.getDestType().getShape()) {
6731 auto newDestType = unPackOp.getDestType().clone(shape: destShape);
6732 dest = tensor::CastOp::create(builder&: rewriter, location: loc, dest: newDestType,
6733 source: unPackOp.getDest());
6734 }
6735 UnPackOp newOp = UnPackOp::create(
6736 builder&: rewriter, location: loc, source, dest, innerDimsPos: unPackOp.getInnerDimsPos(),
6737 innerTiles: unPackOp.getMixedTiles(), outerDimsPerm: unPackOp.getOuterDimsPerm());
6738 rewriter.replaceOpWithNewOp<tensor::CastOp>(
6739 op: unPackOp, args: unPackOp.getResult().getType(), args: newOp.getResult());
6740 return success();
6741 }
6742
6743 return failure();
6744}
6745
6746bool UnPackOp::canFoldSliceOp(tensor::ExtractSliceOp sliceOp) {
6747 // Rank-reduced folding is not supported.
6748 if (sliceOp.getResultType().getRank() != this->getDestType().getRank())
6749 return false;
6750 if (!areAllConstantIntValue(ofrs: sliceOp.getMixedOffsets(), value: 0) ||
6751 !areAllConstantIntValue(ofrs: sliceOp.getMixedStrides(), value: 1))
6752 return false;
6753 RankedTensorType unpackedTypeAfterFold = sliceOp.getResultType();
6754 SmallVector<int64_t> outerShapeWithoutTranspose =
6755 getPackedOuterShapeWithoutTransposition(packOrUnPack: *this);
6756 SmallVector<bool> areOuterDimsTiled(outerShapeWithoutTranspose.size(), false);
6757 for (auto [pos, tileSize] :
6758 llvm::zip_equal(t: this->getInnerDimsPos(), u: this->getStaticInnerTiles())) {
6759 areOuterDimsTiled[pos] = true;
6760 if (unpackedTypeAfterFold.isDynamicDim(idx: pos))
6761 return false;
6762 if (ShapedType::isDynamic(dValue: outerShapeWithoutTranspose[pos]))
6763 return false;
6764 if (ShapedType::isDynamic(dValue: tileSize))
6765 return false;
6766 int64_t paddingSize = outerShapeWithoutTranspose[pos] * tileSize -
6767 unpackedTypeAfterFold.getDimSize(idx: pos);
6768 if (paddingSize >= tileSize)
6769 return false;
6770 }
6771 // extract_slice must not affect dimensions that are not being unpacked
6772 for (int64_t pos = 0, e = outerShapeWithoutTranspose.size(); pos < e; ++pos) {
6773 if (areOuterDimsTiled[pos])
6774 continue;
6775 int64_t dim = outerShapeWithoutTranspose[pos];
6776 if (ShapedType::isDynamic(dValue: dim))
6777 return false;
6778 if (dim != unpackedTypeAfterFold.getDimSize(idx: pos))
6779 return false;
6780 }
6781 return true;
6782}
6783
6784bool UnPackOp::isLikeUnPad() {
6785 ShapedType packedTensorType = getSourceType();
6786 return isLikePadUnPad(packOp: *this, packedTensorType);
6787}
6788
6789::mlir::LogicalResult
6790UnPackOp::fold(FoldAdaptor adaptor,
6791 ::llvm::SmallVectorImpl<OpFoldResult> &results) {
6792 // Pack/unpack memref transformations are unsupported. The memref forms
6793 // are mainly for bufferization and scalar lowering. Other uses are not
6794 // recommended, see #225650 for details.
6795 if (!hasPureTensorSemantics())
6796 return failure();
6797
6798 if (OpFoldResult reshapedSource = reshapeConstantSource(
6799 source: llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getSource()),
6800 result: cast<TensorType>(Val: getResult().getType()))) {
6801 results.push_back(Elt: reshapedSource);
6802 return success();
6803 }
6804 return failure();
6805}
6806
6807/// Folds a tensor.cast op into a consuming UnPackOp op if the
6808/// `tensor.cast` has source that is more static than the consuming op.
6809///
6810/// Example:
6811/// ```mlir
6812/// %1 = tensor.cast %0 : tensor<1x1x8x1xi32> to tensor<1x1x?x1xi32>
6813/// %2 = tensor.unpack %1 ... : tensor<1x1x?x1xi32> -> tensor<7x?xi32>
6814/// ```
6815///
6816/// folds into:
6817///
6818/// ```mlir
6819/// %2 = tensor.unpack %0 ... tensor<1x1x8x1xi32> -> tensor<7x?xi32>
6820/// ```
6821struct FoldTensorCastUnPackOp : public OpRewritePattern<UnPackOp> {
6822 using OpRewritePattern<UnPackOp>::OpRewritePattern;
6823
6824 LogicalResult matchAndRewrite(UnPackOp op,
6825 PatternRewriter &rewriter) const override {
6826 // Pack/unpack memref transformations are unsupported. The memref forms
6827 // are mainly for bufferization and scalar lowering. Other uses are not
6828 // recommended, see #225650 for details.
6829 if (!op.hasPureTensorSemantics())
6830 return failure();
6831
6832 if (!tensor::hasFoldableTensorCastOperand(op))
6833 return failure();
6834
6835 SmallVector<Type> newResultTypes(op->getResultTypes());
6836 SmallVector<Value> newOperands =
6837 tensor::getUpdatedOperandsAfterCastOpFolding(op, newResTy&: newResultTypes);
6838 Value sourceTensor = newOperands[0];
6839
6840 // Get the updated mixed-tile-sizes attribute.
6841 SmallVector<OpFoldResult> newMixedTileSizes = getNewMixedTileSizes(
6842 rewriter, newPackedTy: sourceTensor.getType(), mixedTiles: op.getMixedTiles());
6843
6844 // Clone op.
6845 // TODO: Strictly speaking, discardable attributes should be _discarded_ at
6846 // this point. However, in practice, we use them for things that we'd like
6847 // to preserve. Implement a better abstraction.
6848 UnPackOp newOp = UnPackOp::create(builder&: rewriter, location: op.getLoc(), source: sourceTensor,
6849 dest: newOperands[1], innerDimsPos: op.getInnerDimsPos(),
6850 innerTiles: newMixedTileSizes, outerDimsPerm: op.getOuterDimsPerm());
6851 newOp->setDiscardableAttrs(op->getDiscardableAttrDictionary());
6852
6853 // Replace op.
6854 Value oldResult = op.getResult();
6855 Value newResult = newOp.getResult();
6856 Value replacement =
6857 (newResult.getType() != oldResult.getType())
6858 ? tensor::CastOp::create(builder&: rewriter, location: op->getLoc(),
6859 dest: oldResult.getType(), source: newResult)
6860 : newResult;
6861
6862 rewriter.replaceOp(op, newValues: {replacement});
6863
6864 return success();
6865 }
6866};
6867
6868//===----------------------------------------------------------------------===//
6869// BatchReduceMatmulOp
6870//===----------------------------------------------------------------------===//
6871SmallVector<utils::IteratorType> BatchReduceMatmulOp::getIteratorTypesArray() {
6872 return SmallVector<utils::IteratorType>{
6873 utils::IteratorType::reduction, utils::IteratorType::parallel,
6874 utils::IteratorType::parallel, utils::IteratorType::reduction};
6875}
6876
6877SmallVector<AffineMap>
6878BatchReduceMatmulOp::getDefaultIndexingMaps(MLIRContext *context) {
6879 AffineExpr d0, d1, d2, d3;
6880 SmallVector<AffineMap> indexingMaps;
6881 bindDims(ctx: context, exprs&: d0, exprs&: d1, exprs&: d2, exprs&: d3);
6882 indexingMaps.push_back(Elt: AffineMap::get(dimCount: 4, symbolCount: 0, results: {d0, d1, d3}, context));
6883 indexingMaps.push_back(Elt: AffineMap::get(dimCount: 4, symbolCount: 0, results: {d0, d3, d2}, context));
6884 indexingMaps.push_back(Elt: AffineMap::get(dimCount: 4, symbolCount: 0, results: {d1, d2}, context));
6885 return indexingMaps;
6886}
6887
6888bool BatchReduceMatmulOp::isDefaultIndexingMaps(Attribute attr) {
6889 ArrayAttr maps = dyn_cast<ArrayAttr>(Val&: attr);
6890 if (!maps)
6891 return false;
6892 if (maps.size() != 3)
6893 return false;
6894 auto positions = getAffineResultPositions(maps);
6895 if (failed(Result: positions))
6896 return false;
6897 return (*positions)[0] == SmallVector<int64_t>{0, 1, 3} &&
6898 (*positions)[1] == SmallVector<int64_t>{0, 3, 2} &&
6899 (*positions)[2] == SmallVector<int64_t>{1, 2};
6900}
6901unsigned BatchReduceMatmulOp::getNumRegionArgs() { return 3; }
6902
6903std::string BatchReduceMatmulOp::getLibraryCallName() {
6904 return generateLibraryCallName(op: getOperation());
6905}
6906
6907/// Check if the op has broadcast and/or transpose semantic. Returns true if
6908/// the user defined indexing maps are not equal to default map.
6909bool BatchReduceMatmulOp::hasUserDefinedMaps() {
6910 SmallVector<AffineMap, 3> defaultMaps =
6911 getDefaultIndexingMaps(context: this->getContext());
6912 SmallVector<AffineMap, 3> explicitMaps = getIndexingMapsArray();
6913 return defaultMaps != explicitMaps;
6914}
6915
6916/// Returns true if the given bcastMap map is a valid broadcast map. A valid
6917/// broadcast map must include K dimension.
6918/// TODO: Strict inclusion of K dimension in the broadcast map is not
6919/// necessary for both input matrices simultaneously. We can relax this
6920/// condition to have K dimension for one input matrix map and infer the K
6921/// dimension for other input matrix map from the one already having K
6922/// dimension.
6923bool BatchReduceMatmulOp::isValidLhsRhsBroadcastMap(AffineMap bcastMap,
6924 bool isLHS) {
6925 assert(bcastMap.getNumResults() < 3 &&
6926 "Expected less than 3 result dim expr.");
6927 bool isValid = false;
6928 enum Indices { batchPos, mPos, nPos, kPos };
6929 if (bcastMap.getNumResults() == 1) {
6930 AffineExpr expr = bcastMap.getResult(idx: 0);
6931 isValid = expr.isFunctionOfDim(position: kPos);
6932 } else if (bcastMap.getNumResults() == 2) {
6933 AffineExpr expr0 = bcastMap.getResult(idx: 0);
6934 AffineExpr expr1 = bcastMap.getResult(idx: 1);
6935 isValid =
6936 isLHS ? ((expr0.isFunctionOfDim(position: batchPos) ||
6937 expr0.isFunctionOfDim(position: mPos)) &&
6938 expr1.isFunctionOfDim(position: kPos))
6939 : ((expr0.isFunctionOfDim(position: batchPos) &&
6940 expr1.isFunctionOfDim(position: kPos)) ||
6941 (expr0.isFunctionOfDim(position: kPos) && expr1.isFunctionOfDim(position: nPos)));
6942 }
6943 return isValid;
6944}
6945
6946void BatchReduceMatmulOp::regionBuilder(
6947 ImplicitLocOpBuilder &b, Block &block, ArrayRef<NamedAttribute> attrs,
6948 function_ref<InFlightDiagnostic()> emitError) {
6949 if (emitError && block.getNumArguments() != 3) {
6950 emitError() << "BatchReduceMatmulOp regionBuilder expects 3 args, got "
6951 << block.getNumArguments();
6952 return;
6953 }
6954 assert(block.getNumArguments() == 3 &&
6955 "BatchReduceMatmulOp regionBuilder expects 3 args");
6956 RegionBuilderHelper helper(b, block);
6957 SmallVector<Value> yields;
6958
6959 auto toType = block.getArgument(i: 2).getType();
6960 Value castValA =
6961 helper.buildTypeFn(typeFn: TypeFn::cast_signed, toType, operand: block.getArgument(i: 0));
6962 Value castValB =
6963 helper.buildTypeFn(typeFn: TypeFn::cast_signed, toType, operand: block.getArgument(i: 1));
6964 Value mulVal =
6965 helper.buildBinaryFn(binaryFn: BinaryFn::mul, arg0: castValA, arg1: castValB, emitError);
6966 if (!castValA || !castValB || !mulVal)
6967 return;
6968 Value addVal =
6969 helper.buildBinaryFn(binaryFn: BinaryFn::add, arg0: block.getArgument(i: 2), arg1: mulVal);
6970 if (!addVal)
6971 return;
6972 yields.push_back(Elt: addVal);
6973 helper.yieldOutputs(values: yields);
6974}
6975
6976ParseResult BatchReduceMatmulOp::parse(OpAsmParser &parser,
6977 OperationState &result) {
6978 SmallVector<Attribute, 3> indexingMapsAttr;
6979 Attribute mapAttr;
6980 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "indexing_maps"))) {
6981 if (parser.parseEqual())
6982 return failure();
6983 if (parser.parseLSquare())
6984 return failure();
6985
6986 do {
6987 if (parser.parseAttribute(result&: mapAttr))
6988 return failure();
6989 if (!isa<AffineMapAttr>(Val: mapAttr)) {
6990 return parser.emitError(loc: parser.getCurrentLocation(),
6991 message: "expected affine map attribute");
6992 }
6993 indexingMapsAttr.push_back(Elt: mapAttr);
6994
6995 if (parser.parseOptionalComma())
6996 break;
6997 } while (true);
6998
6999 if (parser.parseRSquare())
7000 return failure();
7001 }
7002 // Initialize indexingMaps, if not supplied explicitly.
7003 if (indexingMapsAttr.empty()) {
7004 indexingMapsAttr = llvm::map_to_vector(
7005 C: BatchReduceMatmulOp::getDefaultIndexingMaps(context: parser.getContext()),
7006 F: [](AffineMap map) -> Attribute { return AffineMapAttr::get(value: map); });
7007 }
7008 result.addAttribute(name: "indexing_maps",
7009 attr: parser.getBuilder().getArrayAttr(value: indexingMapsAttr));
7010 return ::parseNamedStructuredOp(parser, result,
7011 numRegionArgs: BatchReduceMatmulOp::getNumRegionArgs(),
7012 regionBuilder: BatchReduceMatmulOp::getRegionBuilder());
7013}
7014
7015void BatchReduceMatmulOp::print(OpAsmPrinter &p) {
7016 SmallVector<Attribute, 3> indexingMaps = llvm::map_to_vector(
7017 C: BatchReduceMatmulOp::getDefaultIndexingMaps(context: getContext()),
7018 F: [](AffineMap map) -> Attribute { return AffineMapAttr::get(value: map); });
7019
7020 if (!llvm::equal(LRange: getIndexingMaps(), RRange&: indexingMaps)) {
7021 p << " indexing_maps = [";
7022 llvm::interleaveComma(c: getIndexingMaps(), os&: p,
7023 each_fn: [&](Attribute attr) { p.printAttribute(attr); });
7024 p << "]";
7025 }
7026
7027 SmallVector<StringRef, 3> elidedAttrs = {
7028 "operandSegmentSizes", "linalg.memoized_indexing_maps", "indexing_maps"};
7029 ::printNamedStructuredOp(p, op: getOperation(), inputs: getInputs(), outputs: getOutputs(),
7030 elidedAttrs);
7031}
7032
7033/// Verify the user defined indexing maps.
7034LogicalResult BatchReduceMatmulOp::verify() {
7035 // Verification of pure batch_reduce_matmul is handled by
7036 // verifyStructuredOpInterface().
7037 if (!hasUserDefinedMaps())
7038 return success();
7039
7040 for (unsigned opIndex = 0; opIndex < 3; opIndex++) {
7041 if (failed(Result: verifyExtendedBatchVariantMatmulSemantic(batchVariantMatmulOp: *this, opIndex)))
7042 return failure();
7043 }
7044 return success();
7045}
7046LogicalResult BatchReduceMatmulOp::fold(FoldAdaptor,
7047 SmallVectorImpl<OpFoldResult> &) {
7048 return memref::foldMemRefCast(op: *this);
7049}
7050void BatchReduceMatmulOp::getEffects(
7051 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
7052 &effects) {
7053 if (hasPureTensorSemantics())
7054 return;
7055 getGenericEffectsImpl(effects, linalgOp: cast<LinalgOp>(Val: getOperation()));
7056}
7057
7058Speculation::Speculatability BatchReduceMatmulOp::getSpeculatability() {
7059 return getGenericSpeculatabilityImpl(linalgOp: cast<LinalgOp>(Val: getOperation()));
7060}
7061
7062//===----------------------------------------------------------------------===//
7063// ScaledContractOp
7064//===----------------------------------------------------------------------===//
7065
7066SmallVector<utils::IteratorType> ScaledContractOp::getIteratorTypesArray() {
7067 AffineMap outAffineMap = getIndexingMapsArray().pop_back_val();
7068 // Infer iterator types based on the output.
7069 SmallVector<bool> dimsInOutput(outAffineMap.getNumDims(), false);
7070 for (auto result : outAffineMap.getResults()) {
7071 auto dimExpr = dyn_cast<AffineDimExpr>(Val&: result);
7072 assert(dimExpr && "affine_map is a projected permutation");
7073 dimsInOutput[dimExpr.getPosition()] = true;
7074 }
7075
7076 SmallVector<utils::IteratorType> iteratorTypes;
7077 for (auto dimOccursInOutput : dimsInOutput)
7078 iteratorTypes.push_back(Elt: dimOccursInOutput ? utils::IteratorType::parallel
7079 : utils::IteratorType::reduction);
7080
7081 return iteratorTypes;
7082}
7083
7084unsigned ScaledContractOp::getNumRegionArgs() { return 5; }
7085
7086/// Implement block region builder, which is called by 'fillStructuredOpRegion'.
7087void ScaledContractOp::regionBuilder(
7088 ImplicitLocOpBuilder &b, Block &block, ArrayRef<NamedAttribute> attrs,
7089 function_ref<InFlightDiagnostic()> emitError) {
7090 if (emitError && block.getNumArguments() != 5) {
7091 emitError() << "ScaledContractOp regionBuilder expects 5 args, got "
7092 << block.getNumArguments();
7093 return;
7094 }
7095 assert(block.getNumArguments() == 5 &&
7096 "ScaledContractOp regionBuilder expects 5 args");
7097 RegionBuilderHelper helper(b, block);
7098
7099 TypeFn castSignedness = TypeFn::cast_signed;
7100 auto castIter = llvm::find_if(Range&: attrs, P: [&](const NamedAttribute &attr) {
7101 return attr.getName() == "cast";
7102 });
7103 if (castIter != attrs.end()) {
7104 if (auto attr = llvm::dyn_cast<TypeFnAttr>(Val: castIter->getValue()))
7105 castSignedness = attr.getValue();
7106 }
7107
7108 // TODO: Support fields with operators besides mult & add.
7109 Type outType = block.getArgument(i: 4).getType();
7110
7111 // Build input data value scaling.
7112 // Uses specialized arith ops when possible.
7113 // Otherwise, constructs computation manually.
7114 auto buildScaledValue = [&](Value data, Value scale) -> Value {
7115 auto dataFloatTy = dyn_cast<FloatType>(Val: data.getType());
7116 auto outFloatTy = dyn_cast<FloatType>(Val&: outType);
7117 if (dataFloatTy && dyn_cast<FloatType>(Val: scale.getType()) && outFloatTy) {
7118 unsigned dataWidth = dataFloatTy.getWidth();
7119 unsigned outWidth = outFloatTy.getWidth();
7120 if (dataWidth < outWidth)
7121 return arith::ScalingExtFOp::create(builder&: b, out: outType, in: data, scale,
7122 /*fastmath=*/nullptr);
7123 }
7124 Value dataAtOutType = helper.buildTypeFn(typeFn: castSignedness, toType: outType, operand: data);
7125 Value scaleAtOutType = helper.buildTypeFn(typeFn: castSignedness, toType: outType, operand: scale);
7126 return helper.buildBinaryFn(binaryFn: BinaryFn::mul, arg0: dataAtOutType, arg1: scaleAtOutType,
7127 emitError);
7128 };
7129
7130 Value scaledLhs =
7131 buildScaledValue(block.getArgument(i: 0), block.getArgument(i: 1));
7132 if (!scaledLhs)
7133 return;
7134 Value scaledRhs =
7135 buildScaledValue(block.getArgument(i: 2), block.getArgument(i: 3));
7136 if (!scaledRhs)
7137 return;
7138 Value productAtOutType =
7139 helper.buildBinaryFn(binaryFn: BinaryFn::mul, arg0: scaledLhs, arg1: scaledRhs, emitError);
7140 if (!productAtOutType)
7141 return;
7142 Value result = helper.buildBinaryFn(binaryFn: BinaryFn::add, arg0: block.getArgument(i: 4),
7143 arg1: productAtOutType, emitError);
7144 if (!result)
7145 return;
7146 helper.yieldOutputs(values: {result});
7147}
7148
7149ParseResult ScaledContractOp::parse(OpAsmParser &parser,
7150 OperationState &result) {
7151 FailureOr<ArrayAttr> indexingMapsAttr = parseIndexingMapsAttr(parser);
7152 if (failed(Result: indexingMapsAttr) || *indexingMapsAttr == nullptr)
7153 return parser.emitError(loc: parser.getCurrentLocation(),
7154 message: "expected 'indexing_maps' attribute");
7155 result.addAttribute(name: "indexing_maps", attr: *indexingMapsAttr);
7156
7157 return parseNamedStructuredOp(parser, result, numRegionArgs: getNumRegionArgs(),
7158 regionBuilder);
7159}
7160
7161void ScaledContractOp::print(OpAsmPrinter &p) {
7162 p << " indexing_maps = " << llvm::interleaved_array(R: getIndexingMaps());
7163 printNamedStructuredOp(
7164 p, op: getOperation(), inputs: getInputs(), outputs: getOutputs(),
7165 /*elidedAttrs=*/{"indexing_maps", "operandSegmentSizes"});
7166}
7167
7168LogicalResult ScaledContractOp::verify() {
7169 int iterationSpaceDims = -1;
7170 // Map iter space dims to #occurrences in inputs' and output's affine_maps:
7171 // e.g., inOccurrences[0] will hold #times that dim (with index) 0 is used to
7172 // access an input operand (so occurrence count can be at most 2) and
7173 // outOccurrences[1] will indicate whether dim 1 occurred in the output, etc.
7174 SmallVector<size_t> inOccurrences;
7175 SmallVector<size_t> outOccurrences;
7176
7177 // Validate inputs and contraction semantics.
7178 SmallVector<AffineMap, 5> maps = getIndexingMapsArray();
7179 if (maps.size() != 5 || getNumOperands() != 5)
7180 return emitOpError(message: "expected 5 indexing maps and operands");
7181
7182 SmallVector<Type, 5> types = llvm::to_vector(Range: getOperandTypes());
7183 Type outputElementType = getElementTypeOrSelf(type: types[4]);
7184 auto outputFloatType = dyn_cast<FloatType>(Val&: outputElementType);
7185 if (!outputFloatType)
7186 return emitOpError(message: "expected output element type to be floating-point");
7187
7188 for (Type inputType : ArrayRef<Type>(types).take_front(N: 4)) {
7189 Type inputElementType = getElementTypeOrSelf(type: inputType);
7190 if (!inputElementType.isIntOrFloat())
7191 return emitOpError(
7192 message: "expected input element types to be integer or floating-point");
7193 if (inputElementType.getIntOrFloatBitWidth() > outputFloatType.getWidth())
7194 return emitOpError(message: "expected input element type bitwidth to be no "
7195 "greater than output element type bitwidth");
7196 }
7197
7198 for (auto &&[affineMap, operandType, isInput] :
7199 llvm::zip(t: SmallVector<AffineMap>{maps[0], maps[2], maps[4]},
7200 u: SmallVector<Type>{types[0], types[2], types[4]},
7201 args: SmallVector<bool>{true, true, false})) {
7202 if (failed(Result: checkContractionAffineMapAndType(
7203 affineMap, operandType, isInput, iterationSpaceDims, inOccurrences,
7204 outOccurrences, emitError: [&]() { return emitError(); })))
7205 return failure(); // NB: Validation helper emits relevant error.
7206 }
7207
7208 if (failed(Result: verifyContractionDims(iterationSpaceDims: static_cast<size_t>(iterationSpaceDims),
7209 inOccurrences, outOccurrences,
7210 emitError: [&]() { return emitError(); })))
7211 return failure(); // NB: Validation helper emits relevant error.
7212
7213 // Validate scales and scaling semantics.
7214 auto checkScaleAffineMapAndType = [&](AffineMap affineMap,
7215 Type operandType) -> LogicalResult {
7216 // If scale's map is not a projected permutation, then it must follow
7217 // specific scaling scheme semantics.
7218 if (!affineMap.isProjectedPermutation()) {
7219 if (affineMap.getNumSymbols() > 0)
7220 return emitError(message: "scale affine_map must not contain symbols");
7221 if (affineMap.getNumResults() > affineMap.getNumInputs())
7222 return emitError(
7223 message: "scale affine_map must not have more results than inputs");
7224
7225 SmallVector<bool, 8> seen(affineMap.getNumInputs(), false);
7226 // Allow, at most, only one instance of each input dimension in the result
7227 // expressions.
7228 for (auto expr : affineMap.getResults()) {
7229 AffineDimExpr dim = nullptr;
7230 if (isa<AffineDimExpr>(Val: expr)) {
7231 // Scaling over whole dimension.
7232 dim = dyn_cast<AffineDimExpr>(Val&: expr);
7233 } else if (auto binExpr = dyn_cast<AffineBinaryOpExpr>(Val&: expr)) {
7234 // Scaling over a part of the dimension.
7235 // Note: Currently support limited to block scaling i.e.,
7236 // one scale per a fixed number of contiguous scalar elements
7237 // in a given dimension.
7238 if (binExpr.getKind() != AffineExprKind::FloorDiv)
7239 return emitError(
7240 message: "only block scale with floordiv is supported for now");
7241 auto scaleDim = dyn_cast<AffineDimExpr>(Val: binExpr.getLHS());
7242 if (!scaleDim)
7243 return emitError(message: "block scale LHS must be dim");
7244 auto scaleFactor = dyn_cast<AffineConstantExpr>(Val: binExpr.getRHS());
7245 if (!scaleFactor)
7246 return emitError(message: "block scale RHS must be constant");
7247 if (scaleFactor.getValue() <= 0)
7248 return emitError(message: "block scale factor must be positive");
7249 dim = scaleDim;
7250 } else {
7251 return emitError(message: "unsupported scaling variant");
7252 }
7253
7254 if (!dim)
7255 return emitError(message: "invalid scale affine_map result expression");
7256 if (seen[dim.getPosition()])
7257 return emitError(
7258 message: "scale affine_map must not have duplicate result dimensions");
7259 seen[dim.getPosition()] = true;
7260 }
7261 }
7262
7263 if (auto shapedType = dyn_cast<ShapedType>(Val&: operandType)) {
7264 if (affineMap.getNumResults() != shapedType.getRank())
7265 return emitError(
7266 message: "scale ranks of shaped operand and results of corresponding "
7267 "affine_map differ");
7268 } else if (affineMap.getNumResults() != 0) {
7269 return emitError(
7270 message: "scale affine_map specifies shaped access while operand has "
7271 "non-shaped type");
7272 }
7273
7274 return success();
7275 };
7276
7277 // Validate scales' maps.
7278 for (auto &&[affineMap, operandType] :
7279 llvm::zip(t: SmallVector<AffineMap>{maps[1], maps[3]},
7280 u: SmallVector<Type>{types[1], types[3]})) {
7281 if (failed(Result: checkScaleAffineMapAndType(affineMap, operandType)))
7282 return failure(); // NB: Validation helper emits relevant error.
7283 }
7284
7285 // Cross-validate maps of operand and their scale.
7286 for (auto &&[inputMap, inputType, scaleMap, scaleType] :
7287 llvm::zip(t: SmallVector<AffineMap>{maps[0], maps[2]},
7288 u: SmallVector<Type>{types[0], types[2]},
7289 args: SmallVector<AffineMap>{maps[1], maps[3]},
7290 args: SmallVector<Type>{types[1], types[3]})) {
7291 if (inputMap.getNumResults() < scaleMap.getNumResults())
7292 return emitError(message: "scale must have at most the same rank as input");
7293 if (scaleMap.getNumResults() == 0)
7294 continue;
7295
7296 auto inputShape = dyn_cast<ShapedType>(Val&: inputType).getShape();
7297 auto scaleShape = dyn_cast<ShapedType>(Val&: scaleType).getShape();
7298
7299 // Each scale dim must reference a dim present in the input map:
7300 // - a missing scale dim indicates scaling over the whole input dimension
7301 // - a scale dim with floordiv indicates reusing the scaling factor over
7302 // parts of the input dimension; the factor must match the ratio of
7303 // input dim and scale dim sizes.
7304 for (auto [scaleIdx, scaleExpr] : llvm::enumerate(First: scaleMap.getResults())) {
7305 AffineDimExpr scaleDimExpr = nullptr;
7306 std::optional<int64_t> scaleFactor;
7307 if (auto dimExpr = dyn_cast<AffineDimExpr>(Val: scaleExpr)) {
7308 // Scaling over the whole dimension.
7309 scaleDimExpr = dimExpr;
7310 } else if (auto scaleBinExpr = dyn_cast<AffineBinaryOpExpr>(Val: scaleExpr)) {
7311 // Block scaling over a part of the dimension.
7312 assert(scaleBinExpr.getKind() == AffineExprKind::FloorDiv &&
7313 "only floordiv is supported for now");
7314 auto scaleDim = dyn_cast<AffineDimExpr>(Val: scaleBinExpr.getLHS());
7315 assert(scaleDim && "block scale LHS is a dim expression");
7316 scaleDimExpr = scaleDim;
7317 scaleFactor =
7318 dyn_cast<AffineConstantExpr>(Val: scaleBinExpr.getRHS()).getValue();
7319 } else {
7320 llvm_unreachable("unknown scale expression");
7321 }
7322 assert(scaleDimExpr && "failed to find scale dim expression");
7323
7324 std::optional<unsigned> inputIdx =
7325 inputMap.getResultPosition(input: scaleDimExpr);
7326 if (!inputIdx)
7327 return emitError(
7328 message: "scale map must contain corresponding input dimensions only");
7329
7330 // Validate block scaling factor for static shapes.
7331 // For dynamic shapes, it is assumed that all sizes are correct.
7332 if (scaleFactor && inputShape[*inputIdx] != ShapedType::kDynamic &&
7333 scaleShape[scaleIdx] != ShapedType::kDynamic &&
7334 llvm::divideCeilSigned(Numerator: inputShape[*inputIdx], Denominator: *scaleFactor) !=
7335 static_cast<int64_t>(scaleShape[scaleIdx])) {
7336 return emitError() << "Invalid scale shape at dim " << *inputIdx
7337 << ", expected "
7338 << llvm::divideCeilSigned(Numerator: inputShape[*inputIdx],
7339 Denominator: *scaleFactor)
7340 << " but got " << scaleShape[scaleIdx];
7341 }
7342 }
7343 }
7344
7345 return success();
7346}
7347
7348LogicalResult ScaledContractOp::fold(FoldAdaptor,
7349 SmallVectorImpl<OpFoldResult> &) {
7350 return memref::foldMemRefCast(op: *this);
7351}
7352
7353void ScaledContractOp::getEffects(
7354 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
7355 &effects) {
7356 if (hasPureTensorSemantics())
7357 return;
7358 getGenericEffectsImpl(effects, linalgOp: cast<LinalgOp>(Val: getOperation()));
7359}
7360
7361Speculation::Speculatability ScaledContractOp::getSpeculatability() {
7362 return getGenericSpeculatabilityImpl(linalgOp: cast<LinalgOp>(Val: getOperation()));
7363}
7364
7365} // namespace linalg
7366} // namespace mlir
7367
7368//===----------------------------------------------------------------------===//
7369// LinalgDialect
7370//===----------------------------------------------------------------------===//
7371
7372void LinalgDialect::getCanonicalizationPatterns(
7373 RewritePatternSet &results) const {
7374 results.add<EraseDeadLinalgOp, FoldTensorCastConsumerOp, FoldTensorCastPackOp,
7375 FoldTensorCastUnPackOp, InferStaticShapeOfOperands>(arg: getContext());
7376}
7377
7378Operation *LinalgDialect::materializeConstant(OpBuilder &builder,
7379 Attribute value, Type type,
7380 Location loc) {
7381 return arith::ConstantOp::materialize(builder, value, type, loc);
7382}
7383