1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Arith/IR/Arith.h"
10#include "mlir/Dialect/Arith/Utils/Utils.h"
11#include "mlir/Dialect/MemRef/IR/MemRef.h"
12#include "mlir/Dialect/Utils/ReshapeOpsUtils.h"
13#include "mlir/Dialect/Utils/StaticValueUtils.h"
14#include "mlir/Dialect/Utils/VerificationUtils.h"
15#include "mlir/IR/AffineMap.h"
16#include "mlir/IR/Builders.h"
17#include "mlir/IR/BuiltinTypes.h"
18#include "mlir/IR/Matchers.h"
19#include "mlir/IR/OpDefinition.h"
20#include "mlir/IR/PatternMatch.h"
21#include "mlir/IR/TypeUtilities.h"
22#include "mlir/Interfaces/InferTypeOpInterface.h"
23#include "mlir/Interfaces/SideEffectInterfaces.h"
24#include "mlir/Interfaces/Utils/InferIntRangeCommon.h"
25#include "mlir/Interfaces/ViewLikeInterface.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/SmallBitVector.h"
28#include "llvm/ADT/SmallVectorExtras.h"
29
30using namespace mlir;
31using namespace mlir::memref;
32
33/// Materialize a single constant operation from a given attribute value with
34/// the desired resultant type.
35Operation *MemRefDialect::materializeConstant(OpBuilder &builder,
36 Attribute value, Type type,
37 Location loc) {
38 return arith::ConstantOp::materialize(builder, value, type, loc);
39}
40
41//===----------------------------------------------------------------------===//
42// Common canonicalization pattern support logic
43//===----------------------------------------------------------------------===//
44
45/// This is a common class used for patterns of the form
46/// "someop(memrefcast) -> someop". It folds the source of any memref.cast
47/// into the root operation directly.
48LogicalResult mlir::memref::foldMemRefCast(Operation *op, Value inner) {
49 bool folded = false;
50 for (OpOperand &operand : op->getOpOperands()) {
51 auto cast = operand.get().getDefiningOp<CastOp>();
52 if (cast && operand.get() != inner &&
53 !llvm::isa<UnrankedMemRefType>(Val: cast.getOperand().getType())) {
54 operand.set(cast.getOperand());
55 folded = true;
56 }
57 }
58 return success(IsSuccess: folded);
59}
60
61/// Return an unranked/ranked tensor type for the given unranked/ranked memref
62/// type.
63Type mlir::memref::getTensorTypeFromMemRefType(Type type) {
64 if (auto memref = llvm::dyn_cast<MemRefType>(Val&: type))
65 return RankedTensorType::get(shape: memref.getShape(), elementType: memref.getElementType());
66 if (auto memref = llvm::dyn_cast<UnrankedMemRefType>(Val&: type))
67 return UnrankedTensorType::get(elementType: memref.getElementType());
68 return NoneType::get(context: type.getContext());
69}
70
71OpFoldResult memref::getMixedSize(OpBuilder &builder, Location loc, Value value,
72 int64_t dim) {
73 auto memrefType = llvm::cast<MemRefType>(Val: value.getType());
74 if (memrefType.isDynamicDim(idx: dim))
75 return builder.createOrFold<memref::DimOp>(location: loc, args&: value, args&: dim);
76
77 return builder.getIndexAttr(value: memrefType.getDimSize(idx: dim));
78}
79
80SmallVector<OpFoldResult> memref::getMixedSizes(OpBuilder &builder,
81 Location loc, Value value) {
82 auto memrefType = llvm::cast<MemRefType>(Val: value.getType());
83 SmallVector<OpFoldResult> result;
84 for (int64_t i = 0; i < memrefType.getRank(); ++i)
85 result.push_back(Elt: getMixedSize(builder, loc, value, dim: i));
86 return result;
87}
88
89//===----------------------------------------------------------------------===//
90// Utility functions for propagating static information
91//===----------------------------------------------------------------------===//
92
93/// Helper function that sets values[i] to constValues[i] if the latter is a
94/// static value, as indicated by ShapedType::kDynamic.
95///
96/// If constValues[i] is dynamic, tries to extract a constant value from
97/// value[i] to allow for additional folding opportunities. Also convertes all
98/// existing attributes to index attributes. (They may be i64 attributes.)
99static void constifyIndexValues(SmallVectorImpl<OpFoldResult> &values,
100 ArrayRef<int64_t> constValues) {
101 assert(constValues.size() == values.size() &&
102 "incorrect number of const values");
103 for (auto [i, cstVal] : llvm::enumerate(First&: constValues)) {
104 Builder builder(values[i].getContext());
105 if (ShapedType::isStatic(dValue: cstVal)) {
106 // Constant value is known, use it directly.
107 values[i] = builder.getIndexAttr(value: cstVal);
108 continue;
109 }
110 if (std::optional<int64_t> cst = getConstantIntValue(ofr: values[i])) {
111 // Try to extract a constant or convert an existing to index.
112 values[i] = builder.getIndexAttr(value: *cst);
113 }
114 }
115}
116
117/// Helper function to retrieve a lossless memory-space cast, and the
118/// corresponding new result memref type.
119static std::tuple<MemorySpaceCastOpInterface, PtrLikeTypeInterface, Type>
120getMemorySpaceCastInfo(BaseMemRefType resultTy, Value src) {
121 MemorySpaceCastOpInterface castOp =
122 MemorySpaceCastOpInterface::getIfPromotableCast(value: src);
123
124 // Bail if the cast is not lossless.
125 if (!castOp)
126 return {};
127
128 // Transform the source and target type of `castOp` to have the same metadata
129 // as `resultTy`. Bail if not possible.
130 FailureOr<PtrLikeTypeInterface> srcTy = resultTy.clonePtrWith(
131 memorySpace: castOp.getSourcePtr().getType().getMemorySpace(), elementType: std::nullopt);
132 if (failed(Result: srcTy))
133 return {};
134
135 FailureOr<PtrLikeTypeInterface> tgtTy = resultTy.clonePtrWith(
136 memorySpace: castOp.getTargetPtr().getType().getMemorySpace(), elementType: std::nullopt);
137 if (failed(Result: tgtTy))
138 return {};
139
140 // Check if this is a valid memory-space cast.
141 if (!castOp.isValidMemorySpaceCast(tgt: *tgtTy, src: *srcTy))
142 return {};
143
144 return std::make_tuple(args&: castOp, args&: *tgtTy, args&: *srcTy);
145}
146
147/// Implementation of `bubbleDownCasts` method for memref operations that
148/// return a single memref result.
149template <typename ConcreteOpTy>
150static FailureOr<std::optional<SmallVector<Value>>>
151bubbleDownCastsPassthroughOpImpl(ConcreteOpTy op, OpBuilder &builder,
152 OpOperand &src) {
153 auto [castOp, tgtTy, resTy] = getMemorySpaceCastInfo(op.getType(), src.get());
154 // Bail if we cannot cast.
155 if (!castOp)
156 return failure();
157
158 // Create the new operands.
159 SmallVector<Value> operands;
160 llvm::append_range(operands, op->getOperands());
161 operands[src.getOperandNumber()] = castOp.getSourcePtr();
162
163 // Create the new op and results.
164 auto newOp = ConcreteOpTy::create(
165 builder, op.getLoc(), TypeRange(resTy), operands, op.getProperties(),
166 op->getDiscardableAttrDictionary().getValue());
167
168 // Insert a memory-space cast to the original memory space of the op.
169 MemorySpaceCastOpInterface result = castOp.cloneMemorySpaceCastOp(
170 builder, tgtTy,
171 cast<TypedValue<PtrLikeTypeInterface>>(newOp.getResult()));
172 return std::optional<SmallVector<Value>>(
173 SmallVector<Value>({result.getTargetPtr()}));
174}
175
176//===----------------------------------------------------------------------===//
177// AllocOp / AllocaOp
178//===----------------------------------------------------------------------===//
179
180void AllocOp::getAsmResultNames(
181 function_ref<void(Value, StringRef)> setNameFn) {
182 setNameFn(getResult(), "alloc");
183}
184
185void AllocaOp::getAsmResultNames(
186 function_ref<void(Value, StringRef)> setNameFn) {
187 setNameFn(getResult(), "alloca");
188}
189
190template <typename AllocLikeOp>
191static LogicalResult verifyAllocLikeOp(AllocLikeOp op) {
192 static_assert(llvm::is_one_of<AllocLikeOp, AllocOp, AllocaOp>::value,
193 "applies to only alloc or alloca");
194 auto memRefType = llvm::dyn_cast<MemRefType>(op.getResult().getType());
195 if (!memRefType)
196 return op.emitOpError("result must be a memref");
197
198 if (failed(verifyDynamicDimensionCount(op, memRefType, op.getDynamicSizes())))
199 return failure();
200
201 unsigned numSymbols = 0;
202 if (!memRefType.getLayout().isIdentity())
203 numSymbols = memRefType.getLayout().getAffineMap().getNumSymbols();
204 if (op.getSymbolOperands().size() != numSymbols)
205 return op.emitOpError("symbol operand count does not equal memref symbol "
206 "count: expected ")
207 << numSymbols << ", got " << op.getSymbolOperands().size();
208
209 return success();
210}
211
212LogicalResult AllocOp::verify() { return verifyAllocLikeOp(op: *this); }
213
214LogicalResult AllocaOp::verify() {
215 // An alloca op needs to have an ancestor with an allocation scope trait.
216 if (!(*this)->getParentWithTrait<OpTrait::AutomaticAllocationScope>())
217 return emitOpError(
218 message: "requires an ancestor op with AutomaticAllocationScope trait");
219
220 return verifyAllocLikeOp(op: *this);
221}
222
223namespace {
224/// Fold constant dimensions into an alloc like operation.
225template <typename AllocLikeOp>
226struct SimplifyAllocConst : public OpRewritePattern<AllocLikeOp> {
227 using OpRewritePattern<AllocLikeOp>::OpRewritePattern;
228
229 LogicalResult matchAndRewrite(AllocLikeOp alloc,
230 PatternRewriter &rewriter) const override {
231 // Check to see if any dimensions operands are constants. If so, we can
232 // substitute and drop them.
233 if (llvm::none_of(alloc.getDynamicSizes(), [](Value operand) {
234 APInt constSizeArg;
235 if (!matchPattern(value: operand, pattern: m_ConstantInt(bind_value: &constSizeArg)))
236 return false;
237 return constSizeArg.isNonNegative();
238 }))
239 return failure();
240
241 auto memrefType = alloc.getType();
242
243 // Ok, we have one or more constant operands. Collect the non-constant ones
244 // and keep track of the resultant memref type to build.
245 SmallVector<int64_t, 4> newShapeConstants;
246 newShapeConstants.reserve(N: memrefType.getRank());
247 SmallVector<Value, 4> dynamicSizes;
248
249 unsigned dynamicDimPos = 0;
250 for (unsigned dim = 0, e = memrefType.getRank(); dim < e; ++dim) {
251 int64_t dimSize = memrefType.getDimSize(dim);
252 // If this is already static dimension, keep it.
253 if (ShapedType::isStatic(dValue: dimSize)) {
254 newShapeConstants.push_back(Elt: dimSize);
255 continue;
256 }
257 auto dynamicSize = alloc.getDynamicSizes()[dynamicDimPos];
258 APInt constSizeArg;
259 if (matchPattern(dynamicSize, m_ConstantInt(bind_value: &constSizeArg)) &&
260 constSizeArg.isNonNegative()) {
261 // Dynamic shape dimension will be folded.
262 newShapeConstants.push_back(Elt: constSizeArg.getZExtValue());
263 } else {
264 // Dynamic shape dimension not folded; copy dynamicSize from old memref.
265 newShapeConstants.push_back(Elt: ShapedType::kDynamic);
266 dynamicSizes.push_back(Elt: dynamicSize);
267 }
268 dynamicDimPos++;
269 }
270
271 // Create new memref type (which will have fewer dynamic dimensions).
272 MemRefType newMemRefType =
273 MemRefType::Builder(memrefType).setShape(newShapeConstants);
274 assert(dynamicSizes.size() == newMemRefType.getNumDynamicDims());
275
276 // Create and insert the alloc op for the new memref.
277 auto newAlloc = AllocLikeOp::create(rewriter, alloc.getLoc(), newMemRefType,
278 dynamicSizes, alloc.getSymbolOperands(),
279 alloc.getAlignmentAttr());
280 // Insert a cast so we have the same type as the old alloc.
281 rewriter.replaceOpWithNewOp<CastOp>(alloc, alloc.getType(), newAlloc);
282 return success();
283 }
284};
285
286/// Fold alloc operations with no users or only store and dealloc uses.
287template <typename T>
288struct SimplifyDeadAlloc : public OpRewritePattern<T> {
289 using OpRewritePattern<T>::OpRewritePattern;
290
291 LogicalResult matchAndRewrite(T alloc,
292 PatternRewriter &rewriter) const override {
293 if (llvm::any_of(alloc->getUsers(), [&](Operation *op) {
294 if (auto storeOp = dyn_cast<StoreOp>(Val: op))
295 return storeOp.getValue() == alloc;
296 return !isa<DeallocOp>(Val: op);
297 }))
298 return failure();
299
300 for (Operation *user : llvm::make_early_inc_range(alloc->getUsers()))
301 rewriter.eraseOp(op: user);
302
303 rewriter.eraseOp(op: alloc);
304 return success();
305 }
306};
307} // namespace
308
309void AllocOp::getCanonicalizationPatterns(RewritePatternSet &results,
310 MLIRContext *context) {
311 results.add<SimplifyAllocConst<AllocOp>, SimplifyDeadAlloc<AllocOp>>(arg&: context);
312}
313
314void AllocaOp::getCanonicalizationPatterns(RewritePatternSet &results,
315 MLIRContext *context) {
316 results.add<SimplifyAllocConst<AllocaOp>, SimplifyDeadAlloc<AllocaOp>>(
317 arg&: context);
318}
319
320//===----------------------------------------------------------------------===//
321// ReallocOp
322//===----------------------------------------------------------------------===//
323
324LogicalResult ReallocOp::verify() {
325 auto sourceType = llvm::cast<MemRefType>(Val: getOperand(i: 0).getType());
326 MemRefType resultType = getType();
327
328 // The source memref should have identity layout (or none).
329 if (!sourceType.getLayout().isIdentity())
330 return emitError(message: "unsupported layout for source memref type ")
331 << sourceType;
332
333 // The result memref should have identity layout (or none).
334 if (!resultType.getLayout().isIdentity())
335 return emitError(message: "unsupported layout for result memref type ")
336 << resultType;
337
338 // The source memref and the result memref should be in the same memory space.
339 if (sourceType.getMemorySpace() != resultType.getMemorySpace())
340 return emitError(message: "different memory spaces specified for source memref "
341 "type ")
342 << sourceType << " and result memref type " << resultType;
343
344 // The source memref and the result memref should have the same element type.
345 if (failed(Result: verifyElementTypesMatch(op: *this, lhs: sourceType, rhs: resultType, lhsName: "source",
346 rhsName: "result")))
347 return failure();
348
349 // Verify that we have the dynamic dimension operand when it is needed.
350 if (resultType.getNumDynamicDims() && !getDynamicResultSize())
351 return emitError(message: "missing dimension operand for result type ")
352 << resultType;
353 if (!resultType.getNumDynamicDims() && getDynamicResultSize())
354 return emitError(message: "unnecessary dimension operand for result type ")
355 << resultType;
356
357 return success();
358}
359
360void ReallocOp::getCanonicalizationPatterns(RewritePatternSet &results,
361 MLIRContext *context) {
362 results.add<SimplifyDeadAlloc<ReallocOp>>(arg&: context);
363}
364
365//===----------------------------------------------------------------------===//
366// AllocaScopeOp
367//===----------------------------------------------------------------------===//
368
369void AllocaScopeOp::print(OpAsmPrinter &p) {
370 bool printBlockTerminators = false;
371
372 p << ' ';
373 if (!getResults().empty()) {
374 p << " -> (" << getResultTypes() << ")";
375 printBlockTerminators = true;
376 }
377 p << ' ';
378 p.printRegion(blocks&: getBodyRegion(),
379 /*printEntryBlockArgs=*/false,
380 /*printBlockTerminators=*/printBlockTerminators);
381 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary());
382}
383
384ParseResult AllocaScopeOp::parse(OpAsmParser &parser, OperationState &result) {
385 // Create a region for the body.
386 result.regions.reserve(N: 1);
387 Region *bodyRegion = result.addRegion();
388
389 // Parse optional results type list.
390 if (parser.parseOptionalArrowTypeList(result&: result.types))
391 return failure();
392
393 // Parse the body region.
394 if (parser.parseRegion(region&: *bodyRegion, /*arguments=*/{}))
395 return failure();
396 AllocaScopeOp::ensureTerminator(region&: *bodyRegion, builder&: parser.getBuilder(),
397 loc: result.location);
398
399 // Parse the optional attribute list.
400 if (parser.parseOptionalAttrDict(result&: result.attributes))
401 return failure();
402
403 return success();
404}
405
406void AllocaScopeOp::getSuccessorRegions(
407 RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> &regions) {
408 if (!point.isParent()) {
409 regions.push_back(Elt: RegionSuccessor(getOperation()));
410 return;
411 }
412
413 regions.push_back(Elt: RegionSuccessor(&getBodyRegion()));
414}
415
416ValueRange AllocaScopeOp::getSuccessorInputs(RegionSuccessor successor) {
417 return successor.isOperation() ? ValueRange(getResults()) : ValueRange();
418}
419
420/// Given an operation, return whether this op is guaranteed to
421/// allocate an AutomaticAllocationScopeResource
422static bool isGuaranteedAutomaticAllocation(Operation *op) {
423 MemoryEffectOpInterface interface = dyn_cast<MemoryEffectOpInterface>(Val: op);
424 if (!interface)
425 return false;
426 for (auto res : op->getResults()) {
427 if (auto effect =
428 interface.getEffectOnValue<MemoryEffects::Allocate>(value: res)) {
429 if (isa<SideEffects::AutomaticAllocationScopeResource>(
430 Val: effect->getResource()))
431 return true;
432 }
433 }
434 return false;
435}
436
437/// Given an operation, return whether this op itself could
438/// allocate an AutomaticAllocationScopeResource. Note that
439/// this will not check whether an operation contained within
440/// the op can allocate.
441static bool isOpItselfPotentialAutomaticAllocation(Operation *op) {
442 // This op itself doesn't create a stack allocation,
443 // the inner allocation should be handled separately.
444 if (op->hasTrait<OpTrait::HasRecursiveMemoryEffects>())
445 return false;
446 MemoryEffectOpInterface interface = dyn_cast<MemoryEffectOpInterface>(Val: op);
447 if (!interface)
448 return true;
449 for (auto res : op->getResults()) {
450 if (auto effect =
451 interface.getEffectOnValue<MemoryEffects::Allocate>(value: res)) {
452 if (isa<SideEffects::AutomaticAllocationScopeResource>(
453 Val: effect->getResource()))
454 return true;
455 }
456 }
457 return false;
458}
459
460/// Return whether this op is the last non terminating op
461/// in a region. That is to say, it is in a one-block region
462/// and is only followed by a terminator. This prevents
463/// extending the lifetime of allocations.
464static bool lastNonTerminatorInRegion(Operation *op) {
465 return op->getBlock()->mightHaveTerminator() &&
466 op->getNextNode() == op->getBlock()->getTerminator() &&
467 op->getParentRegion()->hasOneBlock();
468}
469
470/// Inline an AllocaScopeOp if either the direct parent is an allocation scope
471/// or it contains no allocation.
472struct AllocaScopeInliner : public OpRewritePattern<AllocaScopeOp> {
473 using OpRewritePattern<AllocaScopeOp>::OpRewritePattern;
474
475 LogicalResult matchAndRewrite(AllocaScopeOp op,
476 PatternRewriter &rewriter) const override {
477 bool hasPotentialAlloca =
478 op->walk<WalkOrder::PreOrder>(callback: [&](Operation *alloc) {
479 if (alloc == op)
480 return WalkResult::advance();
481 if (isOpItselfPotentialAutomaticAllocation(op: alloc))
482 return WalkResult::interrupt();
483 if (alloc->hasTrait<OpTrait::AutomaticAllocationScope>())
484 return WalkResult::skip();
485 return WalkResult::advance();
486 }).wasInterrupted();
487
488 // If this contains no potential allocation, it is always legal to
489 // inline. Otherwise, consider two conditions:
490 if (hasPotentialAlloca) {
491 // If the parent isn't an allocation scope, or we are not the last
492 // non-terminator op in the parent, we will extend the lifetime.
493 if (!op->getParentOp()->hasTrait<OpTrait::AutomaticAllocationScope>())
494 return failure();
495 if (!lastNonTerminatorInRegion(op))
496 return failure();
497 }
498
499 Block *block = &op.getRegion().front();
500 Operation *terminator = block->getTerminator();
501 ValueRange results = terminator->getOperands();
502 rewriter.inlineBlockBefore(source: block, op);
503 rewriter.replaceOp(op, newValues: results);
504 rewriter.eraseOp(op: terminator);
505 return success();
506 }
507};
508
509/// Move allocations into an allocation scope, if it is legal to
510/// move them (e.g. their operands are available at the location
511/// the op would be moved to).
512struct AllocaScopeHoister : public OpRewritePattern<AllocaScopeOp> {
513 using OpRewritePattern<AllocaScopeOp>::OpRewritePattern;
514
515 LogicalResult matchAndRewrite(AllocaScopeOp op,
516 PatternRewriter &rewriter) const override {
517
518 if (!op->getParentWithTrait<OpTrait::AutomaticAllocationScope>())
519 return failure();
520
521 Operation *lastParentWithoutScope = op->getParentOp();
522
523 if (!lastParentWithoutScope ||
524 lastParentWithoutScope->hasTrait<OpTrait::AutomaticAllocationScope>())
525 return failure();
526
527 // Only apply to if this is this last non-terminator
528 // op in the block (lest lifetime be extended) of a one
529 // block region
530 if (!lastNonTerminatorInRegion(op) ||
531 !lastNonTerminatorInRegion(op: lastParentWithoutScope))
532 return failure();
533
534 while (!lastParentWithoutScope->getParentOp()
535 ->hasTrait<OpTrait::AutomaticAllocationScope>()) {
536 lastParentWithoutScope = lastParentWithoutScope->getParentOp();
537 if (!lastParentWithoutScope ||
538 !lastNonTerminatorInRegion(op: lastParentWithoutScope))
539 return failure();
540 }
541 assert(lastParentWithoutScope->getParentOp()
542 ->hasTrait<OpTrait::AutomaticAllocationScope>());
543
544 Region *containingRegion = nullptr;
545 for (auto &r : lastParentWithoutScope->getRegions()) {
546 if (r.isAncestor(other: op->getParentRegion())) {
547 assert(containingRegion == nullptr &&
548 "only one region can contain the op");
549 containingRegion = &r;
550 }
551 }
552 assert(containingRegion && "op must be contained in a region");
553
554 SmallVector<Operation *> toHoist;
555 op->walk(callback: [&](Operation *alloc) {
556 if (!isGuaranteedAutomaticAllocation(op: alloc))
557 return WalkResult::skip();
558
559 // If any operand is not defined before the location of
560 // lastParentWithoutScope (i.e. where we would hoist to), skip.
561 if (llvm::any_of(Range: alloc->getOperands(), P: [&](Value v) {
562 return containingRegion->isAncestor(other: v.getParentRegion());
563 }))
564 return WalkResult::skip();
565 toHoist.push_back(Elt: alloc);
566 return WalkResult::advance();
567 });
568
569 if (toHoist.empty())
570 return failure();
571 rewriter.setInsertionPoint(lastParentWithoutScope);
572 for (auto *op : toHoist) {
573 auto *cloned = rewriter.clone(op&: *op);
574 rewriter.replaceOp(op, newValues: cloned->getResults());
575 }
576 return success();
577 }
578};
579
580void AllocaScopeOp::getCanonicalizationPatterns(RewritePatternSet &results,
581 MLIRContext *context) {
582 results.add<AllocaScopeInliner, AllocaScopeHoister>(arg&: context);
583}
584
585//===----------------------------------------------------------------------===//
586// AssumeAlignmentOp
587//===----------------------------------------------------------------------===//
588
589LogicalResult AssumeAlignmentOp::verify() {
590 if (!llvm::isPowerOf2_32(Value: getAlignment()))
591 return emitOpError(message: "alignment must be power of 2");
592 return success();
593}
594
595void AssumeAlignmentOp::getAsmResultNames(
596 function_ref<void(Value, StringRef)> setNameFn) {
597 setNameFn(getResult(), "assume_align");
598}
599
600OpFoldResult AssumeAlignmentOp::fold(FoldAdaptor adaptor) {
601 auto source = getMemref().getDefiningOp<AssumeAlignmentOp>();
602 if (!source)
603 return {};
604 if (source.getAlignment() != getAlignment())
605 return {};
606 return getMemref();
607}
608
609FailureOr<std::optional<SmallVector<Value>>>
610AssumeAlignmentOp::bubbleDownCasts(OpBuilder &builder) {
611 return bubbleDownCastsPassthroughOpImpl(op: *this, builder, src&: getMemrefMutable());
612}
613
614FailureOr<OpFoldResult> AssumeAlignmentOp::reifyDimOfResult(OpBuilder &builder,
615 int resultIndex,
616 int dim) {
617 assert(resultIndex == 0 && "AssumeAlignmentOp has a single result");
618 return getMixedSize(builder, loc: getLoc(), value: getMemref(), dim);
619}
620
621//===----------------------------------------------------------------------===//
622// DistinctObjectsOp
623//===----------------------------------------------------------------------===//
624
625LogicalResult DistinctObjectsOp::verify() {
626 if (getOperandTypes() != getResultTypes())
627 return emitOpError(message: "operand types and result types must match");
628
629 if (getOperandTypes().empty())
630 return emitOpError(message: "expected at least one operand");
631
632 return success();
633}
634
635LogicalResult DistinctObjectsOp::inferReturnTypes(
636 MLIRContext * /*context*/, std::optional<Location> /*location*/,
637 ValueRange operands, DictionaryAttr /*attributes*/,
638 PropertyRef /*properties*/, RegionRange /*regions*/,
639 SmallVectorImpl<Type> &inferredReturnTypes) {
640 llvm::copy(Range: operands.getTypes(), Out: std::back_inserter(x&: inferredReturnTypes));
641 return success();
642}
643
644//===----------------------------------------------------------------------===//
645// CastOp
646//===----------------------------------------------------------------------===//
647
648void CastOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) {
649 setNameFn(getResult(), "cast");
650}
651
652/// Determines whether MemRef_CastOp casts to a more dynamic version of the
653/// source memref. This is useful to fold a memref.cast into a consuming op
654/// and implement canonicalization patterns for ops in different dialects that
655/// may consume the results of memref.cast operations. Such foldable memref.cast
656/// operations are typically inserted as `view` and `subview` ops are
657/// canonicalized, to preserve the type compatibility of their uses.
658///
659/// Returns true when all conditions are met:
660/// 1. source and result are ranked memrefs with strided semantics and same
661/// element type and rank.
662/// 2. each of the source's size, offset or stride has more static information
663/// than the corresponding result's size, offset or stride.
664///
665/// Example 1:
666/// ```mlir
667/// %1 = memref.cast %0 : memref<8x16xf32> to memref<?x?xf32>
668/// %2 = consumer %1 ... : memref<?x?xf32> ...
669/// ```
670///
671/// may fold into:
672///
673/// ```mlir
674/// %2 = consumer %0 ... : memref<8x16xf32> ...
675/// ```
676///
677/// Example 2:
678/// ```
679/// %1 = memref.cast %0 : memref<?x16xf32, affine_map<(i, j)->(16 * i + j)>>
680/// to memref<?x?xf32>
681/// consumer %1 : memref<?x?xf32> ...
682/// ```
683///
684/// may fold into:
685///
686/// ```
687/// consumer %0 ... : memref<?x16xf32, affine_map<(i, j)->(16 * i + j)>>
688/// ```
689bool CastOp::canFoldIntoConsumerOp(CastOp castOp) {
690 MemRefType sourceType =
691 llvm::dyn_cast<MemRefType>(Val: castOp.getSource().getType());
692 MemRefType resultType = llvm::dyn_cast<MemRefType>(Val: castOp.getType());
693
694 // Requires ranked MemRefType.
695 if (!sourceType || !resultType)
696 return false;
697
698 // Requires same elemental type.
699 if (sourceType.getElementType() != resultType.getElementType())
700 return false;
701
702 // Requires same rank.
703 if (sourceType.getRank() != resultType.getRank())
704 return false;
705
706 // Only fold casts between strided memref forms.
707 int64_t sourceOffset, resultOffset;
708 SmallVector<int64_t, 4> sourceStrides, resultStrides;
709 if (failed(Result: sourceType.getStridesAndOffset(strides&: sourceStrides, offset&: sourceOffset)) ||
710 failed(Result: resultType.getStridesAndOffset(strides&: resultStrides, offset&: resultOffset)))
711 return false;
712
713 // If cast is towards more static sizes along any dimension, don't fold.
714 for (auto it : llvm::zip(t: sourceType.getShape(), u: resultType.getShape())) {
715 auto ss = std::get<0>(t&: it), st = std::get<1>(t&: it);
716 if (ss != st)
717 if (ShapedType::isDynamic(dValue: ss) && ShapedType::isStatic(dValue: st))
718 return false;
719 }
720
721 // If cast is towards more static offset along any dimension, don't fold.
722 if (sourceOffset != resultOffset)
723 if (ShapedType::isDynamic(dValue: sourceOffset) &&
724 ShapedType::isStatic(dValue: resultOffset))
725 return false;
726
727 // If cast is towards more static strides along any dimension, don't fold.
728 for (auto it : llvm::zip(t&: sourceStrides, u&: resultStrides)) {
729 auto ss = std::get<0>(t&: it), st = std::get<1>(t&: it);
730 if (ss != st)
731 if (ShapedType::isDynamic(dValue: ss) && ShapedType::isStatic(dValue: st))
732 return false;
733 }
734
735 return true;
736}
737
738bool CastOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
739 if (inputs.size() != 1 || outputs.size() != 1)
740 return false;
741 if (inputs == outputs)
742 return true;
743 Type a = inputs.front(), b = outputs.front();
744 auto aT = llvm::dyn_cast<MemRefType>(Val&: a);
745 auto bT = llvm::dyn_cast<MemRefType>(Val&: b);
746
747 auto uaT = llvm::dyn_cast<UnrankedMemRefType>(Val&: a);
748 auto ubT = llvm::dyn_cast<UnrankedMemRefType>(Val&: b);
749
750 if (aT && bT) {
751 if (aT.getElementType() != bT.getElementType())
752 return false;
753 if (aT.getLayout() != bT.getLayout()) {
754 int64_t aOffset, bOffset;
755 SmallVector<int64_t, 4> aStrides, bStrides;
756 if (failed(Result: aT.getStridesAndOffset(strides&: aStrides, offset&: aOffset)) ||
757 failed(Result: bT.getStridesAndOffset(strides&: bStrides, offset&: bOffset)) ||
758 aStrides.size() != bStrides.size())
759 return false;
760
761 // Strides along a dimension/offset are compatible if the value in the
762 // source memref is static and the value in the target memref is the
763 // same. They are also compatible if either one is dynamic (see
764 // description of MemRefCastOp for details).
765 // Note that for dimensions of size 1, the stride can differ.
766 auto checkCompatible = [](int64_t a, int64_t b) {
767 return (ShapedType::isDynamic(dValue: a) || ShapedType::isDynamic(dValue: b) || a == b);
768 };
769 if (!checkCompatible(aOffset, bOffset))
770 return false;
771 for (const auto &[index, aStride] : enumerate(First&: aStrides)) {
772 if (aT.getDimSize(idx: index) == 1 || bT.getDimSize(idx: index) == 1)
773 continue;
774 if (!checkCompatible(aStride, bStrides[index]))
775 return false;
776 }
777 }
778 if (aT.getMemorySpace() != bT.getMemorySpace())
779 return false;
780
781 // They must have the same rank, and any specified dimensions must match.
782 if (aT.getRank() != bT.getRank())
783 return false;
784
785 for (unsigned i = 0, e = aT.getRank(); i != e; ++i) {
786 int64_t aDim = aT.getDimSize(idx: i), bDim = bT.getDimSize(idx: i);
787 if (ShapedType::isStatic(dValue: aDim) && ShapedType::isStatic(dValue: bDim) &&
788 aDim != bDim)
789 return false;
790 }
791 return true;
792 } else {
793 if (!aT && !uaT)
794 return false;
795 if (!bT && !ubT)
796 return false;
797 // Unranked to unranked casting is unsupported
798 if (uaT && ubT)
799 return false;
800
801 auto aEltType = (aT) ? aT.getElementType() : uaT.getElementType();
802 auto bEltType = (bT) ? bT.getElementType() : ubT.getElementType();
803 if (aEltType != bEltType)
804 return false;
805
806 auto aMemSpace = (aT) ? aT.getMemorySpace() : uaT.getMemorySpace();
807 auto bMemSpace = (bT) ? bT.getMemorySpace() : ubT.getMemorySpace();
808 return aMemSpace == bMemSpace;
809 }
810
811 return false;
812}
813
814OpFoldResult CastOp::fold(FoldAdaptor adaptor) {
815 return succeeded(Result: foldMemRefCast(op: *this)) ? getResult() : Value();
816}
817
818FailureOr<std::optional<SmallVector<Value>>>
819CastOp::bubbleDownCasts(OpBuilder &builder) {
820 return bubbleDownCastsPassthroughOpImpl(op: *this, builder, src&: getSourceMutable());
821}
822
823//===----------------------------------------------------------------------===//
824// CopyOp
825//===----------------------------------------------------------------------===//
826
827namespace {
828
829/// Fold memref.copy(%x, %x).
830struct FoldSelfCopy : public OpRewritePattern<CopyOp> {
831 using OpRewritePattern<CopyOp>::OpRewritePattern;
832
833 LogicalResult matchAndRewrite(CopyOp copyOp,
834 PatternRewriter &rewriter) const override {
835 if (copyOp.getSource() != copyOp.getTarget())
836 return failure();
837
838 rewriter.eraseOp(op: copyOp);
839 return success();
840 }
841};
842
843struct FoldEmptyCopy final : public OpRewritePattern<CopyOp> {
844 using OpRewritePattern<CopyOp>::OpRewritePattern;
845
846 static bool isEmptyMemRef(BaseMemRefType type) {
847 return type.hasRank() && llvm::is_contained(Range: type.getShape(), Element: 0);
848 }
849
850 LogicalResult matchAndRewrite(CopyOp copyOp,
851 PatternRewriter &rewriter) const override {
852 if (isEmptyMemRef(type: copyOp.getSource().getType()) ||
853 isEmptyMemRef(type: copyOp.getTarget().getType())) {
854 rewriter.eraseOp(op: copyOp);
855 return success();
856 }
857
858 return failure();
859 }
860};
861} // namespace
862
863void CopyOp::getCanonicalizationPatterns(RewritePatternSet &results,
864 MLIRContext *context) {
865 results.add<FoldEmptyCopy, FoldSelfCopy>(arg&: context);
866}
867
868/// If the source/target of a CopyOp is a CastOp that does not modify the shape
869/// and element type, the cast can be skipped. Such CastOps only cast the layout
870/// of the type.
871static LogicalResult foldCopyOfCast(CopyOp op) {
872 for (OpOperand &operand : op->getOpOperands()) {
873 auto castOp = operand.get().getDefiningOp<memref::CastOp>();
874 if (castOp && memref::CastOp::canFoldIntoConsumerOp(castOp)) {
875 operand.set(castOp.getOperand());
876 return success();
877 }
878 }
879 return failure();
880}
881
882LogicalResult CopyOp::fold(FoldAdaptor adaptor,
883 SmallVectorImpl<OpFoldResult> &results) {
884
885 /// copy(memrefcast) -> copy
886 return foldCopyOfCast(op: *this);
887}
888
889//===----------------------------------------------------------------------===//
890// DeallocOp
891//===----------------------------------------------------------------------===//
892
893LogicalResult DeallocOp::fold(FoldAdaptor adaptor,
894 SmallVectorImpl<OpFoldResult> &results) {
895 /// dealloc(memrefcast) -> dealloc
896 return foldMemRefCast(op: *this);
897}
898
899//===----------------------------------------------------------------------===//
900// DimOp
901//===----------------------------------------------------------------------===//
902
903void DimOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) {
904 setNameFn(getResult(), "dim");
905}
906
907void DimOp::build(OpBuilder &builder, OperationState &result, Value source,
908 int64_t index) {
909 auto loc = result.location;
910 Value indexValue = arith::ConstantIndexOp::create(builder, location: loc, value: index);
911 build(odsBuilder&: builder, odsState&: result, source, index: indexValue);
912}
913
914std::optional<int64_t> DimOp::getConstantIndex() {
915 return getConstantIntValue(ofr: getIndex());
916}
917
918Speculation::Speculatability DimOp::getSpeculatability() {
919 auto constantIndex = getConstantIndex();
920 if (!constantIndex)
921 return Speculation::NotSpeculatable;
922
923 auto rankedSourceType = dyn_cast<MemRefType>(Val: getSource().getType());
924 if (!rankedSourceType)
925 return Speculation::NotSpeculatable;
926
927 if (rankedSourceType.getRank() <= constantIndex)
928 return Speculation::NotSpeculatable;
929
930 return Speculation::Speculatable;
931}
932
933void DimOp::inferResultRangesFromOptional(ArrayRef<IntegerValueRange> argRanges,
934 SetIntLatticeFn setResultRange) {
935 setResultRange(getResult(),
936 intrange::inferShapedDimOpInterface(op: *this, maybeDim: argRanges[1]));
937}
938
939/// Return a map with key being elements in `vals` and data being number of
940/// occurences of it. Use std::map, since the `vals` here are strides and the
941/// dynamic stride value is the same as the tombstone value for
942/// `DenseMap<int64_t>`.
943static std::map<int64_t, unsigned> getNumOccurences(ArrayRef<int64_t> vals) {
944 std::map<int64_t, unsigned> numOccurences;
945 for (auto val : vals)
946 numOccurences[val]++;
947 return numOccurences;
948}
949
950/// Returns the set of source dimensions that are dropped in a rank reduction.
951/// For each result dimension in order, matches the leftmost unmatched source
952/// dimension with the same size. Source dimensions not matched are dropped.
953///
954/// Example: memref<1x8x1x3> to memref<1x8x3>. Source sizes [1, 8, 1, 3], result
955/// [1, 8, 3]. Match result[0]=1 -> source dim 0, result[1]=8 -> source dim 1,
956/// result[2]=3 -> source dim 3. Source dim 2 is unmatched and dropped.
957static FailureOr<llvm::SmallBitVector>
958computeMemRefRankReductionMaskByPosition(MemRefType originalType,
959 MemRefType reducedType,
960 ArrayRef<OpFoldResult> sizes) {
961 int64_t rankReduction = originalType.getRank() - reducedType.getRank();
962 if (rankReduction <= 0)
963 return llvm::SmallBitVector(originalType.getRank());
964
965 // Build source sizes from subview sizes (one per source dim).
966 SmallVector<int64_t> sourceSizes(originalType.getRank());
967 for (const auto &it : llvm::enumerate(First&: sizes)) {
968 if (std::optional<int64_t> cst = getConstantIntValue(ofr: it.value()))
969 sourceSizes[it.index()] = *cst;
970 else
971 sourceSizes[it.index()] = ShapedType::kDynamic;
972 }
973
974 ArrayRef<int64_t> resultSizes = reducedType.getShape();
975 llvm::SmallBitVector usedSourceDims(originalType.getRank());
976 int64_t startJ = 0;
977 for (int64_t resultSize : resultSizes) {
978 bool matched = false;
979 for (int64_t j = startJ; j < originalType.getRank(); ++j) {
980 if (sourceSizes[j] == resultSize) {
981 usedSourceDims.set(j);
982 matched = true;
983 startJ = j + 1;
984 break;
985 }
986 }
987 if (!matched)
988 return failure();
989 }
990
991 llvm::SmallBitVector unusedDims(originalType.getRank());
992 for (int64_t i = 0; i < originalType.getRank(); ++i)
993 if (!usedSourceDims.test(Idx: i))
994 unusedDims.set(i);
995 return unusedDims;
996}
997
998/// Returns the set of source dimensions that are dropped in a rank reduction.
999/// A dimension is dropped if its stride is dropped; uses stride occurrence
1000/// counting to disambiguate when multiple unit dims exist.
1001///
1002/// Example: memref<1x1x?xf32, strided<[?, 4, 1]>> to memref<1x4xf32,
1003/// strided<[4, 1]>>. Source strides [?, 4, 1], candidate [4, 1]. Dim 0 (stride
1004/// ?) can be dropped; dim 1 (stride 4) must be kept. Source dim 0 is dropped.
1005static FailureOr<llvm::SmallBitVector> computeMemRefRankReductionMaskByStrides(
1006 MemRefType originalType, MemRefType reducedType,
1007 ArrayRef<int64_t> originalStrides, ArrayRef<int64_t> candidateStrides,
1008 llvm::SmallBitVector unusedDims) {
1009 // Track the number of occurences of the strides in the original type
1010 // and the candidate type. For each unused dim that stride should not be
1011 // present in the candidate type. Note that there could be multiple dimensions
1012 // that have the same size. We dont need to exactly figure out which dim
1013 // corresponds to which stride, we just need to verify that the number of
1014 // reptitions of a stride in the original + number of unused dims with that
1015 // stride == number of repititions of a stride in the candidate.
1016 std::map<int64_t, unsigned> currUnaccountedStrides =
1017 getNumOccurences(vals: originalStrides);
1018 std::map<int64_t, unsigned> candidateStridesNumOccurences =
1019 getNumOccurences(vals: candidateStrides);
1020 for (size_t dim = 0, e = unusedDims.size(); dim != e; ++dim) {
1021 if (!unusedDims.test(Idx: dim))
1022 continue;
1023 int64_t originalStride = originalStrides[dim];
1024 if (currUnaccountedStrides[originalStride] >
1025 candidateStridesNumOccurences[originalStride]) {
1026 // This dim can be treated as dropped.
1027 currUnaccountedStrides[originalStride]--;
1028 continue;
1029 }
1030 if (currUnaccountedStrides[originalStride] ==
1031 candidateStridesNumOccurences[originalStride]) {
1032 // The stride for this is not dropped. Keep as is.
1033 unusedDims.reset(Idx: dim);
1034 continue;
1035 }
1036 if (currUnaccountedStrides[originalStride] <
1037 candidateStridesNumOccurences[originalStride]) {
1038 // This should never happen. Cant have a stride in the reduced rank type
1039 // that wasnt in the original one.
1040 return failure();
1041 }
1042 }
1043 if (static_cast<int64_t>(unusedDims.count()) + reducedType.getRank() !=
1044 originalType.getRank())
1045 return failure();
1046 return unusedDims;
1047}
1048
1049/// Given the `originalType` and a `candidateReducedType` whose shape is assumed
1050/// to be a subset of `originalType` with some `1` entries erased, return the
1051/// set of indices that specifies which of the entries of `originalShape` are
1052/// dropped to obtain `reducedShape`.
1053/// This accounts for cases where there are multiple unit-dims, but only a
1054/// subset of those are dropped. For MemRefTypes these can be disambiguated
1055/// using the strides. If a dimension is dropped the stride must be dropped too.
1056static FailureOr<llvm::SmallBitVector>
1057computeMemRefRankReductionMask(MemRefType originalType, MemRefType reducedType,
1058 ArrayRef<OpFoldResult> sizes) {
1059 llvm::SmallBitVector unusedDims(originalType.getRank());
1060 if (originalType.getRank() == reducedType.getRank())
1061 return unusedDims;
1062
1063 for (const auto &dim : llvm::enumerate(First&: sizes))
1064 if (auto attr = llvm::dyn_cast_if_present<Attribute>(Val: dim.value()))
1065 if (llvm::cast<IntegerAttr>(Val&: attr).getInt() == 1)
1066 unusedDims.set(dim.index());
1067
1068 // Early exit for the case where the number of unused dims matches the number
1069 // of ranks reduced.
1070 if (static_cast<int64_t>(unusedDims.count()) + reducedType.getRank() ==
1071 originalType.getRank())
1072 return unusedDims;
1073
1074 SmallVector<int64_t> originalStrides, candidateStrides;
1075 int64_t originalOffset, candidateOffset;
1076 if (failed(
1077 Result: originalType.getStridesAndOffset(strides&: originalStrides, offset&: originalOffset)) ||
1078 failed(
1079 Result: reducedType.getStridesAndOffset(strides&: candidateStrides, offset&: candidateOffset)))
1080 return failure();
1081
1082 // Try stride-based first when we have meaningful static stride info
1083 // (preserves static strides). Fall back to position-based otherwise.
1084 auto hasNonTrivialStaticStride = [](ArrayRef<int64_t> strides) {
1085 // The innermost stride 1 is trivial for row-major and does not help
1086 // disambiguate.
1087 if (strides.size() <= 1)
1088 return false;
1089 return llvm::any_of(Range: strides.drop_back(),
1090 P: [](int64_t s) { return !ShapedType::isDynamic(dValue: s); });
1091 };
1092 if (hasNonTrivialStaticStride(originalStrides) ||
1093 hasNonTrivialStaticStride(candidateStrides)) {
1094 FailureOr<llvm::SmallBitVector> strideBased =
1095 computeMemRefRankReductionMaskByStrides(originalType, reducedType,
1096 originalStrides,
1097 candidateStrides, unusedDims);
1098 if (succeeded(Result: strideBased))
1099 return *strideBased;
1100 }
1101 return computeMemRefRankReductionMaskByPosition(originalType, reducedType,
1102 sizes);
1103}
1104
1105llvm::SmallBitVector SubViewOp::getDroppedDims() {
1106 MemRefType sourceType = getSourceType();
1107 MemRefType resultType = getType();
1108 FailureOr<llvm::SmallBitVector> unusedDims =
1109 computeMemRefRankReductionMask(originalType: sourceType, reducedType: resultType, sizes: getMixedSizes());
1110 assert(succeeded(unusedDims) && "unable to find unused dims of subview");
1111 return *unusedDims;
1112}
1113
1114OpFoldResult DimOp::fold(FoldAdaptor adaptor) {
1115 // All forms of folding require a known index.
1116 std::optional<int64_t> index = getConstantIndex();
1117 if (!index)
1118 return {};
1119
1120 // Folding for unranked types (UnrankedMemRefType) is not supported.
1121 auto memrefType = llvm::dyn_cast<MemRefType>(Val: getSource().getType());
1122 if (!memrefType)
1123 return {};
1124
1125 // Out of bound indices produce undefined behavior but are still valid IR.
1126 // Don't choke on them.
1127 int64_t indexVal = index.value();
1128 if (indexVal < 0 || indexVal >= memrefType.getRank())
1129 return {};
1130
1131 // Fold if the shape extent along the given index is known.
1132 if (!memrefType.isDynamicDim(idx: indexVal)) {
1133 Builder builder(getContext());
1134 return builder.getIndexAttr(value: memrefType.getShape()[indexVal]);
1135 }
1136
1137 // The size at the given index is now known to be a dynamic size.
1138 // Fold dim to the size argument for an `AllocOp`, `ViewOp`, or `SubViewOp`.
1139 Operation *definingOp = getSource().getDefiningOp();
1140
1141 if (auto alloc = dyn_cast_or_null<AllocOp>(Val: definingOp))
1142 return *(alloc.getDynamicSizes().begin() +
1143 memrefType.getDynamicDimIndex(index: indexVal));
1144
1145 if (auto alloca = dyn_cast_or_null<AllocaOp>(Val: definingOp))
1146 return *(alloca.getDynamicSizes().begin() +
1147 memrefType.getDynamicDimIndex(index: indexVal));
1148
1149 if (auto view = dyn_cast_or_null<ViewOp>(Val: definingOp))
1150 return *(view.getDynamicSizes().begin() +
1151 memrefType.getDynamicDimIndex(index: indexVal));
1152
1153 if (auto subview = dyn_cast_or_null<SubViewOp>(Val: definingOp)) {
1154 // The result dim is dynamic (the static case was handled above). Dropped
1155 // dims always have static size 1, so dynamic source sizes are never
1156 // dropped and map in order to the dynamic result dims. Find the k-th
1157 // dynamic source size, where k is the dynamic dim index of the result dim.
1158 unsigned dynamicResultDimIdx = memrefType.getDynamicDimIndex(index: indexVal);
1159 unsigned dynamicIdx = 0;
1160 for (OpFoldResult size : subview.getMixedSizes()) {
1161 if (llvm::isa<Attribute>(Val: size))
1162 continue;
1163 if (dynamicIdx == dynamicResultDimIdx)
1164 return size;
1165 dynamicIdx++;
1166 }
1167 return {};
1168 }
1169
1170 // dim(memrefcast) -> dim
1171 if (succeeded(Result: foldMemRefCast(op: *this)))
1172 return getResult();
1173
1174 return {};
1175}
1176
1177namespace {
1178/// Fold dim of a memref reshape operation to a load into the reshape's shape
1179/// operand.
1180struct DimOfMemRefReshape : public OpRewritePattern<DimOp> {
1181 using OpRewritePattern<DimOp>::OpRewritePattern;
1182
1183 LogicalResult matchAndRewrite(DimOp dim,
1184 PatternRewriter &rewriter) const override {
1185 auto reshape = dim.getSource().getDefiningOp<ReshapeOp>();
1186
1187 if (!reshape)
1188 return rewriter.notifyMatchFailure(
1189 arg&: dim, msg: "Dim op is not defined by a reshape op.");
1190
1191 // dim of a memref reshape can be folded if dim.getIndex() dominates the
1192 // reshape. Instead of using `DominanceInfo` (which is usually costly) we
1193 // cheaply check that either of the following conditions hold:
1194 // 1. dim.getIndex() is defined in the same block as reshape but before
1195 // reshape.
1196 // 2. dim.getIndex() is defined in a parent block of
1197 // reshape.
1198
1199 // Check condition 1
1200 if (dim.getIndex().getParentBlock() == reshape->getBlock()) {
1201 if (auto *definingOp = dim.getIndex().getDefiningOp()) {
1202 if (reshape->isBeforeInBlock(other: definingOp)) {
1203 return rewriter.notifyMatchFailure(
1204 arg&: dim,
1205 msg: "dim.getIndex is not defined before reshape in the same block.");
1206 }
1207 } // else dim.getIndex is a block argument to reshape->getBlock and
1208 // dominates reshape
1209 } // Check condition 2
1210 else if (dim->getBlock() != reshape->getBlock() &&
1211 !dim.getIndex().getParentRegion()->isProperAncestor(
1212 other: reshape->getParentRegion())) {
1213 // If dim and reshape are in the same block but dim.getIndex() isn't, we
1214 // already know dim.getIndex() dominates reshape without calling
1215 // `isProperAncestor`
1216 return rewriter.notifyMatchFailure(
1217 arg&: dim, msg: "dim.getIndex does not dominate reshape.");
1218 }
1219
1220 // Place the load directly after the reshape to ensure that the shape memref
1221 // was not mutated.
1222 rewriter.setInsertionPointAfter(reshape);
1223 Location loc = dim.getLoc();
1224 Value load =
1225 LoadOp::create(builder&: rewriter, location: loc, memref: reshape.getShape(), indices: dim.getIndex());
1226 if (load.getType() != dim.getType())
1227 load = arith::IndexCastOp::create(builder&: rewriter, location: loc, out: dim.getType(), in: load);
1228 rewriter.replaceOp(op: dim, newValues: load);
1229 return success();
1230 }
1231};
1232
1233} // namespace
1234
1235void DimOp::getCanonicalizationPatterns(RewritePatternSet &results,
1236 MLIRContext *context) {
1237 results.add<DimOfMemRefReshape>(arg&: context);
1238}
1239
1240// ---------------------------------------------------------------------------
1241// DmaStartOp
1242// ---------------------------------------------------------------------------
1243
1244void DmaStartOp::build(OpBuilder &builder, OperationState &result,
1245 Value srcMemRef, ValueRange srcIndices, Value destMemRef,
1246 ValueRange destIndices, Value numElements,
1247 Value tagMemRef, ValueRange tagIndices, Value stride,
1248 Value elementsPerStride) {
1249 result.addOperands(newOperands: srcMemRef);
1250 result.addOperands(newOperands: srcIndices);
1251 result.addOperands(newOperands: destMemRef);
1252 result.addOperands(newOperands: destIndices);
1253 result.addOperands(newOperands: {numElements, tagMemRef});
1254 result.addOperands(newOperands: tagIndices);
1255 if (stride)
1256 result.addOperands(newOperands: {stride, elementsPerStride});
1257}
1258
1259void DmaStartOp::print(OpAsmPrinter &p) {
1260 p << " " << getSrcMemRef() << '[' << getSrcIndices() << "], "
1261 << getDstMemRef() << '[' << getDstIndices() << "], " << getNumElements()
1262 << ", " << getTagMemRef() << '[' << getTagIndices() << ']';
1263 if (isStrided())
1264 p << ", " << getStride() << ", " << getNumElementsPerStride();
1265
1266 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary());
1267 p << " : " << getSrcMemRef().getType() << ", " << getDstMemRef().getType()
1268 << ", " << getTagMemRef().getType();
1269}
1270
1271// Parse DmaStartOp.
1272// Ex:
1273// %dma_id = dma_start %src[%i, %j], %dst[%k, %l], %size,
1274// %tag[%index], %stride, %num_elt_per_stride :
1275// : memref<3076 x f32, 0>,
1276// memref<1024 x f32, 2>,
1277// memref<1 x i32>
1278//
1279ParseResult DmaStartOp::parse(OpAsmParser &parser, OperationState &result) {
1280 OpAsmParser::UnresolvedOperand srcMemRefInfo;
1281 SmallVector<OpAsmParser::UnresolvedOperand, 4> srcIndexInfos;
1282 OpAsmParser::UnresolvedOperand dstMemRefInfo;
1283 SmallVector<OpAsmParser::UnresolvedOperand, 4> dstIndexInfos;
1284 OpAsmParser::UnresolvedOperand numElementsInfo;
1285 OpAsmParser::UnresolvedOperand tagMemrefInfo;
1286 SmallVector<OpAsmParser::UnresolvedOperand, 4> tagIndexInfos;
1287 SmallVector<OpAsmParser::UnresolvedOperand, 2> strideInfo;
1288
1289 SmallVector<Type, 3> types;
1290 auto indexType = parser.getBuilder().getIndexType();
1291
1292 // Parse and resolve the following list of operands:
1293 // *) source memref followed by its indices (in square brackets).
1294 // *) destination memref followed by its indices (in square brackets).
1295 // *) dma size in KiB.
1296 if (parser.parseOperand(result&: srcMemRefInfo) ||
1297 parser.parseOperandList(result&: srcIndexInfos, delimiter: OpAsmParser::Delimiter::Square) ||
1298 parser.parseComma() || parser.parseOperand(result&: dstMemRefInfo) ||
1299 parser.parseOperandList(result&: dstIndexInfos, delimiter: OpAsmParser::Delimiter::Square) ||
1300 parser.parseComma() || parser.parseOperand(result&: numElementsInfo) ||
1301 parser.parseComma() || parser.parseOperand(result&: tagMemrefInfo) ||
1302 parser.parseOperandList(result&: tagIndexInfos, delimiter: OpAsmParser::Delimiter::Square))
1303 return failure();
1304
1305 // Parse optional stride and elements per stride.
1306 if (parser.parseTrailingOperandList(result&: strideInfo))
1307 return failure();
1308
1309 bool isStrided = strideInfo.size() == 2;
1310 if (!strideInfo.empty() && !isStrided) {
1311 return parser.emitError(loc: parser.getNameLoc(),
1312 message: "expected two stride related operands");
1313 }
1314
1315 if (parser.parseColonTypeList(result&: types))
1316 return failure();
1317 if (types.size() != 3)
1318 return parser.emitError(loc: parser.getNameLoc(), message: "fewer/more types expected");
1319
1320 if (parser.resolveOperand(operand: srcMemRefInfo, type: types[0], result&: result.operands) ||
1321 parser.resolveOperands(operands&: srcIndexInfos, type: indexType, result&: result.operands) ||
1322 parser.resolveOperand(operand: dstMemRefInfo, type: types[1], result&: result.operands) ||
1323 parser.resolveOperands(operands&: dstIndexInfos, type: indexType, result&: result.operands) ||
1324 // size should be an index.
1325 parser.resolveOperand(operand: numElementsInfo, type: indexType, result&: result.operands) ||
1326 parser.resolveOperand(operand: tagMemrefInfo, type: types[2], result&: result.operands) ||
1327 // tag indices should be index.
1328 parser.resolveOperands(operands&: tagIndexInfos, type: indexType, result&: result.operands))
1329 return failure();
1330
1331 if (isStrided) {
1332 if (parser.resolveOperands(operands&: strideInfo, type: indexType, result&: result.operands))
1333 return failure();
1334 }
1335
1336 return success();
1337}
1338
1339LogicalResult DmaStartOp::verify() {
1340 unsigned numOperands = getNumOperands();
1341
1342 // Mandatory non-variadic operands are: src memref, dst memref, tag memref and
1343 // the number of elements.
1344 if (numOperands < 4)
1345 return emitOpError(message: "expected at least 4 operands");
1346
1347 // Check types of operands. The order of these calls is important: the later
1348 // calls rely on some type properties to compute the operand position.
1349 // 1. Source memref.
1350 if (!llvm::isa<MemRefType>(Val: getSrcMemRef().getType()))
1351 return emitOpError(message: "expected source to be of memref type");
1352 if (numOperands < getSrcMemRefRank() + 4)
1353 return emitOpError() << "expected at least " << getSrcMemRefRank() + 4
1354 << " operands";
1355 if (!getSrcIndices().empty() &&
1356 !llvm::all_of(Range: getSrcIndices().getTypes(),
1357 P: [](Type t) { return t.isIndex(); }))
1358 return emitOpError(message: "expected source indices to be of index type");
1359
1360 // 2. Destination memref.
1361 if (!llvm::isa<MemRefType>(Val: getDstMemRef().getType()))
1362 return emitOpError(message: "expected destination to be of memref type");
1363 unsigned numExpectedOperands = getSrcMemRefRank() + getDstMemRefRank() + 4;
1364 if (numOperands < numExpectedOperands)
1365 return emitOpError() << "expected at least " << numExpectedOperands
1366 << " operands";
1367 if (!getDstIndices().empty() &&
1368 !llvm::all_of(Range: getDstIndices().getTypes(),
1369 P: [](Type t) { return t.isIndex(); }))
1370 return emitOpError(message: "expected destination indices to be of index type");
1371
1372 // 3. Number of elements.
1373 if (!getNumElements().getType().isIndex())
1374 return emitOpError(message: "expected num elements to be of index type");
1375
1376 // 4. Tag memref.
1377 if (!llvm::isa<MemRefType>(Val: getTagMemRef().getType()))
1378 return emitOpError(message: "expected tag to be of memref type");
1379 numExpectedOperands += getTagMemRefRank();
1380 if (numOperands < numExpectedOperands)
1381 return emitOpError() << "expected at least " << numExpectedOperands
1382 << " operands";
1383 if (!getTagIndices().empty() &&
1384 !llvm::all_of(Range: getTagIndices().getTypes(),
1385 P: [](Type t) { return t.isIndex(); }))
1386 return emitOpError(message: "expected tag indices to be of index type");
1387
1388 // Optional stride-related operands must be either both present or both
1389 // absent.
1390 if (numOperands != numExpectedOperands &&
1391 numOperands != numExpectedOperands + 2)
1392 return emitOpError(message: "incorrect number of operands");
1393
1394 // 5. Strides.
1395 if (isStrided()) {
1396 if (!getStride().getType().isIndex() ||
1397 !getNumElementsPerStride().getType().isIndex())
1398 return emitOpError(
1399 message: "expected stride and num elements per stride to be of type index");
1400 }
1401
1402 return success();
1403}
1404
1405LogicalResult DmaStartOp::fold(FoldAdaptor adaptor,
1406 SmallVectorImpl<OpFoldResult> &results) {
1407 /// dma_start(memrefcast) -> dma_start
1408 return foldMemRefCast(op: *this);
1409}
1410
1411void DmaStartOp::setMemrefsAndIndices(RewriterBase &rewriter, Value newSrc,
1412 ValueRange newSrcIndices, Value newDst,
1413 ValueRange newDstIndices) {
1414 /// dma_start has special handling for variadic rank
1415 SmallVector<Value> newOperands;
1416 newOperands.push_back(Elt: newSrc);
1417 llvm::append_range(C&: newOperands, R&: newSrcIndices);
1418 newOperands.push_back(Elt: newDst);
1419 llvm::append_range(C&: newOperands, R&: newDstIndices);
1420 newOperands.push_back(Elt: getNumElements());
1421 newOperands.push_back(Elt: getTagMemRef());
1422 llvm::append_range(C&: newOperands, R: getTagIndices());
1423 if (isStrided()) {
1424 newOperands.push_back(Elt: getStride());
1425 newOperands.push_back(Elt: getNumElementsPerStride());
1426 }
1427
1428 rewriter.modifyOpInPlace(root: *this, callable: [&]() { (*this)->setOperands(newOperands); });
1429}
1430
1431// ---------------------------------------------------------------------------
1432// DmaWaitOp
1433// ---------------------------------------------------------------------------
1434
1435LogicalResult DmaWaitOp::fold(FoldAdaptor adaptor,
1436 SmallVectorImpl<OpFoldResult> &results) {
1437 /// dma_wait(memrefcast) -> dma_wait
1438 return foldMemRefCast(op: *this);
1439}
1440
1441LogicalResult DmaWaitOp::verify() {
1442 // Check that the number of tag indices matches the tagMemRef rank.
1443 unsigned numTagIndices = getTagIndices().size();
1444 unsigned tagMemRefRank = getTagMemRefRank();
1445 if (numTagIndices != tagMemRefRank)
1446 return emitOpError() << "expected tagIndices to have the same number of "
1447 "elements as the tagMemRef rank, expected "
1448 << tagMemRefRank << ", but got " << numTagIndices;
1449 return success();
1450}
1451
1452//===----------------------------------------------------------------------===//
1453// ExtractAlignedPointerAsIndexOp
1454//===----------------------------------------------------------------------===//
1455
1456void ExtractAlignedPointerAsIndexOp::getAsmResultNames(
1457 function_ref<void(Value, StringRef)> setNameFn) {
1458 setNameFn(getResult(), "intptr");
1459}
1460
1461//===----------------------------------------------------------------------===//
1462// ExtractStridedMetadataOp
1463//===----------------------------------------------------------------------===//
1464
1465/// The number and type of the results are inferred from the
1466/// shape of the source.
1467LogicalResult ExtractStridedMetadataOp::inferReturnTypes(
1468 MLIRContext *context, std::optional<Location> location,
1469 ExtractStridedMetadataOp::Adaptor adaptor,
1470 SmallVectorImpl<Type> &inferredReturnTypes) {
1471 auto sourceType = llvm::dyn_cast<MemRefType>(Val: adaptor.getSource().getType());
1472 if (!sourceType)
1473 return failure();
1474
1475 unsigned sourceRank = sourceType.getRank();
1476 IndexType indexType = IndexType::get(context);
1477 auto memrefType =
1478 MemRefType::get(shape: {}, elementType: sourceType.getElementType(),
1479 layout: MemRefLayoutAttrInterface{}, memorySpace: sourceType.getMemorySpace());
1480 // Base.
1481 inferredReturnTypes.push_back(Elt: memrefType);
1482 // Offset.
1483 inferredReturnTypes.push_back(Elt: indexType);
1484 // Sizes and strides.
1485 for (unsigned i = 0; i < sourceRank * 2; ++i)
1486 inferredReturnTypes.push_back(Elt: indexType);
1487 return success();
1488}
1489
1490void ExtractStridedMetadataOp::getAsmResultNames(
1491 function_ref<void(Value, StringRef)> setNameFn) {
1492 setNameFn(getBaseBuffer(), "base_buffer");
1493 setNameFn(getOffset(), "offset");
1494 // For multi-result to work properly with pretty names and packed syntax `x:3`
1495 // we can only give a pretty name to the first value in the pack.
1496 if (!getSizes().empty()) {
1497 setNameFn(getSizes().front(), "sizes");
1498 setNameFn(getStrides().front(), "strides");
1499 }
1500}
1501
1502/// Helper function to perform the replacement of all constant uses of `values`
1503/// by a materialized constant extracted from `maybeConstants`.
1504/// `values` and `maybeConstants` are expected to have the same size.
1505template <typename Container>
1506static bool replaceConstantUsesOf(OpBuilder &rewriter, Location loc,
1507 Container values,
1508 ArrayRef<OpFoldResult> maybeConstants) {
1509 assert(values.size() == maybeConstants.size() &&
1510 " expected values and maybeConstants of the same size");
1511 bool atLeastOneReplacement = false;
1512 for (auto [maybeConstant, result] : llvm::zip(maybeConstants, values)) {
1513 // Don't materialize a constant if there are no uses: this would indice
1514 // infinite loops in the driver.
1515 if (result.use_empty() || maybeConstant == getAsOpFoldResult(result))
1516 continue;
1517 assert(isa<Attribute>(maybeConstant) &&
1518 "The constified value should be either unchanged (i.e., == result) "
1519 "or a constant");
1520 Value constantVal = arith::ConstantIndexOp::create(
1521 rewriter, loc,
1522 llvm::cast<IntegerAttr>(cast<Attribute>(maybeConstant)).getInt());
1523 for (Operation *op : llvm::make_early_inc_range(result.getUsers())) {
1524 // modifyOpInPlace: lambda cannot capture structured bindings in C++17
1525 // yet.
1526 op->replaceUsesOfWith(from: result, to: constantVal);
1527 atLeastOneReplacement = true;
1528 }
1529 }
1530 return atLeastOneReplacement;
1531}
1532
1533LogicalResult
1534ExtractStridedMetadataOp::fold(FoldAdaptor adaptor,
1535 SmallVectorImpl<OpFoldResult> &results) {
1536 OpBuilder builder(*this);
1537
1538 bool atLeastOneReplacement = replaceConstantUsesOf(
1539 rewriter&: builder, loc: getLoc(), values: ArrayRef<TypedValue<IndexType>>(getOffset()),
1540 maybeConstants: getConstifiedMixedOffset());
1541 atLeastOneReplacement |= replaceConstantUsesOf(rewriter&: builder, loc: getLoc(), values: getSizes(),
1542 maybeConstants: getConstifiedMixedSizes());
1543 atLeastOneReplacement |= replaceConstantUsesOf(
1544 rewriter&: builder, loc: getLoc(), values: getStrides(), maybeConstants: getConstifiedMixedStrides());
1545
1546 // extract_strided_metadata(cast(x)) -> extract_strided_metadata(x).
1547 if (auto prev = getSource().getDefiningOp<CastOp>())
1548 if (isa<MemRefType>(Val: prev.getSource().getType())) {
1549 getSourceMutable().assign(value: prev.getSource());
1550 atLeastOneReplacement = true;
1551 }
1552
1553 return success(IsSuccess: atLeastOneReplacement);
1554}
1555
1556SmallVector<OpFoldResult> ExtractStridedMetadataOp::getConstifiedMixedSizes() {
1557 SmallVector<OpFoldResult> values = getAsOpFoldResult(values: getSizes());
1558 constifyIndexValues(values, constValues: getSource().getType().getShape());
1559 return values;
1560}
1561
1562SmallVector<OpFoldResult>
1563ExtractStridedMetadataOp::getConstifiedMixedStrides() {
1564 SmallVector<OpFoldResult> values = getAsOpFoldResult(values: getStrides());
1565 SmallVector<int64_t> staticValues;
1566 int64_t unused;
1567 LogicalResult status =
1568 getSource().getType().getStridesAndOffset(strides&: staticValues, offset&: unused);
1569 (void)status;
1570 assert(succeeded(status) && "could not get strides from type");
1571 constifyIndexValues(values, constValues: staticValues);
1572 return values;
1573}
1574
1575OpFoldResult ExtractStridedMetadataOp::getConstifiedMixedOffset() {
1576 OpFoldResult offsetOfr = getAsOpFoldResult(val: getOffset());
1577 SmallVector<OpFoldResult> values(1, offsetOfr);
1578 SmallVector<int64_t> staticValues, unused;
1579 int64_t offset;
1580 LogicalResult status =
1581 getSource().getType().getStridesAndOffset(strides&: unused, offset);
1582 (void)status;
1583 assert(succeeded(status) && "could not get offset from type");
1584 staticValues.push_back(Elt: offset);
1585 constifyIndexValues(values, constValues: staticValues);
1586 return values[0];
1587}
1588
1589//===----------------------------------------------------------------------===//
1590// GenericAtomicRMWOp
1591//===----------------------------------------------------------------------===//
1592
1593void GenericAtomicRMWOp::build(OpBuilder &builder, OperationState &result,
1594 Value memref, ValueRange ivs) {
1595 OpBuilder::InsertionGuard g(builder);
1596 result.addOperands(newOperands: memref);
1597 result.addOperands(newOperands: ivs);
1598
1599 if (auto memrefType = llvm::dyn_cast<MemRefType>(Val: memref.getType())) {
1600 Type elementType = memrefType.getElementType();
1601 result.addTypes(newTypes: elementType);
1602
1603 Region *bodyRegion = result.addRegion();
1604 builder.createBlock(parent: bodyRegion);
1605 bodyRegion->addArgument(type: elementType, loc: memref.getLoc());
1606 }
1607}
1608
1609LogicalResult GenericAtomicRMWOp::verify() {
1610 auto &body = getRegion();
1611 if (body.getNumArguments() != 1)
1612 return emitOpError(message: "expected single number of entry block arguments");
1613
1614 if (getResult().getType() != body.getArgument(i: 0).getType())
1615 return emitOpError(message: "expected block argument of the same type result type");
1616
1617 bool hasSideEffects =
1618 body.walk(callback: [&](Operation *nestedOp) {
1619 if (isMemoryEffectFree(op: nestedOp))
1620 return WalkResult::advance();
1621 nestedOp->emitError(
1622 message: "body of 'memref.generic_atomic_rmw' should contain "
1623 "only operations with no side effects");
1624 return WalkResult::interrupt();
1625 })
1626 .wasInterrupted();
1627 return hasSideEffects ? failure() : success();
1628}
1629
1630ParseResult GenericAtomicRMWOp::parse(OpAsmParser &parser,
1631 OperationState &result) {
1632 OpAsmParser::UnresolvedOperand memref;
1633 Type memrefType;
1634 SmallVector<OpAsmParser::UnresolvedOperand, 4> ivs;
1635
1636 Type indexType = parser.getBuilder().getIndexType();
1637 if (parser.parseOperand(result&: memref) ||
1638 parser.parseOperandList(result&: ivs, delimiter: OpAsmParser::Delimiter::Square) ||
1639 parser.parseColonType(result&: memrefType) ||
1640 parser.resolveOperand(operand: memref, type: memrefType, result&: result.operands) ||
1641 parser.resolveOperands(operands&: ivs, type: indexType, result&: result.operands))
1642 return failure();
1643
1644 Region *body = result.addRegion();
1645 if (parser.parseRegion(region&: *body, arguments: {}) ||
1646 parser.parseOptionalAttrDict(result&: result.attributes))
1647 return failure();
1648 result.types.push_back(Elt: llvm::cast<MemRefType>(Val&: memrefType).getElementType());
1649 return success();
1650}
1651
1652void GenericAtomicRMWOp::print(OpAsmPrinter &p) {
1653 p << ' ' << getMemref() << "[" << getIndices()
1654 << "] : " << getMemref().getType() << ' ';
1655 p.printRegion(blocks&: getRegion());
1656 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary());
1657}
1658
1659TypedValue<MemRefType> GenericAtomicRMWOp::getAccessedMemref() {
1660 return getMemref();
1661}
1662
1663std::optional<SmallVector<Value>> GenericAtomicRMWOp::updateMemrefAndIndices(
1664 RewriterBase &rewriter, Value newMemref, ValueRange newIndices) {
1665 rewriter.modifyOpInPlace(root: *this, callable: [&]() {
1666 getMemrefMutable().assign(value: newMemref);
1667 getIndicesMutable().assign(values: newIndices);
1668 });
1669 return std::nullopt;
1670}
1671
1672//===----------------------------------------------------------------------===//
1673// AtomicYieldOp
1674//===----------------------------------------------------------------------===//
1675
1676LogicalResult AtomicYieldOp::verify() {
1677 Type parentType = (*this)->getParentOp()->getResultTypes().front();
1678 Type resultType = getResult().getType();
1679 if (parentType != resultType)
1680 return emitOpError() << "types mismatch between yield op: " << resultType
1681 << " and its parent: " << parentType;
1682 return success();
1683}
1684
1685//===----------------------------------------------------------------------===//
1686// GlobalOp
1687//===----------------------------------------------------------------------===//
1688
1689static void printGlobalMemrefOpTypeAndInitialValue(OpAsmPrinter &p, GlobalOp op,
1690 TypeAttr type,
1691 Attribute initialValue) {
1692 p << type;
1693 if (!op.isExternal()) {
1694 p << " = ";
1695 if (op.isUninitialized())
1696 p << "uninitialized";
1697 else
1698 p.printAttributeWithoutType(attr: initialValue);
1699 }
1700}
1701
1702static ParseResult
1703parseGlobalMemrefOpTypeAndInitialValue(OpAsmParser &parser, TypeAttr &typeAttr,
1704 Attribute &initialValue) {
1705 Type type;
1706 if (parser.parseType(result&: type))
1707 return failure();
1708
1709 auto memrefType = llvm::dyn_cast<MemRefType>(Val&: type);
1710 if (!memrefType || !memrefType.hasStaticShape())
1711 return parser.emitError(loc: parser.getNameLoc())
1712 << "type should be static shaped memref, but got " << type;
1713 typeAttr = TypeAttr::get(type);
1714
1715 if (parser.parseOptionalEqual())
1716 return success();
1717
1718 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "uninitialized"))) {
1719 initialValue = UnitAttr::get(context: parser.getContext());
1720 return success();
1721 }
1722
1723 Type tensorType = getTensorTypeFromMemRefType(type: memrefType);
1724 if (parser.parseAttribute(result&: initialValue, type: tensorType))
1725 return failure();
1726 if (!llvm::isa<ElementsAttr>(Val: initialValue))
1727 return parser.emitError(loc: parser.getNameLoc())
1728 << "initial value should be a unit or elements attribute";
1729 return success();
1730}
1731
1732LogicalResult GlobalOp::verify() {
1733 auto memrefType = llvm::dyn_cast<MemRefType>(Val: getType());
1734 if (!memrefType || !memrefType.hasStaticShape())
1735 return emitOpError(message: "type should be static shaped memref, but got ")
1736 << getType();
1737
1738 // Verify that the initial value, if present, is either a unit attribute or
1739 // an elements attribute.
1740 if (getInitialValue().has_value()) {
1741 Attribute initValue = getInitialValue().value();
1742 if (!llvm::isa<UnitAttr>(Val: initValue) && !llvm::isa<ElementsAttr>(Val: initValue))
1743 return emitOpError(message: "initial value should be a unit or elements "
1744 "attribute, but got ")
1745 << initValue;
1746
1747 // Check that the type of the initial value is compatible with the type of
1748 // the global variable.
1749 if (auto elementsAttr = llvm::dyn_cast<ElementsAttr>(Val&: initValue)) {
1750 // Check the element types match.
1751 auto initElementType =
1752 cast<TensorType>(Val: elementsAttr.getType()).getElementType();
1753 auto memrefElementType = memrefType.getElementType();
1754
1755 if (initElementType != memrefElementType)
1756 return emitOpError(message: "initial value element expected to be of type ")
1757 << memrefElementType << ", but was of type " << initElementType;
1758
1759 // Check the shapes match, given that memref globals can only produce
1760 // statically shaped memrefs and elements literal type must have a static
1761 // shape we can assume both types are shaped.
1762 auto initShape = elementsAttr.getShapedType().getShape();
1763 auto memrefShape = memrefType.getShape();
1764 if (initShape != memrefShape)
1765 return emitOpError(message: "initial value shape expected to be ")
1766 << memrefShape << " but was " << initShape;
1767 }
1768 }
1769
1770 // TODO: verify visibility for declarations.
1771 return success();
1772}
1773
1774ElementsAttr GlobalOp::getConstantInitValue() {
1775 auto initVal = getInitialValue();
1776 if (getConstant() && initVal.has_value())
1777 return llvm::cast<ElementsAttr>(Val&: initVal.value());
1778 return {};
1779}
1780
1781//===----------------------------------------------------------------------===//
1782// GetGlobalOp
1783//===----------------------------------------------------------------------===//
1784
1785LogicalResult
1786GetGlobalOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
1787 // Verify that the result type is same as the type of the referenced
1788 // memref.global op.
1789 auto global =
1790 symbolTable.lookupNearestSymbolFrom<GlobalOp>(from: *this, symbol: getNameAttr());
1791 if (!global)
1792 return emitOpError(message: "'")
1793 << getName() << "' does not reference a valid global memref";
1794
1795 Type resultType = getResult().getType();
1796 if (global.getType() != resultType)
1797 return emitOpError(message: "result type ")
1798 << resultType << " does not match type " << global.getType()
1799 << " of the global memref @" << getName();
1800 return success();
1801}
1802
1803//===----------------------------------------------------------------------===//
1804// LoadOp
1805//===----------------------------------------------------------------------===//
1806
1807static ParseResult parseBoolAttr(OpAsmParser &parser, BoolAttr &result) {
1808 Attribute attr;
1809 if (parser.parseAttribute(result&: attr))
1810 return failure();
1811 result = dyn_cast<BoolAttr>(Val&: attr);
1812 if (!result)
1813 return parser.emitError(loc: parser.getCurrentLocation(),
1814 message: "expected boolean attribute");
1815 return success();
1816}
1817
1818static void printBoolAttr(OpAsmPrinter &printer, Operation *, BoolAttr attr) {
1819 printer.printAttribute(attr);
1820}
1821
1822OpFoldResult LoadOp::fold(FoldAdaptor adaptor) {
1823 /// load(memrefcast) -> load
1824 if (succeeded(Result: foldMemRefCast(op: *this)))
1825 return getResult();
1826
1827 // Fold load from a global constant memref.
1828 auto getGlobalOp = getMemref().getDefiningOp<memref::GetGlobalOp>();
1829 if (!getGlobalOp)
1830 return {};
1831
1832 // Get to the memref.global defining the symbol.
1833 auto global = SymbolTable::lookupNearestSymbolFrom<memref::GlobalOp>(
1834 from: getGlobalOp, symbol: getGlobalOp.getNameAttr());
1835 if (!global)
1836 return {};
1837 // If it's a splat constant, we can fold irrespective of indices.
1838 auto splatAttr =
1839 dyn_cast_or_null<SplatElementsAttr>(Val: global.getConstantInitValue());
1840 if (!splatAttr)
1841 return {};
1842
1843 return splatAttr.getSplatValue<Attribute>();
1844}
1845
1846TypedValue<MemRefType> LoadOp::getAccessedMemref() { return getMemref(); }
1847
1848std::optional<SmallVector<Value>>
1849LoadOp::updateMemrefAndIndices(RewriterBase &rewriter, Value newMemref,
1850 ValueRange newIndices) {
1851 rewriter.modifyOpInPlace(root: *this, callable: [&]() {
1852 getMemrefMutable().assign(value: newMemref);
1853 getIndicesMutable().assign(values: newIndices);
1854 });
1855 return std::nullopt;
1856}
1857
1858FailureOr<std::optional<SmallVector<Value>>>
1859LoadOp::bubbleDownCasts(OpBuilder &builder) {
1860 return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getMemrefMutable(),
1861 results: getResult());
1862}
1863
1864//===----------------------------------------------------------------------===//
1865// MemorySpaceCastOp
1866//===----------------------------------------------------------------------===//
1867
1868void MemorySpaceCastOp::getAsmResultNames(
1869 function_ref<void(Value, StringRef)> setNameFn) {
1870 setNameFn(getResult(), "memspacecast");
1871}
1872
1873bool MemorySpaceCastOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
1874 if (inputs.size() != 1 || outputs.size() != 1)
1875 return false;
1876 Type a = inputs.front(), b = outputs.front();
1877 auto aT = llvm::dyn_cast<MemRefType>(Val&: a);
1878 auto bT = llvm::dyn_cast<MemRefType>(Val&: b);
1879
1880 auto uaT = llvm::dyn_cast<UnrankedMemRefType>(Val&: a);
1881 auto ubT = llvm::dyn_cast<UnrankedMemRefType>(Val&: b);
1882
1883 if (aT && bT) {
1884 if (aT.getElementType() != bT.getElementType())
1885 return false;
1886 if (aT.getLayout() != bT.getLayout())
1887 return false;
1888 if (aT.getShape() != bT.getShape())
1889 return false;
1890 return true;
1891 }
1892 if (uaT && ubT) {
1893 return uaT.getElementType() == ubT.getElementType();
1894 }
1895 return false;
1896}
1897
1898OpFoldResult MemorySpaceCastOp::fold(FoldAdaptor adaptor) {
1899 // memory_space_cast(memory_space_cast(v, t1), t2) -> memory_space_cast(v,
1900 // t2)
1901 if (auto parentCast = getSource().getDefiningOp<MemorySpaceCastOp>()) {
1902 getSourceMutable().assign(value: parentCast.getSource());
1903 return getResult();
1904 }
1905 return Value{};
1906}
1907
1908TypedValue<PtrLikeTypeInterface> MemorySpaceCastOp::getSourcePtr() {
1909 return getSource();
1910}
1911
1912TypedValue<PtrLikeTypeInterface> MemorySpaceCastOp::getTargetPtr() {
1913 return getDest();
1914}
1915
1916bool MemorySpaceCastOp::isValidMemorySpaceCast(PtrLikeTypeInterface tgt,
1917 PtrLikeTypeInterface src) {
1918 return isa<BaseMemRefType>(Val: tgt) &&
1919 tgt.clonePtrWith(memorySpace: src.getMemorySpace(), elementType: std::nullopt) == src;
1920}
1921
1922MemorySpaceCastOpInterface MemorySpaceCastOp::cloneMemorySpaceCastOp(
1923 OpBuilder &b, PtrLikeTypeInterface tgt,
1924 TypedValue<PtrLikeTypeInterface> src) {
1925 assert(isValidMemorySpaceCast(tgt, src.getType()) && "invalid arguments");
1926 return MemorySpaceCastOp::create(builder&: b, location: getLoc(), dest: tgt, source: src);
1927}
1928
1929/// The only cast we recognize as promotable is to the generic space.
1930bool MemorySpaceCastOp::isSourcePromotable() {
1931 return getDest().getType().getMemorySpace() == nullptr;
1932}
1933
1934//===----------------------------------------------------------------------===//
1935// PrefetchOp
1936//===----------------------------------------------------------------------===//
1937
1938void PrefetchOp::print(OpAsmPrinter &p) {
1939 p << " " << getMemref() << '[';
1940 p.printOperands(operands: getIndices());
1941 p << ']' << ", " << (getIsWrite() ? "write" : "read");
1942 p << ", locality<" << getLocalityHint();
1943 p << ">, " << (getIsDataCache() ? "data" : "instr");
1944 p.printOptionalAttrDict(
1945 attrs: (*this)->getDiscardableAttrDictionary(),
1946 /*elidedAttrs=*/{"localityHint", "isWrite", "isDataCache"});
1947 p << " : " << getMemRefType();
1948}
1949
1950ParseResult PrefetchOp::parse(OpAsmParser &parser, OperationState &result) {
1951 OpAsmParser::UnresolvedOperand memrefInfo;
1952 SmallVector<OpAsmParser::UnresolvedOperand, 4> indexInfo;
1953 IntegerAttr localityHint;
1954 MemRefType type;
1955 StringRef readOrWrite, cacheType;
1956
1957 auto indexTy = parser.getBuilder().getIndexType();
1958 auto i32Type = parser.getBuilder().getIntegerType(width: 32);
1959 if (parser.parseOperand(result&: memrefInfo) ||
1960 parser.parseOperandList(result&: indexInfo, delimiter: OpAsmParser::Delimiter::Square) ||
1961 parser.parseComma() || parser.parseKeyword(keyword: &readOrWrite) ||
1962 parser.parseComma() || parser.parseKeyword(keyword: "locality") ||
1963 parser.parseLess() ||
1964 parser.parseAttribute(result&: localityHint, type: i32Type, attrName: "localityHint",
1965 attrs&: result.attributes) ||
1966 parser.parseGreater() || parser.parseComma() ||
1967 parser.parseKeyword(keyword: &cacheType) || parser.parseColonType(result&: type) ||
1968 parser.resolveOperand(operand: memrefInfo, type, result&: result.operands) ||
1969 parser.resolveOperands(operands&: indexInfo, type: indexTy, result&: result.operands))
1970 return failure();
1971
1972 if (readOrWrite != "read" && readOrWrite != "write")
1973 return parser.emitError(loc: parser.getNameLoc(),
1974 message: "rw specifier has to be 'read' or 'write'");
1975 result.addAttribute(name: PrefetchOp::getIsWriteAttrStrName(),
1976 attr: parser.getBuilder().getBoolAttr(value: readOrWrite == "write"));
1977
1978 if (cacheType != "data" && cacheType != "instr")
1979 return parser.emitError(loc: parser.getNameLoc(),
1980 message: "cache type has to be 'data' or 'instr'");
1981
1982 result.addAttribute(name: PrefetchOp::getIsDataCacheAttrStrName(),
1983 attr: parser.getBuilder().getBoolAttr(value: cacheType == "data"));
1984
1985 return success();
1986}
1987
1988LogicalResult PrefetchOp::verify() {
1989 if (getNumOperands() != 1 + getMemRefType().getRank())
1990 return emitOpError(message: "too few indices");
1991
1992 return success();
1993}
1994
1995LogicalResult PrefetchOp::fold(FoldAdaptor adaptor,
1996 SmallVectorImpl<OpFoldResult> &results) {
1997 // prefetch(memrefcast) -> prefetch
1998 return foldMemRefCast(op: *this);
1999}
2000
2001TypedValue<MemRefType> PrefetchOp::getAccessedMemref() { return getMemref(); }
2002
2003std::optional<SmallVector<Value>>
2004PrefetchOp::updateMemrefAndIndices(RewriterBase &rewriter, Value newMemref,
2005 ValueRange newIndices) {
2006 rewriter.modifyOpInPlace(root: *this, callable: [&]() {
2007 getMemrefMutable().assign(value: newMemref);
2008 getIndicesMutable().assign(values: newIndices);
2009 });
2010 return std::nullopt;
2011}
2012
2013//===----------------------------------------------------------------------===//
2014// RankOp
2015//===----------------------------------------------------------------------===//
2016
2017OpFoldResult RankOp::fold(FoldAdaptor adaptor) {
2018 // Constant fold rank when the rank of the operand is known.
2019 auto type = getOperand().getType();
2020 auto shapedType = llvm::dyn_cast<ShapedType>(Val&: type);
2021 if (shapedType && shapedType.hasRank())
2022 return IntegerAttr::get(type: IndexType::get(context: getContext()), value: shapedType.getRank());
2023 return IntegerAttr();
2024}
2025
2026//===----------------------------------------------------------------------===//
2027// ReinterpretCastOp
2028//===----------------------------------------------------------------------===//
2029
2030namespace {
2031
2032struct PrintDynamicOrValue {
2033 int64_t value;
2034};
2035
2036Diagnostic &operator<<(Diagnostic &diag, PrintDynamicOrValue printed) {
2037 if (ShapedType::isDynamic(dValue: printed.value))
2038 return diag << "dynamic";
2039 return diag << printed.value;
2040}
2041
2042} // namespace
2043
2044void ReinterpretCastOp::getAsmResultNames(
2045 function_ref<void(Value, StringRef)> setNameFn) {
2046 setNameFn(getResult(), "reinterpret_cast");
2047}
2048
2049/// Build a ReinterpretCastOp with all dynamic entries: `staticOffsets`,
2050/// `staticSizes` and `staticStrides` are automatically filled with
2051/// source-memref-rank sentinel values that encode dynamic entries.
2052void ReinterpretCastOp::build(OpBuilder &b, OperationState &result,
2053 MemRefType resultType, Value source,
2054 OpFoldResult offset, ArrayRef<OpFoldResult> sizes,
2055 ArrayRef<OpFoldResult> strides,
2056 ArrayRef<NamedAttribute> attrs) {
2057 SmallVector<int64_t> staticOffsets, staticSizes, staticStrides;
2058 SmallVector<Value> dynamicOffsets, dynamicSizes, dynamicStrides;
2059 dispatchIndexOpFoldResults(ofrs: offset, dynamicVec&: dynamicOffsets, staticVec&: staticOffsets);
2060 dispatchIndexOpFoldResults(ofrs: sizes, dynamicVec&: dynamicSizes, staticVec&: staticSizes);
2061 dispatchIndexOpFoldResults(ofrs: strides, dynamicVec&: dynamicStrides, staticVec&: staticStrides);
2062 result.addAttributes(newAttributes: attrs);
2063 build(odsBuilder&: b, odsState&: result, result: resultType, source, offsets: dynamicOffsets, sizes: dynamicSizes,
2064 strides: dynamicStrides, static_offsets: b.getDenseI64ArrayAttr(values: staticOffsets),
2065 static_sizes: b.getDenseI64ArrayAttr(values: staticSizes),
2066 static_strides: b.getDenseI64ArrayAttr(values: staticStrides));
2067}
2068
2069void ReinterpretCastOp::build(OpBuilder &b, OperationState &result,
2070 Value source, OpFoldResult offset,
2071 ArrayRef<OpFoldResult> sizes,
2072 ArrayRef<OpFoldResult> strides,
2073 ArrayRef<NamedAttribute> attrs) {
2074 auto sourceType = cast<BaseMemRefType>(Val: source.getType());
2075 SmallVector<int64_t> staticOffsets, staticSizes, staticStrides;
2076 SmallVector<Value> dynamicOffsets, dynamicSizes, dynamicStrides;
2077 dispatchIndexOpFoldResults(ofrs: offset, dynamicVec&: dynamicOffsets, staticVec&: staticOffsets);
2078 dispatchIndexOpFoldResults(ofrs: sizes, dynamicVec&: dynamicSizes, staticVec&: staticSizes);
2079 dispatchIndexOpFoldResults(ofrs: strides, dynamicVec&: dynamicStrides, staticVec&: staticStrides);
2080 auto stridedLayout = StridedLayoutAttr::get(
2081 context: b.getContext(), offset: staticOffsets.front(), strides: staticStrides);
2082 auto resultType = MemRefType::get(shape: staticSizes, elementType: sourceType.getElementType(),
2083 layout: stridedLayout, memorySpace: sourceType.getMemorySpace());
2084 build(b, result, resultType, source, offset, sizes, strides, attrs);
2085}
2086
2087void ReinterpretCastOp::build(OpBuilder &b, OperationState &result,
2088 MemRefType resultType, Value source,
2089 int64_t offset, ArrayRef<int64_t> sizes,
2090 ArrayRef<int64_t> strides,
2091 ArrayRef<NamedAttribute> attrs) {
2092 SmallVector<OpFoldResult> sizeValues = llvm::map_to_vector<4>(
2093 C&: sizes, F: [&](int64_t v) -> OpFoldResult { return b.getI64IntegerAttr(value: v); });
2094 SmallVector<OpFoldResult> strideValues =
2095 llvm::map_to_vector<4>(C&: strides, F: [&](int64_t v) -> OpFoldResult {
2096 return b.getI64IntegerAttr(value: v);
2097 });
2098 build(b, result, resultType, source, offset: b.getI64IntegerAttr(value: offset), sizes: sizeValues,
2099 strides: strideValues, attrs);
2100}
2101
2102void ReinterpretCastOp::build(OpBuilder &b, OperationState &result,
2103 MemRefType resultType, Value source, Value offset,
2104 ValueRange sizes, ValueRange strides,
2105 ArrayRef<NamedAttribute> attrs) {
2106 SmallVector<OpFoldResult> sizeValues =
2107 llvm::map_to_vector<4>(C&: sizes, F: [](Value v) -> OpFoldResult { return v; });
2108 SmallVector<OpFoldResult> strideValues = llvm::map_to_vector<4>(
2109 C&: strides, F: [](Value v) -> OpFoldResult { return v; });
2110 build(b, result, resultType, source, offset, sizes: sizeValues, strides: strideValues, attrs);
2111}
2112
2113// TODO: ponder whether we want to allow missing trailing sizes/strides that are
2114// completed automatically, like we have for subview and extract_slice.
2115LogicalResult ReinterpretCastOp::verify() {
2116 // The source and result memrefs should be in the same memory space.
2117 auto srcType = llvm::cast<BaseMemRefType>(Val: getSource().getType());
2118 auto resultType = llvm::cast<MemRefType>(Val: getType());
2119 if (srcType.getMemorySpace() != resultType.getMemorySpace())
2120 return emitError(message: "different memory spaces specified for source type ")
2121 << srcType << " and result memref type " << resultType;
2122 if (failed(Result: verifyElementTypesMatch(op: *this, lhs: srcType, rhs: resultType, lhsName: "source",
2123 rhsName: "result")))
2124 return failure();
2125
2126 // Match sizes in result memref type and in static_sizes attribute.
2127 for (auto [idx, resultSize, expectedSize] :
2128 llvm::enumerate(First: resultType.getShape(), Rest: getStaticSizes())) {
2129 if (resultSize != expectedSize)
2130 return emitError(message: "expected result type with size = ")
2131 << PrintDynamicOrValue{.value: expectedSize} << " instead of "
2132 << PrintDynamicOrValue{.value: resultSize} << " in dim = " << idx;
2133 }
2134
2135 // Match offset and strides in static_offset and static_strides attributes. If
2136 // result memref type has no affine map specified, this will assume an
2137 // identity layout.
2138 int64_t resultOffset;
2139 SmallVector<int64_t, 4> resultStrides;
2140 if (failed(Result: resultType.getStridesAndOffset(strides&: resultStrides, offset&: resultOffset)))
2141 return emitError(message: "expected result type to have strided layout but found ")
2142 << resultType;
2143
2144 // Match offset in result memref type and in static_offsets attribute.
2145 int64_t expectedOffset = getStaticOffsets().front();
2146 if (resultOffset != expectedOffset)
2147 return emitError(message: "expected result type with offset = ")
2148 << PrintDynamicOrValue{.value: expectedOffset} << " instead of "
2149 << PrintDynamicOrValue{.value: resultOffset};
2150
2151 // Match strides in result memref type and in static_strides attribute.
2152 for (auto [idx, resultStride, expectedStride] :
2153 llvm::enumerate(First&: resultStrides, Rest: getStaticStrides())) {
2154 if (resultStride != expectedStride)
2155 return emitError(message: "expected result type with stride = ")
2156 << PrintDynamicOrValue{.value: expectedStride} << " instead of "
2157 << PrintDynamicOrValue{.value: resultStride} << " in dim = " << idx;
2158 }
2159
2160 return success();
2161}
2162
2163OpFoldResult ReinterpretCastOp::fold(FoldAdaptor /*operands*/) {
2164 Value src = getSource();
2165 auto getPrevSrc = [&]() -> Value {
2166 // reinterpret_cast(reinterpret_cast(x)) -> reinterpret_cast(x).
2167 if (auto prev = src.getDefiningOp<ReinterpretCastOp>())
2168 return prev.getSource();
2169
2170 // reinterpret_cast(cast(x)) -> reinterpret_cast(x).
2171 if (auto prev = src.getDefiningOp<CastOp>())
2172 return prev.getSource();
2173
2174 // reinterpret_cast(subview(x)) -> reinterpret_cast(x) if subview offsets
2175 // are 0.
2176 if (auto prev = src.getDefiningOp<SubViewOp>())
2177 if (llvm::all_of(Range: prev.getMixedOffsets(), P: isZeroInteger))
2178 return prev.getSource();
2179
2180 return nullptr;
2181 };
2182
2183 if (auto prevSrc = getPrevSrc()) {
2184 getSourceMutable().assign(value: prevSrc);
2185 return getResult();
2186 }
2187
2188 // reinterpret_cast(x) w/o offset/shape/stride changes -> x
2189 if (ShapedType::isStaticShape(dSizes: getType().getShape()) &&
2190 src.getType() == getType() && getStaticOffsets().front() == 0) {
2191 return src;
2192 }
2193
2194 return nullptr;
2195}
2196
2197SmallVector<OpFoldResult> ReinterpretCastOp::getConstifiedMixedSizes() {
2198 SmallVector<OpFoldResult> values = getMixedSizes();
2199 constifyIndexValues(values, constValues: getType().getShape());
2200 return values;
2201}
2202
2203SmallVector<OpFoldResult> ReinterpretCastOp::getConstifiedMixedStrides() {
2204 SmallVector<OpFoldResult> values = getMixedStrides();
2205 SmallVector<int64_t> staticValues;
2206 int64_t unused;
2207 LogicalResult status = getType().getStridesAndOffset(strides&: staticValues, offset&: unused);
2208 (void)status;
2209 assert(succeeded(status) && "could not get strides from type");
2210 constifyIndexValues(values, constValues: staticValues);
2211 return values;
2212}
2213
2214OpFoldResult ReinterpretCastOp::getConstifiedMixedOffset() {
2215 SmallVector<OpFoldResult> values = getMixedOffsets();
2216 assert(values.size() == 1 &&
2217 "reinterpret_cast must have one and only one offset");
2218 SmallVector<int64_t> staticValues, unused;
2219 int64_t offset;
2220 LogicalResult status = getType().getStridesAndOffset(strides&: unused, offset);
2221 (void)status;
2222 assert(succeeded(status) && "could not get offset from type");
2223 staticValues.push_back(Elt: offset);
2224 constifyIndexValues(values, constValues: staticValues);
2225 return values[0];
2226}
2227
2228namespace {
2229/// Replace the sequence:
2230/// ```
2231/// base, offset, sizes, strides = extract_strided_metadata src
2232/// dst = reinterpret_cast base to offset, sizes, strides
2233/// ```
2234/// With
2235///
2236/// ```
2237/// dst = memref.cast src
2238/// ```
2239///
2240/// Note: The cast operation is only inserted when the type of dst and src
2241/// are not the same. E.g., when going from <4xf32> to <?xf32>.
2242///
2243/// This pattern also matches when the offset, sizes, and strides don't come
2244/// directly from the `extract_strided_metadata`'s results but it can be
2245/// statically proven that they would hold the same values.
2246///
2247/// For instance, the following sequence would be replaced:
2248/// ```
2249/// base, offset, sizes, strides =
2250/// extract_strided_metadata memref : memref<3x4xty>
2251/// dst = reinterpret_cast base to 0, [3, 4], strides
2252/// ```
2253/// Because we know (thanks to the type of the input memref) that variable
2254/// `offset` and `sizes` will respectively hold 0 and [3, 4].
2255///
2256/// Similarly, the following sequence would be replaced:
2257/// ```
2258/// c0 = arith.constant 0
2259/// c4 = arith.constant 4
2260/// base, offset, sizes, strides =
2261/// extract_strided_metadata memref : memref<3x4xty>
2262/// dst = reinterpret_cast base to c0, [3, c4], strides
2263/// ```
2264/// Because we know that `offset`and `c0` will hold 0
2265/// and `c4` will hold 4.
2266///
2267/// If the pattern above does not match, the input of the
2268/// extract_strided_metadata is always folded into the input of the
2269/// reinterpret_cast operator. This allows for dead code elimination to get rid
2270/// of the extract_strided_metadata in some cases.
2271struct ReinterpretCastOpExtractStridedMetadataFolder
2272 : public OpRewritePattern<ReinterpretCastOp> {
2273public:
2274 using OpRewritePattern<ReinterpretCastOp>::OpRewritePattern;
2275
2276 LogicalResult matchAndRewrite(ReinterpretCastOp op,
2277 PatternRewriter &rewriter) const override {
2278 auto extractStridedMetadata =
2279 op.getSource().getDefiningOp<ExtractStridedMetadataOp>();
2280 if (!extractStridedMetadata)
2281 return failure();
2282
2283 // Check if the reinterpret cast reconstructs a memref with the exact same
2284 // properties as the extract strided metadata.
2285 auto isReinterpretCastNoop = [&]() -> bool {
2286 // First, check that the strides are the same.
2287 if (!llvm::equal(LRange: extractStridedMetadata.getConstifiedMixedStrides(),
2288 RRange: op.getConstifiedMixedStrides()))
2289 return false;
2290
2291 // Second, check the sizes.
2292 if (!llvm::equal(LRange: extractStridedMetadata.getConstifiedMixedSizes(),
2293 RRange: op.getConstifiedMixedSizes()))
2294 return false;
2295
2296 // Finally, check the offset.
2297 assert(op.getMixedOffsets().size() == 1 &&
2298 "reinterpret_cast with more than one offset should have been "
2299 "rejected by the verifier");
2300 return extractStridedMetadata.getConstifiedMixedOffset() ==
2301 op.getConstifiedMixedOffset();
2302 };
2303
2304 if (!isReinterpretCastNoop()) {
2305 // If the extract_strided_metadata / reinterpret_cast pair can't be
2306 // completely folded, then we could fold the input of the
2307 // extract_strided_metadata into the input of the reinterpret_cast
2308 // input. For some cases (e.g., static dimensions) the
2309 // the extract_strided_metadata is eliminated by dead code elimination.
2310 //
2311 // reinterpret_cast(extract_strided_metadata(x)) -> reinterpret_cast(x).
2312 //
2313 // We can always fold the input of a extract_strided_metadata operator
2314 // to the input of a reinterpret_cast operator, because they point to
2315 // the same memory. Note that the reinterpret_cast does not use the
2316 // layout of its input memref, only its base memory pointer which is
2317 // the same as the base pointer returned by the extract_strided_metadata
2318 // operator and the base pointer of the extract_strided_metadata memref
2319 // input.
2320 rewriter.modifyOpInPlace(root: op, callable: [&]() {
2321 op.getSourceMutable().assign(value: extractStridedMetadata.getSource());
2322 });
2323 return success();
2324 }
2325
2326 // At this point, we know that the back and forth between extract strided
2327 // metadata and reinterpret cast is a noop. However, the final type of the
2328 // reinterpret cast may not be exactly the same as the original memref.
2329 // E.g., it could be changing a dimension from static to dynamic. Check that
2330 // here and add a cast if necessary.
2331 Type srcTy = extractStridedMetadata.getSource().getType();
2332 if (srcTy == op.getResult().getType())
2333 rewriter.replaceOp(op, newValues: extractStridedMetadata.getSource());
2334 else
2335 rewriter.replaceOpWithNewOp<CastOp>(op, args: op.getType(),
2336 args: extractStridedMetadata.getSource());
2337
2338 return success();
2339 }
2340};
2341
2342struct ReinterpretCastOpConstantFolder
2343 : public OpRewritePattern<ReinterpretCastOp> {
2344public:
2345 using OpRewritePattern<ReinterpretCastOp>::OpRewritePattern;
2346
2347 LogicalResult matchAndRewrite(ReinterpretCastOp op,
2348 PatternRewriter &rewriter) const override {
2349 auto srcStaticCount = llvm::count_if(
2350 Range: llvm::concat<OpFoldResult>(Ranges: op.getMixedOffsets(), Ranges: op.getMixedSizes(),
2351 Ranges: op.getMixedStrides()),
2352 P: [](OpFoldResult ofr) { return isa<Attribute>(Val: ofr); });
2353
2354 SmallVector<OpFoldResult> offsets = {op.getConstifiedMixedOffset()};
2355 SmallVector<OpFoldResult> sizes = op.getConstifiedMixedSizes();
2356 SmallVector<OpFoldResult> strides = op.getConstifiedMixedStrides();
2357
2358 // If the offset is a negative constant, we can't fold it because the
2359 // resulting memref type would be invalid. In that case, we keep the
2360 // original offset.
2361 if (auto cst = getConstantIntValue(ofr: offsets[0]))
2362 if (*cst < 0)
2363 offsets[0] = op.getMixedOffsets()[0];
2364
2365 // If the size is a negative constant, we can't fold it because the
2366 // resulting memref type would be invalid. In that case, we keep the
2367 // original size.
2368 for (auto it : llvm::zip(t: op.getMixedSizes(), u&: sizes)) {
2369 auto &srcSizeOfr = std::get<0>(t&: it);
2370 auto &sizeOfr = std::get<1>(t&: it);
2371 if (auto cst = getConstantIntValue(ofr: sizeOfr))
2372 if (*cst < 0)
2373 sizeOfr = srcSizeOfr;
2374 }
2375
2376 // TODO: Using counting comparison instead of direct comparison because
2377 // getMixedValues (and therefore ReinterpretCastOp::getMixed...) returns
2378 // IntegerAttrs, while constifyIndexValues (and therefore
2379 // ReinterpretCastOp::getConstifiedMixed...) returns IndexAttrs.
2380 if (srcStaticCount ==
2381 llvm::count_if(Range: llvm::concat<OpFoldResult>(Ranges&: offsets, Ranges&: sizes, Ranges&: strides),
2382 P: [](OpFoldResult ofr) { return isa<Attribute>(Val: ofr); }))
2383 return failure();
2384
2385 auto newReinterpretCast = ReinterpretCastOp::create(
2386 builder&: rewriter, location: op->getLoc(), source: op.getSource(), offset: offsets[0], sizes, strides);
2387
2388 rewriter.replaceOpWithNewOp<CastOp>(op, args: op.getType(), args&: newReinterpretCast);
2389 return success();
2390 }
2391};
2392} // namespace
2393
2394void ReinterpretCastOp::getCanonicalizationPatterns(RewritePatternSet &results,
2395 MLIRContext *context) {
2396 results.add<ReinterpretCastOpExtractStridedMetadataFolder,
2397 ReinterpretCastOpConstantFolder>(arg&: context);
2398}
2399
2400FailureOr<std::optional<SmallVector<Value>>>
2401ReinterpretCastOp::bubbleDownCasts(OpBuilder &builder) {
2402 return bubbleDownCastsPassthroughOpImpl(op: *this, builder, src&: getSourceMutable());
2403}
2404
2405//===----------------------------------------------------------------------===//
2406// Reassociative reshape ops
2407//===----------------------------------------------------------------------===//
2408
2409void CollapseShapeOp::getAsmResultNames(
2410 function_ref<void(Value, StringRef)> setNameFn) {
2411 setNameFn(getResult(), "collapse_shape");
2412}
2413
2414void ExpandShapeOp::getAsmResultNames(
2415 function_ref<void(Value, StringRef)> setNameFn) {
2416 setNameFn(getResult(), "expand_shape");
2417}
2418
2419LogicalResult ExpandShapeOp::reifyResultShapes(
2420 OpBuilder &builder, ReifiedRankedShapedTypeDims &reifiedResultShapes) {
2421 reifiedResultShapes = {
2422 getMixedValues(staticValues: getStaticOutputShape(), dynamicValues: getOutputShape(), b&: builder)};
2423 return success();
2424}
2425
2426/// Helper function for verifying the shape of ExpandShapeOp and ResultShapeOp
2427/// result and operand. Layout maps are verified separately.
2428///
2429/// If `allowMultipleDynamicDimsPerGroup`, multiple dynamic dimensions are
2430/// allowed in a reassocation group.
2431static LogicalResult
2432verifyCollapsedShape(Operation *op, ArrayRef<int64_t> collapsedShape,
2433 ArrayRef<int64_t> expandedShape,
2434 ArrayRef<ReassociationIndices> reassociation,
2435 bool allowMultipleDynamicDimsPerGroup) {
2436 // There must be one reassociation group per collapsed dimension.
2437 if (collapsedShape.size() != reassociation.size())
2438 return op->emitOpError(message: "invalid number of reassociation groups: found ")
2439 << reassociation.size() << ", expected " << collapsedShape.size();
2440
2441 // The next expected expanded dimension index (while iterating over
2442 // reassociation indices).
2443 int64_t nextDim = 0;
2444 for (const auto &it : llvm::enumerate(First&: reassociation)) {
2445 ReassociationIndices group = it.value();
2446 int64_t collapsedDim = it.index();
2447
2448 bool foundDynamic = false;
2449 for (int64_t expandedDim : group) {
2450 if (expandedDim != nextDim++)
2451 return op->emitOpError(message: "reassociation indices must be contiguous");
2452
2453 if (expandedDim >= static_cast<int64_t>(expandedShape.size()))
2454 return op->emitOpError(message: "reassociation index ")
2455 << expandedDim << " is out of bounds";
2456
2457 // Check if there are multiple dynamic dims in a reassociation group.
2458 if (ShapedType::isDynamic(dValue: expandedShape[expandedDim])) {
2459 if (foundDynamic && !allowMultipleDynamicDimsPerGroup)
2460 return op->emitOpError(
2461 message: "at most one dimension in a reassociation group may be dynamic");
2462 foundDynamic = true;
2463 }
2464 }
2465
2466 // ExpandShapeOp/CollapseShapeOp may not be used to cast dynamicity.
2467 if (ShapedType::isDynamic(dValue: collapsedShape[collapsedDim]) != foundDynamic)
2468 return op->emitOpError(message: "collapsed dim (")
2469 << collapsedDim
2470 << ") must be dynamic if and only if reassociation group is "
2471 "dynamic";
2472
2473 // If all dims in the reassociation group are static, the size of the
2474 // collapsed dim can be verified.
2475 if (!foundDynamic) {
2476 int64_t groupSize = 1;
2477 for (int64_t expandedDim : group)
2478 groupSize *= expandedShape[expandedDim];
2479 if (groupSize != collapsedShape[collapsedDim])
2480 return op->emitOpError(message: "collapsed dim size (")
2481 << collapsedShape[collapsedDim]
2482 << ") must equal reassociation group size (" << groupSize << ")";
2483 }
2484 }
2485
2486 if (collapsedShape.empty()) {
2487 // Rank 0: All expanded dimensions must be 1.
2488 for (int64_t d : expandedShape)
2489 if (d != 1)
2490 return op->emitOpError(
2491 message: "rank 0 memrefs can only be extended/collapsed with/from ones");
2492 } else if (nextDim != static_cast<int64_t>(expandedShape.size())) {
2493 // Rank >= 1: Number of dimensions among all reassociation groups must match
2494 // the result memref rank.
2495 return op->emitOpError(message: "expanded rank (")
2496 << expandedShape.size()
2497 << ") inconsistent with number of reassociation indices (" << nextDim
2498 << ")";
2499 }
2500
2501 return success();
2502}
2503
2504SmallVector<AffineMap, 4> CollapseShapeOp::getReassociationMaps() {
2505 return getSymbolLessAffineMaps(reassociation: getReassociationExprs());
2506}
2507
2508SmallVector<ReassociationExprs, 4> CollapseShapeOp::getReassociationExprs() {
2509 return convertReassociationIndicesToExprs(context: getContext(),
2510 reassociationIndices: getReassociationIndices());
2511}
2512
2513SmallVector<AffineMap, 4> ExpandShapeOp::getReassociationMaps() {
2514 return getSymbolLessAffineMaps(reassociation: getReassociationExprs());
2515}
2516
2517SmallVector<ReassociationExprs, 4> ExpandShapeOp::getReassociationExprs() {
2518 return convertReassociationIndicesToExprs(context: getContext(),
2519 reassociationIndices: getReassociationIndices());
2520}
2521
2522/// Compute the layout map after expanding a given source MemRef type with the
2523/// specified reassociation indices.
2524static FailureOr<StridedLayoutAttr>
2525computeExpandedLayoutMap(MemRefType srcType, ArrayRef<int64_t> resultShape,
2526 ArrayRef<ReassociationIndices> reassociation) {
2527 int64_t srcOffset;
2528 SmallVector<int64_t> srcStrides;
2529 if (failed(Result: srcType.getStridesAndOffset(strides&: srcStrides, offset&: srcOffset)))
2530 return failure();
2531 assert(srcStrides.size() == reassociation.size() && "invalid reassociation");
2532
2533 // 1-1 mapping between srcStrides and reassociation packs.
2534 // Each srcStride starts with the given value and gets expanded according to
2535 // the proper entries in resultShape.
2536 // Example:
2537 // srcStrides = [10000, 1 , 100 ],
2538 // reassociations = [ [0], [1], [2, 3, 4]],
2539 // resultSizes = [2, 5, 4, 3, 2] = [ [2], [5], [4, 3, 2]]
2540 // -> For the purpose of stride calculation, the useful sizes are:
2541 // [x, x, x, 3, 2] = [ [x], [x], [x, 3, 2]].
2542 // resultStrides = [10000, 1, 600, 200, 100]
2543 // Note that a stride does not get expanded along the first entry of each
2544 // shape pack.
2545 SmallVector<int64_t> reverseResultStrides;
2546 reverseResultStrides.reserve(N: resultShape.size());
2547 unsigned shapeIndex = resultShape.size() - 1;
2548 for (auto it : llvm::reverse(C: llvm::zip(t&: reassociation, u&: srcStrides))) {
2549 ReassociationIndices reassoc = std::get<0>(t&: it);
2550 int64_t currentStrideToExpand = std::get<1>(t&: it);
2551 for (unsigned idx = 0, e = reassoc.size(); idx < e; ++idx) {
2552 reverseResultStrides.push_back(Elt: currentStrideToExpand);
2553 currentStrideToExpand =
2554 (SaturatedInteger::wrap(v: currentStrideToExpand) *
2555 SaturatedInteger::wrap(v: resultShape[shapeIndex--]))
2556 .asInteger();
2557 }
2558 }
2559 auto resultStrides = llvm::to_vector<8>(Range: llvm::reverse(C&: reverseResultStrides));
2560 resultStrides.resize(N: resultShape.size(), NV: 1);
2561 return StridedLayoutAttr::get(context: srcType.getContext(), offset: srcOffset, strides: resultStrides);
2562}
2563
2564FailureOr<MemRefType> ExpandShapeOp::computeExpandedType(
2565 MemRefType srcType, ArrayRef<int64_t> resultShape,
2566 ArrayRef<ReassociationIndices> reassociation) {
2567 if (srcType.getLayout().isIdentity()) {
2568 // If the source is contiguous (i.e., no layout map specified), so is the
2569 // result.
2570 MemRefLayoutAttrInterface layout;
2571 return MemRefType::get(shape: resultShape, elementType: srcType.getElementType(), layout,
2572 memorySpace: srcType.getMemorySpace());
2573 }
2574
2575 // Source may not be contiguous. Compute the layout map.
2576 FailureOr<StridedLayoutAttr> computedLayout =
2577 computeExpandedLayoutMap(srcType, resultShape, reassociation);
2578 if (failed(Result: computedLayout))
2579 return failure();
2580 return MemRefType::get(shape: resultShape, elementType: srcType.getElementType(), layout: *computedLayout,
2581 memorySpace: srcType.getMemorySpace());
2582}
2583
2584FailureOr<SmallVector<OpFoldResult>>
2585ExpandShapeOp::inferOutputShape(OpBuilder &b, Location loc,
2586 MemRefType expandedType,
2587 ArrayRef<ReassociationIndices> reassociation,
2588 ArrayRef<OpFoldResult> inputShape) {
2589 std::optional<SmallVector<OpFoldResult>> outputShape =
2590 inferExpandShapeOutputShape(b, loc, expandedType, reassociation,
2591 inputShape);
2592 if (!outputShape)
2593 return failure();
2594 return *outputShape;
2595}
2596
2597void ExpandShapeOp::build(OpBuilder &builder, OperationState &result,
2598 Type resultType, Value src,
2599 ArrayRef<ReassociationIndices> reassociation,
2600 ArrayRef<OpFoldResult> outputShape) {
2601 auto [staticOutputShape, dynamicOutputShape] =
2602 decomposeMixedValues(mixedValues: SmallVector<OpFoldResult>(outputShape));
2603 build(odsBuilder&: builder, odsState&: result, result: llvm::cast<MemRefType>(Val&: resultType), src,
2604 reassociation: getReassociationIndicesAttribute(b&: builder, reassociation),
2605 output_shape: dynamicOutputShape, static_output_shape: staticOutputShape);
2606}
2607
2608void ExpandShapeOp::build(OpBuilder &builder, OperationState &result,
2609 Type resultType, Value src,
2610 ArrayRef<ReassociationIndices> reassociation) {
2611 SmallVector<OpFoldResult> inputShape =
2612 getMixedSizes(builder, loc: result.location, value: src);
2613 MemRefType memrefResultTy = llvm::cast<MemRefType>(Val&: resultType);
2614 FailureOr<SmallVector<OpFoldResult>> outputShape = inferOutputShape(
2615 b&: builder, loc: result.location, expandedType: memrefResultTy, reassociation, inputShape);
2616 // Failure of this assertion usually indicates presence of multiple
2617 // dynamic dimensions in the same reassociation group.
2618 assert(succeeded(outputShape) && "unable to infer output shape");
2619 build(builder, result, resultType: memrefResultTy, src, reassociation, outputShape: *outputShape);
2620}
2621
2622void ExpandShapeOp::build(OpBuilder &builder, OperationState &result,
2623 ArrayRef<int64_t> resultShape, Value src,
2624 ArrayRef<ReassociationIndices> reassociation) {
2625 // Only ranked memref source values are supported.
2626 auto srcType = llvm::cast<MemRefType>(Val: src.getType());
2627 FailureOr<MemRefType> resultType =
2628 ExpandShapeOp::computeExpandedType(srcType, resultShape, reassociation);
2629 // Failure of this assertion usually indicates a problem with the source
2630 // type, e.g., could not get strides/offset.
2631 assert(succeeded(resultType) && "could not compute layout");
2632 build(builder, result, resultType: *resultType, src, reassociation);
2633}
2634
2635void ExpandShapeOp::build(OpBuilder &builder, OperationState &result,
2636 ArrayRef<int64_t> resultShape, Value src,
2637 ArrayRef<ReassociationIndices> reassociation,
2638 ArrayRef<OpFoldResult> outputShape) {
2639 // Only ranked memref source values are supported.
2640 auto srcType = llvm::cast<MemRefType>(Val: src.getType());
2641 FailureOr<MemRefType> resultType =
2642 ExpandShapeOp::computeExpandedType(srcType, resultShape, reassociation);
2643 // Failure of this assertion usually indicates a problem with the source
2644 // type, e.g., could not get strides/offset.
2645 assert(succeeded(resultType) && "could not compute layout");
2646 build(builder, result, resultType: *resultType, src, reassociation, outputShape);
2647}
2648
2649LogicalResult ExpandShapeOp::verify() {
2650 if (failed(Result: verifyReassociationIndicesNotEmpty(op: *this)))
2651 return failure();
2652
2653 MemRefType srcType = getSrcType();
2654 MemRefType resultType = getResultType();
2655
2656 if (srcType.getRank() > resultType.getRank()) {
2657 auto r0 = srcType.getRank();
2658 auto r1 = resultType.getRank();
2659 return emitOpError(message: "has source rank ")
2660 << r0 << " and result rank " << r1 << ". This is not an expansion ("
2661 << r0 << " > " << r1 << ").";
2662 }
2663
2664 // Verify result shape.
2665 if (failed(Result: verifyCollapsedShape(op: getOperation(), collapsedShape: srcType.getShape(),
2666 expandedShape: resultType.getShape(),
2667 reassociation: getReassociationIndices(),
2668 /*allowMultipleDynamicDimsPerGroup=*/true)))
2669 return failure();
2670
2671 // Compute expected result type (including layout map).
2672 FailureOr<MemRefType> expectedResultType = ExpandShapeOp::computeExpandedType(
2673 srcType, resultShape: resultType.getShape(), reassociation: getReassociationIndices());
2674 if (failed(Result: expectedResultType))
2675 return emitOpError(message: "invalid source layout map");
2676
2677 // Check actual result type.
2678 if (*expectedResultType != resultType)
2679 return emitOpError(message: "expected expanded type to be ")
2680 << *expectedResultType << " but found " << resultType;
2681
2682 if ((int64_t)getStaticOutputShape().size() != resultType.getRank())
2683 return emitOpError(message: "expected number of static shape bounds to be equal to "
2684 "the output rank (")
2685 << resultType.getRank() << ") but found "
2686 << getStaticOutputShape().size() << " inputs instead";
2687
2688 if ((int64_t)getOutputShape().size() !=
2689 llvm::count(Range: getStaticOutputShape(), Element: ShapedType::kDynamic))
2690 return emitOpError(message: "mismatch in dynamic dims in output_shape and "
2691 "static_output_shape: static_output_shape has ")
2692 << llvm::count(Range: getStaticOutputShape(), Element: ShapedType::kDynamic)
2693 << " dynamic dims while output_shape has " << getOutputShape().size()
2694 << " values";
2695
2696 // Verify that the number of dynamic dims in output_shape matches the number
2697 // of dynamic dims in the result type.
2698 if (failed(Result: verifyDynamicDimensionCount(op: getOperation(), type: resultType,
2699 dynamicSizes: getOutputShape())))
2700 return failure();
2701
2702 // Verify if provided output shapes are in agreement with output type.
2703 DenseI64ArrayAttr staticOutputShapes = getStaticOutputShapeAttr();
2704 ArrayRef<int64_t> resShape = getResult().getType().getShape();
2705 for (auto [pos, shape] : llvm::enumerate(First&: resShape)) {
2706 if (ShapedType::isStatic(dValue: shape) && shape != staticOutputShapes[pos]) {
2707 return emitOpError(message: "invalid output shape provided at pos ") << pos;
2708 }
2709 }
2710
2711 return success();
2712}
2713
2714struct ExpandShapeOpMemRefCastFolder : public OpRewritePattern<ExpandShapeOp> {
2715public:
2716 using OpRewritePattern<ExpandShapeOp>::OpRewritePattern;
2717
2718 LogicalResult matchAndRewrite(ExpandShapeOp op,
2719 PatternRewriter &rewriter) const override {
2720 auto cast = op.getSrc().getDefiningOp<CastOp>();
2721 if (!cast)
2722 return failure();
2723
2724 if (!CastOp::canFoldIntoConsumerOp(castOp: cast))
2725 return failure();
2726
2727 SmallVector<OpFoldResult> originalOutputShape = op.getMixedOutputShape();
2728 SmallVector<OpFoldResult> newOutputShape = originalOutputShape;
2729 SmallVector<int64_t> newOutputShapeSizes;
2730
2731 // Convert output shape dims from dynamic to static where possible.
2732 for (auto [dimIdx, dimSize] : enumerate(First&: originalOutputShape)) {
2733 std::optional<int64_t> sizeOpt = getConstantIntValue(ofr: dimSize);
2734 if (!sizeOpt.has_value()) {
2735 newOutputShapeSizes.push_back(Elt: ShapedType::kDynamic);
2736 continue;
2737 }
2738
2739 newOutputShapeSizes.push_back(Elt: sizeOpt.value());
2740 newOutputShape[dimIdx] = rewriter.getIndexAttr(value: sizeOpt.value());
2741 }
2742
2743 Value castSource = cast.getSource();
2744 auto castSourceType = llvm::cast<MemRefType>(Val: castSource.getType());
2745 SmallVector<ReassociationIndices> reassociationIndices =
2746 op.getReassociationIndices();
2747 for (auto [idx, group] : llvm::enumerate(First&: reassociationIndices)) {
2748 auto newOutputShapeSizesSlice =
2749 ArrayRef(newOutputShapeSizes).slice(N: group.front(), M: group.size());
2750 bool newOutputDynamic =
2751 llvm::is_contained(Range&: newOutputShapeSizesSlice, Element: ShapedType::kDynamic);
2752 if (castSourceType.isDynamicDim(idx) != newOutputDynamic)
2753 return rewriter.notifyMatchFailure(
2754 arg&: op, msg: "folding cast will result in changing dynamicity in "
2755 "reassociation group");
2756 }
2757
2758 FailureOr<MemRefType> newResultTypeOrFailure =
2759 ExpandShapeOp::computeExpandedType(srcType: castSourceType, resultShape: newOutputShapeSizes,
2760 reassociation: reassociationIndices);
2761
2762 if (failed(Result: newResultTypeOrFailure))
2763 return rewriter.notifyMatchFailure(
2764 arg&: op, msg: "could not compute new expanded type after folding cast");
2765
2766 if (*newResultTypeOrFailure == op.getResultType()) {
2767 rewriter.modifyOpInPlace(
2768 root: op, callable: [&]() { op.getSrcMutable().assign(value: castSource); });
2769 } else {
2770 Value newOp = ExpandShapeOp::create(builder&: rewriter, location: op->getLoc(),
2771 resultType: *newResultTypeOrFailure, src: castSource,
2772 reassociation: reassociationIndices, outputShape: newOutputShape);
2773 rewriter.replaceOpWithNewOp<CastOp>(op, args: op.getType(), args&: newOp);
2774 }
2775 return success();
2776 }
2777};
2778
2779void ExpandShapeOp::getCanonicalizationPatterns(RewritePatternSet &results,
2780 MLIRContext *context) {
2781 results.add<
2782 ComposeReassociativeReshapeOps<ExpandShapeOp, ReshapeOpKind::kExpand>,
2783 ComposeExpandOfCollapseOp<ExpandShapeOp, CollapseShapeOp, CastOp>,
2784 ExpandShapeOpMemRefCastFolder>(arg&: context);
2785}
2786
2787FailureOr<std::optional<SmallVector<Value>>>
2788ExpandShapeOp::bubbleDownCasts(OpBuilder &builder) {
2789 return bubbleDownCastsPassthroughOpImpl(op: *this, builder, src&: getSrcMutable());
2790}
2791
2792/// Compute the layout map after collapsing a given source MemRef type with the
2793/// specified reassociation indices.
2794///
2795/// Note: All collapsed dims in a reassociation group must be contiguous. It is
2796/// not possible to check this by inspecting a MemRefType in the general case.
2797/// If non-contiguity cannot be checked statically, the collapse is assumed to
2798/// be valid (and thus accepted by this function) unless `strict = true`.
2799static FailureOr<StridedLayoutAttr>
2800computeCollapsedLayoutMap(MemRefType srcType,
2801 ArrayRef<ReassociationIndices> reassociation,
2802 bool strict = false) {
2803 int64_t srcOffset;
2804 SmallVector<int64_t> srcStrides;
2805 auto srcShape = srcType.getShape();
2806 if (failed(Result: srcType.getStridesAndOffset(strides&: srcStrides, offset&: srcOffset)))
2807 return failure();
2808
2809 // The result stride of a reassociation group is the stride of the last entry
2810 // of the reassociation. (TODO: Should be the minimum stride in the
2811 // reassociation because strides are not necessarily sorted. E.g., when using
2812 // memref.transpose.) Dimensions of size 1 should be skipped, because their
2813 // strides are meaningless and could have any arbitrary value.
2814 SmallVector<int64_t> resultStrides;
2815 resultStrides.reserve(N: reassociation.size());
2816 for (const ReassociationIndices &reassoc : reassociation) {
2817 ArrayRef<int64_t> ref = llvm::ArrayRef(reassoc);
2818 while (srcShape[ref.back()] == 1 && ref.size() > 1)
2819 ref = ref.drop_back();
2820 if (ShapedType::isStatic(dValue: srcShape[ref.back()]) || ref.size() == 1) {
2821 resultStrides.push_back(Elt: srcStrides[ref.back()]);
2822 } else {
2823 // Dynamically-sized dims may turn out to be dims of size 1 at runtime, so
2824 // the corresponding stride may have to be skipped. (See above comment.)
2825 // Therefore, the result stride cannot be statically determined and must
2826 // be dynamic.
2827 resultStrides.push_back(Elt: ShapedType::kDynamic);
2828 }
2829 }
2830
2831 // Validate that each reassociation group is contiguous.
2832 unsigned resultStrideIndex = resultStrides.size() - 1;
2833 for (const ReassociationIndices &reassoc : llvm::reverse(C&: reassociation)) {
2834 auto trailingReassocs = ArrayRef<int64_t>(reassoc).drop_front();
2835 auto stride = SaturatedInteger::wrap(v: resultStrides[resultStrideIndex--]);
2836 for (int64_t idx : llvm::reverse(C&: trailingReassocs)) {
2837 stride = stride * SaturatedInteger::wrap(v: srcShape[idx]);
2838
2839 // Dimensions of size 1 should be skipped, because their strides are
2840 // meaningless and could have any arbitrary value.
2841 if (srcShape[idx - 1] == 1)
2842 continue;
2843
2844 // Both source and result stride must have the same static value. In that
2845 // case, we can be sure, that the dimensions are collapsible (because they
2846 // are contiguous).
2847 // If `strict = false` (default during op verification), we accept cases
2848 // where one or both strides are dynamic. This is best effort: We reject
2849 // ops where obviously non-contiguous dims are collapsed, but accept ops
2850 // where we cannot be sure statically. Such ops may fail at runtime. See
2851 // the op documentation for details.
2852 auto srcStride = SaturatedInteger::wrap(v: srcStrides[idx - 1]);
2853 if (strict && (stride.saturated || srcStride.saturated))
2854 return failure();
2855
2856 if (!stride.saturated && !srcStride.saturated && stride != srcStride)
2857 return failure();
2858 }
2859 }
2860 return StridedLayoutAttr::get(context: srcType.getContext(), offset: srcOffset, strides: resultStrides);
2861}
2862
2863bool CollapseShapeOp::isGuaranteedCollapsible(
2864 MemRefType srcType, ArrayRef<ReassociationIndices> reassociation) {
2865 // MemRefs with identity layout are always collapsible.
2866 if (srcType.getLayout().isIdentity())
2867 return true;
2868
2869 return succeeded(Result: computeCollapsedLayoutMap(srcType, reassociation,
2870 /*strict=*/true));
2871}
2872
2873MemRefType CollapseShapeOp::computeCollapsedType(
2874 MemRefType srcType, ArrayRef<ReassociationIndices> reassociation) {
2875 SmallVector<int64_t> resultShape;
2876 resultShape.reserve(N: reassociation.size());
2877 for (const ReassociationIndices &group : reassociation) {
2878 auto groupSize = SaturatedInteger::wrap(v: 1);
2879 for (int64_t srcDim : group)
2880 groupSize =
2881 groupSize * SaturatedInteger::wrap(v: srcType.getDimSize(idx: srcDim));
2882 resultShape.push_back(Elt: groupSize.asInteger());
2883 }
2884
2885 if (srcType.getLayout().isIdentity()) {
2886 // If the source is contiguous (i.e., no layout map specified), so is the
2887 // result.
2888 MemRefLayoutAttrInterface layout;
2889 return MemRefType::get(shape: resultShape, elementType: srcType.getElementType(), layout,
2890 memorySpace: srcType.getMemorySpace());
2891 }
2892
2893 // Source may not be fully contiguous. Compute the layout map.
2894 // Note: Dimensions that are collapsed into a single dim are assumed to be
2895 // contiguous.
2896 FailureOr<StridedLayoutAttr> computedLayout =
2897 computeCollapsedLayoutMap(srcType, reassociation);
2898 assert(succeeded(computedLayout) &&
2899 "invalid source layout map or collapsing non-contiguous dims");
2900 return MemRefType::get(shape: resultShape, elementType: srcType.getElementType(), layout: *computedLayout,
2901 memorySpace: srcType.getMemorySpace());
2902}
2903
2904void CollapseShapeOp::build(OpBuilder &b, OperationState &result, Value src,
2905 ArrayRef<ReassociationIndices> reassociation,
2906 ArrayRef<NamedAttribute> attrs) {
2907 auto srcType = llvm::cast<MemRefType>(Val: src.getType());
2908 MemRefType resultType =
2909 CollapseShapeOp::computeCollapsedType(srcType, reassociation);
2910 buildPropertiesAndDiscardableAttributes(odsState&: result, attributes: attrs);
2911 result.getOrAddProperties<Properties>().reassociation =
2912 getReassociationIndicesAttribute(b, reassociation);
2913 result.addOperands(newOperands: src);
2914 result.addTypes(newTypes: resultType);
2915}
2916
2917LogicalResult CollapseShapeOp::verify() {
2918 if (failed(Result: verifyReassociationIndicesNotEmpty(op: *this)))
2919 return failure();
2920
2921 MemRefType srcType = getSrcType();
2922 MemRefType resultType = getResultType();
2923
2924 if (srcType.getRank() < resultType.getRank()) {
2925 auto r0 = srcType.getRank();
2926 auto r1 = resultType.getRank();
2927 return emitOpError(message: "has source rank ")
2928 << r0 << " and result rank " << r1 << ". This is not a collapse ("
2929 << r0 << " < " << r1 << ").";
2930 }
2931
2932 // Verify result shape.
2933 if (failed(Result: verifyCollapsedShape(op: getOperation(), collapsedShape: resultType.getShape(),
2934 expandedShape: srcType.getShape(), reassociation: getReassociationIndices(),
2935 /*allowMultipleDynamicDimsPerGroup=*/true)))
2936 return failure();
2937
2938 // Compute expected result type (including layout map).
2939 MemRefType expectedResultType;
2940 if (srcType.getLayout().isIdentity()) {
2941 // If the source is contiguous (i.e., no layout map specified), so is the
2942 // result.
2943 MemRefLayoutAttrInterface layout;
2944 expectedResultType =
2945 MemRefType::get(shape: resultType.getShape(), elementType: srcType.getElementType(), layout,
2946 memorySpace: srcType.getMemorySpace());
2947 } else {
2948 // Source may not be fully contiguous. Compute the layout map.
2949 // Note: Dimensions that are collapsed into a single dim are assumed to be
2950 // contiguous.
2951 FailureOr<StridedLayoutAttr> computedLayout =
2952 computeCollapsedLayoutMap(srcType, reassociation: getReassociationIndices());
2953 if (failed(Result: computedLayout))
2954 return emitOpError(
2955 message: "invalid source layout map or collapsing non-contiguous dims");
2956 expectedResultType =
2957 MemRefType::get(shape: resultType.getShape(), elementType: srcType.getElementType(),
2958 layout: *computedLayout, memorySpace: srcType.getMemorySpace());
2959 }
2960
2961 if (expectedResultType != resultType)
2962 return emitOpError(message: "expected collapsed type to be ")
2963 << expectedResultType << " but found " << resultType;
2964
2965 return success();
2966}
2967
2968struct CollapseShapeOpMemRefCastFolder
2969 : public OpRewritePattern<CollapseShapeOp> {
2970public:
2971 using OpRewritePattern<CollapseShapeOp>::OpRewritePattern;
2972
2973 LogicalResult matchAndRewrite(CollapseShapeOp op,
2974 PatternRewriter &rewriter) const override {
2975 auto cast = op.getOperand().getDefiningOp<CastOp>();
2976 if (!cast)
2977 return failure();
2978
2979 if (!CastOp::canFoldIntoConsumerOp(castOp: cast))
2980 return failure();
2981
2982 Type newResultType = CollapseShapeOp::computeCollapsedType(
2983 srcType: llvm::cast<MemRefType>(Val: cast.getOperand().getType()),
2984 reassociation: op.getReassociationIndices());
2985
2986 if (newResultType == op.getResultType()) {
2987 rewriter.modifyOpInPlace(
2988 root: op, callable: [&]() { op.getSrcMutable().assign(value: cast.getSource()); });
2989 } else {
2990 Value newOp =
2991 CollapseShapeOp::create(builder&: rewriter, location: op->getLoc(), src: cast.getSource(),
2992 reassociation: op.getReassociationIndices());
2993 rewriter.replaceOpWithNewOp<CastOp>(op, args: op.getType(), args&: newOp);
2994 }
2995 return success();
2996 }
2997};
2998
2999void CollapseShapeOp::getCanonicalizationPatterns(RewritePatternSet &results,
3000 MLIRContext *context) {
3001 results.add<
3002 ComposeReassociativeReshapeOps<CollapseShapeOp, ReshapeOpKind::kCollapse>,
3003 ComposeCollapseOfExpandOp<CollapseShapeOp, ExpandShapeOp, CastOp,
3004 memref::DimOp, MemRefType>,
3005 CollapseShapeOpMemRefCastFolder>(arg&: context);
3006}
3007
3008OpFoldResult ExpandShapeOp::fold(FoldAdaptor adaptor) {
3009 return foldReshapeOp<ExpandShapeOp, CollapseShapeOp>(reshapeOp: *this,
3010 operands: adaptor.getOperands());
3011}
3012
3013OpFoldResult CollapseShapeOp::fold(FoldAdaptor adaptor) {
3014 return foldReshapeOp<CollapseShapeOp, ExpandShapeOp>(reshapeOp: *this,
3015 operands: adaptor.getOperands());
3016}
3017
3018FailureOr<std::optional<SmallVector<Value>>>
3019CollapseShapeOp::bubbleDownCasts(OpBuilder &builder) {
3020 return bubbleDownCastsPassthroughOpImpl(op: *this, builder, src&: getSrcMutable());
3021}
3022
3023//===----------------------------------------------------------------------===//
3024// ReshapeOp
3025//===----------------------------------------------------------------------===//
3026
3027void ReshapeOp::getAsmResultNames(
3028 function_ref<void(Value, StringRef)> setNameFn) {
3029 setNameFn(getResult(), "reshape");
3030}
3031
3032LogicalResult ReshapeOp::verify() {
3033 Type operandType = getSource().getType();
3034 Type resultType = getResult().getType();
3035
3036 Type operandElementType =
3037 llvm::cast<ShapedType>(Val&: operandType).getElementType();
3038 Type resultElementType = llvm::cast<ShapedType>(Val&: resultType).getElementType();
3039 if (operandElementType != resultElementType)
3040 return emitOpError(message: "element types of source and destination memref "
3041 "types should be the same");
3042
3043 if (auto operandMemRefType = llvm::dyn_cast<MemRefType>(Val&: operandType))
3044 if (!operandMemRefType.getLayout().isIdentity())
3045 return emitOpError(message: "source memref type should have identity affine map");
3046
3047 int64_t shapeSize =
3048 llvm::cast<MemRefType>(Val: getShape().getType()).getDimSize(idx: 0);
3049 auto resultMemRefType = llvm::dyn_cast<MemRefType>(Val&: resultType);
3050 if (resultMemRefType) {
3051 if (!resultMemRefType.getLayout().isIdentity())
3052 return emitOpError(message: "result memref type should have identity affine map");
3053 if (ShapedType::isDynamic(dValue: shapeSize))
3054 return emitOpError(message: "cannot use shape operand with dynamic length to "
3055 "reshape to statically-ranked memref type");
3056 if (shapeSize != resultMemRefType.getRank())
3057 return emitOpError(
3058 message: "length of shape operand differs from the result's memref rank");
3059 } else if (ShapedType::isStatic(dValue: shapeSize)) {
3060 return emitOpError(message: "cannot use shape operand with static length to "
3061 "reshape to unranked memref type");
3062 }
3063 return success();
3064}
3065
3066FailureOr<std::optional<SmallVector<Value>>>
3067ReshapeOp::bubbleDownCasts(OpBuilder &builder) {
3068 return bubbleDownCastsPassthroughOpImpl(op: *this, builder, src&: getSourceMutable());
3069}
3070
3071//===----------------------------------------------------------------------===//
3072// StoreOp
3073//===----------------------------------------------------------------------===//
3074
3075LogicalResult StoreOp::fold(FoldAdaptor adaptor,
3076 SmallVectorImpl<OpFoldResult> &results) {
3077 /// store(memrefcast) -> store
3078 return foldMemRefCast(op: *this, inner: getValueToStore());
3079}
3080
3081TypedValue<MemRefType> StoreOp::getAccessedMemref() { return getMemref(); }
3082
3083std::optional<SmallVector<Value>>
3084StoreOp::updateMemrefAndIndices(RewriterBase &rewriter, Value newMemref,
3085 ValueRange newIndices) {
3086 rewriter.modifyOpInPlace(root: *this, callable: [&]() {
3087 getMemrefMutable().assign(value: newMemref);
3088 getIndicesMutable().assign(values: newIndices);
3089 });
3090 return std::nullopt;
3091}
3092
3093FailureOr<std::optional<SmallVector<Value>>>
3094StoreOp::bubbleDownCasts(OpBuilder &builder) {
3095 return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getMemrefMutable(),
3096 results: ValueRange());
3097}
3098
3099//===----------------------------------------------------------------------===//
3100// SubViewOp
3101//===----------------------------------------------------------------------===//
3102
3103void SubViewOp::getAsmResultNames(
3104 function_ref<void(Value, StringRef)> setNameFn) {
3105 setNameFn(getResult(), "subview");
3106}
3107
3108/// A subview result type can be fully inferred from the source type and the
3109/// static representation of offsets, sizes and strides. Special sentinels
3110/// encode the dynamic case.
3111MemRefType SubViewOp::inferResultType(MemRefType sourceMemRefType,
3112 ArrayRef<int64_t> staticOffsets,
3113 ArrayRef<int64_t> staticSizes,
3114 ArrayRef<int64_t> staticStrides) {
3115 unsigned rank = sourceMemRefType.getRank();
3116 (void)rank;
3117 assert(staticOffsets.size() == rank && "staticOffsets length mismatch");
3118 assert(staticSizes.size() == rank && "staticSizes length mismatch");
3119 assert(staticStrides.size() == rank && "staticStrides length mismatch");
3120
3121 // Extract source offset and strides.
3122 auto [sourceStrides, sourceOffset] = sourceMemRefType.getStridesAndOffset();
3123
3124 // Compute target offset whose value is:
3125 // `sourceOffset + sum_i(staticOffset_i * sourceStrides_i)`.
3126 int64_t targetOffset = sourceOffset;
3127 for (auto it : llvm::zip(t&: staticOffsets, u&: sourceStrides)) {
3128 auto staticOffset = std::get<0>(t&: it), sourceStride = std::get<1>(t&: it);
3129 targetOffset = (SaturatedInteger::wrap(v: targetOffset) +
3130 SaturatedInteger::wrap(v: staticOffset) *
3131 SaturatedInteger::wrap(v: sourceStride))
3132 .asInteger();
3133 }
3134
3135 // Compute target stride whose value is:
3136 // `sourceStrides_i * staticStrides_i`.
3137 SmallVector<int64_t, 4> targetStrides;
3138 targetStrides.reserve(N: staticOffsets.size());
3139 for (auto it : llvm::zip(t&: sourceStrides, u&: staticStrides)) {
3140 auto sourceStride = std::get<0>(t&: it), staticStride = std::get<1>(t&: it);
3141 targetStrides.push_back(Elt: (SaturatedInteger::wrap(v: sourceStride) *
3142 SaturatedInteger::wrap(v: staticStride))
3143 .asInteger());
3144 }
3145
3146 // The type is now known.
3147 return MemRefType::get(shape: staticSizes, elementType: sourceMemRefType.getElementType(),
3148 layout: StridedLayoutAttr::get(context: sourceMemRefType.getContext(),
3149 offset: targetOffset, strides: targetStrides),
3150 memorySpace: sourceMemRefType.getMemorySpace());
3151}
3152
3153MemRefType SubViewOp::inferResultType(MemRefType sourceMemRefType,
3154 ArrayRef<OpFoldResult> offsets,
3155 ArrayRef<OpFoldResult> sizes,
3156 ArrayRef<OpFoldResult> strides) {
3157 SmallVector<int64_t> staticOffsets, staticSizes, staticStrides;
3158 SmallVector<Value> dynamicOffsets, dynamicSizes, dynamicStrides;
3159 dispatchIndexOpFoldResults(ofrs: offsets, dynamicVec&: dynamicOffsets, staticVec&: staticOffsets);
3160 dispatchIndexOpFoldResults(ofrs: sizes, dynamicVec&: dynamicSizes, staticVec&: staticSizes);
3161 dispatchIndexOpFoldResults(ofrs: strides, dynamicVec&: dynamicStrides, staticVec&: staticStrides);
3162 if (!hasValidSizesOffsets(sizesOrOffsets: staticOffsets))
3163 return {};
3164 if (!hasValidSizesOffsets(sizesOrOffsets: staticSizes))
3165 return {};
3166 if (!hasValidStrides(strides: staticStrides))
3167 return {};
3168 return SubViewOp::inferResultType(sourceMemRefType, staticOffsets,
3169 staticSizes, staticStrides);
3170}
3171
3172MemRefType SubViewOp::inferRankReducedResultType(
3173 ArrayRef<int64_t> resultShape, MemRefType sourceRankedTensorType,
3174 ArrayRef<int64_t> offsets, ArrayRef<int64_t> sizes,
3175 ArrayRef<int64_t> strides) {
3176 MemRefType inferredType =
3177 inferResultType(sourceMemRefType: sourceRankedTensorType, staticOffsets: offsets, staticSizes: sizes, staticStrides: strides);
3178 assert(inferredType.getRank() >= static_cast<int64_t>(resultShape.size()) &&
3179 "expected ");
3180 if (inferredType.getRank() == static_cast<int64_t>(resultShape.size()))
3181 return inferredType;
3182
3183 // Compute which dimensions are dropped.
3184 std::optional<llvm::SmallDenseSet<unsigned>> dimsToProject =
3185 computeRankReductionMask(originalShape: inferredType.getShape(), reducedShape: resultShape);
3186 assert(dimsToProject.has_value() && "invalid rank reduction");
3187
3188 // Compute the layout and result type.
3189 auto inferredLayout = llvm::cast<StridedLayoutAttr>(Val: inferredType.getLayout());
3190 SmallVector<int64_t> rankReducedStrides;
3191 rankReducedStrides.reserve(N: resultShape.size());
3192 for (auto [idx, value] : llvm::enumerate(First: inferredLayout.getStrides())) {
3193 if (!dimsToProject->contains(V: idx))
3194 rankReducedStrides.push_back(Elt: value);
3195 }
3196 return MemRefType::get(shape: resultShape, elementType: inferredType.getElementType(),
3197 layout: StridedLayoutAttr::get(context: inferredLayout.getContext(),
3198 offset: inferredLayout.getOffset(),
3199 strides: rankReducedStrides),
3200 memorySpace: inferredType.getMemorySpace());
3201}
3202
3203MemRefType SubViewOp::inferRankReducedResultType(
3204 ArrayRef<int64_t> resultShape, MemRefType sourceRankedTensorType,
3205 ArrayRef<OpFoldResult> offsets, ArrayRef<OpFoldResult> sizes,
3206 ArrayRef<OpFoldResult> strides) {
3207 SmallVector<int64_t> staticOffsets, staticSizes, staticStrides;
3208 SmallVector<Value> dynamicOffsets, dynamicSizes, dynamicStrides;
3209 dispatchIndexOpFoldResults(ofrs: offsets, dynamicVec&: dynamicOffsets, staticVec&: staticOffsets);
3210 dispatchIndexOpFoldResults(ofrs: sizes, dynamicVec&: dynamicSizes, staticVec&: staticSizes);
3211 dispatchIndexOpFoldResults(ofrs: strides, dynamicVec&: dynamicStrides, staticVec&: staticStrides);
3212 return SubViewOp::inferRankReducedResultType(
3213 resultShape, sourceRankedTensorType, offsets: staticOffsets, sizes: staticSizes,
3214 strides: staticStrides);
3215}
3216
3217// Build a SubViewOp with mixed static and dynamic entries and custom result
3218// type. If the type passed is nullptr, it is inferred.
3219void SubViewOp::build(OpBuilder &b, OperationState &result,
3220 MemRefType resultType, Value source,
3221 ArrayRef<OpFoldResult> offsets,
3222 ArrayRef<OpFoldResult> sizes,
3223 ArrayRef<OpFoldResult> strides,
3224 ArrayRef<NamedAttribute> attrs) {
3225 SmallVector<int64_t> staticOffsets, staticSizes, staticStrides;
3226 SmallVector<Value> dynamicOffsets, dynamicSizes, dynamicStrides;
3227 dispatchIndexOpFoldResults(ofrs: offsets, dynamicVec&: dynamicOffsets, staticVec&: staticOffsets);
3228 dispatchIndexOpFoldResults(ofrs: sizes, dynamicVec&: dynamicSizes, staticVec&: staticSizes);
3229 dispatchIndexOpFoldResults(ofrs: strides, dynamicVec&: dynamicStrides, staticVec&: staticStrides);
3230 auto sourceMemRefType = llvm::cast<MemRefType>(Val: source.getType());
3231 // Structuring implementation this way avoids duplication between builders.
3232 if (!resultType) {
3233 resultType = SubViewOp::inferResultType(sourceMemRefType, staticOffsets,
3234 staticSizes, staticStrides);
3235 }
3236 result.addAttributes(newAttributes: attrs);
3237 build(odsBuilder&: b, odsState&: result, result: resultType, source, offsets: dynamicOffsets, sizes: dynamicSizes,
3238 strides: dynamicStrides, static_offsets: b.getDenseI64ArrayAttr(values: staticOffsets),
3239 static_sizes: b.getDenseI64ArrayAttr(values: staticSizes),
3240 static_strides: b.getDenseI64ArrayAttr(values: staticStrides));
3241}
3242
3243// Build a SubViewOp with mixed static and dynamic entries and inferred result
3244// type.
3245void SubViewOp::build(OpBuilder &b, OperationState &result, Value source,
3246 ArrayRef<OpFoldResult> offsets,
3247 ArrayRef<OpFoldResult> sizes,
3248 ArrayRef<OpFoldResult> strides,
3249 ArrayRef<NamedAttribute> attrs) {
3250 build(b, result, resultType: MemRefType(), source, offsets, sizes, strides, attrs);
3251}
3252
3253// Build a SubViewOp with static entries and inferred result type.
3254void SubViewOp::build(OpBuilder &b, OperationState &result, Value source,
3255 ArrayRef<int64_t> offsets, ArrayRef<int64_t> sizes,
3256 ArrayRef<int64_t> strides,
3257 ArrayRef<NamedAttribute> attrs) {
3258 SmallVector<OpFoldResult> offsetValues =
3259 llvm::map_to_vector<4>(C&: offsets, F: [&](int64_t v) -> OpFoldResult {
3260 return b.getI64IntegerAttr(value: v);
3261 });
3262 SmallVector<OpFoldResult> sizeValues = llvm::map_to_vector<4>(
3263 C&: sizes, F: [&](int64_t v) -> OpFoldResult { return b.getI64IntegerAttr(value: v); });
3264 SmallVector<OpFoldResult> strideValues =
3265 llvm::map_to_vector<4>(C&: strides, F: [&](int64_t v) -> OpFoldResult {
3266 return b.getI64IntegerAttr(value: v);
3267 });
3268 build(b, result, source, offsets: offsetValues, sizes: sizeValues, strides: strideValues, attrs);
3269}
3270
3271// Build a SubViewOp with dynamic entries and custom result type. If the
3272// type passed is nullptr, it is inferred.
3273void SubViewOp::build(OpBuilder &b, OperationState &result,
3274 MemRefType resultType, Value source,
3275 ArrayRef<int64_t> offsets, ArrayRef<int64_t> sizes,
3276 ArrayRef<int64_t> strides,
3277 ArrayRef<NamedAttribute> attrs) {
3278 SmallVector<OpFoldResult> offsetValues =
3279 llvm::map_to_vector<4>(C&: offsets, F: [&](int64_t v) -> OpFoldResult {
3280 return b.getI64IntegerAttr(value: v);
3281 });
3282 SmallVector<OpFoldResult> sizeValues = llvm::map_to_vector<4>(
3283 C&: sizes, F: [&](int64_t v) -> OpFoldResult { return b.getI64IntegerAttr(value: v); });
3284 SmallVector<OpFoldResult> strideValues =
3285 llvm::map_to_vector<4>(C&: strides, F: [&](int64_t v) -> OpFoldResult {
3286 return b.getI64IntegerAttr(value: v);
3287 });
3288 build(b, result, resultType, source, offsets: offsetValues, sizes: sizeValues, strides: strideValues,
3289 attrs);
3290}
3291
3292// Build a SubViewOp with dynamic entries and custom result type. If the type
3293// passed is nullptr, it is inferred.
3294void SubViewOp::build(OpBuilder &b, OperationState &result,
3295 MemRefType resultType, Value source, ValueRange offsets,
3296 ValueRange sizes, ValueRange strides,
3297 ArrayRef<NamedAttribute> attrs) {
3298 SmallVector<OpFoldResult> offsetValues = llvm::map_to_vector<4>(
3299 C&: offsets, F: [](Value v) -> OpFoldResult { return v; });
3300 SmallVector<OpFoldResult> sizeValues =
3301 llvm::map_to_vector<4>(C&: sizes, F: [](Value v) -> OpFoldResult { return v; });
3302 SmallVector<OpFoldResult> strideValues = llvm::map_to_vector<4>(
3303 C&: strides, F: [](Value v) -> OpFoldResult { return v; });
3304 build(b, result, resultType, source, offsets: offsetValues, sizes: sizeValues, strides: strideValues);
3305}
3306
3307// Build a SubViewOp with dynamic entries and inferred result type.
3308void SubViewOp::build(OpBuilder &b, OperationState &result, Value source,
3309 ValueRange offsets, ValueRange sizes, ValueRange strides,
3310 ArrayRef<NamedAttribute> attrs) {
3311 build(b, result, resultType: MemRefType(), source, offsets, sizes, strides, attrs);
3312}
3313
3314/// For ViewLikeOpInterface.
3315Value SubViewOp::getViewSource() { return getSource(); }
3316
3317/// Return true if `t1` and `t2` have equal offsets (both dynamic or of same
3318/// static value).
3319static bool haveCompatibleOffsets(MemRefType t1, MemRefType t2) {
3320 int64_t t1Offset, t2Offset;
3321 SmallVector<int64_t> t1Strides, t2Strides;
3322 auto res1 = t1.getStridesAndOffset(strides&: t1Strides, offset&: t1Offset);
3323 auto res2 = t2.getStridesAndOffset(strides&: t2Strides, offset&: t2Offset);
3324 return succeeded(Result: res1) && succeeded(Result: res2) && t1Offset == t2Offset;
3325}
3326
3327/// Return true if `t1` and `t2` have equal strides (both dynamic or of same
3328/// static value). Dimensions of `t1` may be dropped in `t2`; these must be
3329/// marked as dropped in `droppedDims`.
3330static bool haveCompatibleStrides(MemRefType t1, MemRefType t2,
3331 const llvm::SmallBitVector &droppedDims) {
3332 assert(size_t(t1.getRank()) == droppedDims.size() &&
3333 "incorrect number of bits");
3334 assert(size_t(t1.getRank() - t2.getRank()) == droppedDims.count() &&
3335 "incorrect number of dropped dims");
3336 int64_t t1Offset, t2Offset;
3337 SmallVector<int64_t> t1Strides, t2Strides;
3338 auto res1 = t1.getStridesAndOffset(strides&: t1Strides, offset&: t1Offset);
3339 auto res2 = t2.getStridesAndOffset(strides&: t2Strides, offset&: t2Offset);
3340 if (failed(Result: res1) || failed(Result: res2))
3341 return false;
3342 for (int64_t i = 0, j = 0, e = t1.getRank(); i < e; ++i) {
3343 if (droppedDims[i])
3344 continue;
3345 if (t1Strides[i] != t2Strides[j])
3346 return false;
3347 ++j;
3348 }
3349 return true;
3350}
3351
3352static LogicalResult produceSubViewErrorMsg(SliceVerificationResult result,
3353 SubViewOp op, Type expectedType) {
3354 auto memrefType = llvm::cast<ShapedType>(Val&: expectedType);
3355 switch (result) {
3356 case SliceVerificationResult::Success:
3357 return success();
3358 case SliceVerificationResult::RankTooLarge:
3359 return op->emitError(message: "expected result rank to be smaller or equal to ")
3360 << "the source rank, but got " << op.getType();
3361 case SliceVerificationResult::SizeMismatch:
3362 return op->emitError(message: "expected result type to be ")
3363 << expectedType
3364 << " or a rank-reduced version. (mismatch of result sizes), but got "
3365 << op.getType();
3366 case SliceVerificationResult::ElemTypeMismatch:
3367 return op->emitError(message: "expected result element type to be ")
3368 << memrefType.getElementType() << ", but got " << op.getType();
3369 case SliceVerificationResult::MemSpaceMismatch:
3370 return op->emitError(
3371 message: "expected result and source memory spaces to match, but got ")
3372 << op.getType();
3373 case SliceVerificationResult::LayoutMismatch:
3374 return op->emitError(message: "expected result type to be ")
3375 << expectedType
3376 << " or a rank-reduced version. (mismatch of result layout), but "
3377 "got "
3378 << op.getType();
3379 }
3380 llvm_unreachable("unexpected subview verification result");
3381}
3382
3383/// Verifier for SubViewOp.
3384LogicalResult SubViewOp::verify() {
3385 MemRefType baseType = getSourceType();
3386 MemRefType subViewType = getType();
3387 ArrayRef<int64_t> staticOffsets = getStaticOffsets();
3388 ArrayRef<int64_t> staticSizes = getStaticSizes();
3389 ArrayRef<int64_t> staticStrides = getStaticStrides();
3390
3391 // The base memref and the view memref should be in the same memory space.
3392 if (baseType.getMemorySpace() != subViewType.getMemorySpace())
3393 return emitError(message: "different memory spaces specified for base memref "
3394 "type ")
3395 << baseType << " and subview memref type " << subViewType;
3396
3397 // Verify that the base memref type has a strided layout map.
3398 if (!baseType.isStrided())
3399 return emitError(message: "base type ") << baseType << " is not strided";
3400
3401 // Compute the expected result type, assuming that there are no rank
3402 // reductions.
3403 MemRefType expectedType = SubViewOp::inferResultType(
3404 sourceMemRefType: baseType, staticOffsets, staticSizes, staticStrides);
3405
3406 // Verify all properties of a shaped type: rank, element type and dimension
3407 // sizes. This takes into account potential rank reductions.
3408 auto shapedTypeVerification = isRankReducedType(
3409 /*originalType=*/expectedType, /*candidateReducedType=*/subViewType);
3410 if (shapedTypeVerification != SliceVerificationResult::Success)
3411 return produceSubViewErrorMsg(result: shapedTypeVerification, op: *this, expectedType);
3412
3413 // Make sure that the memory space did not change.
3414 if (expectedType.getMemorySpace() != subViewType.getMemorySpace())
3415 return produceSubViewErrorMsg(result: SliceVerificationResult::MemSpaceMismatch,
3416 op: *this, expectedType);
3417
3418 // Verify the offset of the layout map.
3419 if (!haveCompatibleOffsets(t1: expectedType, t2: subViewType))
3420 return produceSubViewErrorMsg(result: SliceVerificationResult::LayoutMismatch,
3421 op: *this, expectedType);
3422
3423 // The only thing that's left to verify now are the strides. First, compute
3424 // the unused dimensions due to rank reductions. We have to look at sizes and
3425 // strides to decide which dimensions were dropped. This function also
3426 // partially verifies strides in case of rank reductions.
3427 auto unusedDims = computeMemRefRankReductionMask(originalType: expectedType, reducedType: subViewType,
3428 sizes: getMixedSizes());
3429 if (failed(Result: unusedDims))
3430 return produceSubViewErrorMsg(result: SliceVerificationResult::LayoutMismatch,
3431 op: *this, expectedType);
3432
3433 // Strides must match.
3434 if (!haveCompatibleStrides(t1: expectedType, t2: subViewType, droppedDims: *unusedDims))
3435 return produceSubViewErrorMsg(result: SliceVerificationResult::LayoutMismatch,
3436 op: *this, expectedType);
3437
3438 // Verify that offsets, sizes, strides do not run out-of-bounds with respect
3439 // to the base memref.
3440 SliceBoundsVerificationResult boundsResult =
3441 verifyInBoundsSlice(shape: baseType.getShape(), staticOffsets, staticSizes,
3442 staticStrides, /*generateErrorMessage=*/true);
3443 if (!boundsResult.isValid)
3444 return getOperation()->emitError(message: boundsResult.errorMessage);
3445
3446 return success();
3447}
3448
3449raw_ostream &mlir::operator<<(raw_ostream &os, const Range &range) {
3450 return os << "range " << range.offset << ":" << range.size << ":"
3451 << range.stride;
3452}
3453
3454/// Return the list of Range (i.e. offset, size, stride). Each Range
3455/// entry contains either the dynamic value or a ConstantIndexOp constructed
3456/// with `b` at location `loc`.
3457SmallVector<Range, 8> mlir::getOrCreateRanges(OffsetSizeAndStrideOpInterface op,
3458 OpBuilder &b, Location loc) {
3459 std::array<unsigned, 3> ranks = op.getArrayAttrMaxRanks();
3460 assert(ranks[0] == ranks[1] && "expected offset and sizes of equal ranks");
3461 assert(ranks[1] == ranks[2] && "expected sizes and strides of equal ranks");
3462 SmallVector<Range, 8> res;
3463 unsigned rank = ranks[0];
3464 res.reserve(N: rank);
3465 for (unsigned idx = 0; idx < rank; ++idx) {
3466 Value offset =
3467 op.isDynamicOffset(idx)
3468 ? op.getDynamicOffset(idx)
3469 : arith::ConstantIndexOp::create(builder&: b, location: loc, value: op.getStaticOffset(idx));
3470 Value size =
3471 op.isDynamicSize(idx)
3472 ? op.getDynamicSize(idx)
3473 : arith::ConstantIndexOp::create(builder&: b, location: loc, value: op.getStaticSize(idx));
3474 Value stride =
3475 op.isDynamicStride(idx)
3476 ? op.getDynamicStride(idx)
3477 : arith::ConstantIndexOp::create(builder&: b, location: loc, value: op.getStaticStride(idx));
3478 res.emplace_back(Args: Range{.offset: offset, .size: size, .stride: stride});
3479 }
3480 return res;
3481}
3482
3483/// Compute the canonical result type of a SubViewOp. Call `inferResultType`
3484/// to deduce the result type for the given `sourceType`. Additionally, reduce
3485/// the rank of the inferred result type if `currentResultType` is lower rank
3486/// than `currentSourceType`. Use this signature if `sourceType` is updated
3487/// together with the result type. In this case, it is important to compute
3488/// the dropped dimensions using `currentSourceType` whose strides align with
3489/// `currentResultType`.
3490static MemRefType getCanonicalSubViewResultType(
3491 MemRefType currentResultType, MemRefType currentSourceType,
3492 MemRefType sourceType, ArrayRef<OpFoldResult> mixedOffsets,
3493 ArrayRef<OpFoldResult> mixedSizes, ArrayRef<OpFoldResult> mixedStrides) {
3494 MemRefType nonRankReducedType = SubViewOp::inferResultType(
3495 sourceMemRefType: sourceType, offsets: mixedOffsets, sizes: mixedSizes, strides: mixedStrides);
3496 FailureOr<llvm::SmallBitVector> unusedDims = computeMemRefRankReductionMask(
3497 originalType: currentSourceType, reducedType: currentResultType, sizes: mixedSizes);
3498 if (failed(Result: unusedDims))
3499 return nullptr;
3500
3501 auto layout = llvm::cast<StridedLayoutAttr>(Val: nonRankReducedType.getLayout());
3502 SmallVector<int64_t> shape, strides;
3503 unsigned numDimsAfterReduction =
3504 nonRankReducedType.getRank() - unusedDims->count();
3505 shape.reserve(N: numDimsAfterReduction);
3506 strides.reserve(N: numDimsAfterReduction);
3507 for (const auto &[idx, size, stride] :
3508 llvm::zip(t: llvm::seq<unsigned>(Begin: 0, End: nonRankReducedType.getRank()),
3509 u: nonRankReducedType.getShape(), args: layout.getStrides())) {
3510 if (unusedDims->test(Idx: idx))
3511 continue;
3512 shape.push_back(Elt: size);
3513 strides.push_back(Elt: stride);
3514 }
3515
3516 return MemRefType::get(shape, elementType: nonRankReducedType.getElementType(),
3517 layout: StridedLayoutAttr::get(context: sourceType.getContext(),
3518 offset: layout.getOffset(), strides),
3519 memorySpace: nonRankReducedType.getMemorySpace());
3520}
3521
3522Value mlir::memref::createCanonicalRankReducingSubViewOp(
3523 OpBuilder &b, Location loc, Value memref, ArrayRef<int64_t> targetShape) {
3524 auto memrefType = llvm::cast<MemRefType>(Val: memref.getType());
3525 unsigned rank = memrefType.getRank();
3526 SmallVector<OpFoldResult> offsets(rank, b.getIndexAttr(value: 0));
3527 SmallVector<OpFoldResult> sizes = getMixedSizes(builder&: b, loc, value: memref);
3528 SmallVector<OpFoldResult> strides(rank, b.getIndexAttr(value: 1));
3529 MemRefType targetType = SubViewOp::inferRankReducedResultType(
3530 resultShape: targetShape, sourceRankedTensorType: memrefType, offsets, sizes, strides);
3531 return b.createOrFold<memref::SubViewOp>(location: loc, args&: targetType, args&: memref, args&: offsets,
3532 args&: sizes, args&: strides);
3533}
3534
3535FailureOr<Value> SubViewOp::rankReduceIfNeeded(OpBuilder &b, Location loc,
3536 Value value,
3537 ArrayRef<int64_t> desiredShape) {
3538 auto sourceMemrefType = llvm::dyn_cast<MemRefType>(Val: value.getType());
3539 assert(sourceMemrefType && "not a ranked memref type");
3540 auto sourceShape = sourceMemrefType.getShape();
3541 if (sourceShape.equals(RHS: desiredShape))
3542 return value;
3543 auto maybeRankReductionMask =
3544 mlir::computeRankReductionMask(originalShape: sourceShape, reducedShape: desiredShape);
3545 if (!maybeRankReductionMask)
3546 return failure();
3547 return createCanonicalRankReducingSubViewOp(b, loc, memref: value, targetShape: desiredShape);
3548}
3549
3550/// Helper method to check if a `subview` operation is trivially a no-op. This
3551/// is the case if the all offsets are zero, all strides are 1, and the source
3552/// shape is same as the size of the subview. In such cases, the subview can
3553/// be folded into its source.
3554static bool isTrivialSubViewOp(SubViewOp subViewOp) {
3555 if (subViewOp.getSourceType().getRank() != subViewOp.getType().getRank())
3556 return false;
3557
3558 auto mixedOffsets = subViewOp.getMixedOffsets();
3559 auto mixedSizes = subViewOp.getMixedSizes();
3560 auto mixedStrides = subViewOp.getMixedStrides();
3561
3562 // Check offsets are zero.
3563 if (llvm::any_of(Range&: mixedOffsets, P: [](OpFoldResult ofr) {
3564 std::optional<int64_t> intValue = getConstantIntValue(ofr);
3565 return !intValue || intValue.value() != 0;
3566 }))
3567 return false;
3568
3569 // Check strides are one.
3570 if (llvm::any_of(Range&: mixedStrides, P: [](OpFoldResult ofr) {
3571 std::optional<int64_t> intValue = getConstantIntValue(ofr);
3572 return !intValue || intValue.value() != 1;
3573 }))
3574 return false;
3575
3576 // Check all size values are static and matches the (static) source shape.
3577 ArrayRef<int64_t> sourceShape = subViewOp.getSourceType().getShape();
3578 for (const auto &size : llvm::enumerate(First&: mixedSizes)) {
3579 std::optional<int64_t> intValue = getConstantIntValue(ofr: size.value());
3580 if (!intValue || *intValue != sourceShape[size.index()])
3581 return false;
3582 }
3583 // All conditions met. The `SubViewOp` is foldable as a no-op.
3584 return true;
3585}
3586
3587namespace {
3588/// Pattern to rewrite a subview op with MemRefCast arguments.
3589/// This essentially pushes memref.cast past its consuming subview when
3590/// `canFoldIntoConsumerOp` is true.
3591///
3592/// Example:
3593/// ```
3594/// %0 = memref.cast %V : memref<16x16xf32> to memref<?x?xf32>
3595/// %1 = memref.subview %0[0, 0][3, 4][1, 1] :
3596/// memref<?x?xf32> to memref<3x4xf32, strided<[?, 1], offset: ?>>
3597/// ```
3598/// is rewritten into:
3599/// ```
3600/// %0 = memref.subview %V: memref<16x16xf32> to memref<3x4xf32, #[[map0]]>
3601/// %1 = memref.cast %0: memref<3x4xf32, strided<[16, 1], offset: 0>> to
3602/// memref<3x4xf32, strided<[?, 1], offset: ?>>
3603/// ```
3604class SubViewOpMemRefCastFolder final : public OpRewritePattern<SubViewOp> {
3605public:
3606 using OpRewritePattern<SubViewOp>::OpRewritePattern;
3607
3608 LogicalResult matchAndRewrite(SubViewOp subViewOp,
3609 PatternRewriter &rewriter) const override {
3610 // Any constant operand, just return to let SubViewOpConstantFolder kick
3611 // in.
3612 if (llvm::any_of(Range: subViewOp.getOperands(), P: [](Value operand) {
3613 return matchPattern(value: operand, pattern: matchConstantIndex());
3614 }))
3615 return failure();
3616
3617 auto castOp = subViewOp.getSource().getDefiningOp<CastOp>();
3618 if (!castOp)
3619 return failure();
3620
3621 if (!CastOp::canFoldIntoConsumerOp(castOp))
3622 return failure();
3623
3624 // Compute the SubViewOp result type after folding the MemRefCastOp. Use
3625 // the MemRefCastOp source operand type to infer the result type and the
3626 // current SubViewOp source operand type to compute the dropped dimensions
3627 // if the operation is rank-reducing.
3628 auto resultType = getCanonicalSubViewResultType(
3629 currentResultType: subViewOp.getType(), currentSourceType: subViewOp.getSourceType(),
3630 sourceType: llvm::cast<MemRefType>(Val: castOp.getSource().getType()),
3631 mixedOffsets: subViewOp.getMixedOffsets(), mixedSizes: subViewOp.getMixedSizes(),
3632 mixedStrides: subViewOp.getMixedStrides());
3633 if (!resultType)
3634 return failure();
3635
3636 Value newSubView = SubViewOp::create(
3637 builder&: rewriter, location: subViewOp.getLoc(), result: resultType, source: castOp.getSource(),
3638 offsets: subViewOp.getOffsets(), sizes: subViewOp.getSizes(), strides: subViewOp.getStrides(),
3639 static_offsets: subViewOp.getStaticOffsets(), static_sizes: subViewOp.getStaticSizes(),
3640 static_strides: subViewOp.getStaticStrides());
3641 rewriter.replaceOpWithNewOp<CastOp>(op: subViewOp, args: subViewOp.getType(),
3642 args&: newSubView);
3643 return success();
3644 }
3645};
3646
3647/// Canonicalize subview ops that are no-ops. When the source shape is not
3648/// same as a result shape due to use of `affine_map`.
3649class TrivialSubViewOpFolder final : public OpRewritePattern<SubViewOp> {
3650public:
3651 using OpRewritePattern<SubViewOp>::OpRewritePattern;
3652
3653 LogicalResult matchAndRewrite(SubViewOp subViewOp,
3654 PatternRewriter &rewriter) const override {
3655 if (!isTrivialSubViewOp(subViewOp))
3656 return failure();
3657 if (subViewOp.getSourceType() == subViewOp.getType()) {
3658 rewriter.replaceOp(op: subViewOp, newValues: subViewOp.getSource());
3659 return success();
3660 }
3661 rewriter.replaceOpWithNewOp<CastOp>(op: subViewOp, args: subViewOp.getType(),
3662 args: subViewOp.getSource());
3663 return success();
3664 }
3665};
3666} // namespace
3667
3668/// Return the canonical type of the result of a subview.
3669struct SubViewReturnTypeCanonicalizer {
3670 MemRefType operator()(SubViewOp op, ArrayRef<OpFoldResult> mixedOffsets,
3671 ArrayRef<OpFoldResult> mixedSizes,
3672 ArrayRef<OpFoldResult> mixedStrides) {
3673 // Infer a memref type without taking into account any rank reductions.
3674 MemRefType resTy = SubViewOp::inferResultType(
3675 sourceMemRefType: op.getSourceType(), offsets: mixedOffsets, sizes: mixedSizes, strides: mixedStrides);
3676 if (!resTy)
3677 return {};
3678 MemRefType nonReducedType = resTy;
3679
3680 // Directly return the non-rank reduced type if there are no dropped dims.
3681 llvm::SmallBitVector droppedDims = op.getDroppedDims();
3682 if (droppedDims.none())
3683 return nonReducedType;
3684
3685 // Take the strides and offset from the non-rank reduced type.
3686 auto [nonReducedStrides, offset] = nonReducedType.getStridesAndOffset();
3687
3688 // Drop dims from shape and strides.
3689 SmallVector<int64_t> targetShape;
3690 SmallVector<int64_t> targetStrides;
3691 for (int64_t i = 0; i < static_cast<int64_t>(mixedSizes.size()); ++i) {
3692 if (droppedDims.test(Idx: i))
3693 continue;
3694 targetStrides.push_back(Elt: nonReducedStrides[i]);
3695 targetShape.push_back(Elt: nonReducedType.getDimSize(idx: i));
3696 }
3697
3698 return MemRefType::get(shape: targetShape, elementType: nonReducedType.getElementType(),
3699 layout: StridedLayoutAttr::get(context: nonReducedType.getContext(),
3700 offset, strides: targetStrides),
3701 memorySpace: nonReducedType.getMemorySpace());
3702 }
3703};
3704
3705/// A canonicalizer wrapper to replace SubViewOps.
3706struct SubViewCanonicalizer {
3707 void operator()(PatternRewriter &rewriter, SubViewOp op, SubViewOp newOp) {
3708 rewriter.replaceOpWithNewOp<CastOp>(op, args: op.getType(), args&: newOp);
3709 }
3710};
3711
3712void SubViewOp::getCanonicalizationPatterns(RewritePatternSet &results,
3713 MLIRContext *context) {
3714 results
3715 .add<OpWithOffsetSizesAndStridesConstantArgumentFolder<
3716 SubViewOp, SubViewReturnTypeCanonicalizer, SubViewCanonicalizer>,
3717 SubViewOpMemRefCastFolder, TrivialSubViewOpFolder>(arg&: context);
3718}
3719
3720OpFoldResult SubViewOp::fold(FoldAdaptor adaptor) {
3721 MemRefType sourceMemrefType = getSource().getType();
3722 MemRefType resultMemrefType = getResult().getType();
3723 auto resultLayout =
3724 dyn_cast_if_present<StridedLayoutAttr>(Val: resultMemrefType.getLayout());
3725
3726 if (resultMemrefType == sourceMemrefType &&
3727 resultMemrefType.hasStaticShape() &&
3728 (!resultLayout || resultLayout.hasStaticLayout())) {
3729 return getViewSource();
3730 }
3731
3732 // Fold subview(subview(x)), where both subviews have the same size and the
3733 // second subview's offsets are all zero. (I.e., the second subview is a
3734 // no-op.)
3735 if (auto srcSubview = getViewSource().getDefiningOp<SubViewOp>()) {
3736 auto srcSizes = srcSubview.getMixedSizes();
3737 auto sizes = getMixedSizes();
3738 auto offsets = getMixedOffsets();
3739 bool allOffsetsZero = llvm::all_of(Range&: offsets, P: isZeroInteger);
3740 auto strides = getMixedStrides();
3741 bool allStridesOne = llvm::all_of(Range&: strides, P: isOneInteger);
3742 bool allSizesSame = llvm::equal(LRange&: sizes, RRange&: srcSizes);
3743 if (allOffsetsZero && allStridesOne && allSizesSame &&
3744 resultMemrefType == sourceMemrefType)
3745 return getViewSource();
3746 }
3747
3748 return {};
3749}
3750
3751FailureOr<std::optional<SmallVector<Value>>>
3752SubViewOp::bubbleDownCasts(OpBuilder &builder) {
3753 return bubbleDownCastsPassthroughOpImpl(op: *this, builder, src&: getSourceMutable());
3754}
3755
3756void SubViewOp::inferStridedMetadataRanges(
3757 ArrayRef<StridedMetadataRange> ranges, GetIntRangeFn getIntRange,
3758 SetStridedMetadataRangeFn setMetadata, int32_t indexBitwidth) {
3759 auto isUninitialized =
3760 +[](IntegerValueRange range) { return range.isUninitialized(); };
3761
3762 // Bail early if any of the operands metadata is not ready:
3763 SmallVector<IntegerValueRange> offsetOperands =
3764 getIntValueRanges(values: getMixedOffsets(), getIntRange, indexBitwidth);
3765 if (llvm::any_of(Range&: offsetOperands, P: isUninitialized))
3766 return;
3767
3768 SmallVector<IntegerValueRange> sizeOperands =
3769 getIntValueRanges(values: getMixedSizes(), getIntRange, indexBitwidth);
3770 if (llvm::any_of(Range&: sizeOperands, P: isUninitialized))
3771 return;
3772
3773 SmallVector<IntegerValueRange> stridesOperands =
3774 getIntValueRanges(values: getMixedStrides(), getIntRange, indexBitwidth);
3775 if (llvm::any_of(Range&: stridesOperands, P: isUninitialized))
3776 return;
3777
3778 StridedMetadataRange sourceRange =
3779 ranges[getSourceMutable().getOperandNumber()];
3780 if (sourceRange.isUninitialized())
3781 return;
3782
3783 ArrayRef<ConstantIntRanges> srcStrides = sourceRange.getStrides();
3784
3785 // Get the dropped dims.
3786 llvm::SmallBitVector droppedDims = getDroppedDims();
3787
3788 // Compute the new offset, strides and sizes.
3789 ConstantIntRanges offset = sourceRange.getOffsets()[0];
3790 SmallVector<ConstantIntRanges> strides, sizes;
3791
3792 for (size_t i = 0, e = droppedDims.size(); i < e; ++i) {
3793 bool dropped = droppedDims.test(Idx: i);
3794 // Compute the new offset.
3795 ConstantIntRanges off =
3796 intrange::inferMul(argRanges: {offsetOperands[i].getValue(), srcStrides[i]});
3797 offset = intrange::inferAdd(argRanges: {offset, off});
3798
3799 // Skip dropped dimensions.
3800 if (dropped)
3801 continue;
3802 // Multiply the strides.
3803 strides.push_back(
3804 Elt: intrange::inferMul(argRanges: {stridesOperands[i].getValue(), srcStrides[i]}));
3805 // Get the sizes.
3806 sizes.push_back(Elt: sizeOperands[i].getValue());
3807 }
3808
3809 setMetadata(getResult(),
3810 StridedMetadataRange::getRanked(
3811 offsets: SmallVector<ConstantIntRanges>({std::move(offset)}),
3812 sizes: std::move(sizes), strides: std::move(strides)));
3813}
3814
3815//===----------------------------------------------------------------------===//
3816// TransposeOp
3817//===----------------------------------------------------------------------===//
3818
3819void TransposeOp::getAsmResultNames(
3820 function_ref<void(Value, StringRef)> setNameFn) {
3821 setNameFn(getResult(), "transpose");
3822}
3823
3824/// Build a strided memref type by applying `permutationMap` to `memRefType`.
3825static MemRefType inferTransposeResultType(MemRefType memRefType,
3826 AffineMap permutationMap) {
3827 auto originalSizes = memRefType.getShape();
3828 auto [originalStrides, offset] = memRefType.getStridesAndOffset();
3829 assert(originalStrides.size() == static_cast<unsigned>(memRefType.getRank()));
3830
3831 // Compute permuted sizes and strides.
3832 auto sizes = applyPermutationMap<int64_t>(map: permutationMap, source: originalSizes);
3833 auto strides = applyPermutationMap<int64_t>(map: permutationMap, source: originalStrides);
3834
3835 return MemRefType::Builder(memRefType)
3836 .setShape(sizes)
3837 .setLayout(
3838 StridedLayoutAttr::get(context: memRefType.getContext(), offset, strides));
3839}
3840
3841Value TransposeOp::getViewSource() { return getIn(); }
3842
3843void TransposeOp::build(OpBuilder &b, OperationState &result, Value in,
3844 AffineMapAttr permutation,
3845 ArrayRef<NamedAttribute> attrs) {
3846 auto permutationMap = permutation.getValue();
3847 assert(permutationMap);
3848
3849 auto memRefType = llvm::cast<MemRefType>(Val: in.getType());
3850 // Compute result type.
3851 MemRefType resultType = inferTransposeResultType(memRefType, permutationMap);
3852
3853 buildPropertiesAndDiscardableAttributes(odsState&: result, attributes: attrs);
3854 result.getOrAddProperties<Properties>().permutation = permutation;
3855 result.addOperands(newOperands: in);
3856 result.addTypes(newTypes: resultType);
3857}
3858
3859// transpose $in $permutation attr-dict : type($in) `to` type(results)
3860void TransposeOp::print(OpAsmPrinter &p) {
3861 p << " " << getIn() << " " << getPermutation();
3862 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary(),
3863 elidedAttrs: {getPermutationAttrStrName()});
3864 p << " : " << getIn().getType() << " to " << getType();
3865}
3866
3867ParseResult TransposeOp::parse(OpAsmParser &parser, OperationState &result) {
3868 OpAsmParser::UnresolvedOperand in;
3869 AffineMap permutation;
3870 MemRefType srcType, dstType;
3871 if (parser.parseOperand(result&: in) || parser.parseAffineMap(map&: permutation) ||
3872 parser.parseOptionalAttrDict(result&: result.attributes) ||
3873 parser.parseColonType(result&: srcType) ||
3874 parser.resolveOperand(operand: in, type: srcType, result&: result.operands) ||
3875 parser.parseKeywordType(keyword: "to", result&: dstType) ||
3876 parser.addTypeToList(type: dstType, result&: result.types))
3877 return failure();
3878
3879 result.addAttribute(name: TransposeOp::getPermutationAttrStrName(),
3880 attr: AffineMapAttr::get(value: permutation));
3881 return success();
3882}
3883
3884LogicalResult TransposeOp::verify() {
3885 if (!getPermutation().isPermutation())
3886 return emitOpError(message: "expected a permutation map");
3887 if (getPermutation().getNumDims() != getIn().getType().getRank())
3888 return emitOpError(message: "expected a permutation map of same rank as the input");
3889
3890 auto srcType = llvm::cast<MemRefType>(Val: getIn().getType());
3891 auto resultType = llvm::cast<MemRefType>(Val: getType());
3892 auto canonicalResultType = inferTransposeResultType(memRefType: srcType, permutationMap: getPermutation())
3893 .canonicalizeStridedLayout();
3894
3895 if (resultType.canonicalizeStridedLayout() != canonicalResultType)
3896 return emitOpError(message: "result type ")
3897 << resultType
3898 << " is not equivalent to the canonical transposed input type "
3899 << canonicalResultType;
3900 return success();
3901}
3902
3903OpFoldResult TransposeOp::fold(FoldAdaptor) {
3904 // First check for identity permutation, we can fold it away if input and
3905 // result types are identical already.
3906 if (getPermutation().isIdentity() && getType() == getIn().getType())
3907 return getIn();
3908 // Fold two consecutive memref.transpose Ops into one by composing their
3909 // permutation maps.
3910 if (auto otherTransposeOp = getIn().getDefiningOp<memref::TransposeOp>()) {
3911 AffineMap composedPermutation =
3912 getPermutation().compose(map: otherTransposeOp.getPermutation());
3913 getInMutable().assign(value: otherTransposeOp.getIn());
3914 setPermutation(composedPermutation);
3915 return getResult();
3916 }
3917 return {};
3918}
3919
3920FailureOr<std::optional<SmallVector<Value>>>
3921TransposeOp::bubbleDownCasts(OpBuilder &builder) {
3922 return bubbleDownCastsPassthroughOpImpl(op: *this, builder, src&: getInMutable());
3923}
3924
3925//===----------------------------------------------------------------------===//
3926// ViewOp
3927//===----------------------------------------------------------------------===//
3928
3929void ViewOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) {
3930 setNameFn(getResult(), "view");
3931}
3932
3933LogicalResult ViewOp::verify() {
3934 auto baseType = llvm::cast<MemRefType>(Val: getOperand(i: 0).getType());
3935 auto viewType = getType();
3936
3937 // The base memref should have identity layout map (or none).
3938 if (!baseType.getLayout().isIdentity())
3939 return emitError(message: "unsupported map for base memref type ") << baseType;
3940
3941 // The result memref should have identity layout map (or none).
3942 if (!viewType.getLayout().isIdentity())
3943 return emitError(message: "unsupported map for result memref type ") << viewType;
3944
3945 // The base memref and the view memref should be in the same memory space.
3946 if (baseType.getMemorySpace() != viewType.getMemorySpace())
3947 return emitError(message: "different memory spaces specified for base memref "
3948 "type ")
3949 << baseType << " and view memref type " << viewType;
3950
3951 // Verify that we have the correct number of sizes for the result type.
3952 if (failed(Result: verifyDynamicDimensionCount(op: getOperation(), type: viewType, dynamicSizes: getSizes())))
3953 return failure();
3954
3955 return success();
3956}
3957
3958Value ViewOp::getViewSource() { return getSource(); }
3959
3960OpFoldResult ViewOp::fold(FoldAdaptor adaptor) {
3961 MemRefType sourceMemrefType = getSource().getType();
3962 MemRefType resultMemrefType = getResult().getType();
3963
3964 if (resultMemrefType == sourceMemrefType &&
3965 resultMemrefType.hasStaticShape() && isZeroInteger(v: getByteShift()))
3966 return getViewSource();
3967
3968 return {};
3969}
3970
3971SmallVector<OpFoldResult> ViewOp::getMixedSizes() {
3972 SmallVector<OpFoldResult> result;
3973 unsigned ctr = 0;
3974 Builder b(getContext());
3975 for (int64_t dim : getType().getShape()) {
3976 if (ShapedType::isDynamic(dValue: dim)) {
3977 result.push_back(Elt: getSizes()[ctr++]);
3978 } else {
3979 result.push_back(Elt: b.getIndexAttr(value: dim));
3980 }
3981 }
3982 return result;
3983}
3984
3985namespace {
3986/// Given a memref type and a range of values that defines its dynamic
3987/// dimension sizes, turn all dynamic sizes that have a constant value into
3988/// static dimension sizes.
3989static MemRefType
3990foldDynamicToStaticDimSizes(MemRefType type, ValueRange dynamicSizes,
3991 SmallVectorImpl<Value> &foldedDynamicSizes) {
3992 SmallVector<int64_t> staticShape(type.getShape());
3993 assert(type.getNumDynamicDims() == dynamicSizes.size() &&
3994 "incorrect number of dynamic sizes");
3995
3996 // Compute new static and dynamic sizes.
3997 unsigned ctr = 0;
3998 for (auto [dim, dimSize] : llvm::enumerate(First: type.getShape())) {
3999 if (ShapedType::isStatic(dValue: dimSize))
4000 continue;
4001
4002 Value dynamicSize = dynamicSizes[ctr++];
4003 if (auto cst = getConstantIntValue(ofr: dynamicSize)) {
4004 // Dynamic size must be non-negative.
4005 if (cst.value() < 0) {
4006 foldedDynamicSizes.push_back(Elt: dynamicSize);
4007 continue;
4008 }
4009 staticShape[dim] = cst.value();
4010 } else {
4011 foldedDynamicSizes.push_back(Elt: dynamicSize);
4012 }
4013 }
4014
4015 return MemRefType::Builder(type).setShape(staticShape);
4016}
4017
4018/// Change the result type of a `memref.view` by making originally dynamic
4019/// dimensions static when their sizes come from `constant` ops.
4020/// Example:
4021/// ```
4022/// %c5 = arith.constant 5: index
4023/// %0 = memref.view %src[%offset][%c5] : memref<?xi8> to memref<?x4xf32>
4024/// ```
4025/// to
4026/// ```
4027/// %0 = memref.view %src[%offset][] : memref<?xi8> to memref<5x4xf32>
4028/// ```
4029struct ViewOpShapeFolder : public OpRewritePattern<ViewOp> {
4030 using Base::Base;
4031
4032 LogicalResult matchAndRewrite(ViewOp viewOp,
4033 PatternRewriter &rewriter) const override {
4034 SmallVector<Value> foldedDynamicSizes;
4035 MemRefType resultType = viewOp.getType();
4036 MemRefType foldedMemRefType = foldDynamicToStaticDimSizes(
4037 type: resultType, dynamicSizes: viewOp.getSizes(), foldedDynamicSizes);
4038
4039 // Stop here if no dynamic size was promoted to static.
4040 if (foldedMemRefType == resultType)
4041 return failure();
4042
4043 // Create new ViewOp.
4044 auto newViewOp = ViewOp::create(builder&: rewriter, location: viewOp.getLoc(), resultType0: foldedMemRefType,
4045 source: viewOp.getSource(), byte_shift: viewOp.getByteShift(),
4046 sizes: foldedDynamicSizes);
4047 // Insert a cast so we have the same type as the old memref type.
4048 rewriter.replaceOpWithNewOp<CastOp>(op: viewOp, args&: resultType, args&: newViewOp);
4049 return success();
4050 }
4051};
4052
4053/// view(memref.cast(%source)) -> view(%source).
4054struct ViewOpMemrefCastFolder : public OpRewritePattern<ViewOp> {
4055 using Base::Base;
4056
4057 LogicalResult matchAndRewrite(ViewOp viewOp,
4058 PatternRewriter &rewriter) const override {
4059 auto memrefCastOp = viewOp.getSource().getDefiningOp<CastOp>();
4060 if (!memrefCastOp)
4061 return failure();
4062
4063 rewriter.replaceOpWithNewOp<ViewOp>(
4064 op: viewOp, args: viewOp.getType(), args: memrefCastOp.getSource(),
4065 args: viewOp.getByteShift(), args: viewOp.getSizes());
4066 return success();
4067 }
4068};
4069} // namespace
4070
4071void ViewOp::getCanonicalizationPatterns(RewritePatternSet &results,
4072 MLIRContext *context) {
4073 results.add<ViewOpShapeFolder, ViewOpMemrefCastFolder>(arg&: context);
4074}
4075
4076FailureOr<std::optional<SmallVector<Value>>>
4077ViewOp::bubbleDownCasts(OpBuilder &builder) {
4078 return bubbleDownCastsPassthroughOpImpl(op: *this, builder, src&: getSourceMutable());
4079}
4080
4081//===----------------------------------------------------------------------===//
4082// AtomicRMWOp
4083//===----------------------------------------------------------------------===//
4084
4085LogicalResult AtomicRMWOp::verify() {
4086 switch (getKind()) {
4087 case arith::AtomicRMWKind::addf:
4088 case arith::AtomicRMWKind::maximumf:
4089 case arith::AtomicRMWKind::minimumf:
4090 case arith::AtomicRMWKind::mulf:
4091 if (!llvm::isa<FloatType>(Val: getValue().getType()))
4092 return emitOpError() << "with kind '"
4093 << arith::stringifyAtomicRMWKind(getKind())
4094 << "' expects a floating-point type";
4095 break;
4096 case arith::AtomicRMWKind::addi:
4097 case arith::AtomicRMWKind::maxs:
4098 case arith::AtomicRMWKind::maxu:
4099 case arith::AtomicRMWKind::mins:
4100 case arith::AtomicRMWKind::minu:
4101 case arith::AtomicRMWKind::muli:
4102 case arith::AtomicRMWKind::ori:
4103 case arith::AtomicRMWKind::xori:
4104 case arith::AtomicRMWKind::andi:
4105 if (!llvm::isa<IntegerType>(Val: getValue().getType()))
4106 return emitOpError() << "with kind '"
4107 << arith::stringifyAtomicRMWKind(getKind())
4108 << "' expects an integer type";
4109 break;
4110 default:
4111 break;
4112 }
4113 return success();
4114}
4115
4116OpFoldResult AtomicRMWOp::fold(FoldAdaptor adaptor) {
4117 /// atomicrmw(memrefcast) -> atomicrmw
4118 if (succeeded(Result: foldMemRefCast(op: *this, inner: getValue())))
4119 return getResult();
4120 return OpFoldResult();
4121}
4122
4123FailureOr<std::optional<SmallVector<Value>>>
4124AtomicRMWOp::bubbleDownCasts(OpBuilder &builder) {
4125 return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getMemrefMutable(),
4126 results: getResult());
4127}
4128
4129TypedValue<MemRefType> AtomicRMWOp::getAccessedMemref() { return getMemref(); }
4130
4131std::optional<SmallVector<Value>>
4132AtomicRMWOp::updateMemrefAndIndices(RewriterBase &rewriter, Value newMemref,
4133 ValueRange newIndices) {
4134 rewriter.modifyOpInPlace(root: *this, callable: [&]() {
4135 getMemrefMutable().assign(value: newMemref);
4136 getIndicesMutable().assign(values: newIndices);
4137 });
4138 return std::nullopt;
4139}
4140
4141//===----------------------------------------------------------------------===//
4142// TableGen'd op method definitions
4143//===----------------------------------------------------------------------===//
4144
4145#define GET_OP_CLASSES
4146#include "mlir/Dialect/MemRef/IR/MemRefOps.cpp.inc"
4147