| 1 | //===----------------------------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 10 | #include "mlir/Dialect/Arith/Utils/Utils.h" |
| 11 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 12 | #include "mlir/Dialect/Utils/ReshapeOpsUtils.h" |
| 13 | #include "mlir/Dialect/Utils/StaticValueUtils.h" |
| 14 | #include "mlir/Dialect/Utils/VerificationUtils.h" |
| 15 | #include "mlir/IR/AffineMap.h" |
| 16 | #include "mlir/IR/Builders.h" |
| 17 | #include "mlir/IR/BuiltinTypes.h" |
| 18 | #include "mlir/IR/Matchers.h" |
| 19 | #include "mlir/IR/OpDefinition.h" |
| 20 | #include "mlir/IR/PatternMatch.h" |
| 21 | #include "mlir/IR/TypeUtilities.h" |
| 22 | #include "mlir/Interfaces/InferTypeOpInterface.h" |
| 23 | #include "mlir/Interfaces/SideEffectInterfaces.h" |
| 24 | #include "mlir/Interfaces/Utils/InferIntRangeCommon.h" |
| 25 | #include "mlir/Interfaces/ViewLikeInterface.h" |
| 26 | #include "llvm/ADT/STLExtras.h" |
| 27 | #include "llvm/ADT/SmallBitVector.h" |
| 28 | #include "llvm/ADT/SmallVectorExtras.h" |
| 29 | |
| 30 | using namespace mlir; |
| 31 | using namespace mlir::memref; |
| 32 | |
| 33 | /// Materialize a single constant operation from a given attribute value with |
| 34 | /// the desired resultant type. |
| 35 | Operation *MemRefDialect::materializeConstant(OpBuilder &builder, |
| 36 | Attribute value, Type type, |
| 37 | Location loc) { |
| 38 | return arith::ConstantOp::materialize(builder, value, type, loc); |
| 39 | } |
| 40 | |
| 41 | //===----------------------------------------------------------------------===// |
| 42 | // Common canonicalization pattern support logic |
| 43 | //===----------------------------------------------------------------------===// |
| 44 | |
| 45 | /// This is a common class used for patterns of the form |
| 46 | /// "someop(memrefcast) -> someop". It folds the source of any memref.cast |
| 47 | /// into the root operation directly. |
| 48 | LogicalResult mlir::memref::foldMemRefCast(Operation *op, Value inner) { |
| 49 | bool folded = false; |
| 50 | for (OpOperand &operand : op->getOpOperands()) { |
| 51 | auto cast = operand.get().getDefiningOp<CastOp>(); |
| 52 | if (cast && operand.get() != inner && |
| 53 | !llvm::isa<UnrankedMemRefType>(Val: cast.getOperand().getType())) { |
| 54 | operand.set(cast.getOperand()); |
| 55 | folded = true; |
| 56 | } |
| 57 | } |
| 58 | return success(IsSuccess: folded); |
| 59 | } |
| 60 | |
| 61 | /// Return an unranked/ranked tensor type for the given unranked/ranked memref |
| 62 | /// type. |
| 63 | Type mlir::memref::getTensorTypeFromMemRefType(Type type) { |
| 64 | if (auto memref = llvm::dyn_cast<MemRefType>(Val&: type)) |
| 65 | return RankedTensorType::get(shape: memref.getShape(), elementType: memref.getElementType()); |
| 66 | if (auto memref = llvm::dyn_cast<UnrankedMemRefType>(Val&: type)) |
| 67 | return UnrankedTensorType::get(elementType: memref.getElementType()); |
| 68 | return NoneType::get(context: type.getContext()); |
| 69 | } |
| 70 | |
| 71 | OpFoldResult memref::getMixedSize(OpBuilder &builder, Location loc, Value value, |
| 72 | int64_t dim) { |
| 73 | auto memrefType = llvm::cast<MemRefType>(Val: value.getType()); |
| 74 | if (memrefType.isDynamicDim(idx: dim)) |
| 75 | return builder.createOrFold<memref::DimOp>(location: loc, args&: value, args&: dim); |
| 76 | |
| 77 | return builder.getIndexAttr(value: memrefType.getDimSize(idx: dim)); |
| 78 | } |
| 79 | |
| 80 | SmallVector<OpFoldResult> memref::getMixedSizes(OpBuilder &builder, |
| 81 | Location loc, Value value) { |
| 82 | auto memrefType = llvm::cast<MemRefType>(Val: value.getType()); |
| 83 | SmallVector<OpFoldResult> result; |
| 84 | for (int64_t i = 0; i < memrefType.getRank(); ++i) |
| 85 | result.push_back(Elt: getMixedSize(builder, loc, value, dim: i)); |
| 86 | return result; |
| 87 | } |
| 88 | |
| 89 | //===----------------------------------------------------------------------===// |
| 90 | // Utility functions for propagating static information |
| 91 | //===----------------------------------------------------------------------===// |
| 92 | |
| 93 | /// Helper function that sets values[i] to constValues[i] if the latter is a |
| 94 | /// static value, as indicated by ShapedType::kDynamic. |
| 95 | /// |
| 96 | /// If constValues[i] is dynamic, tries to extract a constant value from |
| 97 | /// value[i] to allow for additional folding opportunities. Also convertes all |
| 98 | /// existing attributes to index attributes. (They may be i64 attributes.) |
| 99 | static void constifyIndexValues(SmallVectorImpl<OpFoldResult> &values, |
| 100 | ArrayRef<int64_t> constValues) { |
| 101 | assert(constValues.size() == values.size() && |
| 102 | "incorrect number of const values" ); |
| 103 | for (auto [i, cstVal] : llvm::enumerate(First&: constValues)) { |
| 104 | Builder builder(values[i].getContext()); |
| 105 | if (ShapedType::isStatic(dValue: cstVal)) { |
| 106 | // Constant value is known, use it directly. |
| 107 | values[i] = builder.getIndexAttr(value: cstVal); |
| 108 | continue; |
| 109 | } |
| 110 | if (std::optional<int64_t> cst = getConstantIntValue(ofr: values[i])) { |
| 111 | // Try to extract a constant or convert an existing to index. |
| 112 | values[i] = builder.getIndexAttr(value: *cst); |
| 113 | } |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | /// Helper function to retrieve a lossless memory-space cast, and the |
| 118 | /// corresponding new result memref type. |
| 119 | static std::tuple<MemorySpaceCastOpInterface, PtrLikeTypeInterface, Type> |
| 120 | getMemorySpaceCastInfo(BaseMemRefType resultTy, Value src) { |
| 121 | MemorySpaceCastOpInterface castOp = |
| 122 | MemorySpaceCastOpInterface::getIfPromotableCast(value: src); |
| 123 | |
| 124 | // Bail if the cast is not lossless. |
| 125 | if (!castOp) |
| 126 | return {}; |
| 127 | |
| 128 | // Transform the source and target type of `castOp` to have the same metadata |
| 129 | // as `resultTy`. Bail if not possible. |
| 130 | FailureOr<PtrLikeTypeInterface> srcTy = resultTy.clonePtrWith( |
| 131 | memorySpace: castOp.getSourcePtr().getType().getMemorySpace(), elementType: std::nullopt); |
| 132 | if (failed(Result: srcTy)) |
| 133 | return {}; |
| 134 | |
| 135 | FailureOr<PtrLikeTypeInterface> tgtTy = resultTy.clonePtrWith( |
| 136 | memorySpace: castOp.getTargetPtr().getType().getMemorySpace(), elementType: std::nullopt); |
| 137 | if (failed(Result: tgtTy)) |
| 138 | return {}; |
| 139 | |
| 140 | // Check if this is a valid memory-space cast. |
| 141 | if (!castOp.isValidMemorySpaceCast(tgt: *tgtTy, src: *srcTy)) |
| 142 | return {}; |
| 143 | |
| 144 | return std::make_tuple(args&: castOp, args&: *tgtTy, args&: *srcTy); |
| 145 | } |
| 146 | |
| 147 | /// Implementation of `bubbleDownCasts` method for memref operations that |
| 148 | /// return a single memref result. |
| 149 | template <typename ConcreteOpTy> |
| 150 | static FailureOr<std::optional<SmallVector<Value>>> |
| 151 | bubbleDownCastsPassthroughOpImpl(ConcreteOpTy op, OpBuilder &builder, |
| 152 | OpOperand &src) { |
| 153 | auto [castOp, tgtTy, resTy] = getMemorySpaceCastInfo(op.getType(), src.get()); |
| 154 | // Bail if we cannot cast. |
| 155 | if (!castOp) |
| 156 | return failure(); |
| 157 | |
| 158 | // Create the new operands. |
| 159 | SmallVector<Value> operands; |
| 160 | llvm::append_range(operands, op->getOperands()); |
| 161 | operands[src.getOperandNumber()] = castOp.getSourcePtr(); |
| 162 | |
| 163 | // Create the new op and results. |
| 164 | auto newOp = ConcreteOpTy::create( |
| 165 | builder, op.getLoc(), TypeRange(resTy), operands, op.getProperties(), |
| 166 | op->getDiscardableAttrDictionary().getValue()); |
| 167 | |
| 168 | // Insert a memory-space cast to the original memory space of the op. |
| 169 | MemorySpaceCastOpInterface result = castOp.cloneMemorySpaceCastOp( |
| 170 | builder, tgtTy, |
| 171 | cast<TypedValue<PtrLikeTypeInterface>>(newOp.getResult())); |
| 172 | return std::optional<SmallVector<Value>>( |
| 173 | SmallVector<Value>({result.getTargetPtr()})); |
| 174 | } |
| 175 | |
| 176 | //===----------------------------------------------------------------------===// |
| 177 | // AllocOp / AllocaOp |
| 178 | //===----------------------------------------------------------------------===// |
| 179 | |
| 180 | void AllocOp::getAsmResultNames( |
| 181 | function_ref<void(Value, StringRef)> setNameFn) { |
| 182 | setNameFn(getResult(), "alloc" ); |
| 183 | } |
| 184 | |
| 185 | void AllocaOp::getAsmResultNames( |
| 186 | function_ref<void(Value, StringRef)> setNameFn) { |
| 187 | setNameFn(getResult(), "alloca" ); |
| 188 | } |
| 189 | |
| 190 | template <typename AllocLikeOp> |
| 191 | static LogicalResult verifyAllocLikeOp(AllocLikeOp op) { |
| 192 | static_assert(llvm::is_one_of<AllocLikeOp, AllocOp, AllocaOp>::value, |
| 193 | "applies to only alloc or alloca" ); |
| 194 | auto memRefType = llvm::dyn_cast<MemRefType>(op.getResult().getType()); |
| 195 | if (!memRefType) |
| 196 | return op.emitOpError("result must be a memref" ); |
| 197 | |
| 198 | if (failed(verifyDynamicDimensionCount(op, memRefType, op.getDynamicSizes()))) |
| 199 | return failure(); |
| 200 | |
| 201 | unsigned numSymbols = 0; |
| 202 | if (!memRefType.getLayout().isIdentity()) |
| 203 | numSymbols = memRefType.getLayout().getAffineMap().getNumSymbols(); |
| 204 | if (op.getSymbolOperands().size() != numSymbols) |
| 205 | return op.emitOpError("symbol operand count does not equal memref symbol " |
| 206 | "count: expected " ) |
| 207 | << numSymbols << ", got " << op.getSymbolOperands().size(); |
| 208 | |
| 209 | return success(); |
| 210 | } |
| 211 | |
| 212 | LogicalResult AllocOp::verify() { return verifyAllocLikeOp(op: *this); } |
| 213 | |
| 214 | LogicalResult AllocaOp::verify() { |
| 215 | // An alloca op needs to have an ancestor with an allocation scope trait. |
| 216 | if (!(*this)->getParentWithTrait<OpTrait::AutomaticAllocationScope>()) |
| 217 | return emitOpError( |
| 218 | message: "requires an ancestor op with AutomaticAllocationScope trait" ); |
| 219 | |
| 220 | return verifyAllocLikeOp(op: *this); |
| 221 | } |
| 222 | |
| 223 | namespace { |
| 224 | /// Fold constant dimensions into an alloc like operation. |
| 225 | template <typename AllocLikeOp> |
| 226 | struct SimplifyAllocConst : public OpRewritePattern<AllocLikeOp> { |
| 227 | using OpRewritePattern<AllocLikeOp>::OpRewritePattern; |
| 228 | |
| 229 | LogicalResult matchAndRewrite(AllocLikeOp alloc, |
| 230 | PatternRewriter &rewriter) const override { |
| 231 | // Check to see if any dimensions operands are constants. If so, we can |
| 232 | // substitute and drop them. |
| 233 | if (llvm::none_of(alloc.getDynamicSizes(), [](Value operand) { |
| 234 | APInt constSizeArg; |
| 235 | if (!matchPattern(value: operand, pattern: m_ConstantInt(bind_value: &constSizeArg))) |
| 236 | return false; |
| 237 | return constSizeArg.isNonNegative(); |
| 238 | })) |
| 239 | return failure(); |
| 240 | |
| 241 | auto memrefType = alloc.getType(); |
| 242 | |
| 243 | // Ok, we have one or more constant operands. Collect the non-constant ones |
| 244 | // and keep track of the resultant memref type to build. |
| 245 | SmallVector<int64_t, 4> newShapeConstants; |
| 246 | newShapeConstants.reserve(N: memrefType.getRank()); |
| 247 | SmallVector<Value, 4> dynamicSizes; |
| 248 | |
| 249 | unsigned dynamicDimPos = 0; |
| 250 | for (unsigned dim = 0, e = memrefType.getRank(); dim < e; ++dim) { |
| 251 | int64_t dimSize = memrefType.getDimSize(dim); |
| 252 | // If this is already static dimension, keep it. |
| 253 | if (ShapedType::isStatic(dValue: dimSize)) { |
| 254 | newShapeConstants.push_back(Elt: dimSize); |
| 255 | continue; |
| 256 | } |
| 257 | auto dynamicSize = alloc.getDynamicSizes()[dynamicDimPos]; |
| 258 | APInt constSizeArg; |
| 259 | if (matchPattern(dynamicSize, m_ConstantInt(bind_value: &constSizeArg)) && |
| 260 | constSizeArg.isNonNegative()) { |
| 261 | // Dynamic shape dimension will be folded. |
| 262 | newShapeConstants.push_back(Elt: constSizeArg.getZExtValue()); |
| 263 | } else { |
| 264 | // Dynamic shape dimension not folded; copy dynamicSize from old memref. |
| 265 | newShapeConstants.push_back(Elt: ShapedType::kDynamic); |
| 266 | dynamicSizes.push_back(Elt: dynamicSize); |
| 267 | } |
| 268 | dynamicDimPos++; |
| 269 | } |
| 270 | |
| 271 | // Create new memref type (which will have fewer dynamic dimensions). |
| 272 | MemRefType newMemRefType = |
| 273 | MemRefType::Builder(memrefType).setShape(newShapeConstants); |
| 274 | assert(dynamicSizes.size() == newMemRefType.getNumDynamicDims()); |
| 275 | |
| 276 | // Create and insert the alloc op for the new memref. |
| 277 | auto newAlloc = AllocLikeOp::create(rewriter, alloc.getLoc(), newMemRefType, |
| 278 | dynamicSizes, alloc.getSymbolOperands(), |
| 279 | alloc.getAlignmentAttr()); |
| 280 | // Insert a cast so we have the same type as the old alloc. |
| 281 | rewriter.replaceOpWithNewOp<CastOp>(alloc, alloc.getType(), newAlloc); |
| 282 | return success(); |
| 283 | } |
| 284 | }; |
| 285 | |
| 286 | /// Fold alloc operations with no users or only store and dealloc uses. |
| 287 | template <typename T> |
| 288 | struct SimplifyDeadAlloc : public OpRewritePattern<T> { |
| 289 | using OpRewritePattern<T>::OpRewritePattern; |
| 290 | |
| 291 | LogicalResult matchAndRewrite(T alloc, |
| 292 | PatternRewriter &rewriter) const override { |
| 293 | if (llvm::any_of(alloc->getUsers(), [&](Operation *op) { |
| 294 | if (auto storeOp = dyn_cast<StoreOp>(Val: op)) |
| 295 | return storeOp.getValue() == alloc; |
| 296 | return !isa<DeallocOp>(Val: op); |
| 297 | })) |
| 298 | return failure(); |
| 299 | |
| 300 | for (Operation *user : llvm::make_early_inc_range(alloc->getUsers())) |
| 301 | rewriter.eraseOp(op: user); |
| 302 | |
| 303 | rewriter.eraseOp(op: alloc); |
| 304 | return success(); |
| 305 | } |
| 306 | }; |
| 307 | } // namespace |
| 308 | |
| 309 | void AllocOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 310 | MLIRContext *context) { |
| 311 | results.add<SimplifyAllocConst<AllocOp>, SimplifyDeadAlloc<AllocOp>>(arg&: context); |
| 312 | } |
| 313 | |
| 314 | void AllocaOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 315 | MLIRContext *context) { |
| 316 | results.add<SimplifyAllocConst<AllocaOp>, SimplifyDeadAlloc<AllocaOp>>( |
| 317 | arg&: context); |
| 318 | } |
| 319 | |
| 320 | //===----------------------------------------------------------------------===// |
| 321 | // ReallocOp |
| 322 | //===----------------------------------------------------------------------===// |
| 323 | |
| 324 | LogicalResult ReallocOp::verify() { |
| 325 | auto sourceType = llvm::cast<MemRefType>(Val: getOperand(i: 0).getType()); |
| 326 | MemRefType resultType = getType(); |
| 327 | |
| 328 | // The source memref should have identity layout (or none). |
| 329 | if (!sourceType.getLayout().isIdentity()) |
| 330 | return emitError(message: "unsupported layout for source memref type " ) |
| 331 | << sourceType; |
| 332 | |
| 333 | // The result memref should have identity layout (or none). |
| 334 | if (!resultType.getLayout().isIdentity()) |
| 335 | return emitError(message: "unsupported layout for result memref type " ) |
| 336 | << resultType; |
| 337 | |
| 338 | // The source memref and the result memref should be in the same memory space. |
| 339 | if (sourceType.getMemorySpace() != resultType.getMemorySpace()) |
| 340 | return emitError(message: "different memory spaces specified for source memref " |
| 341 | "type " ) |
| 342 | << sourceType << " and result memref type " << resultType; |
| 343 | |
| 344 | // The source memref and the result memref should have the same element type. |
| 345 | if (failed(Result: verifyElementTypesMatch(op: *this, lhs: sourceType, rhs: resultType, lhsName: "source" , |
| 346 | rhsName: "result" ))) |
| 347 | return failure(); |
| 348 | |
| 349 | // Verify that we have the dynamic dimension operand when it is needed. |
| 350 | if (resultType.getNumDynamicDims() && !getDynamicResultSize()) |
| 351 | return emitError(message: "missing dimension operand for result type " ) |
| 352 | << resultType; |
| 353 | if (!resultType.getNumDynamicDims() && getDynamicResultSize()) |
| 354 | return emitError(message: "unnecessary dimension operand for result type " ) |
| 355 | << resultType; |
| 356 | |
| 357 | return success(); |
| 358 | } |
| 359 | |
| 360 | void ReallocOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 361 | MLIRContext *context) { |
| 362 | results.add<SimplifyDeadAlloc<ReallocOp>>(arg&: context); |
| 363 | } |
| 364 | |
| 365 | //===----------------------------------------------------------------------===// |
| 366 | // AllocaScopeOp |
| 367 | //===----------------------------------------------------------------------===// |
| 368 | |
| 369 | void AllocaScopeOp::print(OpAsmPrinter &p) { |
| 370 | bool printBlockTerminators = false; |
| 371 | |
| 372 | p << ' '; |
| 373 | if (!getResults().empty()) { |
| 374 | p << " -> (" << getResultTypes() << ")" ; |
| 375 | printBlockTerminators = true; |
| 376 | } |
| 377 | p << ' '; |
| 378 | p.printRegion(blocks&: getBodyRegion(), |
| 379 | /*printEntryBlockArgs=*/false, |
| 380 | /*printBlockTerminators=*/printBlockTerminators); |
| 381 | p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary()); |
| 382 | } |
| 383 | |
| 384 | ParseResult AllocaScopeOp::parse(OpAsmParser &parser, OperationState &result) { |
| 385 | // Create a region for the body. |
| 386 | result.regions.reserve(N: 1); |
| 387 | Region *bodyRegion = result.addRegion(); |
| 388 | |
| 389 | // Parse optional results type list. |
| 390 | if (parser.parseOptionalArrowTypeList(result&: result.types)) |
| 391 | return failure(); |
| 392 | |
| 393 | // Parse the body region. |
| 394 | if (parser.parseRegion(region&: *bodyRegion, /*arguments=*/{})) |
| 395 | return failure(); |
| 396 | AllocaScopeOp::ensureTerminator(region&: *bodyRegion, builder&: parser.getBuilder(), |
| 397 | loc: result.location); |
| 398 | |
| 399 | // Parse the optional attribute list. |
| 400 | if (parser.parseOptionalAttrDict(result&: result.attributes)) |
| 401 | return failure(); |
| 402 | |
| 403 | return success(); |
| 404 | } |
| 405 | |
| 406 | void AllocaScopeOp::getSuccessorRegions( |
| 407 | RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 408 | if (!point.isParent()) { |
| 409 | regions.push_back(Elt: RegionSuccessor(getOperation())); |
| 410 | return; |
| 411 | } |
| 412 | |
| 413 | regions.push_back(Elt: RegionSuccessor(&getBodyRegion())); |
| 414 | } |
| 415 | |
| 416 | ValueRange AllocaScopeOp::getSuccessorInputs(RegionSuccessor successor) { |
| 417 | return successor.isOperation() ? ValueRange(getResults()) : ValueRange(); |
| 418 | } |
| 419 | |
| 420 | /// Given an operation, return whether this op is guaranteed to |
| 421 | /// allocate an AutomaticAllocationScopeResource |
| 422 | static bool isGuaranteedAutomaticAllocation(Operation *op) { |
| 423 | MemoryEffectOpInterface interface = dyn_cast<MemoryEffectOpInterface>(Val: op); |
| 424 | if (!interface) |
| 425 | return false; |
| 426 | for (auto res : op->getResults()) { |
| 427 | if (auto effect = |
| 428 | interface.getEffectOnValue<MemoryEffects::Allocate>(value: res)) { |
| 429 | if (isa<SideEffects::AutomaticAllocationScopeResource>( |
| 430 | Val: effect->getResource())) |
| 431 | return true; |
| 432 | } |
| 433 | } |
| 434 | return false; |
| 435 | } |
| 436 | |
| 437 | /// Given an operation, return whether this op itself could |
| 438 | /// allocate an AutomaticAllocationScopeResource. Note that |
| 439 | /// this will not check whether an operation contained within |
| 440 | /// the op can allocate. |
| 441 | static bool isOpItselfPotentialAutomaticAllocation(Operation *op) { |
| 442 | // This op itself doesn't create a stack allocation, |
| 443 | // the inner allocation should be handled separately. |
| 444 | if (op->hasTrait<OpTrait::HasRecursiveMemoryEffects>()) |
| 445 | return false; |
| 446 | MemoryEffectOpInterface interface = dyn_cast<MemoryEffectOpInterface>(Val: op); |
| 447 | if (!interface) |
| 448 | return true; |
| 449 | for (auto res : op->getResults()) { |
| 450 | if (auto effect = |
| 451 | interface.getEffectOnValue<MemoryEffects::Allocate>(value: res)) { |
| 452 | if (isa<SideEffects::AutomaticAllocationScopeResource>( |
| 453 | Val: effect->getResource())) |
| 454 | return true; |
| 455 | } |
| 456 | } |
| 457 | return false; |
| 458 | } |
| 459 | |
| 460 | /// Return whether this op is the last non terminating op |
| 461 | /// in a region. That is to say, it is in a one-block region |
| 462 | /// and is only followed by a terminator. This prevents |
| 463 | /// extending the lifetime of allocations. |
| 464 | static bool lastNonTerminatorInRegion(Operation *op) { |
| 465 | return op->getBlock()->mightHaveTerminator() && |
| 466 | op->getNextNode() == op->getBlock()->getTerminator() && |
| 467 | op->getParentRegion()->hasOneBlock(); |
| 468 | } |
| 469 | |
| 470 | /// Inline an AllocaScopeOp if either the direct parent is an allocation scope |
| 471 | /// or it contains no allocation. |
| 472 | struct AllocaScopeInliner : public OpRewritePattern<AllocaScopeOp> { |
| 473 | using OpRewritePattern<AllocaScopeOp>::OpRewritePattern; |
| 474 | |
| 475 | LogicalResult matchAndRewrite(AllocaScopeOp op, |
| 476 | PatternRewriter &rewriter) const override { |
| 477 | bool hasPotentialAlloca = |
| 478 | op->walk<WalkOrder::PreOrder>(callback: [&](Operation *alloc) { |
| 479 | if (alloc == op) |
| 480 | return WalkResult::advance(); |
| 481 | if (isOpItselfPotentialAutomaticAllocation(op: alloc)) |
| 482 | return WalkResult::interrupt(); |
| 483 | if (alloc->hasTrait<OpTrait::AutomaticAllocationScope>()) |
| 484 | return WalkResult::skip(); |
| 485 | return WalkResult::advance(); |
| 486 | }).wasInterrupted(); |
| 487 | |
| 488 | // If this contains no potential allocation, it is always legal to |
| 489 | // inline. Otherwise, consider two conditions: |
| 490 | if (hasPotentialAlloca) { |
| 491 | // If the parent isn't an allocation scope, or we are not the last |
| 492 | // non-terminator op in the parent, we will extend the lifetime. |
| 493 | if (!op->getParentOp()->hasTrait<OpTrait::AutomaticAllocationScope>()) |
| 494 | return failure(); |
| 495 | if (!lastNonTerminatorInRegion(op)) |
| 496 | return failure(); |
| 497 | } |
| 498 | |
| 499 | Block *block = &op.getRegion().front(); |
| 500 | Operation *terminator = block->getTerminator(); |
| 501 | ValueRange results = terminator->getOperands(); |
| 502 | rewriter.inlineBlockBefore(source: block, op); |
| 503 | rewriter.replaceOp(op, newValues: results); |
| 504 | rewriter.eraseOp(op: terminator); |
| 505 | return success(); |
| 506 | } |
| 507 | }; |
| 508 | |
| 509 | /// Move allocations into an allocation scope, if it is legal to |
| 510 | /// move them (e.g. their operands are available at the location |
| 511 | /// the op would be moved to). |
| 512 | struct AllocaScopeHoister : public OpRewritePattern<AllocaScopeOp> { |
| 513 | using OpRewritePattern<AllocaScopeOp>::OpRewritePattern; |
| 514 | |
| 515 | LogicalResult matchAndRewrite(AllocaScopeOp op, |
| 516 | PatternRewriter &rewriter) const override { |
| 517 | |
| 518 | if (!op->getParentWithTrait<OpTrait::AutomaticAllocationScope>()) |
| 519 | return failure(); |
| 520 | |
| 521 | Operation *lastParentWithoutScope = op->getParentOp(); |
| 522 | |
| 523 | if (!lastParentWithoutScope || |
| 524 | lastParentWithoutScope->hasTrait<OpTrait::AutomaticAllocationScope>()) |
| 525 | return failure(); |
| 526 | |
| 527 | // Only apply to if this is this last non-terminator |
| 528 | // op in the block (lest lifetime be extended) of a one |
| 529 | // block region |
| 530 | if (!lastNonTerminatorInRegion(op) || |
| 531 | !lastNonTerminatorInRegion(op: lastParentWithoutScope)) |
| 532 | return failure(); |
| 533 | |
| 534 | while (!lastParentWithoutScope->getParentOp() |
| 535 | ->hasTrait<OpTrait::AutomaticAllocationScope>()) { |
| 536 | lastParentWithoutScope = lastParentWithoutScope->getParentOp(); |
| 537 | if (!lastParentWithoutScope || |
| 538 | !lastNonTerminatorInRegion(op: lastParentWithoutScope)) |
| 539 | return failure(); |
| 540 | } |
| 541 | assert(lastParentWithoutScope->getParentOp() |
| 542 | ->hasTrait<OpTrait::AutomaticAllocationScope>()); |
| 543 | |
| 544 | Region *containingRegion = nullptr; |
| 545 | for (auto &r : lastParentWithoutScope->getRegions()) { |
| 546 | if (r.isAncestor(other: op->getParentRegion())) { |
| 547 | assert(containingRegion == nullptr && |
| 548 | "only one region can contain the op" ); |
| 549 | containingRegion = &r; |
| 550 | } |
| 551 | } |
| 552 | assert(containingRegion && "op must be contained in a region" ); |
| 553 | |
| 554 | SmallVector<Operation *> toHoist; |
| 555 | op->walk(callback: [&](Operation *alloc) { |
| 556 | if (!isGuaranteedAutomaticAllocation(op: alloc)) |
| 557 | return WalkResult::skip(); |
| 558 | |
| 559 | // If any operand is not defined before the location of |
| 560 | // lastParentWithoutScope (i.e. where we would hoist to), skip. |
| 561 | if (llvm::any_of(Range: alloc->getOperands(), P: [&](Value v) { |
| 562 | return containingRegion->isAncestor(other: v.getParentRegion()); |
| 563 | })) |
| 564 | return WalkResult::skip(); |
| 565 | toHoist.push_back(Elt: alloc); |
| 566 | return WalkResult::advance(); |
| 567 | }); |
| 568 | |
| 569 | if (toHoist.empty()) |
| 570 | return failure(); |
| 571 | rewriter.setInsertionPoint(lastParentWithoutScope); |
| 572 | for (auto *op : toHoist) { |
| 573 | auto *cloned = rewriter.clone(op&: *op); |
| 574 | rewriter.replaceOp(op, newValues: cloned->getResults()); |
| 575 | } |
| 576 | return success(); |
| 577 | } |
| 578 | }; |
| 579 | |
| 580 | void AllocaScopeOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 581 | MLIRContext *context) { |
| 582 | results.add<AllocaScopeInliner, AllocaScopeHoister>(arg&: context); |
| 583 | } |
| 584 | |
| 585 | //===----------------------------------------------------------------------===// |
| 586 | // AssumeAlignmentOp |
| 587 | //===----------------------------------------------------------------------===// |
| 588 | |
| 589 | LogicalResult AssumeAlignmentOp::verify() { |
| 590 | if (!llvm::isPowerOf2_32(Value: getAlignment())) |
| 591 | return emitOpError(message: "alignment must be power of 2" ); |
| 592 | return success(); |
| 593 | } |
| 594 | |
| 595 | void AssumeAlignmentOp::getAsmResultNames( |
| 596 | function_ref<void(Value, StringRef)> setNameFn) { |
| 597 | setNameFn(getResult(), "assume_align" ); |
| 598 | } |
| 599 | |
| 600 | OpFoldResult AssumeAlignmentOp::fold(FoldAdaptor adaptor) { |
| 601 | auto source = getMemref().getDefiningOp<AssumeAlignmentOp>(); |
| 602 | if (!source) |
| 603 | return {}; |
| 604 | if (source.getAlignment() != getAlignment()) |
| 605 | return {}; |
| 606 | return getMemref(); |
| 607 | } |
| 608 | |
| 609 | FailureOr<std::optional<SmallVector<Value>>> |
| 610 | AssumeAlignmentOp::bubbleDownCasts(OpBuilder &builder) { |
| 611 | return bubbleDownCastsPassthroughOpImpl(op: *this, builder, src&: getMemrefMutable()); |
| 612 | } |
| 613 | |
| 614 | FailureOr<OpFoldResult> AssumeAlignmentOp::reifyDimOfResult(OpBuilder &builder, |
| 615 | int resultIndex, |
| 616 | int dim) { |
| 617 | assert(resultIndex == 0 && "AssumeAlignmentOp has a single result" ); |
| 618 | return getMixedSize(builder, loc: getLoc(), value: getMemref(), dim); |
| 619 | } |
| 620 | |
| 621 | //===----------------------------------------------------------------------===// |
| 622 | // DistinctObjectsOp |
| 623 | //===----------------------------------------------------------------------===// |
| 624 | |
| 625 | LogicalResult DistinctObjectsOp::verify() { |
| 626 | if (getOperandTypes() != getResultTypes()) |
| 627 | return emitOpError(message: "operand types and result types must match" ); |
| 628 | |
| 629 | if (getOperandTypes().empty()) |
| 630 | return emitOpError(message: "expected at least one operand" ); |
| 631 | |
| 632 | return success(); |
| 633 | } |
| 634 | |
| 635 | LogicalResult DistinctObjectsOp::inferReturnTypes( |
| 636 | MLIRContext * /*context*/, std::optional<Location> /*location*/, |
| 637 | ValueRange operands, DictionaryAttr /*attributes*/, |
| 638 | PropertyRef /*properties*/, RegionRange /*regions*/, |
| 639 | SmallVectorImpl<Type> &inferredReturnTypes) { |
| 640 | llvm::copy(Range: operands.getTypes(), Out: std::back_inserter(x&: inferredReturnTypes)); |
| 641 | return success(); |
| 642 | } |
| 643 | |
| 644 | //===----------------------------------------------------------------------===// |
| 645 | // CastOp |
| 646 | //===----------------------------------------------------------------------===// |
| 647 | |
| 648 | void CastOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) { |
| 649 | setNameFn(getResult(), "cast" ); |
| 650 | } |
| 651 | |
| 652 | /// Determines whether MemRef_CastOp casts to a more dynamic version of the |
| 653 | /// source memref. This is useful to fold a memref.cast into a consuming op |
| 654 | /// and implement canonicalization patterns for ops in different dialects that |
| 655 | /// may consume the results of memref.cast operations. Such foldable memref.cast |
| 656 | /// operations are typically inserted as `view` and `subview` ops are |
| 657 | /// canonicalized, to preserve the type compatibility of their uses. |
| 658 | /// |
| 659 | /// Returns true when all conditions are met: |
| 660 | /// 1. source and result are ranked memrefs with strided semantics and same |
| 661 | /// element type and rank. |
| 662 | /// 2. each of the source's size, offset or stride has more static information |
| 663 | /// than the corresponding result's size, offset or stride. |
| 664 | /// |
| 665 | /// Example 1: |
| 666 | /// ```mlir |
| 667 | /// %1 = memref.cast %0 : memref<8x16xf32> to memref<?x?xf32> |
| 668 | /// %2 = consumer %1 ... : memref<?x?xf32> ... |
| 669 | /// ``` |
| 670 | /// |
| 671 | /// may fold into: |
| 672 | /// |
| 673 | /// ```mlir |
| 674 | /// %2 = consumer %0 ... : memref<8x16xf32> ... |
| 675 | /// ``` |
| 676 | /// |
| 677 | /// Example 2: |
| 678 | /// ``` |
| 679 | /// %1 = memref.cast %0 : memref<?x16xf32, affine_map<(i, j)->(16 * i + j)>> |
| 680 | /// to memref<?x?xf32> |
| 681 | /// consumer %1 : memref<?x?xf32> ... |
| 682 | /// ``` |
| 683 | /// |
| 684 | /// may fold into: |
| 685 | /// |
| 686 | /// ``` |
| 687 | /// consumer %0 ... : memref<?x16xf32, affine_map<(i, j)->(16 * i + j)>> |
| 688 | /// ``` |
| 689 | bool CastOp::canFoldIntoConsumerOp(CastOp castOp) { |
| 690 | MemRefType sourceType = |
| 691 | llvm::dyn_cast<MemRefType>(Val: castOp.getSource().getType()); |
| 692 | MemRefType resultType = llvm::dyn_cast<MemRefType>(Val: castOp.getType()); |
| 693 | |
| 694 | // Requires ranked MemRefType. |
| 695 | if (!sourceType || !resultType) |
| 696 | return false; |
| 697 | |
| 698 | // Requires same elemental type. |
| 699 | if (sourceType.getElementType() != resultType.getElementType()) |
| 700 | return false; |
| 701 | |
| 702 | // Requires same rank. |
| 703 | if (sourceType.getRank() != resultType.getRank()) |
| 704 | return false; |
| 705 | |
| 706 | // Only fold casts between strided memref forms. |
| 707 | int64_t sourceOffset, resultOffset; |
| 708 | SmallVector<int64_t, 4> sourceStrides, resultStrides; |
| 709 | if (failed(Result: sourceType.getStridesAndOffset(strides&: sourceStrides, offset&: sourceOffset)) || |
| 710 | failed(Result: resultType.getStridesAndOffset(strides&: resultStrides, offset&: resultOffset))) |
| 711 | return false; |
| 712 | |
| 713 | // If cast is towards more static sizes along any dimension, don't fold. |
| 714 | for (auto it : llvm::zip(t: sourceType.getShape(), u: resultType.getShape())) { |
| 715 | auto ss = std::get<0>(t&: it), st = std::get<1>(t&: it); |
| 716 | if (ss != st) |
| 717 | if (ShapedType::isDynamic(dValue: ss) && ShapedType::isStatic(dValue: st)) |
| 718 | return false; |
| 719 | } |
| 720 | |
| 721 | // If cast is towards more static offset along any dimension, don't fold. |
| 722 | if (sourceOffset != resultOffset) |
| 723 | if (ShapedType::isDynamic(dValue: sourceOffset) && |
| 724 | ShapedType::isStatic(dValue: resultOffset)) |
| 725 | return false; |
| 726 | |
| 727 | // If cast is towards more static strides along any dimension, don't fold. |
| 728 | for (auto it : llvm::zip(t&: sourceStrides, u&: resultStrides)) { |
| 729 | auto ss = std::get<0>(t&: it), st = std::get<1>(t&: it); |
| 730 | if (ss != st) |
| 731 | if (ShapedType::isDynamic(dValue: ss) && ShapedType::isStatic(dValue: st)) |
| 732 | return false; |
| 733 | } |
| 734 | |
| 735 | return true; |
| 736 | } |
| 737 | |
| 738 | bool CastOp::areCastCompatible(TypeRange inputs, TypeRange outputs) { |
| 739 | if (inputs.size() != 1 || outputs.size() != 1) |
| 740 | return false; |
| 741 | if (inputs == outputs) |
| 742 | return true; |
| 743 | Type a = inputs.front(), b = outputs.front(); |
| 744 | auto aT = llvm::dyn_cast<MemRefType>(Val&: a); |
| 745 | auto bT = llvm::dyn_cast<MemRefType>(Val&: b); |
| 746 | |
| 747 | auto uaT = llvm::dyn_cast<UnrankedMemRefType>(Val&: a); |
| 748 | auto ubT = llvm::dyn_cast<UnrankedMemRefType>(Val&: b); |
| 749 | |
| 750 | if (aT && bT) { |
| 751 | if (aT.getElementType() != bT.getElementType()) |
| 752 | return false; |
| 753 | if (aT.getLayout() != bT.getLayout()) { |
| 754 | int64_t aOffset, bOffset; |
| 755 | SmallVector<int64_t, 4> aStrides, bStrides; |
| 756 | if (failed(Result: aT.getStridesAndOffset(strides&: aStrides, offset&: aOffset)) || |
| 757 | failed(Result: bT.getStridesAndOffset(strides&: bStrides, offset&: bOffset)) || |
| 758 | aStrides.size() != bStrides.size()) |
| 759 | return false; |
| 760 | |
| 761 | // Strides along a dimension/offset are compatible if the value in the |
| 762 | // source memref is static and the value in the target memref is the |
| 763 | // same. They are also compatible if either one is dynamic (see |
| 764 | // description of MemRefCastOp for details). |
| 765 | // Note that for dimensions of size 1, the stride can differ. |
| 766 | auto checkCompatible = [](int64_t a, int64_t b) { |
| 767 | return (ShapedType::isDynamic(dValue: a) || ShapedType::isDynamic(dValue: b) || a == b); |
| 768 | }; |
| 769 | if (!checkCompatible(aOffset, bOffset)) |
| 770 | return false; |
| 771 | for (const auto &[index, aStride] : enumerate(First&: aStrides)) { |
| 772 | if (aT.getDimSize(idx: index) == 1 || bT.getDimSize(idx: index) == 1) |
| 773 | continue; |
| 774 | if (!checkCompatible(aStride, bStrides[index])) |
| 775 | return false; |
| 776 | } |
| 777 | } |
| 778 | if (aT.getMemorySpace() != bT.getMemorySpace()) |
| 779 | return false; |
| 780 | |
| 781 | // They must have the same rank, and any specified dimensions must match. |
| 782 | if (aT.getRank() != bT.getRank()) |
| 783 | return false; |
| 784 | |
| 785 | for (unsigned i = 0, e = aT.getRank(); i != e; ++i) { |
| 786 | int64_t aDim = aT.getDimSize(idx: i), bDim = bT.getDimSize(idx: i); |
| 787 | if (ShapedType::isStatic(dValue: aDim) && ShapedType::isStatic(dValue: bDim) && |
| 788 | aDim != bDim) |
| 789 | return false; |
| 790 | } |
| 791 | return true; |
| 792 | } else { |
| 793 | if (!aT && !uaT) |
| 794 | return false; |
| 795 | if (!bT && !ubT) |
| 796 | return false; |
| 797 | // Unranked to unranked casting is unsupported |
| 798 | if (uaT && ubT) |
| 799 | return false; |
| 800 | |
| 801 | auto aEltType = (aT) ? aT.getElementType() : uaT.getElementType(); |
| 802 | auto bEltType = (bT) ? bT.getElementType() : ubT.getElementType(); |
| 803 | if (aEltType != bEltType) |
| 804 | return false; |
| 805 | |
| 806 | auto aMemSpace = (aT) ? aT.getMemorySpace() : uaT.getMemorySpace(); |
| 807 | auto bMemSpace = (bT) ? bT.getMemorySpace() : ubT.getMemorySpace(); |
| 808 | return aMemSpace == bMemSpace; |
| 809 | } |
| 810 | |
| 811 | return false; |
| 812 | } |
| 813 | |
| 814 | OpFoldResult CastOp::fold(FoldAdaptor adaptor) { |
| 815 | return succeeded(Result: foldMemRefCast(op: *this)) ? getResult() : Value(); |
| 816 | } |
| 817 | |
| 818 | FailureOr<std::optional<SmallVector<Value>>> |
| 819 | CastOp::bubbleDownCasts(OpBuilder &builder) { |
| 820 | return bubbleDownCastsPassthroughOpImpl(op: *this, builder, src&: getSourceMutable()); |
| 821 | } |
| 822 | |
| 823 | //===----------------------------------------------------------------------===// |
| 824 | // CopyOp |
| 825 | //===----------------------------------------------------------------------===// |
| 826 | |
| 827 | namespace { |
| 828 | |
| 829 | /// Fold memref.copy(%x, %x). |
| 830 | struct FoldSelfCopy : public OpRewritePattern<CopyOp> { |
| 831 | using OpRewritePattern<CopyOp>::OpRewritePattern; |
| 832 | |
| 833 | LogicalResult matchAndRewrite(CopyOp copyOp, |
| 834 | PatternRewriter &rewriter) const override { |
| 835 | if (copyOp.getSource() != copyOp.getTarget()) |
| 836 | return failure(); |
| 837 | |
| 838 | rewriter.eraseOp(op: copyOp); |
| 839 | return success(); |
| 840 | } |
| 841 | }; |
| 842 | |
| 843 | struct FoldEmptyCopy final : public OpRewritePattern<CopyOp> { |
| 844 | using OpRewritePattern<CopyOp>::OpRewritePattern; |
| 845 | |
| 846 | static bool isEmptyMemRef(BaseMemRefType type) { |
| 847 | return type.hasRank() && llvm::is_contained(Range: type.getShape(), Element: 0); |
| 848 | } |
| 849 | |
| 850 | LogicalResult matchAndRewrite(CopyOp copyOp, |
| 851 | PatternRewriter &rewriter) const override { |
| 852 | if (isEmptyMemRef(type: copyOp.getSource().getType()) || |
| 853 | isEmptyMemRef(type: copyOp.getTarget().getType())) { |
| 854 | rewriter.eraseOp(op: copyOp); |
| 855 | return success(); |
| 856 | } |
| 857 | |
| 858 | return failure(); |
| 859 | } |
| 860 | }; |
| 861 | } // namespace |
| 862 | |
| 863 | void CopyOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 864 | MLIRContext *context) { |
| 865 | results.add<FoldEmptyCopy, FoldSelfCopy>(arg&: context); |
| 866 | } |
| 867 | |
| 868 | /// If the source/target of a CopyOp is a CastOp that does not modify the shape |
| 869 | /// and element type, the cast can be skipped. Such CastOps only cast the layout |
| 870 | /// of the type. |
| 871 | static LogicalResult foldCopyOfCast(CopyOp op) { |
| 872 | for (OpOperand &operand : op->getOpOperands()) { |
| 873 | auto castOp = operand.get().getDefiningOp<memref::CastOp>(); |
| 874 | if (castOp && memref::CastOp::canFoldIntoConsumerOp(castOp)) { |
| 875 | operand.set(castOp.getOperand()); |
| 876 | return success(); |
| 877 | } |
| 878 | } |
| 879 | return failure(); |
| 880 | } |
| 881 | |
| 882 | LogicalResult CopyOp::fold(FoldAdaptor adaptor, |
| 883 | SmallVectorImpl<OpFoldResult> &results) { |
| 884 | |
| 885 | /// copy(memrefcast) -> copy |
| 886 | return foldCopyOfCast(op: *this); |
| 887 | } |
| 888 | |
| 889 | //===----------------------------------------------------------------------===// |
| 890 | // DeallocOp |
| 891 | //===----------------------------------------------------------------------===// |
| 892 | |
| 893 | LogicalResult DeallocOp::fold(FoldAdaptor adaptor, |
| 894 | SmallVectorImpl<OpFoldResult> &results) { |
| 895 | /// dealloc(memrefcast) -> dealloc |
| 896 | return foldMemRefCast(op: *this); |
| 897 | } |
| 898 | |
| 899 | //===----------------------------------------------------------------------===// |
| 900 | // DimOp |
| 901 | //===----------------------------------------------------------------------===// |
| 902 | |
| 903 | void DimOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) { |
| 904 | setNameFn(getResult(), "dim" ); |
| 905 | } |
| 906 | |
| 907 | void DimOp::build(OpBuilder &builder, OperationState &result, Value source, |
| 908 | int64_t index) { |
| 909 | auto loc = result.location; |
| 910 | Value indexValue = arith::ConstantIndexOp::create(builder, location: loc, value: index); |
| 911 | build(odsBuilder&: builder, odsState&: result, source, index: indexValue); |
| 912 | } |
| 913 | |
| 914 | std::optional<int64_t> DimOp::getConstantIndex() { |
| 915 | return getConstantIntValue(ofr: getIndex()); |
| 916 | } |
| 917 | |
| 918 | Speculation::Speculatability DimOp::getSpeculatability() { |
| 919 | auto constantIndex = getConstantIndex(); |
| 920 | if (!constantIndex) |
| 921 | return Speculation::NotSpeculatable; |
| 922 | |
| 923 | auto rankedSourceType = dyn_cast<MemRefType>(Val: getSource().getType()); |
| 924 | if (!rankedSourceType) |
| 925 | return Speculation::NotSpeculatable; |
| 926 | |
| 927 | if (rankedSourceType.getRank() <= constantIndex) |
| 928 | return Speculation::NotSpeculatable; |
| 929 | |
| 930 | return Speculation::Speculatable; |
| 931 | } |
| 932 | |
| 933 | void DimOp::inferResultRangesFromOptional(ArrayRef<IntegerValueRange> argRanges, |
| 934 | SetIntLatticeFn setResultRange) { |
| 935 | setResultRange(getResult(), |
| 936 | intrange::inferShapedDimOpInterface(op: *this, maybeDim: argRanges[1])); |
| 937 | } |
| 938 | |
| 939 | /// Return a map with key being elements in `vals` and data being number of |
| 940 | /// occurences of it. Use std::map, since the `vals` here are strides and the |
| 941 | /// dynamic stride value is the same as the tombstone value for |
| 942 | /// `DenseMap<int64_t>`. |
| 943 | static std::map<int64_t, unsigned> getNumOccurences(ArrayRef<int64_t> vals) { |
| 944 | std::map<int64_t, unsigned> numOccurences; |
| 945 | for (auto val : vals) |
| 946 | numOccurences[val]++; |
| 947 | return numOccurences; |
| 948 | } |
| 949 | |
| 950 | /// Returns the set of source dimensions that are dropped in a rank reduction. |
| 951 | /// For each result dimension in order, matches the leftmost unmatched source |
| 952 | /// dimension with the same size. Source dimensions not matched are dropped. |
| 953 | /// |
| 954 | /// Example: memref<1x8x1x3> to memref<1x8x3>. Source sizes [1, 8, 1, 3], result |
| 955 | /// [1, 8, 3]. Match result[0]=1 -> source dim 0, result[1]=8 -> source dim 1, |
| 956 | /// result[2]=3 -> source dim 3. Source dim 2 is unmatched and dropped. |
| 957 | static FailureOr<llvm::SmallBitVector> |
| 958 | computeMemRefRankReductionMaskByPosition(MemRefType originalType, |
| 959 | MemRefType reducedType, |
| 960 | ArrayRef<OpFoldResult> sizes) { |
| 961 | int64_t rankReduction = originalType.getRank() - reducedType.getRank(); |
| 962 | if (rankReduction <= 0) |
| 963 | return llvm::SmallBitVector(originalType.getRank()); |
| 964 | |
| 965 | // Build source sizes from subview sizes (one per source dim). |
| 966 | SmallVector<int64_t> sourceSizes(originalType.getRank()); |
| 967 | for (const auto &it : llvm::enumerate(First&: sizes)) { |
| 968 | if (std::optional<int64_t> cst = getConstantIntValue(ofr: it.value())) |
| 969 | sourceSizes[it.index()] = *cst; |
| 970 | else |
| 971 | sourceSizes[it.index()] = ShapedType::kDynamic; |
| 972 | } |
| 973 | |
| 974 | ArrayRef<int64_t> resultSizes = reducedType.getShape(); |
| 975 | llvm::SmallBitVector usedSourceDims(originalType.getRank()); |
| 976 | int64_t startJ = 0; |
| 977 | for (int64_t resultSize : resultSizes) { |
| 978 | bool matched = false; |
| 979 | for (int64_t j = startJ; j < originalType.getRank(); ++j) { |
| 980 | if (sourceSizes[j] == resultSize) { |
| 981 | usedSourceDims.set(j); |
| 982 | matched = true; |
| 983 | startJ = j + 1; |
| 984 | break; |
| 985 | } |
| 986 | } |
| 987 | if (!matched) |
| 988 | return failure(); |
| 989 | } |
| 990 | |
| 991 | llvm::SmallBitVector unusedDims(originalType.getRank()); |
| 992 | for (int64_t i = 0; i < originalType.getRank(); ++i) |
| 993 | if (!usedSourceDims.test(Idx: i)) |
| 994 | unusedDims.set(i); |
| 995 | return unusedDims; |
| 996 | } |
| 997 | |
| 998 | /// Returns the set of source dimensions that are dropped in a rank reduction. |
| 999 | /// A dimension is dropped if its stride is dropped; uses stride occurrence |
| 1000 | /// counting to disambiguate when multiple unit dims exist. |
| 1001 | /// |
| 1002 | /// Example: memref<1x1x?xf32, strided<[?, 4, 1]>> to memref<1x4xf32, |
| 1003 | /// strided<[4, 1]>>. Source strides [?, 4, 1], candidate [4, 1]. Dim 0 (stride |
| 1004 | /// ?) can be dropped; dim 1 (stride 4) must be kept. Source dim 0 is dropped. |
| 1005 | static FailureOr<llvm::SmallBitVector> computeMemRefRankReductionMaskByStrides( |
| 1006 | MemRefType originalType, MemRefType reducedType, |
| 1007 | ArrayRef<int64_t> originalStrides, ArrayRef<int64_t> candidateStrides, |
| 1008 | llvm::SmallBitVector unusedDims) { |
| 1009 | // Track the number of occurences of the strides in the original type |
| 1010 | // and the candidate type. For each unused dim that stride should not be |
| 1011 | // present in the candidate type. Note that there could be multiple dimensions |
| 1012 | // that have the same size. We dont need to exactly figure out which dim |
| 1013 | // corresponds to which stride, we just need to verify that the number of |
| 1014 | // reptitions of a stride in the original + number of unused dims with that |
| 1015 | // stride == number of repititions of a stride in the candidate. |
| 1016 | std::map<int64_t, unsigned> currUnaccountedStrides = |
| 1017 | getNumOccurences(vals: originalStrides); |
| 1018 | std::map<int64_t, unsigned> candidateStridesNumOccurences = |
| 1019 | getNumOccurences(vals: candidateStrides); |
| 1020 | for (size_t dim = 0, e = unusedDims.size(); dim != e; ++dim) { |
| 1021 | if (!unusedDims.test(Idx: dim)) |
| 1022 | continue; |
| 1023 | int64_t originalStride = originalStrides[dim]; |
| 1024 | if (currUnaccountedStrides[originalStride] > |
| 1025 | candidateStridesNumOccurences[originalStride]) { |
| 1026 | // This dim can be treated as dropped. |
| 1027 | currUnaccountedStrides[originalStride]--; |
| 1028 | continue; |
| 1029 | } |
| 1030 | if (currUnaccountedStrides[originalStride] == |
| 1031 | candidateStridesNumOccurences[originalStride]) { |
| 1032 | // The stride for this is not dropped. Keep as is. |
| 1033 | unusedDims.reset(Idx: dim); |
| 1034 | continue; |
| 1035 | } |
| 1036 | if (currUnaccountedStrides[originalStride] < |
| 1037 | candidateStridesNumOccurences[originalStride]) { |
| 1038 | // This should never happen. Cant have a stride in the reduced rank type |
| 1039 | // that wasnt in the original one. |
| 1040 | return failure(); |
| 1041 | } |
| 1042 | } |
| 1043 | if (static_cast<int64_t>(unusedDims.count()) + reducedType.getRank() != |
| 1044 | originalType.getRank()) |
| 1045 | return failure(); |
| 1046 | return unusedDims; |
| 1047 | } |
| 1048 | |
| 1049 | /// Given the `originalType` and a `candidateReducedType` whose shape is assumed |
| 1050 | /// to be a subset of `originalType` with some `1` entries erased, return the |
| 1051 | /// set of indices that specifies which of the entries of `originalShape` are |
| 1052 | /// dropped to obtain `reducedShape`. |
| 1053 | /// This accounts for cases where there are multiple unit-dims, but only a |
| 1054 | /// subset of those are dropped. For MemRefTypes these can be disambiguated |
| 1055 | /// using the strides. If a dimension is dropped the stride must be dropped too. |
| 1056 | static FailureOr<llvm::SmallBitVector> |
| 1057 | computeMemRefRankReductionMask(MemRefType originalType, MemRefType reducedType, |
| 1058 | ArrayRef<OpFoldResult> sizes) { |
| 1059 | llvm::SmallBitVector unusedDims(originalType.getRank()); |
| 1060 | if (originalType.getRank() == reducedType.getRank()) |
| 1061 | return unusedDims; |
| 1062 | |
| 1063 | for (const auto &dim : llvm::enumerate(First&: sizes)) |
| 1064 | if (auto attr = llvm::dyn_cast_if_present<Attribute>(Val: dim.value())) |
| 1065 | if (llvm::cast<IntegerAttr>(Val&: attr).getInt() == 1) |
| 1066 | unusedDims.set(dim.index()); |
| 1067 | |
| 1068 | // Early exit for the case where the number of unused dims matches the number |
| 1069 | // of ranks reduced. |
| 1070 | if (static_cast<int64_t>(unusedDims.count()) + reducedType.getRank() == |
| 1071 | originalType.getRank()) |
| 1072 | return unusedDims; |
| 1073 | |
| 1074 | SmallVector<int64_t> originalStrides, candidateStrides; |
| 1075 | int64_t originalOffset, candidateOffset; |
| 1076 | if (failed( |
| 1077 | Result: originalType.getStridesAndOffset(strides&: originalStrides, offset&: originalOffset)) || |
| 1078 | failed( |
| 1079 | Result: reducedType.getStridesAndOffset(strides&: candidateStrides, offset&: candidateOffset))) |
| 1080 | return failure(); |
| 1081 | |
| 1082 | // Try stride-based first when we have meaningful static stride info |
| 1083 | // (preserves static strides). Fall back to position-based otherwise. |
| 1084 | auto hasNonTrivialStaticStride = [](ArrayRef<int64_t> strides) { |
| 1085 | // The innermost stride 1 is trivial for row-major and does not help |
| 1086 | // disambiguate. |
| 1087 | if (strides.size() <= 1) |
| 1088 | return false; |
| 1089 | return llvm::any_of(Range: strides.drop_back(), |
| 1090 | P: [](int64_t s) { return !ShapedType::isDynamic(dValue: s); }); |
| 1091 | }; |
| 1092 | if (hasNonTrivialStaticStride(originalStrides) || |
| 1093 | hasNonTrivialStaticStride(candidateStrides)) { |
| 1094 | FailureOr<llvm::SmallBitVector> strideBased = |
| 1095 | computeMemRefRankReductionMaskByStrides(originalType, reducedType, |
| 1096 | originalStrides, |
| 1097 | candidateStrides, unusedDims); |
| 1098 | if (succeeded(Result: strideBased)) |
| 1099 | return *strideBased; |
| 1100 | } |
| 1101 | return computeMemRefRankReductionMaskByPosition(originalType, reducedType, |
| 1102 | sizes); |
| 1103 | } |
| 1104 | |
| 1105 | llvm::SmallBitVector SubViewOp::getDroppedDims() { |
| 1106 | MemRefType sourceType = getSourceType(); |
| 1107 | MemRefType resultType = getType(); |
| 1108 | FailureOr<llvm::SmallBitVector> unusedDims = |
| 1109 | computeMemRefRankReductionMask(originalType: sourceType, reducedType: resultType, sizes: getMixedSizes()); |
| 1110 | assert(succeeded(unusedDims) && "unable to find unused dims of subview" ); |
| 1111 | return *unusedDims; |
| 1112 | } |
| 1113 | |
| 1114 | OpFoldResult DimOp::fold(FoldAdaptor adaptor) { |
| 1115 | // All forms of folding require a known index. |
| 1116 | std::optional<int64_t> index = getConstantIndex(); |
| 1117 | if (!index) |
| 1118 | return {}; |
| 1119 | |
| 1120 | // Folding for unranked types (UnrankedMemRefType) is not supported. |
| 1121 | auto memrefType = llvm::dyn_cast<MemRefType>(Val: getSource().getType()); |
| 1122 | if (!memrefType) |
| 1123 | return {}; |
| 1124 | |
| 1125 | // Out of bound indices produce undefined behavior but are still valid IR. |
| 1126 | // Don't choke on them. |
| 1127 | int64_t indexVal = index.value(); |
| 1128 | if (indexVal < 0 || indexVal >= memrefType.getRank()) |
| 1129 | return {}; |
| 1130 | |
| 1131 | // Fold if the shape extent along the given index is known. |
| 1132 | if (!memrefType.isDynamicDim(idx: indexVal)) { |
| 1133 | Builder builder(getContext()); |
| 1134 | return builder.getIndexAttr(value: memrefType.getShape()[indexVal]); |
| 1135 | } |
| 1136 | |
| 1137 | // The size at the given index is now known to be a dynamic size. |
| 1138 | // Fold dim to the size argument for an `AllocOp`, `ViewOp`, or `SubViewOp`. |
| 1139 | Operation *definingOp = getSource().getDefiningOp(); |
| 1140 | |
| 1141 | if (auto alloc = dyn_cast_or_null<AllocOp>(Val: definingOp)) |
| 1142 | return *(alloc.getDynamicSizes().begin() + |
| 1143 | memrefType.getDynamicDimIndex(index: indexVal)); |
| 1144 | |
| 1145 | if (auto alloca = dyn_cast_or_null<AllocaOp>(Val: definingOp)) |
| 1146 | return *(alloca.getDynamicSizes().begin() + |
| 1147 | memrefType.getDynamicDimIndex(index: indexVal)); |
| 1148 | |
| 1149 | if (auto view = dyn_cast_or_null<ViewOp>(Val: definingOp)) |
| 1150 | return *(view.getDynamicSizes().begin() + |
| 1151 | memrefType.getDynamicDimIndex(index: indexVal)); |
| 1152 | |
| 1153 | if (auto subview = dyn_cast_or_null<SubViewOp>(Val: definingOp)) { |
| 1154 | // The result dim is dynamic (the static case was handled above). Dropped |
| 1155 | // dims always have static size 1, so dynamic source sizes are never |
| 1156 | // dropped and map in order to the dynamic result dims. Find the k-th |
| 1157 | // dynamic source size, where k is the dynamic dim index of the result dim. |
| 1158 | unsigned dynamicResultDimIdx = memrefType.getDynamicDimIndex(index: indexVal); |
| 1159 | unsigned dynamicIdx = 0; |
| 1160 | for (OpFoldResult size : subview.getMixedSizes()) { |
| 1161 | if (llvm::isa<Attribute>(Val: size)) |
| 1162 | continue; |
| 1163 | if (dynamicIdx == dynamicResultDimIdx) |
| 1164 | return size; |
| 1165 | dynamicIdx++; |
| 1166 | } |
| 1167 | return {}; |
| 1168 | } |
| 1169 | |
| 1170 | // dim(memrefcast) -> dim |
| 1171 | if (succeeded(Result: foldMemRefCast(op: *this))) |
| 1172 | return getResult(); |
| 1173 | |
| 1174 | return {}; |
| 1175 | } |
| 1176 | |
| 1177 | namespace { |
| 1178 | /// Fold dim of a memref reshape operation to a load into the reshape's shape |
| 1179 | /// operand. |
| 1180 | struct DimOfMemRefReshape : public OpRewritePattern<DimOp> { |
| 1181 | using OpRewritePattern<DimOp>::OpRewritePattern; |
| 1182 | |
| 1183 | LogicalResult matchAndRewrite(DimOp dim, |
| 1184 | PatternRewriter &rewriter) const override { |
| 1185 | auto reshape = dim.getSource().getDefiningOp<ReshapeOp>(); |
| 1186 | |
| 1187 | if (!reshape) |
| 1188 | return rewriter.notifyMatchFailure( |
| 1189 | arg&: dim, msg: "Dim op is not defined by a reshape op." ); |
| 1190 | |
| 1191 | // dim of a memref reshape can be folded if dim.getIndex() dominates the |
| 1192 | // reshape. Instead of using `DominanceInfo` (which is usually costly) we |
| 1193 | // cheaply check that either of the following conditions hold: |
| 1194 | // 1. dim.getIndex() is defined in the same block as reshape but before |
| 1195 | // reshape. |
| 1196 | // 2. dim.getIndex() is defined in a parent block of |
| 1197 | // reshape. |
| 1198 | |
| 1199 | // Check condition 1 |
| 1200 | if (dim.getIndex().getParentBlock() == reshape->getBlock()) { |
| 1201 | if (auto *definingOp = dim.getIndex().getDefiningOp()) { |
| 1202 | if (reshape->isBeforeInBlock(other: definingOp)) { |
| 1203 | return rewriter.notifyMatchFailure( |
| 1204 | arg&: dim, |
| 1205 | msg: "dim.getIndex is not defined before reshape in the same block." ); |
| 1206 | } |
| 1207 | } // else dim.getIndex is a block argument to reshape->getBlock and |
| 1208 | // dominates reshape |
| 1209 | } // Check condition 2 |
| 1210 | else if (dim->getBlock() != reshape->getBlock() && |
| 1211 | !dim.getIndex().getParentRegion()->isProperAncestor( |
| 1212 | other: reshape->getParentRegion())) { |
| 1213 | // If dim and reshape are in the same block but dim.getIndex() isn't, we |
| 1214 | // already know dim.getIndex() dominates reshape without calling |
| 1215 | // `isProperAncestor` |
| 1216 | return rewriter.notifyMatchFailure( |
| 1217 | arg&: dim, msg: "dim.getIndex does not dominate reshape." ); |
| 1218 | } |
| 1219 | |
| 1220 | // Place the load directly after the reshape to ensure that the shape memref |
| 1221 | // was not mutated. |
| 1222 | rewriter.setInsertionPointAfter(reshape); |
| 1223 | Location loc = dim.getLoc(); |
| 1224 | Value load = |
| 1225 | LoadOp::create(builder&: rewriter, location: loc, memref: reshape.getShape(), indices: dim.getIndex()); |
| 1226 | if (load.getType() != dim.getType()) |
| 1227 | load = arith::IndexCastOp::create(builder&: rewriter, location: loc, out: dim.getType(), in: load); |
| 1228 | rewriter.replaceOp(op: dim, newValues: load); |
| 1229 | return success(); |
| 1230 | } |
| 1231 | }; |
| 1232 | |
| 1233 | } // namespace |
| 1234 | |
| 1235 | void DimOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 1236 | MLIRContext *context) { |
| 1237 | results.add<DimOfMemRefReshape>(arg&: context); |
| 1238 | } |
| 1239 | |
| 1240 | // --------------------------------------------------------------------------- |
| 1241 | // DmaStartOp |
| 1242 | // --------------------------------------------------------------------------- |
| 1243 | |
| 1244 | void DmaStartOp::build(OpBuilder &builder, OperationState &result, |
| 1245 | Value srcMemRef, ValueRange srcIndices, Value destMemRef, |
| 1246 | ValueRange destIndices, Value numElements, |
| 1247 | Value tagMemRef, ValueRange tagIndices, Value stride, |
| 1248 | Value elementsPerStride) { |
| 1249 | result.addOperands(newOperands: srcMemRef); |
| 1250 | result.addOperands(newOperands: srcIndices); |
| 1251 | result.addOperands(newOperands: destMemRef); |
| 1252 | result.addOperands(newOperands: destIndices); |
| 1253 | result.addOperands(newOperands: {numElements, tagMemRef}); |
| 1254 | result.addOperands(newOperands: tagIndices); |
| 1255 | if (stride) |
| 1256 | result.addOperands(newOperands: {stride, elementsPerStride}); |
| 1257 | } |
| 1258 | |
| 1259 | void DmaStartOp::print(OpAsmPrinter &p) { |
| 1260 | p << " " << getSrcMemRef() << '[' << getSrcIndices() << "], " |
| 1261 | << getDstMemRef() << '[' << getDstIndices() << "], " << getNumElements() |
| 1262 | << ", " << getTagMemRef() << '[' << getTagIndices() << ']'; |
| 1263 | if (isStrided()) |
| 1264 | p << ", " << getStride() << ", " << getNumElementsPerStride(); |
| 1265 | |
| 1266 | p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary()); |
| 1267 | p << " : " << getSrcMemRef().getType() << ", " << getDstMemRef().getType() |
| 1268 | << ", " << getTagMemRef().getType(); |
| 1269 | } |
| 1270 | |
| 1271 | // Parse DmaStartOp. |
| 1272 | // Ex: |
| 1273 | // %dma_id = dma_start %src[%i, %j], %dst[%k, %l], %size, |
| 1274 | // %tag[%index], %stride, %num_elt_per_stride : |
| 1275 | // : memref<3076 x f32, 0>, |
| 1276 | // memref<1024 x f32, 2>, |
| 1277 | // memref<1 x i32> |
| 1278 | // |
| 1279 | ParseResult DmaStartOp::parse(OpAsmParser &parser, OperationState &result) { |
| 1280 | OpAsmParser::UnresolvedOperand srcMemRefInfo; |
| 1281 | SmallVector<OpAsmParser::UnresolvedOperand, 4> srcIndexInfos; |
| 1282 | OpAsmParser::UnresolvedOperand dstMemRefInfo; |
| 1283 | SmallVector<OpAsmParser::UnresolvedOperand, 4> dstIndexInfos; |
| 1284 | OpAsmParser::UnresolvedOperand numElementsInfo; |
| 1285 | OpAsmParser::UnresolvedOperand tagMemrefInfo; |
| 1286 | SmallVector<OpAsmParser::UnresolvedOperand, 4> tagIndexInfos; |
| 1287 | SmallVector<OpAsmParser::UnresolvedOperand, 2> strideInfo; |
| 1288 | |
| 1289 | SmallVector<Type, 3> types; |
| 1290 | auto indexType = parser.getBuilder().getIndexType(); |
| 1291 | |
| 1292 | // Parse and resolve the following list of operands: |
| 1293 | // *) source memref followed by its indices (in square brackets). |
| 1294 | // *) destination memref followed by its indices (in square brackets). |
| 1295 | // *) dma size in KiB. |
| 1296 | if (parser.parseOperand(result&: srcMemRefInfo) || |
| 1297 | parser.parseOperandList(result&: srcIndexInfos, delimiter: OpAsmParser::Delimiter::Square) || |
| 1298 | parser.parseComma() || parser.parseOperand(result&: dstMemRefInfo) || |
| 1299 | parser.parseOperandList(result&: dstIndexInfos, delimiter: OpAsmParser::Delimiter::Square) || |
| 1300 | parser.parseComma() || parser.parseOperand(result&: numElementsInfo) || |
| 1301 | parser.parseComma() || parser.parseOperand(result&: tagMemrefInfo) || |
| 1302 | parser.parseOperandList(result&: tagIndexInfos, delimiter: OpAsmParser::Delimiter::Square)) |
| 1303 | return failure(); |
| 1304 | |
| 1305 | // Parse optional stride and elements per stride. |
| 1306 | if (parser.parseTrailingOperandList(result&: strideInfo)) |
| 1307 | return failure(); |
| 1308 | |
| 1309 | bool isStrided = strideInfo.size() == 2; |
| 1310 | if (!strideInfo.empty() && !isStrided) { |
| 1311 | return parser.emitError(loc: parser.getNameLoc(), |
| 1312 | message: "expected two stride related operands" ); |
| 1313 | } |
| 1314 | |
| 1315 | if (parser.parseColonTypeList(result&: types)) |
| 1316 | return failure(); |
| 1317 | if (types.size() != 3) |
| 1318 | return parser.emitError(loc: parser.getNameLoc(), message: "fewer/more types expected" ); |
| 1319 | |
| 1320 | if (parser.resolveOperand(operand: srcMemRefInfo, type: types[0], result&: result.operands) || |
| 1321 | parser.resolveOperands(operands&: srcIndexInfos, type: indexType, result&: result.operands) || |
| 1322 | parser.resolveOperand(operand: dstMemRefInfo, type: types[1], result&: result.operands) || |
| 1323 | parser.resolveOperands(operands&: dstIndexInfos, type: indexType, result&: result.operands) || |
| 1324 | // size should be an index. |
| 1325 | parser.resolveOperand(operand: numElementsInfo, type: indexType, result&: result.operands) || |
| 1326 | parser.resolveOperand(operand: tagMemrefInfo, type: types[2], result&: result.operands) || |
| 1327 | // tag indices should be index. |
| 1328 | parser.resolveOperands(operands&: tagIndexInfos, type: indexType, result&: result.operands)) |
| 1329 | return failure(); |
| 1330 | |
| 1331 | if (isStrided) { |
| 1332 | if (parser.resolveOperands(operands&: strideInfo, type: indexType, result&: result.operands)) |
| 1333 | return failure(); |
| 1334 | } |
| 1335 | |
| 1336 | return success(); |
| 1337 | } |
| 1338 | |
| 1339 | LogicalResult DmaStartOp::verify() { |
| 1340 | unsigned numOperands = getNumOperands(); |
| 1341 | |
| 1342 | // Mandatory non-variadic operands are: src memref, dst memref, tag memref and |
| 1343 | // the number of elements. |
| 1344 | if (numOperands < 4) |
| 1345 | return emitOpError(message: "expected at least 4 operands" ); |
| 1346 | |
| 1347 | // Check types of operands. The order of these calls is important: the later |
| 1348 | // calls rely on some type properties to compute the operand position. |
| 1349 | // 1. Source memref. |
| 1350 | if (!llvm::isa<MemRefType>(Val: getSrcMemRef().getType())) |
| 1351 | return emitOpError(message: "expected source to be of memref type" ); |
| 1352 | if (numOperands < getSrcMemRefRank() + 4) |
| 1353 | return emitOpError() << "expected at least " << getSrcMemRefRank() + 4 |
| 1354 | << " operands" ; |
| 1355 | if (!getSrcIndices().empty() && |
| 1356 | !llvm::all_of(Range: getSrcIndices().getTypes(), |
| 1357 | P: [](Type t) { return t.isIndex(); })) |
| 1358 | return emitOpError(message: "expected source indices to be of index type" ); |
| 1359 | |
| 1360 | // 2. Destination memref. |
| 1361 | if (!llvm::isa<MemRefType>(Val: getDstMemRef().getType())) |
| 1362 | return emitOpError(message: "expected destination to be of memref type" ); |
| 1363 | unsigned numExpectedOperands = getSrcMemRefRank() + getDstMemRefRank() + 4; |
| 1364 | if (numOperands < numExpectedOperands) |
| 1365 | return emitOpError() << "expected at least " << numExpectedOperands |
| 1366 | << " operands" ; |
| 1367 | if (!getDstIndices().empty() && |
| 1368 | !llvm::all_of(Range: getDstIndices().getTypes(), |
| 1369 | P: [](Type t) { return t.isIndex(); })) |
| 1370 | return emitOpError(message: "expected destination indices to be of index type" ); |
| 1371 | |
| 1372 | // 3. Number of elements. |
| 1373 | if (!getNumElements().getType().isIndex()) |
| 1374 | return emitOpError(message: "expected num elements to be of index type" ); |
| 1375 | |
| 1376 | // 4. Tag memref. |
| 1377 | if (!llvm::isa<MemRefType>(Val: getTagMemRef().getType())) |
| 1378 | return emitOpError(message: "expected tag to be of memref type" ); |
| 1379 | numExpectedOperands += getTagMemRefRank(); |
| 1380 | if (numOperands < numExpectedOperands) |
| 1381 | return emitOpError() << "expected at least " << numExpectedOperands |
| 1382 | << " operands" ; |
| 1383 | if (!getTagIndices().empty() && |
| 1384 | !llvm::all_of(Range: getTagIndices().getTypes(), |
| 1385 | P: [](Type t) { return t.isIndex(); })) |
| 1386 | return emitOpError(message: "expected tag indices to be of index type" ); |
| 1387 | |
| 1388 | // Optional stride-related operands must be either both present or both |
| 1389 | // absent. |
| 1390 | if (numOperands != numExpectedOperands && |
| 1391 | numOperands != numExpectedOperands + 2) |
| 1392 | return emitOpError(message: "incorrect number of operands" ); |
| 1393 | |
| 1394 | // 5. Strides. |
| 1395 | if (isStrided()) { |
| 1396 | if (!getStride().getType().isIndex() || |
| 1397 | !getNumElementsPerStride().getType().isIndex()) |
| 1398 | return emitOpError( |
| 1399 | message: "expected stride and num elements per stride to be of type index" ); |
| 1400 | } |
| 1401 | |
| 1402 | return success(); |
| 1403 | } |
| 1404 | |
| 1405 | LogicalResult DmaStartOp::fold(FoldAdaptor adaptor, |
| 1406 | SmallVectorImpl<OpFoldResult> &results) { |
| 1407 | /// dma_start(memrefcast) -> dma_start |
| 1408 | return foldMemRefCast(op: *this); |
| 1409 | } |
| 1410 | |
| 1411 | void DmaStartOp::setMemrefsAndIndices(RewriterBase &rewriter, Value newSrc, |
| 1412 | ValueRange newSrcIndices, Value newDst, |
| 1413 | ValueRange newDstIndices) { |
| 1414 | /// dma_start has special handling for variadic rank |
| 1415 | SmallVector<Value> newOperands; |
| 1416 | newOperands.push_back(Elt: newSrc); |
| 1417 | llvm::append_range(C&: newOperands, R&: newSrcIndices); |
| 1418 | newOperands.push_back(Elt: newDst); |
| 1419 | llvm::append_range(C&: newOperands, R&: newDstIndices); |
| 1420 | newOperands.push_back(Elt: getNumElements()); |
| 1421 | newOperands.push_back(Elt: getTagMemRef()); |
| 1422 | llvm::append_range(C&: newOperands, R: getTagIndices()); |
| 1423 | if (isStrided()) { |
| 1424 | newOperands.push_back(Elt: getStride()); |
| 1425 | newOperands.push_back(Elt: getNumElementsPerStride()); |
| 1426 | } |
| 1427 | |
| 1428 | rewriter.modifyOpInPlace(root: *this, callable: [&]() { (*this)->setOperands(newOperands); }); |
| 1429 | } |
| 1430 | |
| 1431 | // --------------------------------------------------------------------------- |
| 1432 | // DmaWaitOp |
| 1433 | // --------------------------------------------------------------------------- |
| 1434 | |
| 1435 | LogicalResult DmaWaitOp::fold(FoldAdaptor adaptor, |
| 1436 | SmallVectorImpl<OpFoldResult> &results) { |
| 1437 | /// dma_wait(memrefcast) -> dma_wait |
| 1438 | return foldMemRefCast(op: *this); |
| 1439 | } |
| 1440 | |
| 1441 | LogicalResult DmaWaitOp::verify() { |
| 1442 | // Check that the number of tag indices matches the tagMemRef rank. |
| 1443 | unsigned numTagIndices = getTagIndices().size(); |
| 1444 | unsigned tagMemRefRank = getTagMemRefRank(); |
| 1445 | if (numTagIndices != tagMemRefRank) |
| 1446 | return emitOpError() << "expected tagIndices to have the same number of " |
| 1447 | "elements as the tagMemRef rank, expected " |
| 1448 | << tagMemRefRank << ", but got " << numTagIndices; |
| 1449 | return success(); |
| 1450 | } |
| 1451 | |
| 1452 | //===----------------------------------------------------------------------===// |
| 1453 | // ExtractAlignedPointerAsIndexOp |
| 1454 | //===----------------------------------------------------------------------===// |
| 1455 | |
| 1456 | void ExtractAlignedPointerAsIndexOp::( |
| 1457 | function_ref<void(Value, StringRef)> setNameFn) { |
| 1458 | setNameFn(getResult(), "intptr" ); |
| 1459 | } |
| 1460 | |
| 1461 | //===----------------------------------------------------------------------===// |
| 1462 | // ExtractStridedMetadataOp |
| 1463 | //===----------------------------------------------------------------------===// |
| 1464 | |
| 1465 | /// The number and type of the results are inferred from the |
| 1466 | /// shape of the source. |
| 1467 | LogicalResult ExtractStridedMetadataOp::( |
| 1468 | MLIRContext *context, std::optional<Location> location, |
| 1469 | ExtractStridedMetadataOp::Adaptor adaptor, |
| 1470 | SmallVectorImpl<Type> &inferredReturnTypes) { |
| 1471 | auto sourceType = llvm::dyn_cast<MemRefType>(Val: adaptor.getSource().getType()); |
| 1472 | if (!sourceType) |
| 1473 | return failure(); |
| 1474 | |
| 1475 | unsigned sourceRank = sourceType.getRank(); |
| 1476 | IndexType indexType = IndexType::get(context); |
| 1477 | auto memrefType = |
| 1478 | MemRefType::get(shape: {}, elementType: sourceType.getElementType(), |
| 1479 | layout: MemRefLayoutAttrInterface{}, memorySpace: sourceType.getMemorySpace()); |
| 1480 | // Base. |
| 1481 | inferredReturnTypes.push_back(Elt: memrefType); |
| 1482 | // Offset. |
| 1483 | inferredReturnTypes.push_back(Elt: indexType); |
| 1484 | // Sizes and strides. |
| 1485 | for (unsigned i = 0; i < sourceRank * 2; ++i) |
| 1486 | inferredReturnTypes.push_back(Elt: indexType); |
| 1487 | return success(); |
| 1488 | } |
| 1489 | |
| 1490 | void ExtractStridedMetadataOp::( |
| 1491 | function_ref<void(Value, StringRef)> setNameFn) { |
| 1492 | setNameFn(getBaseBuffer(), "base_buffer" ); |
| 1493 | setNameFn(getOffset(), "offset" ); |
| 1494 | // For multi-result to work properly with pretty names and packed syntax `x:3` |
| 1495 | // we can only give a pretty name to the first value in the pack. |
| 1496 | if (!getSizes().empty()) { |
| 1497 | setNameFn(getSizes().front(), "sizes" ); |
| 1498 | setNameFn(getStrides().front(), "strides" ); |
| 1499 | } |
| 1500 | } |
| 1501 | |
| 1502 | /// Helper function to perform the replacement of all constant uses of `values` |
| 1503 | /// by a materialized constant extracted from `maybeConstants`. |
| 1504 | /// `values` and `maybeConstants` are expected to have the same size. |
| 1505 | template <typename Container> |
| 1506 | static bool replaceConstantUsesOf(OpBuilder &rewriter, Location loc, |
| 1507 | Container values, |
| 1508 | ArrayRef<OpFoldResult> maybeConstants) { |
| 1509 | assert(values.size() == maybeConstants.size() && |
| 1510 | " expected values and maybeConstants of the same size" ); |
| 1511 | bool atLeastOneReplacement = false; |
| 1512 | for (auto [maybeConstant, result] : llvm::zip(maybeConstants, values)) { |
| 1513 | // Don't materialize a constant if there are no uses: this would indice |
| 1514 | // infinite loops in the driver. |
| 1515 | if (result.use_empty() || maybeConstant == getAsOpFoldResult(result)) |
| 1516 | continue; |
| 1517 | assert(isa<Attribute>(maybeConstant) && |
| 1518 | "The constified value should be either unchanged (i.e., == result) " |
| 1519 | "or a constant" ); |
| 1520 | Value constantVal = arith::ConstantIndexOp::create( |
| 1521 | rewriter, loc, |
| 1522 | llvm::cast<IntegerAttr>(cast<Attribute>(maybeConstant)).getInt()); |
| 1523 | for (Operation *op : llvm::make_early_inc_range(result.getUsers())) { |
| 1524 | // modifyOpInPlace: lambda cannot capture structured bindings in C++17 |
| 1525 | // yet. |
| 1526 | op->replaceUsesOfWith(from: result, to: constantVal); |
| 1527 | atLeastOneReplacement = true; |
| 1528 | } |
| 1529 | } |
| 1530 | return atLeastOneReplacement; |
| 1531 | } |
| 1532 | |
| 1533 | LogicalResult |
| 1534 | ExtractStridedMetadataOp::(FoldAdaptor adaptor, |
| 1535 | SmallVectorImpl<OpFoldResult> &results) { |
| 1536 | OpBuilder builder(*this); |
| 1537 | |
| 1538 | bool atLeastOneReplacement = replaceConstantUsesOf( |
| 1539 | rewriter&: builder, loc: getLoc(), values: ArrayRef<TypedValue<IndexType>>(getOffset()), |
| 1540 | maybeConstants: getConstifiedMixedOffset()); |
| 1541 | atLeastOneReplacement |= replaceConstantUsesOf(rewriter&: builder, loc: getLoc(), values: getSizes(), |
| 1542 | maybeConstants: getConstifiedMixedSizes()); |
| 1543 | atLeastOneReplacement |= replaceConstantUsesOf( |
| 1544 | rewriter&: builder, loc: getLoc(), values: getStrides(), maybeConstants: getConstifiedMixedStrides()); |
| 1545 | |
| 1546 | // extract_strided_metadata(cast(x)) -> extract_strided_metadata(x). |
| 1547 | if (auto prev = getSource().getDefiningOp<CastOp>()) |
| 1548 | if (isa<MemRefType>(Val: prev.getSource().getType())) { |
| 1549 | getSourceMutable().assign(value: prev.getSource()); |
| 1550 | atLeastOneReplacement = true; |
| 1551 | } |
| 1552 | |
| 1553 | return success(IsSuccess: atLeastOneReplacement); |
| 1554 | } |
| 1555 | |
| 1556 | SmallVector<OpFoldResult> ExtractStridedMetadataOp::() { |
| 1557 | SmallVector<OpFoldResult> values = getAsOpFoldResult(values: getSizes()); |
| 1558 | constifyIndexValues(values, constValues: getSource().getType().getShape()); |
| 1559 | return values; |
| 1560 | } |
| 1561 | |
| 1562 | SmallVector<OpFoldResult> |
| 1563 | ExtractStridedMetadataOp::() { |
| 1564 | SmallVector<OpFoldResult> values = getAsOpFoldResult(values: getStrides()); |
| 1565 | SmallVector<int64_t> staticValues; |
| 1566 | int64_t unused; |
| 1567 | LogicalResult status = |
| 1568 | getSource().getType().getStridesAndOffset(strides&: staticValues, offset&: unused); |
| 1569 | (void)status; |
| 1570 | assert(succeeded(status) && "could not get strides from type" ); |
| 1571 | constifyIndexValues(values, constValues: staticValues); |
| 1572 | return values; |
| 1573 | } |
| 1574 | |
| 1575 | OpFoldResult ExtractStridedMetadataOp::() { |
| 1576 | OpFoldResult offsetOfr = getAsOpFoldResult(val: getOffset()); |
| 1577 | SmallVector<OpFoldResult> values(1, offsetOfr); |
| 1578 | SmallVector<int64_t> staticValues, unused; |
| 1579 | int64_t offset; |
| 1580 | LogicalResult status = |
| 1581 | getSource().getType().getStridesAndOffset(strides&: unused, offset); |
| 1582 | (void)status; |
| 1583 | assert(succeeded(status) && "could not get offset from type" ); |
| 1584 | staticValues.push_back(Elt: offset); |
| 1585 | constifyIndexValues(values, constValues: staticValues); |
| 1586 | return values[0]; |
| 1587 | } |
| 1588 | |
| 1589 | //===----------------------------------------------------------------------===// |
| 1590 | // GenericAtomicRMWOp |
| 1591 | //===----------------------------------------------------------------------===// |
| 1592 | |
| 1593 | void GenericAtomicRMWOp::build(OpBuilder &builder, OperationState &result, |
| 1594 | Value memref, ValueRange ivs) { |
| 1595 | OpBuilder::InsertionGuard g(builder); |
| 1596 | result.addOperands(newOperands: memref); |
| 1597 | result.addOperands(newOperands: ivs); |
| 1598 | |
| 1599 | if (auto memrefType = llvm::dyn_cast<MemRefType>(Val: memref.getType())) { |
| 1600 | Type elementType = memrefType.getElementType(); |
| 1601 | result.addTypes(newTypes: elementType); |
| 1602 | |
| 1603 | Region *bodyRegion = result.addRegion(); |
| 1604 | builder.createBlock(parent: bodyRegion); |
| 1605 | bodyRegion->addArgument(type: elementType, loc: memref.getLoc()); |
| 1606 | } |
| 1607 | } |
| 1608 | |
| 1609 | LogicalResult GenericAtomicRMWOp::verify() { |
| 1610 | auto &body = getRegion(); |
| 1611 | if (body.getNumArguments() != 1) |
| 1612 | return emitOpError(message: "expected single number of entry block arguments" ); |
| 1613 | |
| 1614 | if (getResult().getType() != body.getArgument(i: 0).getType()) |
| 1615 | return emitOpError(message: "expected block argument of the same type result type" ); |
| 1616 | |
| 1617 | bool hasSideEffects = |
| 1618 | body.walk(callback: [&](Operation *nestedOp) { |
| 1619 | if (isMemoryEffectFree(op: nestedOp)) |
| 1620 | return WalkResult::advance(); |
| 1621 | nestedOp->emitError( |
| 1622 | message: "body of 'memref.generic_atomic_rmw' should contain " |
| 1623 | "only operations with no side effects" ); |
| 1624 | return WalkResult::interrupt(); |
| 1625 | }) |
| 1626 | .wasInterrupted(); |
| 1627 | return hasSideEffects ? failure() : success(); |
| 1628 | } |
| 1629 | |
| 1630 | ParseResult GenericAtomicRMWOp::parse(OpAsmParser &parser, |
| 1631 | OperationState &result) { |
| 1632 | OpAsmParser::UnresolvedOperand memref; |
| 1633 | Type memrefType; |
| 1634 | SmallVector<OpAsmParser::UnresolvedOperand, 4> ivs; |
| 1635 | |
| 1636 | Type indexType = parser.getBuilder().getIndexType(); |
| 1637 | if (parser.parseOperand(result&: memref) || |
| 1638 | parser.parseOperandList(result&: ivs, delimiter: OpAsmParser::Delimiter::Square) || |
| 1639 | parser.parseColonType(result&: memrefType) || |
| 1640 | parser.resolveOperand(operand: memref, type: memrefType, result&: result.operands) || |
| 1641 | parser.resolveOperands(operands&: ivs, type: indexType, result&: result.operands)) |
| 1642 | return failure(); |
| 1643 | |
| 1644 | Region *body = result.addRegion(); |
| 1645 | if (parser.parseRegion(region&: *body, arguments: {}) || |
| 1646 | parser.parseOptionalAttrDict(result&: result.attributes)) |
| 1647 | return failure(); |
| 1648 | result.types.push_back(Elt: llvm::cast<MemRefType>(Val&: memrefType).getElementType()); |
| 1649 | return success(); |
| 1650 | } |
| 1651 | |
| 1652 | void GenericAtomicRMWOp::print(OpAsmPrinter &p) { |
| 1653 | p << ' ' << getMemref() << "[" << getIndices() |
| 1654 | << "] : " << getMemref().getType() << ' '; |
| 1655 | p.printRegion(blocks&: getRegion()); |
| 1656 | p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary()); |
| 1657 | } |
| 1658 | |
| 1659 | TypedValue<MemRefType> GenericAtomicRMWOp::getAccessedMemref() { |
| 1660 | return getMemref(); |
| 1661 | } |
| 1662 | |
| 1663 | std::optional<SmallVector<Value>> GenericAtomicRMWOp::updateMemrefAndIndices( |
| 1664 | RewriterBase &rewriter, Value newMemref, ValueRange newIndices) { |
| 1665 | rewriter.modifyOpInPlace(root: *this, callable: [&]() { |
| 1666 | getMemrefMutable().assign(value: newMemref); |
| 1667 | getIndicesMutable().assign(values: newIndices); |
| 1668 | }); |
| 1669 | return std::nullopt; |
| 1670 | } |
| 1671 | |
| 1672 | //===----------------------------------------------------------------------===// |
| 1673 | // AtomicYieldOp |
| 1674 | //===----------------------------------------------------------------------===// |
| 1675 | |
| 1676 | LogicalResult AtomicYieldOp::verify() { |
| 1677 | Type parentType = (*this)->getParentOp()->getResultTypes().front(); |
| 1678 | Type resultType = getResult().getType(); |
| 1679 | if (parentType != resultType) |
| 1680 | return emitOpError() << "types mismatch between yield op: " << resultType |
| 1681 | << " and its parent: " << parentType; |
| 1682 | return success(); |
| 1683 | } |
| 1684 | |
| 1685 | //===----------------------------------------------------------------------===// |
| 1686 | // GlobalOp |
| 1687 | //===----------------------------------------------------------------------===// |
| 1688 | |
| 1689 | static void printGlobalMemrefOpTypeAndInitialValue(OpAsmPrinter &p, GlobalOp op, |
| 1690 | TypeAttr type, |
| 1691 | Attribute initialValue) { |
| 1692 | p << type; |
| 1693 | if (!op.isExternal()) { |
| 1694 | p << " = " ; |
| 1695 | if (op.isUninitialized()) |
| 1696 | p << "uninitialized" ; |
| 1697 | else |
| 1698 | p.printAttributeWithoutType(attr: initialValue); |
| 1699 | } |
| 1700 | } |
| 1701 | |
| 1702 | static ParseResult |
| 1703 | parseGlobalMemrefOpTypeAndInitialValue(OpAsmParser &parser, TypeAttr &typeAttr, |
| 1704 | Attribute &initialValue) { |
| 1705 | Type type; |
| 1706 | if (parser.parseType(result&: type)) |
| 1707 | return failure(); |
| 1708 | |
| 1709 | auto memrefType = llvm::dyn_cast<MemRefType>(Val&: type); |
| 1710 | if (!memrefType || !memrefType.hasStaticShape()) |
| 1711 | return parser.emitError(loc: parser.getNameLoc()) |
| 1712 | << "type should be static shaped memref, but got " << type; |
| 1713 | typeAttr = TypeAttr::get(type); |
| 1714 | |
| 1715 | if (parser.parseOptionalEqual()) |
| 1716 | return success(); |
| 1717 | |
| 1718 | if (succeeded(Result: parser.parseOptionalKeyword(keyword: "uninitialized" ))) { |
| 1719 | initialValue = UnitAttr::get(context: parser.getContext()); |
| 1720 | return success(); |
| 1721 | } |
| 1722 | |
| 1723 | Type tensorType = getTensorTypeFromMemRefType(type: memrefType); |
| 1724 | if (parser.parseAttribute(result&: initialValue, type: tensorType)) |
| 1725 | return failure(); |
| 1726 | if (!llvm::isa<ElementsAttr>(Val: initialValue)) |
| 1727 | return parser.emitError(loc: parser.getNameLoc()) |
| 1728 | << "initial value should be a unit or elements attribute" ; |
| 1729 | return success(); |
| 1730 | } |
| 1731 | |
| 1732 | LogicalResult GlobalOp::verify() { |
| 1733 | auto memrefType = llvm::dyn_cast<MemRefType>(Val: getType()); |
| 1734 | if (!memrefType || !memrefType.hasStaticShape()) |
| 1735 | return emitOpError(message: "type should be static shaped memref, but got " ) |
| 1736 | << getType(); |
| 1737 | |
| 1738 | // Verify that the initial value, if present, is either a unit attribute or |
| 1739 | // an elements attribute. |
| 1740 | if (getInitialValue().has_value()) { |
| 1741 | Attribute initValue = getInitialValue().value(); |
| 1742 | if (!llvm::isa<UnitAttr>(Val: initValue) && !llvm::isa<ElementsAttr>(Val: initValue)) |
| 1743 | return emitOpError(message: "initial value should be a unit or elements " |
| 1744 | "attribute, but got " ) |
| 1745 | << initValue; |
| 1746 | |
| 1747 | // Check that the type of the initial value is compatible with the type of |
| 1748 | // the global variable. |
| 1749 | if (auto elementsAttr = llvm::dyn_cast<ElementsAttr>(Val&: initValue)) { |
| 1750 | // Check the element types match. |
| 1751 | auto initElementType = |
| 1752 | cast<TensorType>(Val: elementsAttr.getType()).getElementType(); |
| 1753 | auto memrefElementType = memrefType.getElementType(); |
| 1754 | |
| 1755 | if (initElementType != memrefElementType) |
| 1756 | return emitOpError(message: "initial value element expected to be of type " ) |
| 1757 | << memrefElementType << ", but was of type " << initElementType; |
| 1758 | |
| 1759 | // Check the shapes match, given that memref globals can only produce |
| 1760 | // statically shaped memrefs and elements literal type must have a static |
| 1761 | // shape we can assume both types are shaped. |
| 1762 | auto initShape = elementsAttr.getShapedType().getShape(); |
| 1763 | auto memrefShape = memrefType.getShape(); |
| 1764 | if (initShape != memrefShape) |
| 1765 | return emitOpError(message: "initial value shape expected to be " ) |
| 1766 | << memrefShape << " but was " << initShape; |
| 1767 | } |
| 1768 | } |
| 1769 | |
| 1770 | // TODO: verify visibility for declarations. |
| 1771 | return success(); |
| 1772 | } |
| 1773 | |
| 1774 | ElementsAttr GlobalOp::getConstantInitValue() { |
| 1775 | auto initVal = getInitialValue(); |
| 1776 | if (getConstant() && initVal.has_value()) |
| 1777 | return llvm::cast<ElementsAttr>(Val&: initVal.value()); |
| 1778 | return {}; |
| 1779 | } |
| 1780 | |
| 1781 | //===----------------------------------------------------------------------===// |
| 1782 | // GetGlobalOp |
| 1783 | //===----------------------------------------------------------------------===// |
| 1784 | |
| 1785 | LogicalResult |
| 1786 | GetGlobalOp::verifySymbolUses(SymbolTableCollection &symbolTable) { |
| 1787 | // Verify that the result type is same as the type of the referenced |
| 1788 | // memref.global op. |
| 1789 | auto global = |
| 1790 | symbolTable.lookupNearestSymbolFrom<GlobalOp>(from: *this, symbol: getNameAttr()); |
| 1791 | if (!global) |
| 1792 | return emitOpError(message: "'" ) |
| 1793 | << getName() << "' does not reference a valid global memref" ; |
| 1794 | |
| 1795 | Type resultType = getResult().getType(); |
| 1796 | if (global.getType() != resultType) |
| 1797 | return emitOpError(message: "result type " ) |
| 1798 | << resultType << " does not match type " << global.getType() |
| 1799 | << " of the global memref @" << getName(); |
| 1800 | return success(); |
| 1801 | } |
| 1802 | |
| 1803 | //===----------------------------------------------------------------------===// |
| 1804 | // LoadOp |
| 1805 | //===----------------------------------------------------------------------===// |
| 1806 | |
| 1807 | static ParseResult parseBoolAttr(OpAsmParser &parser, BoolAttr &result) { |
| 1808 | Attribute attr; |
| 1809 | if (parser.parseAttribute(result&: attr)) |
| 1810 | return failure(); |
| 1811 | result = dyn_cast<BoolAttr>(Val&: attr); |
| 1812 | if (!result) |
| 1813 | return parser.emitError(loc: parser.getCurrentLocation(), |
| 1814 | message: "expected boolean attribute" ); |
| 1815 | return success(); |
| 1816 | } |
| 1817 | |
| 1818 | static void printBoolAttr(OpAsmPrinter &printer, Operation *, BoolAttr attr) { |
| 1819 | printer.printAttribute(attr); |
| 1820 | } |
| 1821 | |
| 1822 | OpFoldResult LoadOp::fold(FoldAdaptor adaptor) { |
| 1823 | /// load(memrefcast) -> load |
| 1824 | if (succeeded(Result: foldMemRefCast(op: *this))) |
| 1825 | return getResult(); |
| 1826 | |
| 1827 | // Fold load from a global constant memref. |
| 1828 | auto getGlobalOp = getMemref().getDefiningOp<memref::GetGlobalOp>(); |
| 1829 | if (!getGlobalOp) |
| 1830 | return {}; |
| 1831 | |
| 1832 | // Get to the memref.global defining the symbol. |
| 1833 | auto global = SymbolTable::lookupNearestSymbolFrom<memref::GlobalOp>( |
| 1834 | from: getGlobalOp, symbol: getGlobalOp.getNameAttr()); |
| 1835 | if (!global) |
| 1836 | return {}; |
| 1837 | // If it's a splat constant, we can fold irrespective of indices. |
| 1838 | auto splatAttr = |
| 1839 | dyn_cast_or_null<SplatElementsAttr>(Val: global.getConstantInitValue()); |
| 1840 | if (!splatAttr) |
| 1841 | return {}; |
| 1842 | |
| 1843 | return splatAttr.getSplatValue<Attribute>(); |
| 1844 | } |
| 1845 | |
| 1846 | TypedValue<MemRefType> LoadOp::getAccessedMemref() { return getMemref(); } |
| 1847 | |
| 1848 | std::optional<SmallVector<Value>> |
| 1849 | LoadOp::updateMemrefAndIndices(RewriterBase &rewriter, Value newMemref, |
| 1850 | ValueRange newIndices) { |
| 1851 | rewriter.modifyOpInPlace(root: *this, callable: [&]() { |
| 1852 | getMemrefMutable().assign(value: newMemref); |
| 1853 | getIndicesMutable().assign(values: newIndices); |
| 1854 | }); |
| 1855 | return std::nullopt; |
| 1856 | } |
| 1857 | |
| 1858 | FailureOr<std::optional<SmallVector<Value>>> |
| 1859 | LoadOp::bubbleDownCasts(OpBuilder &builder) { |
| 1860 | return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getMemrefMutable(), |
| 1861 | results: getResult()); |
| 1862 | } |
| 1863 | |
| 1864 | //===----------------------------------------------------------------------===// |
| 1865 | // MemorySpaceCastOp |
| 1866 | //===----------------------------------------------------------------------===// |
| 1867 | |
| 1868 | void MemorySpaceCastOp::getAsmResultNames( |
| 1869 | function_ref<void(Value, StringRef)> setNameFn) { |
| 1870 | setNameFn(getResult(), "memspacecast" ); |
| 1871 | } |
| 1872 | |
| 1873 | bool MemorySpaceCastOp::areCastCompatible(TypeRange inputs, TypeRange outputs) { |
| 1874 | if (inputs.size() != 1 || outputs.size() != 1) |
| 1875 | return false; |
| 1876 | Type a = inputs.front(), b = outputs.front(); |
| 1877 | auto aT = llvm::dyn_cast<MemRefType>(Val&: a); |
| 1878 | auto bT = llvm::dyn_cast<MemRefType>(Val&: b); |
| 1879 | |
| 1880 | auto uaT = llvm::dyn_cast<UnrankedMemRefType>(Val&: a); |
| 1881 | auto ubT = llvm::dyn_cast<UnrankedMemRefType>(Val&: b); |
| 1882 | |
| 1883 | if (aT && bT) { |
| 1884 | if (aT.getElementType() != bT.getElementType()) |
| 1885 | return false; |
| 1886 | if (aT.getLayout() != bT.getLayout()) |
| 1887 | return false; |
| 1888 | if (aT.getShape() != bT.getShape()) |
| 1889 | return false; |
| 1890 | return true; |
| 1891 | } |
| 1892 | if (uaT && ubT) { |
| 1893 | return uaT.getElementType() == ubT.getElementType(); |
| 1894 | } |
| 1895 | return false; |
| 1896 | } |
| 1897 | |
| 1898 | OpFoldResult MemorySpaceCastOp::fold(FoldAdaptor adaptor) { |
| 1899 | // memory_space_cast(memory_space_cast(v, t1), t2) -> memory_space_cast(v, |
| 1900 | // t2) |
| 1901 | if (auto parentCast = getSource().getDefiningOp<MemorySpaceCastOp>()) { |
| 1902 | getSourceMutable().assign(value: parentCast.getSource()); |
| 1903 | return getResult(); |
| 1904 | } |
| 1905 | return Value{}; |
| 1906 | } |
| 1907 | |
| 1908 | TypedValue<PtrLikeTypeInterface> MemorySpaceCastOp::getSourcePtr() { |
| 1909 | return getSource(); |
| 1910 | } |
| 1911 | |
| 1912 | TypedValue<PtrLikeTypeInterface> MemorySpaceCastOp::getTargetPtr() { |
| 1913 | return getDest(); |
| 1914 | } |
| 1915 | |
| 1916 | bool MemorySpaceCastOp::isValidMemorySpaceCast(PtrLikeTypeInterface tgt, |
| 1917 | PtrLikeTypeInterface src) { |
| 1918 | return isa<BaseMemRefType>(Val: tgt) && |
| 1919 | tgt.clonePtrWith(memorySpace: src.getMemorySpace(), elementType: std::nullopt) == src; |
| 1920 | } |
| 1921 | |
| 1922 | MemorySpaceCastOpInterface MemorySpaceCastOp::cloneMemorySpaceCastOp( |
| 1923 | OpBuilder &b, PtrLikeTypeInterface tgt, |
| 1924 | TypedValue<PtrLikeTypeInterface> src) { |
| 1925 | assert(isValidMemorySpaceCast(tgt, src.getType()) && "invalid arguments" ); |
| 1926 | return MemorySpaceCastOp::create(builder&: b, location: getLoc(), dest: tgt, source: src); |
| 1927 | } |
| 1928 | |
| 1929 | /// The only cast we recognize as promotable is to the generic space. |
| 1930 | bool MemorySpaceCastOp::isSourcePromotable() { |
| 1931 | return getDest().getType().getMemorySpace() == nullptr; |
| 1932 | } |
| 1933 | |
| 1934 | //===----------------------------------------------------------------------===// |
| 1935 | // PrefetchOp |
| 1936 | //===----------------------------------------------------------------------===// |
| 1937 | |
| 1938 | void PrefetchOp::print(OpAsmPrinter &p) { |
| 1939 | p << " " << getMemref() << '['; |
| 1940 | p.printOperands(operands: getIndices()); |
| 1941 | p << ']' << ", " << (getIsWrite() ? "write" : "read" ); |
| 1942 | p << ", locality<" << getLocalityHint(); |
| 1943 | p << ">, " << (getIsDataCache() ? "data" : "instr" ); |
| 1944 | p.printOptionalAttrDict( |
| 1945 | attrs: (*this)->getDiscardableAttrDictionary(), |
| 1946 | /*elidedAttrs=*/{"localityHint" , "isWrite" , "isDataCache" }); |
| 1947 | p << " : " << getMemRefType(); |
| 1948 | } |
| 1949 | |
| 1950 | ParseResult PrefetchOp::parse(OpAsmParser &parser, OperationState &result) { |
| 1951 | OpAsmParser::UnresolvedOperand memrefInfo; |
| 1952 | SmallVector<OpAsmParser::UnresolvedOperand, 4> indexInfo; |
| 1953 | IntegerAttr localityHint; |
| 1954 | MemRefType type; |
| 1955 | StringRef readOrWrite, cacheType; |
| 1956 | |
| 1957 | auto indexTy = parser.getBuilder().getIndexType(); |
| 1958 | auto i32Type = parser.getBuilder().getIntegerType(width: 32); |
| 1959 | if (parser.parseOperand(result&: memrefInfo) || |
| 1960 | parser.parseOperandList(result&: indexInfo, delimiter: OpAsmParser::Delimiter::Square) || |
| 1961 | parser.parseComma() || parser.parseKeyword(keyword: &readOrWrite) || |
| 1962 | parser.parseComma() || parser.parseKeyword(keyword: "locality" ) || |
| 1963 | parser.parseLess() || |
| 1964 | parser.parseAttribute(result&: localityHint, type: i32Type, attrName: "localityHint" , |
| 1965 | attrs&: result.attributes) || |
| 1966 | parser.parseGreater() || parser.parseComma() || |
| 1967 | parser.parseKeyword(keyword: &cacheType) || parser.parseColonType(result&: type) || |
| 1968 | parser.resolveOperand(operand: memrefInfo, type, result&: result.operands) || |
| 1969 | parser.resolveOperands(operands&: indexInfo, type: indexTy, result&: result.operands)) |
| 1970 | return failure(); |
| 1971 | |
| 1972 | if (readOrWrite != "read" && readOrWrite != "write" ) |
| 1973 | return parser.emitError(loc: parser.getNameLoc(), |
| 1974 | message: "rw specifier has to be 'read' or 'write'" ); |
| 1975 | result.addAttribute(name: PrefetchOp::getIsWriteAttrStrName(), |
| 1976 | attr: parser.getBuilder().getBoolAttr(value: readOrWrite == "write" )); |
| 1977 | |
| 1978 | if (cacheType != "data" && cacheType != "instr" ) |
| 1979 | return parser.emitError(loc: parser.getNameLoc(), |
| 1980 | message: "cache type has to be 'data' or 'instr'" ); |
| 1981 | |
| 1982 | result.addAttribute(name: PrefetchOp::getIsDataCacheAttrStrName(), |
| 1983 | attr: parser.getBuilder().getBoolAttr(value: cacheType == "data" )); |
| 1984 | |
| 1985 | return success(); |
| 1986 | } |
| 1987 | |
| 1988 | LogicalResult PrefetchOp::verify() { |
| 1989 | if (getNumOperands() != 1 + getMemRefType().getRank()) |
| 1990 | return emitOpError(message: "too few indices" ); |
| 1991 | |
| 1992 | return success(); |
| 1993 | } |
| 1994 | |
| 1995 | LogicalResult PrefetchOp::fold(FoldAdaptor adaptor, |
| 1996 | SmallVectorImpl<OpFoldResult> &results) { |
| 1997 | // prefetch(memrefcast) -> prefetch |
| 1998 | return foldMemRefCast(op: *this); |
| 1999 | } |
| 2000 | |
| 2001 | TypedValue<MemRefType> PrefetchOp::getAccessedMemref() { return getMemref(); } |
| 2002 | |
| 2003 | std::optional<SmallVector<Value>> |
| 2004 | PrefetchOp::updateMemrefAndIndices(RewriterBase &rewriter, Value newMemref, |
| 2005 | ValueRange newIndices) { |
| 2006 | rewriter.modifyOpInPlace(root: *this, callable: [&]() { |
| 2007 | getMemrefMutable().assign(value: newMemref); |
| 2008 | getIndicesMutable().assign(values: newIndices); |
| 2009 | }); |
| 2010 | return std::nullopt; |
| 2011 | } |
| 2012 | |
| 2013 | //===----------------------------------------------------------------------===// |
| 2014 | // RankOp |
| 2015 | //===----------------------------------------------------------------------===// |
| 2016 | |
| 2017 | OpFoldResult RankOp::fold(FoldAdaptor adaptor) { |
| 2018 | // Constant fold rank when the rank of the operand is known. |
| 2019 | auto type = getOperand().getType(); |
| 2020 | auto shapedType = llvm::dyn_cast<ShapedType>(Val&: type); |
| 2021 | if (shapedType && shapedType.hasRank()) |
| 2022 | return IntegerAttr::get(type: IndexType::get(context: getContext()), value: shapedType.getRank()); |
| 2023 | return IntegerAttr(); |
| 2024 | } |
| 2025 | |
| 2026 | //===----------------------------------------------------------------------===// |
| 2027 | // ReinterpretCastOp |
| 2028 | //===----------------------------------------------------------------------===// |
| 2029 | |
| 2030 | namespace { |
| 2031 | |
| 2032 | struct PrintDynamicOrValue { |
| 2033 | int64_t value; |
| 2034 | }; |
| 2035 | |
| 2036 | Diagnostic &operator<<(Diagnostic &diag, PrintDynamicOrValue printed) { |
| 2037 | if (ShapedType::isDynamic(dValue: printed.value)) |
| 2038 | return diag << "dynamic" ; |
| 2039 | return diag << printed.value; |
| 2040 | } |
| 2041 | |
| 2042 | } // namespace |
| 2043 | |
| 2044 | void ReinterpretCastOp::getAsmResultNames( |
| 2045 | function_ref<void(Value, StringRef)> setNameFn) { |
| 2046 | setNameFn(getResult(), "reinterpret_cast" ); |
| 2047 | } |
| 2048 | |
| 2049 | /// Build a ReinterpretCastOp with all dynamic entries: `staticOffsets`, |
| 2050 | /// `staticSizes` and `staticStrides` are automatically filled with |
| 2051 | /// source-memref-rank sentinel values that encode dynamic entries. |
| 2052 | void ReinterpretCastOp::build(OpBuilder &b, OperationState &result, |
| 2053 | MemRefType resultType, Value source, |
| 2054 | OpFoldResult offset, ArrayRef<OpFoldResult> sizes, |
| 2055 | ArrayRef<OpFoldResult> strides, |
| 2056 | ArrayRef<NamedAttribute> attrs) { |
| 2057 | SmallVector<int64_t> staticOffsets, staticSizes, staticStrides; |
| 2058 | SmallVector<Value> dynamicOffsets, dynamicSizes, dynamicStrides; |
| 2059 | dispatchIndexOpFoldResults(ofrs: offset, dynamicVec&: dynamicOffsets, staticVec&: staticOffsets); |
| 2060 | dispatchIndexOpFoldResults(ofrs: sizes, dynamicVec&: dynamicSizes, staticVec&: staticSizes); |
| 2061 | dispatchIndexOpFoldResults(ofrs: strides, dynamicVec&: dynamicStrides, staticVec&: staticStrides); |
| 2062 | result.addAttributes(newAttributes: attrs); |
| 2063 | build(odsBuilder&: b, odsState&: result, result: resultType, source, offsets: dynamicOffsets, sizes: dynamicSizes, |
| 2064 | strides: dynamicStrides, static_offsets: b.getDenseI64ArrayAttr(values: staticOffsets), |
| 2065 | static_sizes: b.getDenseI64ArrayAttr(values: staticSizes), |
| 2066 | static_strides: b.getDenseI64ArrayAttr(values: staticStrides)); |
| 2067 | } |
| 2068 | |
| 2069 | void ReinterpretCastOp::build(OpBuilder &b, OperationState &result, |
| 2070 | Value source, OpFoldResult offset, |
| 2071 | ArrayRef<OpFoldResult> sizes, |
| 2072 | ArrayRef<OpFoldResult> strides, |
| 2073 | ArrayRef<NamedAttribute> attrs) { |
| 2074 | auto sourceType = cast<BaseMemRefType>(Val: source.getType()); |
| 2075 | SmallVector<int64_t> staticOffsets, staticSizes, staticStrides; |
| 2076 | SmallVector<Value> dynamicOffsets, dynamicSizes, dynamicStrides; |
| 2077 | dispatchIndexOpFoldResults(ofrs: offset, dynamicVec&: dynamicOffsets, staticVec&: staticOffsets); |
| 2078 | dispatchIndexOpFoldResults(ofrs: sizes, dynamicVec&: dynamicSizes, staticVec&: staticSizes); |
| 2079 | dispatchIndexOpFoldResults(ofrs: strides, dynamicVec&: dynamicStrides, staticVec&: staticStrides); |
| 2080 | auto stridedLayout = StridedLayoutAttr::get( |
| 2081 | context: b.getContext(), offset: staticOffsets.front(), strides: staticStrides); |
| 2082 | auto resultType = MemRefType::get(shape: staticSizes, elementType: sourceType.getElementType(), |
| 2083 | layout: stridedLayout, memorySpace: sourceType.getMemorySpace()); |
| 2084 | build(b, result, resultType, source, offset, sizes, strides, attrs); |
| 2085 | } |
| 2086 | |
| 2087 | void ReinterpretCastOp::build(OpBuilder &b, OperationState &result, |
| 2088 | MemRefType resultType, Value source, |
| 2089 | int64_t offset, ArrayRef<int64_t> sizes, |
| 2090 | ArrayRef<int64_t> strides, |
| 2091 | ArrayRef<NamedAttribute> attrs) { |
| 2092 | SmallVector<OpFoldResult> sizeValues = llvm::map_to_vector<4>( |
| 2093 | C&: sizes, F: [&](int64_t v) -> OpFoldResult { return b.getI64IntegerAttr(value: v); }); |
| 2094 | SmallVector<OpFoldResult> strideValues = |
| 2095 | llvm::map_to_vector<4>(C&: strides, F: [&](int64_t v) -> OpFoldResult { |
| 2096 | return b.getI64IntegerAttr(value: v); |
| 2097 | }); |
| 2098 | build(b, result, resultType, source, offset: b.getI64IntegerAttr(value: offset), sizes: sizeValues, |
| 2099 | strides: strideValues, attrs); |
| 2100 | } |
| 2101 | |
| 2102 | void ReinterpretCastOp::build(OpBuilder &b, OperationState &result, |
| 2103 | MemRefType resultType, Value source, Value offset, |
| 2104 | ValueRange sizes, ValueRange strides, |
| 2105 | ArrayRef<NamedAttribute> attrs) { |
| 2106 | SmallVector<OpFoldResult> sizeValues = |
| 2107 | llvm::map_to_vector<4>(C&: sizes, F: [](Value v) -> OpFoldResult { return v; }); |
| 2108 | SmallVector<OpFoldResult> strideValues = llvm::map_to_vector<4>( |
| 2109 | C&: strides, F: [](Value v) -> OpFoldResult { return v; }); |
| 2110 | build(b, result, resultType, source, offset, sizes: sizeValues, strides: strideValues, attrs); |
| 2111 | } |
| 2112 | |
| 2113 | // TODO: ponder whether we want to allow missing trailing sizes/strides that are |
| 2114 | // completed automatically, like we have for subview and extract_slice. |
| 2115 | LogicalResult ReinterpretCastOp::verify() { |
| 2116 | // The source and result memrefs should be in the same memory space. |
| 2117 | auto srcType = llvm::cast<BaseMemRefType>(Val: getSource().getType()); |
| 2118 | auto resultType = llvm::cast<MemRefType>(Val: getType()); |
| 2119 | if (srcType.getMemorySpace() != resultType.getMemorySpace()) |
| 2120 | return emitError(message: "different memory spaces specified for source type " ) |
| 2121 | << srcType << " and result memref type " << resultType; |
| 2122 | if (failed(Result: verifyElementTypesMatch(op: *this, lhs: srcType, rhs: resultType, lhsName: "source" , |
| 2123 | rhsName: "result" ))) |
| 2124 | return failure(); |
| 2125 | |
| 2126 | // Match sizes in result memref type and in static_sizes attribute. |
| 2127 | for (auto [idx, resultSize, expectedSize] : |
| 2128 | llvm::enumerate(First: resultType.getShape(), Rest: getStaticSizes())) { |
| 2129 | if (resultSize != expectedSize) |
| 2130 | return emitError(message: "expected result type with size = " ) |
| 2131 | << PrintDynamicOrValue{.value: expectedSize} << " instead of " |
| 2132 | << PrintDynamicOrValue{.value: resultSize} << " in dim = " << idx; |
| 2133 | } |
| 2134 | |
| 2135 | // Match offset and strides in static_offset and static_strides attributes. If |
| 2136 | // result memref type has no affine map specified, this will assume an |
| 2137 | // identity layout. |
| 2138 | int64_t resultOffset; |
| 2139 | SmallVector<int64_t, 4> resultStrides; |
| 2140 | if (failed(Result: resultType.getStridesAndOffset(strides&: resultStrides, offset&: resultOffset))) |
| 2141 | return emitError(message: "expected result type to have strided layout but found " ) |
| 2142 | << resultType; |
| 2143 | |
| 2144 | // Match offset in result memref type and in static_offsets attribute. |
| 2145 | int64_t expectedOffset = getStaticOffsets().front(); |
| 2146 | if (resultOffset != expectedOffset) |
| 2147 | return emitError(message: "expected result type with offset = " ) |
| 2148 | << PrintDynamicOrValue{.value: expectedOffset} << " instead of " |
| 2149 | << PrintDynamicOrValue{.value: resultOffset}; |
| 2150 | |
| 2151 | // Match strides in result memref type and in static_strides attribute. |
| 2152 | for (auto [idx, resultStride, expectedStride] : |
| 2153 | llvm::enumerate(First&: resultStrides, Rest: getStaticStrides())) { |
| 2154 | if (resultStride != expectedStride) |
| 2155 | return emitError(message: "expected result type with stride = " ) |
| 2156 | << PrintDynamicOrValue{.value: expectedStride} << " instead of " |
| 2157 | << PrintDynamicOrValue{.value: resultStride} << " in dim = " << idx; |
| 2158 | } |
| 2159 | |
| 2160 | return success(); |
| 2161 | } |
| 2162 | |
| 2163 | OpFoldResult ReinterpretCastOp::fold(FoldAdaptor /*operands*/) { |
| 2164 | Value src = getSource(); |
| 2165 | auto getPrevSrc = [&]() -> Value { |
| 2166 | // reinterpret_cast(reinterpret_cast(x)) -> reinterpret_cast(x). |
| 2167 | if (auto prev = src.getDefiningOp<ReinterpretCastOp>()) |
| 2168 | return prev.getSource(); |
| 2169 | |
| 2170 | // reinterpret_cast(cast(x)) -> reinterpret_cast(x). |
| 2171 | if (auto prev = src.getDefiningOp<CastOp>()) |
| 2172 | return prev.getSource(); |
| 2173 | |
| 2174 | // reinterpret_cast(subview(x)) -> reinterpret_cast(x) if subview offsets |
| 2175 | // are 0. |
| 2176 | if (auto prev = src.getDefiningOp<SubViewOp>()) |
| 2177 | if (llvm::all_of(Range: prev.getMixedOffsets(), P: isZeroInteger)) |
| 2178 | return prev.getSource(); |
| 2179 | |
| 2180 | return nullptr; |
| 2181 | }; |
| 2182 | |
| 2183 | if (auto prevSrc = getPrevSrc()) { |
| 2184 | getSourceMutable().assign(value: prevSrc); |
| 2185 | return getResult(); |
| 2186 | } |
| 2187 | |
| 2188 | // reinterpret_cast(x) w/o offset/shape/stride changes -> x |
| 2189 | if (ShapedType::isStaticShape(dSizes: getType().getShape()) && |
| 2190 | src.getType() == getType() && getStaticOffsets().front() == 0) { |
| 2191 | return src; |
| 2192 | } |
| 2193 | |
| 2194 | return nullptr; |
| 2195 | } |
| 2196 | |
| 2197 | SmallVector<OpFoldResult> ReinterpretCastOp::getConstifiedMixedSizes() { |
| 2198 | SmallVector<OpFoldResult> values = getMixedSizes(); |
| 2199 | constifyIndexValues(values, constValues: getType().getShape()); |
| 2200 | return values; |
| 2201 | } |
| 2202 | |
| 2203 | SmallVector<OpFoldResult> ReinterpretCastOp::getConstifiedMixedStrides() { |
| 2204 | SmallVector<OpFoldResult> values = getMixedStrides(); |
| 2205 | SmallVector<int64_t> staticValues; |
| 2206 | int64_t unused; |
| 2207 | LogicalResult status = getType().getStridesAndOffset(strides&: staticValues, offset&: unused); |
| 2208 | (void)status; |
| 2209 | assert(succeeded(status) && "could not get strides from type" ); |
| 2210 | constifyIndexValues(values, constValues: staticValues); |
| 2211 | return values; |
| 2212 | } |
| 2213 | |
| 2214 | OpFoldResult ReinterpretCastOp::getConstifiedMixedOffset() { |
| 2215 | SmallVector<OpFoldResult> values = getMixedOffsets(); |
| 2216 | assert(values.size() == 1 && |
| 2217 | "reinterpret_cast must have one and only one offset" ); |
| 2218 | SmallVector<int64_t> staticValues, unused; |
| 2219 | int64_t offset; |
| 2220 | LogicalResult status = getType().getStridesAndOffset(strides&: unused, offset); |
| 2221 | (void)status; |
| 2222 | assert(succeeded(status) && "could not get offset from type" ); |
| 2223 | staticValues.push_back(Elt: offset); |
| 2224 | constifyIndexValues(values, constValues: staticValues); |
| 2225 | return values[0]; |
| 2226 | } |
| 2227 | |
| 2228 | namespace { |
| 2229 | /// Replace the sequence: |
| 2230 | /// ``` |
| 2231 | /// base, offset, sizes, strides = extract_strided_metadata src |
| 2232 | /// dst = reinterpret_cast base to offset, sizes, strides |
| 2233 | /// ``` |
| 2234 | /// With |
| 2235 | /// |
| 2236 | /// ``` |
| 2237 | /// dst = memref.cast src |
| 2238 | /// ``` |
| 2239 | /// |
| 2240 | /// Note: The cast operation is only inserted when the type of dst and src |
| 2241 | /// are not the same. E.g., when going from <4xf32> to <?xf32>. |
| 2242 | /// |
| 2243 | /// This pattern also matches when the offset, sizes, and strides don't come |
| 2244 | /// directly from the `extract_strided_metadata`'s results but it can be |
| 2245 | /// statically proven that they would hold the same values. |
| 2246 | /// |
| 2247 | /// For instance, the following sequence would be replaced: |
| 2248 | /// ``` |
| 2249 | /// base, offset, sizes, strides = |
| 2250 | /// extract_strided_metadata memref : memref<3x4xty> |
| 2251 | /// dst = reinterpret_cast base to 0, [3, 4], strides |
| 2252 | /// ``` |
| 2253 | /// Because we know (thanks to the type of the input memref) that variable |
| 2254 | /// `offset` and `sizes` will respectively hold 0 and [3, 4]. |
| 2255 | /// |
| 2256 | /// Similarly, the following sequence would be replaced: |
| 2257 | /// ``` |
| 2258 | /// c0 = arith.constant 0 |
| 2259 | /// c4 = arith.constant 4 |
| 2260 | /// base, offset, sizes, strides = |
| 2261 | /// extract_strided_metadata memref : memref<3x4xty> |
| 2262 | /// dst = reinterpret_cast base to c0, [3, c4], strides |
| 2263 | /// ``` |
| 2264 | /// Because we know that `offset`and `c0` will hold 0 |
| 2265 | /// and `c4` will hold 4. |
| 2266 | /// |
| 2267 | /// If the pattern above does not match, the input of the |
| 2268 | /// extract_strided_metadata is always folded into the input of the |
| 2269 | /// reinterpret_cast operator. This allows for dead code elimination to get rid |
| 2270 | /// of the extract_strided_metadata in some cases. |
| 2271 | struct |
| 2272 | : public OpRewritePattern<ReinterpretCastOp> { |
| 2273 | public: |
| 2274 | using OpRewritePattern<ReinterpretCastOp>::OpRewritePattern; |
| 2275 | |
| 2276 | LogicalResult matchAndRewrite(ReinterpretCastOp op, |
| 2277 | PatternRewriter &rewriter) const override { |
| 2278 | auto = |
| 2279 | op.getSource().getDefiningOp<ExtractStridedMetadataOp>(); |
| 2280 | if (!extractStridedMetadata) |
| 2281 | return failure(); |
| 2282 | |
| 2283 | // Check if the reinterpret cast reconstructs a memref with the exact same |
| 2284 | // properties as the extract strided metadata. |
| 2285 | auto isReinterpretCastNoop = [&]() -> bool { |
| 2286 | // First, check that the strides are the same. |
| 2287 | if (!llvm::equal(LRange: extractStridedMetadata.getConstifiedMixedStrides(), |
| 2288 | RRange: op.getConstifiedMixedStrides())) |
| 2289 | return false; |
| 2290 | |
| 2291 | // Second, check the sizes. |
| 2292 | if (!llvm::equal(LRange: extractStridedMetadata.getConstifiedMixedSizes(), |
| 2293 | RRange: op.getConstifiedMixedSizes())) |
| 2294 | return false; |
| 2295 | |
| 2296 | // Finally, check the offset. |
| 2297 | assert(op.getMixedOffsets().size() == 1 && |
| 2298 | "reinterpret_cast with more than one offset should have been " |
| 2299 | "rejected by the verifier" ); |
| 2300 | return extractStridedMetadata.getConstifiedMixedOffset() == |
| 2301 | op.getConstifiedMixedOffset(); |
| 2302 | }; |
| 2303 | |
| 2304 | if (!isReinterpretCastNoop()) { |
| 2305 | // If the extract_strided_metadata / reinterpret_cast pair can't be |
| 2306 | // completely folded, then we could fold the input of the |
| 2307 | // extract_strided_metadata into the input of the reinterpret_cast |
| 2308 | // input. For some cases (e.g., static dimensions) the |
| 2309 | // the extract_strided_metadata is eliminated by dead code elimination. |
| 2310 | // |
| 2311 | // reinterpret_cast(extract_strided_metadata(x)) -> reinterpret_cast(x). |
| 2312 | // |
| 2313 | // We can always fold the input of a extract_strided_metadata operator |
| 2314 | // to the input of a reinterpret_cast operator, because they point to |
| 2315 | // the same memory. Note that the reinterpret_cast does not use the |
| 2316 | // layout of its input memref, only its base memory pointer which is |
| 2317 | // the same as the base pointer returned by the extract_strided_metadata |
| 2318 | // operator and the base pointer of the extract_strided_metadata memref |
| 2319 | // input. |
| 2320 | rewriter.modifyOpInPlace(root: op, callable: [&]() { |
| 2321 | op.getSourceMutable().assign(value: extractStridedMetadata.getSource()); |
| 2322 | }); |
| 2323 | return success(); |
| 2324 | } |
| 2325 | |
| 2326 | // At this point, we know that the back and forth between extract strided |
| 2327 | // metadata and reinterpret cast is a noop. However, the final type of the |
| 2328 | // reinterpret cast may not be exactly the same as the original memref. |
| 2329 | // E.g., it could be changing a dimension from static to dynamic. Check that |
| 2330 | // here and add a cast if necessary. |
| 2331 | Type srcTy = extractStridedMetadata.getSource().getType(); |
| 2332 | if (srcTy == op.getResult().getType()) |
| 2333 | rewriter.replaceOp(op, newValues: extractStridedMetadata.getSource()); |
| 2334 | else |
| 2335 | rewriter.replaceOpWithNewOp<CastOp>(op, args: op.getType(), |
| 2336 | args: extractStridedMetadata.getSource()); |
| 2337 | |
| 2338 | return success(); |
| 2339 | } |
| 2340 | }; |
| 2341 | |
| 2342 | struct ReinterpretCastOpConstantFolder |
| 2343 | : public OpRewritePattern<ReinterpretCastOp> { |
| 2344 | public: |
| 2345 | using OpRewritePattern<ReinterpretCastOp>::OpRewritePattern; |
| 2346 | |
| 2347 | LogicalResult matchAndRewrite(ReinterpretCastOp op, |
| 2348 | PatternRewriter &rewriter) const override { |
| 2349 | auto srcStaticCount = llvm::count_if( |
| 2350 | Range: llvm::concat<OpFoldResult>(Ranges: op.getMixedOffsets(), Ranges: op.getMixedSizes(), |
| 2351 | Ranges: op.getMixedStrides()), |
| 2352 | P: [](OpFoldResult ofr) { return isa<Attribute>(Val: ofr); }); |
| 2353 | |
| 2354 | SmallVector<OpFoldResult> offsets = {op.getConstifiedMixedOffset()}; |
| 2355 | SmallVector<OpFoldResult> sizes = op.getConstifiedMixedSizes(); |
| 2356 | SmallVector<OpFoldResult> strides = op.getConstifiedMixedStrides(); |
| 2357 | |
| 2358 | // If the offset is a negative constant, we can't fold it because the |
| 2359 | // resulting memref type would be invalid. In that case, we keep the |
| 2360 | // original offset. |
| 2361 | if (auto cst = getConstantIntValue(ofr: offsets[0])) |
| 2362 | if (*cst < 0) |
| 2363 | offsets[0] = op.getMixedOffsets()[0]; |
| 2364 | |
| 2365 | // If the size is a negative constant, we can't fold it because the |
| 2366 | // resulting memref type would be invalid. In that case, we keep the |
| 2367 | // original size. |
| 2368 | for (auto it : llvm::zip(t: op.getMixedSizes(), u&: sizes)) { |
| 2369 | auto &srcSizeOfr = std::get<0>(t&: it); |
| 2370 | auto &sizeOfr = std::get<1>(t&: it); |
| 2371 | if (auto cst = getConstantIntValue(ofr: sizeOfr)) |
| 2372 | if (*cst < 0) |
| 2373 | sizeOfr = srcSizeOfr; |
| 2374 | } |
| 2375 | |
| 2376 | // TODO: Using counting comparison instead of direct comparison because |
| 2377 | // getMixedValues (and therefore ReinterpretCastOp::getMixed...) returns |
| 2378 | // IntegerAttrs, while constifyIndexValues (and therefore |
| 2379 | // ReinterpretCastOp::getConstifiedMixed...) returns IndexAttrs. |
| 2380 | if (srcStaticCount == |
| 2381 | llvm::count_if(Range: llvm::concat<OpFoldResult>(Ranges&: offsets, Ranges&: sizes, Ranges&: strides), |
| 2382 | P: [](OpFoldResult ofr) { return isa<Attribute>(Val: ofr); })) |
| 2383 | return failure(); |
| 2384 | |
| 2385 | auto newReinterpretCast = ReinterpretCastOp::create( |
| 2386 | builder&: rewriter, location: op->getLoc(), source: op.getSource(), offset: offsets[0], sizes, strides); |
| 2387 | |
| 2388 | rewriter.replaceOpWithNewOp<CastOp>(op, args: op.getType(), args&: newReinterpretCast); |
| 2389 | return success(); |
| 2390 | } |
| 2391 | }; |
| 2392 | } // namespace |
| 2393 | |
| 2394 | void ReinterpretCastOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 2395 | MLIRContext *context) { |
| 2396 | results.add<ReinterpretCastOpExtractStridedMetadataFolder, |
| 2397 | ReinterpretCastOpConstantFolder>(arg&: context); |
| 2398 | } |
| 2399 | |
| 2400 | FailureOr<std::optional<SmallVector<Value>>> |
| 2401 | ReinterpretCastOp::bubbleDownCasts(OpBuilder &builder) { |
| 2402 | return bubbleDownCastsPassthroughOpImpl(op: *this, builder, src&: getSourceMutable()); |
| 2403 | } |
| 2404 | |
| 2405 | //===----------------------------------------------------------------------===// |
| 2406 | // Reassociative reshape ops |
| 2407 | //===----------------------------------------------------------------------===// |
| 2408 | |
| 2409 | void CollapseShapeOp::getAsmResultNames( |
| 2410 | function_ref<void(Value, StringRef)> setNameFn) { |
| 2411 | setNameFn(getResult(), "collapse_shape" ); |
| 2412 | } |
| 2413 | |
| 2414 | void ExpandShapeOp::getAsmResultNames( |
| 2415 | function_ref<void(Value, StringRef)> setNameFn) { |
| 2416 | setNameFn(getResult(), "expand_shape" ); |
| 2417 | } |
| 2418 | |
| 2419 | LogicalResult ExpandShapeOp::reifyResultShapes( |
| 2420 | OpBuilder &builder, ReifiedRankedShapedTypeDims &reifiedResultShapes) { |
| 2421 | reifiedResultShapes = { |
| 2422 | getMixedValues(staticValues: getStaticOutputShape(), dynamicValues: getOutputShape(), b&: builder)}; |
| 2423 | return success(); |
| 2424 | } |
| 2425 | |
| 2426 | /// Helper function for verifying the shape of ExpandShapeOp and ResultShapeOp |
| 2427 | /// result and operand. Layout maps are verified separately. |
| 2428 | /// |
| 2429 | /// If `allowMultipleDynamicDimsPerGroup`, multiple dynamic dimensions are |
| 2430 | /// allowed in a reassocation group. |
| 2431 | static LogicalResult |
| 2432 | verifyCollapsedShape(Operation *op, ArrayRef<int64_t> collapsedShape, |
| 2433 | ArrayRef<int64_t> expandedShape, |
| 2434 | ArrayRef<ReassociationIndices> reassociation, |
| 2435 | bool allowMultipleDynamicDimsPerGroup) { |
| 2436 | // There must be one reassociation group per collapsed dimension. |
| 2437 | if (collapsedShape.size() != reassociation.size()) |
| 2438 | return op->emitOpError(message: "invalid number of reassociation groups: found " ) |
| 2439 | << reassociation.size() << ", expected " << collapsedShape.size(); |
| 2440 | |
| 2441 | // The next expected expanded dimension index (while iterating over |
| 2442 | // reassociation indices). |
| 2443 | int64_t nextDim = 0; |
| 2444 | for (const auto &it : llvm::enumerate(First&: reassociation)) { |
| 2445 | ReassociationIndices group = it.value(); |
| 2446 | int64_t collapsedDim = it.index(); |
| 2447 | |
| 2448 | bool foundDynamic = false; |
| 2449 | for (int64_t expandedDim : group) { |
| 2450 | if (expandedDim != nextDim++) |
| 2451 | return op->emitOpError(message: "reassociation indices must be contiguous" ); |
| 2452 | |
| 2453 | if (expandedDim >= static_cast<int64_t>(expandedShape.size())) |
| 2454 | return op->emitOpError(message: "reassociation index " ) |
| 2455 | << expandedDim << " is out of bounds" ; |
| 2456 | |
| 2457 | // Check if there are multiple dynamic dims in a reassociation group. |
| 2458 | if (ShapedType::isDynamic(dValue: expandedShape[expandedDim])) { |
| 2459 | if (foundDynamic && !allowMultipleDynamicDimsPerGroup) |
| 2460 | return op->emitOpError( |
| 2461 | message: "at most one dimension in a reassociation group may be dynamic" ); |
| 2462 | foundDynamic = true; |
| 2463 | } |
| 2464 | } |
| 2465 | |
| 2466 | // ExpandShapeOp/CollapseShapeOp may not be used to cast dynamicity. |
| 2467 | if (ShapedType::isDynamic(dValue: collapsedShape[collapsedDim]) != foundDynamic) |
| 2468 | return op->emitOpError(message: "collapsed dim (" ) |
| 2469 | << collapsedDim |
| 2470 | << ") must be dynamic if and only if reassociation group is " |
| 2471 | "dynamic" ; |
| 2472 | |
| 2473 | // If all dims in the reassociation group are static, the size of the |
| 2474 | // collapsed dim can be verified. |
| 2475 | if (!foundDynamic) { |
| 2476 | int64_t groupSize = 1; |
| 2477 | for (int64_t expandedDim : group) |
| 2478 | groupSize *= expandedShape[expandedDim]; |
| 2479 | if (groupSize != collapsedShape[collapsedDim]) |
| 2480 | return op->emitOpError(message: "collapsed dim size (" ) |
| 2481 | << collapsedShape[collapsedDim] |
| 2482 | << ") must equal reassociation group size (" << groupSize << ")" ; |
| 2483 | } |
| 2484 | } |
| 2485 | |
| 2486 | if (collapsedShape.empty()) { |
| 2487 | // Rank 0: All expanded dimensions must be 1. |
| 2488 | for (int64_t d : expandedShape) |
| 2489 | if (d != 1) |
| 2490 | return op->emitOpError( |
| 2491 | message: "rank 0 memrefs can only be extended/collapsed with/from ones" ); |
| 2492 | } else if (nextDim != static_cast<int64_t>(expandedShape.size())) { |
| 2493 | // Rank >= 1: Number of dimensions among all reassociation groups must match |
| 2494 | // the result memref rank. |
| 2495 | return op->emitOpError(message: "expanded rank (" ) |
| 2496 | << expandedShape.size() |
| 2497 | << ") inconsistent with number of reassociation indices (" << nextDim |
| 2498 | << ")" ; |
| 2499 | } |
| 2500 | |
| 2501 | return success(); |
| 2502 | } |
| 2503 | |
| 2504 | SmallVector<AffineMap, 4> CollapseShapeOp::getReassociationMaps() { |
| 2505 | return getSymbolLessAffineMaps(reassociation: getReassociationExprs()); |
| 2506 | } |
| 2507 | |
| 2508 | SmallVector<ReassociationExprs, 4> CollapseShapeOp::getReassociationExprs() { |
| 2509 | return convertReassociationIndicesToExprs(context: getContext(), |
| 2510 | reassociationIndices: getReassociationIndices()); |
| 2511 | } |
| 2512 | |
| 2513 | SmallVector<AffineMap, 4> ExpandShapeOp::getReassociationMaps() { |
| 2514 | return getSymbolLessAffineMaps(reassociation: getReassociationExprs()); |
| 2515 | } |
| 2516 | |
| 2517 | SmallVector<ReassociationExprs, 4> ExpandShapeOp::getReassociationExprs() { |
| 2518 | return convertReassociationIndicesToExprs(context: getContext(), |
| 2519 | reassociationIndices: getReassociationIndices()); |
| 2520 | } |
| 2521 | |
| 2522 | /// Compute the layout map after expanding a given source MemRef type with the |
| 2523 | /// specified reassociation indices. |
| 2524 | static FailureOr<StridedLayoutAttr> |
| 2525 | computeExpandedLayoutMap(MemRefType srcType, ArrayRef<int64_t> resultShape, |
| 2526 | ArrayRef<ReassociationIndices> reassociation) { |
| 2527 | int64_t srcOffset; |
| 2528 | SmallVector<int64_t> srcStrides; |
| 2529 | if (failed(Result: srcType.getStridesAndOffset(strides&: srcStrides, offset&: srcOffset))) |
| 2530 | return failure(); |
| 2531 | assert(srcStrides.size() == reassociation.size() && "invalid reassociation" ); |
| 2532 | |
| 2533 | // 1-1 mapping between srcStrides and reassociation packs. |
| 2534 | // Each srcStride starts with the given value and gets expanded according to |
| 2535 | // the proper entries in resultShape. |
| 2536 | // Example: |
| 2537 | // srcStrides = [10000, 1 , 100 ], |
| 2538 | // reassociations = [ [0], [1], [2, 3, 4]], |
| 2539 | // resultSizes = [2, 5, 4, 3, 2] = [ [2], [5], [4, 3, 2]] |
| 2540 | // -> For the purpose of stride calculation, the useful sizes are: |
| 2541 | // [x, x, x, 3, 2] = [ [x], [x], [x, 3, 2]]. |
| 2542 | // resultStrides = [10000, 1, 600, 200, 100] |
| 2543 | // Note that a stride does not get expanded along the first entry of each |
| 2544 | // shape pack. |
| 2545 | SmallVector<int64_t> reverseResultStrides; |
| 2546 | reverseResultStrides.reserve(N: resultShape.size()); |
| 2547 | unsigned shapeIndex = resultShape.size() - 1; |
| 2548 | for (auto it : llvm::reverse(C: llvm::zip(t&: reassociation, u&: srcStrides))) { |
| 2549 | ReassociationIndices reassoc = std::get<0>(t&: it); |
| 2550 | int64_t currentStrideToExpand = std::get<1>(t&: it); |
| 2551 | for (unsigned idx = 0, e = reassoc.size(); idx < e; ++idx) { |
| 2552 | reverseResultStrides.push_back(Elt: currentStrideToExpand); |
| 2553 | currentStrideToExpand = |
| 2554 | (SaturatedInteger::wrap(v: currentStrideToExpand) * |
| 2555 | SaturatedInteger::wrap(v: resultShape[shapeIndex--])) |
| 2556 | .asInteger(); |
| 2557 | } |
| 2558 | } |
| 2559 | auto resultStrides = llvm::to_vector<8>(Range: llvm::reverse(C&: reverseResultStrides)); |
| 2560 | resultStrides.resize(N: resultShape.size(), NV: 1); |
| 2561 | return StridedLayoutAttr::get(context: srcType.getContext(), offset: srcOffset, strides: resultStrides); |
| 2562 | } |
| 2563 | |
| 2564 | FailureOr<MemRefType> ExpandShapeOp::computeExpandedType( |
| 2565 | MemRefType srcType, ArrayRef<int64_t> resultShape, |
| 2566 | ArrayRef<ReassociationIndices> reassociation) { |
| 2567 | if (srcType.getLayout().isIdentity()) { |
| 2568 | // If the source is contiguous (i.e., no layout map specified), so is the |
| 2569 | // result. |
| 2570 | MemRefLayoutAttrInterface layout; |
| 2571 | return MemRefType::get(shape: resultShape, elementType: srcType.getElementType(), layout, |
| 2572 | memorySpace: srcType.getMemorySpace()); |
| 2573 | } |
| 2574 | |
| 2575 | // Source may not be contiguous. Compute the layout map. |
| 2576 | FailureOr<StridedLayoutAttr> computedLayout = |
| 2577 | computeExpandedLayoutMap(srcType, resultShape, reassociation); |
| 2578 | if (failed(Result: computedLayout)) |
| 2579 | return failure(); |
| 2580 | return MemRefType::get(shape: resultShape, elementType: srcType.getElementType(), layout: *computedLayout, |
| 2581 | memorySpace: srcType.getMemorySpace()); |
| 2582 | } |
| 2583 | |
| 2584 | FailureOr<SmallVector<OpFoldResult>> |
| 2585 | ExpandShapeOp::inferOutputShape(OpBuilder &b, Location loc, |
| 2586 | MemRefType expandedType, |
| 2587 | ArrayRef<ReassociationIndices> reassociation, |
| 2588 | ArrayRef<OpFoldResult> inputShape) { |
| 2589 | std::optional<SmallVector<OpFoldResult>> outputShape = |
| 2590 | inferExpandShapeOutputShape(b, loc, expandedType, reassociation, |
| 2591 | inputShape); |
| 2592 | if (!outputShape) |
| 2593 | return failure(); |
| 2594 | return *outputShape; |
| 2595 | } |
| 2596 | |
| 2597 | void ExpandShapeOp::build(OpBuilder &builder, OperationState &result, |
| 2598 | Type resultType, Value src, |
| 2599 | ArrayRef<ReassociationIndices> reassociation, |
| 2600 | ArrayRef<OpFoldResult> outputShape) { |
| 2601 | auto [staticOutputShape, dynamicOutputShape] = |
| 2602 | decomposeMixedValues(mixedValues: SmallVector<OpFoldResult>(outputShape)); |
| 2603 | build(odsBuilder&: builder, odsState&: result, result: llvm::cast<MemRefType>(Val&: resultType), src, |
| 2604 | reassociation: getReassociationIndicesAttribute(b&: builder, reassociation), |
| 2605 | output_shape: dynamicOutputShape, static_output_shape: staticOutputShape); |
| 2606 | } |
| 2607 | |
| 2608 | void ExpandShapeOp::build(OpBuilder &builder, OperationState &result, |
| 2609 | Type resultType, Value src, |
| 2610 | ArrayRef<ReassociationIndices> reassociation) { |
| 2611 | SmallVector<OpFoldResult> inputShape = |
| 2612 | getMixedSizes(builder, loc: result.location, value: src); |
| 2613 | MemRefType memrefResultTy = llvm::cast<MemRefType>(Val&: resultType); |
| 2614 | FailureOr<SmallVector<OpFoldResult>> outputShape = inferOutputShape( |
| 2615 | b&: builder, loc: result.location, expandedType: memrefResultTy, reassociation, inputShape); |
| 2616 | // Failure of this assertion usually indicates presence of multiple |
| 2617 | // dynamic dimensions in the same reassociation group. |
| 2618 | assert(succeeded(outputShape) && "unable to infer output shape" ); |
| 2619 | build(builder, result, resultType: memrefResultTy, src, reassociation, outputShape: *outputShape); |
| 2620 | } |
| 2621 | |
| 2622 | void ExpandShapeOp::build(OpBuilder &builder, OperationState &result, |
| 2623 | ArrayRef<int64_t> resultShape, Value src, |
| 2624 | ArrayRef<ReassociationIndices> reassociation) { |
| 2625 | // Only ranked memref source values are supported. |
| 2626 | auto srcType = llvm::cast<MemRefType>(Val: src.getType()); |
| 2627 | FailureOr<MemRefType> resultType = |
| 2628 | ExpandShapeOp::computeExpandedType(srcType, resultShape, reassociation); |
| 2629 | // Failure of this assertion usually indicates a problem with the source |
| 2630 | // type, e.g., could not get strides/offset. |
| 2631 | assert(succeeded(resultType) && "could not compute layout" ); |
| 2632 | build(builder, result, resultType: *resultType, src, reassociation); |
| 2633 | } |
| 2634 | |
| 2635 | void ExpandShapeOp::build(OpBuilder &builder, OperationState &result, |
| 2636 | ArrayRef<int64_t> resultShape, Value src, |
| 2637 | ArrayRef<ReassociationIndices> reassociation, |
| 2638 | ArrayRef<OpFoldResult> outputShape) { |
| 2639 | // Only ranked memref source values are supported. |
| 2640 | auto srcType = llvm::cast<MemRefType>(Val: src.getType()); |
| 2641 | FailureOr<MemRefType> resultType = |
| 2642 | ExpandShapeOp::computeExpandedType(srcType, resultShape, reassociation); |
| 2643 | // Failure of this assertion usually indicates a problem with the source |
| 2644 | // type, e.g., could not get strides/offset. |
| 2645 | assert(succeeded(resultType) && "could not compute layout" ); |
| 2646 | build(builder, result, resultType: *resultType, src, reassociation, outputShape); |
| 2647 | } |
| 2648 | |
| 2649 | LogicalResult ExpandShapeOp::verify() { |
| 2650 | if (failed(Result: verifyReassociationIndicesNotEmpty(op: *this))) |
| 2651 | return failure(); |
| 2652 | |
| 2653 | MemRefType srcType = getSrcType(); |
| 2654 | MemRefType resultType = getResultType(); |
| 2655 | |
| 2656 | if (srcType.getRank() > resultType.getRank()) { |
| 2657 | auto r0 = srcType.getRank(); |
| 2658 | auto r1 = resultType.getRank(); |
| 2659 | return emitOpError(message: "has source rank " ) |
| 2660 | << r0 << " and result rank " << r1 << ". This is not an expansion (" |
| 2661 | << r0 << " > " << r1 << ")." ; |
| 2662 | } |
| 2663 | |
| 2664 | // Verify result shape. |
| 2665 | if (failed(Result: verifyCollapsedShape(op: getOperation(), collapsedShape: srcType.getShape(), |
| 2666 | expandedShape: resultType.getShape(), |
| 2667 | reassociation: getReassociationIndices(), |
| 2668 | /*allowMultipleDynamicDimsPerGroup=*/true))) |
| 2669 | return failure(); |
| 2670 | |
| 2671 | // Compute expected result type (including layout map). |
| 2672 | FailureOr<MemRefType> expectedResultType = ExpandShapeOp::computeExpandedType( |
| 2673 | srcType, resultShape: resultType.getShape(), reassociation: getReassociationIndices()); |
| 2674 | if (failed(Result: expectedResultType)) |
| 2675 | return emitOpError(message: "invalid source layout map" ); |
| 2676 | |
| 2677 | // Check actual result type. |
| 2678 | if (*expectedResultType != resultType) |
| 2679 | return emitOpError(message: "expected expanded type to be " ) |
| 2680 | << *expectedResultType << " but found " << resultType; |
| 2681 | |
| 2682 | if ((int64_t)getStaticOutputShape().size() != resultType.getRank()) |
| 2683 | return emitOpError(message: "expected number of static shape bounds to be equal to " |
| 2684 | "the output rank (" ) |
| 2685 | << resultType.getRank() << ") but found " |
| 2686 | << getStaticOutputShape().size() << " inputs instead" ; |
| 2687 | |
| 2688 | if ((int64_t)getOutputShape().size() != |
| 2689 | llvm::count(Range: getStaticOutputShape(), Element: ShapedType::kDynamic)) |
| 2690 | return emitOpError(message: "mismatch in dynamic dims in output_shape and " |
| 2691 | "static_output_shape: static_output_shape has " ) |
| 2692 | << llvm::count(Range: getStaticOutputShape(), Element: ShapedType::kDynamic) |
| 2693 | << " dynamic dims while output_shape has " << getOutputShape().size() |
| 2694 | << " values" ; |
| 2695 | |
| 2696 | // Verify that the number of dynamic dims in output_shape matches the number |
| 2697 | // of dynamic dims in the result type. |
| 2698 | if (failed(Result: verifyDynamicDimensionCount(op: getOperation(), type: resultType, |
| 2699 | dynamicSizes: getOutputShape()))) |
| 2700 | return failure(); |
| 2701 | |
| 2702 | // Verify if provided output shapes are in agreement with output type. |
| 2703 | DenseI64ArrayAttr staticOutputShapes = getStaticOutputShapeAttr(); |
| 2704 | ArrayRef<int64_t> resShape = getResult().getType().getShape(); |
| 2705 | for (auto [pos, shape] : llvm::enumerate(First&: resShape)) { |
| 2706 | if (ShapedType::isStatic(dValue: shape) && shape != staticOutputShapes[pos]) { |
| 2707 | return emitOpError(message: "invalid output shape provided at pos " ) << pos; |
| 2708 | } |
| 2709 | } |
| 2710 | |
| 2711 | return success(); |
| 2712 | } |
| 2713 | |
| 2714 | struct ExpandShapeOpMemRefCastFolder : public OpRewritePattern<ExpandShapeOp> { |
| 2715 | public: |
| 2716 | using OpRewritePattern<ExpandShapeOp>::OpRewritePattern; |
| 2717 | |
| 2718 | LogicalResult matchAndRewrite(ExpandShapeOp op, |
| 2719 | PatternRewriter &rewriter) const override { |
| 2720 | auto cast = op.getSrc().getDefiningOp<CastOp>(); |
| 2721 | if (!cast) |
| 2722 | return failure(); |
| 2723 | |
| 2724 | if (!CastOp::canFoldIntoConsumerOp(castOp: cast)) |
| 2725 | return failure(); |
| 2726 | |
| 2727 | SmallVector<OpFoldResult> originalOutputShape = op.getMixedOutputShape(); |
| 2728 | SmallVector<OpFoldResult> newOutputShape = originalOutputShape; |
| 2729 | SmallVector<int64_t> newOutputShapeSizes; |
| 2730 | |
| 2731 | // Convert output shape dims from dynamic to static where possible. |
| 2732 | for (auto [dimIdx, dimSize] : enumerate(First&: originalOutputShape)) { |
| 2733 | std::optional<int64_t> sizeOpt = getConstantIntValue(ofr: dimSize); |
| 2734 | if (!sizeOpt.has_value()) { |
| 2735 | newOutputShapeSizes.push_back(Elt: ShapedType::kDynamic); |
| 2736 | continue; |
| 2737 | } |
| 2738 | |
| 2739 | newOutputShapeSizes.push_back(Elt: sizeOpt.value()); |
| 2740 | newOutputShape[dimIdx] = rewriter.getIndexAttr(value: sizeOpt.value()); |
| 2741 | } |
| 2742 | |
| 2743 | Value castSource = cast.getSource(); |
| 2744 | auto castSourceType = llvm::cast<MemRefType>(Val: castSource.getType()); |
| 2745 | SmallVector<ReassociationIndices> reassociationIndices = |
| 2746 | op.getReassociationIndices(); |
| 2747 | for (auto [idx, group] : llvm::enumerate(First&: reassociationIndices)) { |
| 2748 | auto newOutputShapeSizesSlice = |
| 2749 | ArrayRef(newOutputShapeSizes).slice(N: group.front(), M: group.size()); |
| 2750 | bool newOutputDynamic = |
| 2751 | llvm::is_contained(Range&: newOutputShapeSizesSlice, Element: ShapedType::kDynamic); |
| 2752 | if (castSourceType.isDynamicDim(idx) != newOutputDynamic) |
| 2753 | return rewriter.notifyMatchFailure( |
| 2754 | arg&: op, msg: "folding cast will result in changing dynamicity in " |
| 2755 | "reassociation group" ); |
| 2756 | } |
| 2757 | |
| 2758 | FailureOr<MemRefType> newResultTypeOrFailure = |
| 2759 | ExpandShapeOp::computeExpandedType(srcType: castSourceType, resultShape: newOutputShapeSizes, |
| 2760 | reassociation: reassociationIndices); |
| 2761 | |
| 2762 | if (failed(Result: newResultTypeOrFailure)) |
| 2763 | return rewriter.notifyMatchFailure( |
| 2764 | arg&: op, msg: "could not compute new expanded type after folding cast" ); |
| 2765 | |
| 2766 | if (*newResultTypeOrFailure == op.getResultType()) { |
| 2767 | rewriter.modifyOpInPlace( |
| 2768 | root: op, callable: [&]() { op.getSrcMutable().assign(value: castSource); }); |
| 2769 | } else { |
| 2770 | Value newOp = ExpandShapeOp::create(builder&: rewriter, location: op->getLoc(), |
| 2771 | resultType: *newResultTypeOrFailure, src: castSource, |
| 2772 | reassociation: reassociationIndices, outputShape: newOutputShape); |
| 2773 | rewriter.replaceOpWithNewOp<CastOp>(op, args: op.getType(), args&: newOp); |
| 2774 | } |
| 2775 | return success(); |
| 2776 | } |
| 2777 | }; |
| 2778 | |
| 2779 | void ExpandShapeOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 2780 | MLIRContext *context) { |
| 2781 | results.add< |
| 2782 | ComposeReassociativeReshapeOps<ExpandShapeOp, ReshapeOpKind::kExpand>, |
| 2783 | ComposeExpandOfCollapseOp<ExpandShapeOp, CollapseShapeOp, CastOp>, |
| 2784 | ExpandShapeOpMemRefCastFolder>(arg&: context); |
| 2785 | } |
| 2786 | |
| 2787 | FailureOr<std::optional<SmallVector<Value>>> |
| 2788 | ExpandShapeOp::bubbleDownCasts(OpBuilder &builder) { |
| 2789 | return bubbleDownCastsPassthroughOpImpl(op: *this, builder, src&: getSrcMutable()); |
| 2790 | } |
| 2791 | |
| 2792 | /// Compute the layout map after collapsing a given source MemRef type with the |
| 2793 | /// specified reassociation indices. |
| 2794 | /// |
| 2795 | /// Note: All collapsed dims in a reassociation group must be contiguous. It is |
| 2796 | /// not possible to check this by inspecting a MemRefType in the general case. |
| 2797 | /// If non-contiguity cannot be checked statically, the collapse is assumed to |
| 2798 | /// be valid (and thus accepted by this function) unless `strict = true`. |
| 2799 | static FailureOr<StridedLayoutAttr> |
| 2800 | computeCollapsedLayoutMap(MemRefType srcType, |
| 2801 | ArrayRef<ReassociationIndices> reassociation, |
| 2802 | bool strict = false) { |
| 2803 | int64_t srcOffset; |
| 2804 | SmallVector<int64_t> srcStrides; |
| 2805 | auto srcShape = srcType.getShape(); |
| 2806 | if (failed(Result: srcType.getStridesAndOffset(strides&: srcStrides, offset&: srcOffset))) |
| 2807 | return failure(); |
| 2808 | |
| 2809 | // The result stride of a reassociation group is the stride of the last entry |
| 2810 | // of the reassociation. (TODO: Should be the minimum stride in the |
| 2811 | // reassociation because strides are not necessarily sorted. E.g., when using |
| 2812 | // memref.transpose.) Dimensions of size 1 should be skipped, because their |
| 2813 | // strides are meaningless and could have any arbitrary value. |
| 2814 | SmallVector<int64_t> resultStrides; |
| 2815 | resultStrides.reserve(N: reassociation.size()); |
| 2816 | for (const ReassociationIndices &reassoc : reassociation) { |
| 2817 | ArrayRef<int64_t> ref = llvm::ArrayRef(reassoc); |
| 2818 | while (srcShape[ref.back()] == 1 && ref.size() > 1) |
| 2819 | ref = ref.drop_back(); |
| 2820 | if (ShapedType::isStatic(dValue: srcShape[ref.back()]) || ref.size() == 1) { |
| 2821 | resultStrides.push_back(Elt: srcStrides[ref.back()]); |
| 2822 | } else { |
| 2823 | // Dynamically-sized dims may turn out to be dims of size 1 at runtime, so |
| 2824 | // the corresponding stride may have to be skipped. (See above comment.) |
| 2825 | // Therefore, the result stride cannot be statically determined and must |
| 2826 | // be dynamic. |
| 2827 | resultStrides.push_back(Elt: ShapedType::kDynamic); |
| 2828 | } |
| 2829 | } |
| 2830 | |
| 2831 | // Validate that each reassociation group is contiguous. |
| 2832 | unsigned resultStrideIndex = resultStrides.size() - 1; |
| 2833 | for (const ReassociationIndices &reassoc : llvm::reverse(C&: reassociation)) { |
| 2834 | auto trailingReassocs = ArrayRef<int64_t>(reassoc).drop_front(); |
| 2835 | auto stride = SaturatedInteger::wrap(v: resultStrides[resultStrideIndex--]); |
| 2836 | for (int64_t idx : llvm::reverse(C&: trailingReassocs)) { |
| 2837 | stride = stride * SaturatedInteger::wrap(v: srcShape[idx]); |
| 2838 | |
| 2839 | // Dimensions of size 1 should be skipped, because their strides are |
| 2840 | // meaningless and could have any arbitrary value. |
| 2841 | if (srcShape[idx - 1] == 1) |
| 2842 | continue; |
| 2843 | |
| 2844 | // Both source and result stride must have the same static value. In that |
| 2845 | // case, we can be sure, that the dimensions are collapsible (because they |
| 2846 | // are contiguous). |
| 2847 | // If `strict = false` (default during op verification), we accept cases |
| 2848 | // where one or both strides are dynamic. This is best effort: We reject |
| 2849 | // ops where obviously non-contiguous dims are collapsed, but accept ops |
| 2850 | // where we cannot be sure statically. Such ops may fail at runtime. See |
| 2851 | // the op documentation for details. |
| 2852 | auto srcStride = SaturatedInteger::wrap(v: srcStrides[idx - 1]); |
| 2853 | if (strict && (stride.saturated || srcStride.saturated)) |
| 2854 | return failure(); |
| 2855 | |
| 2856 | if (!stride.saturated && !srcStride.saturated && stride != srcStride) |
| 2857 | return failure(); |
| 2858 | } |
| 2859 | } |
| 2860 | return StridedLayoutAttr::get(context: srcType.getContext(), offset: srcOffset, strides: resultStrides); |
| 2861 | } |
| 2862 | |
| 2863 | bool CollapseShapeOp::isGuaranteedCollapsible( |
| 2864 | MemRefType srcType, ArrayRef<ReassociationIndices> reassociation) { |
| 2865 | // MemRefs with identity layout are always collapsible. |
| 2866 | if (srcType.getLayout().isIdentity()) |
| 2867 | return true; |
| 2868 | |
| 2869 | return succeeded(Result: computeCollapsedLayoutMap(srcType, reassociation, |
| 2870 | /*strict=*/true)); |
| 2871 | } |
| 2872 | |
| 2873 | MemRefType CollapseShapeOp::computeCollapsedType( |
| 2874 | MemRefType srcType, ArrayRef<ReassociationIndices> reassociation) { |
| 2875 | SmallVector<int64_t> resultShape; |
| 2876 | resultShape.reserve(N: reassociation.size()); |
| 2877 | for (const ReassociationIndices &group : reassociation) { |
| 2878 | auto groupSize = SaturatedInteger::wrap(v: 1); |
| 2879 | for (int64_t srcDim : group) |
| 2880 | groupSize = |
| 2881 | groupSize * SaturatedInteger::wrap(v: srcType.getDimSize(idx: srcDim)); |
| 2882 | resultShape.push_back(Elt: groupSize.asInteger()); |
| 2883 | } |
| 2884 | |
| 2885 | if (srcType.getLayout().isIdentity()) { |
| 2886 | // If the source is contiguous (i.e., no layout map specified), so is the |
| 2887 | // result. |
| 2888 | MemRefLayoutAttrInterface layout; |
| 2889 | return MemRefType::get(shape: resultShape, elementType: srcType.getElementType(), layout, |
| 2890 | memorySpace: srcType.getMemorySpace()); |
| 2891 | } |
| 2892 | |
| 2893 | // Source may not be fully contiguous. Compute the layout map. |
| 2894 | // Note: Dimensions that are collapsed into a single dim are assumed to be |
| 2895 | // contiguous. |
| 2896 | FailureOr<StridedLayoutAttr> computedLayout = |
| 2897 | computeCollapsedLayoutMap(srcType, reassociation); |
| 2898 | assert(succeeded(computedLayout) && |
| 2899 | "invalid source layout map or collapsing non-contiguous dims" ); |
| 2900 | return MemRefType::get(shape: resultShape, elementType: srcType.getElementType(), layout: *computedLayout, |
| 2901 | memorySpace: srcType.getMemorySpace()); |
| 2902 | } |
| 2903 | |
| 2904 | void CollapseShapeOp::build(OpBuilder &b, OperationState &result, Value src, |
| 2905 | ArrayRef<ReassociationIndices> reassociation, |
| 2906 | ArrayRef<NamedAttribute> attrs) { |
| 2907 | auto srcType = llvm::cast<MemRefType>(Val: src.getType()); |
| 2908 | MemRefType resultType = |
| 2909 | CollapseShapeOp::computeCollapsedType(srcType, reassociation); |
| 2910 | buildPropertiesAndDiscardableAttributes(odsState&: result, attributes: attrs); |
| 2911 | result.getOrAddProperties<Properties>().reassociation = |
| 2912 | getReassociationIndicesAttribute(b, reassociation); |
| 2913 | result.addOperands(newOperands: src); |
| 2914 | result.addTypes(newTypes: resultType); |
| 2915 | } |
| 2916 | |
| 2917 | LogicalResult CollapseShapeOp::verify() { |
| 2918 | if (failed(Result: verifyReassociationIndicesNotEmpty(op: *this))) |
| 2919 | return failure(); |
| 2920 | |
| 2921 | MemRefType srcType = getSrcType(); |
| 2922 | MemRefType resultType = getResultType(); |
| 2923 | |
| 2924 | if (srcType.getRank() < resultType.getRank()) { |
| 2925 | auto r0 = srcType.getRank(); |
| 2926 | auto r1 = resultType.getRank(); |
| 2927 | return emitOpError(message: "has source rank " ) |
| 2928 | << r0 << " and result rank " << r1 << ". This is not a collapse (" |
| 2929 | << r0 << " < " << r1 << ")." ; |
| 2930 | } |
| 2931 | |
| 2932 | // Verify result shape. |
| 2933 | if (failed(Result: verifyCollapsedShape(op: getOperation(), collapsedShape: resultType.getShape(), |
| 2934 | expandedShape: srcType.getShape(), reassociation: getReassociationIndices(), |
| 2935 | /*allowMultipleDynamicDimsPerGroup=*/true))) |
| 2936 | return failure(); |
| 2937 | |
| 2938 | // Compute expected result type (including layout map). |
| 2939 | MemRefType expectedResultType; |
| 2940 | if (srcType.getLayout().isIdentity()) { |
| 2941 | // If the source is contiguous (i.e., no layout map specified), so is the |
| 2942 | // result. |
| 2943 | MemRefLayoutAttrInterface layout; |
| 2944 | expectedResultType = |
| 2945 | MemRefType::get(shape: resultType.getShape(), elementType: srcType.getElementType(), layout, |
| 2946 | memorySpace: srcType.getMemorySpace()); |
| 2947 | } else { |
| 2948 | // Source may not be fully contiguous. Compute the layout map. |
| 2949 | // Note: Dimensions that are collapsed into a single dim are assumed to be |
| 2950 | // contiguous. |
| 2951 | FailureOr<StridedLayoutAttr> computedLayout = |
| 2952 | computeCollapsedLayoutMap(srcType, reassociation: getReassociationIndices()); |
| 2953 | if (failed(Result: computedLayout)) |
| 2954 | return emitOpError( |
| 2955 | message: "invalid source layout map or collapsing non-contiguous dims" ); |
| 2956 | expectedResultType = |
| 2957 | MemRefType::get(shape: resultType.getShape(), elementType: srcType.getElementType(), |
| 2958 | layout: *computedLayout, memorySpace: srcType.getMemorySpace()); |
| 2959 | } |
| 2960 | |
| 2961 | if (expectedResultType != resultType) |
| 2962 | return emitOpError(message: "expected collapsed type to be " ) |
| 2963 | << expectedResultType << " but found " << resultType; |
| 2964 | |
| 2965 | return success(); |
| 2966 | } |
| 2967 | |
| 2968 | struct CollapseShapeOpMemRefCastFolder |
| 2969 | : public OpRewritePattern<CollapseShapeOp> { |
| 2970 | public: |
| 2971 | using OpRewritePattern<CollapseShapeOp>::OpRewritePattern; |
| 2972 | |
| 2973 | LogicalResult matchAndRewrite(CollapseShapeOp op, |
| 2974 | PatternRewriter &rewriter) const override { |
| 2975 | auto cast = op.getOperand().getDefiningOp<CastOp>(); |
| 2976 | if (!cast) |
| 2977 | return failure(); |
| 2978 | |
| 2979 | if (!CastOp::canFoldIntoConsumerOp(castOp: cast)) |
| 2980 | return failure(); |
| 2981 | |
| 2982 | Type newResultType = CollapseShapeOp::computeCollapsedType( |
| 2983 | srcType: llvm::cast<MemRefType>(Val: cast.getOperand().getType()), |
| 2984 | reassociation: op.getReassociationIndices()); |
| 2985 | |
| 2986 | if (newResultType == op.getResultType()) { |
| 2987 | rewriter.modifyOpInPlace( |
| 2988 | root: op, callable: [&]() { op.getSrcMutable().assign(value: cast.getSource()); }); |
| 2989 | } else { |
| 2990 | Value newOp = |
| 2991 | CollapseShapeOp::create(builder&: rewriter, location: op->getLoc(), src: cast.getSource(), |
| 2992 | reassociation: op.getReassociationIndices()); |
| 2993 | rewriter.replaceOpWithNewOp<CastOp>(op, args: op.getType(), args&: newOp); |
| 2994 | } |
| 2995 | return success(); |
| 2996 | } |
| 2997 | }; |
| 2998 | |
| 2999 | void CollapseShapeOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 3000 | MLIRContext *context) { |
| 3001 | results.add< |
| 3002 | ComposeReassociativeReshapeOps<CollapseShapeOp, ReshapeOpKind::kCollapse>, |
| 3003 | ComposeCollapseOfExpandOp<CollapseShapeOp, ExpandShapeOp, CastOp, |
| 3004 | memref::DimOp, MemRefType>, |
| 3005 | CollapseShapeOpMemRefCastFolder>(arg&: context); |
| 3006 | } |
| 3007 | |
| 3008 | OpFoldResult ExpandShapeOp::fold(FoldAdaptor adaptor) { |
| 3009 | return foldReshapeOp<ExpandShapeOp, CollapseShapeOp>(reshapeOp: *this, |
| 3010 | operands: adaptor.getOperands()); |
| 3011 | } |
| 3012 | |
| 3013 | OpFoldResult CollapseShapeOp::fold(FoldAdaptor adaptor) { |
| 3014 | return foldReshapeOp<CollapseShapeOp, ExpandShapeOp>(reshapeOp: *this, |
| 3015 | operands: adaptor.getOperands()); |
| 3016 | } |
| 3017 | |
| 3018 | FailureOr<std::optional<SmallVector<Value>>> |
| 3019 | CollapseShapeOp::bubbleDownCasts(OpBuilder &builder) { |
| 3020 | return bubbleDownCastsPassthroughOpImpl(op: *this, builder, src&: getSrcMutable()); |
| 3021 | } |
| 3022 | |
| 3023 | //===----------------------------------------------------------------------===// |
| 3024 | // ReshapeOp |
| 3025 | //===----------------------------------------------------------------------===// |
| 3026 | |
| 3027 | void ReshapeOp::getAsmResultNames( |
| 3028 | function_ref<void(Value, StringRef)> setNameFn) { |
| 3029 | setNameFn(getResult(), "reshape" ); |
| 3030 | } |
| 3031 | |
| 3032 | LogicalResult ReshapeOp::verify() { |
| 3033 | Type operandType = getSource().getType(); |
| 3034 | Type resultType = getResult().getType(); |
| 3035 | |
| 3036 | Type operandElementType = |
| 3037 | llvm::cast<ShapedType>(Val&: operandType).getElementType(); |
| 3038 | Type resultElementType = llvm::cast<ShapedType>(Val&: resultType).getElementType(); |
| 3039 | if (operandElementType != resultElementType) |
| 3040 | return emitOpError(message: "element types of source and destination memref " |
| 3041 | "types should be the same" ); |
| 3042 | |
| 3043 | if (auto operandMemRefType = llvm::dyn_cast<MemRefType>(Val&: operandType)) |
| 3044 | if (!operandMemRefType.getLayout().isIdentity()) |
| 3045 | return emitOpError(message: "source memref type should have identity affine map" ); |
| 3046 | |
| 3047 | int64_t shapeSize = |
| 3048 | llvm::cast<MemRefType>(Val: getShape().getType()).getDimSize(idx: 0); |
| 3049 | auto resultMemRefType = llvm::dyn_cast<MemRefType>(Val&: resultType); |
| 3050 | if (resultMemRefType) { |
| 3051 | if (!resultMemRefType.getLayout().isIdentity()) |
| 3052 | return emitOpError(message: "result memref type should have identity affine map" ); |
| 3053 | if (ShapedType::isDynamic(dValue: shapeSize)) |
| 3054 | return emitOpError(message: "cannot use shape operand with dynamic length to " |
| 3055 | "reshape to statically-ranked memref type" ); |
| 3056 | if (shapeSize != resultMemRefType.getRank()) |
| 3057 | return emitOpError( |
| 3058 | message: "length of shape operand differs from the result's memref rank" ); |
| 3059 | } else if (ShapedType::isStatic(dValue: shapeSize)) { |
| 3060 | return emitOpError(message: "cannot use shape operand with static length to " |
| 3061 | "reshape to unranked memref type" ); |
| 3062 | } |
| 3063 | return success(); |
| 3064 | } |
| 3065 | |
| 3066 | FailureOr<std::optional<SmallVector<Value>>> |
| 3067 | ReshapeOp::bubbleDownCasts(OpBuilder &builder) { |
| 3068 | return bubbleDownCastsPassthroughOpImpl(op: *this, builder, src&: getSourceMutable()); |
| 3069 | } |
| 3070 | |
| 3071 | //===----------------------------------------------------------------------===// |
| 3072 | // StoreOp |
| 3073 | //===----------------------------------------------------------------------===// |
| 3074 | |
| 3075 | LogicalResult StoreOp::fold(FoldAdaptor adaptor, |
| 3076 | SmallVectorImpl<OpFoldResult> &results) { |
| 3077 | /// store(memrefcast) -> store |
| 3078 | return foldMemRefCast(op: *this, inner: getValueToStore()); |
| 3079 | } |
| 3080 | |
| 3081 | TypedValue<MemRefType> StoreOp::getAccessedMemref() { return getMemref(); } |
| 3082 | |
| 3083 | std::optional<SmallVector<Value>> |
| 3084 | StoreOp::updateMemrefAndIndices(RewriterBase &rewriter, Value newMemref, |
| 3085 | ValueRange newIndices) { |
| 3086 | rewriter.modifyOpInPlace(root: *this, callable: [&]() { |
| 3087 | getMemrefMutable().assign(value: newMemref); |
| 3088 | getIndicesMutable().assign(values: newIndices); |
| 3089 | }); |
| 3090 | return std::nullopt; |
| 3091 | } |
| 3092 | |
| 3093 | FailureOr<std::optional<SmallVector<Value>>> |
| 3094 | StoreOp::bubbleDownCasts(OpBuilder &builder) { |
| 3095 | return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getMemrefMutable(), |
| 3096 | results: ValueRange()); |
| 3097 | } |
| 3098 | |
| 3099 | //===----------------------------------------------------------------------===// |
| 3100 | // SubViewOp |
| 3101 | //===----------------------------------------------------------------------===// |
| 3102 | |
| 3103 | void SubViewOp::getAsmResultNames( |
| 3104 | function_ref<void(Value, StringRef)> setNameFn) { |
| 3105 | setNameFn(getResult(), "subview" ); |
| 3106 | } |
| 3107 | |
| 3108 | /// A subview result type can be fully inferred from the source type and the |
| 3109 | /// static representation of offsets, sizes and strides. Special sentinels |
| 3110 | /// encode the dynamic case. |
| 3111 | MemRefType SubViewOp::inferResultType(MemRefType sourceMemRefType, |
| 3112 | ArrayRef<int64_t> staticOffsets, |
| 3113 | ArrayRef<int64_t> staticSizes, |
| 3114 | ArrayRef<int64_t> staticStrides) { |
| 3115 | unsigned rank = sourceMemRefType.getRank(); |
| 3116 | (void)rank; |
| 3117 | assert(staticOffsets.size() == rank && "staticOffsets length mismatch" ); |
| 3118 | assert(staticSizes.size() == rank && "staticSizes length mismatch" ); |
| 3119 | assert(staticStrides.size() == rank && "staticStrides length mismatch" ); |
| 3120 | |
| 3121 | // Extract source offset and strides. |
| 3122 | auto [sourceStrides, sourceOffset] = sourceMemRefType.getStridesAndOffset(); |
| 3123 | |
| 3124 | // Compute target offset whose value is: |
| 3125 | // `sourceOffset + sum_i(staticOffset_i * sourceStrides_i)`. |
| 3126 | int64_t targetOffset = sourceOffset; |
| 3127 | for (auto it : llvm::zip(t&: staticOffsets, u&: sourceStrides)) { |
| 3128 | auto staticOffset = std::get<0>(t&: it), sourceStride = std::get<1>(t&: it); |
| 3129 | targetOffset = (SaturatedInteger::wrap(v: targetOffset) + |
| 3130 | SaturatedInteger::wrap(v: staticOffset) * |
| 3131 | SaturatedInteger::wrap(v: sourceStride)) |
| 3132 | .asInteger(); |
| 3133 | } |
| 3134 | |
| 3135 | // Compute target stride whose value is: |
| 3136 | // `sourceStrides_i * staticStrides_i`. |
| 3137 | SmallVector<int64_t, 4> targetStrides; |
| 3138 | targetStrides.reserve(N: staticOffsets.size()); |
| 3139 | for (auto it : llvm::zip(t&: sourceStrides, u&: staticStrides)) { |
| 3140 | auto sourceStride = std::get<0>(t&: it), staticStride = std::get<1>(t&: it); |
| 3141 | targetStrides.push_back(Elt: (SaturatedInteger::wrap(v: sourceStride) * |
| 3142 | SaturatedInteger::wrap(v: staticStride)) |
| 3143 | .asInteger()); |
| 3144 | } |
| 3145 | |
| 3146 | // The type is now known. |
| 3147 | return MemRefType::get(shape: staticSizes, elementType: sourceMemRefType.getElementType(), |
| 3148 | layout: StridedLayoutAttr::get(context: sourceMemRefType.getContext(), |
| 3149 | offset: targetOffset, strides: targetStrides), |
| 3150 | memorySpace: sourceMemRefType.getMemorySpace()); |
| 3151 | } |
| 3152 | |
| 3153 | MemRefType SubViewOp::inferResultType(MemRefType sourceMemRefType, |
| 3154 | ArrayRef<OpFoldResult> offsets, |
| 3155 | ArrayRef<OpFoldResult> sizes, |
| 3156 | ArrayRef<OpFoldResult> strides) { |
| 3157 | SmallVector<int64_t> staticOffsets, staticSizes, staticStrides; |
| 3158 | SmallVector<Value> dynamicOffsets, dynamicSizes, dynamicStrides; |
| 3159 | dispatchIndexOpFoldResults(ofrs: offsets, dynamicVec&: dynamicOffsets, staticVec&: staticOffsets); |
| 3160 | dispatchIndexOpFoldResults(ofrs: sizes, dynamicVec&: dynamicSizes, staticVec&: staticSizes); |
| 3161 | dispatchIndexOpFoldResults(ofrs: strides, dynamicVec&: dynamicStrides, staticVec&: staticStrides); |
| 3162 | if (!hasValidSizesOffsets(sizesOrOffsets: staticOffsets)) |
| 3163 | return {}; |
| 3164 | if (!hasValidSizesOffsets(sizesOrOffsets: staticSizes)) |
| 3165 | return {}; |
| 3166 | if (!hasValidStrides(strides: staticStrides)) |
| 3167 | return {}; |
| 3168 | return SubViewOp::inferResultType(sourceMemRefType, staticOffsets, |
| 3169 | staticSizes, staticStrides); |
| 3170 | } |
| 3171 | |
| 3172 | MemRefType SubViewOp::inferRankReducedResultType( |
| 3173 | ArrayRef<int64_t> resultShape, MemRefType sourceRankedTensorType, |
| 3174 | ArrayRef<int64_t> offsets, ArrayRef<int64_t> sizes, |
| 3175 | ArrayRef<int64_t> strides) { |
| 3176 | MemRefType inferredType = |
| 3177 | inferResultType(sourceMemRefType: sourceRankedTensorType, staticOffsets: offsets, staticSizes: sizes, staticStrides: strides); |
| 3178 | assert(inferredType.getRank() >= static_cast<int64_t>(resultShape.size()) && |
| 3179 | "expected " ); |
| 3180 | if (inferredType.getRank() == static_cast<int64_t>(resultShape.size())) |
| 3181 | return inferredType; |
| 3182 | |
| 3183 | // Compute which dimensions are dropped. |
| 3184 | std::optional<llvm::SmallDenseSet<unsigned>> dimsToProject = |
| 3185 | computeRankReductionMask(originalShape: inferredType.getShape(), reducedShape: resultShape); |
| 3186 | assert(dimsToProject.has_value() && "invalid rank reduction" ); |
| 3187 | |
| 3188 | // Compute the layout and result type. |
| 3189 | auto inferredLayout = llvm::cast<StridedLayoutAttr>(Val: inferredType.getLayout()); |
| 3190 | SmallVector<int64_t> rankReducedStrides; |
| 3191 | rankReducedStrides.reserve(N: resultShape.size()); |
| 3192 | for (auto [idx, value] : llvm::enumerate(First: inferredLayout.getStrides())) { |
| 3193 | if (!dimsToProject->contains(V: idx)) |
| 3194 | rankReducedStrides.push_back(Elt: value); |
| 3195 | } |
| 3196 | return MemRefType::get(shape: resultShape, elementType: inferredType.getElementType(), |
| 3197 | layout: StridedLayoutAttr::get(context: inferredLayout.getContext(), |
| 3198 | offset: inferredLayout.getOffset(), |
| 3199 | strides: rankReducedStrides), |
| 3200 | memorySpace: inferredType.getMemorySpace()); |
| 3201 | } |
| 3202 | |
| 3203 | MemRefType SubViewOp::inferRankReducedResultType( |
| 3204 | ArrayRef<int64_t> resultShape, MemRefType sourceRankedTensorType, |
| 3205 | ArrayRef<OpFoldResult> offsets, ArrayRef<OpFoldResult> sizes, |
| 3206 | ArrayRef<OpFoldResult> strides) { |
| 3207 | SmallVector<int64_t> staticOffsets, staticSizes, staticStrides; |
| 3208 | SmallVector<Value> dynamicOffsets, dynamicSizes, dynamicStrides; |
| 3209 | dispatchIndexOpFoldResults(ofrs: offsets, dynamicVec&: dynamicOffsets, staticVec&: staticOffsets); |
| 3210 | dispatchIndexOpFoldResults(ofrs: sizes, dynamicVec&: dynamicSizes, staticVec&: staticSizes); |
| 3211 | dispatchIndexOpFoldResults(ofrs: strides, dynamicVec&: dynamicStrides, staticVec&: staticStrides); |
| 3212 | return SubViewOp::inferRankReducedResultType( |
| 3213 | resultShape, sourceRankedTensorType, offsets: staticOffsets, sizes: staticSizes, |
| 3214 | strides: staticStrides); |
| 3215 | } |
| 3216 | |
| 3217 | // Build a SubViewOp with mixed static and dynamic entries and custom result |
| 3218 | // type. If the type passed is nullptr, it is inferred. |
| 3219 | void SubViewOp::build(OpBuilder &b, OperationState &result, |
| 3220 | MemRefType resultType, Value source, |
| 3221 | ArrayRef<OpFoldResult> offsets, |
| 3222 | ArrayRef<OpFoldResult> sizes, |
| 3223 | ArrayRef<OpFoldResult> strides, |
| 3224 | ArrayRef<NamedAttribute> attrs) { |
| 3225 | SmallVector<int64_t> staticOffsets, staticSizes, staticStrides; |
| 3226 | SmallVector<Value> dynamicOffsets, dynamicSizes, dynamicStrides; |
| 3227 | dispatchIndexOpFoldResults(ofrs: offsets, dynamicVec&: dynamicOffsets, staticVec&: staticOffsets); |
| 3228 | dispatchIndexOpFoldResults(ofrs: sizes, dynamicVec&: dynamicSizes, staticVec&: staticSizes); |
| 3229 | dispatchIndexOpFoldResults(ofrs: strides, dynamicVec&: dynamicStrides, staticVec&: staticStrides); |
| 3230 | auto sourceMemRefType = llvm::cast<MemRefType>(Val: source.getType()); |
| 3231 | // Structuring implementation this way avoids duplication between builders. |
| 3232 | if (!resultType) { |
| 3233 | resultType = SubViewOp::inferResultType(sourceMemRefType, staticOffsets, |
| 3234 | staticSizes, staticStrides); |
| 3235 | } |
| 3236 | result.addAttributes(newAttributes: attrs); |
| 3237 | build(odsBuilder&: b, odsState&: result, result: resultType, source, offsets: dynamicOffsets, sizes: dynamicSizes, |
| 3238 | strides: dynamicStrides, static_offsets: b.getDenseI64ArrayAttr(values: staticOffsets), |
| 3239 | static_sizes: b.getDenseI64ArrayAttr(values: staticSizes), |
| 3240 | static_strides: b.getDenseI64ArrayAttr(values: staticStrides)); |
| 3241 | } |
| 3242 | |
| 3243 | // Build a SubViewOp with mixed static and dynamic entries and inferred result |
| 3244 | // type. |
| 3245 | void SubViewOp::build(OpBuilder &b, OperationState &result, Value source, |
| 3246 | ArrayRef<OpFoldResult> offsets, |
| 3247 | ArrayRef<OpFoldResult> sizes, |
| 3248 | ArrayRef<OpFoldResult> strides, |
| 3249 | ArrayRef<NamedAttribute> attrs) { |
| 3250 | build(b, result, resultType: MemRefType(), source, offsets, sizes, strides, attrs); |
| 3251 | } |
| 3252 | |
| 3253 | // Build a SubViewOp with static entries and inferred result type. |
| 3254 | void SubViewOp::build(OpBuilder &b, OperationState &result, Value source, |
| 3255 | ArrayRef<int64_t> offsets, ArrayRef<int64_t> sizes, |
| 3256 | ArrayRef<int64_t> strides, |
| 3257 | ArrayRef<NamedAttribute> attrs) { |
| 3258 | SmallVector<OpFoldResult> offsetValues = |
| 3259 | llvm::map_to_vector<4>(C&: offsets, F: [&](int64_t v) -> OpFoldResult { |
| 3260 | return b.getI64IntegerAttr(value: v); |
| 3261 | }); |
| 3262 | SmallVector<OpFoldResult> sizeValues = llvm::map_to_vector<4>( |
| 3263 | C&: sizes, F: [&](int64_t v) -> OpFoldResult { return b.getI64IntegerAttr(value: v); }); |
| 3264 | SmallVector<OpFoldResult> strideValues = |
| 3265 | llvm::map_to_vector<4>(C&: strides, F: [&](int64_t v) -> OpFoldResult { |
| 3266 | return b.getI64IntegerAttr(value: v); |
| 3267 | }); |
| 3268 | build(b, result, source, offsets: offsetValues, sizes: sizeValues, strides: strideValues, attrs); |
| 3269 | } |
| 3270 | |
| 3271 | // Build a SubViewOp with dynamic entries and custom result type. If the |
| 3272 | // type passed is nullptr, it is inferred. |
| 3273 | void SubViewOp::build(OpBuilder &b, OperationState &result, |
| 3274 | MemRefType resultType, Value source, |
| 3275 | ArrayRef<int64_t> offsets, ArrayRef<int64_t> sizes, |
| 3276 | ArrayRef<int64_t> strides, |
| 3277 | ArrayRef<NamedAttribute> attrs) { |
| 3278 | SmallVector<OpFoldResult> offsetValues = |
| 3279 | llvm::map_to_vector<4>(C&: offsets, F: [&](int64_t v) -> OpFoldResult { |
| 3280 | return b.getI64IntegerAttr(value: v); |
| 3281 | }); |
| 3282 | SmallVector<OpFoldResult> sizeValues = llvm::map_to_vector<4>( |
| 3283 | C&: sizes, F: [&](int64_t v) -> OpFoldResult { return b.getI64IntegerAttr(value: v); }); |
| 3284 | SmallVector<OpFoldResult> strideValues = |
| 3285 | llvm::map_to_vector<4>(C&: strides, F: [&](int64_t v) -> OpFoldResult { |
| 3286 | return b.getI64IntegerAttr(value: v); |
| 3287 | }); |
| 3288 | build(b, result, resultType, source, offsets: offsetValues, sizes: sizeValues, strides: strideValues, |
| 3289 | attrs); |
| 3290 | } |
| 3291 | |
| 3292 | // Build a SubViewOp with dynamic entries and custom result type. If the type |
| 3293 | // passed is nullptr, it is inferred. |
| 3294 | void SubViewOp::build(OpBuilder &b, OperationState &result, |
| 3295 | MemRefType resultType, Value source, ValueRange offsets, |
| 3296 | ValueRange sizes, ValueRange strides, |
| 3297 | ArrayRef<NamedAttribute> attrs) { |
| 3298 | SmallVector<OpFoldResult> offsetValues = llvm::map_to_vector<4>( |
| 3299 | C&: offsets, F: [](Value v) -> OpFoldResult { return v; }); |
| 3300 | SmallVector<OpFoldResult> sizeValues = |
| 3301 | llvm::map_to_vector<4>(C&: sizes, F: [](Value v) -> OpFoldResult { return v; }); |
| 3302 | SmallVector<OpFoldResult> strideValues = llvm::map_to_vector<4>( |
| 3303 | C&: strides, F: [](Value v) -> OpFoldResult { return v; }); |
| 3304 | build(b, result, resultType, source, offsets: offsetValues, sizes: sizeValues, strides: strideValues); |
| 3305 | } |
| 3306 | |
| 3307 | // Build a SubViewOp with dynamic entries and inferred result type. |
| 3308 | void SubViewOp::build(OpBuilder &b, OperationState &result, Value source, |
| 3309 | ValueRange offsets, ValueRange sizes, ValueRange strides, |
| 3310 | ArrayRef<NamedAttribute> attrs) { |
| 3311 | build(b, result, resultType: MemRefType(), source, offsets, sizes, strides, attrs); |
| 3312 | } |
| 3313 | |
| 3314 | /// For ViewLikeOpInterface. |
| 3315 | Value SubViewOp::getViewSource() { return getSource(); } |
| 3316 | |
| 3317 | /// Return true if `t1` and `t2` have equal offsets (both dynamic or of same |
| 3318 | /// static value). |
| 3319 | static bool haveCompatibleOffsets(MemRefType t1, MemRefType t2) { |
| 3320 | int64_t t1Offset, t2Offset; |
| 3321 | SmallVector<int64_t> t1Strides, t2Strides; |
| 3322 | auto res1 = t1.getStridesAndOffset(strides&: t1Strides, offset&: t1Offset); |
| 3323 | auto res2 = t2.getStridesAndOffset(strides&: t2Strides, offset&: t2Offset); |
| 3324 | return succeeded(Result: res1) && succeeded(Result: res2) && t1Offset == t2Offset; |
| 3325 | } |
| 3326 | |
| 3327 | /// Return true if `t1` and `t2` have equal strides (both dynamic or of same |
| 3328 | /// static value). Dimensions of `t1` may be dropped in `t2`; these must be |
| 3329 | /// marked as dropped in `droppedDims`. |
| 3330 | static bool haveCompatibleStrides(MemRefType t1, MemRefType t2, |
| 3331 | const llvm::SmallBitVector &droppedDims) { |
| 3332 | assert(size_t(t1.getRank()) == droppedDims.size() && |
| 3333 | "incorrect number of bits" ); |
| 3334 | assert(size_t(t1.getRank() - t2.getRank()) == droppedDims.count() && |
| 3335 | "incorrect number of dropped dims" ); |
| 3336 | int64_t t1Offset, t2Offset; |
| 3337 | SmallVector<int64_t> t1Strides, t2Strides; |
| 3338 | auto res1 = t1.getStridesAndOffset(strides&: t1Strides, offset&: t1Offset); |
| 3339 | auto res2 = t2.getStridesAndOffset(strides&: t2Strides, offset&: t2Offset); |
| 3340 | if (failed(Result: res1) || failed(Result: res2)) |
| 3341 | return false; |
| 3342 | for (int64_t i = 0, j = 0, e = t1.getRank(); i < e; ++i) { |
| 3343 | if (droppedDims[i]) |
| 3344 | continue; |
| 3345 | if (t1Strides[i] != t2Strides[j]) |
| 3346 | return false; |
| 3347 | ++j; |
| 3348 | } |
| 3349 | return true; |
| 3350 | } |
| 3351 | |
| 3352 | static LogicalResult produceSubViewErrorMsg(SliceVerificationResult result, |
| 3353 | SubViewOp op, Type expectedType) { |
| 3354 | auto memrefType = llvm::cast<ShapedType>(Val&: expectedType); |
| 3355 | switch (result) { |
| 3356 | case SliceVerificationResult::Success: |
| 3357 | return success(); |
| 3358 | case SliceVerificationResult::RankTooLarge: |
| 3359 | return op->emitError(message: "expected result rank to be smaller or equal to " ) |
| 3360 | << "the source rank, but got " << op.getType(); |
| 3361 | case SliceVerificationResult::SizeMismatch: |
| 3362 | return op->emitError(message: "expected result type to be " ) |
| 3363 | << expectedType |
| 3364 | << " or a rank-reduced version. (mismatch of result sizes), but got " |
| 3365 | << op.getType(); |
| 3366 | case SliceVerificationResult::ElemTypeMismatch: |
| 3367 | return op->emitError(message: "expected result element type to be " ) |
| 3368 | << memrefType.getElementType() << ", but got " << op.getType(); |
| 3369 | case SliceVerificationResult::MemSpaceMismatch: |
| 3370 | return op->emitError( |
| 3371 | message: "expected result and source memory spaces to match, but got " ) |
| 3372 | << op.getType(); |
| 3373 | case SliceVerificationResult::LayoutMismatch: |
| 3374 | return op->emitError(message: "expected result type to be " ) |
| 3375 | << expectedType |
| 3376 | << " or a rank-reduced version. (mismatch of result layout), but " |
| 3377 | "got " |
| 3378 | << op.getType(); |
| 3379 | } |
| 3380 | llvm_unreachable("unexpected subview verification result" ); |
| 3381 | } |
| 3382 | |
| 3383 | /// Verifier for SubViewOp. |
| 3384 | LogicalResult SubViewOp::verify() { |
| 3385 | MemRefType baseType = getSourceType(); |
| 3386 | MemRefType subViewType = getType(); |
| 3387 | ArrayRef<int64_t> staticOffsets = getStaticOffsets(); |
| 3388 | ArrayRef<int64_t> staticSizes = getStaticSizes(); |
| 3389 | ArrayRef<int64_t> staticStrides = getStaticStrides(); |
| 3390 | |
| 3391 | // The base memref and the view memref should be in the same memory space. |
| 3392 | if (baseType.getMemorySpace() != subViewType.getMemorySpace()) |
| 3393 | return emitError(message: "different memory spaces specified for base memref " |
| 3394 | "type " ) |
| 3395 | << baseType << " and subview memref type " << subViewType; |
| 3396 | |
| 3397 | // Verify that the base memref type has a strided layout map. |
| 3398 | if (!baseType.isStrided()) |
| 3399 | return emitError(message: "base type " ) << baseType << " is not strided" ; |
| 3400 | |
| 3401 | // Compute the expected result type, assuming that there are no rank |
| 3402 | // reductions. |
| 3403 | MemRefType expectedType = SubViewOp::inferResultType( |
| 3404 | sourceMemRefType: baseType, staticOffsets, staticSizes, staticStrides); |
| 3405 | |
| 3406 | // Verify all properties of a shaped type: rank, element type and dimension |
| 3407 | // sizes. This takes into account potential rank reductions. |
| 3408 | auto shapedTypeVerification = isRankReducedType( |
| 3409 | /*originalType=*/expectedType, /*candidateReducedType=*/subViewType); |
| 3410 | if (shapedTypeVerification != SliceVerificationResult::Success) |
| 3411 | return produceSubViewErrorMsg(result: shapedTypeVerification, op: *this, expectedType); |
| 3412 | |
| 3413 | // Make sure that the memory space did not change. |
| 3414 | if (expectedType.getMemorySpace() != subViewType.getMemorySpace()) |
| 3415 | return produceSubViewErrorMsg(result: SliceVerificationResult::MemSpaceMismatch, |
| 3416 | op: *this, expectedType); |
| 3417 | |
| 3418 | // Verify the offset of the layout map. |
| 3419 | if (!haveCompatibleOffsets(t1: expectedType, t2: subViewType)) |
| 3420 | return produceSubViewErrorMsg(result: SliceVerificationResult::LayoutMismatch, |
| 3421 | op: *this, expectedType); |
| 3422 | |
| 3423 | // The only thing that's left to verify now are the strides. First, compute |
| 3424 | // the unused dimensions due to rank reductions. We have to look at sizes and |
| 3425 | // strides to decide which dimensions were dropped. This function also |
| 3426 | // partially verifies strides in case of rank reductions. |
| 3427 | auto unusedDims = computeMemRefRankReductionMask(originalType: expectedType, reducedType: subViewType, |
| 3428 | sizes: getMixedSizes()); |
| 3429 | if (failed(Result: unusedDims)) |
| 3430 | return produceSubViewErrorMsg(result: SliceVerificationResult::LayoutMismatch, |
| 3431 | op: *this, expectedType); |
| 3432 | |
| 3433 | // Strides must match. |
| 3434 | if (!haveCompatibleStrides(t1: expectedType, t2: subViewType, droppedDims: *unusedDims)) |
| 3435 | return produceSubViewErrorMsg(result: SliceVerificationResult::LayoutMismatch, |
| 3436 | op: *this, expectedType); |
| 3437 | |
| 3438 | // Verify that offsets, sizes, strides do not run out-of-bounds with respect |
| 3439 | // to the base memref. |
| 3440 | SliceBoundsVerificationResult boundsResult = |
| 3441 | verifyInBoundsSlice(shape: baseType.getShape(), staticOffsets, staticSizes, |
| 3442 | staticStrides, /*generateErrorMessage=*/true); |
| 3443 | if (!boundsResult.isValid) |
| 3444 | return getOperation()->emitError(message: boundsResult.errorMessage); |
| 3445 | |
| 3446 | return success(); |
| 3447 | } |
| 3448 | |
| 3449 | raw_ostream &mlir::operator<<(raw_ostream &os, const Range &range) { |
| 3450 | return os << "range " << range.offset << ":" << range.size << ":" |
| 3451 | << range.stride; |
| 3452 | } |
| 3453 | |
| 3454 | /// Return the list of Range (i.e. offset, size, stride). Each Range |
| 3455 | /// entry contains either the dynamic value or a ConstantIndexOp constructed |
| 3456 | /// with `b` at location `loc`. |
| 3457 | SmallVector<Range, 8> mlir::getOrCreateRanges(OffsetSizeAndStrideOpInterface op, |
| 3458 | OpBuilder &b, Location loc) { |
| 3459 | std::array<unsigned, 3> ranks = op.getArrayAttrMaxRanks(); |
| 3460 | assert(ranks[0] == ranks[1] && "expected offset and sizes of equal ranks" ); |
| 3461 | assert(ranks[1] == ranks[2] && "expected sizes and strides of equal ranks" ); |
| 3462 | SmallVector<Range, 8> res; |
| 3463 | unsigned rank = ranks[0]; |
| 3464 | res.reserve(N: rank); |
| 3465 | for (unsigned idx = 0; idx < rank; ++idx) { |
| 3466 | Value offset = |
| 3467 | op.isDynamicOffset(idx) |
| 3468 | ? op.getDynamicOffset(idx) |
| 3469 | : arith::ConstantIndexOp::create(builder&: b, location: loc, value: op.getStaticOffset(idx)); |
| 3470 | Value size = |
| 3471 | op.isDynamicSize(idx) |
| 3472 | ? op.getDynamicSize(idx) |
| 3473 | : arith::ConstantIndexOp::create(builder&: b, location: loc, value: op.getStaticSize(idx)); |
| 3474 | Value stride = |
| 3475 | op.isDynamicStride(idx) |
| 3476 | ? op.getDynamicStride(idx) |
| 3477 | : arith::ConstantIndexOp::create(builder&: b, location: loc, value: op.getStaticStride(idx)); |
| 3478 | res.emplace_back(Args: Range{.offset: offset, .size: size, .stride: stride}); |
| 3479 | } |
| 3480 | return res; |
| 3481 | } |
| 3482 | |
| 3483 | /// Compute the canonical result type of a SubViewOp. Call `inferResultType` |
| 3484 | /// to deduce the result type for the given `sourceType`. Additionally, reduce |
| 3485 | /// the rank of the inferred result type if `currentResultType` is lower rank |
| 3486 | /// than `currentSourceType`. Use this signature if `sourceType` is updated |
| 3487 | /// together with the result type. In this case, it is important to compute |
| 3488 | /// the dropped dimensions using `currentSourceType` whose strides align with |
| 3489 | /// `currentResultType`. |
| 3490 | static MemRefType getCanonicalSubViewResultType( |
| 3491 | MemRefType currentResultType, MemRefType currentSourceType, |
| 3492 | MemRefType sourceType, ArrayRef<OpFoldResult> mixedOffsets, |
| 3493 | ArrayRef<OpFoldResult> mixedSizes, ArrayRef<OpFoldResult> mixedStrides) { |
| 3494 | MemRefType nonRankReducedType = SubViewOp::inferResultType( |
| 3495 | sourceMemRefType: sourceType, offsets: mixedOffsets, sizes: mixedSizes, strides: mixedStrides); |
| 3496 | FailureOr<llvm::SmallBitVector> unusedDims = computeMemRefRankReductionMask( |
| 3497 | originalType: currentSourceType, reducedType: currentResultType, sizes: mixedSizes); |
| 3498 | if (failed(Result: unusedDims)) |
| 3499 | return nullptr; |
| 3500 | |
| 3501 | auto layout = llvm::cast<StridedLayoutAttr>(Val: nonRankReducedType.getLayout()); |
| 3502 | SmallVector<int64_t> shape, strides; |
| 3503 | unsigned numDimsAfterReduction = |
| 3504 | nonRankReducedType.getRank() - unusedDims->count(); |
| 3505 | shape.reserve(N: numDimsAfterReduction); |
| 3506 | strides.reserve(N: numDimsAfterReduction); |
| 3507 | for (const auto &[idx, size, stride] : |
| 3508 | llvm::zip(t: llvm::seq<unsigned>(Begin: 0, End: nonRankReducedType.getRank()), |
| 3509 | u: nonRankReducedType.getShape(), args: layout.getStrides())) { |
| 3510 | if (unusedDims->test(Idx: idx)) |
| 3511 | continue; |
| 3512 | shape.push_back(Elt: size); |
| 3513 | strides.push_back(Elt: stride); |
| 3514 | } |
| 3515 | |
| 3516 | return MemRefType::get(shape, elementType: nonRankReducedType.getElementType(), |
| 3517 | layout: StridedLayoutAttr::get(context: sourceType.getContext(), |
| 3518 | offset: layout.getOffset(), strides), |
| 3519 | memorySpace: nonRankReducedType.getMemorySpace()); |
| 3520 | } |
| 3521 | |
| 3522 | Value mlir::memref::createCanonicalRankReducingSubViewOp( |
| 3523 | OpBuilder &b, Location loc, Value memref, ArrayRef<int64_t> targetShape) { |
| 3524 | auto memrefType = llvm::cast<MemRefType>(Val: memref.getType()); |
| 3525 | unsigned rank = memrefType.getRank(); |
| 3526 | SmallVector<OpFoldResult> offsets(rank, b.getIndexAttr(value: 0)); |
| 3527 | SmallVector<OpFoldResult> sizes = getMixedSizes(builder&: b, loc, value: memref); |
| 3528 | SmallVector<OpFoldResult> strides(rank, b.getIndexAttr(value: 1)); |
| 3529 | MemRefType targetType = SubViewOp::inferRankReducedResultType( |
| 3530 | resultShape: targetShape, sourceRankedTensorType: memrefType, offsets, sizes, strides); |
| 3531 | return b.createOrFold<memref::SubViewOp>(location: loc, args&: targetType, args&: memref, args&: offsets, |
| 3532 | args&: sizes, args&: strides); |
| 3533 | } |
| 3534 | |
| 3535 | FailureOr<Value> SubViewOp::rankReduceIfNeeded(OpBuilder &b, Location loc, |
| 3536 | Value value, |
| 3537 | ArrayRef<int64_t> desiredShape) { |
| 3538 | auto sourceMemrefType = llvm::dyn_cast<MemRefType>(Val: value.getType()); |
| 3539 | assert(sourceMemrefType && "not a ranked memref type" ); |
| 3540 | auto sourceShape = sourceMemrefType.getShape(); |
| 3541 | if (sourceShape.equals(RHS: desiredShape)) |
| 3542 | return value; |
| 3543 | auto maybeRankReductionMask = |
| 3544 | mlir::computeRankReductionMask(originalShape: sourceShape, reducedShape: desiredShape); |
| 3545 | if (!maybeRankReductionMask) |
| 3546 | return failure(); |
| 3547 | return createCanonicalRankReducingSubViewOp(b, loc, memref: value, targetShape: desiredShape); |
| 3548 | } |
| 3549 | |
| 3550 | /// Helper method to check if a `subview` operation is trivially a no-op. This |
| 3551 | /// is the case if the all offsets are zero, all strides are 1, and the source |
| 3552 | /// shape is same as the size of the subview. In such cases, the subview can |
| 3553 | /// be folded into its source. |
| 3554 | static bool isTrivialSubViewOp(SubViewOp subViewOp) { |
| 3555 | if (subViewOp.getSourceType().getRank() != subViewOp.getType().getRank()) |
| 3556 | return false; |
| 3557 | |
| 3558 | auto mixedOffsets = subViewOp.getMixedOffsets(); |
| 3559 | auto mixedSizes = subViewOp.getMixedSizes(); |
| 3560 | auto mixedStrides = subViewOp.getMixedStrides(); |
| 3561 | |
| 3562 | // Check offsets are zero. |
| 3563 | if (llvm::any_of(Range&: mixedOffsets, P: [](OpFoldResult ofr) { |
| 3564 | std::optional<int64_t> intValue = getConstantIntValue(ofr); |
| 3565 | return !intValue || intValue.value() != 0; |
| 3566 | })) |
| 3567 | return false; |
| 3568 | |
| 3569 | // Check strides are one. |
| 3570 | if (llvm::any_of(Range&: mixedStrides, P: [](OpFoldResult ofr) { |
| 3571 | std::optional<int64_t> intValue = getConstantIntValue(ofr); |
| 3572 | return !intValue || intValue.value() != 1; |
| 3573 | })) |
| 3574 | return false; |
| 3575 | |
| 3576 | // Check all size values are static and matches the (static) source shape. |
| 3577 | ArrayRef<int64_t> sourceShape = subViewOp.getSourceType().getShape(); |
| 3578 | for (const auto &size : llvm::enumerate(First&: mixedSizes)) { |
| 3579 | std::optional<int64_t> intValue = getConstantIntValue(ofr: size.value()); |
| 3580 | if (!intValue || *intValue != sourceShape[size.index()]) |
| 3581 | return false; |
| 3582 | } |
| 3583 | // All conditions met. The `SubViewOp` is foldable as a no-op. |
| 3584 | return true; |
| 3585 | } |
| 3586 | |
| 3587 | namespace { |
| 3588 | /// Pattern to rewrite a subview op with MemRefCast arguments. |
| 3589 | /// This essentially pushes memref.cast past its consuming subview when |
| 3590 | /// `canFoldIntoConsumerOp` is true. |
| 3591 | /// |
| 3592 | /// Example: |
| 3593 | /// ``` |
| 3594 | /// %0 = memref.cast %V : memref<16x16xf32> to memref<?x?xf32> |
| 3595 | /// %1 = memref.subview %0[0, 0][3, 4][1, 1] : |
| 3596 | /// memref<?x?xf32> to memref<3x4xf32, strided<[?, 1], offset: ?>> |
| 3597 | /// ``` |
| 3598 | /// is rewritten into: |
| 3599 | /// ``` |
| 3600 | /// %0 = memref.subview %V: memref<16x16xf32> to memref<3x4xf32, #[[map0]]> |
| 3601 | /// %1 = memref.cast %0: memref<3x4xf32, strided<[16, 1], offset: 0>> to |
| 3602 | /// memref<3x4xf32, strided<[?, 1], offset: ?>> |
| 3603 | /// ``` |
| 3604 | class SubViewOpMemRefCastFolder final : public OpRewritePattern<SubViewOp> { |
| 3605 | public: |
| 3606 | using OpRewritePattern<SubViewOp>::OpRewritePattern; |
| 3607 | |
| 3608 | LogicalResult matchAndRewrite(SubViewOp subViewOp, |
| 3609 | PatternRewriter &rewriter) const override { |
| 3610 | // Any constant operand, just return to let SubViewOpConstantFolder kick |
| 3611 | // in. |
| 3612 | if (llvm::any_of(Range: subViewOp.getOperands(), P: [](Value operand) { |
| 3613 | return matchPattern(value: operand, pattern: matchConstantIndex()); |
| 3614 | })) |
| 3615 | return failure(); |
| 3616 | |
| 3617 | auto castOp = subViewOp.getSource().getDefiningOp<CastOp>(); |
| 3618 | if (!castOp) |
| 3619 | return failure(); |
| 3620 | |
| 3621 | if (!CastOp::canFoldIntoConsumerOp(castOp)) |
| 3622 | return failure(); |
| 3623 | |
| 3624 | // Compute the SubViewOp result type after folding the MemRefCastOp. Use |
| 3625 | // the MemRefCastOp source operand type to infer the result type and the |
| 3626 | // current SubViewOp source operand type to compute the dropped dimensions |
| 3627 | // if the operation is rank-reducing. |
| 3628 | auto resultType = getCanonicalSubViewResultType( |
| 3629 | currentResultType: subViewOp.getType(), currentSourceType: subViewOp.getSourceType(), |
| 3630 | sourceType: llvm::cast<MemRefType>(Val: castOp.getSource().getType()), |
| 3631 | mixedOffsets: subViewOp.getMixedOffsets(), mixedSizes: subViewOp.getMixedSizes(), |
| 3632 | mixedStrides: subViewOp.getMixedStrides()); |
| 3633 | if (!resultType) |
| 3634 | return failure(); |
| 3635 | |
| 3636 | Value newSubView = SubViewOp::create( |
| 3637 | builder&: rewriter, location: subViewOp.getLoc(), result: resultType, source: castOp.getSource(), |
| 3638 | offsets: subViewOp.getOffsets(), sizes: subViewOp.getSizes(), strides: subViewOp.getStrides(), |
| 3639 | static_offsets: subViewOp.getStaticOffsets(), static_sizes: subViewOp.getStaticSizes(), |
| 3640 | static_strides: subViewOp.getStaticStrides()); |
| 3641 | rewriter.replaceOpWithNewOp<CastOp>(op: subViewOp, args: subViewOp.getType(), |
| 3642 | args&: newSubView); |
| 3643 | return success(); |
| 3644 | } |
| 3645 | }; |
| 3646 | |
| 3647 | /// Canonicalize subview ops that are no-ops. When the source shape is not |
| 3648 | /// same as a result shape due to use of `affine_map`. |
| 3649 | class TrivialSubViewOpFolder final : public OpRewritePattern<SubViewOp> { |
| 3650 | public: |
| 3651 | using OpRewritePattern<SubViewOp>::OpRewritePattern; |
| 3652 | |
| 3653 | LogicalResult matchAndRewrite(SubViewOp subViewOp, |
| 3654 | PatternRewriter &rewriter) const override { |
| 3655 | if (!isTrivialSubViewOp(subViewOp)) |
| 3656 | return failure(); |
| 3657 | if (subViewOp.getSourceType() == subViewOp.getType()) { |
| 3658 | rewriter.replaceOp(op: subViewOp, newValues: subViewOp.getSource()); |
| 3659 | return success(); |
| 3660 | } |
| 3661 | rewriter.replaceOpWithNewOp<CastOp>(op: subViewOp, args: subViewOp.getType(), |
| 3662 | args: subViewOp.getSource()); |
| 3663 | return success(); |
| 3664 | } |
| 3665 | }; |
| 3666 | } // namespace |
| 3667 | |
| 3668 | /// Return the canonical type of the result of a subview. |
| 3669 | struct SubViewReturnTypeCanonicalizer { |
| 3670 | MemRefType operator()(SubViewOp op, ArrayRef<OpFoldResult> mixedOffsets, |
| 3671 | ArrayRef<OpFoldResult> mixedSizes, |
| 3672 | ArrayRef<OpFoldResult> mixedStrides) { |
| 3673 | // Infer a memref type without taking into account any rank reductions. |
| 3674 | MemRefType resTy = SubViewOp::inferResultType( |
| 3675 | sourceMemRefType: op.getSourceType(), offsets: mixedOffsets, sizes: mixedSizes, strides: mixedStrides); |
| 3676 | if (!resTy) |
| 3677 | return {}; |
| 3678 | MemRefType nonReducedType = resTy; |
| 3679 | |
| 3680 | // Directly return the non-rank reduced type if there are no dropped dims. |
| 3681 | llvm::SmallBitVector droppedDims = op.getDroppedDims(); |
| 3682 | if (droppedDims.none()) |
| 3683 | return nonReducedType; |
| 3684 | |
| 3685 | // Take the strides and offset from the non-rank reduced type. |
| 3686 | auto [nonReducedStrides, offset] = nonReducedType.getStridesAndOffset(); |
| 3687 | |
| 3688 | // Drop dims from shape and strides. |
| 3689 | SmallVector<int64_t> targetShape; |
| 3690 | SmallVector<int64_t> targetStrides; |
| 3691 | for (int64_t i = 0; i < static_cast<int64_t>(mixedSizes.size()); ++i) { |
| 3692 | if (droppedDims.test(Idx: i)) |
| 3693 | continue; |
| 3694 | targetStrides.push_back(Elt: nonReducedStrides[i]); |
| 3695 | targetShape.push_back(Elt: nonReducedType.getDimSize(idx: i)); |
| 3696 | } |
| 3697 | |
| 3698 | return MemRefType::get(shape: targetShape, elementType: nonReducedType.getElementType(), |
| 3699 | layout: StridedLayoutAttr::get(context: nonReducedType.getContext(), |
| 3700 | offset, strides: targetStrides), |
| 3701 | memorySpace: nonReducedType.getMemorySpace()); |
| 3702 | } |
| 3703 | }; |
| 3704 | |
| 3705 | /// A canonicalizer wrapper to replace SubViewOps. |
| 3706 | struct SubViewCanonicalizer { |
| 3707 | void operator()(PatternRewriter &rewriter, SubViewOp op, SubViewOp newOp) { |
| 3708 | rewriter.replaceOpWithNewOp<CastOp>(op, args: op.getType(), args&: newOp); |
| 3709 | } |
| 3710 | }; |
| 3711 | |
| 3712 | void SubViewOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 3713 | MLIRContext *context) { |
| 3714 | results |
| 3715 | .add<OpWithOffsetSizesAndStridesConstantArgumentFolder< |
| 3716 | SubViewOp, SubViewReturnTypeCanonicalizer, SubViewCanonicalizer>, |
| 3717 | SubViewOpMemRefCastFolder, TrivialSubViewOpFolder>(arg&: context); |
| 3718 | } |
| 3719 | |
| 3720 | OpFoldResult SubViewOp::fold(FoldAdaptor adaptor) { |
| 3721 | MemRefType sourceMemrefType = getSource().getType(); |
| 3722 | MemRefType resultMemrefType = getResult().getType(); |
| 3723 | auto resultLayout = |
| 3724 | dyn_cast_if_present<StridedLayoutAttr>(Val: resultMemrefType.getLayout()); |
| 3725 | |
| 3726 | if (resultMemrefType == sourceMemrefType && |
| 3727 | resultMemrefType.hasStaticShape() && |
| 3728 | (!resultLayout || resultLayout.hasStaticLayout())) { |
| 3729 | return getViewSource(); |
| 3730 | } |
| 3731 | |
| 3732 | // Fold subview(subview(x)), where both subviews have the same size and the |
| 3733 | // second subview's offsets are all zero. (I.e., the second subview is a |
| 3734 | // no-op.) |
| 3735 | if (auto srcSubview = getViewSource().getDefiningOp<SubViewOp>()) { |
| 3736 | auto srcSizes = srcSubview.getMixedSizes(); |
| 3737 | auto sizes = getMixedSizes(); |
| 3738 | auto offsets = getMixedOffsets(); |
| 3739 | bool allOffsetsZero = llvm::all_of(Range&: offsets, P: isZeroInteger); |
| 3740 | auto strides = getMixedStrides(); |
| 3741 | bool allStridesOne = llvm::all_of(Range&: strides, P: isOneInteger); |
| 3742 | bool allSizesSame = llvm::equal(LRange&: sizes, RRange&: srcSizes); |
| 3743 | if (allOffsetsZero && allStridesOne && allSizesSame && |
| 3744 | resultMemrefType == sourceMemrefType) |
| 3745 | return getViewSource(); |
| 3746 | } |
| 3747 | |
| 3748 | return {}; |
| 3749 | } |
| 3750 | |
| 3751 | FailureOr<std::optional<SmallVector<Value>>> |
| 3752 | SubViewOp::bubbleDownCasts(OpBuilder &builder) { |
| 3753 | return bubbleDownCastsPassthroughOpImpl(op: *this, builder, src&: getSourceMutable()); |
| 3754 | } |
| 3755 | |
| 3756 | void SubViewOp::inferStridedMetadataRanges( |
| 3757 | ArrayRef<StridedMetadataRange> ranges, GetIntRangeFn getIntRange, |
| 3758 | SetStridedMetadataRangeFn setMetadata, int32_t indexBitwidth) { |
| 3759 | auto isUninitialized = |
| 3760 | +[](IntegerValueRange range) { return range.isUninitialized(); }; |
| 3761 | |
| 3762 | // Bail early if any of the operands metadata is not ready: |
| 3763 | SmallVector<IntegerValueRange> offsetOperands = |
| 3764 | getIntValueRanges(values: getMixedOffsets(), getIntRange, indexBitwidth); |
| 3765 | if (llvm::any_of(Range&: offsetOperands, P: isUninitialized)) |
| 3766 | return; |
| 3767 | |
| 3768 | SmallVector<IntegerValueRange> sizeOperands = |
| 3769 | getIntValueRanges(values: getMixedSizes(), getIntRange, indexBitwidth); |
| 3770 | if (llvm::any_of(Range&: sizeOperands, P: isUninitialized)) |
| 3771 | return; |
| 3772 | |
| 3773 | SmallVector<IntegerValueRange> stridesOperands = |
| 3774 | getIntValueRanges(values: getMixedStrides(), getIntRange, indexBitwidth); |
| 3775 | if (llvm::any_of(Range&: stridesOperands, P: isUninitialized)) |
| 3776 | return; |
| 3777 | |
| 3778 | StridedMetadataRange sourceRange = |
| 3779 | ranges[getSourceMutable().getOperandNumber()]; |
| 3780 | if (sourceRange.isUninitialized()) |
| 3781 | return; |
| 3782 | |
| 3783 | ArrayRef<ConstantIntRanges> srcStrides = sourceRange.getStrides(); |
| 3784 | |
| 3785 | // Get the dropped dims. |
| 3786 | llvm::SmallBitVector droppedDims = getDroppedDims(); |
| 3787 | |
| 3788 | // Compute the new offset, strides and sizes. |
| 3789 | ConstantIntRanges offset = sourceRange.getOffsets()[0]; |
| 3790 | SmallVector<ConstantIntRanges> strides, sizes; |
| 3791 | |
| 3792 | for (size_t i = 0, e = droppedDims.size(); i < e; ++i) { |
| 3793 | bool dropped = droppedDims.test(Idx: i); |
| 3794 | // Compute the new offset. |
| 3795 | ConstantIntRanges off = |
| 3796 | intrange::inferMul(argRanges: {offsetOperands[i].getValue(), srcStrides[i]}); |
| 3797 | offset = intrange::inferAdd(argRanges: {offset, off}); |
| 3798 | |
| 3799 | // Skip dropped dimensions. |
| 3800 | if (dropped) |
| 3801 | continue; |
| 3802 | // Multiply the strides. |
| 3803 | strides.push_back( |
| 3804 | Elt: intrange::inferMul(argRanges: {stridesOperands[i].getValue(), srcStrides[i]})); |
| 3805 | // Get the sizes. |
| 3806 | sizes.push_back(Elt: sizeOperands[i].getValue()); |
| 3807 | } |
| 3808 | |
| 3809 | setMetadata(getResult(), |
| 3810 | StridedMetadataRange::getRanked( |
| 3811 | offsets: SmallVector<ConstantIntRanges>({std::move(offset)}), |
| 3812 | sizes: std::move(sizes), strides: std::move(strides))); |
| 3813 | } |
| 3814 | |
| 3815 | //===----------------------------------------------------------------------===// |
| 3816 | // TransposeOp |
| 3817 | //===----------------------------------------------------------------------===// |
| 3818 | |
| 3819 | void TransposeOp::getAsmResultNames( |
| 3820 | function_ref<void(Value, StringRef)> setNameFn) { |
| 3821 | setNameFn(getResult(), "transpose" ); |
| 3822 | } |
| 3823 | |
| 3824 | /// Build a strided memref type by applying `permutationMap` to `memRefType`. |
| 3825 | static MemRefType inferTransposeResultType(MemRefType memRefType, |
| 3826 | AffineMap permutationMap) { |
| 3827 | auto originalSizes = memRefType.getShape(); |
| 3828 | auto [originalStrides, offset] = memRefType.getStridesAndOffset(); |
| 3829 | assert(originalStrides.size() == static_cast<unsigned>(memRefType.getRank())); |
| 3830 | |
| 3831 | // Compute permuted sizes and strides. |
| 3832 | auto sizes = applyPermutationMap<int64_t>(map: permutationMap, source: originalSizes); |
| 3833 | auto strides = applyPermutationMap<int64_t>(map: permutationMap, source: originalStrides); |
| 3834 | |
| 3835 | return MemRefType::Builder(memRefType) |
| 3836 | .setShape(sizes) |
| 3837 | .setLayout( |
| 3838 | StridedLayoutAttr::get(context: memRefType.getContext(), offset, strides)); |
| 3839 | } |
| 3840 | |
| 3841 | Value TransposeOp::getViewSource() { return getIn(); } |
| 3842 | |
| 3843 | void TransposeOp::build(OpBuilder &b, OperationState &result, Value in, |
| 3844 | AffineMapAttr permutation, |
| 3845 | ArrayRef<NamedAttribute> attrs) { |
| 3846 | auto permutationMap = permutation.getValue(); |
| 3847 | assert(permutationMap); |
| 3848 | |
| 3849 | auto memRefType = llvm::cast<MemRefType>(Val: in.getType()); |
| 3850 | // Compute result type. |
| 3851 | MemRefType resultType = inferTransposeResultType(memRefType, permutationMap); |
| 3852 | |
| 3853 | buildPropertiesAndDiscardableAttributes(odsState&: result, attributes: attrs); |
| 3854 | result.getOrAddProperties<Properties>().permutation = permutation; |
| 3855 | result.addOperands(newOperands: in); |
| 3856 | result.addTypes(newTypes: resultType); |
| 3857 | } |
| 3858 | |
| 3859 | // transpose $in $permutation attr-dict : type($in) `to` type(results) |
| 3860 | void TransposeOp::print(OpAsmPrinter &p) { |
| 3861 | p << " " << getIn() << " " << getPermutation(); |
| 3862 | p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary(), |
| 3863 | elidedAttrs: {getPermutationAttrStrName()}); |
| 3864 | p << " : " << getIn().getType() << " to " << getType(); |
| 3865 | } |
| 3866 | |
| 3867 | ParseResult TransposeOp::parse(OpAsmParser &parser, OperationState &result) { |
| 3868 | OpAsmParser::UnresolvedOperand in; |
| 3869 | AffineMap permutation; |
| 3870 | MemRefType srcType, dstType; |
| 3871 | if (parser.parseOperand(result&: in) || parser.parseAffineMap(map&: permutation) || |
| 3872 | parser.parseOptionalAttrDict(result&: result.attributes) || |
| 3873 | parser.parseColonType(result&: srcType) || |
| 3874 | parser.resolveOperand(operand: in, type: srcType, result&: result.operands) || |
| 3875 | parser.parseKeywordType(keyword: "to" , result&: dstType) || |
| 3876 | parser.addTypeToList(type: dstType, result&: result.types)) |
| 3877 | return failure(); |
| 3878 | |
| 3879 | result.addAttribute(name: TransposeOp::getPermutationAttrStrName(), |
| 3880 | attr: AffineMapAttr::get(value: permutation)); |
| 3881 | return success(); |
| 3882 | } |
| 3883 | |
| 3884 | LogicalResult TransposeOp::verify() { |
| 3885 | if (!getPermutation().isPermutation()) |
| 3886 | return emitOpError(message: "expected a permutation map" ); |
| 3887 | if (getPermutation().getNumDims() != getIn().getType().getRank()) |
| 3888 | return emitOpError(message: "expected a permutation map of same rank as the input" ); |
| 3889 | |
| 3890 | auto srcType = llvm::cast<MemRefType>(Val: getIn().getType()); |
| 3891 | auto resultType = llvm::cast<MemRefType>(Val: getType()); |
| 3892 | auto canonicalResultType = inferTransposeResultType(memRefType: srcType, permutationMap: getPermutation()) |
| 3893 | .canonicalizeStridedLayout(); |
| 3894 | |
| 3895 | if (resultType.canonicalizeStridedLayout() != canonicalResultType) |
| 3896 | return emitOpError(message: "result type " ) |
| 3897 | << resultType |
| 3898 | << " is not equivalent to the canonical transposed input type " |
| 3899 | << canonicalResultType; |
| 3900 | return success(); |
| 3901 | } |
| 3902 | |
| 3903 | OpFoldResult TransposeOp::fold(FoldAdaptor) { |
| 3904 | // First check for identity permutation, we can fold it away if input and |
| 3905 | // result types are identical already. |
| 3906 | if (getPermutation().isIdentity() && getType() == getIn().getType()) |
| 3907 | return getIn(); |
| 3908 | // Fold two consecutive memref.transpose Ops into one by composing their |
| 3909 | // permutation maps. |
| 3910 | if (auto otherTransposeOp = getIn().getDefiningOp<memref::TransposeOp>()) { |
| 3911 | AffineMap composedPermutation = |
| 3912 | getPermutation().compose(map: otherTransposeOp.getPermutation()); |
| 3913 | getInMutable().assign(value: otherTransposeOp.getIn()); |
| 3914 | setPermutation(composedPermutation); |
| 3915 | return getResult(); |
| 3916 | } |
| 3917 | return {}; |
| 3918 | } |
| 3919 | |
| 3920 | FailureOr<std::optional<SmallVector<Value>>> |
| 3921 | TransposeOp::bubbleDownCasts(OpBuilder &builder) { |
| 3922 | return bubbleDownCastsPassthroughOpImpl(op: *this, builder, src&: getInMutable()); |
| 3923 | } |
| 3924 | |
| 3925 | //===----------------------------------------------------------------------===// |
| 3926 | // ViewOp |
| 3927 | //===----------------------------------------------------------------------===// |
| 3928 | |
| 3929 | void ViewOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) { |
| 3930 | setNameFn(getResult(), "view" ); |
| 3931 | } |
| 3932 | |
| 3933 | LogicalResult ViewOp::verify() { |
| 3934 | auto baseType = llvm::cast<MemRefType>(Val: getOperand(i: 0).getType()); |
| 3935 | auto viewType = getType(); |
| 3936 | |
| 3937 | // The base memref should have identity layout map (or none). |
| 3938 | if (!baseType.getLayout().isIdentity()) |
| 3939 | return emitError(message: "unsupported map for base memref type " ) << baseType; |
| 3940 | |
| 3941 | // The result memref should have identity layout map (or none). |
| 3942 | if (!viewType.getLayout().isIdentity()) |
| 3943 | return emitError(message: "unsupported map for result memref type " ) << viewType; |
| 3944 | |
| 3945 | // The base memref and the view memref should be in the same memory space. |
| 3946 | if (baseType.getMemorySpace() != viewType.getMemorySpace()) |
| 3947 | return emitError(message: "different memory spaces specified for base memref " |
| 3948 | "type " ) |
| 3949 | << baseType << " and view memref type " << viewType; |
| 3950 | |
| 3951 | // Verify that we have the correct number of sizes for the result type. |
| 3952 | if (failed(Result: verifyDynamicDimensionCount(op: getOperation(), type: viewType, dynamicSizes: getSizes()))) |
| 3953 | return failure(); |
| 3954 | |
| 3955 | return success(); |
| 3956 | } |
| 3957 | |
| 3958 | Value ViewOp::getViewSource() { return getSource(); } |
| 3959 | |
| 3960 | OpFoldResult ViewOp::fold(FoldAdaptor adaptor) { |
| 3961 | MemRefType sourceMemrefType = getSource().getType(); |
| 3962 | MemRefType resultMemrefType = getResult().getType(); |
| 3963 | |
| 3964 | if (resultMemrefType == sourceMemrefType && |
| 3965 | resultMemrefType.hasStaticShape() && isZeroInteger(v: getByteShift())) |
| 3966 | return getViewSource(); |
| 3967 | |
| 3968 | return {}; |
| 3969 | } |
| 3970 | |
| 3971 | SmallVector<OpFoldResult> ViewOp::getMixedSizes() { |
| 3972 | SmallVector<OpFoldResult> result; |
| 3973 | unsigned ctr = 0; |
| 3974 | Builder b(getContext()); |
| 3975 | for (int64_t dim : getType().getShape()) { |
| 3976 | if (ShapedType::isDynamic(dValue: dim)) { |
| 3977 | result.push_back(Elt: getSizes()[ctr++]); |
| 3978 | } else { |
| 3979 | result.push_back(Elt: b.getIndexAttr(value: dim)); |
| 3980 | } |
| 3981 | } |
| 3982 | return result; |
| 3983 | } |
| 3984 | |
| 3985 | namespace { |
| 3986 | /// Given a memref type and a range of values that defines its dynamic |
| 3987 | /// dimension sizes, turn all dynamic sizes that have a constant value into |
| 3988 | /// static dimension sizes. |
| 3989 | static MemRefType |
| 3990 | foldDynamicToStaticDimSizes(MemRefType type, ValueRange dynamicSizes, |
| 3991 | SmallVectorImpl<Value> &foldedDynamicSizes) { |
| 3992 | SmallVector<int64_t> staticShape(type.getShape()); |
| 3993 | assert(type.getNumDynamicDims() == dynamicSizes.size() && |
| 3994 | "incorrect number of dynamic sizes" ); |
| 3995 | |
| 3996 | // Compute new static and dynamic sizes. |
| 3997 | unsigned ctr = 0; |
| 3998 | for (auto [dim, dimSize] : llvm::enumerate(First: type.getShape())) { |
| 3999 | if (ShapedType::isStatic(dValue: dimSize)) |
| 4000 | continue; |
| 4001 | |
| 4002 | Value dynamicSize = dynamicSizes[ctr++]; |
| 4003 | if (auto cst = getConstantIntValue(ofr: dynamicSize)) { |
| 4004 | // Dynamic size must be non-negative. |
| 4005 | if (cst.value() < 0) { |
| 4006 | foldedDynamicSizes.push_back(Elt: dynamicSize); |
| 4007 | continue; |
| 4008 | } |
| 4009 | staticShape[dim] = cst.value(); |
| 4010 | } else { |
| 4011 | foldedDynamicSizes.push_back(Elt: dynamicSize); |
| 4012 | } |
| 4013 | } |
| 4014 | |
| 4015 | return MemRefType::Builder(type).setShape(staticShape); |
| 4016 | } |
| 4017 | |
| 4018 | /// Change the result type of a `memref.view` by making originally dynamic |
| 4019 | /// dimensions static when their sizes come from `constant` ops. |
| 4020 | /// Example: |
| 4021 | /// ``` |
| 4022 | /// %c5 = arith.constant 5: index |
| 4023 | /// %0 = memref.view %src[%offset][%c5] : memref<?xi8> to memref<?x4xf32> |
| 4024 | /// ``` |
| 4025 | /// to |
| 4026 | /// ``` |
| 4027 | /// %0 = memref.view %src[%offset][] : memref<?xi8> to memref<5x4xf32> |
| 4028 | /// ``` |
| 4029 | struct ViewOpShapeFolder : public OpRewritePattern<ViewOp> { |
| 4030 | using Base::Base; |
| 4031 | |
| 4032 | LogicalResult matchAndRewrite(ViewOp viewOp, |
| 4033 | PatternRewriter &rewriter) const override { |
| 4034 | SmallVector<Value> foldedDynamicSizes; |
| 4035 | MemRefType resultType = viewOp.getType(); |
| 4036 | MemRefType foldedMemRefType = foldDynamicToStaticDimSizes( |
| 4037 | type: resultType, dynamicSizes: viewOp.getSizes(), foldedDynamicSizes); |
| 4038 | |
| 4039 | // Stop here if no dynamic size was promoted to static. |
| 4040 | if (foldedMemRefType == resultType) |
| 4041 | return failure(); |
| 4042 | |
| 4043 | // Create new ViewOp. |
| 4044 | auto newViewOp = ViewOp::create(builder&: rewriter, location: viewOp.getLoc(), resultType0: foldedMemRefType, |
| 4045 | source: viewOp.getSource(), byte_shift: viewOp.getByteShift(), |
| 4046 | sizes: foldedDynamicSizes); |
| 4047 | // Insert a cast so we have the same type as the old memref type. |
| 4048 | rewriter.replaceOpWithNewOp<CastOp>(op: viewOp, args&: resultType, args&: newViewOp); |
| 4049 | return success(); |
| 4050 | } |
| 4051 | }; |
| 4052 | |
| 4053 | /// view(memref.cast(%source)) -> view(%source). |
| 4054 | struct ViewOpMemrefCastFolder : public OpRewritePattern<ViewOp> { |
| 4055 | using Base::Base; |
| 4056 | |
| 4057 | LogicalResult matchAndRewrite(ViewOp viewOp, |
| 4058 | PatternRewriter &rewriter) const override { |
| 4059 | auto memrefCastOp = viewOp.getSource().getDefiningOp<CastOp>(); |
| 4060 | if (!memrefCastOp) |
| 4061 | return failure(); |
| 4062 | |
| 4063 | rewriter.replaceOpWithNewOp<ViewOp>( |
| 4064 | op: viewOp, args: viewOp.getType(), args: memrefCastOp.getSource(), |
| 4065 | args: viewOp.getByteShift(), args: viewOp.getSizes()); |
| 4066 | return success(); |
| 4067 | } |
| 4068 | }; |
| 4069 | } // namespace |
| 4070 | |
| 4071 | void ViewOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 4072 | MLIRContext *context) { |
| 4073 | results.add<ViewOpShapeFolder, ViewOpMemrefCastFolder>(arg&: context); |
| 4074 | } |
| 4075 | |
| 4076 | FailureOr<std::optional<SmallVector<Value>>> |
| 4077 | ViewOp::bubbleDownCasts(OpBuilder &builder) { |
| 4078 | return bubbleDownCastsPassthroughOpImpl(op: *this, builder, src&: getSourceMutable()); |
| 4079 | } |
| 4080 | |
| 4081 | //===----------------------------------------------------------------------===// |
| 4082 | // AtomicRMWOp |
| 4083 | //===----------------------------------------------------------------------===// |
| 4084 | |
| 4085 | LogicalResult AtomicRMWOp::verify() { |
| 4086 | switch (getKind()) { |
| 4087 | case arith::AtomicRMWKind::addf: |
| 4088 | case arith::AtomicRMWKind::maximumf: |
| 4089 | case arith::AtomicRMWKind::minimumf: |
| 4090 | case arith::AtomicRMWKind::mulf: |
| 4091 | if (!llvm::isa<FloatType>(Val: getValue().getType())) |
| 4092 | return emitOpError() << "with kind '" |
| 4093 | << arith::stringifyAtomicRMWKind(getKind()) |
| 4094 | << "' expects a floating-point type" ; |
| 4095 | break; |
| 4096 | case arith::AtomicRMWKind::addi: |
| 4097 | case arith::AtomicRMWKind::maxs: |
| 4098 | case arith::AtomicRMWKind::maxu: |
| 4099 | case arith::AtomicRMWKind::mins: |
| 4100 | case arith::AtomicRMWKind::minu: |
| 4101 | case arith::AtomicRMWKind::muli: |
| 4102 | case arith::AtomicRMWKind::ori: |
| 4103 | case arith::AtomicRMWKind::xori: |
| 4104 | case arith::AtomicRMWKind::andi: |
| 4105 | if (!llvm::isa<IntegerType>(Val: getValue().getType())) |
| 4106 | return emitOpError() << "with kind '" |
| 4107 | << arith::stringifyAtomicRMWKind(getKind()) |
| 4108 | << "' expects an integer type" ; |
| 4109 | break; |
| 4110 | default: |
| 4111 | break; |
| 4112 | } |
| 4113 | return success(); |
| 4114 | } |
| 4115 | |
| 4116 | OpFoldResult AtomicRMWOp::fold(FoldAdaptor adaptor) { |
| 4117 | /// atomicrmw(memrefcast) -> atomicrmw |
| 4118 | if (succeeded(Result: foldMemRefCast(op: *this, inner: getValue()))) |
| 4119 | return getResult(); |
| 4120 | return OpFoldResult(); |
| 4121 | } |
| 4122 | |
| 4123 | FailureOr<std::optional<SmallVector<Value>>> |
| 4124 | AtomicRMWOp::bubbleDownCasts(OpBuilder &builder) { |
| 4125 | return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getMemrefMutable(), |
| 4126 | results: getResult()); |
| 4127 | } |
| 4128 | |
| 4129 | TypedValue<MemRefType> AtomicRMWOp::getAccessedMemref() { return getMemref(); } |
| 4130 | |
| 4131 | std::optional<SmallVector<Value>> |
| 4132 | AtomicRMWOp::updateMemrefAndIndices(RewriterBase &rewriter, Value newMemref, |
| 4133 | ValueRange newIndices) { |
| 4134 | rewriter.modifyOpInPlace(root: *this, callable: [&]() { |
| 4135 | getMemrefMutable().assign(value: newMemref); |
| 4136 | getIndicesMutable().assign(values: newIndices); |
| 4137 | }); |
| 4138 | return std::nullopt; |
| 4139 | } |
| 4140 | |
| 4141 | //===----------------------------------------------------------------------===// |
| 4142 | // TableGen'd op method definitions |
| 4143 | //===----------------------------------------------------------------------===// |
| 4144 | |
| 4145 | #define GET_OP_CLASSES |
| 4146 | #include "mlir/Dialect/MemRef/IR/MemRefOps.cpp.inc" |
| 4147 | |