| 1 | //===- MemRefTransformOps.cpp - Implementation of Memref transform ops ----===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.h" |
| 10 | |
| 11 | #include "mlir/Analysis/DataLayoutAnalysis.h" |
| 12 | #include "mlir/Conversion/LLVMCommon/TypeConverter.h" |
| 13 | #include "mlir/Dialect/Affine/IR/AffineDialect.h" |
| 14 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 15 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 16 | #include "mlir/Dialect/MemRef/Transforms/Passes.h" |
| 17 | #include "mlir/Dialect/MemRef/Transforms/Transforms.h" |
| 18 | #include "mlir/Dialect/MemRef/Utils/MemRefUtils.h" |
| 19 | #include "mlir/Dialect/NVGPU/IR/NVGPUDialectDecl.h" |
| 20 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 21 | #include "mlir/Dialect/Transform/IR/TransformDialect.h" |
| 22 | #include "mlir/Dialect/Transform/IR/TransformTypes.h" |
| 23 | #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" |
| 24 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 25 | #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h" |
| 26 | #include "mlir/Interfaces/LoopLikeInterface.h" |
| 27 | #include "llvm/Support/Debug.h" |
| 28 | |
| 29 | using namespace mlir; |
| 30 | |
| 31 | #define DEBUG_TYPE "memref-transforms" |
| 32 | #define DBGS() (llvm::dbgs() << '[' << DEBUG_TYPE << "] ") |
| 33 | |
| 34 | namespace mlir::transform { |
| 35 | namespace { |
| 36 | ParseResult parseLLVMTypeConverterOptions(OpAsmParser &parser, |
| 37 | BoolAttr &useAlignedAlloc, |
| 38 | IntegerAttr &indexBitwidth, |
| 39 | BoolAttr &useGenericFunctions, |
| 40 | BoolAttr &useBarePtrCallConv, |
| 41 | StringAttr &dataLayout) { |
| 42 | bool seenUseAlignedAlloc = false; |
| 43 | bool seenIndexBitwidth = false; |
| 44 | bool seenUseGenericFunctions = false; |
| 45 | bool seenUseBarePtrCallConv = false; |
| 46 | bool seenDataLayout = false; |
| 47 | |
| 48 | auto parseDuplicate = [&](StringRef name, bool &seen) -> ParseResult { |
| 49 | if (seen) |
| 50 | return parser.emitError(loc: parser.getCurrentLocation()) |
| 51 | << "duplicate '" << name << "' option" ; |
| 52 | seen = true; |
| 53 | return ParseResult::success(); |
| 54 | }; |
| 55 | |
| 56 | while (true) { |
| 57 | if (succeeded(Result: parser.parseOptionalKeyword(keyword: "use_aligned_alloc" ))) { |
| 58 | if (failed(Result: parseDuplicate("use_aligned_alloc" , seenUseAlignedAlloc)) || |
| 59 | parser.parseEqual() || |
| 60 | parser.parseAttribute(result&: useAlignedAlloc, |
| 61 | type: parser.getBuilder().getI1Type())) |
| 62 | return failure(); |
| 63 | continue; |
| 64 | } |
| 65 | if (succeeded(Result: parser.parseOptionalKeyword(keyword: "index_bitwidth" ))) { |
| 66 | if (failed(Result: parseDuplicate("index_bitwidth" , seenIndexBitwidth)) || |
| 67 | parser.parseEqual() || |
| 68 | parser.parseAttribute(result&: indexBitwidth, |
| 69 | type: parser.getBuilder().getI64Type())) |
| 70 | return failure(); |
| 71 | continue; |
| 72 | } |
| 73 | if (succeeded(Result: parser.parseOptionalKeyword(keyword: "use_generic_functions" ))) { |
| 74 | if (failed(Result: parseDuplicate("use_generic_functions" , |
| 75 | seenUseGenericFunctions)) || |
| 76 | parser.parseEqual() || |
| 77 | parser.parseAttribute(result&: useGenericFunctions, |
| 78 | type: parser.getBuilder().getI1Type())) |
| 79 | return failure(); |
| 80 | continue; |
| 81 | } |
| 82 | if (succeeded(Result: parser.parseOptionalKeyword(keyword: "use_bare_ptr_call_conv" ))) { |
| 83 | if (failed(Result: parseDuplicate("use_bare_ptr_call_conv" , |
| 84 | seenUseBarePtrCallConv)) || |
| 85 | parser.parseEqual() || |
| 86 | parser.parseAttribute(result&: useBarePtrCallConv, |
| 87 | type: parser.getBuilder().getI1Type())) |
| 88 | return failure(); |
| 89 | continue; |
| 90 | } |
| 91 | if (succeeded(Result: parser.parseOptionalKeyword(keyword: "data_layout" ))) { |
| 92 | if (failed(Result: parseDuplicate("data_layout" , seenDataLayout)) || |
| 93 | parser.parseEqual() || parser.parseAttribute(result&: dataLayout)) |
| 94 | return failure(); |
| 95 | continue; |
| 96 | } |
| 97 | break; |
| 98 | } |
| 99 | return success(); |
| 100 | } |
| 101 | |
| 102 | void printLLVMTypeConverterOptions(OpAsmPrinter &printer, Operation *, |
| 103 | BoolAttr useAlignedAlloc, |
| 104 | IntegerAttr indexBitwidth, |
| 105 | BoolAttr useGenericFunctions, |
| 106 | BoolAttr useBarePtrCallConv, |
| 107 | StringAttr dataLayout) { |
| 108 | bool needsSpace = false; |
| 109 | auto printOption = [&](StringRef name, Attribute value) { |
| 110 | if (!value) |
| 111 | return; |
| 112 | if (needsSpace) |
| 113 | printer << ' '; |
| 114 | printer << name << " = " ; |
| 115 | printer.printAttributeWithoutType(attr: value); |
| 116 | needsSpace = true; |
| 117 | }; |
| 118 | |
| 119 | printOption("use_aligned_alloc" , useAlignedAlloc); |
| 120 | printOption("index_bitwidth" , indexBitwidth); |
| 121 | printOption("use_generic_functions" , useGenericFunctions); |
| 122 | printOption("use_bare_ptr_call_conv" , useBarePtrCallConv); |
| 123 | printOption("data_layout" , dataLayout); |
| 124 | } |
| 125 | } // namespace |
| 126 | } // namespace mlir::transform |
| 127 | |
| 128 | //===----------------------------------------------------------------------===// |
| 129 | // Apply...ConversionPatternsOp |
| 130 | //===----------------------------------------------------------------------===// |
| 131 | |
| 132 | std::unique_ptr<TypeConverter> |
| 133 | transform::MemrefToLLVMTypeConverterOp::getTypeConverter() { |
| 134 | LowerToLLVMOptions options(getContext()); |
| 135 | options.allocLowering = |
| 136 | (getUseAlignedAlloc() ? LowerToLLVMOptions::AllocLowering::AlignedAlloc |
| 137 | : LowerToLLVMOptions::AllocLowering::Malloc); |
| 138 | options.useGenericFunctions = getUseGenericFunctions(); |
| 139 | |
| 140 | if (getIndexBitwidth() != kDeriveIndexBitwidthFromDataLayout) |
| 141 | options.overrideIndexBitwidth(bitwidth: getIndexBitwidth()); |
| 142 | |
| 143 | // TODO: the following two options don't really make sense for |
| 144 | // memref_to_llvm_type_converter specifically but we should have a single |
| 145 | // to_llvm_type_converter. |
| 146 | if (getDataLayout().has_value()) |
| 147 | options.dataLayout = llvm::DataLayout(getDataLayout().value()); |
| 148 | options.useBarePtrCallConv = getUseBarePtrCallConv(); |
| 149 | |
| 150 | return std::make_unique<LLVMTypeConverter>(args: getContext(), args&: options); |
| 151 | } |
| 152 | |
| 153 | StringRef transform::MemrefToLLVMTypeConverterOp::getTypeConverterType() { |
| 154 | return "LLVMTypeConverter" ; |
| 155 | } |
| 156 | |
| 157 | //===----------------------------------------------------------------------===// |
| 158 | // Apply...PatternsOp |
| 159 | //===----------------------------------------------------------------------===// |
| 160 | |
| 161 | namespace { |
| 162 | class AllocToAllocaPattern : public OpRewritePattern<memref::AllocOp> { |
| 163 | public: |
| 164 | explicit AllocToAllocaPattern(Operation *analysisRoot, int64_t maxSize = 0) |
| 165 | : OpRewritePattern<memref::AllocOp>(analysisRoot->getContext()), |
| 166 | dataLayoutAnalysis(analysisRoot), maxSize(maxSize) {} |
| 167 | |
| 168 | LogicalResult matchAndRewrite(memref::AllocOp op, |
| 169 | PatternRewriter &rewriter) const override { |
| 170 | return success(IsSuccess: memref::allocToAlloca( |
| 171 | rewriter, alloc: op, filter: [this](memref::AllocOp alloc, memref::DeallocOp dealloc) { |
| 172 | MemRefType type = alloc.getMemref().getType(); |
| 173 | if (!type.hasStaticShape()) |
| 174 | return false; |
| 175 | |
| 176 | const DataLayout &dataLayout = dataLayoutAnalysis.getAtOrAbove(operation: alloc); |
| 177 | int64_t elementSize = dataLayout.getTypeSize(t: type.getElementType()); |
| 178 | return maxSize == 0 || type.getNumElements() * elementSize < maxSize; |
| 179 | })); |
| 180 | } |
| 181 | |
| 182 | private: |
| 183 | DataLayoutAnalysis dataLayoutAnalysis; |
| 184 | int64_t maxSize; |
| 185 | }; |
| 186 | } // namespace |
| 187 | |
| 188 | void transform::ApplyAllocToAllocaOp::populatePatterns( |
| 189 | RewritePatternSet &patterns) {} |
| 190 | |
| 191 | void transform::ApplyAllocToAllocaOp::populatePatternsWithState( |
| 192 | RewritePatternSet &patterns, transform::TransformState &state) { |
| 193 | patterns.insert<AllocToAllocaPattern>( |
| 194 | arg: state.getTopLevel(), args: static_cast<int64_t>(getSizeLimit().value_or(u: 0))); |
| 195 | } |
| 196 | |
| 197 | void transform::ApplyExpandOpsPatternsOp::populatePatterns( |
| 198 | RewritePatternSet &patterns) { |
| 199 | memref::populateExpandOpsPatterns(patterns); |
| 200 | } |
| 201 | |
| 202 | void transform::ApplyExpandStridedMetadataPatternsOp::populatePatterns( |
| 203 | RewritePatternSet &patterns) { |
| 204 | memref::populateExpandStridedMetadataPatterns(patterns); |
| 205 | } |
| 206 | |
| 207 | void transform::ApplyExtractAddressComputationsPatternsOp::( |
| 208 | RewritePatternSet &patterns) { |
| 209 | memref::populateExtractAddressComputationsPatterns(patterns); |
| 210 | } |
| 211 | |
| 212 | void transform::ApplyFoldMemrefAliasOpsPatternsOp::populatePatterns( |
| 213 | RewritePatternSet &patterns) { |
| 214 | memref::populateFoldMemRefAliasOpPatterns(patterns); |
| 215 | } |
| 216 | |
| 217 | void transform::ApplyResolveRankedShapedTypeResultDimsPatternsOp:: |
| 218 | populatePatterns(RewritePatternSet &patterns) { |
| 219 | memref::populateResolveRankedShapedTypeResultDimsPatterns(patterns); |
| 220 | } |
| 221 | |
| 222 | //===----------------------------------------------------------------------===// |
| 223 | // Alloc and alloca to global utilities |
| 224 | //===----------------------------------------------------------------------===// |
| 225 | |
| 226 | /// Checks whether an allocation operation can be converted to a |
| 227 | /// `memref.global`. |
| 228 | template <typename AllocLikeOp> |
| 229 | static DiagnosedSilenceableFailure |
| 230 | checkAllocToGlobalPreconditions(AllocLikeOp allocLikeOp) { |
| 231 | MemRefType memrefType = allocLikeOp.getType(); |
| 232 | if (!memrefType.hasStaticShape()) { |
| 233 | return emitSilenceableFailure(allocLikeOp) |
| 234 | << "conversion to a global op requires statically shaped memrefs, " |
| 235 | "but got " |
| 236 | << memrefType; |
| 237 | } |
| 238 | |
| 239 | if (!allocLikeOp.getSymbolOperands().empty()) { |
| 240 | return emitSilenceableFailure(allocLikeOp) |
| 241 | << "conversion to a global op does not support symbol operands, but " |
| 242 | "got " |
| 243 | << memrefType; |
| 244 | } |
| 245 | |
| 246 | int64_t offset; |
| 247 | SmallVector<int64_t, 4> strides; |
| 248 | if (failed(Result: memrefType.getStridesAndOffset(strides, offset))) { |
| 249 | return emitSilenceableFailure(allocLikeOp) |
| 250 | << "conversion to a global op requires strided layout, but got " |
| 251 | << memrefType; |
| 252 | } |
| 253 | if (!ShapedType::isStatic(dValue: offset) || !ShapedType::isStaticShape(dSizes: strides)) { |
| 254 | return emitSilenceableFailure(allocLikeOp) |
| 255 | << "conversion to a global op does not support dynamic offset or " |
| 256 | "strides, but got " |
| 257 | << memrefType; |
| 258 | } |
| 259 | |
| 260 | return DiagnosedSilenceableFailure::success(); |
| 261 | } |
| 262 | |
| 263 | /// Converts an allocation operation (`memref.alloca` or `memref.alloc`) to a |
| 264 | /// `memref.global` operation in the nearest symbol table, and replaces the |
| 265 | /// allocation with a `memref.get_global` operation. Any `memref.dealloc` |
| 266 | /// operations referencing the allocation are erased. |
| 267 | template <typename AllocLikeOp> |
| 268 | static DiagnosedSilenceableFailure |
| 269 | allocLikeToGlobal(transform::TransformRewriter &rewriter, |
| 270 | AllocLikeOp allocLikeOp, StringRef globalName, |
| 271 | memref::GlobalOp &globalOp, |
| 272 | memref::GetGlobalOp &getGlobalOp) { |
| 273 | if (DiagnosedSilenceableFailure failure = |
| 274 | checkAllocToGlobalPreconditions(allocLikeOp); |
| 275 | !failure.succeeded()) |
| 276 | return failure; |
| 277 | |
| 278 | MLIRContext *ctx = rewriter.getContext(); |
| 279 | Location loc = allocLikeOp->getLoc(); |
| 280 | |
| 281 | // Find nearest symbol table. |
| 282 | Operation *symbolTableOp = SymbolTable::getNearestSymbolTable(from: allocLikeOp); |
| 283 | assert(symbolTableOp && "expected payload to be in symbol table" ); |
| 284 | SymbolTable symbolTable(symbolTableOp); |
| 285 | |
| 286 | // Insert a `memref.global` into the symbol table. |
| 287 | Type resultType = allocLikeOp.getResult().getType(); |
| 288 | OpBuilder builder(rewriter.getContext()); |
| 289 | // TODO: Add a better builder for this. |
| 290 | globalOp = memref::GlobalOp::create( |
| 291 | builder, loc, StringAttr::get(context: ctx, bytes: globalName), |
| 292 | StringAttr::get(context: ctx, bytes: "private" ), TypeAttr::get(type: resultType), Attribute{}, |
| 293 | UnitAttr{}, allocLikeOp.getAlignmentAttr()); |
| 294 | symbolTable.insert(symbol: globalOp); |
| 295 | |
| 296 | // Remove any `memref.dealloc` operations referencing this allocation. |
| 297 | // We assume that the allocation does not escape the current container |
| 298 | // (e.g., via return or interprocedural function calls) and is not passed |
| 299 | // through control-flow or alias operations (e.g., `scf.if`, `cf.cond_br`, |
| 300 | // `select`, `memref.subview`), so any deallocation is a direct user of the |
| 301 | // allocation. Indirect deallocations are not removed and must be handled |
| 302 | // separately. |
| 303 | for (Operation *user : llvm::make_early_inc_range(allocLikeOp->getUsers())) { |
| 304 | if (auto dealloc = dyn_cast<memref::DeallocOp>(Val: user)) |
| 305 | rewriter.eraseOp(op: dealloc); |
| 306 | } |
| 307 | |
| 308 | // Replace the allocation with a `memref.get_global` accessing the |
| 309 | // global symbol inserted above. |
| 310 | rewriter.setInsertionPoint(allocLikeOp); |
| 311 | getGlobalOp = rewriter.replaceOpWithNewOp<memref::GetGlobalOp>( |
| 312 | allocLikeOp, globalOp.getType(), globalOp.getName()); |
| 313 | |
| 314 | return DiagnosedSilenceableFailure::success(); |
| 315 | } |
| 316 | |
| 317 | //===----------------------------------------------------------------------===// |
| 318 | // AllocaToGlobalOp |
| 319 | //===----------------------------------------------------------------------===// |
| 320 | |
| 321 | DiagnosedSilenceableFailure |
| 322 | transform::MemRefAllocaToGlobalOp::apply(transform::TransformRewriter &rewriter, |
| 323 | transform::TransformResults &results, |
| 324 | transform::TransformState &state) { |
| 325 | auto allocaOps = state.getPayloadOps(value: getAlloca()); |
| 326 | |
| 327 | SmallVector<memref::GlobalOp> globalOps; |
| 328 | SmallVector<memref::GetGlobalOp> getGlobalOps; |
| 329 | |
| 330 | // Transform `memref.alloca`s. |
| 331 | for (auto *op : allocaOps) { |
| 332 | auto alloca = cast<memref::AllocaOp>(Val: op); |
| 333 | memref::GlobalOp globalOp; |
| 334 | memref::GetGlobalOp getGlobalOp; |
| 335 | DiagnosedSilenceableFailure diag = |
| 336 | allocLikeToGlobal(rewriter, allocLikeOp: alloca, globalName: "alloca" , globalOp, getGlobalOp); |
| 337 | if (!diag.succeeded()) |
| 338 | return diag; |
| 339 | |
| 340 | globalOps.push_back(Elt: globalOp); |
| 341 | getGlobalOps.push_back(Elt: getGlobalOp); |
| 342 | } |
| 343 | |
| 344 | // Assemble results. |
| 345 | results.set(value: cast<OpResult>(Val: getGlobal()), ops&: globalOps); |
| 346 | results.set(value: cast<OpResult>(Val: getGetGlobal()), ops&: getGlobalOps); |
| 347 | |
| 348 | return DiagnosedSilenceableFailure::success(); |
| 349 | } |
| 350 | |
| 351 | void transform::MemRefAllocaToGlobalOp::getEffects( |
| 352 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 353 | producesHandle(handles: getOperation()->getOpResults(), effects); |
| 354 | consumesHandle(handles: getAllocaMutable(), effects); |
| 355 | modifiesPayload(effects); |
| 356 | } |
| 357 | |
| 358 | //===----------------------------------------------------------------------===// |
| 359 | // AllocToGlobalOp |
| 360 | //===----------------------------------------------------------------------===// |
| 361 | |
| 362 | DiagnosedSilenceableFailure |
| 363 | transform::MemRefAllocToGlobalOp::apply(transform::TransformRewriter &rewriter, |
| 364 | transform::TransformResults &results, |
| 365 | transform::TransformState &state) { |
| 366 | auto allocOps = state.getPayloadOps(value: getAlloc()); |
| 367 | |
| 368 | SmallVector<memref::GlobalOp> globalOps; |
| 369 | SmallVector<memref::GetGlobalOp> getGlobalOps; |
| 370 | |
| 371 | // Transform `memref.alloc`s. |
| 372 | for (auto *op : allocOps) { |
| 373 | auto alloc = cast<memref::AllocOp>(Val: op); |
| 374 | memref::GlobalOp globalOp; |
| 375 | memref::GetGlobalOp getGlobalOp; |
| 376 | DiagnosedSilenceableFailure diag = |
| 377 | allocLikeToGlobal(rewriter, allocLikeOp: alloc, globalName: "alloc" , globalOp, getGlobalOp); |
| 378 | if (!diag.succeeded()) |
| 379 | return diag; |
| 380 | |
| 381 | globalOps.push_back(Elt: globalOp); |
| 382 | getGlobalOps.push_back(Elt: getGlobalOp); |
| 383 | } |
| 384 | |
| 385 | // Assemble results. |
| 386 | results.set(value: cast<OpResult>(Val: getGlobal()), ops&: globalOps); |
| 387 | results.set(value: cast<OpResult>(Val: getGetGlobal()), ops&: getGlobalOps); |
| 388 | |
| 389 | return DiagnosedSilenceableFailure::success(); |
| 390 | } |
| 391 | |
| 392 | void transform::MemRefAllocToGlobalOp::getEffects( |
| 393 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 394 | producesHandle(handles: getOperation()->getOpResults(), effects); |
| 395 | consumesHandle(handles: getAllocMutable(), effects); |
| 396 | modifiesPayload(effects); |
| 397 | } |
| 398 | |
| 399 | //===----------------------------------------------------------------------===// |
| 400 | // MemRefMultiBufferOp |
| 401 | //===----------------------------------------------------------------------===// |
| 402 | |
| 403 | DiagnosedSilenceableFailure transform::MemRefMultiBufferOp::apply( |
| 404 | transform::TransformRewriter &rewriter, |
| 405 | transform::TransformResults &transformResults, |
| 406 | transform::TransformState &state) { |
| 407 | SmallVector<Operation *> results; |
| 408 | for (Operation *op : state.getPayloadOps(value: getTarget())) { |
| 409 | bool canApplyMultiBuffer = true; |
| 410 | auto target = cast<memref::AllocOp>(Val: op); |
| 411 | LLVM_DEBUG(DBGS() << "Start multibuffer transform op: " << target << "\n" ;); |
| 412 | // Skip allocations not used in a loop. |
| 413 | for (Operation *user : target->getUsers()) { |
| 414 | if (isa<memref::DeallocOp>(Val: user)) |
| 415 | continue; |
| 416 | auto loop = user->getParentOfType<LoopLikeOpInterface>(); |
| 417 | if (!loop) { |
| 418 | LLVM_DEBUG(DBGS() << "--allocation not used in a loop\n" ; |
| 419 | DBGS() << "----due to user: " << *user;); |
| 420 | canApplyMultiBuffer = false; |
| 421 | break; |
| 422 | } |
| 423 | } |
| 424 | if (!canApplyMultiBuffer) { |
| 425 | LLVM_DEBUG(DBGS() << "--cannot apply multibuffering -> Skip\n" ;); |
| 426 | continue; |
| 427 | } |
| 428 | |
| 429 | auto newBuffer = |
| 430 | memref::multiBuffer(rewriter, allocOp: target, multiplier: getFactor(), skipOverrideAnalysis: getSkipAnalysis()); |
| 431 | |
| 432 | if (failed(Result: newBuffer)) { |
| 433 | LLVM_DEBUG(DBGS() << "--op failed to multibuffer\n" ;); |
| 434 | return emitSilenceableFailure(loc: target->getLoc()) |
| 435 | << "op failed to multibuffer" ; |
| 436 | } |
| 437 | |
| 438 | results.push_back(Elt: *newBuffer); |
| 439 | } |
| 440 | transformResults.set(value: cast<OpResult>(Val: getResult()), ops&: results); |
| 441 | return DiagnosedSilenceableFailure::success(); |
| 442 | } |
| 443 | |
| 444 | //===----------------------------------------------------------------------===// |
| 445 | // MemRefEraseDeadAllocAndStoresOp |
| 446 | //===----------------------------------------------------------------------===// |
| 447 | |
| 448 | DiagnosedSilenceableFailure |
| 449 | transform::MemRefEraseDeadAllocAndStoresOp::applyToOne( |
| 450 | transform::TransformRewriter &rewriter, Operation *target, |
| 451 | transform::ApplyToEachResultList &results, |
| 452 | transform::TransformState &state) { |
| 453 | // Apply store to load forwarding and dead store elimination. |
| 454 | vector::transferOpflowOpt(rewriter, rootOp: target); |
| 455 | memref::eraseDeadAllocAndStores(rewriter, parentOp: target); |
| 456 | return DiagnosedSilenceableFailure::success(); |
| 457 | } |
| 458 | |
| 459 | void transform::MemRefEraseDeadAllocAndStoresOp::getEffects( |
| 460 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 461 | transform::onlyReadsHandle(handles: getTargetMutable(), effects); |
| 462 | transform::modifiesPayload(effects); |
| 463 | } |
| 464 | void transform::MemRefEraseDeadAllocAndStoresOp::build(OpBuilder &builder, |
| 465 | OperationState &result, |
| 466 | Value target) { |
| 467 | result.addOperands(newOperands: target); |
| 468 | } |
| 469 | |
| 470 | //===----------------------------------------------------------------------===// |
| 471 | // MemRefMakeLoopIndependentOp |
| 472 | //===----------------------------------------------------------------------===// |
| 473 | |
| 474 | DiagnosedSilenceableFailure transform::MemRefMakeLoopIndependentOp::applyToOne( |
| 475 | transform::TransformRewriter &rewriter, Operation *target, |
| 476 | transform::ApplyToEachResultList &results, |
| 477 | transform::TransformState &state) { |
| 478 | // Gather IVs. |
| 479 | SmallVector<Value> ivs; |
| 480 | Operation *nextOp = target; |
| 481 | for (uint64_t i = 0, e = getNumLoops(); i < e; ++i) { |
| 482 | nextOp = nextOp->getParentOfType<scf::ForOp>(); |
| 483 | if (!nextOp) { |
| 484 | DiagnosedSilenceableFailure diag = emitSilenceableError() |
| 485 | << "could not find " << i |
| 486 | << "-th enclosing loop" ; |
| 487 | diag.attachNote(loc: target->getLoc()) << "target op" ; |
| 488 | return diag; |
| 489 | } |
| 490 | ivs.push_back(Elt: cast<scf::ForOp>(Val: nextOp).getInductionVar()); |
| 491 | } |
| 492 | |
| 493 | // Rewrite IR. |
| 494 | FailureOr<Value> replacement = failure(); |
| 495 | if (auto allocaOp = dyn_cast<memref::AllocaOp>(Val: target)) { |
| 496 | replacement = memref::replaceWithIndependentOp(rewriter, allocaOp, independencies: ivs); |
| 497 | } else { |
| 498 | DiagnosedSilenceableFailure diag = emitSilenceableError() |
| 499 | << "unsupported target op" ; |
| 500 | diag.attachNote(loc: target->getLoc()) << "target op" ; |
| 501 | return diag; |
| 502 | } |
| 503 | if (failed(Result: replacement)) { |
| 504 | DiagnosedSilenceableFailure diag = |
| 505 | emitSilenceableError() << "could not make target op loop-independent" ; |
| 506 | diag.attachNote(loc: target->getLoc()) << "target op" ; |
| 507 | return diag; |
| 508 | } |
| 509 | results.push_back(op: replacement->getDefiningOp()); |
| 510 | return DiagnosedSilenceableFailure::success(); |
| 511 | } |
| 512 | |
| 513 | //===----------------------------------------------------------------------===// |
| 514 | // Transform op registration |
| 515 | //===----------------------------------------------------------------------===// |
| 516 | |
| 517 | namespace { |
| 518 | class MemRefTransformDialectExtension |
| 519 | : public transform::TransformDialectExtension< |
| 520 | MemRefTransformDialectExtension> { |
| 521 | public: |
| 522 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(MemRefTransformDialectExtension) |
| 523 | |
| 524 | using Base::Base; |
| 525 | |
| 526 | void init() { |
| 527 | declareGeneratedDialect<affine::AffineDialect>(); |
| 528 | declareGeneratedDialect<arith::ArithDialect>(); |
| 529 | declareGeneratedDialect<memref::MemRefDialect>(); |
| 530 | declareGeneratedDialect<nvgpu::NVGPUDialect>(); |
| 531 | declareGeneratedDialect<vector::VectorDialect>(); |
| 532 | |
| 533 | registerTransformOps< |
| 534 | #define GET_OP_LIST |
| 535 | #include "mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp.inc" |
| 536 | >(); |
| 537 | } |
| 538 | }; |
| 539 | } // namespace |
| 540 | |
| 541 | #define GET_OP_CLASSES |
| 542 | #include "mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp.inc" |
| 543 | |
| 544 | void mlir::memref::registerTransformDialectExtension( |
| 545 | DialectRegistry ®istry) { |
| 546 | registry.addExtensions<MemRefTransformDialectExtension>(); |
| 547 | } |
| 548 | |