1//===- MemRefTransformOps.cpp - Implementation of Memref transform ops ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.h"
10
11#include "mlir/Analysis/DataLayoutAnalysis.h"
12#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
13#include "mlir/Dialect/Affine/IR/AffineDialect.h"
14#include "mlir/Dialect/Arith/IR/Arith.h"
15#include "mlir/Dialect/MemRef/IR/MemRef.h"
16#include "mlir/Dialect/MemRef/Transforms/Passes.h"
17#include "mlir/Dialect/MemRef/Transforms/Transforms.h"
18#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
19#include "mlir/Dialect/NVGPU/IR/NVGPUDialectDecl.h"
20#include "mlir/Dialect/SCF/IR/SCF.h"
21#include "mlir/Dialect/Transform/IR/TransformDialect.h"
22#include "mlir/Dialect/Transform/IR/TransformTypes.h"
23#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
24#include "mlir/Dialect/Vector/IR/VectorOps.h"
25#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
26#include "mlir/Interfaces/LoopLikeInterface.h"
27#include "llvm/Support/Debug.h"
28
29using namespace mlir;
30
31#define DEBUG_TYPE "memref-transforms"
32#define DBGS() (llvm::dbgs() << '[' << DEBUG_TYPE << "] ")
33
34namespace mlir::transform {
35namespace {
36ParseResult parseLLVMTypeConverterOptions(OpAsmParser &parser,
37 BoolAttr &useAlignedAlloc,
38 IntegerAttr &indexBitwidth,
39 BoolAttr &useGenericFunctions,
40 BoolAttr &useBarePtrCallConv,
41 StringAttr &dataLayout) {
42 bool seenUseAlignedAlloc = false;
43 bool seenIndexBitwidth = false;
44 bool seenUseGenericFunctions = false;
45 bool seenUseBarePtrCallConv = false;
46 bool seenDataLayout = false;
47
48 auto parseDuplicate = [&](StringRef name, bool &seen) -> ParseResult {
49 if (seen)
50 return parser.emitError(loc: parser.getCurrentLocation())
51 << "duplicate '" << name << "' option";
52 seen = true;
53 return ParseResult::success();
54 };
55
56 while (true) {
57 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "use_aligned_alloc"))) {
58 if (failed(Result: parseDuplicate("use_aligned_alloc", seenUseAlignedAlloc)) ||
59 parser.parseEqual() ||
60 parser.parseAttribute(result&: useAlignedAlloc,
61 type: parser.getBuilder().getI1Type()))
62 return failure();
63 continue;
64 }
65 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "index_bitwidth"))) {
66 if (failed(Result: parseDuplicate("index_bitwidth", seenIndexBitwidth)) ||
67 parser.parseEqual() ||
68 parser.parseAttribute(result&: indexBitwidth,
69 type: parser.getBuilder().getI64Type()))
70 return failure();
71 continue;
72 }
73 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "use_generic_functions"))) {
74 if (failed(Result: parseDuplicate("use_generic_functions",
75 seenUseGenericFunctions)) ||
76 parser.parseEqual() ||
77 parser.parseAttribute(result&: useGenericFunctions,
78 type: parser.getBuilder().getI1Type()))
79 return failure();
80 continue;
81 }
82 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "use_bare_ptr_call_conv"))) {
83 if (failed(Result: parseDuplicate("use_bare_ptr_call_conv",
84 seenUseBarePtrCallConv)) ||
85 parser.parseEqual() ||
86 parser.parseAttribute(result&: useBarePtrCallConv,
87 type: parser.getBuilder().getI1Type()))
88 return failure();
89 continue;
90 }
91 if (succeeded(Result: parser.parseOptionalKeyword(keyword: "data_layout"))) {
92 if (failed(Result: parseDuplicate("data_layout", seenDataLayout)) ||
93 parser.parseEqual() || parser.parseAttribute(result&: dataLayout))
94 return failure();
95 continue;
96 }
97 break;
98 }
99 return success();
100}
101
102void printLLVMTypeConverterOptions(OpAsmPrinter &printer, Operation *,
103 BoolAttr useAlignedAlloc,
104 IntegerAttr indexBitwidth,
105 BoolAttr useGenericFunctions,
106 BoolAttr useBarePtrCallConv,
107 StringAttr dataLayout) {
108 bool needsSpace = false;
109 auto printOption = [&](StringRef name, Attribute value) {
110 if (!value)
111 return;
112 if (needsSpace)
113 printer << ' ';
114 printer << name << " = ";
115 printer.printAttributeWithoutType(attr: value);
116 needsSpace = true;
117 };
118
119 printOption("use_aligned_alloc", useAlignedAlloc);
120 printOption("index_bitwidth", indexBitwidth);
121 printOption("use_generic_functions", useGenericFunctions);
122 printOption("use_bare_ptr_call_conv", useBarePtrCallConv);
123 printOption("data_layout", dataLayout);
124}
125} // namespace
126} // namespace mlir::transform
127
128//===----------------------------------------------------------------------===//
129// Apply...ConversionPatternsOp
130//===----------------------------------------------------------------------===//
131
132std::unique_ptr<TypeConverter>
133transform::MemrefToLLVMTypeConverterOp::getTypeConverter() {
134 LowerToLLVMOptions options(getContext());
135 options.allocLowering =
136 (getUseAlignedAlloc() ? LowerToLLVMOptions::AllocLowering::AlignedAlloc
137 : LowerToLLVMOptions::AllocLowering::Malloc);
138 options.useGenericFunctions = getUseGenericFunctions();
139
140 if (getIndexBitwidth() != kDeriveIndexBitwidthFromDataLayout)
141 options.overrideIndexBitwidth(bitwidth: getIndexBitwidth());
142
143 // TODO: the following two options don't really make sense for
144 // memref_to_llvm_type_converter specifically but we should have a single
145 // to_llvm_type_converter.
146 if (getDataLayout().has_value())
147 options.dataLayout = llvm::DataLayout(getDataLayout().value());
148 options.useBarePtrCallConv = getUseBarePtrCallConv();
149
150 return std::make_unique<LLVMTypeConverter>(args: getContext(), args&: options);
151}
152
153StringRef transform::MemrefToLLVMTypeConverterOp::getTypeConverterType() {
154 return "LLVMTypeConverter";
155}
156
157//===----------------------------------------------------------------------===//
158// Apply...PatternsOp
159//===----------------------------------------------------------------------===//
160
161namespace {
162class AllocToAllocaPattern : public OpRewritePattern<memref::AllocOp> {
163public:
164 explicit AllocToAllocaPattern(Operation *analysisRoot, int64_t maxSize = 0)
165 : OpRewritePattern<memref::AllocOp>(analysisRoot->getContext()),
166 dataLayoutAnalysis(analysisRoot), maxSize(maxSize) {}
167
168 LogicalResult matchAndRewrite(memref::AllocOp op,
169 PatternRewriter &rewriter) const override {
170 return success(IsSuccess: memref::allocToAlloca(
171 rewriter, alloc: op, filter: [this](memref::AllocOp alloc, memref::DeallocOp dealloc) {
172 MemRefType type = alloc.getMemref().getType();
173 if (!type.hasStaticShape())
174 return false;
175
176 const DataLayout &dataLayout = dataLayoutAnalysis.getAtOrAbove(operation: alloc);
177 int64_t elementSize = dataLayout.getTypeSize(t: type.getElementType());
178 return maxSize == 0 || type.getNumElements() * elementSize < maxSize;
179 }));
180 }
181
182private:
183 DataLayoutAnalysis dataLayoutAnalysis;
184 int64_t maxSize;
185};
186} // namespace
187
188void transform::ApplyAllocToAllocaOp::populatePatterns(
189 RewritePatternSet &patterns) {}
190
191void transform::ApplyAllocToAllocaOp::populatePatternsWithState(
192 RewritePatternSet &patterns, transform::TransformState &state) {
193 patterns.insert<AllocToAllocaPattern>(
194 arg: state.getTopLevel(), args: static_cast<int64_t>(getSizeLimit().value_or(u: 0)));
195}
196
197void transform::ApplyExpandOpsPatternsOp::populatePatterns(
198 RewritePatternSet &patterns) {
199 memref::populateExpandOpsPatterns(patterns);
200}
201
202void transform::ApplyExpandStridedMetadataPatternsOp::populatePatterns(
203 RewritePatternSet &patterns) {
204 memref::populateExpandStridedMetadataPatterns(patterns);
205}
206
207void transform::ApplyExtractAddressComputationsPatternsOp::populatePatterns(
208 RewritePatternSet &patterns) {
209 memref::populateExtractAddressComputationsPatterns(patterns);
210}
211
212void transform::ApplyFoldMemrefAliasOpsPatternsOp::populatePatterns(
213 RewritePatternSet &patterns) {
214 memref::populateFoldMemRefAliasOpPatterns(patterns);
215}
216
217void transform::ApplyResolveRankedShapedTypeResultDimsPatternsOp::
218 populatePatterns(RewritePatternSet &patterns) {
219 memref::populateResolveRankedShapedTypeResultDimsPatterns(patterns);
220}
221
222//===----------------------------------------------------------------------===//
223// Alloc and alloca to global utilities
224//===----------------------------------------------------------------------===//
225
226/// Checks whether an allocation operation can be converted to a
227/// `memref.global`.
228template <typename AllocLikeOp>
229static DiagnosedSilenceableFailure
230checkAllocToGlobalPreconditions(AllocLikeOp allocLikeOp) {
231 MemRefType memrefType = allocLikeOp.getType();
232 if (!memrefType.hasStaticShape()) {
233 return emitSilenceableFailure(allocLikeOp)
234 << "conversion to a global op requires statically shaped memrefs, "
235 "but got "
236 << memrefType;
237 }
238
239 if (!allocLikeOp.getSymbolOperands().empty()) {
240 return emitSilenceableFailure(allocLikeOp)
241 << "conversion to a global op does not support symbol operands, but "
242 "got "
243 << memrefType;
244 }
245
246 int64_t offset;
247 SmallVector<int64_t, 4> strides;
248 if (failed(Result: memrefType.getStridesAndOffset(strides, offset))) {
249 return emitSilenceableFailure(allocLikeOp)
250 << "conversion to a global op requires strided layout, but got "
251 << memrefType;
252 }
253 if (!ShapedType::isStatic(dValue: offset) || !ShapedType::isStaticShape(dSizes: strides)) {
254 return emitSilenceableFailure(allocLikeOp)
255 << "conversion to a global op does not support dynamic offset or "
256 "strides, but got "
257 << memrefType;
258 }
259
260 return DiagnosedSilenceableFailure::success();
261}
262
263/// Converts an allocation operation (`memref.alloca` or `memref.alloc`) to a
264/// `memref.global` operation in the nearest symbol table, and replaces the
265/// allocation with a `memref.get_global` operation. Any `memref.dealloc`
266/// operations referencing the allocation are erased.
267template <typename AllocLikeOp>
268static DiagnosedSilenceableFailure
269allocLikeToGlobal(transform::TransformRewriter &rewriter,
270 AllocLikeOp allocLikeOp, StringRef globalName,
271 memref::GlobalOp &globalOp,
272 memref::GetGlobalOp &getGlobalOp) {
273 if (DiagnosedSilenceableFailure failure =
274 checkAllocToGlobalPreconditions(allocLikeOp);
275 !failure.succeeded())
276 return failure;
277
278 MLIRContext *ctx = rewriter.getContext();
279 Location loc = allocLikeOp->getLoc();
280
281 // Find nearest symbol table.
282 Operation *symbolTableOp = SymbolTable::getNearestSymbolTable(from: allocLikeOp);
283 assert(symbolTableOp && "expected payload to be in symbol table");
284 SymbolTable symbolTable(symbolTableOp);
285
286 // Insert a `memref.global` into the symbol table.
287 Type resultType = allocLikeOp.getResult().getType();
288 OpBuilder builder(rewriter.getContext());
289 // TODO: Add a better builder for this.
290 globalOp = memref::GlobalOp::create(
291 builder, loc, StringAttr::get(context: ctx, bytes: globalName),
292 StringAttr::get(context: ctx, bytes: "private"), TypeAttr::get(type: resultType), Attribute{},
293 UnitAttr{}, allocLikeOp.getAlignmentAttr());
294 symbolTable.insert(symbol: globalOp);
295
296 // Remove any `memref.dealloc` operations referencing this allocation.
297 // We assume that the allocation does not escape the current container
298 // (e.g., via return or interprocedural function calls) and is not passed
299 // through control-flow or alias operations (e.g., `scf.if`, `cf.cond_br`,
300 // `select`, `memref.subview`), so any deallocation is a direct user of the
301 // allocation. Indirect deallocations are not removed and must be handled
302 // separately.
303 for (Operation *user : llvm::make_early_inc_range(allocLikeOp->getUsers())) {
304 if (auto dealloc = dyn_cast<memref::DeallocOp>(Val: user))
305 rewriter.eraseOp(op: dealloc);
306 }
307
308 // Replace the allocation with a `memref.get_global` accessing the
309 // global symbol inserted above.
310 rewriter.setInsertionPoint(allocLikeOp);
311 getGlobalOp = rewriter.replaceOpWithNewOp<memref::GetGlobalOp>(
312 allocLikeOp, globalOp.getType(), globalOp.getName());
313
314 return DiagnosedSilenceableFailure::success();
315}
316
317//===----------------------------------------------------------------------===//
318// AllocaToGlobalOp
319//===----------------------------------------------------------------------===//
320
321DiagnosedSilenceableFailure
322transform::MemRefAllocaToGlobalOp::apply(transform::TransformRewriter &rewriter,
323 transform::TransformResults &results,
324 transform::TransformState &state) {
325 auto allocaOps = state.getPayloadOps(value: getAlloca());
326
327 SmallVector<memref::GlobalOp> globalOps;
328 SmallVector<memref::GetGlobalOp> getGlobalOps;
329
330 // Transform `memref.alloca`s.
331 for (auto *op : allocaOps) {
332 auto alloca = cast<memref::AllocaOp>(Val: op);
333 memref::GlobalOp globalOp;
334 memref::GetGlobalOp getGlobalOp;
335 DiagnosedSilenceableFailure diag =
336 allocLikeToGlobal(rewriter, allocLikeOp: alloca, globalName: "alloca", globalOp, getGlobalOp);
337 if (!diag.succeeded())
338 return diag;
339
340 globalOps.push_back(Elt: globalOp);
341 getGlobalOps.push_back(Elt: getGlobalOp);
342 }
343
344 // Assemble results.
345 results.set(value: cast<OpResult>(Val: getGlobal()), ops&: globalOps);
346 results.set(value: cast<OpResult>(Val: getGetGlobal()), ops&: getGlobalOps);
347
348 return DiagnosedSilenceableFailure::success();
349}
350
351void transform::MemRefAllocaToGlobalOp::getEffects(
352 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
353 producesHandle(handles: getOperation()->getOpResults(), effects);
354 consumesHandle(handles: getAllocaMutable(), effects);
355 modifiesPayload(effects);
356}
357
358//===----------------------------------------------------------------------===//
359// AllocToGlobalOp
360//===----------------------------------------------------------------------===//
361
362DiagnosedSilenceableFailure
363transform::MemRefAllocToGlobalOp::apply(transform::TransformRewriter &rewriter,
364 transform::TransformResults &results,
365 transform::TransformState &state) {
366 auto allocOps = state.getPayloadOps(value: getAlloc());
367
368 SmallVector<memref::GlobalOp> globalOps;
369 SmallVector<memref::GetGlobalOp> getGlobalOps;
370
371 // Transform `memref.alloc`s.
372 for (auto *op : allocOps) {
373 auto alloc = cast<memref::AllocOp>(Val: op);
374 memref::GlobalOp globalOp;
375 memref::GetGlobalOp getGlobalOp;
376 DiagnosedSilenceableFailure diag =
377 allocLikeToGlobal(rewriter, allocLikeOp: alloc, globalName: "alloc", globalOp, getGlobalOp);
378 if (!diag.succeeded())
379 return diag;
380
381 globalOps.push_back(Elt: globalOp);
382 getGlobalOps.push_back(Elt: getGlobalOp);
383 }
384
385 // Assemble results.
386 results.set(value: cast<OpResult>(Val: getGlobal()), ops&: globalOps);
387 results.set(value: cast<OpResult>(Val: getGetGlobal()), ops&: getGlobalOps);
388
389 return DiagnosedSilenceableFailure::success();
390}
391
392void transform::MemRefAllocToGlobalOp::getEffects(
393 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
394 producesHandle(handles: getOperation()->getOpResults(), effects);
395 consumesHandle(handles: getAllocMutable(), effects);
396 modifiesPayload(effects);
397}
398
399//===----------------------------------------------------------------------===//
400// MemRefMultiBufferOp
401//===----------------------------------------------------------------------===//
402
403DiagnosedSilenceableFailure transform::MemRefMultiBufferOp::apply(
404 transform::TransformRewriter &rewriter,
405 transform::TransformResults &transformResults,
406 transform::TransformState &state) {
407 SmallVector<Operation *> results;
408 for (Operation *op : state.getPayloadOps(value: getTarget())) {
409 bool canApplyMultiBuffer = true;
410 auto target = cast<memref::AllocOp>(Val: op);
411 LLVM_DEBUG(DBGS() << "Start multibuffer transform op: " << target << "\n";);
412 // Skip allocations not used in a loop.
413 for (Operation *user : target->getUsers()) {
414 if (isa<memref::DeallocOp>(Val: user))
415 continue;
416 auto loop = user->getParentOfType<LoopLikeOpInterface>();
417 if (!loop) {
418 LLVM_DEBUG(DBGS() << "--allocation not used in a loop\n";
419 DBGS() << "----due to user: " << *user;);
420 canApplyMultiBuffer = false;
421 break;
422 }
423 }
424 if (!canApplyMultiBuffer) {
425 LLVM_DEBUG(DBGS() << "--cannot apply multibuffering -> Skip\n";);
426 continue;
427 }
428
429 auto newBuffer =
430 memref::multiBuffer(rewriter, allocOp: target, multiplier: getFactor(), skipOverrideAnalysis: getSkipAnalysis());
431
432 if (failed(Result: newBuffer)) {
433 LLVM_DEBUG(DBGS() << "--op failed to multibuffer\n";);
434 return emitSilenceableFailure(loc: target->getLoc())
435 << "op failed to multibuffer";
436 }
437
438 results.push_back(Elt: *newBuffer);
439 }
440 transformResults.set(value: cast<OpResult>(Val: getResult()), ops&: results);
441 return DiagnosedSilenceableFailure::success();
442}
443
444//===----------------------------------------------------------------------===//
445// MemRefEraseDeadAllocAndStoresOp
446//===----------------------------------------------------------------------===//
447
448DiagnosedSilenceableFailure
449transform::MemRefEraseDeadAllocAndStoresOp::applyToOne(
450 transform::TransformRewriter &rewriter, Operation *target,
451 transform::ApplyToEachResultList &results,
452 transform::TransformState &state) {
453 // Apply store to load forwarding and dead store elimination.
454 vector::transferOpflowOpt(rewriter, rootOp: target);
455 memref::eraseDeadAllocAndStores(rewriter, parentOp: target);
456 return DiagnosedSilenceableFailure::success();
457}
458
459void transform::MemRefEraseDeadAllocAndStoresOp::getEffects(
460 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
461 transform::onlyReadsHandle(handles: getTargetMutable(), effects);
462 transform::modifiesPayload(effects);
463}
464void transform::MemRefEraseDeadAllocAndStoresOp::build(OpBuilder &builder,
465 OperationState &result,
466 Value target) {
467 result.addOperands(newOperands: target);
468}
469
470//===----------------------------------------------------------------------===//
471// MemRefMakeLoopIndependentOp
472//===----------------------------------------------------------------------===//
473
474DiagnosedSilenceableFailure transform::MemRefMakeLoopIndependentOp::applyToOne(
475 transform::TransformRewriter &rewriter, Operation *target,
476 transform::ApplyToEachResultList &results,
477 transform::TransformState &state) {
478 // Gather IVs.
479 SmallVector<Value> ivs;
480 Operation *nextOp = target;
481 for (uint64_t i = 0, e = getNumLoops(); i < e; ++i) {
482 nextOp = nextOp->getParentOfType<scf::ForOp>();
483 if (!nextOp) {
484 DiagnosedSilenceableFailure diag = emitSilenceableError()
485 << "could not find " << i
486 << "-th enclosing loop";
487 diag.attachNote(loc: target->getLoc()) << "target op";
488 return diag;
489 }
490 ivs.push_back(Elt: cast<scf::ForOp>(Val: nextOp).getInductionVar());
491 }
492
493 // Rewrite IR.
494 FailureOr<Value> replacement = failure();
495 if (auto allocaOp = dyn_cast<memref::AllocaOp>(Val: target)) {
496 replacement = memref::replaceWithIndependentOp(rewriter, allocaOp, independencies: ivs);
497 } else {
498 DiagnosedSilenceableFailure diag = emitSilenceableError()
499 << "unsupported target op";
500 diag.attachNote(loc: target->getLoc()) << "target op";
501 return diag;
502 }
503 if (failed(Result: replacement)) {
504 DiagnosedSilenceableFailure diag =
505 emitSilenceableError() << "could not make target op loop-independent";
506 diag.attachNote(loc: target->getLoc()) << "target op";
507 return diag;
508 }
509 results.push_back(op: replacement->getDefiningOp());
510 return DiagnosedSilenceableFailure::success();
511}
512
513//===----------------------------------------------------------------------===//
514// Transform op registration
515//===----------------------------------------------------------------------===//
516
517namespace {
518class MemRefTransformDialectExtension
519 : public transform::TransformDialectExtension<
520 MemRefTransformDialectExtension> {
521public:
522 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(MemRefTransformDialectExtension)
523
524 using Base::Base;
525
526 void init() {
527 declareGeneratedDialect<affine::AffineDialect>();
528 declareGeneratedDialect<arith::ArithDialect>();
529 declareGeneratedDialect<memref::MemRefDialect>();
530 declareGeneratedDialect<nvgpu::NVGPUDialect>();
531 declareGeneratedDialect<vector::VectorDialect>();
532
533 registerTransformOps<
534#define GET_OP_LIST
535#include "mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp.inc"
536 >();
537 }
538};
539} // namespace
540
541#define GET_OP_CLASSES
542#include "mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp.inc"
543
544void mlir::memref::registerTransformDialectExtension(
545 DialectRegistry &registry) {
546 registry.addExtensions<MemRefTransformDialectExtension>();
547}
548