1//===- SparseTensorPasses.cpp - Pass for autogen sparse tensor code -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Affine/IR/AffineOps.h"
10#include "mlir/Dialect/Arith/IR/Arith.h"
11#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
12#include "mlir/Dialect/Complex/IR/Complex.h"
13#include "mlir/Dialect/Func/IR/FuncOps.h"
14#include "mlir/Dialect/Func/Transforms/FuncConversions.h"
15#include "mlir/Dialect/GPU/IR/GPUDialect.h"
16#include "mlir/Dialect/LLVMIR/LLVMDialectDecl.h"
17#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
18#include "mlir/Dialect/SCF/Transforms/Patterns.h"
19#include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
20#include "mlir/Dialect/SparseTensor/Transforms/Passes.h"
21#include "mlir/Dialect/Tensor/IR/Tensor.h"
22#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
23
24namespace mlir {
25#define GEN_PASS_DEF_SPARSEASSEMBLER
26#define GEN_PASS_DEF_SPARSEREINTERPRETMAP
27#define GEN_PASS_DEF_PRESPARSIFICATIONREWRITE
28#define GEN_PASS_DEF_SPARSIFICATIONPASS
29#define GEN_PASS_DEF_LOWERSPARSEITERATIONTOSCF
30#define GEN_PASS_DEF_LOWERSPARSEOPSTOFOREACH
31#define GEN_PASS_DEF_LOWERFOREACHTOSCF
32#define GEN_PASS_DEF_SPARSETENSORCONVERSIONPASS
33#define GEN_PASS_DEF_SPARSETENSORCODEGEN
34#define GEN_PASS_DEF_SPARSEBUFFERREWRITE
35#define GEN_PASS_DEF_SPARSEVECTORIZATION
36#define GEN_PASS_DEF_SPARSEGPUCODEGEN
37#define GEN_PASS_DEF_STAGESPARSEOPERATIONS
38#define GEN_PASS_DEF_STORAGESPECIFIERTOLLVM
39#include "mlir/Dialect/SparseTensor/Transforms/Passes.h.inc"
40} // namespace mlir
41
42using namespace mlir;
43using namespace mlir::sparse_tensor;
44
45namespace {
46
47//===----------------------------------------------------------------------===//
48// Passes implementation.
49//===----------------------------------------------------------------------===//
50
51static bool supportsCollapsedRangeBetween(LevelType lt) {
52 return isSingletonLT(lt);
53}
54
55static LogicalResult verifyLowerableCollapsedIterSpaces(Operation *op) {
56 WalkResult result = op->walk(callback: [&](ExtractIterSpaceOp op) {
57 auto [lvlLo, lvlHi] = op.getLvlRange();
58 SparseTensorEncodingAttr enc = cast<SparseTensorEncodingAttr>(
59 Val: getRankedTensorType(t: op.getTensor()).getEncoding());
60 for (Level lvl = lvlLo + 1; lvl < lvlHi; ++lvl) {
61 LevelType lt = enc.getLvlType(l: lvl);
62 if (!supportsCollapsedRangeBetween(lt)) {
63 op.emitOpError() << "cannot lower collapsed iteration space with "
64 << toMLIRString(lt) << " level after the first level";
65 return WalkResult::interrupt();
66 }
67 }
68 return WalkResult::advance();
69 });
70 return failure(IsFailure: result.wasInterrupted());
71}
72
73struct SparseAssembler : public impl::SparseAssemblerBase<SparseAssembler> {
74 SparseAssembler() = default;
75 SparseAssembler(const SparseAssembler &pass) = default;
76 SparseAssembler(bool dO) { directOut = dO; }
77
78 void runOnOperation() override {
79 auto *ctx = &getContext();
80 RewritePatternSet patterns(ctx);
81 populateSparseAssembler(patterns, directOut);
82 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
83 }
84};
85
86struct SparseReinterpretMap
87 : public impl::SparseReinterpretMapBase<SparseReinterpretMap> {
88 SparseReinterpretMap() = default;
89 SparseReinterpretMap(const SparseReinterpretMap &pass) = default;
90 SparseReinterpretMap(const SparseReinterpretMapOptions &options) {
91 scope = options.scope;
92 loopOrderingStrategy = options.loopOrderingStrategy;
93 }
94
95 void runOnOperation() override {
96 auto *ctx = &getContext();
97 RewritePatternSet patterns(ctx);
98 populateSparseReinterpretMap(patterns, scope, strategy: loopOrderingStrategy);
99 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
100 }
101};
102
103struct PreSparsificationRewritePass
104 : public impl::PreSparsificationRewriteBase<PreSparsificationRewritePass> {
105 PreSparsificationRewritePass() = default;
106 PreSparsificationRewritePass(const PreSparsificationRewritePass &pass) =
107 default;
108
109 void runOnOperation() override {
110 auto *ctx = &getContext();
111 RewritePatternSet patterns(ctx);
112 populatePreSparsificationRewriting(patterns);
113 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
114 }
115};
116
117struct SparsificationPass
118 : public impl::SparsificationPassBase<SparsificationPass> {
119 SparsificationPass() = default;
120 SparsificationPass(const SparsificationPass &pass) = default;
121 SparsificationPass(const SparsificationOptions &options) {
122 parallelization = options.parallelizationStrategy;
123 sparseEmitStrategy = options.sparseEmitStrategy;
124 enableRuntimeLibrary = options.enableRuntimeLibrary;
125 }
126
127 void runOnOperation() override {
128 auto *ctx = &getContext();
129 // Translate strategy flags to strategy options.
130 SparsificationOptions options(parallelization, sparseEmitStrategy,
131 enableRuntimeLibrary);
132 // Apply sparsification and cleanup rewriting.
133 RewritePatternSet patterns(ctx);
134 populateSparsificationPatterns(patterns, options);
135 scf::ForOp::getCanonicalizationPatterns(results&: patterns, context: ctx);
136 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
137 }
138};
139
140struct StageSparseOperationsPass
141 : public impl::StageSparseOperationsBase<StageSparseOperationsPass> {
142 StageSparseOperationsPass() = default;
143 StageSparseOperationsPass(const StageSparseOperationsPass &pass) = default;
144 void runOnOperation() override {
145 auto *ctx = &getContext();
146 RewritePatternSet patterns(ctx);
147 populateStageSparseOperationsPatterns(patterns);
148 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
149 }
150};
151
152struct LowerSparseOpsToForeachPass
153 : public impl::LowerSparseOpsToForeachBase<LowerSparseOpsToForeachPass> {
154 LowerSparseOpsToForeachPass() = default;
155 LowerSparseOpsToForeachPass(const LowerSparseOpsToForeachPass &pass) =
156 default;
157 LowerSparseOpsToForeachPass(bool enableRT, bool convert) {
158 enableRuntimeLibrary = enableRT;
159 enableConvert = convert;
160 }
161
162 void runOnOperation() override {
163 auto *ctx = &getContext();
164 RewritePatternSet patterns(ctx);
165 populateLowerSparseOpsToForeachPatterns(patterns, enableRT: enableRuntimeLibrary,
166 enableConvert);
167 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
168 }
169};
170
171struct LowerForeachToSCFPass
172 : public impl::LowerForeachToSCFBase<LowerForeachToSCFPass> {
173 LowerForeachToSCFPass() = default;
174 LowerForeachToSCFPass(const LowerForeachToSCFPass &pass) = default;
175
176 void runOnOperation() override {
177 auto *ctx = &getContext();
178 RewritePatternSet patterns(ctx);
179 populateLowerForeachToSCFPatterns(patterns);
180 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
181 }
182};
183
184struct LowerSparseIterationToSCFPass
185 : public impl::LowerSparseIterationToSCFBase<
186 LowerSparseIterationToSCFPass> {
187 LowerSparseIterationToSCFPass() = default;
188 LowerSparseIterationToSCFPass(const LowerSparseIterationToSCFPass &) =
189 default;
190
191 void runOnOperation() override {
192 if (failed(Result: verifyLowerableCollapsedIterSpaces(op: getOperation()))) {
193 signalPassFailure();
194 return;
195 }
196
197 auto *ctx = &getContext();
198 RewritePatternSet patterns(ctx);
199 SparseIterationTypeConverter converter;
200 ConversionTarget target(*ctx);
201
202 // The actual conversion.
203 target.addLegalDialect<arith::ArithDialect, linalg::LinalgDialect,
204 memref::MemRefDialect, scf::SCFDialect,
205 sparse_tensor::SparseTensorDialect>();
206 target.addIllegalOp<CoIterateOp, ExtractIterSpaceOp, ExtractValOp,
207 IterateOp>();
208 target.addLegalOp<UnrealizedConversionCastOp>();
209 populateLowerSparseIterationToSCFPatterns(converter, patterns);
210
211 if (failed(Result: applyPartialConversion(op: getOperation(), target,
212 patterns: std::move(patterns))))
213 signalPassFailure();
214 }
215};
216
217struct SparseTensorConversionPass
218 : public impl::SparseTensorConversionPassBase<SparseTensorConversionPass> {
219 SparseTensorConversionPass() = default;
220 SparseTensorConversionPass(const SparseTensorConversionPass &pass) = default;
221
222 void runOnOperation() override {
223 auto *ctx = &getContext();
224 RewritePatternSet patterns(ctx);
225 SparseTensorTypeToPtrConverter converter;
226 ConversionTarget target(*ctx);
227 // Everything in the sparse dialect must go!
228 target.addIllegalDialect<SparseTensorDialect>();
229 // All dynamic rules below accept new function, call, return, and various
230 // tensor and bufferization operations as legal output of the rewriting
231 // provided that all sparse tensor types have been fully rewritten.
232 target.addDynamicallyLegalOp<func::FuncOp>(callback: [&](func::FuncOp op) {
233 return converter.isSignatureLegal(ty: op.getFunctionType());
234 });
235 target.addDynamicallyLegalOp<func::CallOp>(callback: [&](func::CallOp op) {
236 return converter.isSignatureLegal(ty: op.getCalleeType());
237 });
238 target.addDynamicallyLegalOp<func::ReturnOp>(callback: [&](func::ReturnOp op) {
239 return converter.isLegal(range: op.getOperandTypes());
240 });
241 target.addDynamicallyLegalOp<tensor::DimOp>(callback: [&](tensor::DimOp op) {
242 return converter.isLegal(range: op.getOperandTypes());
243 });
244 target.addDynamicallyLegalOp<tensor::CastOp>(callback: [&](tensor::CastOp op) {
245 return converter.isLegal(type: op.getSource().getType()) &&
246 converter.isLegal(type: op.getDest().getType());
247 });
248 target.addDynamicallyLegalOp<tensor::ExpandShapeOp>(
249 callback: [&](tensor::ExpandShapeOp op) {
250 return converter.isLegal(type: op.getSrc().getType()) &&
251 converter.isLegal(type: op.getResult().getType());
252 });
253 target.addDynamicallyLegalOp<tensor::CollapseShapeOp>(
254 callback: [&](tensor::CollapseShapeOp op) {
255 return converter.isLegal(type: op.getSrc().getType()) &&
256 converter.isLegal(type: op.getResult().getType());
257 });
258 target.addDynamicallyLegalOp<bufferization::AllocTensorOp>(
259 callback: [&](bufferization::AllocTensorOp op) {
260 return converter.isLegal(type: op.getType());
261 });
262 target.addDynamicallyLegalOp<bufferization::DeallocTensorOp>(
263 callback: [&](bufferization::DeallocTensorOp op) {
264 return converter.isLegal(type: op.getTensor().getType());
265 });
266 // The following operations and dialects may be introduced by the
267 // rewriting rules, and are therefore marked as legal.
268 target.addLegalOp<complex::ConstantOp, complex::NotEqualOp, linalg::FillOp,
269 linalg::YieldOp, tensor::ExtractOp,
270 tensor::FromElementsOp>();
271 target.addLegalDialect<
272 arith::ArithDialect, bufferization::BufferizationDialect,
273 LLVM::LLVMDialect, memref::MemRefDialect, scf::SCFDialect>();
274
275 // Populate with rules and apply rewriting rules.
276 populateFunctionOpInterfaceTypeConversionPattern<func::FuncOp>(patterns,
277 converter);
278 populateCallOpTypeConversionPattern(patterns, converter);
279 scf::populateSCFStructuralTypeConversionsAndLegality(typeConverter: converter, patterns,
280 target);
281 populateSparseTensorConversionPatterns(typeConverter: converter, patterns);
282 if (failed(Result: applyPartialConversion(op: getOperation(), target,
283 patterns: std::move(patterns))))
284 signalPassFailure();
285 }
286};
287
288struct SparseTensorCodegenPass
289 : public impl::SparseTensorCodegenBase<SparseTensorCodegenPass> {
290 SparseTensorCodegenPass() = default;
291 SparseTensorCodegenPass(const SparseTensorCodegenPass &pass) = default;
292 SparseTensorCodegenPass(bool createDeallocs, bool enableInit) {
293 createSparseDeallocs = createDeallocs;
294 enableBufferInitialization = enableInit;
295 }
296
297 void runOnOperation() override {
298 auto *ctx = &getContext();
299 RewritePatternSet patterns(ctx);
300 SparseTensorTypeToBufferConverter converter;
301 ConversionTarget target(*ctx);
302 // Most ops in the sparse dialect must go!
303 target.addIllegalDialect<SparseTensorDialect>();
304 target.addLegalOp<SortOp>();
305 target.addLegalOp<PushBackOp>();
306 // Storage specifier outlives sparse tensor pipeline.
307 target.addLegalOp<GetStorageSpecifierOp>();
308 target.addLegalOp<SetStorageSpecifierOp>();
309 target.addLegalOp<StorageSpecifierInitOp>();
310 // Note that tensor::FromElementsOp might be yield after lowering unpack.
311 target.addLegalOp<tensor::FromElementsOp>();
312 // All dynamic rules below accept new function, call, return, and
313 // various tensor and bufferization operations as legal output of the
314 // rewriting provided that all sparse tensor types have been fully
315 // rewritten.
316 target.addDynamicallyLegalOp<func::FuncOp>(callback: [&](func::FuncOp op) {
317 return converter.isSignatureLegal(ty: op.getFunctionType());
318 });
319 target.addDynamicallyLegalOp<func::CallOp>(callback: [&](func::CallOp op) {
320 return converter.isSignatureLegal(ty: op.getCalleeType());
321 });
322 target.addDynamicallyLegalOp<func::ReturnOp>(callback: [&](func::ReturnOp op) {
323 return converter.isLegal(range: op.getOperandTypes());
324 });
325 target.addDynamicallyLegalOp<bufferization::AllocTensorOp>(
326 callback: [&](bufferization::AllocTensorOp op) {
327 return converter.isLegal(type: op.getType());
328 });
329 target.addDynamicallyLegalOp<bufferization::DeallocTensorOp>(
330 callback: [&](bufferization::DeallocTensorOp op) {
331 return converter.isLegal(type: op.getTensor().getType());
332 });
333 // The following operations and dialects may be introduced by the
334 // codegen rules, and are therefore marked as legal.
335 target.addLegalOp<linalg::FillOp, linalg::YieldOp>();
336 target.addLegalDialect<
337 arith::ArithDialect, bufferization::BufferizationDialect,
338 complex::ComplexDialect, memref::MemRefDialect, scf::SCFDialect>();
339 target.addLegalOp<UnrealizedConversionCastOp>();
340 // Populate with rules and apply rewriting rules.
341 populateFunctionOpInterfaceTypeConversionPattern<func::FuncOp>(patterns,
342 converter);
343 scf::populateSCFStructuralTypeConversionsAndLegality(typeConverter: converter, patterns,
344 target);
345 populateSparseTensorCodegenPatterns(
346 typeConverter: converter, patterns, createSparseDeallocs, enableBufferInitialization);
347 if (failed(Result: applyPartialConversion(op: getOperation(), target,
348 patterns: std::move(patterns))))
349 signalPassFailure();
350 }
351};
352
353struct SparseBufferRewritePass
354 : public impl::SparseBufferRewriteBase<SparseBufferRewritePass> {
355 SparseBufferRewritePass() = default;
356 SparseBufferRewritePass(const SparseBufferRewritePass &pass) = default;
357 SparseBufferRewritePass(bool enableInit) {
358 enableBufferInitialization = enableInit;
359 }
360
361 void runOnOperation() override {
362 auto *ctx = &getContext();
363 RewritePatternSet patterns(ctx);
364 populateSparseBufferRewriting(patterns, enableBufferInitialization);
365 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
366 }
367};
368
369struct SparseVectorizationPass
370 : public impl::SparseVectorizationBase<SparseVectorizationPass> {
371 SparseVectorizationPass() = default;
372 SparseVectorizationPass(const SparseVectorizationPass &pass) = default;
373 SparseVectorizationPass(unsigned vl, bool vla, bool sidx32) {
374 vectorLength = vl;
375 enableVLAVectorization = vla;
376 enableSIMDIndex32 = sidx32;
377 }
378
379 void runOnOperation() override {
380 if (vectorLength == 0)
381 return signalPassFailure();
382 auto *ctx = &getContext();
383 RewritePatternSet patterns(ctx);
384 populateSparseVectorizationPatterns(
385 patterns, vectorLength, enableVLAVectorization, enableSIMDIndex32);
386 vector::populateVectorToVectorCanonicalizationPatterns(patterns);
387 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
388 }
389};
390
391struct SparseGPUCodegenPass
392 : public impl::SparseGPUCodegenBase<SparseGPUCodegenPass> {
393 SparseGPUCodegenPass() = default;
394 SparseGPUCodegenPass(const SparseGPUCodegenPass &pass) = default;
395 SparseGPUCodegenPass(unsigned nT, bool enableRT) {
396 numThreads = nT;
397 enableRuntimeLibrary = enableRT;
398 }
399
400 void runOnOperation() override {
401 auto *ctx = &getContext();
402 RewritePatternSet patterns(ctx);
403 if (numThreads == 0)
404 populateSparseGPULibgenPatterns(patterns, enableRT: enableRuntimeLibrary);
405 else
406 populateSparseGPUCodegenPatterns(patterns, numThreads);
407 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
408 }
409};
410
411struct StorageSpecifierToLLVMPass
412 : public impl::StorageSpecifierToLLVMBase<StorageSpecifierToLLVMPass> {
413 StorageSpecifierToLLVMPass() = default;
414
415 void runOnOperation() override {
416 auto *ctx = &getContext();
417 ConversionTarget target(*ctx);
418 RewritePatternSet patterns(ctx);
419 StorageSpecifierToLLVMTypeConverter converter;
420
421 // All ops in the sparse dialect must go!
422 target.addIllegalDialect<SparseTensorDialect>();
423 target.addDynamicallyLegalOp<func::FuncOp>(callback: [&](func::FuncOp op) {
424 return converter.isSignatureLegal(ty: op.getFunctionType());
425 });
426 target.addDynamicallyLegalOp<func::CallOp>(callback: [&](func::CallOp op) {
427 return converter.isSignatureLegal(ty: op.getCalleeType());
428 });
429 target.addDynamicallyLegalOp<func::ReturnOp>(callback: [&](func::ReturnOp op) {
430 return converter.isLegal(range: op.getOperandTypes());
431 });
432 target.addLegalDialect<arith::ArithDialect, LLVM::LLVMDialect>();
433
434 populateFunctionOpInterfaceTypeConversionPattern<func::FuncOp>(patterns,
435 converter);
436 populateCallOpTypeConversionPattern(patterns, converter);
437 populateBranchOpInterfaceTypeConversionPattern(patterns, converter);
438 populateReturnOpTypeConversionPattern(patterns, converter);
439 scf::populateSCFStructuralTypeConversionsAndLegality(typeConverter: converter, patterns,
440 target);
441 populateStorageSpecifierToLLVMPatterns(converter, patterns);
442 if (failed(Result: applyPartialConversion(op: getOperation(), target,
443 patterns: std::move(patterns))))
444 signalPassFailure();
445 }
446};
447
448} // namespace
449
450//===----------------------------------------------------------------------===//
451// Pass creation methods.
452//===----------------------------------------------------------------------===//
453
454std::unique_ptr<Pass> mlir::createSparseAssembler() {
455 return std::make_unique<SparseAssembler>();
456}
457
458std::unique_ptr<Pass> mlir::createSparseReinterpretMapPass() {
459 return std::make_unique<SparseReinterpretMap>();
460}
461
462std::unique_ptr<Pass>
463mlir::createSparseReinterpretMapPass(ReinterpretMapScope scope) {
464 SparseReinterpretMapOptions options;
465 options.scope = scope;
466 return std::make_unique<SparseReinterpretMap>(args&: options);
467}
468
469std::unique_ptr<Pass> mlir::createSparseReinterpretMapPass(
470 ReinterpretMapScope scope, sparse_tensor::LoopOrderingStrategy strategy) {
471 SparseReinterpretMapOptions options;
472 options.scope = scope;
473 options.loopOrderingStrategy = strategy;
474 return std::make_unique<SparseReinterpretMap>(args&: options);
475}
476
477std::unique_ptr<Pass> mlir::createPreSparsificationRewritePass() {
478 return std::make_unique<PreSparsificationRewritePass>();
479}
480
481std::unique_ptr<Pass> mlir::createSparsificationPass() {
482 return std::make_unique<SparsificationPass>();
483}
484
485std::unique_ptr<Pass>
486mlir::createSparsificationPass(const SparsificationOptions &options) {
487 return std::make_unique<SparsificationPass>(args: options);
488}
489
490std::unique_ptr<Pass> mlir::createStageSparseOperationsPass() {
491 return std::make_unique<StageSparseOperationsPass>();
492}
493
494std::unique_ptr<Pass> mlir::createLowerSparseOpsToForeachPass() {
495 return std::make_unique<LowerSparseOpsToForeachPass>();
496}
497
498std::unique_ptr<Pass>
499mlir::createLowerSparseOpsToForeachPass(bool enableRT, bool enableConvert) {
500 return std::make_unique<LowerSparseOpsToForeachPass>(args&: enableRT, args&: enableConvert);
501}
502
503std::unique_ptr<Pass> mlir::createLowerForeachToSCFPass() {
504 return std::make_unique<LowerForeachToSCFPass>();
505}
506
507std::unique_ptr<Pass> mlir::createLowerSparseIterationToSCFPass() {
508 return std::make_unique<LowerSparseIterationToSCFPass>();
509}
510
511std::unique_ptr<Pass> mlir::createSparseTensorConversionPass() {
512 return std::make_unique<SparseTensorConversionPass>();
513}
514
515std::unique_ptr<Pass> mlir::createSparseTensorCodegenPass() {
516 return std::make_unique<SparseTensorCodegenPass>();
517}
518
519std::unique_ptr<Pass>
520mlir::createSparseTensorCodegenPass(bool createSparseDeallocs,
521 bool enableBufferInitialization) {
522 return std::make_unique<SparseTensorCodegenPass>(args&: createSparseDeallocs,
523 args&: enableBufferInitialization);
524}
525
526std::unique_ptr<Pass> mlir::createSparseBufferRewritePass() {
527 return std::make_unique<SparseBufferRewritePass>();
528}
529
530std::unique_ptr<Pass>
531mlir::createSparseBufferRewritePass(bool enableBufferInitialization) {
532 return std::make_unique<SparseBufferRewritePass>(args&: enableBufferInitialization);
533}
534
535std::unique_ptr<Pass> mlir::createSparseVectorizationPass() {
536 return std::make_unique<SparseVectorizationPass>();
537}
538
539std::unique_ptr<Pass>
540mlir::createSparseVectorizationPass(unsigned vectorLength,
541 bool enableVLAVectorization,
542 bool enableSIMDIndex32) {
543 return std::make_unique<SparseVectorizationPass>(
544 args&: vectorLength, args&: enableVLAVectorization, args&: enableSIMDIndex32);
545}
546
547std::unique_ptr<Pass> mlir::createSparseGPUCodegenPass() {
548 return std::make_unique<SparseGPUCodegenPass>();
549}
550
551std::unique_ptr<Pass> mlir::createSparseGPUCodegenPass(unsigned numThreads,
552 bool enableRT) {
553 return std::make_unique<SparseGPUCodegenPass>(args&: numThreads, args&: enableRT);
554}
555
556std::unique_ptr<Pass> mlir::createStorageSpecifierToLLVMPass() {
557 return std::make_unique<StorageSpecifierToLLVMPass>();
558}
559