1//===- SparseAssembler.cpp - adds wrapper method around sparse types ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "Utils/CodegenUtils.h"
10
11#include "mlir/Dialect/LLVMIR/LLVMDialectDecl.h"
12#include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
13#include "mlir/Dialect/SparseTensor/IR/SparseTensorStorageLayout.h"
14#include "mlir/Dialect/SparseTensor/IR/SparseTensorType.h"
15#include "mlir/Dialect/SparseTensor/Transforms/Passes.h"
16#include "llvm/Support/FormatVariadic.h"
17
18using namespace mlir;
19using namespace sparse_tensor;
20
21//===----------------------------------------------------------------------===//
22// Helper methods.
23//===----------------------------------------------------------------------===//
24
25// Convert type range to new types range, with sparse tensors externalized.
26static void convTypes(bool &hasAnnotation, TypeRange types,
27 SmallVectorImpl<Type> &convTypes,
28 SmallVectorImpl<Type> *extraTypes, bool directOut) {
29 for (auto type : types) {
30 // All "dense" data passes through unmodified. Note: getSparseTensorEncoding
31 // also returns non-null for StorageSpecifierType (which is not a
32 // RankedTensorType), so we must check isa<RankedTensorType> as well.
33 if (!getSparseTensorEncoding(type) || !isa<RankedTensorType>(Val: type)) {
34 convTypes.push_back(Elt: type);
35 continue;
36 }
37 hasAnnotation = true;
38
39 // Convert the external representations of the pos/crd/val arrays.
40 const SparseTensorType stt(cast<RankedTensorType>(Val&: type));
41 foreachFieldAndTypeInSparseTensor(
42 stt, [&convTypes, extraTypes, directOut](Type t, FieldIndex,
43 SparseTensorFieldKind kind,
44 Level, LevelType) {
45 if (kind == SparseTensorFieldKind::PosMemRef ||
46 kind == SparseTensorFieldKind::CrdMemRef ||
47 kind == SparseTensorFieldKind::ValMemRef) {
48 auto rtp = cast<ShapedType>(Val&: t);
49 if (!directOut) {
50 rtp = RankedTensorType::get(shape: rtp.getShape(), elementType: rtp.getElementType());
51 if (extraTypes)
52 extraTypes->push_back(Elt: rtp);
53 }
54 convTypes.push_back(Elt: rtp);
55 }
56 return true;
57 });
58 }
59}
60
61// Convert input and output values to [dis]assemble ops for sparse tensors.
62static void convVals(OpBuilder &builder, Location loc, TypeRange types,
63 ValueRange fromVals, ValueRange extraVals,
64 SmallVectorImpl<Value> &toVals, unsigned extra, bool isIn,
65 bool directOut) {
66 unsigned idx = 0;
67 for (auto type : types) {
68 // All "dense" data passes through unmodified. Note: getSparseTensorEncoding
69 // also returns non-null for StorageSpecifierType (which is not a
70 // RankedTensorType), so we must check isa<RankedTensorType> as well.
71 if (!getSparseTensorEncoding(type) || !isa<RankedTensorType>(Val: type)) {
72 toVals.push_back(Elt: fromVals[idx++]);
73 continue;
74 }
75 // Handle sparse data.
76 auto rtp = cast<RankedTensorType>(Val&: type);
77 const SparseTensorType stt(rtp);
78 SmallVector<Value> inputs;
79 SmallVector<Type> retTypes;
80 SmallVector<Type> cntTypes;
81 if (!isIn)
82 inputs.push_back(Elt: fromVals[idx++]); // The sparse tensor to disassemble
83
84 // Collect the external representations of the pos/crd/val arrays.
85 foreachFieldAndTypeInSparseTensor(stt, [&, isIn](Type t, FieldIndex,
86 SparseTensorFieldKind kind,
87 Level lv, LevelType) {
88 if (kind == SparseTensorFieldKind::PosMemRef ||
89 kind == SparseTensorFieldKind::CrdMemRef ||
90 kind == SparseTensorFieldKind::ValMemRef) {
91 if (isIn) {
92 inputs.push_back(Elt: fromVals[idx++]);
93 } else if (directOut) {
94 Value mem;
95 if (kind == SparseTensorFieldKind::PosMemRef)
96 mem = sparse_tensor::ToPositionsOp::create(builder, location: loc, tensor: inputs[0],
97 level: lv);
98 else if (kind == SparseTensorFieldKind::CrdMemRef)
99 mem = sparse_tensor::ToCoordinatesOp::create(builder, location: loc,
100 tensor: inputs[0], level: lv);
101 else
102 mem = sparse_tensor::ToValuesOp::create(builder, location: loc, tensor: inputs[0]);
103 toVals.push_back(Elt: mem);
104 } else {
105 ShapedType rtp = cast<ShapedType>(Val&: t);
106 rtp = RankedTensorType::get(shape: rtp.getShape(), elementType: rtp.getElementType());
107 inputs.push_back(Elt: extraVals[extra++]);
108 retTypes.push_back(Elt: rtp);
109 cntTypes.push_back(Elt: builder.getIndexType());
110 }
111 }
112 return true;
113 });
114
115 if (isIn) {
116 // Assemble multiple inputs into a single sparse tensor.
117 auto a = sparse_tensor::AssembleOp::create(builder, location: loc, resultTypes: rtp, operands: inputs);
118 toVals.push_back(Elt: a.getResult());
119 } else if (!directOut) {
120 // Disassemble a single sparse input into multiple outputs.
121 // Note that this includes the counters, which are dropped.
122 unsigned len = retTypes.size();
123 retTypes.append(RHS: cntTypes);
124 auto d =
125 sparse_tensor::DisassembleOp::create(builder, location: loc, resultTypes: retTypes, operands: inputs);
126 for (unsigned i = 0; i < len; i++)
127 toVals.push_back(Elt: d.getResult(i));
128 }
129 }
130}
131
132//===----------------------------------------------------------------------===//
133// Rewriting rules.
134//===----------------------------------------------------------------------===//
135
136namespace {
137
138// A rewriting rules that converts public entry methods that use sparse tensors
139// as input parameters and/or output return values into wrapper methods that
140// [dis]assemble the individual tensors that constitute the actual storage used
141// externally into MLIR sparse tensors before calling the original method.
142//
143// In particular, each sparse tensor input
144//
145// void foo(..., t, ...) { }
146//
147// makes the original foo() internal and adds the following wrapper method
148//
149// void foo(..., t1..tn, ...) {
150// t = assemble t1..tn
151// _internal_foo(..., t, ...)
152// }
153//
154// and likewise, each output tensor
155//
156// ... T ... bar(...) { return ..., t, ...; }
157//
158// makes the original bar() internal and adds the following wrapper method
159//
160// ... T1..TN ... bar(..., t1'..tn') {
161// ..., t, ... = _internal_bar(...)
162// t1..tn = disassemble t, t1'..tn'
163// return ..., t1..tn, ...
164// }
165//
166// (with a direct-out variant without the disassemble).
167//
168struct SparseFuncAssembler : public OpRewritePattern<func::FuncOp> {
169 using Base::Base;
170
171 SparseFuncAssembler(MLIRContext *context, bool dO)
172 : OpRewritePattern(context), directOut(dO) {}
173
174 LogicalResult matchAndRewrite(func::FuncOp funcOp,
175 PatternRewriter &rewriter) const override {
176 // Only rewrite public entry methods.
177 if (funcOp.isPrivate())
178 return failure();
179
180 // Translate sparse tensor types to external types.
181 SmallVector<Type> inputTypes;
182 SmallVector<Type> outputTypes;
183 SmallVector<Type> extraTypes;
184 bool hasAnnotation = false;
185 convTypes(hasAnnotation, types: funcOp.getArgumentTypes(), convTypes&: inputTypes, extraTypes: nullptr,
186 directOut: false);
187 convTypes(hasAnnotation, types: funcOp.getResultTypes(), convTypes&: outputTypes, extraTypes: &extraTypes,
188 directOut);
189
190 // Only sparse inputs or outputs need a wrapper method.
191 if (!hasAnnotation)
192 return failure();
193
194 // Modify the original method into an internal, private method.
195 auto orgName = funcOp.getName();
196 std::string wrapper = llvm::formatv(Fmt: "_internal_{0}", Vals&: orgName).str();
197 rewriter.modifyOpInPlace(root: funcOp, callable: [&]() {
198 funcOp.setName(wrapper);
199 funcOp.setPrivate();
200 });
201
202 // Start the new public wrapper method with original name.
203 Location loc = funcOp.getLoc();
204 ModuleOp modOp = funcOp->getParentOfType<ModuleOp>();
205 MLIRContext *context = modOp.getContext();
206 OpBuilder moduleBuilder(modOp.getBodyRegion());
207 unsigned extra = inputTypes.size();
208 inputTypes.append(RHS: extraTypes);
209 auto func = func::FuncOp::create(
210 builder&: moduleBuilder, location: loc, name: orgName,
211 type: FunctionType::get(context, inputs: inputTypes, results: outputTypes));
212 func.setPublic();
213
214 // Construct new wrapper method body.
215 OpBuilder::InsertionGuard insertionGuard(rewriter);
216 Block *body = func.addEntryBlock();
217 rewriter.setInsertionPointToStart(body);
218
219 // Convert inputs.
220 SmallVector<Value> inputs;
221 convVals(builder&: rewriter, loc, types: funcOp.getArgumentTypes(), fromVals: body->getArguments(),
222 extraVals: ValueRange(), toVals&: inputs, /*extra=*/0, /*isIn=*/true, directOut);
223
224 // Call the original, now private method. A subsequent inlining pass can
225 // determine whether cloning the method body in place is worthwhile.
226 auto org = SymbolRefAttr::get(ctx: context, value: wrapper);
227 auto call = func::CallOp::create(builder&: rewriter, location: loc, results: funcOp.getResultTypes(),
228 callee: org, operands: inputs);
229
230 // Convert outputs and return.
231 SmallVector<Value> outputs;
232 convVals(builder&: rewriter, loc, types: funcOp.getResultTypes(), fromVals: call.getResults(),
233 extraVals: body->getArguments(), toVals&: outputs, extra, /*isIn=*/false, directOut);
234 func::ReturnOp::create(builder&: rewriter, location: loc, operands: outputs);
235
236 // Finally, migrate a potential c-interface property.
237 if (funcOp->getDiscardableAttrOfType<UnitAttr>(
238 name: LLVM::LLVMDialect::getEmitCWrapperAttrName())) {
239 func->setDiscardableAttr(name: LLVM::LLVMDialect::getEmitCWrapperAttrName(),
240 value: UnitAttr::get(context));
241 rewriter.modifyOpInPlace(root: funcOp, callable: [&]() {
242 funcOp->removeDiscardableAttr(
243 name: LLVM::LLVMDialect::getEmitCWrapperAttrName());
244 });
245 }
246 return success();
247 }
248
249private:
250 const bool directOut;
251};
252
253} // namespace
254
255//===----------------------------------------------------------------------===//
256// Public method for populating conversion rules.
257//===----------------------------------------------------------------------===//
258
259void mlir::populateSparseAssembler(RewritePatternSet &patterns,
260 bool directOut) {
261 patterns.add<SparseFuncAssembler>(arg: patterns.getContext(), args&: directOut);
262}
263