1//===- SparseStorageSpecifierToLLVM.cpp - convert specifier to llvm -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "Utils/CodegenUtils.h"
10#include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h"
11#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
12
13#include "mlir/Conversion/LLVMCommon/StructBuilder.h"
14#include "mlir/Dialect/SparseTensor/IR/SparseTensorStorageLayout.h"
15#include "mlir/Dialect/SparseTensor/Transforms/Passes.h"
16
17#include <optional>
18
19using namespace mlir;
20using namespace sparse_tensor;
21
22namespace {
23
24//===----------------------------------------------------------------------===//
25// Helper methods.
26//===----------------------------------------------------------------------===//
27
28static SmallVector<Type, 4> getSpecifierFields(StorageSpecifierType tp) {
29 MLIRContext *ctx = tp.getContext();
30 auto enc = tp.getEncoding();
31 const Level lvlRank = enc.getLvlRank();
32
33 SmallVector<Type, 4> result;
34 // TODO: how can we get the lowering type for index type in the later pipeline
35 // to be consistent? LLVM::StructureType does not allow index fields.
36 auto sizeType = IntegerType::get(context: tp.getContext(), width: 64);
37 auto lvlSizes = LLVM::LLVMArrayType::get(context: ctx, elementType: sizeType, numElements: lvlRank);
38 auto memSizes = LLVM::LLVMArrayType::get(context: ctx, elementType: sizeType,
39 numElements: getNumDataFieldsFromEncoding(enc));
40 result.push_back(Elt: lvlSizes);
41 result.push_back(Elt: memSizes);
42
43 if (enc.isSlice()) {
44 // Extra fields are required for the slice information.
45 auto dimOffset = LLVM::LLVMArrayType::get(context: ctx, elementType: sizeType, numElements: lvlRank);
46 auto dimStride = LLVM::LLVMArrayType::get(context: ctx, elementType: sizeType, numElements: lvlRank);
47
48 result.push_back(Elt: dimOffset);
49 result.push_back(Elt: dimStride);
50 }
51
52 return result;
53}
54
55static Type convertSpecifier(StorageSpecifierType tp) {
56 return LLVM::LLVMStructType::getLiteral(context: tp.getContext(),
57 types: getSpecifierFields(tp));
58}
59
60//===----------------------------------------------------------------------===//
61// Specifier struct builder.
62//===----------------------------------------------------------------------===//
63
64constexpr uint64_t kLvlSizePosInSpecifier = 0;
65constexpr uint64_t kMemSizePosInSpecifier = 1;
66constexpr uint64_t kDimOffsetPosInSpecifier = 2;
67constexpr uint64_t kDimStridePosInSpecifier = 3;
68
69class SpecifierStructBuilder : public StructBuilder {
70private:
71 Value extractField(OpBuilder &builder, Location loc,
72 ArrayRef<int64_t> indices) const {
73 return genCast(builder, loc,
74 value: LLVM::ExtractValueOp::create(builder, location: loc, container: value, position: indices),
75 dstTy: builder.getIndexType());
76 }
77
78 void insertField(OpBuilder &builder, Location loc, ArrayRef<int64_t> indices,
79 Value v) {
80 value = LLVM::InsertValueOp::create(
81 builder, location: loc, container: value,
82 value: genCast(builder, loc, value: v, dstTy: builder.getIntegerType(width: 64)), position: indices);
83 }
84
85public:
86 explicit SpecifierStructBuilder(Value specifier) : StructBuilder(specifier) {
87 assert(value);
88 }
89
90 // Undef value for dimension sizes, all zero value for memory sizes.
91 static Value getInitValue(OpBuilder &builder, Location loc, Type structType,
92 Value source);
93
94 Value lvlSize(OpBuilder &builder, Location loc, Level lvl) const;
95 void setLvlSize(OpBuilder &builder, Location loc, Level lvl, Value size);
96
97 Value dimOffset(OpBuilder &builder, Location loc, Dimension dim) const;
98 void setDimOffset(OpBuilder &builder, Location loc, Dimension dim,
99 Value size);
100
101 Value dimStride(OpBuilder &builder, Location loc, Dimension dim) const;
102 void setDimStride(OpBuilder &builder, Location loc, Dimension dim,
103 Value size);
104
105 Value memSize(OpBuilder &builder, Location loc, FieldIndex fidx) const;
106 void setMemSize(OpBuilder &builder, Location loc, FieldIndex fidx,
107 Value size);
108
109 Value memSizeArray(OpBuilder &builder, Location loc) const;
110 void setMemSizeArray(OpBuilder &builder, Location loc, Value array);
111};
112
113Value SpecifierStructBuilder::getInitValue(OpBuilder &builder, Location loc,
114 Type structType, Value source) {
115 Value metaData = LLVM::PoisonOp::create(builder, location: loc, res: structType);
116 SpecifierStructBuilder md(metaData);
117 if (!source) {
118 auto memSizeArrayType =
119 cast<LLVM::LLVMArrayType>(Val: cast<LLVM::LLVMStructType>(Val&: structType)
120 .getBody()[kMemSizePosInSpecifier]);
121
122 Value zero = constantZero(builder, loc, tp: memSizeArrayType.getElementType());
123 // Fill memSizes array with zero.
124 for (int i = 0, e = memSizeArrayType.getNumElements(); i < e; i++)
125 md.setMemSize(builder, loc, fidx: i, size: zero);
126 } else {
127 // We copy non-slice information (memory sizes array) from source
128 SpecifierStructBuilder sourceMd(source);
129 md.setMemSizeArray(builder, loc, array: sourceMd.memSizeArray(builder, loc));
130 }
131 return md;
132}
133
134/// Builds IR extracting the pos-th offset from the descriptor.
135Value SpecifierStructBuilder::dimOffset(OpBuilder &builder, Location loc,
136 Dimension dim) const {
137 return extractField(
138 builder, loc,
139 indices: ArrayRef<int64_t>{kDimOffsetPosInSpecifier, static_cast<int64_t>(dim)});
140}
141
142/// Builds IR inserting the pos-th offset into the descriptor.
143void SpecifierStructBuilder::setDimOffset(OpBuilder &builder, Location loc,
144 Dimension dim, Value size) {
145 insertField(
146 builder, loc,
147 indices: ArrayRef<int64_t>{kDimOffsetPosInSpecifier, static_cast<int64_t>(dim)},
148 v: size);
149}
150
151/// Builds IR extracting the `lvl`-th level-size from the descriptor.
152Value SpecifierStructBuilder::lvlSize(OpBuilder &builder, Location loc,
153 Level lvl) const {
154 // This static_cast makes the narrowing of `lvl` explicit, as required
155 // by the braces notation for the ctor.
156 return extractField(
157 builder, loc,
158 indices: ArrayRef<int64_t>{kLvlSizePosInSpecifier, static_cast<int64_t>(lvl)});
159}
160
161/// Builds IR inserting the `lvl`-th level-size into the descriptor.
162void SpecifierStructBuilder::setLvlSize(OpBuilder &builder, Location loc,
163 Level lvl, Value size) {
164 // This static_cast makes the narrowing of `lvl` explicit, as required
165 // by the braces notation for the ctor.
166 insertField(
167 builder, loc,
168 indices: ArrayRef<int64_t>{kLvlSizePosInSpecifier, static_cast<int64_t>(lvl)},
169 v: size);
170}
171
172/// Builds IR extracting the pos-th stride from the descriptor.
173Value SpecifierStructBuilder::dimStride(OpBuilder &builder, Location loc,
174 Dimension dim) const {
175 return extractField(
176 builder, loc,
177 indices: ArrayRef<int64_t>{kDimStridePosInSpecifier, static_cast<int64_t>(dim)});
178}
179
180/// Builds IR inserting the pos-th stride into the descriptor.
181void SpecifierStructBuilder::setDimStride(OpBuilder &builder, Location loc,
182 Dimension dim, Value size) {
183 insertField(
184 builder, loc,
185 indices: ArrayRef<int64_t>{kDimStridePosInSpecifier, static_cast<int64_t>(dim)},
186 v: size);
187}
188
189/// Builds IR extracting the pos-th memory size into the descriptor.
190Value SpecifierStructBuilder::memSize(OpBuilder &builder, Location loc,
191 FieldIndex fidx) const {
192 return extractField(
193 builder, loc,
194 indices: ArrayRef<int64_t>{kMemSizePosInSpecifier, static_cast<int64_t>(fidx)});
195}
196
197/// Builds IR inserting the `fidx`-th memory-size into the descriptor.
198void SpecifierStructBuilder::setMemSize(OpBuilder &builder, Location loc,
199 FieldIndex fidx, Value size) {
200 insertField(
201 builder, loc,
202 indices: ArrayRef<int64_t>{kMemSizePosInSpecifier, static_cast<int64_t>(fidx)},
203 v: size);
204}
205
206/// Builds IR extracting the memory size array from the descriptor.
207Value SpecifierStructBuilder::memSizeArray(OpBuilder &builder,
208 Location loc) const {
209 return LLVM::ExtractValueOp::create(builder, location: loc, container: value,
210 position: kMemSizePosInSpecifier);
211}
212
213/// Builds IR inserting the memory size array into the descriptor.
214void SpecifierStructBuilder::setMemSizeArray(OpBuilder &builder, Location loc,
215 Value array) {
216 value = LLVM::InsertValueOp::create(builder, location: loc, container: value, value: array,
217 position: kMemSizePosInSpecifier);
218}
219
220} // namespace
221
222//===----------------------------------------------------------------------===//
223// The sparse storage specifier type converter (defined in Passes.h).
224//===----------------------------------------------------------------------===//
225
226StorageSpecifierToLLVMTypeConverter::StorageSpecifierToLLVMTypeConverter() {
227 addConversion(callback: [](Type type) { return type; });
228 addConversion(callback&: convertSpecifier);
229}
230
231//===----------------------------------------------------------------------===//
232// Storage specifier conversion rules.
233//===----------------------------------------------------------------------===//
234
235template <typename Base, typename SourceOp>
236class SpecifierGetterSetterOpConverter : public OpConversionPattern<SourceOp> {
237public:
238 using OpAdaptor = typename SourceOp::Adaptor;
239 using OpConversionPattern<SourceOp>::OpConversionPattern;
240
241 LogicalResult
242 matchAndRewrite(SourceOp op, OpAdaptor adaptor,
243 ConversionPatternRewriter &rewriter) const override {
244 SpecifierStructBuilder spec(adaptor.getSpecifier());
245 switch (op.getSpecifierKind()) {
246 case StorageSpecifierKind::LvlSize: {
247 Value v = Base::onLvlSize(rewriter, op, spec, (*op.getLevel()));
248 rewriter.replaceOp(op, v);
249 return success();
250 }
251 case StorageSpecifierKind::DimOffset: {
252 Value v = Base::onDimOffset(rewriter, op, spec, (*op.getLevel()));
253 rewriter.replaceOp(op, v);
254 return success();
255 }
256 case StorageSpecifierKind::DimStride: {
257 Value v = Base::onDimStride(rewriter, op, spec, (*op.getLevel()));
258 rewriter.replaceOp(op, v);
259 return success();
260 }
261 case StorageSpecifierKind::CrdMemSize:
262 case StorageSpecifierKind::PosMemSize:
263 case StorageSpecifierKind::ValMemSize: {
264 auto enc = op.getSpecifier().getType().getEncoding();
265 StorageLayout layout(enc);
266 std::optional<unsigned> lvl;
267 if (op.getLevel())
268 lvl = (*op.getLevel());
269 unsigned idx =
270 layout.getMemRefFieldIndex(kind: toFieldKind(op.getSpecifierKind()), lvl);
271 Value v = Base::onMemSize(rewriter, op, spec, idx);
272 rewriter.replaceOp(op, v);
273 return success();
274 }
275 }
276 llvm_unreachable("unrecognized specifer kind");
277 }
278};
279
280struct StorageSpecifierSetOpConverter
281 : public SpecifierGetterSetterOpConverter<StorageSpecifierSetOpConverter,
282 SetStorageSpecifierOp> {
283 using SpecifierGetterSetterOpConverter::SpecifierGetterSetterOpConverter;
284
285 static Value onLvlSize(OpBuilder &builder, SetStorageSpecifierOp op,
286 SpecifierStructBuilder &spec, Level lvl) {
287 spec.setLvlSize(builder, loc: op.getLoc(), lvl, size: op.getValue());
288 return spec;
289 }
290
291 static Value onDimOffset(OpBuilder &builder, SetStorageSpecifierOp op,
292 SpecifierStructBuilder &spec, Dimension d) {
293 spec.setDimOffset(builder, loc: op.getLoc(), dim: d, size: op.getValue());
294 return spec;
295 }
296
297 static Value onDimStride(OpBuilder &builder, SetStorageSpecifierOp op,
298 SpecifierStructBuilder &spec, Dimension d) {
299 spec.setDimStride(builder, loc: op.getLoc(), dim: d, size: op.getValue());
300 return spec;
301 }
302
303 static Value onMemSize(OpBuilder &builder, SetStorageSpecifierOp op,
304 SpecifierStructBuilder &spec, FieldIndex fidx) {
305 spec.setMemSize(builder, loc: op.getLoc(), fidx, size: op.getValue());
306 return spec;
307 }
308};
309
310struct StorageSpecifierGetOpConverter
311 : public SpecifierGetterSetterOpConverter<StorageSpecifierGetOpConverter,
312 GetStorageSpecifierOp> {
313 using SpecifierGetterSetterOpConverter::SpecifierGetterSetterOpConverter;
314
315 static Value onLvlSize(OpBuilder &builder, GetStorageSpecifierOp op,
316 SpecifierStructBuilder &spec, Level lvl) {
317 return spec.lvlSize(builder, loc: op.getLoc(), lvl);
318 }
319
320 static Value onDimOffset(OpBuilder &builder, GetStorageSpecifierOp op,
321 const SpecifierStructBuilder &spec, Dimension d) {
322 return spec.dimOffset(builder, loc: op.getLoc(), dim: d);
323 }
324
325 static Value onDimStride(OpBuilder &builder, GetStorageSpecifierOp op,
326 const SpecifierStructBuilder &spec, Dimension d) {
327 return spec.dimStride(builder, loc: op.getLoc(), dim: d);
328 }
329
330 static Value onMemSize(OpBuilder &builder, GetStorageSpecifierOp op,
331 SpecifierStructBuilder &spec, FieldIndex fidx) {
332 return spec.memSize(builder, loc: op.getLoc(), fidx);
333 }
334};
335
336struct StorageSpecifierInitOpConverter
337 : public OpConversionPattern<StorageSpecifierInitOp> {
338public:
339 using Base::Base;
340 LogicalResult
341 matchAndRewrite(StorageSpecifierInitOp op, OpAdaptor adaptor,
342 ConversionPatternRewriter &rewriter) const override {
343 Type llvmType = getTypeConverter()->convertType(t: op.getResult().getType());
344 rewriter.replaceOp(
345 op, newValues: SpecifierStructBuilder::getInitValue(
346 builder&: rewriter, loc: op.getLoc(), structType: llvmType, source: adaptor.getSource()));
347 return success();
348 }
349};
350
351//===----------------------------------------------------------------------===//
352// Public method for populating conversion rules.
353//===----------------------------------------------------------------------===//
354
355void mlir::populateStorageSpecifierToLLVMPatterns(
356 const TypeConverter &converter, RewritePatternSet &patterns) {
357 patterns.add<StorageSpecifierGetOpConverter, StorageSpecifierSetOpConverter,
358 StorageSpecifierInitOpConverter>(arg: converter,
359 args: patterns.getContext());
360}
361