1//===- CodegenUtils.h - Utilities for generating MLIR -----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This header file defines utilities for generating MLIR.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_CODEGENUTILS_H_
14#define MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_CODEGENUTILS_H_
15
16#include "mlir/Dialect/Arith/IR/Arith.h"
17#include "mlir/Dialect/Complex/IR/Complex.h"
18#include "mlir/Dialect/Func/IR/FuncOps.h"
19#include "mlir/Dialect/SparseTensor/IR/Enums.h"
20#include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
21#include "mlir/Dialect/SparseTensor/IR/SparseTensorType.h"
22#include "mlir/Dialect/Utils/ReshapeOpsUtils.h"
23#include "mlir/IR/Builders.h"
24
25namespace mlir {
26
27class Location;
28class Type;
29class Value;
30
31namespace sparse_tensor {
32
33/// Shorthand aliases for the `emitCInterface` argument to `getFunc()`,
34/// `createFuncCall()`, and `replaceOpWithFuncCall()`.
35enum class EmitCInterface : bool { Off = false, On = true };
36
37//===----------------------------------------------------------------------===//
38// ExecutionEngine/SparseTensorUtils helper functions.
39//===----------------------------------------------------------------------===//
40
41/// Converts an overhead storage bitwidth to its internal type-encoding.
42OverheadType overheadTypeEncoding(unsigned width);
43
44/// Converts an overhead storage type to its internal type-encoding.
45OverheadType overheadTypeEncoding(Type tp);
46
47/// Converts the internal type-encoding for overhead storage to an mlir::Type.
48Type getOverheadType(Builder &builder, OverheadType ot);
49
50/// Returns the OverheadType for position overhead storage.
51OverheadType posTypeEncoding(SparseTensorEncodingAttr enc);
52
53/// Returns the OverheadType for coordinate overhead storage.
54OverheadType crdTypeEncoding(SparseTensorEncodingAttr enc);
55
56/// Convert OverheadType to its function-name suffix.
57StringRef overheadTypeFunctionSuffix(OverheadType ot);
58
59/// Converts an overhead storage type to its function-name suffix.
60StringRef overheadTypeFunctionSuffix(Type overheadTp);
61
62/// Converts a primary storage type to its internal type-encoding.
63PrimaryType primaryTypeEncoding(Type elemTp);
64
65/// Convert PrimaryType to its function-name suffix.
66StringRef primaryTypeFunctionSuffix(PrimaryType pt);
67
68/// Converts a primary storage type to its function-name suffix.
69StringRef primaryTypeFunctionSuffix(Type elemTp);
70
71//===----------------------------------------------------------------------===//
72// Misc code generators and utilities.
73//===----------------------------------------------------------------------===//
74
75/// A helper class to simplify lowering operations with/without function calls.
76template <class SubClass>
77class FuncCallOrInlineGenerator {
78public:
79 FuncCallOrInlineGenerator(TypeRange retTypes, ValueRange params, bool genCall)
80 : retTypes(retTypes), params(params), genCall(genCall) {}
81
82 // The main API invoked by clients, which abstracts away the details of
83 // creating function calls from clients.
84 SmallVector<Value> genCallOrInline(OpBuilder &builder, Location loc) {
85 if (!genCall)
86 return genImplementation(retTypes, params, builder, loc);
87
88 // Looks up the function.
89 std::string funcName = getMangledFuncName();
90 ModuleOp module = getParentOpOf<ModuleOp>(builder);
91 MLIRContext *context = module.getContext();
92 auto result = SymbolRefAttr::get(ctx: context, value: funcName);
93 auto func = module.lookupSymbol<func::FuncOp>(name: result.getAttr());
94
95 if (!func) {
96 // Create the function if not already exist.
97 OpBuilder::InsertionGuard insertionGuard(builder);
98 builder.setInsertionPoint(getParentOpOf<func::FuncOp>(builder));
99 func = func::FuncOp::create(
100 builder, location: loc, name: funcName,
101 type: FunctionType::get(context, inputs: params.getTypes(), results: retTypes));
102 func.setPrivate();
103 // Set the insertion point to the body of the function.
104 Block *entryBB = func.addEntryBlock();
105 builder.setInsertionPointToStart(entryBB);
106 ValueRange args = entryBB->getArguments();
107 // Delegates to user to generate the actually implementation.
108 SmallVector<Value> result =
109 genImplementation(retTypes, params: args, builder, loc);
110 func::ReturnOp::create(builder, location: loc, operands: result);
111 }
112 // Returns the CallOp result.
113 func::CallOp call = func::CallOp::create(builder, location: loc, callee: func, operands: params);
114 return call.getResults();
115 }
116
117private:
118 template <class OpTp>
119 OpTp getParentOpOf(OpBuilder &builder) {
120 return builder.getInsertionBlock()->getParent()->getParentOfType<OpTp>();
121 }
122
123 // CRTP: get the mangled function name (only called when genCall=true).
124 std::string getMangledFuncName() {
125 return static_cast<SubClass *>(this)->getMangledFuncName();
126 }
127
128 // CRTP: Client implementation.
129 SmallVector<Value> genImplementation(TypeRange retTypes, ValueRange params,
130 OpBuilder &builder, Location loc) {
131 return static_cast<SubClass *>(this)->genImplementation(retTypes, params,
132 builder, loc);
133 }
134
135private:
136 TypeRange retTypes; // The types of all returned results
137 ValueRange params; // The values of all input parameters
138 bool genCall; // Should the implemetantion be wrapped in a function
139};
140
141/// Add type casting between arith and index types when needed.
142Value genCast(OpBuilder &builder, Location loc, Value value, Type dstTy);
143
144/// Add conversion from scalar to given type (possibly a 0-rank tensor).
145Value genScalarToTensor(OpBuilder &builder, Location loc, Value elem,
146 Type dstTp);
147
148/// Generates a pointer/index load from the sparse storage scheme. Narrower
149/// data types need to be zero extended before casting the value into the
150/// index type used for looping and indexing.
151Value genIndexLoad(OpBuilder &builder, Location loc, Value mem, ValueRange s);
152
153/// Generates a 1-valued attribute of the given type. This supports
154/// all the same types as `getZeroAttr`; however, unlike `getZeroAttr`,
155/// for unsupported types we raise `llvm_unreachable` rather than
156/// returning a null attribute.
157TypedAttr getOneAttr(Builder &builder, Type tp);
158
159/// Generates the comparison `v != 0` where `v` is of numeric type.
160/// For floating types, we use the "unordered" comparator (i.e., returns
161/// true if `v` is NaN).
162Value genIsNonzero(OpBuilder &builder, Location loc, Value v);
163
164/// Computes the shape of destination tensor of a reshape operator. This is only
165/// used when operands have dynamic shape. The shape of the destination is
166/// stored into dstShape.
167void genReshapeDstShape(OpBuilder &builder, Location loc,
168 SmallVectorImpl<Value> &dstShape,
169 ArrayRef<Value> srcShape, ArrayRef<Size> staticDstShape,
170 ArrayRef<ReassociationIndices> reassociation);
171
172/// Reshape coordinates during a reshaping operation.
173void reshapeCvs(OpBuilder &builder, Location loc,
174 ArrayRef<ReassociationIndices> reassociation,
175 ValueRange srcSizes, ValueRange srcCvs, // NOLINT
176 ValueRange dstSizes, SmallVectorImpl<Value> &dstCvs);
177
178/// Returns a function reference (first hit also inserts into module). Sets
179/// the "_emit_c_interface" on the function declaration when requested,
180/// so that LLVM lowering generates a wrapper function that takes care
181/// of ABI complications with passing in and returning MemRefs to C functions.
182FlatSymbolRefAttr getFunc(ModuleOp module, StringRef name, TypeRange resultType,
183 ValueRange operands, EmitCInterface emitCInterface);
184
185/// Creates a `CallOp` to the function reference returned by `getFunc()` in
186/// the builder's module.
187func::CallOp createFuncCall(OpBuilder &builder, Location loc, StringRef name,
188 TypeRange resultType, ValueRange operands,
189 EmitCInterface emitCInterface);
190
191/// Returns the equivalent of `void*` for opaque arguments to the
192/// execution engine.
193Type getOpaquePointerType(MLIRContext *ctx);
194Type getOpaquePointerType(Builder &builder);
195
196/// Generates an uninitialized temporary buffer of the given size and
197/// type, but returns it as type `memref<? x $tp>` (rather than as type
198/// `memref<$sz x $tp>`).
199Value genAlloca(OpBuilder &builder, Location loc, Value sz, Type tp);
200
201/// Generates an uninitialized temporary buffer of the given size and
202/// type, and returns it as type `memref<? x $tp>` (staticShape=false) or
203/// `memref<$sz x $tp>` (staticShape=true).
204Value genAlloca(OpBuilder &builder, Location loc, unsigned sz, Type tp,
205 bool staticShape = false);
206
207/// Generates an uninitialized temporary buffer with room for one value
208/// of the given type, and returns the `memref<$tp>`.
209Value genAllocaScalar(OpBuilder &builder, Location loc, Type tp);
210
211/// Generates a temporary buffer, initializes it with the given contents,
212/// and returns it as type `memref<? x $tp>` (rather than specifying the
213/// size of the buffer).
214Value allocaBuffer(OpBuilder &builder, Location loc, ValueRange values);
215
216/// Generates code to allocate a buffer of the given type, and zero
217/// initialize it. If the buffer type has any dynamic sizes, then the
218/// `sizes` parameter should be as filled by sizesFromPtr(); that way
219/// we can reuse the genDimSizeCall() results generated by sizesFromPtr().
220Value allocDenseTensor(OpBuilder &builder, Location loc,
221 RankedTensorType tensorTp, ValueRange sizes);
222
223/// Generates code to deallocate a dense buffer.
224void deallocDenseTensor(OpBuilder &builder, Location loc, Value buffer);
225
226/// Populates given sizes array from dense tensor or sparse tensor constant.
227void sizesFromSrc(OpBuilder &builder, SmallVectorImpl<Value> &sizes,
228 Location loc, Value src);
229
230/// Scans to top of generated loop.
231Operation *getTop(Operation *op);
232
233/// Iterate over a sparse constant, generates constantOp for value
234/// and coordinates. E.g.,
235/// sparse<[ [0], [28], [31] ],
236/// [ (-5.13, 2.0), (3.0, 4.0), (5.0, 6.0) ] >
237/// =>
238/// %c1 = arith.constant 0
239/// %v1 = complex.constant (5.13, 2.0)
240/// callback({%c1}, %v1)
241///
242/// %c2 = arith.constant 28
243/// %v2 = complex.constant (3.0, 4.0)
244/// callback({%c2}, %v2)
245///
246/// %c3 = arith.constant 31
247/// %v3 = complex.constant (5.0, 6.0)
248/// callback({%c3}, %v3)
249void foreachInSparseConstant(
250 OpBuilder &builder, Location loc, SparseElementsAttr attr, AffineMap order,
251 function_ref<void(ArrayRef<Value>, Value)> callback);
252
253/// Loads `size`-many values from the memref, which must have rank-1 and
254/// size greater-or-equal to `size`. If the optional `(offsetIdx,offsetVal)`
255/// arguments are provided, then the `offsetVal` will be added to the
256/// `offsetIdx`-th value after loading.
257SmallVector<Value> loadAll(OpBuilder &builder, Location loc, size_t size,
258 Value mem, size_t offsetIdx = 0,
259 Value offsetVal = Value());
260
261/// Stores all the values of `vs` into the memref `mem`, which must have
262/// rank-1 and size greater-or-equal to `vs.size()`. If the optional
263/// `(offsetIdx,offsetVal)` arguments are provided, then the `offsetVal`
264/// will be added to the `offsetIdx`-th value before storing.
265void storeAll(OpBuilder &builder, Location loc, Value mem, ValueRange vs,
266 size_t offsetIdx = 0, Value offsetVal = Value());
267
268// Generates code to cast a tensor to a memref.
269TypedValue<BaseMemRefType> genToMemref(OpBuilder &builder, Location loc,
270 Value tensor);
271
272/// Generates code to retrieve the slice offset for the sparse tensor slice,
273/// return a constant if the offset is statically known.
274Value createOrFoldSliceOffsetOp(OpBuilder &builder, Location loc, Value tensor,
275 Dimension dim);
276
277/// Generates code to retrieve the slice slice for the sparse tensor slice,
278/// return a constant if the offset is statically known.
279Value createOrFoldSliceStrideOp(OpBuilder &builder, Location loc, Value tensor,
280 Dimension dim);
281
282/// Generates code that opens a reader and sets the dimension sizes.
283Value genReader(OpBuilder &builder, Location loc, SparseTensorType stt,
284 Value tensor,
285 /*out*/ SmallVectorImpl<Value> &dimSizesValues,
286 /*out*/ Value &dimSizesBuffer);
287
288/// Generates code to set up the buffer parameters for a map.
289Value genMapBuffers(OpBuilder &builder, Location loc, SparseTensorType stt,
290 ArrayRef<Value> dimSizesValues, Value dimSizesBuffer,
291 /*out*/ SmallVectorImpl<Value> &lvlSizesValues,
292 /*out*/ Value &dim2lvlBuffer,
293 /*out*/ Value &lvl2dimBuffer);
294
295//===----------------------------------------------------------------------===//
296// Inlined constant generators.
297//
298// All these functions are just wrappers to improve code legibility;
299// therefore, we mark them as `inline` to avoid introducing any additional
300// overhead due to the legibility. Ideally these should move upstream.
301//
302//===----------------------------------------------------------------------===//
303
304/// Generates a 0-valued constant of the given type. In addition to
305/// the scalar types (`ComplexType`, `FloatType`, `IndexType`,
306/// `IntegerType`), this also works for `RankedTensorType` and `VectorType`
307/// (for which it generates a constant `DenseElementsAttr` of zeros).
308inline Value constantZero(OpBuilder &builder, Location loc, Type tp) {
309 if (auto ctp = dyn_cast<ComplexType>(Val&: tp)) {
310 auto zeroe = builder.getZeroAttr(type: ctp.getElementType());
311 auto zeroa = builder.getArrayAttr(value: {zeroe, zeroe});
312 return complex::ConstantOp::create(builder, location: loc, complex: tp, value: zeroa);
313 }
314 return arith::ConstantOp::create(builder, location: loc, result: tp, value: builder.getZeroAttr(type: tp));
315}
316
317/// Generates a 1-valued constant of the given type. This supports all
318/// the same types as `constantZero`.
319inline Value constantOne(OpBuilder &builder, Location loc, Type tp) {
320 if (auto ctp = dyn_cast<ComplexType>(Val&: tp)) {
321 auto zeroe = builder.getZeroAttr(type: ctp.getElementType());
322 auto onee = getOneAttr(builder, tp: ctp.getElementType());
323 auto zeroa = builder.getArrayAttr(value: {onee, zeroe});
324 return complex::ConstantOp::create(builder, location: loc, complex: tp, value: zeroa);
325 }
326 return arith::ConstantOp::create(builder, location: loc, result: tp, value: getOneAttr(builder, tp));
327}
328
329/// Generates a constant of `index` type.
330inline Value constantIndex(OpBuilder &builder, Location loc, int64_t i) {
331 return arith::ConstantIndexOp::create(builder, location: loc, value: i);
332}
333
334/// Generates a constant of `i64` type.
335inline Value constantI64(OpBuilder &builder, Location loc, int64_t i) {
336 return arith::ConstantIntOp::create(builder, location: loc, value: i, width: 64);
337}
338
339/// Generates a constant of `i32` type.
340inline Value constantI32(OpBuilder &builder, Location loc, int32_t i) {
341 return arith::ConstantIntOp::create(builder, location: loc, value: i, width: 32);
342}
343
344/// Generates a constant of `i16` type.
345inline Value constantI16(OpBuilder &builder, Location loc, int16_t i) {
346 return arith::ConstantIntOp::create(builder, location: loc, value: i, width: 16);
347}
348
349/// Generates a constant of `i8` type.
350inline Value constantI8(OpBuilder &builder, Location loc, int8_t i) {
351 return arith::ConstantIntOp::create(builder, location: loc, value: i, width: 8);
352}
353
354/// Generates a constant of `i1` type.
355inline Value constantI1(OpBuilder &builder, Location loc, bool b) {
356 return arith::ConstantIntOp::create(builder, location: loc, value: b, width: 1);
357}
358
359/// Generates a constant of the given `Action`.
360inline Value constantAction(OpBuilder &builder, Location loc, Action action) {
361 return constantI32(builder, loc, i: static_cast<uint32_t>(action));
362}
363
364/// Generates a constant of the internal type-encoding for overhead storage.
365inline Value constantOverheadTypeEncoding(OpBuilder &builder, Location loc,
366 unsigned width) {
367 return constantI32(builder, loc,
368 i: static_cast<uint32_t>(overheadTypeEncoding(width)));
369}
370
371/// Generates a constant of the internal type-encoding for position
372/// overhead storage.
373inline Value constantPosTypeEncoding(OpBuilder &builder, Location loc,
374 SparseTensorEncodingAttr enc) {
375 return constantOverheadTypeEncoding(builder, loc, width: enc.getPosWidth());
376}
377
378/// Generates a constant of the internal type-encoding for coordinate
379/// overhead storage.
380inline Value constantCrdTypeEncoding(OpBuilder &builder, Location loc,
381 SparseTensorEncodingAttr enc) {
382 return constantOverheadTypeEncoding(builder, loc, width: enc.getCrdWidth());
383}
384
385/// Generates a constant of the internal type-encoding for primary storage.
386inline Value constantPrimaryTypeEncoding(OpBuilder &builder, Location loc,
387 Type elemTp) {
388 return constantI32(builder, loc,
389 i: static_cast<uint32_t>(primaryTypeEncoding(elemTp)));
390}
391
392/// Generates a constant of the internal dimension level type encoding.
393inline Value constantLevelTypeEncoding(OpBuilder &builder, Location loc,
394 LevelType lt) {
395 return constantI64(builder, loc, i: static_cast<uint64_t>(lt));
396}
397
398// Generates a constant from a validated value carrying attribute.
399inline Value genValFromAttr(OpBuilder &builder, Location loc, Attribute attr) {
400 if (auto complexAttr = dyn_cast<complex::NumberAttr>(Val&: attr)) {
401 Type tp = cast<ComplexType>(Val: complexAttr.getType()).getElementType();
402 return complex::ConstantOp::create(
403 builder, location: loc, complex: complexAttr.getType(),
404 value: builder.getArrayAttr(value: {FloatAttr::get(type: tp, value: complexAttr.getReal()),
405 FloatAttr::get(type: tp, value: complexAttr.getImag())}));
406 }
407 return arith::ConstantOp::create(builder, location: loc, value: cast<TypedAttr>(Val&: attr));
408}
409
410// TODO: is this at the right place?
411inline bool isZeroRankedTensorOrScalar(Type type) {
412 auto rtp = dyn_cast<RankedTensorType>(Val&: type);
413 return !rtp || rtp.getRank() == 0;
414}
415
416} // namespace sparse_tensor
417} // namespace mlir
418
419#endif // MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_CODEGENUTILS_H_
420