1//===- TypeConverter.cpp - Convert builtin to LLVM dialect types ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
10#include "mlir/Conversion/LLVMCommon/MemRefBuilder.h"
11#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
12#include "mlir/Dialect/LLVMIR/LLVMDialectDecl.h"
13#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
14#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
15#include "mlir/Interfaces/FunctionInterfaces.h"
16#include "llvm/ADT/ScopeExit.h"
17#include "llvm/Support/Threading.h"
18#include <memory>
19#include <mutex>
20#include <optional>
21
22using namespace mlir;
23
24SmallVector<Type> &LLVMTypeConverter::getCurrentThreadRecursiveStack() {
25 {
26 // Most of the time, the entry already exists in the map.
27 std::shared_lock<decltype(callStackMutex)> lock(callStackMutex,
28 std::defer_lock);
29 if (getContext().isMultithreadingEnabled())
30 lock.lock();
31 auto recursiveStack = conversionCallStack.find(Val: llvm::get_threadid());
32 if (recursiveStack != conversionCallStack.end())
33 return *recursiveStack->second;
34 }
35
36 // First time this thread gets here, we have to get an exclusive access to
37 // inset in the map
38 std::unique_lock<decltype(callStackMutex)> lock(callStackMutex);
39 auto recursiveStackInserted = conversionCallStack.insert(KV: std::make_pair(
40 x: llvm::get_threadid(), y: std::make_unique<SmallVector<Type>>()));
41 return *recursiveStackInserted.first->second;
42}
43
44/// Create an LLVMTypeConverter using default LowerToLLVMOptions.
45LLVMTypeConverter::LLVMTypeConverter(MLIRContext *ctx,
46 const DataLayoutAnalysis *analysis)
47 : LLVMTypeConverter(ctx, LowerToLLVMOptions(ctx), analysis) {}
48
49/// Helper function that checks if the given value range is a bare pointer.
50static bool isBarePointer(ValueRange values) {
51 return values.size() == 1 &&
52 isa<LLVM::LLVMPointerType>(Val: values.front().getType());
53}
54
55/// Pack SSA values into an unranked memref descriptor struct.
56static Value packUnrankedMemRefDesc(OpBuilder &builder,
57 UnrankedMemRefType resultType,
58 ValueRange inputs, Location loc,
59 const LLVMTypeConverter &converter) {
60 // Note: Bare pointers are not supported for unranked memrefs because a
61 // memref descriptor cannot be built just from a bare pointer.
62 if (TypeRange(inputs) != converter.getUnrankedMemRefDescriptorFields())
63 return Value();
64 return UnrankedMemRefDescriptor::pack(builder, loc, converter, type: resultType,
65 values: inputs);
66}
67
68/// Pack SSA values into a ranked memref descriptor struct.
69static Value packRankedMemRefDesc(OpBuilder &builder, MemRefType resultType,
70 ValueRange inputs, Location loc,
71 const LLVMTypeConverter &converter) {
72 assert(resultType && "expected non-null result type");
73 if (isBarePointer(values: inputs))
74 return MemRefDescriptor::fromStaticShape(builder, loc, typeConverter: converter,
75 type: resultType, memory: inputs[0]);
76 if (TypeRange(inputs) ==
77 converter.getMemRefDescriptorFields(type: resultType,
78 /*unpackAggregates=*/true))
79 return MemRefDescriptor::pack(builder, loc, converter, type: resultType, values: inputs);
80 // The inputs are neither a bare pointer nor an unpacked memref descriptor.
81 // This materialization function cannot be used.
82 return Value();
83}
84
85/// MemRef descriptor elements -> UnrankedMemRefType
86static Value unrankedMemRefMaterialization(OpBuilder &builder,
87 UnrankedMemRefType resultType,
88 ValueRange inputs, Location loc,
89 const LLVMTypeConverter &converter) {
90 // A source materialization must return a value of type
91 // `resultType`, so insert a cast from the memref descriptor type
92 // (!llvm.struct) to the original memref type.
93 Value packed =
94 packUnrankedMemRefDesc(builder, resultType, inputs, loc, converter);
95 if (!packed)
96 return Value();
97 return UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: resultType, operands: packed)
98 .getResult(i: 0);
99}
100
101/// MemRef descriptor elements -> MemRefType
102static Value rankedMemRefMaterialization(OpBuilder &builder,
103 MemRefType resultType,
104 ValueRange inputs, Location loc,
105 const LLVMTypeConverter &converter) {
106 // A source materialization must return a value of type `resultType`,
107 // so insert a cast from the memref descriptor type (!llvm.struct) to the
108 // original memref type.
109 Value packed =
110 packRankedMemRefDesc(builder, resultType, inputs, loc, converter);
111 if (!packed)
112 return Value();
113 return UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: resultType, operands: packed)
114 .getResult(i: 0);
115}
116
117/// Create an LLVMTypeConverter using custom LowerToLLVMOptions.
118LLVMTypeConverter::LLVMTypeConverter(MLIRContext *ctx,
119 const LowerToLLVMOptions &options,
120 const DataLayoutAnalysis *analysis)
121 : llvmDialect(ctx->getOrLoadDialect<LLVM::LLVMDialect>()), options(options),
122 dataLayoutAnalysis(analysis) {
123 assert(llvmDialect && "LLVM IR dialect is not registered");
124
125 // Register conversions for the builtin types.
126 addConversion(callback: [&](ComplexType type) { return convertComplexType(type); });
127 addConversion(callback: [&](FloatType type) { return convertFloatType(type); });
128 addConversion(callback: [&](FunctionType type) { return convertFunctionType(type); });
129 addConversion(callback: [&](IndexType type) { return convertIndexType(type); });
130 addConversion(callback: [&](IntegerType type) { return convertIntegerType(type); });
131 addConversion(callback: [&](MemRefType type) { return convertMemRefType(type); });
132 addConversion(
133 callback: [&](UnrankedMemRefType type) { return convertUnrankedMemRefType(type); });
134 addConversion(callback: [&](VectorType type) -> std::optional<Type> {
135 FailureOr<Type> llvmType = convertVectorType(type);
136 if (failed(Result: llvmType))
137 return std::nullopt;
138 return llvmType;
139 });
140
141 // LLVM-compatible types are legal, so add a pass-through conversion. Do this
142 // before the conversions below since conversions are attempted in reverse
143 // order and those should take priority.
144 addConversion(callback: [](Type type) {
145 return LLVM::isCompatibleType(type) ? std::optional<Type>(type)
146 : std::nullopt;
147 });
148
149 addConversion(callback: [&](LLVM::LLVMStructType type, SmallVectorImpl<Type> &results)
150 -> std::optional<LogicalResult> {
151 // Fastpath for types that won't be converted by this callback anyway.
152 if (LLVM::isCompatibleType(type)) {
153 results.push_back(Elt: type);
154 return success();
155 }
156
157 if (type.isIdentified()) {
158 auto convertedType = LLVM::LLVMStructType::getIdentified(
159 context: type.getContext(), name: ("_Converted." + type.getName()).str());
160
161 SmallVectorImpl<Type> &recursiveStack = getCurrentThreadRecursiveStack();
162 if (llvm::count(Range&: recursiveStack, Element: type)) {
163 results.push_back(Elt: convertedType);
164 return success();
165 }
166 recursiveStack.push_back(Elt: type);
167 llvm::scope_exit popConversionCallStack(
168 [&recursiveStack]() { recursiveStack.pop_back(); });
169
170 SmallVector<Type> convertedElemTypes;
171 convertedElemTypes.reserve(N: type.getBody().size());
172 if (failed(Result: convertTypes(types: type.getBody(), results&: convertedElemTypes)))
173 return std::nullopt;
174
175 // If the converted type has not been initialized yet, just set its body
176 // to be the converted arguments and return.
177 if (!convertedType.isInitialized()) {
178 if (failed(
179 Result: convertedType.setBody(types: convertedElemTypes, isPacked: type.isPacked()))) {
180 return failure();
181 }
182 results.push_back(Elt: convertedType);
183 return success();
184 }
185
186 // If it has been initialized, has the same body and packed bit, just use
187 // it. This ensures that recursive structs keep being recursive rather
188 // than including a non-updated name.
189 if (TypeRange(convertedType.getBody()) == TypeRange(convertedElemTypes) &&
190 convertedType.isPacked() == type.isPacked()) {
191 results.push_back(Elt: convertedType);
192 return success();
193 }
194
195 return failure();
196 }
197
198 SmallVector<Type> convertedSubtypes;
199 convertedSubtypes.reserve(N: type.getBody().size());
200 if (failed(Result: convertTypes(types: type.getBody(), results&: convertedSubtypes)))
201 return std::nullopt;
202
203 results.push_back(Elt: LLVM::LLVMStructType::getLiteral(
204 context: type.getContext(), types: convertedSubtypes, isPacked: type.isPacked()));
205 return success();
206 });
207 addConversion(callback: [&](LLVM::LLVMArrayType type) -> std::optional<Type> {
208 if (auto element = convertType(t: type.getElementType()))
209 return LLVM::LLVMArrayType::get(elementType: element, numElements: type.getNumElements());
210 return std::nullopt;
211 });
212 addConversion(callback: [&](LLVM::LLVMFunctionType type) -> std::optional<Type> {
213 Type convertedResType = convertType(t: type.getReturnType());
214 if (!convertedResType)
215 return std::nullopt;
216
217 SmallVector<Type> convertedArgTypes;
218 convertedArgTypes.reserve(N: type.getNumParams());
219 if (failed(Result: convertTypes(types: type.getParams(), results&: convertedArgTypes)))
220 return std::nullopt;
221
222 return LLVM::LLVMFunctionType::get(result: convertedResType, arguments: convertedArgTypes,
223 isVarArg: type.isVarArg());
224 });
225
226 // Add generic source and target materializations to handle cases where
227 // non-LLVM types persist after an LLVM conversion.
228 addSourceMaterialization(callback: [&](OpBuilder &builder, Type resultType,
229 ValueRange inputs, Location loc) {
230 return UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: resultType, operands: inputs)
231 .getResult(i: 0);
232 });
233 addTargetMaterialization(callback: [&](OpBuilder &builder, Type resultType,
234 ValueRange inputs, Location loc) {
235 return UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: resultType, operands: inputs)
236 .getResult(i: 0);
237 });
238
239 // Source materializations convert from the new block argument types
240 // (multiple SSA values that make up a memref descriptor) back to the
241 // original block argument type.
242 addSourceMaterialization(callback: [&](OpBuilder &builder,
243 UnrankedMemRefType resultType, ValueRange inputs,
244 Location loc) {
245 return unrankedMemRefMaterialization(builder, resultType, inputs, loc,
246 converter: *this);
247 });
248 addSourceMaterialization(callback: [&](OpBuilder &builder, MemRefType resultType,
249 ValueRange inputs, Location loc) {
250 return rankedMemRefMaterialization(builder, resultType, inputs, loc, converter: *this);
251 });
252
253 // Bare pointer -> Packed MemRef descriptor
254 addTargetMaterialization(callback: [&](OpBuilder &builder, Type resultType,
255 ValueRange inputs, Location loc,
256 Type originalType) -> Value {
257 // The original MemRef type is required to build a MemRef descriptor
258 // because the sizes/strides of the MemRef cannot be inferred from just the
259 // bare pointer.
260 if (!originalType)
261 return Value();
262 if (resultType != convertType(t: originalType))
263 return Value();
264 if (auto memrefType = dyn_cast<MemRefType>(Val&: originalType))
265 return packRankedMemRefDesc(builder, resultType: memrefType, inputs, loc, converter: *this);
266 if (auto unrankedMemrefType = dyn_cast<UnrankedMemRefType>(Val&: originalType))
267 return packUnrankedMemRefDesc(builder, resultType: unrankedMemrefType, inputs, loc,
268 converter: *this);
269 return Value();
270 });
271
272 // Integer memory spaces map to themselves.
273 addTypeAttributeConversion(
274 callback: [](BaseMemRefType memref, IntegerAttr addrspace) { return addrspace; });
275}
276
277/// Returns the MLIR context.
278MLIRContext &LLVMTypeConverter::getContext() const {
279 return *getDialect()->getContext();
280}
281
282Type LLVMTypeConverter::getIndexType() const {
283 return IntegerType::get(context: &getContext(), width: getIndexTypeBitwidth());
284}
285
286unsigned LLVMTypeConverter::getPointerBitwidth(unsigned addressSpace) const {
287 return options.dataLayout.getPointerSizeInBits(AS: addressSpace);
288}
289
290Type LLVMTypeConverter::convertIndexType(IndexType type) const {
291 return getIndexType();
292}
293
294Type LLVMTypeConverter::convertIntegerType(IntegerType type) const {
295 return IntegerType::get(context: &getContext(), width: type.getWidth());
296}
297
298Type LLVMTypeConverter::convertFloatType(FloatType type) const {
299 // Valid LLVM float types are used directly.
300 if (LLVM::isCompatibleType(type))
301 return type;
302
303 // F4, F6, F8 types are converted to integer types with the same bit width.
304 if (isa<Float8E5M2Type, Float8E4M3Type, Float8E4M3FNType, Float8E5M2FNUZType,
305 Float8E4M3FNUZType, Float8E4M3B11FNUZType, Float8E3M4Type,
306 Float4E2M1FNType, Float6E2M3FNType, Float6E3M2FNType,
307 Float8E8M0FNUType, Float8E5M3FNUType>(Val: type))
308 return IntegerType::get(context: &getContext(), width: type.getWidth());
309
310 // Other floating-point types: A custom type conversion rule must be
311 // specified by the user.
312 return Type();
313}
314
315// Convert a `ComplexType` to an LLVM type. The result is a complex number
316// struct with entries for the
317// 1. real part and for the
318// 2. imaginary part.
319Type LLVMTypeConverter::convertComplexType(ComplexType type) const {
320 auto elementType = convertType(t: type.getElementType());
321 return LLVM::LLVMStructType::getLiteral(context: &getContext(),
322 types: {elementType, elementType});
323}
324
325// Except for signatures, MLIR function types are converted into LLVM
326// pointer-to-function types.
327Type LLVMTypeConverter::convertFunctionType(FunctionType type) const {
328 return LLVM::LLVMPointerType::get(context: type.getContext());
329}
330
331/// Returns the `llvm.byval` or `llvm.byref` attributes that are present in the
332/// function arguments. Returns an empty container if none of these attributes
333/// are found in any of the arguments.
334static void
335filterByValRefArgAttrs(FunctionOpInterface funcOp,
336 SmallVectorImpl<std::optional<NamedAttribute>> &result) {
337 assert(result.empty() && "Unexpected non-empty output");
338 result.resize(N: funcOp.getNumArguments(), NV: std::nullopt);
339 bool foundByValByRefAttrs = false;
340 for (int argIdx : llvm::seq(Size: funcOp.getNumArguments())) {
341 for (NamedAttribute namedAttr : funcOp.getArgAttrs(index: argIdx)) {
342 if ((namedAttr.getName() == LLVM::LLVMDialect::getByValAttrName() ||
343 namedAttr.getName() == LLVM::LLVMDialect::getByRefAttrName())) {
344 foundByValByRefAttrs = true;
345 result[argIdx] = namedAttr;
346 break;
347 }
348 }
349 }
350
351 if (!foundByValByRefAttrs)
352 result.clear();
353}
354
355// Function types are converted to LLVM Function types by recursively converting
356// argument and result types. If MLIR Function has zero results, the LLVM
357// Function has one VoidType result. If MLIR Function has more than one result,
358// they are into an LLVM StructType in their order of appearance.
359// If `byValRefNonPtrAttrs` is provided, converted types of `llvm.byval` and
360// `llvm.byref` function arguments which are not LLVM pointers are overridden
361// with LLVM pointers. `llvm.byval` and `llvm.byref` arguments that were already
362// converted to LLVM pointer types are removed from 'byValRefNonPtrAttrs`.
363Type LLVMTypeConverter::convertFunctionSignatureImpl(
364 FunctionType funcTy, bool isVariadic, bool useBarePtrCallConv,
365 LLVMTypeConverter::SignatureConversion &result,
366 SmallVectorImpl<std::optional<NamedAttribute>> *byValRefNonPtrAttrs) const {
367 // Select the argument converter depending on the calling convention.
368 useBarePtrCallConv = useBarePtrCallConv || options.useBarePtrCallConv;
369 auto funcArgConverter = useBarePtrCallConv ? barePtrFuncArgTypeConverter
370 : structFuncArgTypeConverter;
371
372 // Convert argument types one by one and check for errors.
373 for (auto [idx, type] : llvm::enumerate(First: funcTy.getInputs())) {
374 SmallVector<Type, 8> converted;
375 if (failed(Result: funcArgConverter(*this, type, converted)))
376 return {};
377
378 // Rewrite converted type of `llvm.byval` or `llvm.byref` function
379 // argument that was not converted to an LLVM pointer types.
380 if (byValRefNonPtrAttrs != nullptr && !byValRefNonPtrAttrs->empty() &&
381 converted.size() == 1 && (*byValRefNonPtrAttrs)[idx].has_value()) {
382 // If the argument was already converted to an LLVM pointer type, we stop
383 // tracking it as it doesn't need more processing.
384 if (isa<LLVM::LLVMPointerType>(Val: converted[0]))
385 (*byValRefNonPtrAttrs)[idx] = std::nullopt;
386 else
387 converted[0] = LLVM::LLVMPointerType::get(context: &getContext());
388 }
389
390 result.addInputs(origInputNo: idx, types: converted);
391 }
392
393 // If function does not return anything, create the void result type,
394 // if it returns on element, convert it, otherwise pack the result types into
395 // a struct.
396 Type resultType =
397 funcTy.getNumResults() == 0
398 ? LLVM::LLVMVoidType::get(ctx: &getContext())
399 : packFunctionResults(types: funcTy.getResults(), useBarePointerCallConv: useBarePtrCallConv);
400 if (!resultType)
401 return {};
402 return LLVM::LLVMFunctionType::get(result: resultType, arguments: result.getConvertedTypes(),
403 isVarArg: isVariadic);
404}
405
406Type LLVMTypeConverter::convertFunctionSignature(
407 FunctionType funcTy, bool isVariadic, bool useBarePtrCallConv,
408 LLVMTypeConverter::SignatureConversion &result) const {
409 return convertFunctionSignatureImpl(funcTy, isVariadic, useBarePtrCallConv,
410 result,
411 /*byValRefNonPtrAttrs=*/nullptr);
412}
413
414Type LLVMTypeConverter::convertFunctionSignature(
415 FunctionOpInterface funcOp, bool isVariadic, bool useBarePtrCallConv,
416 LLVMTypeConverter::SignatureConversion &result,
417 SmallVectorImpl<std::optional<NamedAttribute>> &byValRefNonPtrAttrs) const {
418 // Gather all `llvm.byval` and `llvm.byref` function arguments. Only those
419 // that were not converted to LLVM pointer types will be returned for further
420 // processing.
421 filterByValRefArgAttrs(funcOp, result&: byValRefNonPtrAttrs);
422 auto funcTy = cast<FunctionType>(Val: funcOp.getFunctionType());
423 return convertFunctionSignatureImpl(funcTy, isVariadic, useBarePtrCallConv,
424 result, byValRefNonPtrAttrs: &byValRefNonPtrAttrs);
425}
426
427/// Converts the function type to a C-compatible format, in particular using
428/// pointers to memref descriptors for arguments.
429std::pair<LLVM::LLVMFunctionType, LLVM::LLVMStructType>
430LLVMTypeConverter::convertFunctionTypeCWrapper(FunctionType type) const {
431 SmallVector<Type, 4> inputs;
432
433 Type resultType = type.getNumResults() == 0
434 ? LLVM::LLVMVoidType::get(ctx: &getContext())
435 : packFunctionResults(types: type.getResults());
436 if (!resultType)
437 return {};
438
439 auto ptrType = LLVM::LLVMPointerType::get(context: type.getContext());
440 auto structType = dyn_cast<LLVM::LLVMStructType>(Val&: resultType);
441 if (structType) {
442 // Struct types cannot be safely returned via C interface. Make this a
443 // pointer argument, instead.
444 inputs.push_back(Elt: ptrType);
445 resultType = LLVM::LLVMVoidType::get(ctx: &getContext());
446 }
447
448 for (Type t : type.getInputs()) {
449 auto converted = convertType(t);
450 if (!converted || !LLVM::isCompatibleType(type: converted))
451 return {};
452 if (isa<MemRefType, UnrankedMemRefType>(Val: t))
453 converted = ptrType;
454 inputs.push_back(Elt: converted);
455 }
456
457 return {LLVM::LLVMFunctionType::get(result: resultType, arguments: inputs), structType};
458}
459
460/// Convert a memref type into a list of LLVM IR types that will form the
461/// memref descriptor. The result contains the following types:
462/// 1. The pointer to the allocated data buffer, followed by
463/// 2. The pointer to the aligned data buffer, followed by
464/// 3. A lowered `index`-type integer containing the distance between the
465/// beginning of the buffer and the first element to be accessed through the
466/// view, followed by
467/// 4. An array containing as many `index`-type integers as the rank of the
468/// MemRef: the array represents the size, in number of elements, of the memref
469/// along the given dimension. For constant MemRef dimensions, the
470/// corresponding size entry is a constant whose runtime value must match the
471/// static value, followed by
472/// 5. A second array containing as many `index`-type integers as the rank of
473/// the MemRef: the second array represents the "stride" (in tensor abstraction
474/// sense), i.e. the number of consecutive elements of the underlying buffer.
475/// TODO: add assertions for the static cases.
476///
477/// If `unpackAggregates` is set to true, the arrays described in (4) and (5)
478/// are expanded into individual index-type elements.
479///
480/// template <typename Elem, typename Index, size_t Rank>
481/// struct {
482/// Elem *allocatedPtr;
483/// Elem *alignedPtr;
484/// Index offset;
485/// Index sizes[Rank]; // omitted when rank == 0
486/// Index strides[Rank]; // omitted when rank == 0
487/// };
488SmallVector<Type, 5>
489LLVMTypeConverter::getMemRefDescriptorFields(MemRefType type,
490 bool unpackAggregates) const {
491 if (!type.isStrided()) {
492 emitError(
493 loc: UnknownLoc::get(context: type.getContext()),
494 message: "conversion to strided form failed either due to non-strided layout "
495 "maps (which should have been normalized away) or other reasons");
496 return {};
497 }
498
499 Type elementType = convertType(t: type.getElementType());
500 if (!elementType)
501 return {};
502
503 FailureOr<unsigned> addressSpace = getMemRefAddressSpace(type);
504 if (failed(Result: addressSpace)) {
505 emitError(loc: UnknownLoc::get(context: type.getContext()),
506 message: "conversion of memref memory space ")
507 << type.getMemorySpace()
508 << " to integer address space "
509 "failed. Consider adding memory space conversions.";
510 return {};
511 }
512 auto ptrTy = LLVM::LLVMPointerType::get(context: type.getContext(), addressSpace: *addressSpace);
513
514 auto indexTy = getIndexType();
515
516 SmallVector<Type, 5> results = {ptrTy, ptrTy, indexTy};
517 auto rank = type.getRank();
518 if (rank == 0)
519 return results;
520
521 if (unpackAggregates)
522 results.insert(I: results.end(), NumToInsert: 2 * rank, Elt: indexTy);
523 else
524 results.insert(I: results.end(), NumToInsert: 2, Elt: LLVM::LLVMArrayType::get(elementType: indexTy, numElements: rank));
525 return results;
526}
527
528unsigned
529LLVMTypeConverter::getMemRefDescriptorSize(MemRefType type,
530 const DataLayout &layout) const {
531 // Compute the descriptor size given that of its components indicated above.
532 unsigned space = *getMemRefAddressSpace(type);
533 return 2 * llvm::divideCeil(Numerator: getPointerBitwidth(addressSpace: space), Denominator: 8) +
534 (1 + 2 * type.getRank()) * layout.getTypeSize(t: getIndexType());
535}
536
537/// Converts MemRefType to LLVMType. A MemRefType is converted to a struct that
538/// packs the descriptor fields as defined by `getMemRefDescriptorFields`.
539Type LLVMTypeConverter::convertMemRefType(MemRefType type) const {
540 // When converting a MemRefType to a struct with descriptor fields, do not
541 // unpack the `sizes` and `strides` arrays.
542 SmallVector<Type, 5> types =
543 getMemRefDescriptorFields(type, /*unpackAggregates=*/false);
544 if (types.empty())
545 return {};
546 return LLVM::LLVMStructType::getLiteral(context: &getContext(), types);
547}
548
549/// Convert an unranked memref type into a list of non-aggregate LLVM IR types
550/// that will form the unranked memref descriptor. In particular, the fields
551/// for an unranked memref descriptor are:
552/// 1. index-typed rank, the dynamic rank of this MemRef
553/// 2. void* ptr, pointer to the static ranked MemRef descriptor. This will be
554/// stack allocated (alloca) copy of a MemRef descriptor that got casted to
555/// be unranked.
556SmallVector<Type, 2>
557LLVMTypeConverter::getUnrankedMemRefDescriptorFields() const {
558 return {getIndexType(), LLVM::LLVMPointerType::get(context: &getContext())};
559}
560
561unsigned LLVMTypeConverter::getUnrankedMemRefDescriptorSize(
562 UnrankedMemRefType type, const DataLayout &layout) const {
563 // Compute the descriptor size given that of its components indicated above.
564 unsigned space = *getMemRefAddressSpace(type);
565 return layout.getTypeSize(t: getIndexType()) +
566 llvm::divideCeil(Numerator: getPointerBitwidth(addressSpace: space), Denominator: 8);
567}
568
569Type LLVMTypeConverter::convertUnrankedMemRefType(
570 UnrankedMemRefType type) const {
571 if (!convertType(t: type.getElementType()))
572 return {};
573 return LLVM::LLVMStructType::getLiteral(context: &getContext(),
574 types: getUnrankedMemRefDescriptorFields());
575}
576
577FailureOr<unsigned>
578LLVMTypeConverter::getMemRefAddressSpace(BaseMemRefType type) const {
579 if (!type.getMemorySpace()) // Default memory space -> 0.
580 return 0;
581 std::optional<Attribute> converted =
582 convertTypeAttribute(type, attr: type.getMemorySpace());
583 if (!converted)
584 return failure();
585 if (!(*converted)) // Conversion to default is 0.
586 return 0;
587 if (auto explicitSpace = dyn_cast_if_present<IntegerAttr>(Val&: *converted)) {
588 if (explicitSpace.getType().isIndex() ||
589 explicitSpace.getType().isSignlessInteger())
590 return explicitSpace.getInt();
591 }
592 return failure();
593}
594
595// Check if a memref type can be converted to a bare pointer.
596bool LLVMTypeConverter::canConvertToBarePtr(BaseMemRefType type) {
597 if (isa<UnrankedMemRefType>(Val: type))
598 // Unranked memref is not supported in the bare pointer calling convention.
599 return false;
600
601 // Check that the memref has static shape, strides and offset. Otherwise, it
602 // cannot be lowered to a bare pointer.
603 auto memrefTy = cast<MemRefType>(Val&: type);
604 if (!memrefTy.hasStaticShape())
605 return false;
606
607 int64_t offset = 0;
608 SmallVector<int64_t, 4> strides;
609 if (failed(Result: memrefTy.getStridesAndOffset(strides, offset)))
610 return false;
611
612 for (int64_t stride : strides)
613 if (ShapedType::isDynamic(dValue: stride))
614 return false;
615
616 return ShapedType::isStatic(dValue: offset);
617}
618
619/// Convert a memref type to a bare pointer to the memref element type.
620Type LLVMTypeConverter::convertMemRefToBarePtr(BaseMemRefType type) const {
621 if (!canConvertToBarePtr(type))
622 return {};
623 Type elementType = convertType(t: type.getElementType());
624 if (!elementType)
625 return {};
626 FailureOr<unsigned> addressSpace = getMemRefAddressSpace(type);
627 if (failed(Result: addressSpace))
628 return {};
629 return LLVM::LLVMPointerType::get(context: type.getContext(), addressSpace: *addressSpace);
630}
631
632/// Convert an n-D vector type to an LLVM vector type:
633/// * 0-D `vector<T>` are converted to vector<1xT>
634/// * 1-D `vector<axT>` remains as is while,
635/// * n>1 `vector<ax...xkxT>` convert via an (n-1)-D array type to
636/// `!llvm.array<ax...array<jxvector<kxT>>>`.
637/// As LLVM supports arrays of scalable vectors, this method will also convert
638/// n-D scalable vectors provided that only the trailing dim is scalable.
639FailureOr<Type> LLVMTypeConverter::convertVectorType(VectorType type) const {
640 auto elementType = convertType(t: type.getElementType());
641 if (!elementType)
642 return {};
643 if (type.getShape().empty())
644 return VectorType::get(shape: {1}, elementType);
645 Type vectorType = VectorType::get(shape: type.getShape().back(), elementType,
646 scalableDims: type.getScalableDims().back());
647 assert(LLVM::isCompatibleVectorType(vectorType) &&
648 "expected vector type compatible with the LLVM dialect");
649 // For n-D vector types for which a _non-trailing_ dim is scalable,
650 // return a failure. Supporting such cases would require LLVM
651 // to support something akin "scalable arrays" of vectors.
652 if (llvm::is_contained(Range: type.getScalableDims().drop_back(), Element: true))
653 return failure();
654 auto shape = type.getShape();
655 for (int i = shape.size() - 2; i >= 0; --i)
656 vectorType = LLVM::LLVMArrayType::get(elementType: vectorType, numElements: shape[i]);
657 return vectorType;
658}
659
660/// Convert a type in the context of the default or bare pointer calling
661/// convention. Calling convention sensitive types, such as MemRefType and
662/// UnrankedMemRefType, are converted following the specific rules for the
663/// calling convention. Calling convention independent types are converted
664/// following the default LLVM type conversions.
665LogicalResult LLVMTypeConverter::convertCallingConventionType(
666 Type type, SmallVectorImpl<Type> &result, bool useBarePtrCallConv) const {
667 if (useBarePtrCallConv) {
668 if (auto memrefTy = dyn_cast<BaseMemRefType>(Val&: type)) {
669 Type converted = convertMemRefToBarePtr(type: memrefTy);
670 if (!converted)
671 return failure();
672 result.push_back(Elt: converted);
673 return success();
674 }
675 }
676
677 return convertType(t: type, results&: result);
678}
679
680/// Convert a non-empty list of types of values produced by an operation into an
681/// LLVM-compatible type. In particular, if more than one value is
682/// produced, create a literal structure with elements that correspond to each
683/// of the types converted with `convertType`.
684Type LLVMTypeConverter::packOperationResults(TypeRange types) const {
685 assert(!types.empty() && "expected non-empty list of type");
686 if (types.size() == 1)
687 return convertType(t: types[0]);
688
689 SmallVector<Type> resultTypes;
690 resultTypes.reserve(N: types.size());
691 for (Type type : types) {
692 Type converted = convertType(t: type);
693 if (!converted || !LLVM::isCompatibleType(type: converted))
694 return {};
695 resultTypes.push_back(Elt: converted);
696 }
697
698 return LLVM::LLVMStructType::getLiteral(context: &getContext(), types: resultTypes);
699}
700
701/// Convert a non-empty list of types to be returned from a function into an
702/// LLVM-compatible type. In particular, if more than one value is returned,
703/// create an LLVM dialect structure type with elements that correspond to each
704/// of the types converted with `convertCallingConventionType`.
705Type LLVMTypeConverter::packFunctionResults(
706 TypeRange types, bool useBarePtrCallConv,
707 SmallVector<SmallVector<Type>> *groupedTypes,
708 int64_t *numConvertedTypes) const {
709 assert(!types.empty() && "expected non-empty list of type");
710 assert((!groupedTypes || groupedTypes->empty()) &&
711 "expected groupedTypes to be empty");
712
713 useBarePtrCallConv |= options.useBarePtrCallConv;
714 SmallVector<Type> resultTypes;
715 resultTypes.reserve(N: types.size());
716 size_t sizeBefore = 0;
717 for (auto t : types) {
718 if (failed(
719 Result: convertCallingConventionType(type: t, result&: resultTypes, useBarePtrCallConv)))
720 return {};
721 if (groupedTypes) {
722 SmallVector<Type> &group = groupedTypes->emplace_back();
723 llvm::append_range(C&: group, R: ArrayRef(resultTypes).drop_front(N: sizeBefore));
724 }
725 sizeBefore = resultTypes.size();
726 }
727
728 if (numConvertedTypes)
729 *numConvertedTypes = resultTypes.size();
730 if (resultTypes.size() == 1)
731 return resultTypes.front();
732 if (resultTypes.empty())
733 return {};
734 return LLVM::LLVMStructType::getLiteral(context: &getContext(), types: resultTypes);
735}
736
737Value LLVMTypeConverter::promoteOneMemRefDescriptor(Location loc, Value operand,
738 OpBuilder &builder) const {
739 // Alloca with proper alignment. We do not expect optimizations of this
740 // alloca op and so we omit allocating at the entry block.
741 auto ptrType = LLVM::LLVMPointerType::get(context: builder.getContext());
742 Value one = LLVM::ConstantOp::create(builder, location: loc, res: builder.getI64Type(),
743 value: builder.getI64IntegerAttr(value: 1));
744 Value allocated =
745 LLVM::AllocaOp::create(builder, location: loc, resultType: ptrType, elementType: operand.getType(), arraySize: one);
746 // Store into the alloca'ed descriptor.
747 LLVM::StoreOp::create(builder, location: loc, value: operand, addr: allocated);
748 return allocated;
749}
750
751SmallVector<Value, 4> LLVMTypeConverter::promoteOperands(
752 Location loc, ValueRange opOperands, ValueRange adaptorOperands,
753 OpBuilder &builder, bool useBarePtrCallConv) const {
754 SmallVector<ValueRange> ranges;
755 for (size_t i = 0, e = adaptorOperands.size(); i < e; i++)
756 ranges.push_back(Elt: adaptorOperands.slice(n: i, m: 1));
757 return promoteOperands(loc, opOperands, adaptorOperands: ranges, builder, useBarePtrCallConv);
758}
759
760SmallVector<Value, 4> LLVMTypeConverter::promoteOperands(
761 Location loc, ValueRange opOperands, ArrayRef<ValueRange> adaptorOperands,
762 OpBuilder &builder, bool useBarePtrCallConv) const {
763 SmallVector<Value, 4> promotedOperands;
764 promotedOperands.reserve(N: adaptorOperands.size());
765 useBarePtrCallConv |= options.useBarePtrCallConv;
766 for (auto [operand, llvmOperand] :
767 llvm::zip_equal(t&: opOperands, u&: adaptorOperands)) {
768 if (useBarePtrCallConv) {
769 // For the bare-ptr calling convention, we only have to extract the
770 // aligned pointer of a memref.
771 if (isa<MemRefType>(Val: operand.getType())) {
772 assert(llvmOperand.size() == 1 && "Expected a single operand");
773 MemRefDescriptor desc(llvmOperand.front());
774 promotedOperands.push_back(Elt: desc.alignedPtr(builder, loc));
775 continue;
776 }
777 if (isa<UnrankedMemRefType>(Val: operand.getType())) {
778 llvm_unreachable("Unranked memrefs are not supported");
779 }
780 } else {
781 if (isa<UnrankedMemRefType>(Val: operand.getType())) {
782 assert(llvmOperand.size() == 1 && "Expected a single operand");
783 UnrankedMemRefDescriptor::unpack(builder, loc, packed: llvmOperand.front(),
784 results&: promotedOperands);
785 continue;
786 }
787 if (auto memrefType = dyn_cast<MemRefType>(Val: operand.getType())) {
788 assert(llvmOperand.size() == 1 && "Expected a single operand");
789 MemRefDescriptor::unpack(builder, loc, packed: llvmOperand.front(), type: memrefType,
790 results&: promotedOperands);
791 continue;
792 }
793 }
794
795 llvm::append_range(C&: promotedOperands, R: llvmOperand);
796 }
797 return promotedOperands;
798}
799
800/// Callback to convert function argument types. It converts a MemRef function
801/// argument to a list of non-aggregate types containing descriptor
802/// information, and an UnrankedmemRef function argument to a list containing
803/// the rank and a pointer to a descriptor struct.
804LogicalResult
805mlir::structFuncArgTypeConverter(const LLVMTypeConverter &converter, Type type,
806 SmallVectorImpl<Type> &result) {
807 if (auto memref = dyn_cast<MemRefType>(Val&: type)) {
808 // In signatures, Memref descriptors are expanded into lists of
809 // non-aggregate values.
810 auto converted =
811 converter.getMemRefDescriptorFields(type: memref, /*unpackAggregates=*/true);
812 if (converted.empty())
813 return failure();
814 result.append(in_start: converted.begin(), in_end: converted.end());
815 return success();
816 }
817 if (isa<UnrankedMemRefType>(Val: type)) {
818 auto converted = converter.getUnrankedMemRefDescriptorFields();
819 if (converted.empty())
820 return failure();
821 result.append(in_start: converted.begin(), in_end: converted.end());
822 return success();
823 }
824 return converter.convertType(t: type, results&: result);
825}
826
827/// Callback to convert function argument types. It converts MemRef function
828/// arguments to bare pointers to the MemRef element type.
829LogicalResult
830mlir::barePtrFuncArgTypeConverter(const LLVMTypeConverter &converter, Type type,
831 SmallVectorImpl<Type> &result) {
832 return converter.convertCallingConventionType(
833 type, result,
834 /*useBarePointerCallConv=*/useBarePtrCallConv: true);
835}
836