| 1 | //===- TypeConverter.cpp - Convert builtin to LLVM dialect types ----------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Conversion/LLVMCommon/TypeConverter.h" |
| 10 | #include "mlir/Conversion/LLVMCommon/MemRefBuilder.h" |
| 11 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 12 | #include "mlir/Dialect/LLVMIR/LLVMDialectDecl.h" |
| 13 | #include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h" |
| 14 | #include "mlir/Dialect/LLVMIR/LLVMTypes.h" |
| 15 | #include "mlir/Interfaces/FunctionInterfaces.h" |
| 16 | #include "llvm/ADT/ScopeExit.h" |
| 17 | #include "llvm/Support/Threading.h" |
| 18 | #include <memory> |
| 19 | #include <mutex> |
| 20 | #include <optional> |
| 21 | |
| 22 | using namespace mlir; |
| 23 | |
| 24 | SmallVector<Type> &LLVMTypeConverter::getCurrentThreadRecursiveStack() { |
| 25 | { |
| 26 | // Most of the time, the entry already exists in the map. |
| 27 | std::shared_lock<decltype(callStackMutex)> lock(callStackMutex, |
| 28 | std::defer_lock); |
| 29 | if (getContext().isMultithreadingEnabled()) |
| 30 | lock.lock(); |
| 31 | auto recursiveStack = conversionCallStack.find(Val: llvm::get_threadid()); |
| 32 | if (recursiveStack != conversionCallStack.end()) |
| 33 | return *recursiveStack->second; |
| 34 | } |
| 35 | |
| 36 | // First time this thread gets here, we have to get an exclusive access to |
| 37 | // inset in the map |
| 38 | std::unique_lock<decltype(callStackMutex)> lock(callStackMutex); |
| 39 | auto recursiveStackInserted = conversionCallStack.insert(KV: std::make_pair( |
| 40 | x: llvm::get_threadid(), y: std::make_unique<SmallVector<Type>>())); |
| 41 | return *recursiveStackInserted.first->second; |
| 42 | } |
| 43 | |
| 44 | /// Create an LLVMTypeConverter using default LowerToLLVMOptions. |
| 45 | LLVMTypeConverter::LLVMTypeConverter(MLIRContext *ctx, |
| 46 | const DataLayoutAnalysis *analysis) |
| 47 | : LLVMTypeConverter(ctx, LowerToLLVMOptions(ctx), analysis) {} |
| 48 | |
| 49 | /// Helper function that checks if the given value range is a bare pointer. |
| 50 | static bool isBarePointer(ValueRange values) { |
| 51 | return values.size() == 1 && |
| 52 | isa<LLVM::LLVMPointerType>(Val: values.front().getType()); |
| 53 | } |
| 54 | |
| 55 | /// Pack SSA values into an unranked memref descriptor struct. |
| 56 | static Value packUnrankedMemRefDesc(OpBuilder &builder, |
| 57 | UnrankedMemRefType resultType, |
| 58 | ValueRange inputs, Location loc, |
| 59 | const LLVMTypeConverter &converter) { |
| 60 | // Note: Bare pointers are not supported for unranked memrefs because a |
| 61 | // memref descriptor cannot be built just from a bare pointer. |
| 62 | if (TypeRange(inputs) != converter.getUnrankedMemRefDescriptorFields()) |
| 63 | return Value(); |
| 64 | return UnrankedMemRefDescriptor::pack(builder, loc, converter, type: resultType, |
| 65 | values: inputs); |
| 66 | } |
| 67 | |
| 68 | /// Pack SSA values into a ranked memref descriptor struct. |
| 69 | static Value packRankedMemRefDesc(OpBuilder &builder, MemRefType resultType, |
| 70 | ValueRange inputs, Location loc, |
| 71 | const LLVMTypeConverter &converter) { |
| 72 | assert(resultType && "expected non-null result type" ); |
| 73 | if (isBarePointer(values: inputs)) |
| 74 | return MemRefDescriptor::fromStaticShape(builder, loc, typeConverter: converter, |
| 75 | type: resultType, memory: inputs[0]); |
| 76 | if (TypeRange(inputs) == |
| 77 | converter.getMemRefDescriptorFields(type: resultType, |
| 78 | /*unpackAggregates=*/true)) |
| 79 | return MemRefDescriptor::pack(builder, loc, converter, type: resultType, values: inputs); |
| 80 | // The inputs are neither a bare pointer nor an unpacked memref descriptor. |
| 81 | // This materialization function cannot be used. |
| 82 | return Value(); |
| 83 | } |
| 84 | |
| 85 | /// MemRef descriptor elements -> UnrankedMemRefType |
| 86 | static Value unrankedMemRefMaterialization(OpBuilder &builder, |
| 87 | UnrankedMemRefType resultType, |
| 88 | ValueRange inputs, Location loc, |
| 89 | const LLVMTypeConverter &converter) { |
| 90 | // A source materialization must return a value of type |
| 91 | // `resultType`, so insert a cast from the memref descriptor type |
| 92 | // (!llvm.struct) to the original memref type. |
| 93 | Value packed = |
| 94 | packUnrankedMemRefDesc(builder, resultType, inputs, loc, converter); |
| 95 | if (!packed) |
| 96 | return Value(); |
| 97 | return UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: resultType, operands: packed) |
| 98 | .getResult(i: 0); |
| 99 | } |
| 100 | |
| 101 | /// MemRef descriptor elements -> MemRefType |
| 102 | static Value rankedMemRefMaterialization(OpBuilder &builder, |
| 103 | MemRefType resultType, |
| 104 | ValueRange inputs, Location loc, |
| 105 | const LLVMTypeConverter &converter) { |
| 106 | // A source materialization must return a value of type `resultType`, |
| 107 | // so insert a cast from the memref descriptor type (!llvm.struct) to the |
| 108 | // original memref type. |
| 109 | Value packed = |
| 110 | packRankedMemRefDesc(builder, resultType, inputs, loc, converter); |
| 111 | if (!packed) |
| 112 | return Value(); |
| 113 | return UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: resultType, operands: packed) |
| 114 | .getResult(i: 0); |
| 115 | } |
| 116 | |
| 117 | /// Create an LLVMTypeConverter using custom LowerToLLVMOptions. |
| 118 | LLVMTypeConverter::LLVMTypeConverter(MLIRContext *ctx, |
| 119 | const LowerToLLVMOptions &options, |
| 120 | const DataLayoutAnalysis *analysis) |
| 121 | : llvmDialect(ctx->getOrLoadDialect<LLVM::LLVMDialect>()), options(options), |
| 122 | dataLayoutAnalysis(analysis) { |
| 123 | assert(llvmDialect && "LLVM IR dialect is not registered" ); |
| 124 | |
| 125 | // Register conversions for the builtin types. |
| 126 | addConversion(callback: [&](ComplexType type) { return convertComplexType(type); }); |
| 127 | addConversion(callback: [&](FloatType type) { return convertFloatType(type); }); |
| 128 | addConversion(callback: [&](FunctionType type) { return convertFunctionType(type); }); |
| 129 | addConversion(callback: [&](IndexType type) { return convertIndexType(type); }); |
| 130 | addConversion(callback: [&](IntegerType type) { return convertIntegerType(type); }); |
| 131 | addConversion(callback: [&](MemRefType type) { return convertMemRefType(type); }); |
| 132 | addConversion( |
| 133 | callback: [&](UnrankedMemRefType type) { return convertUnrankedMemRefType(type); }); |
| 134 | addConversion(callback: [&](VectorType type) -> std::optional<Type> { |
| 135 | FailureOr<Type> llvmType = convertVectorType(type); |
| 136 | if (failed(Result: llvmType)) |
| 137 | return std::nullopt; |
| 138 | return llvmType; |
| 139 | }); |
| 140 | |
| 141 | // LLVM-compatible types are legal, so add a pass-through conversion. Do this |
| 142 | // before the conversions below since conversions are attempted in reverse |
| 143 | // order and those should take priority. |
| 144 | addConversion(callback: [](Type type) { |
| 145 | return LLVM::isCompatibleType(type) ? std::optional<Type>(type) |
| 146 | : std::nullopt; |
| 147 | }); |
| 148 | |
| 149 | addConversion(callback: [&](LLVM::LLVMStructType type, SmallVectorImpl<Type> &results) |
| 150 | -> std::optional<LogicalResult> { |
| 151 | // Fastpath for types that won't be converted by this callback anyway. |
| 152 | if (LLVM::isCompatibleType(type)) { |
| 153 | results.push_back(Elt: type); |
| 154 | return success(); |
| 155 | } |
| 156 | |
| 157 | if (type.isIdentified()) { |
| 158 | auto convertedType = LLVM::LLVMStructType::getIdentified( |
| 159 | context: type.getContext(), name: ("_Converted." + type.getName()).str()); |
| 160 | |
| 161 | SmallVectorImpl<Type> &recursiveStack = getCurrentThreadRecursiveStack(); |
| 162 | if (llvm::count(Range&: recursiveStack, Element: type)) { |
| 163 | results.push_back(Elt: convertedType); |
| 164 | return success(); |
| 165 | } |
| 166 | recursiveStack.push_back(Elt: type); |
| 167 | llvm::scope_exit popConversionCallStack( |
| 168 | [&recursiveStack]() { recursiveStack.pop_back(); }); |
| 169 | |
| 170 | SmallVector<Type> convertedElemTypes; |
| 171 | convertedElemTypes.reserve(N: type.getBody().size()); |
| 172 | if (failed(Result: convertTypes(types: type.getBody(), results&: convertedElemTypes))) |
| 173 | return std::nullopt; |
| 174 | |
| 175 | // If the converted type has not been initialized yet, just set its body |
| 176 | // to be the converted arguments and return. |
| 177 | if (!convertedType.isInitialized()) { |
| 178 | if (failed( |
| 179 | Result: convertedType.setBody(types: convertedElemTypes, isPacked: type.isPacked()))) { |
| 180 | return failure(); |
| 181 | } |
| 182 | results.push_back(Elt: convertedType); |
| 183 | return success(); |
| 184 | } |
| 185 | |
| 186 | // If it has been initialized, has the same body and packed bit, just use |
| 187 | // it. This ensures that recursive structs keep being recursive rather |
| 188 | // than including a non-updated name. |
| 189 | if (TypeRange(convertedType.getBody()) == TypeRange(convertedElemTypes) && |
| 190 | convertedType.isPacked() == type.isPacked()) { |
| 191 | results.push_back(Elt: convertedType); |
| 192 | return success(); |
| 193 | } |
| 194 | |
| 195 | return failure(); |
| 196 | } |
| 197 | |
| 198 | SmallVector<Type> convertedSubtypes; |
| 199 | convertedSubtypes.reserve(N: type.getBody().size()); |
| 200 | if (failed(Result: convertTypes(types: type.getBody(), results&: convertedSubtypes))) |
| 201 | return std::nullopt; |
| 202 | |
| 203 | results.push_back(Elt: LLVM::LLVMStructType::getLiteral( |
| 204 | context: type.getContext(), types: convertedSubtypes, isPacked: type.isPacked())); |
| 205 | return success(); |
| 206 | }); |
| 207 | addConversion(callback: [&](LLVM::LLVMArrayType type) -> std::optional<Type> { |
| 208 | if (auto element = convertType(t: type.getElementType())) |
| 209 | return LLVM::LLVMArrayType::get(elementType: element, numElements: type.getNumElements()); |
| 210 | return std::nullopt; |
| 211 | }); |
| 212 | addConversion(callback: [&](LLVM::LLVMFunctionType type) -> std::optional<Type> { |
| 213 | Type convertedResType = convertType(t: type.getReturnType()); |
| 214 | if (!convertedResType) |
| 215 | return std::nullopt; |
| 216 | |
| 217 | SmallVector<Type> convertedArgTypes; |
| 218 | convertedArgTypes.reserve(N: type.getNumParams()); |
| 219 | if (failed(Result: convertTypes(types: type.getParams(), results&: convertedArgTypes))) |
| 220 | return std::nullopt; |
| 221 | |
| 222 | return LLVM::LLVMFunctionType::get(result: convertedResType, arguments: convertedArgTypes, |
| 223 | isVarArg: type.isVarArg()); |
| 224 | }); |
| 225 | |
| 226 | // Add generic source and target materializations to handle cases where |
| 227 | // non-LLVM types persist after an LLVM conversion. |
| 228 | addSourceMaterialization(callback: [&](OpBuilder &builder, Type resultType, |
| 229 | ValueRange inputs, Location loc) { |
| 230 | return UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: resultType, operands: inputs) |
| 231 | .getResult(i: 0); |
| 232 | }); |
| 233 | addTargetMaterialization(callback: [&](OpBuilder &builder, Type resultType, |
| 234 | ValueRange inputs, Location loc) { |
| 235 | return UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: resultType, operands: inputs) |
| 236 | .getResult(i: 0); |
| 237 | }); |
| 238 | |
| 239 | // Source materializations convert from the new block argument types |
| 240 | // (multiple SSA values that make up a memref descriptor) back to the |
| 241 | // original block argument type. |
| 242 | addSourceMaterialization(callback: [&](OpBuilder &builder, |
| 243 | UnrankedMemRefType resultType, ValueRange inputs, |
| 244 | Location loc) { |
| 245 | return unrankedMemRefMaterialization(builder, resultType, inputs, loc, |
| 246 | converter: *this); |
| 247 | }); |
| 248 | addSourceMaterialization(callback: [&](OpBuilder &builder, MemRefType resultType, |
| 249 | ValueRange inputs, Location loc) { |
| 250 | return rankedMemRefMaterialization(builder, resultType, inputs, loc, converter: *this); |
| 251 | }); |
| 252 | |
| 253 | // Bare pointer -> Packed MemRef descriptor |
| 254 | addTargetMaterialization(callback: [&](OpBuilder &builder, Type resultType, |
| 255 | ValueRange inputs, Location loc, |
| 256 | Type originalType) -> Value { |
| 257 | // The original MemRef type is required to build a MemRef descriptor |
| 258 | // because the sizes/strides of the MemRef cannot be inferred from just the |
| 259 | // bare pointer. |
| 260 | if (!originalType) |
| 261 | return Value(); |
| 262 | if (resultType != convertType(t: originalType)) |
| 263 | return Value(); |
| 264 | if (auto memrefType = dyn_cast<MemRefType>(Val&: originalType)) |
| 265 | return packRankedMemRefDesc(builder, resultType: memrefType, inputs, loc, converter: *this); |
| 266 | if (auto unrankedMemrefType = dyn_cast<UnrankedMemRefType>(Val&: originalType)) |
| 267 | return packUnrankedMemRefDesc(builder, resultType: unrankedMemrefType, inputs, loc, |
| 268 | converter: *this); |
| 269 | return Value(); |
| 270 | }); |
| 271 | |
| 272 | // Integer memory spaces map to themselves. |
| 273 | addTypeAttributeConversion( |
| 274 | callback: [](BaseMemRefType memref, IntegerAttr addrspace) { return addrspace; }); |
| 275 | } |
| 276 | |
| 277 | /// Returns the MLIR context. |
| 278 | MLIRContext &LLVMTypeConverter::getContext() const { |
| 279 | return *getDialect()->getContext(); |
| 280 | } |
| 281 | |
| 282 | Type LLVMTypeConverter::getIndexType() const { |
| 283 | return IntegerType::get(context: &getContext(), width: getIndexTypeBitwidth()); |
| 284 | } |
| 285 | |
| 286 | unsigned LLVMTypeConverter::getPointerBitwidth(unsigned addressSpace) const { |
| 287 | return options.dataLayout.getPointerSizeInBits(AS: addressSpace); |
| 288 | } |
| 289 | |
| 290 | Type LLVMTypeConverter::convertIndexType(IndexType type) const { |
| 291 | return getIndexType(); |
| 292 | } |
| 293 | |
| 294 | Type LLVMTypeConverter::convertIntegerType(IntegerType type) const { |
| 295 | return IntegerType::get(context: &getContext(), width: type.getWidth()); |
| 296 | } |
| 297 | |
| 298 | Type LLVMTypeConverter::convertFloatType(FloatType type) const { |
| 299 | // Valid LLVM float types are used directly. |
| 300 | if (LLVM::isCompatibleType(type)) |
| 301 | return type; |
| 302 | |
| 303 | // F4, F6, F8 types are converted to integer types with the same bit width. |
| 304 | if (isa<Float8E5M2Type, Float8E4M3Type, Float8E4M3FNType, Float8E5M2FNUZType, |
| 305 | Float8E4M3FNUZType, Float8E4M3B11FNUZType, Float8E3M4Type, |
| 306 | Float4E2M1FNType, Float6E2M3FNType, Float6E3M2FNType, |
| 307 | Float8E8M0FNUType, Float8E5M3FNUType>(Val: type)) |
| 308 | return IntegerType::get(context: &getContext(), width: type.getWidth()); |
| 309 | |
| 310 | // Other floating-point types: A custom type conversion rule must be |
| 311 | // specified by the user. |
| 312 | return Type(); |
| 313 | } |
| 314 | |
| 315 | // Convert a `ComplexType` to an LLVM type. The result is a complex number |
| 316 | // struct with entries for the |
| 317 | // 1. real part and for the |
| 318 | // 2. imaginary part. |
| 319 | Type LLVMTypeConverter::convertComplexType(ComplexType type) const { |
| 320 | auto elementType = convertType(t: type.getElementType()); |
| 321 | return LLVM::LLVMStructType::getLiteral(context: &getContext(), |
| 322 | types: {elementType, elementType}); |
| 323 | } |
| 324 | |
| 325 | // Except for signatures, MLIR function types are converted into LLVM |
| 326 | // pointer-to-function types. |
| 327 | Type LLVMTypeConverter::convertFunctionType(FunctionType type) const { |
| 328 | return LLVM::LLVMPointerType::get(context: type.getContext()); |
| 329 | } |
| 330 | |
| 331 | /// Returns the `llvm.byval` or `llvm.byref` attributes that are present in the |
| 332 | /// function arguments. Returns an empty container if none of these attributes |
| 333 | /// are found in any of the arguments. |
| 334 | static void |
| 335 | filterByValRefArgAttrs(FunctionOpInterface funcOp, |
| 336 | SmallVectorImpl<std::optional<NamedAttribute>> &result) { |
| 337 | assert(result.empty() && "Unexpected non-empty output" ); |
| 338 | result.resize(N: funcOp.getNumArguments(), NV: std::nullopt); |
| 339 | bool foundByValByRefAttrs = false; |
| 340 | for (int argIdx : llvm::seq(Size: funcOp.getNumArguments())) { |
| 341 | for (NamedAttribute namedAttr : funcOp.getArgAttrs(index: argIdx)) { |
| 342 | if ((namedAttr.getName() == LLVM::LLVMDialect::getByValAttrName() || |
| 343 | namedAttr.getName() == LLVM::LLVMDialect::getByRefAttrName())) { |
| 344 | foundByValByRefAttrs = true; |
| 345 | result[argIdx] = namedAttr; |
| 346 | break; |
| 347 | } |
| 348 | } |
| 349 | } |
| 350 | |
| 351 | if (!foundByValByRefAttrs) |
| 352 | result.clear(); |
| 353 | } |
| 354 | |
| 355 | // Function types are converted to LLVM Function types by recursively converting |
| 356 | // argument and result types. If MLIR Function has zero results, the LLVM |
| 357 | // Function has one VoidType result. If MLIR Function has more than one result, |
| 358 | // they are into an LLVM StructType in their order of appearance. |
| 359 | // If `byValRefNonPtrAttrs` is provided, converted types of `llvm.byval` and |
| 360 | // `llvm.byref` function arguments which are not LLVM pointers are overridden |
| 361 | // with LLVM pointers. `llvm.byval` and `llvm.byref` arguments that were already |
| 362 | // converted to LLVM pointer types are removed from 'byValRefNonPtrAttrs`. |
| 363 | Type LLVMTypeConverter::convertFunctionSignatureImpl( |
| 364 | FunctionType funcTy, bool isVariadic, bool useBarePtrCallConv, |
| 365 | LLVMTypeConverter::SignatureConversion &result, |
| 366 | SmallVectorImpl<std::optional<NamedAttribute>> *byValRefNonPtrAttrs) const { |
| 367 | // Select the argument converter depending on the calling convention. |
| 368 | useBarePtrCallConv = useBarePtrCallConv || options.useBarePtrCallConv; |
| 369 | auto funcArgConverter = useBarePtrCallConv ? barePtrFuncArgTypeConverter |
| 370 | : structFuncArgTypeConverter; |
| 371 | |
| 372 | // Convert argument types one by one and check for errors. |
| 373 | for (auto [idx, type] : llvm::enumerate(First: funcTy.getInputs())) { |
| 374 | SmallVector<Type, 8> converted; |
| 375 | if (failed(Result: funcArgConverter(*this, type, converted))) |
| 376 | return {}; |
| 377 | |
| 378 | // Rewrite converted type of `llvm.byval` or `llvm.byref` function |
| 379 | // argument that was not converted to an LLVM pointer types. |
| 380 | if (byValRefNonPtrAttrs != nullptr && !byValRefNonPtrAttrs->empty() && |
| 381 | converted.size() == 1 && (*byValRefNonPtrAttrs)[idx].has_value()) { |
| 382 | // If the argument was already converted to an LLVM pointer type, we stop |
| 383 | // tracking it as it doesn't need more processing. |
| 384 | if (isa<LLVM::LLVMPointerType>(Val: converted[0])) |
| 385 | (*byValRefNonPtrAttrs)[idx] = std::nullopt; |
| 386 | else |
| 387 | converted[0] = LLVM::LLVMPointerType::get(context: &getContext()); |
| 388 | } |
| 389 | |
| 390 | result.addInputs(origInputNo: idx, types: converted); |
| 391 | } |
| 392 | |
| 393 | // If function does not return anything, create the void result type, |
| 394 | // if it returns on element, convert it, otherwise pack the result types into |
| 395 | // a struct. |
| 396 | Type resultType = |
| 397 | funcTy.getNumResults() == 0 |
| 398 | ? LLVM::LLVMVoidType::get(ctx: &getContext()) |
| 399 | : packFunctionResults(types: funcTy.getResults(), useBarePointerCallConv: useBarePtrCallConv); |
| 400 | if (!resultType) |
| 401 | return {}; |
| 402 | return LLVM::LLVMFunctionType::get(result: resultType, arguments: result.getConvertedTypes(), |
| 403 | isVarArg: isVariadic); |
| 404 | } |
| 405 | |
| 406 | Type LLVMTypeConverter::convertFunctionSignature( |
| 407 | FunctionType funcTy, bool isVariadic, bool useBarePtrCallConv, |
| 408 | LLVMTypeConverter::SignatureConversion &result) const { |
| 409 | return convertFunctionSignatureImpl(funcTy, isVariadic, useBarePtrCallConv, |
| 410 | result, |
| 411 | /*byValRefNonPtrAttrs=*/nullptr); |
| 412 | } |
| 413 | |
| 414 | Type LLVMTypeConverter::convertFunctionSignature( |
| 415 | FunctionOpInterface funcOp, bool isVariadic, bool useBarePtrCallConv, |
| 416 | LLVMTypeConverter::SignatureConversion &result, |
| 417 | SmallVectorImpl<std::optional<NamedAttribute>> &byValRefNonPtrAttrs) const { |
| 418 | // Gather all `llvm.byval` and `llvm.byref` function arguments. Only those |
| 419 | // that were not converted to LLVM pointer types will be returned for further |
| 420 | // processing. |
| 421 | filterByValRefArgAttrs(funcOp, result&: byValRefNonPtrAttrs); |
| 422 | auto funcTy = cast<FunctionType>(Val: funcOp.getFunctionType()); |
| 423 | return convertFunctionSignatureImpl(funcTy, isVariadic, useBarePtrCallConv, |
| 424 | result, byValRefNonPtrAttrs: &byValRefNonPtrAttrs); |
| 425 | } |
| 426 | |
| 427 | /// Converts the function type to a C-compatible format, in particular using |
| 428 | /// pointers to memref descriptors for arguments. |
| 429 | std::pair<LLVM::LLVMFunctionType, LLVM::LLVMStructType> |
| 430 | LLVMTypeConverter::convertFunctionTypeCWrapper(FunctionType type) const { |
| 431 | SmallVector<Type, 4> inputs; |
| 432 | |
| 433 | Type resultType = type.getNumResults() == 0 |
| 434 | ? LLVM::LLVMVoidType::get(ctx: &getContext()) |
| 435 | : packFunctionResults(types: type.getResults()); |
| 436 | if (!resultType) |
| 437 | return {}; |
| 438 | |
| 439 | auto ptrType = LLVM::LLVMPointerType::get(context: type.getContext()); |
| 440 | auto structType = dyn_cast<LLVM::LLVMStructType>(Val&: resultType); |
| 441 | if (structType) { |
| 442 | // Struct types cannot be safely returned via C interface. Make this a |
| 443 | // pointer argument, instead. |
| 444 | inputs.push_back(Elt: ptrType); |
| 445 | resultType = LLVM::LLVMVoidType::get(ctx: &getContext()); |
| 446 | } |
| 447 | |
| 448 | for (Type t : type.getInputs()) { |
| 449 | auto converted = convertType(t); |
| 450 | if (!converted || !LLVM::isCompatibleType(type: converted)) |
| 451 | return {}; |
| 452 | if (isa<MemRefType, UnrankedMemRefType>(Val: t)) |
| 453 | converted = ptrType; |
| 454 | inputs.push_back(Elt: converted); |
| 455 | } |
| 456 | |
| 457 | return {LLVM::LLVMFunctionType::get(result: resultType, arguments: inputs), structType}; |
| 458 | } |
| 459 | |
| 460 | /// Convert a memref type into a list of LLVM IR types that will form the |
| 461 | /// memref descriptor. The result contains the following types: |
| 462 | /// 1. The pointer to the allocated data buffer, followed by |
| 463 | /// 2. The pointer to the aligned data buffer, followed by |
| 464 | /// 3. A lowered `index`-type integer containing the distance between the |
| 465 | /// beginning of the buffer and the first element to be accessed through the |
| 466 | /// view, followed by |
| 467 | /// 4. An array containing as many `index`-type integers as the rank of the |
| 468 | /// MemRef: the array represents the size, in number of elements, of the memref |
| 469 | /// along the given dimension. For constant MemRef dimensions, the |
| 470 | /// corresponding size entry is a constant whose runtime value must match the |
| 471 | /// static value, followed by |
| 472 | /// 5. A second array containing as many `index`-type integers as the rank of |
| 473 | /// the MemRef: the second array represents the "stride" (in tensor abstraction |
| 474 | /// sense), i.e. the number of consecutive elements of the underlying buffer. |
| 475 | /// TODO: add assertions for the static cases. |
| 476 | /// |
| 477 | /// If `unpackAggregates` is set to true, the arrays described in (4) and (5) |
| 478 | /// are expanded into individual index-type elements. |
| 479 | /// |
| 480 | /// template <typename Elem, typename Index, size_t Rank> |
| 481 | /// struct { |
| 482 | /// Elem *allocatedPtr; |
| 483 | /// Elem *alignedPtr; |
| 484 | /// Index offset; |
| 485 | /// Index sizes[Rank]; // omitted when rank == 0 |
| 486 | /// Index strides[Rank]; // omitted when rank == 0 |
| 487 | /// }; |
| 488 | SmallVector<Type, 5> |
| 489 | LLVMTypeConverter::getMemRefDescriptorFields(MemRefType type, |
| 490 | bool unpackAggregates) const { |
| 491 | if (!type.isStrided()) { |
| 492 | emitError( |
| 493 | loc: UnknownLoc::get(context: type.getContext()), |
| 494 | message: "conversion to strided form failed either due to non-strided layout " |
| 495 | "maps (which should have been normalized away) or other reasons" ); |
| 496 | return {}; |
| 497 | } |
| 498 | |
| 499 | Type elementType = convertType(t: type.getElementType()); |
| 500 | if (!elementType) |
| 501 | return {}; |
| 502 | |
| 503 | FailureOr<unsigned> addressSpace = getMemRefAddressSpace(type); |
| 504 | if (failed(Result: addressSpace)) { |
| 505 | emitError(loc: UnknownLoc::get(context: type.getContext()), |
| 506 | message: "conversion of memref memory space " ) |
| 507 | << type.getMemorySpace() |
| 508 | << " to integer address space " |
| 509 | "failed. Consider adding memory space conversions." ; |
| 510 | return {}; |
| 511 | } |
| 512 | auto ptrTy = LLVM::LLVMPointerType::get(context: type.getContext(), addressSpace: *addressSpace); |
| 513 | |
| 514 | auto indexTy = getIndexType(); |
| 515 | |
| 516 | SmallVector<Type, 5> results = {ptrTy, ptrTy, indexTy}; |
| 517 | auto rank = type.getRank(); |
| 518 | if (rank == 0) |
| 519 | return results; |
| 520 | |
| 521 | if (unpackAggregates) |
| 522 | results.insert(I: results.end(), NumToInsert: 2 * rank, Elt: indexTy); |
| 523 | else |
| 524 | results.insert(I: results.end(), NumToInsert: 2, Elt: LLVM::LLVMArrayType::get(elementType: indexTy, numElements: rank)); |
| 525 | return results; |
| 526 | } |
| 527 | |
| 528 | unsigned |
| 529 | LLVMTypeConverter::getMemRefDescriptorSize(MemRefType type, |
| 530 | const DataLayout &layout) const { |
| 531 | // Compute the descriptor size given that of its components indicated above. |
| 532 | unsigned space = *getMemRefAddressSpace(type); |
| 533 | return 2 * llvm::divideCeil(Numerator: getPointerBitwidth(addressSpace: space), Denominator: 8) + |
| 534 | (1 + 2 * type.getRank()) * layout.getTypeSize(t: getIndexType()); |
| 535 | } |
| 536 | |
| 537 | /// Converts MemRefType to LLVMType. A MemRefType is converted to a struct that |
| 538 | /// packs the descriptor fields as defined by `getMemRefDescriptorFields`. |
| 539 | Type LLVMTypeConverter::convertMemRefType(MemRefType type) const { |
| 540 | // When converting a MemRefType to a struct with descriptor fields, do not |
| 541 | // unpack the `sizes` and `strides` arrays. |
| 542 | SmallVector<Type, 5> types = |
| 543 | getMemRefDescriptorFields(type, /*unpackAggregates=*/false); |
| 544 | if (types.empty()) |
| 545 | return {}; |
| 546 | return LLVM::LLVMStructType::getLiteral(context: &getContext(), types); |
| 547 | } |
| 548 | |
| 549 | /// Convert an unranked memref type into a list of non-aggregate LLVM IR types |
| 550 | /// that will form the unranked memref descriptor. In particular, the fields |
| 551 | /// for an unranked memref descriptor are: |
| 552 | /// 1. index-typed rank, the dynamic rank of this MemRef |
| 553 | /// 2. void* ptr, pointer to the static ranked MemRef descriptor. This will be |
| 554 | /// stack allocated (alloca) copy of a MemRef descriptor that got casted to |
| 555 | /// be unranked. |
| 556 | SmallVector<Type, 2> |
| 557 | LLVMTypeConverter::getUnrankedMemRefDescriptorFields() const { |
| 558 | return {getIndexType(), LLVM::LLVMPointerType::get(context: &getContext())}; |
| 559 | } |
| 560 | |
| 561 | unsigned LLVMTypeConverter::getUnrankedMemRefDescriptorSize( |
| 562 | UnrankedMemRefType type, const DataLayout &layout) const { |
| 563 | // Compute the descriptor size given that of its components indicated above. |
| 564 | unsigned space = *getMemRefAddressSpace(type); |
| 565 | return layout.getTypeSize(t: getIndexType()) + |
| 566 | llvm::divideCeil(Numerator: getPointerBitwidth(addressSpace: space), Denominator: 8); |
| 567 | } |
| 568 | |
| 569 | Type LLVMTypeConverter::convertUnrankedMemRefType( |
| 570 | UnrankedMemRefType type) const { |
| 571 | if (!convertType(t: type.getElementType())) |
| 572 | return {}; |
| 573 | return LLVM::LLVMStructType::getLiteral(context: &getContext(), |
| 574 | types: getUnrankedMemRefDescriptorFields()); |
| 575 | } |
| 576 | |
| 577 | FailureOr<unsigned> |
| 578 | LLVMTypeConverter::getMemRefAddressSpace(BaseMemRefType type) const { |
| 579 | if (!type.getMemorySpace()) // Default memory space -> 0. |
| 580 | return 0; |
| 581 | std::optional<Attribute> converted = |
| 582 | convertTypeAttribute(type, attr: type.getMemorySpace()); |
| 583 | if (!converted) |
| 584 | return failure(); |
| 585 | if (!(*converted)) // Conversion to default is 0. |
| 586 | return 0; |
| 587 | if (auto explicitSpace = dyn_cast_if_present<IntegerAttr>(Val&: *converted)) { |
| 588 | if (explicitSpace.getType().isIndex() || |
| 589 | explicitSpace.getType().isSignlessInteger()) |
| 590 | return explicitSpace.getInt(); |
| 591 | } |
| 592 | return failure(); |
| 593 | } |
| 594 | |
| 595 | // Check if a memref type can be converted to a bare pointer. |
| 596 | bool LLVMTypeConverter::canConvertToBarePtr(BaseMemRefType type) { |
| 597 | if (isa<UnrankedMemRefType>(Val: type)) |
| 598 | // Unranked memref is not supported in the bare pointer calling convention. |
| 599 | return false; |
| 600 | |
| 601 | // Check that the memref has static shape, strides and offset. Otherwise, it |
| 602 | // cannot be lowered to a bare pointer. |
| 603 | auto memrefTy = cast<MemRefType>(Val&: type); |
| 604 | if (!memrefTy.hasStaticShape()) |
| 605 | return false; |
| 606 | |
| 607 | int64_t offset = 0; |
| 608 | SmallVector<int64_t, 4> strides; |
| 609 | if (failed(Result: memrefTy.getStridesAndOffset(strides, offset))) |
| 610 | return false; |
| 611 | |
| 612 | for (int64_t stride : strides) |
| 613 | if (ShapedType::isDynamic(dValue: stride)) |
| 614 | return false; |
| 615 | |
| 616 | return ShapedType::isStatic(dValue: offset); |
| 617 | } |
| 618 | |
| 619 | /// Convert a memref type to a bare pointer to the memref element type. |
| 620 | Type LLVMTypeConverter::convertMemRefToBarePtr(BaseMemRefType type) const { |
| 621 | if (!canConvertToBarePtr(type)) |
| 622 | return {}; |
| 623 | Type elementType = convertType(t: type.getElementType()); |
| 624 | if (!elementType) |
| 625 | return {}; |
| 626 | FailureOr<unsigned> addressSpace = getMemRefAddressSpace(type); |
| 627 | if (failed(Result: addressSpace)) |
| 628 | return {}; |
| 629 | return LLVM::LLVMPointerType::get(context: type.getContext(), addressSpace: *addressSpace); |
| 630 | } |
| 631 | |
| 632 | /// Convert an n-D vector type to an LLVM vector type: |
| 633 | /// * 0-D `vector<T>` are converted to vector<1xT> |
| 634 | /// * 1-D `vector<axT>` remains as is while, |
| 635 | /// * n>1 `vector<ax...xkxT>` convert via an (n-1)-D array type to |
| 636 | /// `!llvm.array<ax...array<jxvector<kxT>>>`. |
| 637 | /// As LLVM supports arrays of scalable vectors, this method will also convert |
| 638 | /// n-D scalable vectors provided that only the trailing dim is scalable. |
| 639 | FailureOr<Type> LLVMTypeConverter::convertVectorType(VectorType type) const { |
| 640 | auto elementType = convertType(t: type.getElementType()); |
| 641 | if (!elementType) |
| 642 | return {}; |
| 643 | if (type.getShape().empty()) |
| 644 | return VectorType::get(shape: {1}, elementType); |
| 645 | Type vectorType = VectorType::get(shape: type.getShape().back(), elementType, |
| 646 | scalableDims: type.getScalableDims().back()); |
| 647 | assert(LLVM::isCompatibleVectorType(vectorType) && |
| 648 | "expected vector type compatible with the LLVM dialect" ); |
| 649 | // For n-D vector types for which a _non-trailing_ dim is scalable, |
| 650 | // return a failure. Supporting such cases would require LLVM |
| 651 | // to support something akin "scalable arrays" of vectors. |
| 652 | if (llvm::is_contained(Range: type.getScalableDims().drop_back(), Element: true)) |
| 653 | return failure(); |
| 654 | auto shape = type.getShape(); |
| 655 | for (int i = shape.size() - 2; i >= 0; --i) |
| 656 | vectorType = LLVM::LLVMArrayType::get(elementType: vectorType, numElements: shape[i]); |
| 657 | return vectorType; |
| 658 | } |
| 659 | |
| 660 | /// Convert a type in the context of the default or bare pointer calling |
| 661 | /// convention. Calling convention sensitive types, such as MemRefType and |
| 662 | /// UnrankedMemRefType, are converted following the specific rules for the |
| 663 | /// calling convention. Calling convention independent types are converted |
| 664 | /// following the default LLVM type conversions. |
| 665 | LogicalResult LLVMTypeConverter::convertCallingConventionType( |
| 666 | Type type, SmallVectorImpl<Type> &result, bool useBarePtrCallConv) const { |
| 667 | if (useBarePtrCallConv) { |
| 668 | if (auto memrefTy = dyn_cast<BaseMemRefType>(Val&: type)) { |
| 669 | Type converted = convertMemRefToBarePtr(type: memrefTy); |
| 670 | if (!converted) |
| 671 | return failure(); |
| 672 | result.push_back(Elt: converted); |
| 673 | return success(); |
| 674 | } |
| 675 | } |
| 676 | |
| 677 | return convertType(t: type, results&: result); |
| 678 | } |
| 679 | |
| 680 | /// Convert a non-empty list of types of values produced by an operation into an |
| 681 | /// LLVM-compatible type. In particular, if more than one value is |
| 682 | /// produced, create a literal structure with elements that correspond to each |
| 683 | /// of the types converted with `convertType`. |
| 684 | Type LLVMTypeConverter::packOperationResults(TypeRange types) const { |
| 685 | assert(!types.empty() && "expected non-empty list of type" ); |
| 686 | if (types.size() == 1) |
| 687 | return convertType(t: types[0]); |
| 688 | |
| 689 | SmallVector<Type> resultTypes; |
| 690 | resultTypes.reserve(N: types.size()); |
| 691 | for (Type type : types) { |
| 692 | Type converted = convertType(t: type); |
| 693 | if (!converted || !LLVM::isCompatibleType(type: converted)) |
| 694 | return {}; |
| 695 | resultTypes.push_back(Elt: converted); |
| 696 | } |
| 697 | |
| 698 | return LLVM::LLVMStructType::getLiteral(context: &getContext(), types: resultTypes); |
| 699 | } |
| 700 | |
| 701 | /// Convert a non-empty list of types to be returned from a function into an |
| 702 | /// LLVM-compatible type. In particular, if more than one value is returned, |
| 703 | /// create an LLVM dialect structure type with elements that correspond to each |
| 704 | /// of the types converted with `convertCallingConventionType`. |
| 705 | Type LLVMTypeConverter::packFunctionResults( |
| 706 | TypeRange types, bool useBarePtrCallConv, |
| 707 | SmallVector<SmallVector<Type>> *groupedTypes, |
| 708 | int64_t *numConvertedTypes) const { |
| 709 | assert(!types.empty() && "expected non-empty list of type" ); |
| 710 | assert((!groupedTypes || groupedTypes->empty()) && |
| 711 | "expected groupedTypes to be empty" ); |
| 712 | |
| 713 | useBarePtrCallConv |= options.useBarePtrCallConv; |
| 714 | SmallVector<Type> resultTypes; |
| 715 | resultTypes.reserve(N: types.size()); |
| 716 | size_t sizeBefore = 0; |
| 717 | for (auto t : types) { |
| 718 | if (failed( |
| 719 | Result: convertCallingConventionType(type: t, result&: resultTypes, useBarePtrCallConv))) |
| 720 | return {}; |
| 721 | if (groupedTypes) { |
| 722 | SmallVector<Type> &group = groupedTypes->emplace_back(); |
| 723 | llvm::append_range(C&: group, R: ArrayRef(resultTypes).drop_front(N: sizeBefore)); |
| 724 | } |
| 725 | sizeBefore = resultTypes.size(); |
| 726 | } |
| 727 | |
| 728 | if (numConvertedTypes) |
| 729 | *numConvertedTypes = resultTypes.size(); |
| 730 | if (resultTypes.size() == 1) |
| 731 | return resultTypes.front(); |
| 732 | if (resultTypes.empty()) |
| 733 | return {}; |
| 734 | return LLVM::LLVMStructType::getLiteral(context: &getContext(), types: resultTypes); |
| 735 | } |
| 736 | |
| 737 | Value LLVMTypeConverter::promoteOneMemRefDescriptor(Location loc, Value operand, |
| 738 | OpBuilder &builder) const { |
| 739 | // Alloca with proper alignment. We do not expect optimizations of this |
| 740 | // alloca op and so we omit allocating at the entry block. |
| 741 | auto ptrType = LLVM::LLVMPointerType::get(context: builder.getContext()); |
| 742 | Value one = LLVM::ConstantOp::create(builder, location: loc, res: builder.getI64Type(), |
| 743 | value: builder.getI64IntegerAttr(value: 1)); |
| 744 | Value allocated = |
| 745 | LLVM::AllocaOp::create(builder, location: loc, resultType: ptrType, elementType: operand.getType(), arraySize: one); |
| 746 | // Store into the alloca'ed descriptor. |
| 747 | LLVM::StoreOp::create(builder, location: loc, value: operand, addr: allocated); |
| 748 | return allocated; |
| 749 | } |
| 750 | |
| 751 | SmallVector<Value, 4> LLVMTypeConverter::promoteOperands( |
| 752 | Location loc, ValueRange opOperands, ValueRange adaptorOperands, |
| 753 | OpBuilder &builder, bool useBarePtrCallConv) const { |
| 754 | SmallVector<ValueRange> ranges; |
| 755 | for (size_t i = 0, e = adaptorOperands.size(); i < e; i++) |
| 756 | ranges.push_back(Elt: adaptorOperands.slice(n: i, m: 1)); |
| 757 | return promoteOperands(loc, opOperands, adaptorOperands: ranges, builder, useBarePtrCallConv); |
| 758 | } |
| 759 | |
| 760 | SmallVector<Value, 4> LLVMTypeConverter::promoteOperands( |
| 761 | Location loc, ValueRange opOperands, ArrayRef<ValueRange> adaptorOperands, |
| 762 | OpBuilder &builder, bool useBarePtrCallConv) const { |
| 763 | SmallVector<Value, 4> promotedOperands; |
| 764 | promotedOperands.reserve(N: adaptorOperands.size()); |
| 765 | useBarePtrCallConv |= options.useBarePtrCallConv; |
| 766 | for (auto [operand, llvmOperand] : |
| 767 | llvm::zip_equal(t&: opOperands, u&: adaptorOperands)) { |
| 768 | if (useBarePtrCallConv) { |
| 769 | // For the bare-ptr calling convention, we only have to extract the |
| 770 | // aligned pointer of a memref. |
| 771 | if (isa<MemRefType>(Val: operand.getType())) { |
| 772 | assert(llvmOperand.size() == 1 && "Expected a single operand" ); |
| 773 | MemRefDescriptor desc(llvmOperand.front()); |
| 774 | promotedOperands.push_back(Elt: desc.alignedPtr(builder, loc)); |
| 775 | continue; |
| 776 | } |
| 777 | if (isa<UnrankedMemRefType>(Val: operand.getType())) { |
| 778 | llvm_unreachable("Unranked memrefs are not supported" ); |
| 779 | } |
| 780 | } else { |
| 781 | if (isa<UnrankedMemRefType>(Val: operand.getType())) { |
| 782 | assert(llvmOperand.size() == 1 && "Expected a single operand" ); |
| 783 | UnrankedMemRefDescriptor::unpack(builder, loc, packed: llvmOperand.front(), |
| 784 | results&: promotedOperands); |
| 785 | continue; |
| 786 | } |
| 787 | if (auto memrefType = dyn_cast<MemRefType>(Val: operand.getType())) { |
| 788 | assert(llvmOperand.size() == 1 && "Expected a single operand" ); |
| 789 | MemRefDescriptor::unpack(builder, loc, packed: llvmOperand.front(), type: memrefType, |
| 790 | results&: promotedOperands); |
| 791 | continue; |
| 792 | } |
| 793 | } |
| 794 | |
| 795 | llvm::append_range(C&: promotedOperands, R: llvmOperand); |
| 796 | } |
| 797 | return promotedOperands; |
| 798 | } |
| 799 | |
| 800 | /// Callback to convert function argument types. It converts a MemRef function |
| 801 | /// argument to a list of non-aggregate types containing descriptor |
| 802 | /// information, and an UnrankedmemRef function argument to a list containing |
| 803 | /// the rank and a pointer to a descriptor struct. |
| 804 | LogicalResult |
| 805 | mlir::structFuncArgTypeConverter(const LLVMTypeConverter &converter, Type type, |
| 806 | SmallVectorImpl<Type> &result) { |
| 807 | if (auto memref = dyn_cast<MemRefType>(Val&: type)) { |
| 808 | // In signatures, Memref descriptors are expanded into lists of |
| 809 | // non-aggregate values. |
| 810 | auto converted = |
| 811 | converter.getMemRefDescriptorFields(type: memref, /*unpackAggregates=*/true); |
| 812 | if (converted.empty()) |
| 813 | return failure(); |
| 814 | result.append(in_start: converted.begin(), in_end: converted.end()); |
| 815 | return success(); |
| 816 | } |
| 817 | if (isa<UnrankedMemRefType>(Val: type)) { |
| 818 | auto converted = converter.getUnrankedMemRefDescriptorFields(); |
| 819 | if (converted.empty()) |
| 820 | return failure(); |
| 821 | result.append(in_start: converted.begin(), in_end: converted.end()); |
| 822 | return success(); |
| 823 | } |
| 824 | return converter.convertType(t: type, results&: result); |
| 825 | } |
| 826 | |
| 827 | /// Callback to convert function argument types. It converts MemRef function |
| 828 | /// arguments to bare pointers to the MemRef element type. |
| 829 | LogicalResult |
| 830 | mlir::barePtrFuncArgTypeConverter(const LLVMTypeConverter &converter, Type type, |
| 831 | SmallVectorImpl<Type> &result) { |
| 832 | return converter.convertCallingConventionType( |
| 833 | type, result, |
| 834 | /*useBarePointerCallConv=*/useBarePtrCallConv: true); |
| 835 | } |
| 836 | |