| 1 | //===- Pattern.cpp - Conversion pattern to the LLVM dialect ---------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Conversion/LLVMCommon/Pattern.h" |
| 10 | #include "mlir/Dialect/LLVMIR/FunctionCallUtils.h" |
| 11 | #include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h" |
| 12 | #include "mlir/Dialect/LLVMIR/LLVMArithOps.h" |
| 13 | #include "mlir/Dialect/LLVMIR/LLVMCallOps.h" |
| 14 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 15 | #include "mlir/Dialect/LLVMIR/LLVMConversionOps.h" |
| 16 | #include "mlir/Dialect/LLVMIR/LLVMMemoryIntrinsicOps.h" |
| 17 | #include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h" |
| 18 | #include "mlir/Dialect/LLVMIR/LLVMModuleOps.h" |
| 19 | #include "mlir/Dialect/LLVMIR/LLVMTypes.h" |
| 20 | #include "mlir/Dialect/LLVMIR/LLVMVectorOps.h" |
| 21 | #include "mlir/IR/AffineMap.h" |
| 22 | #include "mlir/IR/BuiltinAttributes.h" |
| 23 | #include "llvm/Support/CheckedArithmetic.h" |
| 24 | #include "llvm/Support/MathExtras.h" |
| 25 | |
| 26 | using namespace mlir; |
| 27 | |
| 28 | //===----------------------------------------------------------------------===// |
| 29 | // ConvertToLLVMPattern |
| 30 | //===----------------------------------------------------------------------===// |
| 31 | |
| 32 | ConvertToLLVMPattern::ConvertToLLVMPattern( |
| 33 | StringRef rootOpName, MLIRContext *context, |
| 34 | const LLVMTypeConverter &typeConverter, PatternBenefit benefit) |
| 35 | : ConversionPattern(typeConverter, rootOpName, benefit, context) {} |
| 36 | |
| 37 | const LLVMTypeConverter *ConvertToLLVMPattern::getTypeConverter() const { |
| 38 | return static_cast<const LLVMTypeConverter *>( |
| 39 | ConversionPattern::getTypeConverter()); |
| 40 | } |
| 41 | |
| 42 | LLVM::LLVMDialect &ConvertToLLVMPattern::getDialect() const { |
| 43 | return *getTypeConverter()->getDialect(); |
| 44 | } |
| 45 | |
| 46 | Type ConvertToLLVMPattern::getIndexType() const { |
| 47 | return getTypeConverter()->getIndexType(); |
| 48 | } |
| 49 | |
| 50 | Type ConvertToLLVMPattern::getIntPtrType(unsigned addressSpace) const { |
| 51 | return IntegerType::get(context: &getTypeConverter()->getContext(), |
| 52 | width: getTypeConverter()->getPointerBitwidth(addressSpace)); |
| 53 | } |
| 54 | |
| 55 | Type ConvertToLLVMPattern::getVoidType() const { |
| 56 | return LLVM::LLVMVoidType::get(ctx: &getTypeConverter()->getContext()); |
| 57 | } |
| 58 | |
| 59 | Type ConvertToLLVMPattern::getPtrType(unsigned addressSpace) const { |
| 60 | return LLVM::LLVMPointerType::get(context: &getTypeConverter()->getContext(), |
| 61 | addressSpace); |
| 62 | } |
| 63 | |
| 64 | Type ConvertToLLVMPattern::getVoidPtrType() const { return getPtrType(); } |
| 65 | |
| 66 | Value mlir::LLVM::createIndexAttrConstant(OpBuilder &builder, Location loc, |
| 67 | Type resultType, int64_t value) { |
| 68 | return LLVM::ConstantOp::create(builder, location: loc, res: resultType, |
| 69 | value: builder.getIntegerAttr(type: resultType, value)); |
| 70 | } |
| 71 | |
| 72 | Value ConvertToLLVMPattern::createIndexAttrConstant(OpBuilder &builder, |
| 73 | Location loc, |
| 74 | Type resultType, |
| 75 | int64_t value) { |
| 76 | return LLVM::createIndexAttrConstant(builder, loc, resultType, value); |
| 77 | } |
| 78 | |
| 79 | Value ConvertToLLVMPattern::getStridedElementPtr( |
| 80 | ConversionPatternRewriter &rewriter, Location loc, MemRefType type, |
| 81 | Value memRefDesc, ValueRange indices, |
| 82 | LLVM::GEPNoWrapFlags noWrapFlags) const { |
| 83 | return LLVM::getStridedElementPtr(builder&: rewriter, loc, converter: *getTypeConverter(), type, |
| 84 | memRefDesc, indices, noWrapFlags); |
| 85 | } |
| 86 | |
| 87 | // Check if the MemRefType `type` is supported by the lowering. We currently |
| 88 | // only support memrefs with identity maps. |
| 89 | bool ConvertToLLVMPattern::isConvertibleAndHasIdentityMaps( |
| 90 | MemRefType type) const { |
| 91 | if (!type.getLayout().isIdentity()) |
| 92 | return false; |
| 93 | return static_cast<bool>(typeConverter->convertType(t: type)); |
| 94 | } |
| 95 | |
| 96 | Type ConvertToLLVMPattern::getElementPtrType(MemRefType type) const { |
| 97 | auto addressSpace = getTypeConverter()->getMemRefAddressSpace(type); |
| 98 | if (failed(Result: addressSpace)) |
| 99 | return {}; |
| 100 | return LLVM::LLVMPointerType::get(context: type.getContext(), addressSpace: *addressSpace); |
| 101 | } |
| 102 | |
| 103 | void ConvertToLLVMPattern::getMemRefDescriptorSizes( |
| 104 | Location loc, MemRefType memRefType, ValueRange dynamicSizes, |
| 105 | ConversionPatternRewriter &rewriter, SmallVectorImpl<Value> &sizes, |
| 106 | SmallVectorImpl<Value> &strides, Value &size, bool sizeInBytes) const { |
| 107 | assert(isConvertibleAndHasIdentityMaps(memRefType) && |
| 108 | "layout maps must have been normalized away" ); |
| 109 | assert(count(memRefType.getShape(), ShapedType::kDynamic) == |
| 110 | static_cast<ssize_t>(dynamicSizes.size()) && |
| 111 | "dynamicSizes size doesn't match dynamic sizes count in memref shape" ); |
| 112 | |
| 113 | sizes.reserve(N: memRefType.getRank()); |
| 114 | unsigned dynamicIndex = 0; |
| 115 | Type indexType = getIndexType(); |
| 116 | for (int64_t size : memRefType.getShape()) { |
| 117 | sizes.push_back( |
| 118 | Elt: size == ShapedType::kDynamic |
| 119 | ? dynamicSizes[dynamicIndex++] |
| 120 | : createIndexAttrConstant(builder&: rewriter, loc, resultType: indexType, value: size)); |
| 121 | } |
| 122 | |
| 123 | // Strides: iterate sizes in reverse order and multiply. |
| 124 | int64_t stride = 1; |
| 125 | bool overflowed = false; |
| 126 | Value runningStride = createIndexAttrConstant(builder&: rewriter, loc, resultType: indexType, value: 1); |
| 127 | strides.resize(N: memRefType.getRank()); |
| 128 | for (auto i = memRefType.getRank(); i-- > 0;) { |
| 129 | strides[i] = overflowed ? LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: indexType) |
| 130 | : runningStride; |
| 131 | |
| 132 | int64_t staticSize = memRefType.getShape()[i]; |
| 133 | bool useSizeAsStride = stride == 1; |
| 134 | if (staticSize == ShapedType::kDynamic) |
| 135 | stride = ShapedType::kDynamic; |
| 136 | if (stride != ShapedType::kDynamic) { |
| 137 | std::optional<int64_t> res = llvm::checkedMul(LHS: stride, RHS: staticSize); |
| 138 | |
| 139 | if (!res) |
| 140 | overflowed = true; |
| 141 | else |
| 142 | stride = res.value(); |
| 143 | } |
| 144 | |
| 145 | if (overflowed) |
| 146 | runningStride = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: indexType); |
| 147 | else if (useSizeAsStride) |
| 148 | runningStride = sizes[i]; |
| 149 | else if (stride == ShapedType::kDynamic) |
| 150 | runningStride = |
| 151 | LLVM::MulOp::create(builder&: rewriter, location: loc, lhs: runningStride, rhs: sizes[i]); |
| 152 | else |
| 153 | runningStride = createIndexAttrConstant(builder&: rewriter, loc, resultType: indexType, value: stride); |
| 154 | } |
| 155 | if (sizeInBytes) { |
| 156 | // Buffer size in bytes. |
| 157 | Type elementType = typeConverter->convertType(t: memRefType.getElementType()); |
| 158 | auto elementPtrType = LLVM::LLVMPointerType::get(context: rewriter.getContext()); |
| 159 | Value nullPtr = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: elementPtrType); |
| 160 | Value gepPtr = LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: elementPtrType, |
| 161 | elementType, basePtr: nullPtr, indices: runningStride); |
| 162 | size = LLVM::PtrToIntOp::create(builder&: rewriter, location: loc, res: getIndexType(), arg: gepPtr); |
| 163 | } else { |
| 164 | size = runningStride; |
| 165 | } |
| 166 | } |
| 167 | |
| 168 | Value ConvertToLLVMPattern::getSizeInBytes( |
| 169 | Location loc, Type type, ConversionPatternRewriter &rewriter) const { |
| 170 | // Compute the size of an individual element. This emits the MLIR equivalent |
| 171 | // of the following sizeof(...) implementation in LLVM IR: |
| 172 | // %0 = getelementptr %elementType* null, %indexType 1 |
| 173 | // %1 = ptrtoint %elementType* %0 to %indexType |
| 174 | // which is a common pattern of getting the size of a type in bytes. |
| 175 | Type llvmType = typeConverter->convertType(t: type); |
| 176 | auto convertedPtrType = LLVM::LLVMPointerType::get(context: rewriter.getContext()); |
| 177 | auto nullPtr = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: convertedPtrType); |
| 178 | auto gep = LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: convertedPtrType, elementType: llvmType, |
| 179 | basePtr: nullPtr, indices: ArrayRef<LLVM::GEPArg>{1}); |
| 180 | return LLVM::PtrToIntOp::create(builder&: rewriter, location: loc, res: getIndexType(), arg: gep); |
| 181 | } |
| 182 | |
| 183 | Value ConvertToLLVMPattern::getNumElements( |
| 184 | Location loc, MemRefType memRefType, ValueRange dynamicSizes, |
| 185 | ConversionPatternRewriter &rewriter) const { |
| 186 | assert(count(memRefType.getShape(), ShapedType::kDynamic) == |
| 187 | static_cast<ssize_t>(dynamicSizes.size()) && |
| 188 | "dynamicSizes size doesn't match dynamic sizes count in memref shape" ); |
| 189 | |
| 190 | Type indexType = getIndexType(); |
| 191 | Value numElements = memRefType.getRank() == 0 |
| 192 | ? createIndexAttrConstant(builder&: rewriter, loc, resultType: indexType, value: 1) |
| 193 | : nullptr; |
| 194 | unsigned dynamicIndex = 0; |
| 195 | |
| 196 | // Compute the total number of memref elements. |
| 197 | for (int64_t staticSize : memRefType.getShape()) { |
| 198 | if (numElements) { |
| 199 | Value size = |
| 200 | staticSize == ShapedType::kDynamic |
| 201 | ? dynamicSizes[dynamicIndex++] |
| 202 | : createIndexAttrConstant(builder&: rewriter, loc, resultType: indexType, value: staticSize); |
| 203 | numElements = LLVM::MulOp::create(builder&: rewriter, location: loc, lhs: numElements, rhs: size); |
| 204 | } else { |
| 205 | numElements = |
| 206 | staticSize == ShapedType::kDynamic |
| 207 | ? dynamicSizes[dynamicIndex++] |
| 208 | : createIndexAttrConstant(builder&: rewriter, loc, resultType: indexType, value: staticSize); |
| 209 | } |
| 210 | } |
| 211 | return numElements; |
| 212 | } |
| 213 | |
| 214 | /// Creates and populates the memref descriptor struct given all its fields. |
| 215 | MemRefDescriptor ConvertToLLVMPattern::createMemRefDescriptor( |
| 216 | Location loc, MemRefType memRefType, Value allocatedPtr, Value alignedPtr, |
| 217 | ArrayRef<Value> sizes, ArrayRef<Value> strides, |
| 218 | ConversionPatternRewriter &rewriter) const { |
| 219 | auto structType = typeConverter->convertType(t: memRefType); |
| 220 | auto memRefDescriptor = MemRefDescriptor::poison(builder&: rewriter, loc, descriptorType: structType); |
| 221 | |
| 222 | // Field 1: Allocated pointer, used for malloc/free. |
| 223 | memRefDescriptor.setAllocatedPtr(builder&: rewriter, loc, ptr: allocatedPtr); |
| 224 | |
| 225 | // Field 2: Actual aligned pointer to payload. |
| 226 | memRefDescriptor.setAlignedPtr(builder&: rewriter, loc, ptr: alignedPtr); |
| 227 | |
| 228 | // Field 3: Offset in aligned pointer. |
| 229 | Type indexType = getIndexType(); |
| 230 | memRefDescriptor.setOffset( |
| 231 | builder&: rewriter, loc, offset: createIndexAttrConstant(builder&: rewriter, loc, resultType: indexType, value: 0)); |
| 232 | |
| 233 | // Fields 4: Sizes. |
| 234 | for (const auto &en : llvm::enumerate(First&: sizes)) |
| 235 | memRefDescriptor.setSize(builder&: rewriter, loc, pos: en.index(), size: en.value()); |
| 236 | |
| 237 | // Field 5: Strides. |
| 238 | for (const auto &en : llvm::enumerate(First&: strides)) |
| 239 | memRefDescriptor.setStride(builder&: rewriter, loc, pos: en.index(), stride: en.value()); |
| 240 | |
| 241 | return memRefDescriptor; |
| 242 | } |
| 243 | |
| 244 | Value ConvertToLLVMPattern::copyUnrankedDescriptor( |
| 245 | OpBuilder &builder, Location loc, UnrankedMemRefType memRefType, |
| 246 | Value operand, bool toDynamic) const { |
| 247 | // Convert memory space. |
| 248 | FailureOr<unsigned> addressSpace = |
| 249 | getTypeConverter()->getMemRefAddressSpace(type: memRefType); |
| 250 | if (failed(Result: addressSpace)) |
| 251 | return {}; |
| 252 | |
| 253 | // Get frequently used types. |
| 254 | Type indexType = getTypeConverter()->getIndexType(); |
| 255 | |
| 256 | // Find the malloc and free, or declare them if necessary. |
| 257 | auto module = builder.getInsertionPoint()->getParentOfType<ModuleOp>(); |
| 258 | FailureOr<LLVM::LLVMFuncOp> freeFunc, mallocFunc; |
| 259 | if (toDynamic) { |
| 260 | mallocFunc = LLVM::lookupOrCreateMallocFn(b&: builder, moduleOp: module, indexType); |
| 261 | if (failed(Result: mallocFunc)) |
| 262 | return {}; |
| 263 | } |
| 264 | if (!toDynamic) { |
| 265 | freeFunc = LLVM::lookupOrCreateFreeFn(b&: builder, moduleOp: module); |
| 266 | if (failed(Result: freeFunc)) |
| 267 | return {}; |
| 268 | } |
| 269 | |
| 270 | UnrankedMemRefDescriptor desc(operand); |
| 271 | Value allocationSize = UnrankedMemRefDescriptor::computeSize( |
| 272 | builder, loc, typeConverter: *getTypeConverter(), desc, addressSpace: *addressSpace); |
| 273 | |
| 274 | // Allocate memory, copy, and free the source if necessary. |
| 275 | Value memory = toDynamic |
| 276 | ? LLVM::CallOp::create(builder, location: loc, func: mallocFunc.value(), |
| 277 | args: allocationSize) |
| 278 | .getResult() |
| 279 | : LLVM::AllocaOp::create(builder, location: loc, resultType: getPtrType(), |
| 280 | elementType: IntegerType::get(context: getContext(), width: 8), |
| 281 | arraySize: allocationSize, |
| 282 | /*alignment=*/0); |
| 283 | Value source = desc.memRefDescPtr(builder, loc); |
| 284 | LLVM::MemcpyOp::create(builder, location: loc, dst: memory, src: source, len: allocationSize, isVolatile: false); |
| 285 | if (!toDynamic) |
| 286 | LLVM::CallOp::create(builder, location: loc, func: freeFunc.value(), args: source); |
| 287 | |
| 288 | // Create a new descriptor. The same descriptor can be returned multiple |
| 289 | // times, attempting to modify its pointer can lead to memory leaks |
| 290 | // (allocated twice and overwritten) or double frees (the caller does not |
| 291 | // know if the descriptor points to the same memory). |
| 292 | Type descriptorType = getTypeConverter()->convertType(t: memRefType); |
| 293 | if (!descriptorType) |
| 294 | return {}; |
| 295 | auto updatedDesc = |
| 296 | UnrankedMemRefDescriptor::poison(builder, loc, descriptorType); |
| 297 | Value rank = desc.rank(builder, loc); |
| 298 | updatedDesc.setRank(builder, loc, value: rank); |
| 299 | updatedDesc.setMemRefDescPtr(builder, loc, value: memory); |
| 300 | return updatedDesc; |
| 301 | } |
| 302 | |
| 303 | LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors( |
| 304 | OpBuilder &builder, Location loc, TypeRange origTypes, |
| 305 | SmallVectorImpl<Value> &operands, bool toDynamic) const { |
| 306 | assert(origTypes.size() == operands.size() && |
| 307 | "expected as may original types as operands" ); |
| 308 | for (unsigned i = 0, e = operands.size(); i < e; ++i) { |
| 309 | if (auto memRefType = dyn_cast<UnrankedMemRefType>(Val: origTypes[i])) { |
| 310 | Value updatedDesc = copyUnrankedDescriptor(builder, loc, memRefType, |
| 311 | operand: operands[i], toDynamic); |
| 312 | if (!updatedDesc) |
| 313 | return failure(); |
| 314 | operands[i] = updatedDesc; |
| 315 | } |
| 316 | } |
| 317 | return success(); |
| 318 | } |
| 319 | |
| 320 | //===----------------------------------------------------------------------===// |
| 321 | // Detail methods |
| 322 | //===----------------------------------------------------------------------===// |
| 323 | |
| 324 | /// Replaces the given operation "op" with a new operation of type "targetOp" |
| 325 | /// and given operands. |
| 326 | LogicalResult LLVM::detail::oneToOneRewrite( |
| 327 | Operation *op, StringRef targetOp, ValueRange operands, |
| 328 | ArrayRef<NamedAttribute> targetAttrs, Attribute propertiesAttr, |
| 329 | const LLVMTypeConverter &typeConverter, |
| 330 | ConversionPatternRewriter &rewriter) { |
| 331 | unsigned numResults = op->getNumResults(); |
| 332 | |
| 333 | SmallVector<Type> resultTypes; |
| 334 | if (numResults != 0) { |
| 335 | resultTypes.push_back( |
| 336 | Elt: typeConverter.packOperationResults(types: op->getResultTypes())); |
| 337 | if (!resultTypes.back()) |
| 338 | return failure(); |
| 339 | } |
| 340 | |
| 341 | // Create the operation through state since we don't know its C++ type. |
| 342 | OperationState state(op->getLoc(), rewriter.getStringAttr(bytes: targetOp), operands, |
| 343 | resultTypes, targetAttrs); |
| 344 | state.propertiesAttr = propertiesAttr; |
| 345 | Operation *newOp = rewriter.create(state); |
| 346 | |
| 347 | // If the operation produced 0 or 1 result, return them immediately. |
| 348 | if (numResults == 0) |
| 349 | return rewriter.eraseOp(op), success(); |
| 350 | if (numResults == 1) |
| 351 | return rewriter.replaceOp(op, newValues: newOp->getResult(idx: 0)), success(); |
| 352 | |
| 353 | // Otherwise, it had been converted to an operation producing a structure. |
| 354 | // Extract individual results from the structure and return them as list. |
| 355 | SmallVector<Value, 4> results; |
| 356 | results.reserve(N: numResults); |
| 357 | for (unsigned i = 0; i < numResults; ++i) { |
| 358 | results.push_back(Elt: LLVM::ExtractValueOp::create(builder&: rewriter, location: op->getLoc(), |
| 359 | container: newOp->getResult(idx: 0), position: i)); |
| 360 | } |
| 361 | rewriter.replaceOp(op, newValues: results); |
| 362 | return success(); |
| 363 | } |
| 364 | |
| 365 | LogicalResult LLVM::detail::intrinsicRewrite( |
| 366 | Operation *op, StringRef intrinsic, ValueRange operands, |
| 367 | const LLVMTypeConverter &typeConverter, RewriterBase &rewriter) { |
| 368 | auto loc = op->getLoc(); |
| 369 | |
| 370 | if (!llvm::all_of(Range&: operands, P: [](Value value) { |
| 371 | return LLVM::isCompatibleType(type: value.getType()); |
| 372 | })) |
| 373 | return failure(); |
| 374 | |
| 375 | unsigned numResults = op->getNumResults(); |
| 376 | Type resType; |
| 377 | if (numResults != 0) |
| 378 | resType = typeConverter.packOperationResults(types: op->getResultTypes()); |
| 379 | |
| 380 | auto callIntrOp = LLVM::CallIntrinsicOp::create( |
| 381 | builder&: rewriter, location: loc, resultType: resType, intrin: rewriter.getStringAttr(bytes: intrinsic), args: operands); |
| 382 | // Propagate attributes. |
| 383 | SmallVector<NamedAttribute> discardableAttrs; |
| 384 | auto copyAttr = [&](StringAttr name, Attribute attr) { |
| 385 | if (callIntrOp->getInherentAttr(name).has_value()) |
| 386 | callIntrOp->setInherentAttr(name, value: attr); |
| 387 | else |
| 388 | discardableAttrs.emplace_back(Args&: name, Args&: attr); |
| 389 | }; |
| 390 | for (NamedAttribute attr : op->getDiscardableAttrDictionary()) |
| 391 | copyAttr(attr.getName(), attr.getValue()); |
| 392 | op->getName().walkInherentAttrs(op, visitor: [&](StringRef name, Attribute &attr) { |
| 393 | copyAttr(rewriter.getStringAttr(bytes: name), attr); |
| 394 | }); |
| 395 | callIntrOp->setDiscardableAttrs(discardableAttrs); |
| 396 | |
| 397 | if (numResults <= 1) { |
| 398 | // Directly replace the original op. |
| 399 | rewriter.replaceOp(op, newOp: callIntrOp); |
| 400 | return success(); |
| 401 | } |
| 402 | |
| 403 | // Extract individual results from packed structure and use them as |
| 404 | // replacements. |
| 405 | SmallVector<Value, 4> results; |
| 406 | results.reserve(N: numResults); |
| 407 | Value intrRes = callIntrOp.getResults(); |
| 408 | for (unsigned i = 0; i < numResults; ++i) |
| 409 | results.push_back(Elt: LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: intrRes, position: i)); |
| 410 | rewriter.replaceOp(op, newValues: results); |
| 411 | |
| 412 | return success(); |
| 413 | } |
| 414 | |
| 415 | static unsigned getBitWidth(Type type) { |
| 416 | if (type.isIntOrFloat()) |
| 417 | return type.getIntOrFloatBitWidth(); |
| 418 | |
| 419 | auto vec = cast<VectorType>(Val&: type); |
| 420 | assert(!vec.isScalable() && "scalable vectors are not supported" ); |
| 421 | return vec.getNumElements() * getBitWidth(type: vec.getElementType()); |
| 422 | } |
| 423 | |
| 424 | /// Returns true if every leaf in `type` (recursing through LLVM arrays and |
| 425 | /// structs) is either equal to `dstType` or has a fixed bit width. |
| 426 | static bool isFixedSizeAggregate(Type type, Type dstType) { |
| 427 | if (type == dstType) |
| 428 | return true; |
| 429 | if (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(Val&: type)) |
| 430 | return isFixedSizeAggregate(type: arrayType.getElementType(), dstType); |
| 431 | if (auto structType = dyn_cast<LLVM::LLVMStructType>(Val&: type)) |
| 432 | return llvm::all_of(Range: structType.getBody(), P: [&](Type fieldType) { |
| 433 | return isFixedSizeAggregate(type: fieldType, dstType); |
| 434 | }); |
| 435 | if (auto vecTy = dyn_cast<VectorType>(Val&: type)) |
| 436 | return !vecTy.isScalable(); |
| 437 | return type.isIntOrFloat(); |
| 438 | } |
| 439 | |
| 440 | static Value createI32Constant(OpBuilder &builder, Location loc, |
| 441 | int32_t value) { |
| 442 | Type i32 = builder.getI32Type(); |
| 443 | return LLVM::ConstantOp::create(builder, location: loc, type: i32, value); |
| 444 | } |
| 445 | |
| 446 | /// Recursive implementation of decomposeValue. When |
| 447 | /// `permitVariablySizedScalars` is false, callers must ensure |
| 448 | /// isFixedSizeAggregate() holds before calling this. |
| 449 | static void decomposeValueImpl(OpBuilder &builder, Location loc, Value src, |
| 450 | Type dstType, SmallVectorImpl<Value> &result) { |
| 451 | Type srcType = src.getType(); |
| 452 | if (srcType == dstType) { |
| 453 | result.push_back(Elt: src); |
| 454 | return; |
| 455 | } |
| 456 | |
| 457 | if (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(Val&: srcType)) { |
| 458 | for (auto i : llvm::seq(Size: arrayType.getNumElements())) { |
| 459 | Value elem = LLVM::ExtractValueOp::create(builder, location: loc, container: src, position: i); |
| 460 | decomposeValueImpl(builder, loc, src: elem, dstType, result); |
| 461 | } |
| 462 | return; |
| 463 | } |
| 464 | |
| 465 | if (auto structType = dyn_cast<LLVM::LLVMStructType>(Val&: srcType)) { |
| 466 | for (auto [i, fieldType] : llvm::enumerate(First: structType.getBody())) { |
| 467 | Value field = LLVM::ExtractValueOp::create(builder, location: loc, container: src, |
| 468 | position: static_cast<int64_t>(i)); |
| 469 | decomposeValueImpl(builder, loc, src: field, dstType, result); |
| 470 | } |
| 471 | return; |
| 472 | } |
| 473 | |
| 474 | // Variably sized leaf types (e.g., ptr) — pass through as-is. |
| 475 | if (!srcType.isIntOrFloat() && !isa<VectorType>(Val: srcType)) { |
| 476 | result.push_back(Elt: src); |
| 477 | return; |
| 478 | } |
| 479 | |
| 480 | unsigned srcBitWidth = getBitWidth(type: srcType); |
| 481 | unsigned dstBitWidth = getBitWidth(type: dstType); |
| 482 | if (srcBitWidth == dstBitWidth) { |
| 483 | Value cast = LLVM::BitcastOp::create(builder, location: loc, res: dstType, arg: src); |
| 484 | result.push_back(Elt: cast); |
| 485 | return; |
| 486 | } |
| 487 | |
| 488 | if (dstBitWidth > srcBitWidth) { |
| 489 | auto smallerInt = builder.getIntegerType(width: srcBitWidth); |
| 490 | if (srcType != smallerInt) |
| 491 | src = LLVM::BitcastOp::create(builder, location: loc, res: smallerInt, arg: src); |
| 492 | |
| 493 | auto largerInt = builder.getIntegerType(width: dstBitWidth); |
| 494 | Value res = LLVM::ZExtOp::create(builder, location: loc, res: largerInt, arg: src); |
| 495 | result.push_back(Elt: res); |
| 496 | return; |
| 497 | } |
| 498 | int64_t numElements = llvm::divideCeil(Numerator: srcBitWidth, Denominator: dstBitWidth); |
| 499 | int64_t roundedBitWidth = numElements * dstBitWidth; |
| 500 | |
| 501 | // Pad out values that don't decompose evenly before creating a vector. |
| 502 | if (roundedBitWidth != srcBitWidth) { |
| 503 | auto srcInt = builder.getIntegerType(width: srcBitWidth); |
| 504 | if (srcType != srcInt) |
| 505 | src = LLVM::BitcastOp::create(builder, location: loc, res: srcInt, arg: src); |
| 506 | auto roundedInt = builder.getIntegerType(width: roundedBitWidth); |
| 507 | src = LLVM::ZExtOp::create(builder, location: loc, res: roundedInt, arg: src); |
| 508 | } |
| 509 | |
| 510 | auto vecType = VectorType::get(shape: numElements, elementType: dstType); |
| 511 | src = LLVM::BitcastOp::create(builder, location: loc, res: vecType, arg: src); |
| 512 | |
| 513 | for (auto i : llvm::seq(Size: numElements)) { |
| 514 | Value idx = createI32Constant(builder, loc, value: i); |
| 515 | Value elem = LLVM::ExtractElementOp::create(builder, location: loc, vector: src, position: idx); |
| 516 | result.push_back(Elt: elem); |
| 517 | } |
| 518 | } |
| 519 | |
| 520 | LogicalResult mlir::LLVM::decomposeValue(OpBuilder &builder, Location loc, |
| 521 | Value src, Type dstType, |
| 522 | SmallVectorImpl<Value> &result, |
| 523 | bool permitVariablySizedScalars) { |
| 524 | // Check the type tree before emitting any IR, so that a failing pattern |
| 525 | // leaves the IR unmodified. |
| 526 | if (!permitVariablySizedScalars && |
| 527 | !isFixedSizeAggregate(type: src.getType(), dstType)) |
| 528 | return failure(); |
| 529 | |
| 530 | decomposeValueImpl(builder, loc, src, dstType, result); |
| 531 | return success(); |
| 532 | } |
| 533 | |
| 534 | /// Recursive implementation of composeValue. Consumes elements from `src` |
| 535 | /// starting at `offset`, advancing it past the consumed elements. |
| 536 | static Value composeValueImpl(OpBuilder &builder, Location loc, ValueRange src, |
| 537 | size_t &offset, Type dstType) { |
| 538 | if (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(Val&: dstType)) { |
| 539 | Value result = LLVM::PoisonOp::create(builder, location: loc, res: arrayType); |
| 540 | Type elemType = arrayType.getElementType(); |
| 541 | for (auto i : llvm::seq(Size: arrayType.getNumElements())) { |
| 542 | Value elem = composeValueImpl(builder, loc, src, offset, dstType: elemType); |
| 543 | result = LLVM::InsertValueOp::create(builder, location: loc, container: result, value: elem, position: i); |
| 544 | } |
| 545 | return result; |
| 546 | } |
| 547 | |
| 548 | if (auto structType = dyn_cast<LLVM::LLVMStructType>(Val&: dstType)) { |
| 549 | Value result = LLVM::PoisonOp::create(builder, location: loc, res: structType); |
| 550 | for (auto [i, fieldType] : llvm::enumerate(First: structType.getBody())) { |
| 551 | Value field = composeValueImpl(builder, loc, src, offset, dstType: fieldType); |
| 552 | result = LLVM::InsertValueOp::create(builder, location: loc, container: result, value: field, |
| 553 | position: static_cast<int64_t>(i)); |
| 554 | } |
| 555 | return result; |
| 556 | } |
| 557 | |
| 558 | // Variably sized leaf types (e.g., ptr) — consume and return as-is. |
| 559 | if (!dstType.isIntOrFloat() && !isa<VectorType>(Val: dstType)) |
| 560 | return src[offset++]; |
| 561 | |
| 562 | unsigned dstBitWidth = getBitWidth(type: dstType); |
| 563 | |
| 564 | Value front = src[offset]; |
| 565 | if (front.getType() == dstType) { |
| 566 | ++offset; |
| 567 | return front; |
| 568 | } |
| 569 | |
| 570 | // Single element wider than or equal to dst: bitcast/trunc. |
| 571 | if (front.getType().isIntOrFloat() || isa<VectorType>(Val: front.getType())) { |
| 572 | unsigned srcBitWidth = getBitWidth(type: front.getType()); |
| 573 | if (srcBitWidth >= dstBitWidth) { |
| 574 | ++offset; |
| 575 | Value res = front; |
| 576 | if (dstBitWidth < srcBitWidth) { |
| 577 | auto largerInt = builder.getIntegerType(width: srcBitWidth); |
| 578 | if (res.getType() != largerInt) |
| 579 | res = LLVM::BitcastOp::create(builder, location: loc, res: largerInt, arg: res); |
| 580 | |
| 581 | auto smallerInt = builder.getIntegerType(width: dstBitWidth); |
| 582 | res = LLVM::TruncOp::create(builder, location: loc, res: smallerInt, arg: res); |
| 583 | } |
| 584 | if (res.getType() != dstType) |
| 585 | res = LLVM::BitcastOp::create(builder, location: loc, res: dstType, arg: res); |
| 586 | return res; |
| 587 | } |
| 588 | } |
| 589 | |
| 590 | // Multiple elements narrower than dst: gather into a vector and bitcast. |
| 591 | unsigned elemBitWidth = getBitWidth(type: front.getType()); |
| 592 | int64_t numElements = llvm::divideCeil(Numerator: dstBitWidth, Denominator: elemBitWidth); |
| 593 | int64_t roundedBitWidth = numElements * elemBitWidth; |
| 594 | |
| 595 | auto vecType = VectorType::get(shape: numElements, elementType: front.getType()); |
| 596 | Value res = LLVM::PoisonOp::create(builder, location: loc, res: vecType); |
| 597 | for (auto i : llvm::seq(Size: numElements)) { |
| 598 | Value idx = createI32Constant(builder, loc, value: i); |
| 599 | res = LLVM::InsertElementOp::create(builder, location: loc, res: vecType, vector: res, |
| 600 | value: src[offset++], position: idx); |
| 601 | } |
| 602 | |
| 603 | // Undo any padding decomposition might have introduced. |
| 604 | if (roundedBitWidth != dstBitWidth) { |
| 605 | auto roundedInt = builder.getIntegerType(width: roundedBitWidth); |
| 606 | res = LLVM::BitcastOp::create(builder, location: loc, res: roundedInt, arg: res); |
| 607 | auto dstInt = builder.getIntegerType(width: dstBitWidth); |
| 608 | res = LLVM::TruncOp::create(builder, location: loc, res: dstInt, arg: res); |
| 609 | if (dstType != dstInt) |
| 610 | res = LLVM::BitcastOp::create(builder, location: loc, res: dstType, arg: res); |
| 611 | } else { |
| 612 | if (res.getType() != dstType) |
| 613 | res = LLVM::BitcastOp::create(builder, location: loc, res: dstType, arg: res); |
| 614 | } |
| 615 | |
| 616 | return res; |
| 617 | } |
| 618 | |
| 619 | Value mlir::LLVM::composeValue(OpBuilder &builder, Location loc, ValueRange src, |
| 620 | Type dstType) { |
| 621 | assert(!src.empty() && "src range must not be empty" ); |
| 622 | size_t offset = 0; |
| 623 | Value result = composeValueImpl(builder, loc, src, offset, dstType); |
| 624 | assert(offset == src.size() && "not all decomposed values were consumed" ); |
| 625 | return result; |
| 626 | } |
| 627 | |
| 628 | Value mlir::LLVM::getStridedElementPtr(OpBuilder &builder, Location loc, |
| 629 | const LLVMTypeConverter &converter, |
| 630 | MemRefType type, Value memRefDesc, |
| 631 | ValueRange indices, |
| 632 | LLVM::GEPNoWrapFlags noWrapFlags) { |
| 633 | auto [strides, offset] = type.getStridesAndOffset(); |
| 634 | |
| 635 | MemRefDescriptor memRefDescriptor(memRefDesc); |
| 636 | // Use a canonical representation of the start address so that later |
| 637 | // optimizations have a longer sequence of instructions to CSE. |
| 638 | // If we don't do that we would sprinkle the memref.offset in various |
| 639 | // position of the different address computations. |
| 640 | Value base = memRefDescriptor.bufferPtr(builder, loc, converter, type); |
| 641 | |
| 642 | LLVM::IntegerOverflowFlags intOverflowFlags = |
| 643 | LLVM::IntegerOverflowFlags::none; |
| 644 | if (LLVM::bitEnumContainsAny(bits: noWrapFlags, bit: LLVM::GEPNoWrapFlags::nusw)) { |
| 645 | intOverflowFlags = intOverflowFlags | LLVM::IntegerOverflowFlags::nsw; |
| 646 | } |
| 647 | if (LLVM::bitEnumContainsAny(bits: noWrapFlags, bit: LLVM::GEPNoWrapFlags::nuw)) { |
| 648 | intOverflowFlags = intOverflowFlags | LLVM::IntegerOverflowFlags::nuw; |
| 649 | } |
| 650 | |
| 651 | Type indexType = converter.getIndexType(); |
| 652 | Value index; |
| 653 | for (int i = 0, e = indices.size(); i < e; ++i) { |
| 654 | Value increment = indices[i]; |
| 655 | if (strides[i] != 1) { // Skip if stride is 1. |
| 656 | Value stride = ShapedType::isDynamic(dValue: strides[i]) |
| 657 | ? memRefDescriptor.stride(builder, loc, pos: i) |
| 658 | : LLVM::createIndexAttrConstant(builder, loc, |
| 659 | resultType: indexType, value: strides[i]); |
| 660 | increment = LLVM::MulOp::create(builder, location: loc, lhs: increment, rhs: stride, |
| 661 | overflowFlags: intOverflowFlags); |
| 662 | } |
| 663 | index = index ? LLVM::AddOp::create(builder, location: loc, lhs: index, rhs: increment, |
| 664 | overflowFlags: intOverflowFlags) |
| 665 | : increment; |
| 666 | } |
| 667 | |
| 668 | Type elementPtrType = memRefDescriptor.getElementPtrType(); |
| 669 | return index |
| 670 | ? LLVM::GEPOp::create(builder, location: loc, resultType: elementPtrType, |
| 671 | elementType: converter.convertType(t: type.getElementType()), |
| 672 | basePtr: base, indices: index, noWrapFlags) |
| 673 | : base; |
| 674 | } |
| 675 | |
| 676 | /// Return the given type if it's a floating point type. If the given type is |
| 677 | /// a vector type, return its element type if it's a floating point type. |
| 678 | static FloatType getFloatingPointType(Type type) { |
| 679 | if (auto floatType = dyn_cast<FloatType>(Val&: type)) |
| 680 | return floatType; |
| 681 | if (auto vecType = dyn_cast<VectorType>(Val&: type)) |
| 682 | return dyn_cast<FloatType>(Val: vecType.getElementType()); |
| 683 | return nullptr; |
| 684 | } |
| 685 | |
| 686 | bool LLVM::detail::isUnsupportedFloatingPointType( |
| 687 | const TypeConverter &typeConverter, Type type) { |
| 688 | FloatType floatType = getFloatingPointType(type); |
| 689 | if (!floatType) |
| 690 | return false; |
| 691 | Type convertedType = typeConverter.convertType(t: floatType); |
| 692 | if (!convertedType) |
| 693 | return true; |
| 694 | return !isa<FloatType>(Val: convertedType); |
| 695 | } |
| 696 | |
| 697 | bool LLVM::detail::opHasUnsupportedFloatingPointTypes( |
| 698 | Operation *op, const TypeConverter &typeConverter) { |
| 699 | for (Value operand : op->getOperands()) |
| 700 | if (isUnsupportedFloatingPointType(typeConverter, type: operand.getType())) |
| 701 | return true; |
| 702 | return llvm::any_of(Range: op->getResults(), P: [&typeConverter](OpResult r) { |
| 703 | return isUnsupportedFloatingPointType(typeConverter, type: r.getType()); |
| 704 | }); |
| 705 | } |
| 706 | |