1//===- Pattern.cpp - Conversion pattern to the LLVM dialect ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/LLVMCommon/Pattern.h"
10#include "mlir/Dialect/LLVMIR/FunctionCallUtils.h"
11#include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h"
12#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
13#include "mlir/Dialect/LLVMIR/LLVMCallOps.h"
14#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
15#include "mlir/Dialect/LLVMIR/LLVMConversionOps.h"
16#include "mlir/Dialect/LLVMIR/LLVMMemoryIntrinsicOps.h"
17#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
18#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
19#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
20#include "mlir/Dialect/LLVMIR/LLVMVectorOps.h"
21#include "mlir/IR/AffineMap.h"
22#include "mlir/IR/BuiltinAttributes.h"
23#include "llvm/Support/CheckedArithmetic.h"
24#include "llvm/Support/MathExtras.h"
25
26using namespace mlir;
27
28//===----------------------------------------------------------------------===//
29// ConvertToLLVMPattern
30//===----------------------------------------------------------------------===//
31
32ConvertToLLVMPattern::ConvertToLLVMPattern(
33 StringRef rootOpName, MLIRContext *context,
34 const LLVMTypeConverter &typeConverter, PatternBenefit benefit)
35 : ConversionPattern(typeConverter, rootOpName, benefit, context) {}
36
37const LLVMTypeConverter *ConvertToLLVMPattern::getTypeConverter() const {
38 return static_cast<const LLVMTypeConverter *>(
39 ConversionPattern::getTypeConverter());
40}
41
42LLVM::LLVMDialect &ConvertToLLVMPattern::getDialect() const {
43 return *getTypeConverter()->getDialect();
44}
45
46Type ConvertToLLVMPattern::getIndexType() const {
47 return getTypeConverter()->getIndexType();
48}
49
50Type ConvertToLLVMPattern::getIntPtrType(unsigned addressSpace) const {
51 return IntegerType::get(context: &getTypeConverter()->getContext(),
52 width: getTypeConverter()->getPointerBitwidth(addressSpace));
53}
54
55Type ConvertToLLVMPattern::getVoidType() const {
56 return LLVM::LLVMVoidType::get(ctx: &getTypeConverter()->getContext());
57}
58
59Type ConvertToLLVMPattern::getPtrType(unsigned addressSpace) const {
60 return LLVM::LLVMPointerType::get(context: &getTypeConverter()->getContext(),
61 addressSpace);
62}
63
64Type ConvertToLLVMPattern::getVoidPtrType() const { return getPtrType(); }
65
66Value mlir::LLVM::createIndexAttrConstant(OpBuilder &builder, Location loc,
67 Type resultType, int64_t value) {
68 return LLVM::ConstantOp::create(builder, location: loc, res: resultType,
69 value: builder.getIntegerAttr(type: resultType, value));
70}
71
72Value ConvertToLLVMPattern::createIndexAttrConstant(OpBuilder &builder,
73 Location loc,
74 Type resultType,
75 int64_t value) {
76 return LLVM::createIndexAttrConstant(builder, loc, resultType, value);
77}
78
79Value ConvertToLLVMPattern::getStridedElementPtr(
80 ConversionPatternRewriter &rewriter, Location loc, MemRefType type,
81 Value memRefDesc, ValueRange indices,
82 LLVM::GEPNoWrapFlags noWrapFlags) const {
83 return LLVM::getStridedElementPtr(builder&: rewriter, loc, converter: *getTypeConverter(), type,
84 memRefDesc, indices, noWrapFlags);
85}
86
87// Check if the MemRefType `type` is supported by the lowering. We currently
88// only support memrefs with identity maps.
89bool ConvertToLLVMPattern::isConvertibleAndHasIdentityMaps(
90 MemRefType type) const {
91 if (!type.getLayout().isIdentity())
92 return false;
93 return static_cast<bool>(typeConverter->convertType(t: type));
94}
95
96Type ConvertToLLVMPattern::getElementPtrType(MemRefType type) const {
97 auto addressSpace = getTypeConverter()->getMemRefAddressSpace(type);
98 if (failed(Result: addressSpace))
99 return {};
100 return LLVM::LLVMPointerType::get(context: type.getContext(), addressSpace: *addressSpace);
101}
102
103void ConvertToLLVMPattern::getMemRefDescriptorSizes(
104 Location loc, MemRefType memRefType, ValueRange dynamicSizes,
105 ConversionPatternRewriter &rewriter, SmallVectorImpl<Value> &sizes,
106 SmallVectorImpl<Value> &strides, Value &size, bool sizeInBytes) const {
107 assert(isConvertibleAndHasIdentityMaps(memRefType) &&
108 "layout maps must have been normalized away");
109 assert(count(memRefType.getShape(), ShapedType::kDynamic) ==
110 static_cast<ssize_t>(dynamicSizes.size()) &&
111 "dynamicSizes size doesn't match dynamic sizes count in memref shape");
112
113 sizes.reserve(N: memRefType.getRank());
114 unsigned dynamicIndex = 0;
115 Type indexType = getIndexType();
116 for (int64_t size : memRefType.getShape()) {
117 sizes.push_back(
118 Elt: size == ShapedType::kDynamic
119 ? dynamicSizes[dynamicIndex++]
120 : createIndexAttrConstant(builder&: rewriter, loc, resultType: indexType, value: size));
121 }
122
123 // Strides: iterate sizes in reverse order and multiply.
124 int64_t stride = 1;
125 bool overflowed = false;
126 Value runningStride = createIndexAttrConstant(builder&: rewriter, loc, resultType: indexType, value: 1);
127 strides.resize(N: memRefType.getRank());
128 for (auto i = memRefType.getRank(); i-- > 0;) {
129 strides[i] = overflowed ? LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: indexType)
130 : runningStride;
131
132 int64_t staticSize = memRefType.getShape()[i];
133 bool useSizeAsStride = stride == 1;
134 if (staticSize == ShapedType::kDynamic)
135 stride = ShapedType::kDynamic;
136 if (stride != ShapedType::kDynamic) {
137 std::optional<int64_t> res = llvm::checkedMul(LHS: stride, RHS: staticSize);
138
139 if (!res)
140 overflowed = true;
141 else
142 stride = res.value();
143 }
144
145 if (overflowed)
146 runningStride = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: indexType);
147 else if (useSizeAsStride)
148 runningStride = sizes[i];
149 else if (stride == ShapedType::kDynamic)
150 runningStride =
151 LLVM::MulOp::create(builder&: rewriter, location: loc, lhs: runningStride, rhs: sizes[i]);
152 else
153 runningStride = createIndexAttrConstant(builder&: rewriter, loc, resultType: indexType, value: stride);
154 }
155 if (sizeInBytes) {
156 // Buffer size in bytes.
157 Type elementType = typeConverter->convertType(t: memRefType.getElementType());
158 auto elementPtrType = LLVM::LLVMPointerType::get(context: rewriter.getContext());
159 Value nullPtr = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: elementPtrType);
160 Value gepPtr = LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: elementPtrType,
161 elementType, basePtr: nullPtr, indices: runningStride);
162 size = LLVM::PtrToIntOp::create(builder&: rewriter, location: loc, res: getIndexType(), arg: gepPtr);
163 } else {
164 size = runningStride;
165 }
166}
167
168Value ConvertToLLVMPattern::getSizeInBytes(
169 Location loc, Type type, ConversionPatternRewriter &rewriter) const {
170 // Compute the size of an individual element. This emits the MLIR equivalent
171 // of the following sizeof(...) implementation in LLVM IR:
172 // %0 = getelementptr %elementType* null, %indexType 1
173 // %1 = ptrtoint %elementType* %0 to %indexType
174 // which is a common pattern of getting the size of a type in bytes.
175 Type llvmType = typeConverter->convertType(t: type);
176 auto convertedPtrType = LLVM::LLVMPointerType::get(context: rewriter.getContext());
177 auto nullPtr = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: convertedPtrType);
178 auto gep = LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: convertedPtrType, elementType: llvmType,
179 basePtr: nullPtr, indices: ArrayRef<LLVM::GEPArg>{1});
180 return LLVM::PtrToIntOp::create(builder&: rewriter, location: loc, res: getIndexType(), arg: gep);
181}
182
183Value ConvertToLLVMPattern::getNumElements(
184 Location loc, MemRefType memRefType, ValueRange dynamicSizes,
185 ConversionPatternRewriter &rewriter) const {
186 assert(count(memRefType.getShape(), ShapedType::kDynamic) ==
187 static_cast<ssize_t>(dynamicSizes.size()) &&
188 "dynamicSizes size doesn't match dynamic sizes count in memref shape");
189
190 Type indexType = getIndexType();
191 Value numElements = memRefType.getRank() == 0
192 ? createIndexAttrConstant(builder&: rewriter, loc, resultType: indexType, value: 1)
193 : nullptr;
194 unsigned dynamicIndex = 0;
195
196 // Compute the total number of memref elements.
197 for (int64_t staticSize : memRefType.getShape()) {
198 if (numElements) {
199 Value size =
200 staticSize == ShapedType::kDynamic
201 ? dynamicSizes[dynamicIndex++]
202 : createIndexAttrConstant(builder&: rewriter, loc, resultType: indexType, value: staticSize);
203 numElements = LLVM::MulOp::create(builder&: rewriter, location: loc, lhs: numElements, rhs: size);
204 } else {
205 numElements =
206 staticSize == ShapedType::kDynamic
207 ? dynamicSizes[dynamicIndex++]
208 : createIndexAttrConstant(builder&: rewriter, loc, resultType: indexType, value: staticSize);
209 }
210 }
211 return numElements;
212}
213
214/// Creates and populates the memref descriptor struct given all its fields.
215MemRefDescriptor ConvertToLLVMPattern::createMemRefDescriptor(
216 Location loc, MemRefType memRefType, Value allocatedPtr, Value alignedPtr,
217 ArrayRef<Value> sizes, ArrayRef<Value> strides,
218 ConversionPatternRewriter &rewriter) const {
219 auto structType = typeConverter->convertType(t: memRefType);
220 auto memRefDescriptor = MemRefDescriptor::poison(builder&: rewriter, loc, descriptorType: structType);
221
222 // Field 1: Allocated pointer, used for malloc/free.
223 memRefDescriptor.setAllocatedPtr(builder&: rewriter, loc, ptr: allocatedPtr);
224
225 // Field 2: Actual aligned pointer to payload.
226 memRefDescriptor.setAlignedPtr(builder&: rewriter, loc, ptr: alignedPtr);
227
228 // Field 3: Offset in aligned pointer.
229 Type indexType = getIndexType();
230 memRefDescriptor.setOffset(
231 builder&: rewriter, loc, offset: createIndexAttrConstant(builder&: rewriter, loc, resultType: indexType, value: 0));
232
233 // Fields 4: Sizes.
234 for (const auto &en : llvm::enumerate(First&: sizes))
235 memRefDescriptor.setSize(builder&: rewriter, loc, pos: en.index(), size: en.value());
236
237 // Field 5: Strides.
238 for (const auto &en : llvm::enumerate(First&: strides))
239 memRefDescriptor.setStride(builder&: rewriter, loc, pos: en.index(), stride: en.value());
240
241 return memRefDescriptor;
242}
243
244Value ConvertToLLVMPattern::copyUnrankedDescriptor(
245 OpBuilder &builder, Location loc, UnrankedMemRefType memRefType,
246 Value operand, bool toDynamic) const {
247 // Convert memory space.
248 FailureOr<unsigned> addressSpace =
249 getTypeConverter()->getMemRefAddressSpace(type: memRefType);
250 if (failed(Result: addressSpace))
251 return {};
252
253 // Get frequently used types.
254 Type indexType = getTypeConverter()->getIndexType();
255
256 // Find the malloc and free, or declare them if necessary.
257 auto module = builder.getInsertionPoint()->getParentOfType<ModuleOp>();
258 FailureOr<LLVM::LLVMFuncOp> freeFunc, mallocFunc;
259 if (toDynamic) {
260 mallocFunc = LLVM::lookupOrCreateMallocFn(b&: builder, moduleOp: module, indexType);
261 if (failed(Result: mallocFunc))
262 return {};
263 }
264 if (!toDynamic) {
265 freeFunc = LLVM::lookupOrCreateFreeFn(b&: builder, moduleOp: module);
266 if (failed(Result: freeFunc))
267 return {};
268 }
269
270 UnrankedMemRefDescriptor desc(operand);
271 Value allocationSize = UnrankedMemRefDescriptor::computeSize(
272 builder, loc, typeConverter: *getTypeConverter(), desc, addressSpace: *addressSpace);
273
274 // Allocate memory, copy, and free the source if necessary.
275 Value memory = toDynamic
276 ? LLVM::CallOp::create(builder, location: loc, func: mallocFunc.value(),
277 args: allocationSize)
278 .getResult()
279 : LLVM::AllocaOp::create(builder, location: loc, resultType: getPtrType(),
280 elementType: IntegerType::get(context: getContext(), width: 8),
281 arraySize: allocationSize,
282 /*alignment=*/0);
283 Value source = desc.memRefDescPtr(builder, loc);
284 LLVM::MemcpyOp::create(builder, location: loc, dst: memory, src: source, len: allocationSize, isVolatile: false);
285 if (!toDynamic)
286 LLVM::CallOp::create(builder, location: loc, func: freeFunc.value(), args: source);
287
288 // Create a new descriptor. The same descriptor can be returned multiple
289 // times, attempting to modify its pointer can lead to memory leaks
290 // (allocated twice and overwritten) or double frees (the caller does not
291 // know if the descriptor points to the same memory).
292 Type descriptorType = getTypeConverter()->convertType(t: memRefType);
293 if (!descriptorType)
294 return {};
295 auto updatedDesc =
296 UnrankedMemRefDescriptor::poison(builder, loc, descriptorType);
297 Value rank = desc.rank(builder, loc);
298 updatedDesc.setRank(builder, loc, value: rank);
299 updatedDesc.setMemRefDescPtr(builder, loc, value: memory);
300 return updatedDesc;
301}
302
303LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors(
304 OpBuilder &builder, Location loc, TypeRange origTypes,
305 SmallVectorImpl<Value> &operands, bool toDynamic) const {
306 assert(origTypes.size() == operands.size() &&
307 "expected as may original types as operands");
308 for (unsigned i = 0, e = operands.size(); i < e; ++i) {
309 if (auto memRefType = dyn_cast<UnrankedMemRefType>(Val: origTypes[i])) {
310 Value updatedDesc = copyUnrankedDescriptor(builder, loc, memRefType,
311 operand: operands[i], toDynamic);
312 if (!updatedDesc)
313 return failure();
314 operands[i] = updatedDesc;
315 }
316 }
317 return success();
318}
319
320//===----------------------------------------------------------------------===//
321// Detail methods
322//===----------------------------------------------------------------------===//
323
324/// Replaces the given operation "op" with a new operation of type "targetOp"
325/// and given operands.
326LogicalResult LLVM::detail::oneToOneRewrite(
327 Operation *op, StringRef targetOp, ValueRange operands,
328 ArrayRef<NamedAttribute> targetAttrs, Attribute propertiesAttr,
329 const LLVMTypeConverter &typeConverter,
330 ConversionPatternRewriter &rewriter) {
331 unsigned numResults = op->getNumResults();
332
333 SmallVector<Type> resultTypes;
334 if (numResults != 0) {
335 resultTypes.push_back(
336 Elt: typeConverter.packOperationResults(types: op->getResultTypes()));
337 if (!resultTypes.back())
338 return failure();
339 }
340
341 // Create the operation through state since we don't know its C++ type.
342 OperationState state(op->getLoc(), rewriter.getStringAttr(bytes: targetOp), operands,
343 resultTypes, targetAttrs);
344 state.propertiesAttr = propertiesAttr;
345 Operation *newOp = rewriter.create(state);
346
347 // If the operation produced 0 or 1 result, return them immediately.
348 if (numResults == 0)
349 return rewriter.eraseOp(op), success();
350 if (numResults == 1)
351 return rewriter.replaceOp(op, newValues: newOp->getResult(idx: 0)), success();
352
353 // Otherwise, it had been converted to an operation producing a structure.
354 // Extract individual results from the structure and return them as list.
355 SmallVector<Value, 4> results;
356 results.reserve(N: numResults);
357 for (unsigned i = 0; i < numResults; ++i) {
358 results.push_back(Elt: LLVM::ExtractValueOp::create(builder&: rewriter, location: op->getLoc(),
359 container: newOp->getResult(idx: 0), position: i));
360 }
361 rewriter.replaceOp(op, newValues: results);
362 return success();
363}
364
365LogicalResult LLVM::detail::intrinsicRewrite(
366 Operation *op, StringRef intrinsic, ValueRange operands,
367 const LLVMTypeConverter &typeConverter, RewriterBase &rewriter) {
368 auto loc = op->getLoc();
369
370 if (!llvm::all_of(Range&: operands, P: [](Value value) {
371 return LLVM::isCompatibleType(type: value.getType());
372 }))
373 return failure();
374
375 unsigned numResults = op->getNumResults();
376 Type resType;
377 if (numResults != 0)
378 resType = typeConverter.packOperationResults(types: op->getResultTypes());
379
380 auto callIntrOp = LLVM::CallIntrinsicOp::create(
381 builder&: rewriter, location: loc, resultType: resType, intrin: rewriter.getStringAttr(bytes: intrinsic), args: operands);
382 // Propagate attributes.
383 SmallVector<NamedAttribute> discardableAttrs;
384 auto copyAttr = [&](StringAttr name, Attribute attr) {
385 if (callIntrOp->getInherentAttr(name).has_value())
386 callIntrOp->setInherentAttr(name, value: attr);
387 else
388 discardableAttrs.emplace_back(Args&: name, Args&: attr);
389 };
390 for (NamedAttribute attr : op->getDiscardableAttrDictionary())
391 copyAttr(attr.getName(), attr.getValue());
392 op->getName().walkInherentAttrs(op, visitor: [&](StringRef name, Attribute &attr) {
393 copyAttr(rewriter.getStringAttr(bytes: name), attr);
394 });
395 callIntrOp->setDiscardableAttrs(discardableAttrs);
396
397 if (numResults <= 1) {
398 // Directly replace the original op.
399 rewriter.replaceOp(op, newOp: callIntrOp);
400 return success();
401 }
402
403 // Extract individual results from packed structure and use them as
404 // replacements.
405 SmallVector<Value, 4> results;
406 results.reserve(N: numResults);
407 Value intrRes = callIntrOp.getResults();
408 for (unsigned i = 0; i < numResults; ++i)
409 results.push_back(Elt: LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: intrRes, position: i));
410 rewriter.replaceOp(op, newValues: results);
411
412 return success();
413}
414
415static unsigned getBitWidth(Type type) {
416 if (type.isIntOrFloat())
417 return type.getIntOrFloatBitWidth();
418
419 auto vec = cast<VectorType>(Val&: type);
420 assert(!vec.isScalable() && "scalable vectors are not supported");
421 return vec.getNumElements() * getBitWidth(type: vec.getElementType());
422}
423
424/// Returns true if every leaf in `type` (recursing through LLVM arrays and
425/// structs) is either equal to `dstType` or has a fixed bit width.
426static bool isFixedSizeAggregate(Type type, Type dstType) {
427 if (type == dstType)
428 return true;
429 if (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(Val&: type))
430 return isFixedSizeAggregate(type: arrayType.getElementType(), dstType);
431 if (auto structType = dyn_cast<LLVM::LLVMStructType>(Val&: type))
432 return llvm::all_of(Range: structType.getBody(), P: [&](Type fieldType) {
433 return isFixedSizeAggregate(type: fieldType, dstType);
434 });
435 if (auto vecTy = dyn_cast<VectorType>(Val&: type))
436 return !vecTy.isScalable();
437 return type.isIntOrFloat();
438}
439
440static Value createI32Constant(OpBuilder &builder, Location loc,
441 int32_t value) {
442 Type i32 = builder.getI32Type();
443 return LLVM::ConstantOp::create(builder, location: loc, type: i32, value);
444}
445
446/// Recursive implementation of decomposeValue. When
447/// `permitVariablySizedScalars` is false, callers must ensure
448/// isFixedSizeAggregate() holds before calling this.
449static void decomposeValueImpl(OpBuilder &builder, Location loc, Value src,
450 Type dstType, SmallVectorImpl<Value> &result) {
451 Type srcType = src.getType();
452 if (srcType == dstType) {
453 result.push_back(Elt: src);
454 return;
455 }
456
457 if (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(Val&: srcType)) {
458 for (auto i : llvm::seq(Size: arrayType.getNumElements())) {
459 Value elem = LLVM::ExtractValueOp::create(builder, location: loc, container: src, position: i);
460 decomposeValueImpl(builder, loc, src: elem, dstType, result);
461 }
462 return;
463 }
464
465 if (auto structType = dyn_cast<LLVM::LLVMStructType>(Val&: srcType)) {
466 for (auto [i, fieldType] : llvm::enumerate(First: structType.getBody())) {
467 Value field = LLVM::ExtractValueOp::create(builder, location: loc, container: src,
468 position: static_cast<int64_t>(i));
469 decomposeValueImpl(builder, loc, src: field, dstType, result);
470 }
471 return;
472 }
473
474 // Variably sized leaf types (e.g., ptr) — pass through as-is.
475 if (!srcType.isIntOrFloat() && !isa<VectorType>(Val: srcType)) {
476 result.push_back(Elt: src);
477 return;
478 }
479
480 unsigned srcBitWidth = getBitWidth(type: srcType);
481 unsigned dstBitWidth = getBitWidth(type: dstType);
482 if (srcBitWidth == dstBitWidth) {
483 Value cast = LLVM::BitcastOp::create(builder, location: loc, res: dstType, arg: src);
484 result.push_back(Elt: cast);
485 return;
486 }
487
488 if (dstBitWidth > srcBitWidth) {
489 auto smallerInt = builder.getIntegerType(width: srcBitWidth);
490 if (srcType != smallerInt)
491 src = LLVM::BitcastOp::create(builder, location: loc, res: smallerInt, arg: src);
492
493 auto largerInt = builder.getIntegerType(width: dstBitWidth);
494 Value res = LLVM::ZExtOp::create(builder, location: loc, res: largerInt, arg: src);
495 result.push_back(Elt: res);
496 return;
497 }
498 int64_t numElements = llvm::divideCeil(Numerator: srcBitWidth, Denominator: dstBitWidth);
499 int64_t roundedBitWidth = numElements * dstBitWidth;
500
501 // Pad out values that don't decompose evenly before creating a vector.
502 if (roundedBitWidth != srcBitWidth) {
503 auto srcInt = builder.getIntegerType(width: srcBitWidth);
504 if (srcType != srcInt)
505 src = LLVM::BitcastOp::create(builder, location: loc, res: srcInt, arg: src);
506 auto roundedInt = builder.getIntegerType(width: roundedBitWidth);
507 src = LLVM::ZExtOp::create(builder, location: loc, res: roundedInt, arg: src);
508 }
509
510 auto vecType = VectorType::get(shape: numElements, elementType: dstType);
511 src = LLVM::BitcastOp::create(builder, location: loc, res: vecType, arg: src);
512
513 for (auto i : llvm::seq(Size: numElements)) {
514 Value idx = createI32Constant(builder, loc, value: i);
515 Value elem = LLVM::ExtractElementOp::create(builder, location: loc, vector: src, position: idx);
516 result.push_back(Elt: elem);
517 }
518}
519
520LogicalResult mlir::LLVM::decomposeValue(OpBuilder &builder, Location loc,
521 Value src, Type dstType,
522 SmallVectorImpl<Value> &result,
523 bool permitVariablySizedScalars) {
524 // Check the type tree before emitting any IR, so that a failing pattern
525 // leaves the IR unmodified.
526 if (!permitVariablySizedScalars &&
527 !isFixedSizeAggregate(type: src.getType(), dstType))
528 return failure();
529
530 decomposeValueImpl(builder, loc, src, dstType, result);
531 return success();
532}
533
534/// Recursive implementation of composeValue. Consumes elements from `src`
535/// starting at `offset`, advancing it past the consumed elements.
536static Value composeValueImpl(OpBuilder &builder, Location loc, ValueRange src,
537 size_t &offset, Type dstType) {
538 if (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(Val&: dstType)) {
539 Value result = LLVM::PoisonOp::create(builder, location: loc, res: arrayType);
540 Type elemType = arrayType.getElementType();
541 for (auto i : llvm::seq(Size: arrayType.getNumElements())) {
542 Value elem = composeValueImpl(builder, loc, src, offset, dstType: elemType);
543 result = LLVM::InsertValueOp::create(builder, location: loc, container: result, value: elem, position: i);
544 }
545 return result;
546 }
547
548 if (auto structType = dyn_cast<LLVM::LLVMStructType>(Val&: dstType)) {
549 Value result = LLVM::PoisonOp::create(builder, location: loc, res: structType);
550 for (auto [i, fieldType] : llvm::enumerate(First: structType.getBody())) {
551 Value field = composeValueImpl(builder, loc, src, offset, dstType: fieldType);
552 result = LLVM::InsertValueOp::create(builder, location: loc, container: result, value: field,
553 position: static_cast<int64_t>(i));
554 }
555 return result;
556 }
557
558 // Variably sized leaf types (e.g., ptr) — consume and return as-is.
559 if (!dstType.isIntOrFloat() && !isa<VectorType>(Val: dstType))
560 return src[offset++];
561
562 unsigned dstBitWidth = getBitWidth(type: dstType);
563
564 Value front = src[offset];
565 if (front.getType() == dstType) {
566 ++offset;
567 return front;
568 }
569
570 // Single element wider than or equal to dst: bitcast/trunc.
571 if (front.getType().isIntOrFloat() || isa<VectorType>(Val: front.getType())) {
572 unsigned srcBitWidth = getBitWidth(type: front.getType());
573 if (srcBitWidth >= dstBitWidth) {
574 ++offset;
575 Value res = front;
576 if (dstBitWidth < srcBitWidth) {
577 auto largerInt = builder.getIntegerType(width: srcBitWidth);
578 if (res.getType() != largerInt)
579 res = LLVM::BitcastOp::create(builder, location: loc, res: largerInt, arg: res);
580
581 auto smallerInt = builder.getIntegerType(width: dstBitWidth);
582 res = LLVM::TruncOp::create(builder, location: loc, res: smallerInt, arg: res);
583 }
584 if (res.getType() != dstType)
585 res = LLVM::BitcastOp::create(builder, location: loc, res: dstType, arg: res);
586 return res;
587 }
588 }
589
590 // Multiple elements narrower than dst: gather into a vector and bitcast.
591 unsigned elemBitWidth = getBitWidth(type: front.getType());
592 int64_t numElements = llvm::divideCeil(Numerator: dstBitWidth, Denominator: elemBitWidth);
593 int64_t roundedBitWidth = numElements * elemBitWidth;
594
595 auto vecType = VectorType::get(shape: numElements, elementType: front.getType());
596 Value res = LLVM::PoisonOp::create(builder, location: loc, res: vecType);
597 for (auto i : llvm::seq(Size: numElements)) {
598 Value idx = createI32Constant(builder, loc, value: i);
599 res = LLVM::InsertElementOp::create(builder, location: loc, res: vecType, vector: res,
600 value: src[offset++], position: idx);
601 }
602
603 // Undo any padding decomposition might have introduced.
604 if (roundedBitWidth != dstBitWidth) {
605 auto roundedInt = builder.getIntegerType(width: roundedBitWidth);
606 res = LLVM::BitcastOp::create(builder, location: loc, res: roundedInt, arg: res);
607 auto dstInt = builder.getIntegerType(width: dstBitWidth);
608 res = LLVM::TruncOp::create(builder, location: loc, res: dstInt, arg: res);
609 if (dstType != dstInt)
610 res = LLVM::BitcastOp::create(builder, location: loc, res: dstType, arg: res);
611 } else {
612 if (res.getType() != dstType)
613 res = LLVM::BitcastOp::create(builder, location: loc, res: dstType, arg: res);
614 }
615
616 return res;
617}
618
619Value mlir::LLVM::composeValue(OpBuilder &builder, Location loc, ValueRange src,
620 Type dstType) {
621 assert(!src.empty() && "src range must not be empty");
622 size_t offset = 0;
623 Value result = composeValueImpl(builder, loc, src, offset, dstType);
624 assert(offset == src.size() && "not all decomposed values were consumed");
625 return result;
626}
627
628Value mlir::LLVM::getStridedElementPtr(OpBuilder &builder, Location loc,
629 const LLVMTypeConverter &converter,
630 MemRefType type, Value memRefDesc,
631 ValueRange indices,
632 LLVM::GEPNoWrapFlags noWrapFlags) {
633 auto [strides, offset] = type.getStridesAndOffset();
634
635 MemRefDescriptor memRefDescriptor(memRefDesc);
636 // Use a canonical representation of the start address so that later
637 // optimizations have a longer sequence of instructions to CSE.
638 // If we don't do that we would sprinkle the memref.offset in various
639 // position of the different address computations.
640 Value base = memRefDescriptor.bufferPtr(builder, loc, converter, type);
641
642 LLVM::IntegerOverflowFlags intOverflowFlags =
643 LLVM::IntegerOverflowFlags::none;
644 if (LLVM::bitEnumContainsAny(bits: noWrapFlags, bit: LLVM::GEPNoWrapFlags::nusw)) {
645 intOverflowFlags = intOverflowFlags | LLVM::IntegerOverflowFlags::nsw;
646 }
647 if (LLVM::bitEnumContainsAny(bits: noWrapFlags, bit: LLVM::GEPNoWrapFlags::nuw)) {
648 intOverflowFlags = intOverflowFlags | LLVM::IntegerOverflowFlags::nuw;
649 }
650
651 Type indexType = converter.getIndexType();
652 Value index;
653 for (int i = 0, e = indices.size(); i < e; ++i) {
654 Value increment = indices[i];
655 if (strides[i] != 1) { // Skip if stride is 1.
656 Value stride = ShapedType::isDynamic(dValue: strides[i])
657 ? memRefDescriptor.stride(builder, loc, pos: i)
658 : LLVM::createIndexAttrConstant(builder, loc,
659 resultType: indexType, value: strides[i]);
660 increment = LLVM::MulOp::create(builder, location: loc, lhs: increment, rhs: stride,
661 overflowFlags: intOverflowFlags);
662 }
663 index = index ? LLVM::AddOp::create(builder, location: loc, lhs: index, rhs: increment,
664 overflowFlags: intOverflowFlags)
665 : increment;
666 }
667
668 Type elementPtrType = memRefDescriptor.getElementPtrType();
669 return index
670 ? LLVM::GEPOp::create(builder, location: loc, resultType: elementPtrType,
671 elementType: converter.convertType(t: type.getElementType()),
672 basePtr: base, indices: index, noWrapFlags)
673 : base;
674}
675
676/// Return the given type if it's a floating point type. If the given type is
677/// a vector type, return its element type if it's a floating point type.
678static FloatType getFloatingPointType(Type type) {
679 if (auto floatType = dyn_cast<FloatType>(Val&: type))
680 return floatType;
681 if (auto vecType = dyn_cast<VectorType>(Val&: type))
682 return dyn_cast<FloatType>(Val: vecType.getElementType());
683 return nullptr;
684}
685
686bool LLVM::detail::isUnsupportedFloatingPointType(
687 const TypeConverter &typeConverter, Type type) {
688 FloatType floatType = getFloatingPointType(type);
689 if (!floatType)
690 return false;
691 Type convertedType = typeConverter.convertType(t: floatType);
692 if (!convertedType)
693 return true;
694 return !isa<FloatType>(Val: convertedType);
695}
696
697bool LLVM::detail::opHasUnsupportedFloatingPointTypes(
698 Operation *op, const TypeConverter &typeConverter) {
699 for (Value operand : op->getOperands())
700 if (isUnsupportedFloatingPointType(typeConverter, type: operand.getType()))
701 return true;
702 return llvm::any_of(Range: op->getResults(), P: [&typeConverter](OpResult r) {
703 return isUnsupportedFloatingPointType(typeConverter, type: r.getType());
704 });
705}
706