1//===- PtrToLLVM.cpp - Ptr to LLVM dialect conversion ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/PtrToLLVM/PtrToLLVM.h"
10
11#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
12#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
13#include "mlir/Conversion/LLVMCommon/Pattern.h"
14#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
15#include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h"
16#include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
17#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
18#include "mlir/Dialect/LLVMIR/LLVMConversionOps.h"
19#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
20#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
21#include "mlir/Dialect/Ptr/IR/PtrAttrs.h"
22#include "mlir/Dialect/Ptr/IR/PtrOps.h"
23#include "mlir/IR/TypeUtilities.h"
24#include "llvm/Support/LogicalResult.h"
25#include <type_traits>
26
27using namespace mlir;
28
29namespace {
30//===----------------------------------------------------------------------===//
31// FromPtrOpConversion
32//===----------------------------------------------------------------------===//
33struct FromPtrOpConversion : public ConvertOpToLLVMPattern<ptr::FromPtrOp> {
34 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
35 LogicalResult
36 matchAndRewrite(ptr::FromPtrOp op, OpAdaptor adaptor,
37 ConversionPatternRewriter &rewriter) const override;
38};
39
40//===----------------------------------------------------------------------===//
41// GetMetadataOpConversion
42//===----------------------------------------------------------------------===//
43struct GetMetadataOpConversion
44 : public ConvertOpToLLVMPattern<ptr::GetMetadataOp> {
45 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
46 LogicalResult
47 matchAndRewrite(ptr::GetMetadataOp op, OpAdaptor adaptor,
48 ConversionPatternRewriter &rewriter) const override;
49};
50
51//===----------------------------------------------------------------------===//
52// PtrAddOpConversion
53//===----------------------------------------------------------------------===//
54struct PtrAddOpConversion : public ConvertOpToLLVMPattern<ptr::PtrAddOp> {
55 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
56 LogicalResult
57 matchAndRewrite(ptr::PtrAddOp op, OpAdaptor adaptor,
58 ConversionPatternRewriter &rewriter) const override;
59};
60
61//===----------------------------------------------------------------------===//
62// ToPtrOpConversion
63//===----------------------------------------------------------------------===//
64struct ToPtrOpConversion : public ConvertOpToLLVMPattern<ptr::ToPtrOp> {
65 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
66 LogicalResult
67 matchAndRewrite(ptr::ToPtrOp op, OpAdaptor adaptor,
68 ConversionPatternRewriter &rewriter) const override;
69};
70
71//===----------------------------------------------------------------------===//
72// TypeOffsetOpConversion
73//===----------------------------------------------------------------------===//
74struct TypeOffsetOpConversion
75 : public ConvertOpToLLVMPattern<ptr::TypeOffsetOp> {
76 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
77 LogicalResult
78 matchAndRewrite(ptr::TypeOffsetOp op, OpAdaptor adaptor,
79 ConversionPatternRewriter &rewriter) const override;
80};
81
82//===----------------------------------------------------------------------===//
83// ConstantOpConversion
84//===----------------------------------------------------------------------===//
85struct ConstantOpConversion : public ConvertOpToLLVMPattern<ptr::ConstantOp> {
86 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
87 LogicalResult
88 matchAndRewrite(ptr::ConstantOp op, OpAdaptor adaptor,
89 ConversionPatternRewriter &rewriter) const override;
90};
91} // namespace
92
93//===----------------------------------------------------------------------===//
94// Internal functions
95//===----------------------------------------------------------------------===//
96
97// Function to create an LLVM struct type representing a memref metadata.
98static FailureOr<LLVM::LLVMStructType>
99createMemRefMetadataType(MemRefType type,
100 const LLVMTypeConverter &typeConverter) {
101 MLIRContext *context = type.getContext();
102 // Get the address space.
103 FailureOr<unsigned> addressSpace = typeConverter.getMemRefAddressSpace(type);
104 if (failed(Result: addressSpace))
105 return failure();
106
107 // Get pointer type (using address space 0 by default)
108 auto ptrType = LLVM::LLVMPointerType::get(context, addressSpace: *addressSpace);
109
110 // Get the strides offsets and shape.
111 SmallVector<int64_t> strides;
112 int64_t offset;
113 if (failed(Result: type.getStridesAndOffset(strides, offset)))
114 return failure();
115 ArrayRef<int64_t> shape = type.getShape();
116
117 // Use index type from the type converter for the descriptor elements
118 Type indexType = typeConverter.getIndexType();
119
120 // For a ranked memref, the descriptor contains:
121 // 1. The pointer to the allocated data
122 // 2. The pointer to the aligned data
123 // 3. The dynamic offset?
124 // 4. The dynamic sizes?
125 // 5. The dynamic strides?
126 SmallVector<Type, 5> elements;
127
128 // Allocated pointer.
129 elements.push_back(Elt: ptrType);
130
131 // Potentially add the dynamic offset.
132 if (offset == ShapedType::kDynamic)
133 elements.push_back(Elt: indexType);
134
135 // Potentially add the dynamic sizes.
136 for (int64_t dim : shape) {
137 if (dim == ShapedType::kDynamic)
138 elements.push_back(Elt: indexType);
139 }
140
141 // Potentially add the dynamic strides.
142 for (int64_t stride : strides) {
143 if (stride == ShapedType::kDynamic)
144 elements.push_back(Elt: indexType);
145 }
146 return LLVM::LLVMStructType::getLiteral(context, types: elements);
147}
148
149//===----------------------------------------------------------------------===//
150// FromPtrOpConversion
151//===----------------------------------------------------------------------===//
152
153LogicalResult FromPtrOpConversion::matchAndRewrite(
154 ptr::FromPtrOp op, OpAdaptor adaptor,
155 ConversionPatternRewriter &rewriter) const {
156 // Get the target memref type
157 auto mTy = dyn_cast<MemRefType>(Val: op.getResult().getType());
158 if (!mTy)
159 return rewriter.notifyMatchFailure(arg&: op, msg: "Expected memref result type");
160
161 if (!op.getMetadata() && op.getType().hasPtrMetadata()) {
162 return rewriter.notifyMatchFailure(
163 arg&: op, msg: "Can convert only memrefs with metadata");
164 }
165
166 // Convert the result type
167 Type descriptorTy = getTypeConverter()->convertType(t: mTy);
168 if (!descriptorTy)
169 return rewriter.notifyMatchFailure(arg&: op, msg: "Failed to convert result type");
170
171 // Get the strides, offsets and shape.
172 SmallVector<int64_t> strides;
173 int64_t offset;
174 if (failed(Result: mTy.getStridesAndOffset(strides, offset))) {
175 return rewriter.notifyMatchFailure(arg&: op,
176 msg: "Failed to get the strides and offset");
177 }
178 ArrayRef<int64_t> shape = mTy.getShape();
179
180 // Create a new memref descriptor
181 Location loc = op.getLoc();
182 auto desc = MemRefDescriptor::poison(builder&: rewriter, loc, descriptorType: descriptorTy);
183
184 // Set the allocated and aligned pointers.
185 desc.setAllocatedPtr(
186 builder&: rewriter, loc,
187 ptr: LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: adaptor.getMetadata(), position: 0));
188 desc.setAlignedPtr(builder&: rewriter, loc, ptr: adaptor.getPtr());
189
190 // Extract metadata from the passed struct.
191 unsigned fieldIdx = 1;
192
193 // Set dynamic offset if needed.
194 if (offset == ShapedType::kDynamic) {
195 Value offsetValue = LLVM::ExtractValueOp::create(
196 builder&: rewriter, location: loc, container: adaptor.getMetadata(), position: fieldIdx++);
197 desc.setOffset(builder&: rewriter, loc, offset: offsetValue);
198 } else {
199 desc.setConstantOffset(builder&: rewriter, loc, offset);
200 }
201
202 // Set dynamic sizes if needed.
203 for (auto [i, dim] : llvm::enumerate(First&: shape)) {
204 if (dim == ShapedType::kDynamic) {
205 Value sizeValue = LLVM::ExtractValueOp::create(
206 builder&: rewriter, location: loc, container: adaptor.getMetadata(), position: fieldIdx++);
207 desc.setSize(builder&: rewriter, loc, pos: i, size: sizeValue);
208 } else {
209 desc.setConstantSize(builder&: rewriter, loc, pos: i, size: dim);
210 }
211 }
212
213 // Set dynamic strides if needed.
214 for (auto [i, stride] : llvm::enumerate(First&: strides)) {
215 if (stride == ShapedType::kDynamic) {
216 Value strideValue = LLVM::ExtractValueOp::create(
217 builder&: rewriter, location: loc, container: adaptor.getMetadata(), position: fieldIdx++);
218 desc.setStride(builder&: rewriter, loc, pos: i, stride: strideValue);
219 } else {
220 desc.setConstantStride(builder&: rewriter, loc, pos: i, stride);
221 }
222 }
223
224 rewriter.replaceOp(op, newValues: static_cast<Value>(desc));
225 return success();
226}
227
228//===----------------------------------------------------------------------===//
229// GetMetadataOpConversion
230//===----------------------------------------------------------------------===//
231
232LogicalResult GetMetadataOpConversion::matchAndRewrite(
233 ptr::GetMetadataOp op, OpAdaptor adaptor,
234 ConversionPatternRewriter &rewriter) const {
235 auto mTy = dyn_cast<MemRefType>(Val: op.getPtr().getType());
236 if (!mTy)
237 return rewriter.notifyMatchFailure(arg&: op, msg: "Only memref metadata is supported");
238
239 // Get the metadata type.
240 FailureOr<LLVM::LLVMStructType> mdTy =
241 createMemRefMetadataType(type: mTy, typeConverter: *getTypeConverter());
242 if (failed(Result: mdTy)) {
243 return rewriter.notifyMatchFailure(arg&: op,
244 msg: "Failed to create the metadata type");
245 }
246
247 // Get the memref descriptor.
248 MemRefDescriptor descriptor(adaptor.getPtr());
249
250 // Get the strides offsets and shape.
251 SmallVector<int64_t> strides;
252 int64_t offset;
253 if (failed(Result: mTy.getStridesAndOffset(strides, offset))) {
254 return rewriter.notifyMatchFailure(arg&: op,
255 msg: "Failed to get the strides and offset");
256 }
257 ArrayRef<int64_t> shape = mTy.getShape();
258
259 // Create a new LLVM struct to hold the metadata
260 Location loc = op.getLoc();
261 Value sV = LLVM::UndefOp::create(builder&: rewriter, location: loc, res: *mdTy);
262
263 // First element is the allocated pointer.
264 SmallVector<int64_t> pos{0};
265 sV = LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: sV,
266 value: descriptor.allocatedPtr(builder&: rewriter, loc), position: pos);
267
268 // Track the current field index.
269 unsigned fieldIdx = 1;
270
271 // Add dynamic offset if needed.
272 if (offset == ShapedType::kDynamic) {
273 sV = LLVM::InsertValueOp::create(
274 builder&: rewriter, location: loc, container: sV, value: descriptor.offset(builder&: rewriter, loc), position: fieldIdx++);
275 }
276
277 // Add dynamic sizes if needed.
278 for (auto [i, dim] : llvm::enumerate(First&: shape)) {
279 if (dim != ShapedType::kDynamic)
280 continue;
281 sV = LLVM::InsertValueOp::create(
282 builder&: rewriter, location: loc, container: sV, value: descriptor.size(builder&: rewriter, loc, pos: i), position: fieldIdx++);
283 }
284
285 // Add dynamic strides if needed
286 for (auto [i, stride] : llvm::enumerate(First&: strides)) {
287 if (stride != ShapedType::kDynamic)
288 continue;
289 sV = LLVM::InsertValueOp::create(
290 builder&: rewriter, location: loc, container: sV, value: descriptor.stride(builder&: rewriter, loc, pos: i), position: fieldIdx++);
291 }
292 rewriter.replaceOp(op, newValues: sV);
293 return success();
294}
295
296//===----------------------------------------------------------------------===//
297// PtrAddOpConversion
298//===----------------------------------------------------------------------===//
299
300LogicalResult
301PtrAddOpConversion::matchAndRewrite(ptr::PtrAddOp op, OpAdaptor adaptor,
302 ConversionPatternRewriter &rewriter) const {
303 // Get and check the base.
304 Value base = adaptor.getBase();
305 if (!isa<LLVM::LLVMPointerType>(Val: getElementTypeOrSelf(type: base.getType())))
306 return rewriter.notifyMatchFailure(arg&: op, msg: "Incompatible pointer type");
307
308 // A vector offset can broadcast a scalar base to a vector result.
309 Type resultType = getTypeConverter()->convertType(t: op.getType());
310 if (!resultType ||
311 !isa<LLVM::LLVMPointerType>(Val: getElementTypeOrSelf(type: resultType)))
312 return rewriter.notifyMatchFailure(arg&: op, msg: "Incompatible result type");
313
314 // Get the offset.
315 Value offset = adaptor.getOffset();
316
317 // Ptr assumes the offset is in bytes.
318 Type elementType = IntegerType::get(context: rewriter.getContext(), width: 8);
319
320 // Convert the `ptradd` flags.
321 LLVM::GEPNoWrapFlags flags;
322 switch (op.getFlags()) {
323 case ptr::PtrAddFlags::none:
324 flags = LLVM::GEPNoWrapFlags::none;
325 break;
326 case ptr::PtrAddFlags::nusw:
327 flags = LLVM::GEPNoWrapFlags::nusw;
328 break;
329 case ptr::PtrAddFlags::nuw:
330 flags = LLVM::GEPNoWrapFlags::nuw;
331 break;
332 case ptr::PtrAddFlags::inbounds:
333 flags = LLVM::GEPNoWrapFlags::inbounds;
334 break;
335 }
336
337 // Create the GEP operation with appropriate arguments
338 rewriter.replaceOpWithNewOp<LLVM::GEPOp>(op, args&: resultType, args&: elementType, args&: base,
339 args: ValueRange{offset}, args&: flags);
340 return success();
341}
342
343//===----------------------------------------------------------------------===//
344// ToPtrOpConversion
345//===----------------------------------------------------------------------===//
346
347LogicalResult
348ToPtrOpConversion::matchAndRewrite(ptr::ToPtrOp op, OpAdaptor adaptor,
349 ConversionPatternRewriter &rewriter) const {
350 // Bail if it's not a memref.
351 if (!isa<MemRefType>(Val: op.getPtr().getType()))
352 return rewriter.notifyMatchFailure(arg&: op, msg: "Expected a memref input");
353
354 // Extract the aligned pointer from the memref descriptor.
355 rewriter.replaceOp(
356 op, newValues: MemRefDescriptor(adaptor.getPtr()).alignedPtr(builder&: rewriter, loc: op.getLoc()));
357 return success();
358}
359
360//===----------------------------------------------------------------------===//
361// TypeOffsetOpConversion
362//===----------------------------------------------------------------------===//
363
364LogicalResult TypeOffsetOpConversion::matchAndRewrite(
365 ptr::TypeOffsetOp op, OpAdaptor adaptor,
366 ConversionPatternRewriter &rewriter) const {
367 // Convert the type attribute.
368 Type type = getTypeConverter()->convertType(t: op.getElementType());
369 if (!type)
370 return rewriter.notifyMatchFailure(arg&: op, msg: "Couldn't convert the type");
371
372 // Convert the result type.
373 Type rTy = getTypeConverter()->convertType(t: op.getResult().getType());
374 if (!rTy)
375 return rewriter.notifyMatchFailure(arg&: op, msg: "Couldn't convert the result type");
376
377 // TODO: Use MLIR's data layout. We don't use it because overall support is
378 // still flaky.
379
380 // Create an LLVM pointer type for the GEP operation.
381 auto ptrTy = LLVM::LLVMPointerType::get(context: getContext());
382
383 // Create a GEP operation to compute the offset of the type.
384 auto offset =
385 LLVM::GEPOp::create(builder&: rewriter, location: op.getLoc(), resultType: ptrTy, elementType: type,
386 basePtr: LLVM::ZeroOp::create(builder&: rewriter, location: op.getLoc(), res: ptrTy),
387 indices: ArrayRef<LLVM::GEPArg>({LLVM::GEPArg(1)}));
388
389 // Replace the original op with a PtrToIntOp using the computed offset.
390 rewriter.replaceOpWithNewOp<LLVM::PtrToIntOp>(op, args&: rTy, args: offset.getRes());
391 return success();
392}
393
394//===----------------------------------------------------------------------===//
395// ConstantOpConversion
396//===----------------------------------------------------------------------===//
397
398LogicalResult ConstantOpConversion::matchAndRewrite(
399 ptr::ConstantOp op, OpAdaptor adaptor,
400 ConversionPatternRewriter &rewriter) const {
401 TypedAttr value = op.getValue();
402 Type resultType = getTypeConverter()->convertType(t: op.getType());
403 if (!resultType)
404 return rewriter.notifyMatchFailure(arg&: op, msg: "Couldn't convert the result type");
405
406 if (isa<ptr::NullAttr>(Val: value)) {
407 rewriter.replaceOpWithNewOp<LLVM::ZeroOp>(op, args&: resultType);
408 return success();
409 }
410 auto addrAttr = dyn_cast<ptr::AddressAttr>(Val&: value);
411 // Early-exit if unknown attribute.
412 if (!addrAttr) {
413 return rewriter.notifyMatchFailure(
414 arg&: op, msg: "unsupported value attribute kind: " +
415 value.getAbstractAttribute().getName());
416 }
417 Type intType = rewriter.getIntegerType(width: addrAttr.getValue().getBitWidth());
418 Value intConst = LLVM::ConstantOp::create(builder&: rewriter, location: op.getLoc(), type: intType,
419 value: addrAttr.getValue());
420 rewriter.replaceOpWithNewOp<LLVM::IntToPtrOp>(op, args&: resultType, args&: intConst);
421 return success();
422}
423
424//===----------------------------------------------------------------------===//
425// ConvertToLLVMPatternInterface implementation
426//===----------------------------------------------------------------------===//
427
428namespace {
429/// Implement the interface to convert Ptr to LLVM.
430struct PtrToLLVMDialectInterface : public ConvertToLLVMPatternInterface {
431 PtrToLLVMDialectInterface(Dialect *dialect)
432 : ConvertToLLVMPatternInterface(dialect) {}
433
434 void loadDependentDialects(MLIRContext *context) const final {
435 context->loadDialect<LLVM::LLVMDialect>();
436 }
437
438 /// Hook for derived dialect interface to provide conversion patterns
439 /// and mark dialect legal for the conversion target.
440 void populateConvertToLLVMConversionPatterns(
441 ConversionTarget &target, LLVMTypeConverter &converter,
442 RewritePatternSet &patterns) const final {
443 ptr::populatePtrToLLVMConversionPatterns(converter, patterns);
444 }
445};
446} // namespace
447
448//===----------------------------------------------------------------------===//
449// API
450//===----------------------------------------------------------------------===//
451
452void mlir::ptr::populatePtrToLLVMConversionPatterns(
453 LLVMTypeConverter &converter, RewritePatternSet &patterns) {
454 // Add address space conversions.
455 converter.addTypeAttributeConversion(
456 callback: [&](PtrLikeTypeInterface type, ptr::GenericSpaceAttr memorySpace)
457 -> TypeConverter::AttributeConversionResult {
458 if (type.getMemorySpace() != memorySpace)
459 return TypeConverter::AttributeConversionResult::na();
460 return IntegerAttr::get(type: IntegerType::get(context: type.getContext(), width: 32), value: 0);
461 });
462
463 // Add type conversions.
464 converter.addConversion(callback: [&](ptr::PtrType type) -> Type {
465 std::optional<Attribute> maybeAttr =
466 converter.convertTypeAttribute(type, attr: type.getMemorySpace());
467 auto memSpace =
468 maybeAttr ? dyn_cast_or_null<IntegerAttr>(Val&: *maybeAttr) : IntegerAttr();
469 if (!memSpace)
470 return {};
471 return LLVM::LLVMPointerType::get(context: type.getContext(),
472 addressSpace: memSpace.getValue().getSExtValue());
473 });
474
475 // Convert ptr metadata of memref type.
476 converter.addConversion(callback: [&](ptr::PtrMetadataType type) -> Type {
477 auto mTy = dyn_cast<MemRefType>(Val: type.getType());
478 if (!mTy)
479 return {};
480 FailureOr<LLVM::LLVMStructType> res =
481 createMemRefMetadataType(type: mTy, typeConverter: converter);
482 return failed(Result: res) ? Type() : res.value();
483 });
484
485 // Add conversion patterns.
486 patterns.add<FromPtrOpConversion, GetMetadataOpConversion, PtrAddOpConversion,
487 ToPtrOpConversion, TypeOffsetOpConversion, ConstantOpConversion>(
488 arg&: converter);
489}
490
491void mlir::ptr::registerConvertPtrToLLVMInterface(DialectRegistry &registry) {
492 registry.addExtension(extensionFn: +[](MLIRContext *ctx, ptr::PtrDialect *dialect) {
493 dialect->addInterfaces<PtrToLLVMDialectInterface>();
494 });
495}
496