1//===- FuncToLLVM.cpp - Func to LLVM dialect conversion -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a pass to convert MLIR Func and builtin dialects
10// into the LLVM IR dialect.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h"
15
16#include "mlir/Analysis/DataLayoutAnalysis.h"
17#include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
18#include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
19#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
20#include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
21#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
22#include "mlir/Conversion/LLVMCommon/LowerFunctionDiscardablesToLLVM.h"
23#include "mlir/Conversion/LLVMCommon/Pattern.h"
24#include "mlir/Dialect/Func/IR/FuncOps.h"
25#include "mlir/Dialect/LLVMIR/FunctionCallUtils.h"
26#include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h"
27#include "mlir/Dialect/LLVMIR/LLVMCallOps.h"
28#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
29#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
30#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
31#include "mlir/Dialect/LLVMIR/LLVMTerminatorOps.h"
32#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
33#include "mlir/IR/Attributes.h"
34#include "mlir/IR/Builders.h"
35#include "mlir/IR/BuiltinAttributes.h"
36#include "mlir/IR/BuiltinOps.h"
37#include "mlir/IR/PatternMatch.h"
38#include "mlir/IR/SymbolTable.h"
39#include "mlir/IR/TypeUtilities.h"
40#include "mlir/Transforms/DialectConversion.h"
41#include "mlir/Transforms/Passes.h"
42#include "llvm/ADT/SmallVector.h"
43#include "llvm/IR/Type.h"
44#include "llvm/Support/DebugLog.h"
45#include "llvm/Support/FormatVariadic.h"
46#include <optional>
47
48namespace mlir {
49#define GEN_PASS_DEF_CONVERTFUNCTOLLVMPASS
50#define GEN_PASS_DEF_SETLLVMMODULEDATALAYOUTPASS
51#include "mlir/Conversion/Passes.h.inc"
52} // namespace mlir
53
54using namespace mlir;
55
56#define PASS_NAME "convert-func-to-llvm"
57#define DEBUG_TYPE PASS_NAME
58
59static constexpr StringRef varargsAttrName = "func.varargs";
60static constexpr StringRef linkageAttrName = "llvm.linkage";
61static constexpr StringRef barePtrAttrName = "llvm.bareptr";
62
63/// Return `true` if the `op` should use bare pointer calling convention.
64static bool shouldUseBarePtrCallConv(Operation *op,
65 const LLVMTypeConverter *typeConverter) {
66 return (op && op->hasDiscardableAttr(name: barePtrAttrName)) ||
67 typeConverter->getOptions().useBarePtrCallConv;
68}
69
70static bool isDiscardableAttr(StringRef name) {
71 return name == linkageAttrName || name == varargsAttrName ||
72 name == LLVM::LLVMDialect::getReadnoneAttrName();
73}
74
75/// Only retain those attributes that are not constructed by
76/// `LLVMFuncOp::build`.
77static void filterFuncAttributes(FunctionOpInterface func,
78 SmallVectorImpl<NamedAttribute> &result) {
79 for (const NamedAttribute &attr :
80 func->getDiscardableAttrDictionary().getValue()) {
81 if (isDiscardableAttr(name: attr.getName().strref()))
82 continue;
83 result.push_back(Elt: attr);
84 }
85}
86
87/// Propagate argument/results attributes.
88static void propagateArgResAttrs(OpBuilder &builder, bool resultStructType,
89 FunctionOpInterface funcOp,
90 LLVM::LLVMFuncOp wrapperFuncOp) {
91 auto argAttrs = funcOp.getAllArgAttrs();
92 if (!resultStructType) {
93 if (auto resAttrs = funcOp.getAllResultAttrs())
94 wrapperFuncOp.setAllResultAttrs(resAttrs);
95 if (argAttrs)
96 wrapperFuncOp.setAllArgAttrs(argAttrs);
97 } else {
98 SmallVector<Attribute> argAttributes;
99 // Only modify the argument and result attributes when the result is now
100 // an argument.
101 if (argAttrs) {
102 argAttributes.push_back(Elt: builder.getDictionaryAttr(value: {}));
103 argAttributes.append(in_start: argAttrs.begin(), in_end: argAttrs.end());
104 wrapperFuncOp.setAllArgAttrs(argAttributes);
105 }
106 }
107 cast<FunctionOpInterface>(Val: wrapperFuncOp.getOperation())
108 .setVisibility(funcOp.getVisibility());
109}
110
111/// Creates an auxiliary function with pointer-to-memref-descriptor-struct
112/// arguments instead of unpacked arguments. This function can be called from C
113/// by passing a pointer to a C struct corresponding to a memref descriptor.
114/// Similarly, returned memrefs are passed via pointers to a C struct that is
115/// passed as additional argument.
116/// Internally, the auxiliary function unpacks the descriptor into individual
117/// components and forwards them to `newFuncOp` and forwards the results to
118/// the extra arguments.
119static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
120 const LLVMTypeConverter &typeConverter,
121 FunctionOpInterface funcOp,
122 LLVM::LLVMFuncOp newFuncOp) {
123 auto type = cast<FunctionType>(Val: funcOp.getFunctionType());
124 auto [wrapperFuncType, resultStructType] =
125 typeConverter.convertFunctionTypeCWrapper(type);
126
127 SmallVector<NamedAttribute> attributes;
128 filterFuncAttributes(func: funcOp, result&: attributes);
129
130 auto wrapperFuncOp = LLVM::LLVMFuncOp::create(
131 builder&: rewriter, location: loc, name: llvm::formatv(Fmt: "_mlir_ciface_{0}", Vals: funcOp.getName()).str(),
132 type: wrapperFuncType, linkage: LLVM::Linkage::External, /*dsoLocal=*/false,
133 /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr, attrs: attributes);
134 propagateArgResAttrs(builder&: rewriter, resultStructType: !!resultStructType, funcOp, wrapperFuncOp);
135
136 OpBuilder::InsertionGuard guard(rewriter);
137 rewriter.setInsertionPointToStart(wrapperFuncOp.addEntryBlock(builder&: rewriter));
138
139 SmallVector<Value, 8> args;
140 size_t argOffset = resultStructType ? 1 : 0;
141 for (auto [index, argType] : llvm::enumerate(First: type.getInputs())) {
142 Value arg = wrapperFuncOp.getArgument(idx: index + argOffset);
143 if (auto memrefType = dyn_cast<MemRefType>(Val: argType)) {
144 Value loaded = LLVM::LoadOp::create(
145 builder&: rewriter, location: loc, type: typeConverter.convertType(t: memrefType), addr: arg);
146 MemRefDescriptor::unpack(builder&: rewriter, loc, packed: loaded, type: memrefType, results&: args);
147 continue;
148 }
149 if (isa<UnrankedMemRefType>(Val: argType)) {
150 Value loaded = LLVM::LoadOp::create(
151 builder&: rewriter, location: loc, type: typeConverter.convertType(t: argType), addr: arg);
152 UnrankedMemRefDescriptor::unpack(builder&: rewriter, loc, packed: loaded, results&: args);
153 continue;
154 }
155
156 args.push_back(Elt: arg);
157 }
158
159 auto call = LLVM::CallOp::create(builder&: rewriter, location: loc, func: newFuncOp, args);
160
161 if (resultStructType) {
162 LLVM::StoreOp::create(builder&: rewriter, location: loc, value: call.getResult(),
163 addr: wrapperFuncOp.getArgument(idx: 0));
164 LLVM::ReturnOp::create(builder&: rewriter, location: loc, args: ValueRange{});
165 } else {
166 LLVM::ReturnOp::create(builder&: rewriter, location: loc, args: call.getResults());
167 }
168}
169
170/// Creates an auxiliary function with pointer-to-memref-descriptor-struct
171/// arguments instead of unpacked arguments. Creates a body for the (external)
172/// `newFuncOp` that allocates a memref descriptor on stack, packs the
173/// individual arguments into this descriptor and passes a pointer to it into
174/// the auxiliary function. If the result of the function cannot be directly
175/// returned, we write it to a special first argument that provides a pointer
176/// to a corresponding struct. This auxiliary external function is now
177/// compatible with functions defined in C using pointers to C structs
178/// corresponding to a memref descriptor.
179static void wrapExternalFunction(OpBuilder &builder, Location loc,
180 const LLVMTypeConverter &typeConverter,
181 FunctionOpInterface funcOp,
182 LLVM::LLVMFuncOp newFuncOp) {
183 OpBuilder::InsertionGuard guard(builder);
184
185 auto [wrapperType, resultStructType] =
186 typeConverter.convertFunctionTypeCWrapper(
187 type: cast<FunctionType>(Val: funcOp.getFunctionType()));
188 // This conversion can only fail if it could not convert one of the argument
189 // types. But since it has been applied to a non-wrapper function before, it
190 // should have failed earlier and not reach this point at all.
191 assert(wrapperType && "unexpected type conversion failure");
192
193 SmallVector<NamedAttribute, 4> attributes;
194 filterFuncAttributes(func: funcOp, result&: attributes);
195
196 // Create the auxiliary function.
197 auto wrapperFunc = LLVM::LLVMFuncOp::create(
198 builder, location: loc, name: llvm::formatv(Fmt: "_mlir_ciface_{0}", Vals: funcOp.getName()).str(),
199 type: wrapperType, linkage: LLVM::Linkage::External, /*dsoLocal=*/false,
200 /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr, attrs: attributes);
201 propagateArgResAttrs(builder, resultStructType: !!resultStructType, funcOp, wrapperFuncOp: wrapperFunc);
202
203 // The wrapper that we synthetize here should only be visible in this module.
204 newFuncOp.setLinkage(LLVM::Linkage::Private);
205 builder.setInsertionPointToStart(newFuncOp.addEntryBlock(builder));
206
207 // Get a ValueRange containing arguments.
208 FunctionType type = cast<FunctionType>(Val: funcOp.getFunctionType());
209 SmallVector<Value, 8> args;
210 args.reserve(N: type.getNumInputs());
211 ValueRange wrapperArgsRange(newFuncOp.getArguments());
212
213 if (resultStructType) {
214 // Allocate the struct on the stack and pass the pointer.
215 Type resultType = cast<LLVM::LLVMFunctionType>(Val&: wrapperType).getParamType(i: 0);
216 Type indexType = typeConverter.convertType(t: builder.getIndexType());
217 Value one = LLVM::ConstantOp::create(builder, location: loc, res: indexType,
218 value: builder.getIntegerAttr(type: indexType, value: 1));
219 Value result =
220 LLVM::AllocaOp::create(builder, location: loc, resultType, elementType: resultStructType, arraySize: one);
221 args.push_back(Elt: result);
222 }
223
224 // Iterate over the inputs of the original function and pack values into
225 // memref descriptors if the original type is a memref.
226 for (Type input : type.getInputs()) {
227 Value arg;
228 int numToDrop = 1;
229 auto memRefType = dyn_cast<MemRefType>(Val&: input);
230 auto unrankedMemRefType = dyn_cast<UnrankedMemRefType>(Val&: input);
231 if (memRefType || unrankedMemRefType) {
232 numToDrop = memRefType
233 ? MemRefDescriptor::getNumUnpackedValues(type: memRefType)
234 : UnrankedMemRefDescriptor::getNumUnpackedValues();
235 Value packed =
236 memRefType
237 ? MemRefDescriptor::pack(builder, loc, converter: typeConverter, type: memRefType,
238 values: wrapperArgsRange.take_front(n: numToDrop))
239 : UnrankedMemRefDescriptor::pack(
240 builder, loc, converter: typeConverter, type: unrankedMemRefType,
241 values: wrapperArgsRange.take_front(n: numToDrop));
242
243 auto ptrTy = LLVM::LLVMPointerType::get(context: builder.getContext());
244 Type indexType = typeConverter.convertType(t: builder.getIndexType());
245 Value one = LLVM::ConstantOp::create(
246 builder, location: loc, res: indexType, value: builder.getIntegerAttr(type: indexType, value: 1));
247 Value allocated = LLVM::AllocaOp::create(
248 builder, location: loc, resultType: ptrTy, elementType: packed.getType(), arraySize: one, /*alignment=*/0);
249 LLVM::StoreOp::create(builder, location: loc, value: packed, addr: allocated);
250 arg = allocated;
251 } else {
252 arg = wrapperArgsRange[0];
253 }
254
255 args.push_back(Elt: arg);
256 wrapperArgsRange = wrapperArgsRange.drop_front(n: numToDrop);
257 }
258 assert(wrapperArgsRange.empty() && "did not map some of the arguments");
259
260 auto call = LLVM::CallOp::create(builder, location: loc, func: wrapperFunc, args);
261
262 if (resultStructType) {
263 Value result =
264 LLVM::LoadOp::create(builder, location: loc, type: resultStructType, addr: args.front());
265 LLVM::ReturnOp::create(builder, location: loc, arg: result);
266 } else {
267 LLVM::ReturnOp::create(builder, location: loc, args: call.getResults());
268 }
269}
270
271/// Inserts `llvm.load` ops in the function body to restore the expected pointee
272/// value from `llvm.byval`/`llvm.byref` function arguments that were converted
273/// to LLVM pointer types.
274static void restoreByValRefArgumentType(
275 ConversionPatternRewriter &rewriter, const LLVMTypeConverter &typeConverter,
276 ArrayRef<std::optional<NamedAttribute>> byValRefNonPtrAttrs,
277 LLVM::LLVMFuncOp funcOp) {
278 // Nothing to do for function declarations.
279 if (funcOp.isExternal())
280 return;
281
282 ConversionPatternRewriter::InsertionGuard guard(rewriter);
283 rewriter.setInsertionPointToStart(&funcOp.getFunctionBody().front());
284
285 for (const auto &[arg, byValRefAttr] :
286 llvm::zip(t: funcOp.getArguments(), u&: byValRefNonPtrAttrs)) {
287 // Skip argument if no `llvm.byval` or `llvm.byref` attribute.
288 if (!byValRefAttr)
289 continue;
290
291 // Insert load to retrieve the actual argument passed by value/reference.
292 assert(isa<LLVM::LLVMPointerType>(arg.getType()) &&
293 "Expected LLVM pointer type for argument with "
294 "`llvm.byval`/`llvm.byref` attribute");
295 Type resTy = typeConverter.convertType(
296 t: cast<TypeAttr>(Val: byValRefAttr->getValue()).getValue());
297
298 Value valueArg = LLVM::LoadOp::create(builder&: rewriter, location: arg.getLoc(), type: resTy, addr: arg);
299 rewriter.replaceAllUsesWith(from: arg, to: valueArg);
300 }
301}
302
303static FailureOr<LLVM::LLVMFunctionType> convertFuncSignature(
304 FunctionOpInterface funcOp, const LLVMTypeConverter &converter,
305 bool useBarePtrCallConv, TypeConverter::SignatureConversion &result,
306 SmallVectorImpl<std::optional<NamedAttribute>> &byValRefNonPtrAttrs) {
307 auto varargsAttr =
308 funcOp->getDiscardableAttrOfType<BoolAttr>(name: varargsAttrName);
309 auto llvmType = dyn_cast_or_null<LLVM::LLVMFunctionType>(
310 Val: converter.convertFunctionSignature(
311 funcOp, isVariadic: varargsAttr && varargsAttr.getValue(), useBarePtrCallConv,
312 result, byValRefNonPtrAttrs));
313 if (!llvmType)
314 return failure();
315 return llvmType;
316}
317
318static LLVM::LLVMFuncOp createLLVMFuncOp(FunctionOpInterface funcOp,
319 ConversionPatternRewriter &rewriter,
320 LLVM::LLVMFunctionType llvmType,
321 LoweredLLVMFuncAttrs &loweredAttrs,
322 SymbolTableCollection *symbolTables) {
323 Operation *symbolTableOp = funcOp->getParentWithTrait<OpTrait::SymbolTable>();
324 if (symbolTables && symbolTableOp) {
325 SymbolTable &symbolTable = symbolTables->getSymbolTable(op: symbolTableOp);
326 symbolTable.remove(op: funcOp);
327 }
328 loweredAttrs.properties.setCConv(
329 LLVM::CConvAttr::get(context: rewriter.getContext(), CallingConv: LLVM::CConv::C));
330 auto newFuncOp = LLVM::LLVMFuncOp::create(builder&: rewriter, location: funcOp.getLoc(),
331 properties: loweredAttrs.properties,
332 discardableAttributes: loweredAttrs.discardableAttrs);
333
334 if (symbolTables && symbolTableOp) {
335 auto ip = rewriter.getInsertionPoint();
336 SymbolTable &symbolTable = symbolTables->getSymbolTable(op: symbolTableOp);
337 symbolTable.insert(symbol: newFuncOp, insertPt: ip);
338 }
339
340 cast<FunctionOpInterface>(Val: newFuncOp.getOperation())
341 .setVisibility(funcOp.getVisibility());
342
343 // Set readnone memory effects
344 if (funcOp->hasDiscardableAttr(name: LLVM::LLVMDialect::getReadnoneAttrName())) {
345 auto memoryAttr = LLVM::MemoryEffectsAttr::get(
346 context: rewriter.getContext(), memInfoArgs: {/*other=*/LLVM::ModRefInfo::NoModRef,
347 /*argMem=*/LLVM::ModRefInfo::NoModRef,
348 /*inaccessibleMem=*/LLVM::ModRefInfo::NoModRef,
349 /*errnoMem=*/LLVM::ModRefInfo::NoModRef,
350 /*targetMem0=*/LLVM::ModRefInfo::NoModRef,
351 /*targetMem1=*/LLVM::ModRefInfo::NoModRef});
352 newFuncOp.setMemoryEffectsAttr(memoryAttr);
353 }
354
355 return newFuncOp;
356}
357
358static SmallVector<NamedAttribute>
359convertArgumentAttributes(DictionaryAttr attrsDict,
360 ConversionPatternRewriter &rewriter,
361 const LLVMTypeConverter &converter) {
362 SmallVector<NamedAttribute> convertedAttrs;
363 convertedAttrs.reserve(N: attrsDict.size());
364 for (const NamedAttribute &attr : attrsDict) {
365 const auto convert = [&](const NamedAttribute &attr) {
366 return TypeAttr::get(
367 type: converter.convertType(t: cast<TypeAttr>(Val: attr.getValue()).getValue()));
368 };
369 if (attr.getName().getValue() == LLVM::LLVMDialect::getByValAttrName()) {
370 convertedAttrs.push_back(Elt: rewriter.getNamedAttr(
371 name: LLVM::LLVMDialect::getByValAttrName(), val: convert(attr)));
372 } else if (attr.getName().getValue() ==
373 LLVM::LLVMDialect::getByRefAttrName()) {
374 convertedAttrs.push_back(Elt: rewriter.getNamedAttr(
375 name: LLVM::LLVMDialect::getByRefAttrName(), val: convert(attr)));
376 } else if (attr.getName().getValue() ==
377 LLVM::LLVMDialect::getStructRetAttrName()) {
378 convertedAttrs.push_back(Elt: rewriter.getNamedAttr(
379 name: LLVM::LLVMDialect::getStructRetAttrName(), val: convert(attr)));
380 } else if (attr.getName().getValue() ==
381 LLVM::LLVMDialect::getInAllocaAttrName()) {
382 convertedAttrs.push_back(Elt: rewriter.getNamedAttr(
383 name: LLVM::LLVMDialect::getInAllocaAttrName(), val: convert(attr)));
384 } else {
385 convertedAttrs.push_back(Elt: attr);
386 }
387 }
388 return convertedAttrs;
389}
390
391static void propagateFunctionArgResAttrs(
392 FunctionOpInterface funcOp, ConversionPatternRewriter &rewriter,
393 const LLVMTypeConverter &converter, TypeConverter::SignatureConversion &sig,
394 LLVM::LLVMFunctionType llvmType, LLVM::LLVMFuncOp newFuncOp) {
395 // Propagate argument/result attributes to all converted arguments/result
396 // obtained after converting a given original argument/result.
397 if (ArrayAttr resAttrDicts = funcOp.getAllResultAttrs()) {
398 assert(!resAttrDicts.empty() && "expected array to be non-empty");
399 if (funcOp.getNumResults() == 1)
400 newFuncOp.setAllResultAttrs(resAttrDicts);
401 }
402 if (ArrayAttr argAttrDicts = funcOp.getAllArgAttrs()) {
403 SmallVector<Attribute> newArgAttrs(llvmType.getNumParams());
404 for (unsigned i = 0, e = funcOp.getNumArguments(); i < e; ++i) {
405 // Some LLVM IR attribute have a type attached to them. During FuncOp ->
406 // LLVMFuncOp conversion these types may have changed. Account for that
407 // change by converting attributes' types as well.
408 auto attrsDict = cast<DictionaryAttr>(Val: argAttrDicts[i]);
409 SmallVector<NamedAttribute, 4> convertedAttrs =
410 convertArgumentAttributes(attrsDict, rewriter, converter);
411 auto mapping = sig.getInputMapping(input: i);
412 assert(mapping && "unexpected deletion of function argument");
413 // Only attach the new argument attributes if there is a one-to-one
414 // mapping from old to new types. Otherwise, attributes might be
415 // attached to types that they do not support.
416 if (mapping->size == 1) {
417 newArgAttrs[mapping->inputNo] =
418 DictionaryAttr::get(context: rewriter.getContext(), value: convertedAttrs);
419 continue;
420 }
421 // TODO: Implement custom handling for types that expand to multiple
422 // function arguments.
423 for (size_t j = 0; j < mapping->size; ++j)
424 newArgAttrs[mapping->inputNo + j] =
425 DictionaryAttr::get(context: rewriter.getContext(), value: {});
426 }
427 if (!newArgAttrs.empty())
428 newFuncOp.setAllArgAttrs(rewriter.getArrayAttr(value: newArgAttrs));
429 }
430}
431
432static void wrapWithCInterface(FunctionOpInterface funcOp,
433 ConversionPatternRewriter &rewriter,
434 const LLVMTypeConverter &converter,
435 LLVM::LLVMFuncOp newFuncOp) {
436 if (newFuncOp.isExternal())
437 wrapExternalFunction(builder&: rewriter, loc: funcOp->getLoc(), typeConverter: converter, funcOp,
438 newFuncOp);
439 else
440 wrapForExternalCallers(rewriter, loc: funcOp->getLoc(), typeConverter: converter, funcOp,
441 newFuncOp);
442}
443
444/// Conversion steps
445/// - Validate function type
446/// - Convert signature
447/// - Validate C wrapper varargs constraint
448/// - Lower function attrs
449/// - Create llvm.func
450/// - Propagate arg/result attrs
451/// - Inline body + signature conversion
452/// - Restore byval/byref pointee types
453/// - C-wrapper handling
454FailureOr<LLVM::LLVMFuncOp> mlir::convertFuncOpToLLVMFuncOp(
455 FunctionOpInterface funcOp, ConversionPatternRewriter &rewriter,
456 const LLVMTypeConverter &converter, SymbolTableCollection *symbolTables) {
457 // Check the funcOp has `FunctionType`.
458 auto funcTy = dyn_cast<FunctionType>(Val: funcOp.getFunctionType());
459 if (!funcTy)
460 return rewriter.notifyMatchFailure(
461 arg&: funcOp, msg: "Only support FunctionOpInterface with FunctionType");
462
463 bool useBarePtrCallConv = shouldUseBarePtrCallConv(op: funcOp, typeConverter: &converter);
464 // Convert the original function arguments. They are converted using the
465 // LLVMTypeConverter provided to this legalization pattern.
466 // Gather `llvm.byval` and `llvm.byref` arguments whose type convertion was
467 // overriden with an LLVM pointer type for later processing.
468 SmallVector<std::optional<NamedAttribute>> byValRefNonPtrAttrs;
469 TypeConverter::SignatureConversion result(funcOp.getNumArguments());
470 FailureOr<LLVM::LLVMFunctionType> llvmType = convertFuncSignature(
471 funcOp, converter, useBarePtrCallConv, result, byValRefNonPtrAttrs);
472 if (failed(Result: llvmType))
473 return rewriter.notifyMatchFailure(arg&: funcOp, msg: "signature conversion failed");
474
475 // Validate C wrapper varargs constraint
476 bool emitCWrapper = funcOp->hasDiscardableAttrOfType<UnitAttr>(
477 name: LLVM::LLVMDialect::getEmitCWrapperAttrName());
478 if (!useBarePtrCallConv && emitCWrapper && llvmType->isVarArg())
479 return funcOp.emitError(message: "C interface for variadic functions is not "
480 "supported yet.");
481
482 // Lower function attrs
483 FailureOr<LoweredLLVMFuncAttrs> loweredAttrs =
484 lowerDiscardableAttrsForLLVMFunc(funcOp, llvmFuncType: *llvmType);
485 if (failed(Result: loweredAttrs))
486 return rewriter.notifyMatchFailure(arg&: funcOp,
487 msg: "failed to lower func attributes");
488
489 // Create llvm.func
490 auto newFuncOp = createLLVMFuncOp(funcOp, rewriter, llvmType: *llvmType, loweredAttrs&: *loweredAttrs,
491 symbolTables);
492
493 // Propagate arg/result attrs
494 propagateFunctionArgResAttrs(funcOp, rewriter, converter, sig&: result, llvmType: *llvmType,
495 newFuncOp);
496
497 // Inline body + signature conversion
498 rewriter.inlineRegionBefore(region&: funcOp.getFunctionBody(), parent&: newFuncOp.getBody(),
499 before: newFuncOp.end());
500 // Convert just the entry block. The remaining unstructured control flow is
501 // converted by ControlFlowToLLVM.
502 if (!newFuncOp.getBody().empty())
503 rewriter.applySignatureConversion(block: &newFuncOp.getBody().front(), conversion&: result,
504 converter: &converter);
505
506 // Restore byval/byref pointee types
507 // Fix the type mismatch between the materialized `llvm.ptr` and the expected
508 // pointee type in the function body when converting `llvm.byval`/`llvm.byref`
509 // function arguments.
510 restoreByValRefArgumentType(rewriter, typeConverter: converter, byValRefNonPtrAttrs,
511 funcOp: newFuncOp);
512
513 // C-wrapper handling
514 if (!useBarePtrCallConv && emitCWrapper)
515 wrapWithCInterface(funcOp, rewriter, converter, newFuncOp);
516
517 return newFuncOp;
518}
519
520namespace {
521
522/// FuncOp legalization pattern that converts MemRef arguments to pointers to
523/// MemRef descriptors (LLVM struct data types) containing all the MemRef type
524/// information.
525class FuncOpConversion : public ConvertOpToLLVMPattern<func::FuncOp> {
526 SymbolTableCollection *symbolTables = nullptr;
527
528public:
529 explicit FuncOpConversion(const LLVMTypeConverter &converter,
530 SymbolTableCollection *symbolTables = nullptr)
531 : ConvertOpToLLVMPattern(converter), symbolTables(symbolTables) {}
532
533 LogicalResult
534 matchAndRewrite(func::FuncOp funcOp, OpAdaptor adaptor,
535 ConversionPatternRewriter &rewriter) const override {
536 FailureOr<LLVM::LLVMFuncOp> newFuncOp = mlir::convertFuncOpToLLVMFuncOp(
537 funcOp: cast<FunctionOpInterface>(Val: funcOp.getOperation()), rewriter,
538 converter: *getTypeConverter(), symbolTables);
539 if (failed(Result: newFuncOp))
540 return rewriter.notifyMatchFailure(arg&: funcOp, msg: "Could not convert funcop");
541
542 rewriter.eraseOp(op: funcOp);
543 return success();
544 }
545};
546
547struct ConstantOpLowering : public ConvertOpToLLVMPattern<func::ConstantOp> {
548 using ConvertOpToLLVMPattern<func::ConstantOp>::ConvertOpToLLVMPattern;
549
550 LogicalResult
551 matchAndRewrite(func::ConstantOp op, OpAdaptor adaptor,
552 ConversionPatternRewriter &rewriter) const override {
553 auto type = typeConverter->convertType(t: op.getResult().getType());
554 if (!type || !LLVM::isCompatibleType(type))
555 return rewriter.notifyMatchFailure(arg&: op, msg: "failed to convert result type");
556
557 auto newOp =
558 LLVM::AddressOfOp::create(builder&: rewriter, location: op.getLoc(), res: type, global_name: op.getValue());
559 for (const NamedAttribute &attr :
560 op->getDiscardableAttrDictionary().getValue()) {
561 if (attr.getName().strref() == "value")
562 continue;
563 newOp->setDiscardableAttr(name: attr.getName(), value: attr.getValue());
564 }
565 rewriter.replaceOp(op, newValues: newOp->getResults());
566 return success();
567 }
568};
569
570// A CallOp automatically promotes MemRefType to a sequence of alloca/store and
571// passes the pointer to the MemRef across function boundaries.
572template <typename CallOpType>
573struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
574 using ConvertOpToLLVMPattern<CallOpType>::ConvertOpToLLVMPattern;
575 using Super = CallOpInterfaceLowering<CallOpType>;
576 using Base = ConvertOpToLLVMPattern<CallOpType>;
577 using Adaptor = typename ConvertOpToLLVMPattern<CallOpType>::OneToNOpAdaptor;
578
579 LogicalResult matchAndRewriteImpl(CallOpType callOp, Adaptor adaptor,
580 ConversionPatternRewriter &rewriter,
581 bool useBarePtrCallConv = false) const {
582 // Pack the result types into a struct.
583 Type packedResult = nullptr;
584 SmallVector<SmallVector<Type>> groupedResultTypes;
585 unsigned numResults = callOp.getNumResults();
586 auto resultTypes = llvm::to_vector<4>(callOp.getResultTypes());
587 int64_t numConvertedTypes = 0;
588 if (numResults != 0) {
589 if (!(packedResult = this->getTypeConverter()->packFunctionResults(
590 resultTypes, useBarePtrCallConv, &groupedResultTypes,
591 &numConvertedTypes)))
592 return failure();
593 }
594
595 if (useBarePtrCallConv) {
596 for (auto it : callOp->getOperands()) {
597 Type operandType = it.getType();
598 if (isa<UnrankedMemRefType>(Val: operandType)) {
599 // Unranked memref is not supported in the bare pointer calling
600 // convention.
601 return failure();
602 }
603 }
604 }
605 auto promoted = this->getTypeConverter()->promoteOperands(
606 callOp.getLoc(), /*opOperands=*/callOp->getOperands(),
607 adaptor.getOperands(), rewriter, useBarePtrCallConv);
608 LLVM::CallOp::Properties properties{};
609 LLVM::CallOp::populateDefaultProperties(
610 opName: OperationName(LLVM::CallOp::getOperationName(), rewriter.getContext()),
611 properties);
612 properties.operandSegmentSizes = {static_cast<int32_t>(promoted.size()), 0};
613 properties.op_bundle_sizes = rewriter.getDenseI32ArrayAttr(values: {});
614 auto newOp = LLVM::CallOp::create(
615 rewriter, callOp.getLoc(),
616 packedResult ? TypeRange(packedResult) : TypeRange(), promoted,
617 properties, callOp->getDiscardableAttrDictionary().getValue());
618 if constexpr (std::is_same_v<CallOpType, func::CallOp>)
619 newOp.setCalleeAttr(callOp.getCalleeAttr());
620
621 // Helper function that extracts an individual result from the return value
622 // of the new call op. llvm.call ops support only 0 or 1 result. In case of
623 // 2 or more results, the results are packed into a structure.
624 //
625 // The new call op may have more than 2 results because:
626 // a. The original call op has more than 2 results.
627 // b. An original op result type-converted to more than 1 result.
628 auto getUnpackedResult = [&](unsigned i) -> Value {
629 assert(numConvertedTypes > 0 && "convert op has no results");
630 if (numConvertedTypes == 1) {
631 assert(i == 0 && "out of bounds: converted op has only one result");
632 return newOp->getResult(0);
633 }
634 // Results have been converted to a structure. Extract individual results
635 // from the structure.
636 return LLVM::ExtractValueOp::create(rewriter, callOp.getLoc(),
637 newOp->getResult(0), i);
638 };
639
640 // Group the results into a vector of vectors, such that it is clear which
641 // original op result is replaced with which range of values. (In case of a
642 // 1:N conversion, there can be multiple replacements for a single result.)
643 SmallVector<SmallVector<Value>> results;
644 results.reserve(N: numResults);
645 unsigned counter = 0;
646 for (unsigned i = 0; i < numResults; ++i) {
647 SmallVector<Value> &group = results.emplace_back();
648 for (unsigned j = 0, e = groupedResultTypes[i].size(); j < e; ++j)
649 group.push_back(Elt: getUnpackedResult(counter++));
650 }
651
652 // Special handling for MemRef types.
653 for (unsigned i = 0; i < numResults; ++i) {
654 Type origType = resultTypes[i];
655 auto memrefType = dyn_cast<MemRefType>(Val&: origType);
656 auto unrankedMemrefType = dyn_cast<UnrankedMemRefType>(Val&: origType);
657 if (useBarePtrCallConv && memrefType) {
658 // For the bare-ptr calling convention, promote memref results to
659 // descriptors.
660 assert(results[i].size() == 1 && "expected one converted result");
661 results[i].front() = MemRefDescriptor::fromStaticShape(
662 rewriter, callOp.getLoc(), *this->getTypeConverter(), memrefType,
663 results[i].front());
664 }
665 if (unrankedMemrefType) {
666 assert(!useBarePtrCallConv && "unranked memref is not supported in the "
667 "bare-ptr calling convention");
668 assert(results[i].size() == 1 && "expected one converted result");
669 Value desc = this->copyUnrankedDescriptor(
670 rewriter, callOp.getLoc(), unrankedMemrefType, results[i].front(),
671 /*toDynamic=*/false);
672 if (!desc)
673 return failure();
674 results[i].front() = desc;
675 }
676 }
677
678 rewriter.replaceOpWithMultiple(callOp, results);
679 return success();
680 }
681};
682
683class CallOpLowering : public CallOpInterfaceLowering<func::CallOp> {
684public:
685 explicit CallOpLowering(const LLVMTypeConverter &typeConverter,
686 SymbolTableCollection *symbolTables = nullptr,
687 PatternBenefit benefit = 1)
688 : CallOpInterfaceLowering<func::CallOp>(typeConverter, benefit),
689 symbolTables(symbolTables) {}
690
691 LogicalResult
692 matchAndRewrite(func::CallOp callOp, OneToNOpAdaptor adaptor,
693 ConversionPatternRewriter &rewriter) const override {
694 bool useBarePtrCallConv = false;
695 if (getTypeConverter()->getOptions().useBarePtrCallConv) {
696 useBarePtrCallConv = true;
697 } else if (symbolTables != nullptr) {
698 // Fast lookup.
699 Operation *callee =
700 symbolTables->lookupNearestSymbolFrom(from: callOp, symbol: callOp.getCalleeAttr());
701 useBarePtrCallConv =
702 callee != nullptr && callee->hasDiscardableAttr(name: barePtrAttrName);
703 } else {
704 // Warning: This is a linear lookup.
705 Operation *callee =
706 SymbolTable::lookupNearestSymbolFrom(from: callOp, symbol: callOp.getCalleeAttr());
707 useBarePtrCallConv =
708 callee != nullptr && callee->hasDiscardableAttr(name: barePtrAttrName);
709 }
710 return matchAndRewriteImpl(callOp, adaptor, rewriter, useBarePtrCallConv);
711 }
712
713private:
714 SymbolTableCollection *symbolTables = nullptr;
715};
716
717struct CallIndirectOpLowering
718 : public CallOpInterfaceLowering<func::CallIndirectOp> {
719 using Super::Super;
720
721 LogicalResult
722 matchAndRewrite(func::CallIndirectOp callIndirectOp, OneToNOpAdaptor adaptor,
723 ConversionPatternRewriter &rewriter) const override {
724 return matchAndRewriteImpl(callOp: callIndirectOp, adaptor, rewriter);
725 }
726};
727
728struct UnrealizedConversionCastOpLowering
729 : public ConvertOpToLLVMPattern<UnrealizedConversionCastOp> {
730 using ConvertOpToLLVMPattern<
731 UnrealizedConversionCastOp>::ConvertOpToLLVMPattern;
732
733 LogicalResult
734 matchAndRewrite(UnrealizedConversionCastOp op, OpAdaptor adaptor,
735 ConversionPatternRewriter &rewriter) const override {
736 SmallVector<Type> convertedTypes;
737 if (succeeded(Result: typeConverter->convertTypes(types: op.getOutputs().getTypes(),
738 results&: convertedTypes)) &&
739 convertedTypes == adaptor.getInputs().getTypes()) {
740 rewriter.replaceOp(op, newValues: adaptor.getInputs());
741 return success();
742 }
743
744 convertedTypes.clear();
745 if (succeeded(Result: typeConverter->convertTypes(types: adaptor.getInputs().getTypes(),
746 results&: convertedTypes)) &&
747 convertedTypes == op.getOutputs().getType()) {
748 rewriter.replaceOp(op, newValues: adaptor.getInputs());
749 return success();
750 }
751 return failure();
752 }
753};
754
755// Special lowering pattern for `ReturnOps`. Unlike all other operations,
756// `ReturnOp` interacts with the function signature and must have as many
757// operands as the function has return values. Because in LLVM IR, functions
758// can only return 0 or 1 value, we pack multiple values into a structure type.
759// Emit `PoisonOp` followed by `InsertValueOp`s to create such structure if
760// necessary before returning it
761struct ReturnOpLowering : public ConvertOpToLLVMPattern<func::ReturnOp> {
762 using ConvertOpToLLVMPattern<func::ReturnOp>::ConvertOpToLLVMPattern;
763
764 LogicalResult
765 matchAndRewrite(func::ReturnOp op, OneToNOpAdaptor adaptor,
766 ConversionPatternRewriter &rewriter) const override {
767 Location loc = op.getLoc();
768 SmallVector<Value, 4> updatedOperands;
769
770 auto funcOp = op->getParentOfType<LLVM::LLVMFuncOp>();
771 bool useBarePtrCallConv =
772 shouldUseBarePtrCallConv(op: funcOp, typeConverter: this->getTypeConverter());
773
774 for (auto [oldOperand, newOperands] :
775 llvm::zip_equal(t: op->getOperands(), u: adaptor.getOperands())) {
776 Type oldTy = oldOperand.getType();
777 if (auto memRefType = dyn_cast<MemRefType>(Val&: oldTy)) {
778 assert(newOperands.size() == 1 && "expected one converted result");
779 if (useBarePtrCallConv &&
780 getTypeConverter()->canConvertToBarePtr(type: memRefType)) {
781 // For the bare-ptr calling convention, extract the aligned pointer to
782 // be returned from the memref descriptor.
783 MemRefDescriptor memrefDesc(newOperands.front());
784 updatedOperands.push_back(Elt: memrefDesc.allocatedPtr(builder&: rewriter, loc));
785 continue;
786 }
787 } else if (auto unrankedMemRefType =
788 dyn_cast<UnrankedMemRefType>(Val&: oldTy)) {
789 assert(newOperands.size() == 1 && "expected one converted result");
790 if (useBarePtrCallConv) {
791 // Unranked memref is not supported in the bare pointer calling
792 // convention.
793 return failure();
794 }
795 Value updatedDesc =
796 copyUnrankedDescriptor(builder&: rewriter, loc, memRefType: unrankedMemRefType,
797 operand: newOperands.front(), /*toDynamic=*/true);
798 if (!updatedDesc)
799 return failure();
800 updatedOperands.push_back(Elt: updatedDesc);
801 continue;
802 }
803
804 llvm::append_range(C&: updatedOperands, R: newOperands);
805 }
806
807 // If ReturnOp has 0 or 1 operand, create it and return immediately.
808 if (updatedOperands.size() <= 1) {
809 rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(
810 op, args: TypeRange(), args&: updatedOperands,
811 args: op->getDiscardableAttrDictionary().getValue());
812 return success();
813 }
814
815 // Otherwise, we need to pack the arguments into an LLVM struct type before
816 // returning.
817 auto packedType = getTypeConverter()->packFunctionResults(
818 types: op.getOperandTypes(), useBarePointerCallConv: useBarePtrCallConv);
819 if (!packedType) {
820 return rewriter.notifyMatchFailure(arg&: op, msg: "could not convert result types");
821 }
822
823 Value packed = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: packedType);
824 for (auto [idx, operand] : llvm::enumerate(First&: updatedOperands)) {
825 packed = LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: packed, value: operand, position: idx);
826 }
827 rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(
828 op, args: TypeRange(), args&: packed, args: op->getDiscardableAttrDictionary().getValue());
829 return success();
830 }
831};
832} // namespace
833
834void mlir::populateFuncToLLVMFuncOpConversionPattern(
835 const LLVMTypeConverter &converter, RewritePatternSet &patterns,
836 SymbolTableCollection *symbolTables) {
837 patterns.add<FuncOpConversion>(arg: converter, args&: symbolTables);
838}
839
840void mlir::populateFuncToLLVMConversionPatterns(
841 const LLVMTypeConverter &converter, RewritePatternSet &patterns,
842 SymbolTableCollection *symbolTables) {
843 populateFuncToLLVMFuncOpConversionPattern(converter, patterns, symbolTables);
844 patterns.add<CallIndirectOpLowering>(arg: converter);
845 patterns.add<CallOpLowering>(arg: converter, args&: symbolTables);
846 patterns.add<ConstantOpLowering>(arg: converter);
847 patterns.add<ReturnOpLowering>(arg: converter);
848}
849
850namespace {
851/// A pass converting Func operations into the LLVM IR dialect.
852struct ConvertFuncToLLVMPass
853 : public impl::ConvertFuncToLLVMPassBase<ConvertFuncToLLVMPass> {
854 using Base::Base;
855
856 /// Run the dialect converter on the module.
857 void runOnOperation() override {
858 ModuleOp m = getOperation();
859 StringRef dataLayout;
860 auto dataLayoutAttr = dyn_cast_or_null<StringAttr>(
861 Val: m->getDiscardableAttr(name: LLVM::LLVMDialect::getDataLayoutAttrName()));
862 if (dataLayoutAttr)
863 dataLayout = dataLayoutAttr.getValue();
864
865 if (failed(Result: LLVM::LLVMDialect::verifyDataLayoutString(
866 descr: dataLayout, reportError: [this](const Twine &message) {
867 getOperation().emitError() << message.str();
868 }))) {
869 signalPassFailure();
870 return;
871 }
872
873 const auto &dataLayoutAnalysis = getAnalysis<DataLayoutAnalysis>();
874
875 LowerToLLVMOptions options(&getContext(),
876 dataLayoutAnalysis.getAtOrAbove(operation: m));
877 options.useBarePtrCallConv = useBarePtrCallConv;
878 if (indexBitwidth != kDeriveIndexBitwidthFromDataLayout)
879 options.overrideIndexBitwidth(bitwidth: indexBitwidth);
880 options.dataLayout = llvm::DataLayout(dataLayout);
881
882 LLVMTypeConverter typeConverter(&getContext(), options,
883 &dataLayoutAnalysis);
884
885 RewritePatternSet patterns(&getContext());
886 SymbolTableCollection symbolTables;
887
888 populateFuncToLLVMConversionPatterns(converter: typeConverter, patterns,
889 symbolTables: &symbolTables);
890
891 LLVMConversionTarget target(getContext());
892 if (failed(Result: applyPartialConversion(op: m, target, patterns: std::move(patterns))))
893 signalPassFailure();
894 }
895};
896
897struct SetLLVMModuleDataLayoutPass
898 : public impl::SetLLVMModuleDataLayoutPassBase<
899 SetLLVMModuleDataLayoutPass> {
900 using Base::Base;
901
902 /// Run the dialect converter on the module.
903 void runOnOperation() override {
904 if (failed(Result: LLVM::LLVMDialect::verifyDataLayoutString(
905 descr: this->dataLayout, reportError: [this](const Twine &message) {
906 getOperation().emitError() << message.str();
907 }))) {
908 signalPassFailure();
909 return;
910 }
911 ModuleOp m = getOperation();
912 m->setDiscardableAttr(name: LLVM::LLVMDialect::getDataLayoutAttrName(),
913 value: StringAttr::get(context: m.getContext(), bytes: this->dataLayout));
914 }
915};
916} // namespace
917
918//===----------------------------------------------------------------------===//
919// ConvertToLLVMPatternInterface implementation
920//===----------------------------------------------------------------------===//
921
922namespace {
923/// Implement the interface to convert Func to LLVM.
924struct FuncToLLVMDialectInterface : public ConvertToLLVMPatternInterface {
925 FuncToLLVMDialectInterface(Dialect *dialect)
926 : ConvertToLLVMPatternInterface(dialect) {}
927 /// Hook for derived dialect interface to provide conversion patterns
928 /// and mark dialect legal for the conversion target.
929 void populateConvertToLLVMConversionPatterns(
930 ConversionTarget &target, LLVMTypeConverter &typeConverter,
931 RewritePatternSet &patterns) const final {
932 populateFuncToLLVMConversionPatterns(converter: typeConverter, patterns);
933 }
934};
935} // namespace
936
937void mlir::registerConvertFuncToLLVMInterface(DialectRegistry &registry) {
938 registry.addExtension(extensionFn: +[](MLIRContext *ctx, func::FuncDialect *dialect) {
939 dialect->addInterfaces<FuncToLLVMDialectInterface>();
940 });
941}
942