1//===- ModuleTranslation.cpp - MLIR to LLVM conversion --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the translation between an MLIR LLVM dialect module and
10// the corresponding LLVMIR module. It only handles core LLVM IR operations.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Target/LLVMIR/ModuleTranslation.h"
15
16#include "AttrKindDetail.h"
17#include "DebugTranslation.h"
18#include "LoopAnnotationTranslation.h"
19#include "mlir/Analysis/TopologicalSortUtils.h"
20#include "mlir/Dialect/DLTI/DLTI.h"
21#include "mlir/Dialect/LLVMIR/LLVMCallOps.h"
22#include "mlir/Dialect/LLVMIR/LLVMInterfaces.h"
23#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
24#include "mlir/Dialect/LLVMIR/LLVMTerminatorOps.h"
25#include "mlir/Dialect/LLVMIR/Transforms/DIExpressionLegalization.h"
26#include "mlir/Dialect/LLVMIR/Transforms/LegalizeForExport.h"
27#include "mlir/IR/AttrTypeSubElements.h"
28#include "mlir/IR/Attributes.h"
29#include "mlir/IR/BuiltinOps.h"
30#include "mlir/IR/BuiltinTypes.h"
31#include "mlir/IR/DialectResourceBlobManager.h"
32#include "mlir/Support/LLVM.h"
33#include "mlir/Target/LLVMIR/LLVMTranslationInterface.h"
34#include "mlir/Target/LLVMIR/TypeToLLVM.h"
35
36#include "llvm/ADT/DenseSet.h"
37#include "llvm/ADT/STLExtras.h"
38#include "llvm/ADT/StringExtras.h"
39#include "llvm/ADT/TypeSwitch.h"
40#include "llvm/Analysis/TargetFolder.h"
41#include "llvm/IR/BasicBlock.h"
42#include "llvm/IR/CFG.h"
43#include "llvm/IR/Constants.h"
44#include "llvm/IR/DerivedTypes.h"
45#include "llvm/IR/IRBuilder.h"
46#include "llvm/IR/InlineAsm.h"
47#include "llvm/IR/LLVMContext.h"
48#include "llvm/IR/MDBuilder.h"
49#include "llvm/IR/Metadata.h"
50#include "llvm/IR/Module.h"
51#include "llvm/IR/Verifier.h"
52#include "llvm/Support/Debug.h"
53#include "llvm/Support/ErrorHandling.h"
54#include "llvm/Support/VirtualFileSystem.h"
55#include "llvm/Support/raw_ostream.h"
56#include "llvm/Transforms/Utils/BasicBlockUtils.h"
57#include "llvm/Transforms/Utils/Cloning.h"
58#include "llvm/Transforms/Utils/ModuleUtils.h"
59#include <numeric>
60#include <optional>
61
62#define DEBUG_TYPE "llvm-dialect-to-llvm-ir"
63
64using namespace mlir;
65using namespace mlir::LLVM;
66using namespace mlir::LLVM::detail;
67
68#include "mlir/Dialect/LLVMIR/LLVMConversionEnumsToLLVM.inc"
69
70namespace {
71/// A customized inserter for LLVM's IRBuilder that captures all LLVM IR
72/// instructions that are created for future reference.
73///
74/// This is intended to be used with the `CollectionScope` RAII object:
75///
76/// llvm::IRBuilder<..., InstructionCapturingInserter> builder;
77/// {
78/// InstructionCapturingInserter::CollectionScope scope(builder);
79/// // Call IRBuilder methods as usual.
80///
81/// // This will return a list of all instructions created by the builder,
82/// // in order of creation.
83/// builder.getInserter().getCapturedInstructions();
84/// }
85/// // This will return an empty list.
86/// builder.getInserter().getCapturedInstructions();
87///
88/// The capturing functionality is _disabled_ by default for performance
89/// consideration. It needs to be explicitly enabled, which is achieved by
90/// creating a `CollectionScope`.
91class InstructionCapturingInserter : public llvm::IRBuilderCallbackInserter {
92public:
93 /// Constructs the inserter.
94 InstructionCapturingInserter()
95 : llvm::IRBuilderCallbackInserter([this](llvm::Instruction *instruction) {
96 if (LLVM_LIKELY(enabled))
97 capturedInstructions.push_back(Elt: instruction);
98 }) {}
99
100 /// Returns the list of LLVM IR instructions captured since the last cleanup.
101 ArrayRef<llvm::Instruction *> getCapturedInstructions() const {
102 return capturedInstructions;
103 }
104
105 /// Clears the list of captured LLVM IR instructions.
106 void clearCapturedInstructions() { capturedInstructions.clear(); }
107
108 /// RAII object enabling the capture of created LLVM IR instructions.
109 class CollectionScope {
110 public:
111 /// Creates the scope for the given inserter.
112 CollectionScope(llvm::IRBuilderBase &irBuilder, bool isBuilderCapturing);
113
114 /// Ends the scope.
115 ~CollectionScope();
116
117 ArrayRef<llvm::Instruction *> getCapturedInstructions() {
118 if (!inserter)
119 return {};
120 return inserter->getCapturedInstructions();
121 }
122
123 private:
124 /// Back reference to the inserter.
125 InstructionCapturingInserter *inserter = nullptr;
126
127 /// List of instructions in the inserter prior to this scope.
128 SmallVector<llvm::Instruction *> previouslyCollectedInstructions;
129
130 /// Whether the inserter was enabled prior to this scope.
131 bool wasEnabled;
132 };
133
134 /// Enable or disable the capturing mechanism.
135 void setEnabled(bool enabled = true) { this->enabled = enabled; }
136
137private:
138 /// List of captured instructions.
139 SmallVector<llvm::Instruction *> capturedInstructions;
140
141 /// Whether the collection is enabled.
142 bool enabled = false;
143};
144
145using CapturingIRBuilder =
146 llvm::IRBuilder<llvm::TargetFolder, InstructionCapturingInserter>;
147} // namespace
148
149InstructionCapturingInserter::CollectionScope::CollectionScope(
150 llvm::IRBuilderBase &irBuilder, bool isBuilderCapturing) {
151
152 if (!isBuilderCapturing)
153 return;
154
155 auto &capturingIRBuilder = static_cast<CapturingIRBuilder &>(irBuilder);
156 inserter = &capturingIRBuilder.getInserter();
157 wasEnabled = inserter->enabled;
158 if (wasEnabled)
159 previouslyCollectedInstructions.swap(RHS&: inserter->capturedInstructions);
160 inserter->setEnabled(true);
161}
162
163InstructionCapturingInserter::CollectionScope::~CollectionScope() {
164 if (!inserter)
165 return;
166
167 previouslyCollectedInstructions.swap(RHS&: inserter->capturedInstructions);
168 // If collection was enabled (likely in another, surrounding scope), keep
169 // the instructions collected in this scope.
170 if (wasEnabled) {
171 llvm::append_range(C&: inserter->capturedInstructions,
172 R&: previouslyCollectedInstructions);
173 }
174 inserter->setEnabled(wasEnabled);
175}
176
177/// Translates the given data layout spec attribute to the LLVM IR data layout.
178/// Only integer, float, pointer and endianness entries are currently supported.
179static FailureOr<llvm::DataLayout>
180translateDataLayout(DataLayoutSpecInterface attribute,
181 const DataLayout &dataLayout,
182 std::optional<Location> loc = std::nullopt) {
183 if (!loc)
184 loc = UnknownLoc::get(context: attribute.getContext());
185
186 // Translate the endianness attribute.
187 std::string llvmDataLayout;
188 llvm::raw_string_ostream layoutStream(llvmDataLayout);
189 for (DataLayoutEntryInterface entry : attribute.getEntries()) {
190 auto key = llvm::dyn_cast_if_present<StringAttr>(Val: entry.getKey());
191 if (!key)
192 continue;
193 if (key.getValue() == DLTIDialect::kDataLayoutEndiannessKey) {
194 auto value = cast<StringAttr>(Val: entry.getValue());
195 bool isLittleEndian =
196 value.getValue() == DLTIDialect::kDataLayoutEndiannessLittle;
197 layoutStream << "-" << (isLittleEndian ? "e" : "E");
198 continue;
199 }
200 if (key.getValue() == DLTIDialect::kDataLayoutManglingModeKey) {
201 auto value = cast<StringAttr>(Val: entry.getValue());
202 layoutStream << "-m:" << value.getValue();
203 continue;
204 }
205 if (key.getValue() == DLTIDialect::kDataLayoutProgramMemorySpaceKey) {
206 auto value = cast<IntegerAttr>(Val: entry.getValue());
207 uint64_t space = value.getValue().getZExtValue();
208 // Skip the default address space.
209 if (space == 0)
210 continue;
211 layoutStream << "-P" << space;
212 continue;
213 }
214 if (key.getValue() == DLTIDialect::kDataLayoutGlobalMemorySpaceKey) {
215 auto value = cast<IntegerAttr>(Val: entry.getValue());
216 uint64_t space = value.getValue().getZExtValue();
217 // Skip the default address space.
218 if (space == 0)
219 continue;
220 layoutStream << "-G" << space;
221 continue;
222 }
223 if (key.getValue() == DLTIDialect::kDataLayoutAllocaMemorySpaceKey) {
224 auto value = cast<IntegerAttr>(Val: entry.getValue());
225 uint64_t space = value.getValue().getZExtValue();
226 // Skip the default address space.
227 if (space == 0)
228 continue;
229 layoutStream << "-A" << space;
230 continue;
231 }
232 if (key.getValue() == DLTIDialect::kDataLayoutStackAlignmentKey) {
233 auto value = cast<IntegerAttr>(Val: entry.getValue());
234 uint64_t alignment = value.getValue().getZExtValue();
235 // Skip the default stack alignment.
236 if (alignment == 0)
237 continue;
238 layoutStream << "-S" << alignment;
239 continue;
240 }
241 if (key.getValue() == DLTIDialect::kDataLayoutFunctionPointerAlignmentKey) {
242 auto value = cast<FunctionPointerAlignmentAttr>(Val: entry.getValue());
243 uint64_t alignment = value.getAlignment();
244 // Skip the default function pointer alignment.
245 if (alignment == 0)
246 continue;
247 layoutStream << "-F" << (value.getFunctionDependent() ? "n" : "i")
248 << alignment;
249 continue;
250 }
251 if (key.getValue() == DLTIDialect::kDataLayoutLegalIntWidthsKey) {
252 layoutStream << "-n";
253 llvm::interleave(
254 c: cast<DenseI32ArrayAttr>(Val: entry.getValue()).asArrayRef(), os&: layoutStream,
255 each_fn: [&](int32_t val) { layoutStream << val; }, separator: ":");
256 continue;
257 }
258 emitError(loc: *loc) << "unsupported data layout key " << key;
259 return failure();
260 }
261
262 // Go through the list of entries to check which types are explicitly
263 // specified in entries. Where possible, data layout queries are used instead
264 // of directly inspecting the entries.
265 for (DataLayoutEntryInterface entry : attribute.getEntries()) {
266 auto type = llvm::dyn_cast_if_present<Type>(Val: entry.getKey());
267 if (!type)
268 continue;
269 // Data layout for the index type is irrelevant at this point.
270 if (isa<IndexType>(Val: type))
271 continue;
272 layoutStream << "-";
273 LogicalResult result =
274 llvm::TypeSwitch<Type, LogicalResult>(type)
275 .Case<IntegerType, Float16Type, Float32Type, Float64Type,
276 Float80Type, Float128Type>(caseFn: [&](Type type) -> LogicalResult {
277 if (auto intType = dyn_cast<IntegerType>(Val&: type)) {
278 if (intType.getSignedness() != IntegerType::Signless)
279 return emitError(loc: *loc)
280 << "unsupported data layout for non-signless integer "
281 << intType;
282 layoutStream << "i";
283 } else {
284 layoutStream << "f";
285 }
286 uint64_t size = dataLayout.getTypeSizeInBits(t: type);
287 uint64_t abi = dataLayout.getTypeABIAlignment(t: type) * 8u;
288 uint64_t preferred =
289 dataLayout.getTypePreferredAlignment(t: type) * 8u;
290 layoutStream << size << ":" << abi;
291 if (abi != preferred)
292 layoutStream << ":" << preferred;
293 return success();
294 })
295 .Case(caseFn: [&](LLVMPointerType type) {
296 layoutStream << "p" << type.getAddressSpace() << ":";
297 uint64_t size = dataLayout.getTypeSizeInBits(t: type);
298 uint64_t abi = dataLayout.getTypeABIAlignment(t: type) * 8u;
299 uint64_t preferred =
300 dataLayout.getTypePreferredAlignment(t: type) * 8u;
301 uint64_t index = *dataLayout.getTypeIndexBitwidth(t: type);
302 layoutStream << size << ":" << abi << ":" << preferred << ":"
303 << index;
304 return success();
305 })
306 .Default(defaultFn: [loc](Type type) {
307 return emitError(loc: *loc)
308 << "unsupported type in data layout: " << type;
309 });
310 if (failed(Result: result))
311 return failure();
312 }
313 StringRef layoutSpec(llvmDataLayout);
314 layoutSpec.consume_front(Prefix: "-");
315
316 return llvm::DataLayout(layoutSpec);
317}
318
319/// Builds a constant of a sequential LLVM type `type`, potentially containing
320/// other sequential types recursively, from the individual constant values
321/// provided in `constants`. `shape` contains the number of elements in nested
322/// sequential types. Reports errors at `loc` and returns nullptr on error.
323static llvm::Constant *
324buildSequentialConstant(ArrayRef<llvm::Constant *> &constants,
325 ArrayRef<int64_t> shape, llvm::Type *type,
326 Location loc) {
327 if (shape.empty()) {
328 llvm::Constant *result = constants.front();
329 constants = constants.drop_front();
330 return result;
331 }
332
333 llvm::Type *elementType;
334 if (auto *arrayTy = dyn_cast<llvm::ArrayType>(Val: type)) {
335 elementType = arrayTy->getElementType();
336 } else if (auto *vectorTy = dyn_cast<llvm::VectorType>(Val: type)) {
337 elementType = vectorTy->getElementType();
338 } else {
339 emitError(loc) << "expected sequential LLVM types wrapping a scalar";
340 return nullptr;
341 }
342
343 SmallVector<llvm::Constant *, 8> nested;
344 nested.reserve(N: shape.front());
345 for (int64_t i = 0; i < shape.front(); ++i) {
346 nested.push_back(Elt: buildSequentialConstant(constants, shape: shape.drop_front(),
347 type: elementType, loc));
348 if (!nested.back())
349 return nullptr;
350 }
351
352 if (shape.size() == 1 && type->isVectorTy())
353 return llvm::ConstantVector::get(V: nested);
354 return llvm::ConstantArray::get(
355 T: llvm::ArrayType::get(ElementType: elementType, NumElements: shape.front()), V: nested);
356}
357
358/// Returns the first non-sequential type nested in sequential types.
359static llvm::Type *getInnermostElementType(llvm::Type *type) {
360 do {
361 if (auto *arrayTy = dyn_cast<llvm::ArrayType>(Val: type)) {
362 type = arrayTy->getElementType();
363 } else if (auto *vectorTy = dyn_cast<llvm::VectorType>(Val: type)) {
364 type = vectorTy->getElementType();
365 } else {
366 return type;
367 }
368 } while (true);
369}
370
371/// Convert a dense elements attribute to an LLVM IR constant using its raw data
372/// storage if possible. This supports elements attributes of tensor or vector
373/// type and avoids constructing separate objects for individual values of the
374/// innermost dimension. Constants for other dimensions are still constructed
375/// recursively. Returns null if constructing from raw data is not supported for
376/// this type, e.g., element type is not a power-of-two-sized primitive. Reports
377/// other errors at `loc`.
378static llvm::Constant *
379convertDenseElementsAttr(Location loc, DenseElementsAttr denseElementsAttr,
380 llvm::Type *llvmType,
381 const ModuleTranslation &moduleTranslation) {
382 if (!denseElementsAttr)
383 return nullptr;
384
385 llvm::Type *innermostLLVMType = getInnermostElementType(type: llvmType);
386 if (!llvm::ConstantDataSequential::isElementTypeCompatible(Ty: innermostLLVMType))
387 return nullptr;
388
389 ShapedType type = denseElementsAttr.getType();
390 if (type.getNumElements() == 0)
391 return nullptr;
392
393 // Check that the raw data size matches what is expected for the scalar size.
394 // TODO: in theory, we could repack the data here to keep constructing from
395 // raw data.
396 // TODO: we may also need to consider endianness when cross-compiling to an
397 // architecture where it is different.
398 int64_t elementByteSize = denseElementsAttr.getRawData().size() /
399 denseElementsAttr.getNumElements();
400 if (8 * elementByteSize != innermostLLVMType->getScalarSizeInBits())
401 return nullptr;
402
403 // Compute the shape of all dimensions but the innermost. Note that the
404 // innermost dimension may be that of the vector element type.
405 bool hasVectorElementType = isa<VectorType>(Val: type.getElementType());
406 int64_t numAggregates =
407 denseElementsAttr.getNumElements() /
408 (hasVectorElementType ? 1
409 : denseElementsAttr.getType().getShape().back());
410 ArrayRef<int64_t> outerShape = type.getShape();
411 if (!hasVectorElementType)
412 outerShape = outerShape.drop_back();
413
414 // Handle the case of vector splat, LLVM has special support for it.
415 if (denseElementsAttr.isSplat() &&
416 (isa<VectorType>(Val: type) || hasVectorElementType)) {
417 llvm::Constant *splatValue = LLVM::detail::getLLVMConstant(
418 llvmType: innermostLLVMType, attr: denseElementsAttr.getSplatValue<Attribute>(), loc,
419 moduleTranslation);
420 llvm::Constant *splatVector =
421 llvm::ConstantDataVector::getSplat(NumElts: 0, Elt: splatValue);
422 SmallVector<llvm::Constant *> constants(numAggregates, splatVector);
423 ArrayRef<llvm::Constant *> constantsRef = constants;
424 return buildSequentialConstant(constants&: constantsRef, shape: outerShape, type: llvmType, loc);
425 }
426 if (denseElementsAttr.isSplat())
427 return nullptr;
428
429 // In case of non-splat, create a constructor for the innermost constant from
430 // a piece of raw data.
431 std::function<llvm::Constant *(StringRef)> buildCstData;
432 if (isa<TensorType>(Val: type)) {
433 auto vectorElementType = dyn_cast<VectorType>(Val: type.getElementType());
434 if (vectorElementType && vectorElementType.getRank() == 1) {
435 buildCstData = [&](StringRef data) {
436 return llvm::ConstantDataVector::getRaw(
437 Data: data, NumElements: vectorElementType.getShape().back(), ElementTy: innermostLLVMType);
438 };
439 } else if (!vectorElementType) {
440 buildCstData = [&](StringRef data) {
441 return llvm::ConstantDataArray::getRaw(Data: data, NumElements: type.getShape().back(),
442 ElementTy: innermostLLVMType);
443 };
444 }
445 } else if (isa<VectorType>(Val: type)) {
446 buildCstData = [&](StringRef data) {
447 return llvm::ConstantDataVector::getRaw(Data: data, NumElements: type.getShape().back(),
448 ElementTy: innermostLLVMType);
449 };
450 }
451 if (!buildCstData)
452 return nullptr;
453
454 // Create innermost constants and defer to the default constant creation
455 // mechanism for other dimensions.
456 SmallVector<llvm::Constant *> constants;
457 int64_t aggregateSize = denseElementsAttr.getType().getShape().back() *
458 (innermostLLVMType->getScalarSizeInBits() / 8);
459 constants.reserve(N: numAggregates);
460 for (unsigned i = 0; i < numAggregates; ++i) {
461 StringRef data(denseElementsAttr.getRawData().data() + i * aggregateSize,
462 aggregateSize);
463 constants.push_back(Elt: buildCstData(data));
464 }
465
466 ArrayRef<llvm::Constant *> constantsRef = constants;
467 return buildSequentialConstant(constants&: constantsRef, shape: outerShape, type: llvmType, loc);
468}
469
470/// Convert a dense resource elements attribute to an LLVM IR constant using its
471/// raw data storage if possible. This supports elements attributes of tensor or
472/// vector type and avoids constructing separate objects for individual values
473/// of the innermost dimension. Constants for other dimensions are still
474/// constructed recursively. Returns nullptr on failure and emits errors at
475/// `loc`.
476static llvm::Constant *convertDenseResourceElementsAttr(
477 Location loc, DenseResourceElementsAttr denseResourceAttr,
478 llvm::Type *llvmType, const ModuleTranslation &moduleTranslation) {
479 assert(denseResourceAttr && "expected non-null attribute");
480
481 llvm::Type *innermostLLVMType = getInnermostElementType(type: llvmType);
482 if (!llvm::ConstantDataSequential::isElementTypeCompatible(
483 Ty: innermostLLVMType)) {
484 emitError(loc, message: "no known conversion for innermost element type");
485 return nullptr;
486 }
487
488 ShapedType type = denseResourceAttr.getType();
489 assert(type.getNumElements() > 0 && "Expected non-empty elements attribute");
490
491 AsmResourceBlob *blob = denseResourceAttr.getRawHandle().getBlob();
492 if (!blob) {
493 emitError(loc, message: "resource does not exist");
494 return nullptr;
495 }
496
497 ArrayRef<char> rawData = blob->getData();
498
499 // Check that the raw data size matches what is expected for the scalar size.
500 // TODO: in theory, we could repack the data here to keep constructing from
501 // raw data.
502 // TODO: we may also need to consider endianness when cross-compiling to an
503 // architecture where it is different.
504 int64_t numElements = denseResourceAttr.getType().getNumElements();
505 int64_t elementByteSize = rawData.size() / numElements;
506 if (8 * elementByteSize != innermostLLVMType->getScalarSizeInBits()) {
507 emitError(loc, message: "raw data size does not match element type size");
508 return nullptr;
509 }
510
511 // Compute the shape of all dimensions but the innermost. Note that the
512 // innermost dimension may be that of the vector element type.
513 bool hasVectorElementType = isa<VectorType>(Val: type.getElementType());
514 int64_t numAggregates =
515 numElements / (hasVectorElementType
516 ? 1
517 : denseResourceAttr.getType().getShape().back());
518 ArrayRef<int64_t> outerShape = type.getShape();
519 if (!hasVectorElementType)
520 outerShape = outerShape.drop_back();
521
522 // Create a constructor for the innermost constant from a piece of raw data.
523 std::function<llvm::Constant *(StringRef)> buildCstData;
524 if (isa<TensorType>(Val: type)) {
525 auto vectorElementType = dyn_cast<VectorType>(Val: type.getElementType());
526 if (vectorElementType && vectorElementType.getRank() == 1) {
527 buildCstData = [&](StringRef data) {
528 return llvm::ConstantDataVector::getRaw(
529 Data: data, NumElements: vectorElementType.getShape().back(), ElementTy: innermostLLVMType);
530 };
531 } else if (!vectorElementType) {
532 buildCstData = [&](StringRef data) {
533 return llvm::ConstantDataArray::getRaw(Data: data, NumElements: type.getShape().back(),
534 ElementTy: innermostLLVMType);
535 };
536 }
537 } else if (isa<VectorType>(Val: type)) {
538 buildCstData = [&](StringRef data) {
539 return llvm::ConstantDataVector::getRaw(Data: data, NumElements: type.getShape().back(),
540 ElementTy: innermostLLVMType);
541 };
542 }
543 if (!buildCstData) {
544 emitError(loc, message: "unsupported dense_resource type");
545 return nullptr;
546 }
547
548 // Create innermost constants and defer to the default constant creation
549 // mechanism for other dimensions.
550 SmallVector<llvm::Constant *> constants;
551 int64_t aggregateSize = denseResourceAttr.getType().getShape().back() *
552 (innermostLLVMType->getScalarSizeInBits() / 8);
553 constants.reserve(N: numAggregates);
554 for (unsigned i = 0; i < numAggregates; ++i) {
555 StringRef data(rawData.data() + i * aggregateSize, aggregateSize);
556 constants.push_back(Elt: buildCstData(data));
557 }
558
559 ArrayRef<llvm::Constant *> constantsRef = constants;
560 return buildSequentialConstant(constants&: constantsRef, shape: outerShape, type: llvmType, loc);
561}
562
563/// Create an LLVM IR constant of `llvmType` from the MLIR attribute `attr`.
564/// This currently supports integer, floating point, splat and dense element
565/// attributes and combinations thereof. Also, an array attribute with two
566/// elements is supported to represent a complex constant. In case of error,
567/// report it to `loc` and return nullptr.
568llvm::Constant *mlir::LLVM::detail::getLLVMConstant(
569 llvm::Type *llvmType, Attribute attr, Location loc,
570 const ModuleTranslation &moduleTranslation) {
571 if (!attr || isa<UndefAttr>(Val: attr))
572 return llvm::UndefValue::get(T: llvmType);
573 if (isa<ZeroAttr>(Val: attr))
574 return llvm::Constant::getNullValue(Ty: llvmType);
575 if (auto *structType = dyn_cast<::llvm::StructType>(Val: llvmType)) {
576 auto arrayAttr = dyn_cast<ArrayAttr>(Val&: attr);
577 if (!arrayAttr) {
578 emitError(loc, message: "expected an array attribute for a struct constant");
579 return nullptr;
580 }
581 SmallVector<llvm::Constant *> structElements;
582 structElements.reserve(N: structType->getNumElements());
583 for (auto [elemType, elemAttr] :
584 zip_equal(t: structType->elements(), u&: arrayAttr)) {
585 llvm::Constant *element =
586 getLLVMConstant(llvmType: elemType, attr: elemAttr, loc, moduleTranslation);
587 if (!element)
588 return nullptr;
589 structElements.push_back(Elt: element);
590 }
591 return llvm::ConstantStruct::get(T: structType, V: structElements);
592 }
593 // For integer types, we allow a mismatch in sizes as the index type in
594 // MLIR might have a different size than the index type in the LLVM module.
595 if (auto intAttr = dyn_cast<IntegerAttr>(Val&: attr)) {
596 // If the attribute is an unsigned integer or a 1-bit integer, zero-extend
597 // the value to the bit width of the LLVM type. Otherwise, sign-extend.
598 auto intTy = dyn_cast<IntegerType>(Val: intAttr.getType());
599 APInt value;
600 if (intTy && (intTy.isUnsigned() || intTy.getWidth() == 1))
601 value = intAttr.getValue().zextOrTrunc(width: llvmType->getIntegerBitWidth());
602 else
603 value = intAttr.getValue().sextOrTrunc(width: llvmType->getIntegerBitWidth());
604 return llvm::ConstantInt::get(Ty: llvmType, V: value);
605 }
606 if (auto floatAttr = dyn_cast<FloatAttr>(Val&: attr)) {
607 const llvm::fltSemantics &sem = floatAttr.getValue().getSemantics();
608 // Special case for 8-bit floats, which are represented by integers due to
609 // the lack of native fp8 types in LLVM at the moment. Additionally, handle
610 // targets (like AMDGPU) that don't implement bfloat and convert all bfloats
611 // to i16.
612 unsigned floatWidth = APFloat::getSizeInBits(Sem: sem);
613 if (llvmType->isIntegerTy(BitWidth: floatWidth))
614 return llvm::ConstantInt::get(Ty: llvmType,
615 V: floatAttr.getValue().bitcastToAPInt());
616 if (llvmType !=
617 llvm::Type::getFloatingPointTy(C&: llvmType->getContext(),
618 S: floatAttr.getValue().getSemantics())) {
619 emitError(loc, message: "FloatAttr does not match expected type of the constant");
620 return nullptr;
621 }
622 return llvm::ConstantFP::get(Ty: llvmType, V: floatAttr.getValue());
623 }
624 if (auto symAttr = dyn_cast<FlatSymbolRefAttr>(Val&: attr)) {
625 StringRef name = symAttr.getValue();
626 if (llvm::Function *func = moduleTranslation.lookupFunction(name))
627 return llvm::ConstantExpr::getBitCast(C: func, Ty: llvmType);
628 if (llvm::GlobalValue *global = moduleTranslation.lookupGlobal(name))
629 return llvm::ConstantExpr::getBitCast(C: global, Ty: llvmType);
630 emitError(loc, message: "unknown symbol reference '") << name << "' in constant";
631 return nullptr;
632 }
633 if (auto splatAttr = dyn_cast<SplatElementsAttr>(Val&: attr)) {
634 llvm::Type *elementType;
635 uint64_t numElements;
636 bool isScalable = false;
637 if (auto *arrayTy = dyn_cast<llvm::ArrayType>(Val: llvmType)) {
638 elementType = arrayTy->getElementType();
639 numElements = arrayTy->getNumElements();
640 } else if (auto *fVectorTy = dyn_cast<llvm::FixedVectorType>(Val: llvmType)) {
641 elementType = fVectorTy->getElementType();
642 numElements = fVectorTy->getNumElements();
643 } else if (auto *sVectorTy = dyn_cast<llvm::ScalableVectorType>(Val: llvmType)) {
644 elementType = sVectorTy->getElementType();
645 numElements = sVectorTy->getMinNumElements();
646 isScalable = true;
647 } else {
648 llvm_unreachable("unrecognized constant vector type");
649 }
650 // Splat value is a scalar. Extract it only if the element type is not
651 // another sequence type. The recursion terminates because each step removes
652 // one outer sequential type.
653 bool elementTypeSequential =
654 isa<llvm::ArrayType, llvm::VectorType>(Val: elementType);
655 llvm::Constant *child = getLLVMConstant(
656 llvmType: elementType,
657 attr: elementTypeSequential ? splatAttr
658 : splatAttr.getSplatValue<Attribute>(),
659 loc, moduleTranslation);
660 if (!child)
661 return nullptr;
662 if (llvmType->isVectorTy())
663 return llvm::ConstantVector::getSplat(
664 EC: llvm::ElementCount::get(MinVal: numElements, /*Scalable=*/isScalable), Elt: child);
665 if (llvmType->isArrayTy()) {
666 auto *arrayType = llvm::ArrayType::get(ElementType: elementType, NumElements: numElements);
667 if (child->isNullValue() && !elementType->isFPOrFPVectorTy()) {
668 return llvm::ConstantAggregateZero::get(Ty: arrayType);
669 }
670 if (llvm::ConstantDataSequential::isElementTypeCompatible(Ty: elementType)) {
671 if (isa<llvm::IntegerType>(Val: elementType)) {
672 if (llvm::ConstantInt *ci = dyn_cast<llvm::ConstantInt>(Val: child)) {
673 if (ci->getBitWidth() == 8) {
674 SmallVector<int8_t> constants(numElements, ci->getZExtValue());
675 return llvm::ConstantDataArray::get(Context&: elementType->getContext(),
676 Elts&: constants);
677 }
678 if (ci->getBitWidth() == 16) {
679 SmallVector<int16_t> constants(numElements, ci->getZExtValue());
680 return llvm::ConstantDataArray::get(Context&: elementType->getContext(),
681 Elts&: constants);
682 }
683 if (ci->getBitWidth() == 32) {
684 SmallVector<int32_t> constants(numElements, ci->getZExtValue());
685 return llvm::ConstantDataArray::get(Context&: elementType->getContext(),
686 Elts&: constants);
687 }
688 if (ci->getBitWidth() == 64) {
689 SmallVector<int64_t> constants(numElements, ci->getZExtValue());
690 return llvm::ConstantDataArray::get(Context&: elementType->getContext(),
691 Elts&: constants);
692 }
693 }
694 }
695 if (elementType->isFloatingPointTy()) {
696 if (llvm::ConstantFP *cfp = dyn_cast<llvm::ConstantFP>(Val: child)) {
697 APInt bitPattern = cfp->getValueAPF().bitcastToAPInt();
698 uint64_t value = bitPattern.getZExtValue();
699 // TODO: This code only handles 16, 32, and 64 bit floats. Handle
700 // all compatible types, fp8, fp4, etc.
701 if (bitPattern.getBitWidth() == 16) {
702 SmallVector<uint16_t> constants(numElements, value);
703 return llvm::ConstantDataArray::getFP(ElementType: elementType, Elts: constants);
704 }
705 if (bitPattern.getBitWidth() == 32) {
706 SmallVector<uint32_t> constants(numElements, value);
707 return llvm::ConstantDataArray::getFP(ElementType: elementType, Elts: constants);
708 }
709 if (bitPattern.getBitWidth() == 64) {
710 SmallVector<uint64_t> constants(numElements, value);
711 return llvm::ConstantDataArray::getFP(ElementType: elementType, Elts: constants);
712 }
713 }
714 }
715 }
716 // std::vector is used here to accomodate large number of elements that
717 // exceed SmallVector capacity.
718 std::vector<llvm::Constant *> constants(numElements, child);
719 return llvm::ConstantArray::get(T: arrayType, V: constants);
720 }
721 }
722
723 // Try using raw elements data if possible.
724 if (llvm::Constant *result =
725 convertDenseElementsAttr(loc, denseElementsAttr: dyn_cast<DenseElementsAttr>(Val&: attr),
726 llvmType, moduleTranslation)) {
727 return result;
728 }
729
730 if (auto denseResourceAttr = dyn_cast<DenseResourceElementsAttr>(Val&: attr)) {
731 return convertDenseResourceElementsAttr(loc, denseResourceAttr, llvmType,
732 moduleTranslation);
733 }
734
735 // Fall back to element-by-element construction otherwise.
736 if (auto elementsAttr = dyn_cast<ElementsAttr>(Val&: attr)) {
737 assert(elementsAttr.getShapedType().hasStaticShape());
738 assert(!elementsAttr.getShapedType().getShape().empty() &&
739 "unexpected empty elements attribute shape");
740
741 SmallVector<llvm::Constant *, 8> constants;
742 constants.reserve(N: elementsAttr.getNumElements());
743 llvm::Type *innermostType = getInnermostElementType(type: llvmType);
744 for (auto n : elementsAttr.getValues<Attribute>()) {
745 constants.push_back(
746 Elt: getLLVMConstant(llvmType: innermostType, attr: n, loc, moduleTranslation));
747 if (!constants.back())
748 return nullptr;
749 }
750 ArrayRef<llvm::Constant *> constantsRef = constants;
751 llvm::Constant *result = buildSequentialConstant(
752 constants&: constantsRef, shape: elementsAttr.getShapedType().getShape(), type: llvmType, loc);
753 assert(constantsRef.empty() && "did not consume all elemental constants");
754 return result;
755 }
756
757 if (auto stringAttr = dyn_cast<StringAttr>(Val&: attr)) {
758 return llvm::ConstantDataArray::get(Context&: moduleTranslation.getLLVMContext(),
759 Elts: ArrayRef<char>{stringAttr.getValue()});
760 }
761
762 // Handle arrays of structs that cannot be represented as DenseElementsAttr
763 // in MLIR.
764 if (auto arrayAttr = dyn_cast<ArrayAttr>(Val&: attr)) {
765 if (auto *arrayTy = dyn_cast<llvm::ArrayType>(Val: llvmType)) {
766 llvm::Type *elementType = arrayTy->getElementType();
767 Attribute previousElementAttr;
768 llvm::Constant *elementCst = nullptr;
769 SmallVector<llvm::Constant *> constants;
770 constants.reserve(N: arrayTy->getNumElements());
771 for (Attribute elementAttr : arrayAttr) {
772 // Arrays with a single value or with repeating values are quite common.
773 // Short-circuit the translation when the element value is the same as
774 // the previous one.
775 if (!previousElementAttr || previousElementAttr != elementAttr) {
776 previousElementAttr = elementAttr;
777 elementCst =
778 getLLVMConstant(llvmType: elementType, attr: elementAttr, loc, moduleTranslation);
779 if (!elementCst)
780 return nullptr;
781 }
782 constants.push_back(Elt: elementCst);
783 }
784 return llvm::ConstantArray::get(T: arrayTy, V: constants);
785 }
786 }
787
788 emitError(loc, message: "unsupported constant value");
789 return nullptr;
790}
791
792ModuleTranslation::ModuleTranslation(Operation *module,
793 std::unique_ptr<llvm::Module> llvmModule,
794 llvm::vfs::FileSystem *fs)
795 : mlirModule(module), llvmModule(std::move(llvmModule)),
796 debugTranslation(
797 std::make_unique<DebugTranslation>(args&: module, args&: *this->llvmModule)),
798 loopAnnotationTranslation(std::make_unique<LoopAnnotationTranslation>(
799 args&: *this, args&: *this->llvmModule)),
800 fileSystem(fs), typeTranslator(this->llvmModule->getContext()),
801 iface(module->getContext()) {
802 assert(satisfiesLLVMModule(mlirModule) &&
803 "mlirModule should honor LLVM's module semantics.");
804}
805
806ModuleTranslation::~ModuleTranslation() {
807 // Destroy extensions while the rest of the translation state is still alive.
808 extensions.clear();
809}
810
811ModuleTranslation::Extension::~Extension() = default;
812
813void ModuleTranslation::forgetMapping(Region &region) {
814 SmallVector<Region *> toProcess;
815 toProcess.push_back(Elt: &region);
816 while (!toProcess.empty()) {
817 Region *current = toProcess.pop_back_val();
818 for (Block &block : *current) {
819 blockMapping.erase(Val: &block);
820 for (Value arg : block.getArguments())
821 valueMapping.erase(Val: arg);
822 for (Operation &op : block) {
823 for (Value value : op.getResults())
824 valueMapping.erase(Val: value);
825 if (op.hasSuccessors())
826 branchMapping.erase(Val: &op);
827 if (isa<LLVM::GlobalOp>(Val: op))
828 globalsMapping.erase(Val: &op);
829 if (isa<LLVM::AliasOp>(Val: op))
830 aliasesMapping.erase(Val: &op);
831 if (isa<LLVM::IFuncOp>(Val: op))
832 ifuncMapping.erase(Val: &op);
833 if (isa<LLVM::CallOp>(Val: op))
834 callMapping.erase(Val: &op);
835 llvm::append_range(
836 C&: toProcess,
837 R: llvm::map_range(C: op.getRegions(), F: [](Region &r) { return &r; }));
838 }
839 }
840 }
841}
842
843/// Get the SSA value passed to the current block from the terminator operation
844/// of its predecessor.
845static Value getPHISourceValue(Block *current, Block *pred,
846 unsigned numArguments, unsigned index) {
847 Operation &terminator = *pred->getTerminator();
848 if (isa<LLVM::BrOp>(Val: terminator))
849 return terminator.getOperand(idx: index);
850
851#ifndef NDEBUG
852 llvm::SmallPtrSet<Block *, 4> seenSuccessors;
853 for (unsigned i = 0, e = terminator.getNumSuccessors(); i < e; ++i) {
854 Block *successor = terminator.getSuccessor(i);
855 auto branch = cast<BranchOpInterface>(terminator);
856 SuccessorOperands successorOperands = branch.getSuccessorOperands(i);
857 assert(
858 (!seenSuccessors.contains(successor) || successorOperands.empty()) &&
859 "successors with arguments in LLVM branches must be different blocks");
860 seenSuccessors.insert(successor);
861 }
862#endif
863
864 // For instructions that branch based on a condition value, we need to take
865 // the operands for the branch that was taken.
866 if (auto condBranchOp = dyn_cast<LLVM::CondBrOp>(Val&: terminator)) {
867 // For conditional branches, we take the operands from either the "true" or
868 // the "false" branch.
869 return condBranchOp.getSuccessor(i: 0) == current
870 ? condBranchOp.getTrueDestOperands()[index]
871 : condBranchOp.getFalseDestOperands()[index];
872 }
873
874 if (auto switchOp = dyn_cast<LLVM::SwitchOp>(Val&: terminator)) {
875 // For switches, we take the operands from either the default case, or from
876 // the case branch that was taken.
877 if (switchOp.getDefaultDestination() == current)
878 return switchOp.getDefaultOperands()[index];
879 for (const auto &i : llvm::enumerate(First: switchOp.getCaseDestinations()))
880 if (i.value() == current)
881 return switchOp.getCaseOperands(index: i.index())[index];
882 }
883
884 if (auto indBrOp = dyn_cast<LLVM::IndirectBrOp>(Val&: terminator)) {
885 // For indirect branches we take operands for each successor.
886 for (const auto &i : llvm::enumerate(First: indBrOp->getSuccessors())) {
887 if (indBrOp->getSuccessor(index: i.index()) == current)
888 return indBrOp.getSuccessorOperands(index: i.index())[index];
889 }
890 }
891
892 if (auto invokeOp = dyn_cast<LLVM::InvokeOp>(Val&: terminator)) {
893 return invokeOp.getNormalDest() == current
894 ? invokeOp.getNormalDestOperands()[index]
895 : invokeOp.getUnwindDestOperands()[index];
896 }
897
898 llvm_unreachable(
899 "only branch, switch or invoke operations can be terminators "
900 "of a block that has successors");
901}
902
903/// Connect the PHI nodes to the results of preceding blocks.
904void mlir::LLVM::detail::connectPHINodes(Region &region,
905 const ModuleTranslation &state) {
906 // Skip the first block, it cannot be branched to and its arguments correspond
907 // to the arguments of the LLVM function.
908 for (Block &bb : llvm::drop_begin(RangeOrContainer&: region)) {
909 llvm::BasicBlock *llvmBB = state.lookupBlock(block: &bb);
910 auto phis = llvmBB->phis();
911 unsigned numArguments = bb.getNumArguments();
912 assert(numArguments == llvm::range_size(phis));
913 for (auto [index, phiNode] : llvm::enumerate(First&: phis)) {
914 for (auto *pred : bb.getPredecessors()) {
915 // Find the LLVM IR block that contains the converted terminator
916 // instruction and use it in the PHI node. Note that this block is not
917 // necessarily the same as state.lookupBlock(pred), some operations
918 // (in particular, OpenMP operations using OpenMPIRBuilder) may have
919 // split the blocks.
920 llvm::Instruction *terminator =
921 state.lookupBranch(op: pred->getTerminator());
922 assert(terminator && "missing the mapping for a terminator");
923 phiNode.addIncoming(V: state.lookupValue(value: getPHISourceValue(
924 current: &bb, pred, numArguments, index)),
925 BB: terminator->getParent());
926 }
927 }
928 }
929}
930
931llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall(
932 llvm::IRBuilderBase &builder, llvm::Intrinsic::ID intrinsic,
933 ArrayRef<llvm::Value *> args, ArrayRef<llvm::Type *> tys) {
934 return builder.CreateIntrinsicWithoutFolding(ID: intrinsic, OverloadTypes: tys, Args: args);
935}
936
937llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall(
938 llvm::IRBuilderBase &builder, llvm::Intrinsic::ID intrinsic,
939 llvm::Type *retTy, ArrayRef<llvm::Value *> args) {
940 return builder.CreateIntrinsicWithoutFolding(RetTy: retTy, ID: intrinsic, Args: args);
941}
942
943llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall(
944 llvm::IRBuilderBase &builder, ModuleTranslation &moduleTranslation,
945 Operation *intrOp, llvm::Intrinsic::ID intrinsic, unsigned numResults,
946 ArrayRef<unsigned> overloadedResults, ArrayRef<unsigned> overloadedOperands,
947 ArrayRef<unsigned> immArgPositions,
948 ArrayRef<StringLiteral> immArgAttrNames) {
949 assert(immArgPositions.size() == immArgAttrNames.size() &&
950 "LLVM `immArgPositions` and MLIR `immArgAttrNames` should have equal "
951 "length");
952
953 SmallVector<llvm::OperandBundleDef> opBundles;
954 size_t numOpBundleOperands = 0;
955 auto opBundleSizesAttr = cast_if_present<DenseI32ArrayAttr>(
956 Val: intrOp->getInherentAttr(name: LLVMDialect::getOpBundleSizesAttrName())
957 .value_or(u: Attribute{}));
958 auto opBundleTagsAttr = cast_if_present<ArrayAttr>(
959 Val: intrOp->getInherentAttr(name: LLVMDialect::getOpBundleTagsAttrName())
960 .value_or(u: Attribute{}));
961
962 if (opBundleSizesAttr && opBundleTagsAttr) {
963 ArrayRef<int> opBundleSizes = opBundleSizesAttr.asArrayRef();
964 assert(opBundleSizes.size() == opBundleTagsAttr.size() &&
965 "operand bundles and tags do not match");
966
967 numOpBundleOperands = llvm::sum_of(Range&: opBundleSizes);
968 assert(numOpBundleOperands <= intrOp->getNumOperands() &&
969 "operand bundle operands is more than the number of operands");
970
971 ValueRange operands = intrOp->getOperands().take_back(n: numOpBundleOperands);
972 size_t nextOperandIdx = 0;
973 opBundles.reserve(N: opBundleSizesAttr.size());
974
975 for (auto [opBundleTagAttr, bundleSize] :
976 llvm::zip(t&: opBundleTagsAttr, u&: opBundleSizes)) {
977 auto bundleTag = cast<StringAttr>(Val: opBundleTagAttr).str();
978 auto bundleOperands = moduleTranslation.lookupValues(
979 values: operands.slice(n: nextOperandIdx, m: bundleSize));
980 opBundles.emplace_back(Args: std::move(bundleTag), Args: std::move(bundleOperands));
981 nextOperandIdx += bundleSize;
982 }
983 }
984
985 // Map operands and attributes to LLVM values.
986 auto opOperands = intrOp->getOperands().drop_back(n: numOpBundleOperands);
987 auto operands = moduleTranslation.lookupValues(values: opOperands);
988 SmallVector<llvm::Value *> args(immArgPositions.size() + operands.size());
989 for (auto [immArgPos, immArgName] :
990 llvm::zip(t&: immArgPositions, u&: immArgAttrNames)) {
991 Attribute attr = intrOp->getInherentAttr(name: immArgName).value_or(u: Attribute{});
992 if (auto intrinsicIntegerAttr =
993 dyn_cast<LLVM::IntrinsicIntegerAttrInterface>(Val&: attr))
994 attr = intrinsicIntegerAttr.getIntegerAttr();
995 auto typedAttr = llvm::cast<TypedAttr>(Val&: attr);
996 assert(typedAttr.getType().isIntOrFloat() &&
997 "expected int or float immarg");
998 auto *type = moduleTranslation.convertType(type: typedAttr.getType());
999 args[immArgPos] = LLVM::detail::getLLVMConstant(
1000 llvmType: type, attr: typedAttr, loc: intrOp->getLoc(), moduleTranslation);
1001 }
1002 unsigned opArg = 0;
1003 for (auto &arg : args) {
1004 if (!arg)
1005 arg = operands[opArg++];
1006 }
1007
1008 // Resolve overloaded intrinsic declaration.
1009 SmallVector<llvm::Type *> overloadedTypes;
1010 for (unsigned overloadedResultIdx : overloadedResults) {
1011 if (numResults > 1) {
1012 // More than one result is mapped to an LLVM struct.
1013 overloadedTypes.push_back(Elt: moduleTranslation.convertType(
1014 type: llvm::cast<LLVM::LLVMStructType>(Val: intrOp->getResult(idx: 0).getType())
1015 .getBody()[overloadedResultIdx]));
1016 } else {
1017 overloadedTypes.push_back(
1018 Elt: moduleTranslation.convertType(type: intrOp->getResult(idx: 0).getType()));
1019 }
1020 }
1021 for (unsigned overloadedOperandIdx : overloadedOperands)
1022 overloadedTypes.push_back(Elt: args[overloadedOperandIdx]->getType());
1023 llvm::Module *module = builder.getModule();
1024 llvm::Function *llvmIntr = llvm::Intrinsic::getOrInsertDeclaration(
1025 M: module, id: intrinsic, OverloadTys: overloadedTypes);
1026
1027 return builder.CreateCall(Callee: llvmIntr, Args: args, OpBundles: opBundles);
1028}
1029
1030/// Given a single MLIR operation, create the corresponding LLVM IR operation
1031/// using the `builder`.
1032LogicalResult ModuleTranslation::convertOperationImpl(
1033 Operation &op, llvm::IRBuilderBase &builder, bool recordInsertions) {
1034 const LLVMTranslationDialectInterface *opIface = iface.getInterfaceFor(obj: &op);
1035 if (!opIface)
1036 return op.emitError(message: "cannot be converted to LLVM IR: missing "
1037 "`LLVMTranslationDialectInterface` registration for "
1038 "dialect for op: ")
1039 << op.getName();
1040
1041 InstructionCapturingInserter::CollectionScope scope(builder,
1042 recordInsertions);
1043 if (failed(Result: opIface->convertOperation(op: &op, builder, moduleTranslation&: *this)))
1044 return op.emitError(message: "LLVM Translation failed for operation: ")
1045 << op.getName();
1046
1047 return convertDialectAttributes(op: &op, instructions: scope.getCapturedInstructions());
1048}
1049
1050/// Convert block to LLVM IR. Unless `ignoreArguments` is set, emit PHI nodes
1051/// to define values corresponding to the MLIR block arguments. These nodes
1052/// are not connected to the source basic blocks, which may not exist yet. Uses
1053/// `builder` to construct the LLVM IR. Expects the LLVM IR basic block to have
1054/// been created for `bb` and included in the block mapping. Inserts new
1055/// instructions at the end of the block and leaves `builder` in a state
1056/// suitable for further insertion into the end of the block.
1057LogicalResult ModuleTranslation::convertBlockImpl(Block &bb,
1058 bool ignoreArguments,
1059 llvm::IRBuilderBase &builder,
1060 bool recordInsertions) {
1061 builder.SetInsertPoint(lookupBlock(block: &bb));
1062 auto *subprogram = builder.GetInsertBlock()->getParent()->getSubprogram();
1063
1064 // Before traversing operations, make block arguments available through
1065 // value remapping and PHI nodes, but do not add incoming edges for the PHI
1066 // nodes just yet: those values may be defined by this or following blocks.
1067 // This step is omitted if "ignoreArguments" is set. The arguments of the
1068 // first block have been already made available through the remapping of
1069 // LLVM function arguments.
1070 if (!ignoreArguments) {
1071 auto predecessors = bb.getPredecessors();
1072 unsigned numPredecessors =
1073 std::distance(first: predecessors.begin(), last: predecessors.end());
1074 for (auto arg : bb.getArguments()) {
1075 auto wrappedType = arg.getType();
1076 if (!isCompatibleType(type: wrappedType))
1077 return emitError(loc: bb.front().getLoc(),
1078 message: "block argument does not have an LLVM type");
1079 builder.SetCurrentDebugLocation(
1080 debugTranslation->translateLoc(loc: arg.getLoc(), scope: subprogram));
1081 llvm::Type *type = convertType(type: wrappedType);
1082 llvm::PHINode *phi = builder.CreatePHI(Ty: type, NumReservedValues: numPredecessors);
1083 mapValue(mlir: arg, llvm: phi);
1084 }
1085 }
1086
1087 // Traverse operations.
1088 for (auto &op : bb) {
1089 // Set the current debug location within the builder.
1090 builder.SetCurrentDebugLocation(
1091 debugTranslation->translateLoc(loc: op.getLoc(), scope: subprogram));
1092
1093 if (failed(Result: convertOperationImpl(op, builder, recordInsertions)))
1094 return failure();
1095
1096 // Set the branch weight metadata on the translated instruction.
1097 if (auto iface = dyn_cast<WeightedBranchOpInterface>(Val&: op))
1098 setBranchWeightsMetadata(iface);
1099 }
1100
1101 return success();
1102}
1103
1104/// A helper method to get the single Block in an operation honoring LLVM's
1105/// module requirements.
1106static Block &getModuleBody(Operation *module) {
1107 return module->getRegion(index: 0).front();
1108}
1109
1110/// A helper method to decide if a constant must not be set as a global variable
1111/// initializer. For an external linkage variable, the variable with an
1112/// initializer is considered externally visible and defined in this module, the
1113/// variable without an initializer is externally available and is defined
1114/// elsewhere.
1115static bool shouldDropGlobalInitializer(llvm::GlobalValue::LinkageTypes linkage,
1116 llvm::Constant *cst) {
1117 return (linkage == llvm::GlobalVariable::ExternalLinkage && !cst) ||
1118 linkage == llvm::GlobalVariable::ExternalWeakLinkage;
1119}
1120
1121/// Sets the runtime preemption specifier of `gv` to dso_local if
1122/// `dsoLocalRequested` is true, otherwise it is left unchanged.
1123static void addRuntimePreemptionSpecifier(bool dsoLocalRequested,
1124 llvm::GlobalValue *gv) {
1125 if (dsoLocalRequested)
1126 gv->setDSOLocal(true);
1127}
1128
1129/// Attempts to translate an MLIR attribute identified by `key`, optionally with
1130/// the given `value`, into an LLVM IR attribute. Reports errors at `loc` if
1131/// any. If the attribute name corresponds to a known LLVM IR attribute kind,
1132/// creates the LLVM attribute of that kind; otherwise, keeps it as a string
1133/// attribute. Performs additional checks for attributes known to have or not
1134/// have a value in order to avoid assertions inside LLVM upon construction.
1135static FailureOr<llvm::Attribute>
1136convertMLIRAttributeToLLVM(Location loc, llvm::LLVMContext &ctx, StringRef key,
1137 StringRef value = StringRef()) {
1138 auto kind = llvm::Attribute::getAttrKindFromName(AttrName: key);
1139 if (kind == llvm::Attribute::None)
1140 return llvm::Attribute::get(Context&: ctx, Kind: key, Val: value);
1141
1142 if (llvm::Attribute::isIntAttrKind(Kind: kind)) {
1143 if (value.empty())
1144 return emitError(loc) << "LLVM attribute '" << key << "' expects a value";
1145
1146 int64_t result;
1147 if (!value.getAsInteger(/*Radix=*/0, Result&: result))
1148 return llvm::Attribute::get(Context&: ctx, Kind: kind, Val: result);
1149 return llvm::Attribute::get(Context&: ctx, Kind: key, Val: value);
1150 }
1151
1152 if (!value.empty())
1153 return emitError(loc) << "LLVM attribute '" << key
1154 << "' does not expect a value, found '" << value
1155 << "'";
1156
1157 return llvm::Attribute::get(Context&: ctx, Kind: kind);
1158}
1159
1160/// Converts the MLIR attributes listed in the given array attribute into LLVM
1161/// attributes. Returns an `AttrBuilder` containing the converted attributes.
1162/// Reports error to `loc` if any and returns immediately. Expects `arrayAttr`
1163/// to contain either string attributes, treated as value-less LLVM attributes,
1164/// or array attributes containing two string attributes, with the first string
1165/// being the name of the corresponding LLVM attribute and the second string
1166/// beings its value. Note that even integer attributes are expected to have
1167/// their values expressed as strings.
1168static FailureOr<llvm::AttrBuilder>
1169convertMLIRAttributesToLLVM(Location loc, llvm::LLVMContext &ctx,
1170 ArrayAttr arrayAttr, StringRef arrayAttrName) {
1171 llvm::AttrBuilder attrBuilder(ctx);
1172 if (!arrayAttr)
1173 return attrBuilder;
1174
1175 for (Attribute attr : arrayAttr) {
1176 if (auto stringAttr = dyn_cast<StringAttr>(Val&: attr)) {
1177 FailureOr<llvm::Attribute> llvmAttr =
1178 convertMLIRAttributeToLLVM(loc, ctx, key: stringAttr.getValue());
1179 if (failed(Result: llvmAttr))
1180 return failure();
1181 attrBuilder.addAttribute(A: *llvmAttr);
1182 continue;
1183 }
1184
1185 auto arrayAttr = dyn_cast<ArrayAttr>(Val&: attr);
1186 if (!arrayAttr || arrayAttr.size() != 2)
1187 return emitError(loc) << "expected '" << arrayAttrName
1188 << "' to contain string or array attributes";
1189
1190 auto keyAttr = dyn_cast<StringAttr>(Val: arrayAttr[0]);
1191 auto valueAttr = dyn_cast<StringAttr>(Val: arrayAttr[1]);
1192 if (!keyAttr || !valueAttr)
1193 return emitError(loc) << "expected arrays within '" << arrayAttrName
1194 << "' to contain two strings";
1195
1196 FailureOr<llvm::Attribute> llvmAttr = convertMLIRAttributeToLLVM(
1197 loc, ctx, key: keyAttr.getValue(), value: valueAttr.getValue());
1198 if (failed(Result: llvmAttr))
1199 return failure();
1200 attrBuilder.addAttribute(A: *llvmAttr);
1201 }
1202
1203 return attrBuilder;
1204}
1205
1206LogicalResult ModuleTranslation::convertGlobalsAndAliases() {
1207 // Mapping from compile unit to its respective set of global variables.
1208 DenseMap<llvm::DICompileUnit *, SmallVector<llvm::Metadata *>> globalGVars;
1209 // Mapping from subprogram to its respective set of static local variables.
1210 DenseMap<llvm::DISubprogram *, SmallVector<llvm::Metadata *>> staticLocals;
1211
1212 // First, create all global variables and global aliases in LLVM IR. A global
1213 // or alias body may refer to another global/alias or itself, so all the
1214 // mapping needs to happen prior to body conversion.
1215
1216 // Create all llvm::GlobalVariable
1217 for (auto op : getModuleBody(module: mlirModule).getOps<LLVM::GlobalOp>()) {
1218 llvm::Type *type = convertType(type: op.getType());
1219 llvm::Constant *cst = nullptr;
1220 const bool deferValueAttrToPass2 = op.getValueOrNull() &&
1221 !op.getInitializerBlock() &&
1222 !isa<StringAttr>(Val: op.getValueOrNull());
1223 if (op.getValueOrNull() && !deferValueAttrToPass2) {
1224 // String attributes are treated separately because they cannot appear as
1225 // in-function constants and are thus not supported by getLLVMConstant.
1226 if (auto strAttr = dyn_cast_or_null<StringAttr>(Val: op.getValueOrNull())) {
1227 cst = llvm::ConstantDataArray::getString(
1228 Context&: llvmModule->getContext(), Initializer: strAttr.getValue(), /*AddNull=*/false);
1229 type = cst->getType();
1230 }
1231 }
1232
1233 auto linkage = convertLinkageToLLVM(value: op.getLinkage());
1234
1235 // LLVM IR requires constant with linkage other than external or weak
1236 // external to have initializers. If MLIR does not provide an initializer,
1237 // default to undef.
1238 bool dropInitializer = shouldDropGlobalInitializer(linkage, cst);
1239 if (!deferValueAttrToPass2) {
1240 if (!dropInitializer && !cst)
1241 cst = llvm::UndefValue::get(T: type);
1242 else if (dropInitializer && cst)
1243 cst = nullptr;
1244 } else {
1245 cst = nullptr;
1246 }
1247
1248 auto *var = new llvm::GlobalVariable(
1249 *llvmModule, type, op.getConstant(), linkage, cst, op.getSymName(),
1250 /*InsertBefore=*/nullptr, convertThreadLocalModeToLLVM(value: op.getTlsMode()),
1251 op.getAddrSpace(), op.getExternallyInitialized());
1252
1253 if (std::optional<mlir::SymbolRefAttr> comdat = op.getComdat()) {
1254 auto selectorOp = cast<ComdatSelectorOp>(
1255 Val: symbolTable().lookupNearestSymbolFrom(from: op, symbol: *comdat));
1256 var->setComdat(comdatMapping.lookup(Val: selectorOp));
1257 }
1258
1259 if (op.getUnnamedAddr().has_value())
1260 var->setUnnamedAddr(convertUnnamedAddrToLLVM(value: *op.getUnnamedAddr()));
1261
1262 if (op.getSection().has_value())
1263 var->setSection(*op.getSection());
1264
1265 addRuntimePreemptionSpecifier(dsoLocalRequested: op.getDsoLocal(), gv: var);
1266
1267 std::optional<uint64_t> alignment = op.getAlignment();
1268 if (alignment.has_value())
1269 var->setAlignment(llvm::MaybeAlign(alignment.value()));
1270
1271 var->setVisibility(convertVisibilityToLLVM(value: op.getVisibility_()));
1272
1273 globalsMapping.try_emplace(Key: op, Args&: var);
1274 globalsByNameMapping.try_emplace(Key: op.getSymName(), Args&: var);
1275
1276 // Add debug information if present.
1277 if (op.getDbgExprs()) {
1278 for (auto exprAttr :
1279 op.getDbgExprs()->getAsRange<DIGlobalVariableExpressionAttr>()) {
1280 llvm::DIGlobalVariableExpression *diGlobalExpr =
1281 debugTranslation->translateGlobalVariableExpression(attr: exprAttr);
1282 llvm::DIGlobalVariable *diGlobalVar = diGlobalExpr->getVariable();
1283 var->addDebugInfo(GV: diGlobalExpr);
1284
1285 // There is no `globals` field in DICompileUnitAttr which can be
1286 // directly assigned to DICompileUnit. We have to build the list by
1287 // looking at the dbgExpr of all the GlobalOps. The scope of the
1288 // variable is used to get the DICompileUnit in which to add it. But
1289 // there are cases where the scope of a global does not directly point
1290 // to the DICompileUnit and we have to do a bit more work to get to
1291 // it. Some of those cases are:
1292 //
1293 // 1. For the languages that support modules, the scope hierarchy can
1294 // be variable -> DIModule -> DICompileUnit
1295 //
1296 // 2. For the Fortran common block variable, the scope hierarchy can
1297 // be variable -> DICommonBlock -> DISubprogram -> DICompileUnit
1298 //
1299 // 3. For entities like static local variables in C or variable with
1300 // SAVE attribute in Fortran, the scope hierarchy can be
1301 // variable (-> DILocalScope)* -> DISubprogram
1302 llvm::DIScope *scope = diGlobalVar->getScope();
1303 if (auto *mod = dyn_cast_if_present<llvm::DIModule>(Val: scope))
1304 scope = mod->getScope();
1305 else if (auto *cb = dyn_cast_if_present<llvm::DICommonBlock>(Val: scope)) {
1306 if (auto *sp =
1307 dyn_cast_if_present<llvm::DISubprogram>(Val: cb->getScope()))
1308 scope = sp->getUnit();
1309 } else if (auto *lbb =
1310 dyn_cast_if_present<llvm::DILexicalBlockBase>(Val: scope)) {
1311 scope = lbb->getSubprogram();
1312 }
1313
1314 // Get the compile unit (scope) of the the global variable, or the
1315 // subprogram of the static local variable.
1316 if (llvm::DICompileUnit *compileUnit =
1317 dyn_cast_if_present<llvm::DICompileUnit>(Val: scope)) {
1318 // Update the compile unit with this incoming global variable
1319 // expression during the finalizing step later.
1320 globalGVars[compileUnit].push_back(Elt: diGlobalExpr);
1321 } else if (llvm::DISubprogram *sp =
1322 dyn_cast_if_present<llvm::DISubprogram>(Val: scope)) {
1323 // Update the subprogram with this incoming static local variable
1324 // expression during the finalizing step later.
1325 staticLocals[sp].push_back(Elt: diGlobalExpr);
1326 }
1327 }
1328 }
1329
1330 // Forward the target-specific attributes to LLVM.
1331 FailureOr<llvm::AttrBuilder> convertedTargetSpecificAttrs =
1332 convertMLIRAttributesToLLVM(loc: op.getLoc(), ctx&: var->getContext(),
1333 arrayAttr: op.getTargetSpecificAttrsAttr(),
1334 arrayAttrName: op.getTargetSpecificAttrsAttrName());
1335 if (failed(Result: convertedTargetSpecificAttrs))
1336 return failure();
1337 var->addAttributes(AttrBuilder: *convertedTargetSpecificAttrs);
1338 }
1339
1340 // Value-attribute initializers may reference other globals by symbol name.
1341 // Register every global above before materializing those constants.
1342 for (auto op : getModuleBody(module: mlirModule).getOps<LLVM::GlobalOp>()) {
1343 if (!op.getValueOrNull() || op.getInitializerBlock() ||
1344 isa<StringAttr>(Val: op.getValueOrNull()))
1345 continue;
1346
1347 llvm::Type *type = convertType(type: op.getType());
1348 llvm::Constant *cst =
1349 getLLVMConstant(llvmType: type, attr: op.getValueOrNull(), loc: op.getLoc(), moduleTranslation: *this);
1350 if (!cst)
1351 return failure();
1352
1353 auto linkage = convertLinkageToLLVM(value: op.getLinkage());
1354 bool dropInitializer = shouldDropGlobalInitializer(linkage, cst);
1355 auto *var = cast<llvm::GlobalVariable>(Val: lookupGlobal(op));
1356 if (dropInitializer)
1357 var->setInitializer(nullptr);
1358 else
1359 var->setInitializer(cst);
1360 }
1361
1362 // Create all llvm::GlobalAlias
1363 for (auto op : getModuleBody(module: mlirModule).getOps<LLVM::AliasOp>()) {
1364 llvm::Type *type = convertType(type: op.getType());
1365 llvm::Constant *cst = nullptr;
1366 llvm::GlobalValue::LinkageTypes linkage =
1367 convertLinkageToLLVM(value: op.getLinkage());
1368 llvm::Module &llvmMod = *llvmModule;
1369
1370 // Note address space and aliasee info isn't set just yet.
1371 llvm::GlobalAlias *var = llvm::GlobalAlias::create(
1372 Ty: type, AddressSpace: op.getAddrSpace(), Linkage: linkage, Name: op.getSymName(), /*placeholder*/ Aliasee: cst,
1373 Parent: &llvmMod);
1374
1375 var->setThreadLocalMode(convertThreadLocalModeToLLVM(value: op.getTlsMode()));
1376
1377 // Note there is no need to setup the comdat because GlobalAlias calls into
1378 // the aliasee comdat information automatically.
1379
1380 if (op.getUnnamedAddr().has_value())
1381 var->setUnnamedAddr(convertUnnamedAddrToLLVM(value: *op.getUnnamedAddr()));
1382
1383 var->setVisibility(convertVisibilityToLLVM(value: op.getVisibility_()));
1384 addRuntimePreemptionSpecifier(dsoLocalRequested: op.getDsoLocal(), gv: var);
1385
1386 aliasesMapping.try_emplace(Key: op, Args&: var);
1387 }
1388
1389 // Convert global variable bodies.
1390 for (auto op : getModuleBody(module: mlirModule).getOps<LLVM::GlobalOp>()) {
1391 if (Block *initializer = op.getInitializerBlock()) {
1392 llvm::IRBuilder<llvm::TargetFolder> builder(
1393 *llvmModule, llvm::TargetFolder(llvmModule->getDataLayout()));
1394
1395 [[maybe_unused]] int numConstantsHit = 0;
1396 [[maybe_unused]] int numConstantsErased = 0;
1397 DenseMap<llvm::ConstantAggregate *, int> constantAggregateUseMap;
1398
1399 for (auto &op : initializer->without_terminator()) {
1400 if (failed(Result: convertOperation(op, builder)))
1401 return emitError(loc: op.getLoc(), message: "fail to convert global initializer");
1402 auto *cst = dyn_cast<llvm::Constant>(Val: lookupValue(value: op.getResult(idx: 0)));
1403 if (!cst)
1404 return emitError(loc: op.getLoc(), message: "unemittable constant value");
1405
1406 // When emitting an LLVM constant, a new constant is created and the old
1407 // constant may become dangling and take space. We should remove the
1408 // dangling constants to avoid memory explosion especially for constant
1409 // arrays whose number of elements is large.
1410 // Because multiple operations may refer to the same constant, we need
1411 // to count the number of uses of each constant array and remove it only
1412 // when the count becomes zero.
1413 if (auto *agg = dyn_cast<llvm::ConstantAggregate>(Val: cst)) {
1414 numConstantsHit++;
1415 Value result = op.getResult(idx: 0);
1416 int numUsers = std::distance(first: result.use_begin(), last: result.use_end());
1417 auto [iterator, inserted] =
1418 constantAggregateUseMap.try_emplace(Key: agg, Args&: numUsers);
1419 if (!inserted) {
1420 // Key already exists, update the value
1421 iterator->second += numUsers;
1422 }
1423 }
1424 // Scan the operands of the operation to decrement the use count of
1425 // constants. Erase the constant if the use count becomes zero.
1426 for (Value v : op.getOperands()) {
1427 auto *cst = dyn_cast<llvm::ConstantAggregate>(Val: lookupValue(value: v));
1428 if (!cst)
1429 continue;
1430 auto iter = constantAggregateUseMap.find(Val: cst);
1431 assert(iter != constantAggregateUseMap.end() && "constant not found");
1432 iter->second--;
1433 if (iter->second == 0) {
1434 // NOTE: cannot call removeDeadConstantUsers() here because it
1435 // may remove the constant which has uses not be converted yet.
1436 if (cst->user_empty()) {
1437 cst->destroyConstant();
1438 numConstantsErased++;
1439 }
1440 constantAggregateUseMap.erase(I: iter);
1441 }
1442 }
1443 }
1444
1445 ReturnOp ret = cast<ReturnOp>(Val: initializer->getTerminator());
1446 llvm::Constant *cst =
1447 cast<llvm::Constant>(Val: lookupValue(value: ret.getOperand(i: 0)));
1448 auto *global = cast<llvm::GlobalVariable>(Val: lookupGlobal(op));
1449 if (!shouldDropGlobalInitializer(linkage: global->getLinkage(), cst))
1450 global->setInitializer(cst);
1451
1452 // Try to remove the dangling constants again after all operations are
1453 // converted.
1454 for (auto it : constantAggregateUseMap) {
1455 auto *cst = it.first;
1456 cst->removeDeadConstantUsers();
1457 if (cst->user_empty()) {
1458 cst->destroyConstant();
1459 numConstantsErased++;
1460 }
1461 }
1462
1463 LLVM_DEBUG(llvm::dbgs()
1464 << "Convert initializer for " << op.getName() << "\n";
1465 llvm::dbgs() << numConstantsHit << " new constants hit\n";
1466 llvm::dbgs()
1467 << numConstantsErased << " dangling constants erased\n";);
1468 }
1469 }
1470
1471 // Convert llvm.mlir.global_ctors and dtors.
1472 for (Operation &op : getModuleBody(module: mlirModule)) {
1473 auto ctorOp = dyn_cast<GlobalCtorsOp>(Val&: op);
1474 auto dtorOp = dyn_cast<GlobalDtorsOp>(Val&: op);
1475 if (!ctorOp && !dtorOp)
1476 continue;
1477
1478 // The empty / zero initialized version of llvm.global_(c|d)tors cannot be
1479 // handled by appendGlobalFn logic below, which just ignores empty (c|d)tor
1480 // lists. Make sure it gets emitted.
1481 if ((ctorOp && ctorOp.getCtors().empty()) ||
1482 (dtorOp && dtorOp.getDtors().empty())) {
1483 llvm::IRBuilder<llvm::TargetFolder> builder(
1484 *llvmModule, llvm::TargetFolder(llvmModule->getDataLayout()));
1485 llvm::Type *eltTy = llvm::StructType::get(
1486 elt1: builder.getInt32Ty(), elts: builder.getPtrTy(), elts: builder.getPtrTy());
1487 llvm::ArrayType *at = llvm::ArrayType::get(ElementType: eltTy, NumElements: 0);
1488 llvm::Constant *zeroInit = llvm::Constant::getNullValue(Ty: at);
1489 (void)new llvm::GlobalVariable(
1490 *llvmModule, zeroInit->getType(), false,
1491 llvm::GlobalValue::AppendingLinkage, zeroInit,
1492 ctorOp ? "llvm.global_ctors" : "llvm.global_dtors");
1493 } else {
1494 auto range = ctorOp
1495 ? llvm::zip(t: ctorOp.getCtors(), u: ctorOp.getPriorities())
1496 : llvm::zip(t: dtorOp.getDtors(), u: dtorOp.getPriorities());
1497 auto appendGlobalFn =
1498 ctorOp ? llvm::appendToGlobalCtors : llvm::appendToGlobalDtors;
1499 for (const auto &[sym, prio] : range) {
1500 llvm::Function *f =
1501 lookupFunction(name: cast<FlatSymbolRefAttr>(Val: sym).getValue());
1502 appendGlobalFn(*llvmModule, f, cast<IntegerAttr>(Val: prio).getInt(),
1503 /*Data=*/nullptr);
1504 }
1505 }
1506 }
1507
1508 for (auto op : getModuleBody(module: mlirModule).getOps<LLVM::GlobalOp>())
1509 if (failed(Result: convertDialectAttributes(op, instructions: {})))
1510 return failure();
1511
1512 // Finally, update the compile units their respective sets of global variables
1513 // created earlier.
1514 for (const auto &[compileUnit, globals] : globalGVars)
1515 compileUnit->replaceGlobalVariables(
1516 N: llvm::MDTuple::get(Context&: getLLVMContext(), MDs: globals));
1517
1518 // And update the subprograms with their respective sets of static local
1519 // variables.
1520 for (const auto &[sp, globals] : staticLocals)
1521 sp->retainNodes(NodesBegin: globals.begin(), NodesEnd: globals.end());
1522
1523 // Convert global alias bodies.
1524 for (auto op : getModuleBody(module: mlirModule).getOps<LLVM::AliasOp>()) {
1525 Block &initializer = op.getInitializerBlock();
1526 llvm::IRBuilder<llvm::TargetFolder> builder(
1527 *llvmModule, llvm::TargetFolder(llvmModule->getDataLayout()));
1528
1529 for (mlir::Operation &op : initializer.without_terminator()) {
1530 if (failed(Result: convertOperation(op, builder)))
1531 return emitError(loc: op.getLoc(), message: "fail to convert alias initializer");
1532 if (!isa<llvm::Constant>(Val: lookupValue(value: op.getResult(idx: 0))))
1533 return emitError(loc: op.getLoc(), message: "unemittable constant value");
1534 }
1535
1536 auto ret = cast<ReturnOp>(Val: initializer.getTerminator());
1537 auto *cst = cast<llvm::Constant>(Val: lookupValue(value: ret.getOperand(i: 0)));
1538 assert(aliasesMapping.count(op));
1539 auto *alias = cast<llvm::GlobalAlias>(Val: aliasesMapping[op]);
1540 alias->setAliasee(cst);
1541 }
1542
1543 for (auto op : getModuleBody(module: mlirModule).getOps<LLVM::AliasOp>())
1544 if (failed(Result: convertDialectAttributes(op, instructions: {})))
1545 return failure();
1546
1547 return success();
1548}
1549
1550/// Return a representation of `value` as metadata.
1551static llvm::Metadata *convertIntegerToMetadata(llvm::LLVMContext &context,
1552 const llvm::APInt &value) {
1553 llvm::Constant *constant = llvm::ConstantInt::get(Context&: context, V: value);
1554 return llvm::ConstantAsMetadata::get(C: constant);
1555}
1556
1557/// Return a representation of `value` as an MDNode.
1558static llvm::MDNode *convertIntegerToMDNode(llvm::LLVMContext &context,
1559 const llvm::APInt &value) {
1560 return llvm::MDNode::get(Context&: context, MDs: convertIntegerToMetadata(context, value));
1561}
1562
1563/// Return an MDNode encoding `vec_type_hint` metadata.
1564static llvm::MDNode *convertVecTypeHintToMDNode(llvm::LLVMContext &context,
1565 llvm::Type *type,
1566 bool isSigned) {
1567 llvm::Metadata *typeMD =
1568 llvm::ConstantAsMetadata::get(C: llvm::UndefValue::get(T: type));
1569 llvm::Metadata *isSignedMD =
1570 convertIntegerToMetadata(context, value: llvm::APInt(32, isSigned ? 1 : 0));
1571 return llvm::MDNode::get(Context&: context, MDs: {typeMD, isSignedMD});
1572}
1573
1574/// Return an MDNode with a tuple given by the values in `values`.
1575static llvm::MDNode *convertIntegerArrayToMDNode(llvm::LLVMContext &context,
1576 ArrayRef<int32_t> values) {
1577 SmallVector<llvm::Metadata *> mdValues;
1578 llvm::transform(Range&: values, d_first: std::back_inserter(x&: mdValues),
1579 F: [&context](int32_t value) {
1580 return convertIntegerToMetadata(
1581 context, value: llvm::APInt(32, value, /*isSigned=*/true));
1582 });
1583 return llvm::MDNode::get(Context&: context, MDs: mdValues);
1584}
1585
1586FailureOr<llvm::Metadata *> ModuleTranslation::convertMetadataAttr(
1587 Attribute attr, function_ref<InFlightDiagnostic()> emitError) {
1588 llvm::LLVMContext &llvmContext = getLLVMContext();
1589
1590 return llvm::TypeSwitch<Attribute, FailureOr<llvm::Metadata *>>(attr)
1591 .Case(caseFn: [&](MDStringAttr a) -> FailureOr<llvm::Metadata *> {
1592 return llvm::MDString::get(Context&: llvmContext, Str: a.getValue().getValue());
1593 })
1594 .Case(caseFn: [&](MDConstantAttr a) -> FailureOr<llvm::Metadata *> {
1595 IntegerAttr intAttr = llvm::dyn_cast<IntegerAttr>(Val: a.getValue());
1596 if (!intAttr) {
1597 return emitError()
1598 << "expected integer attribute in metadata constant";
1599 }
1600 return llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
1601 Ty: llvm::Type::getIntNTy(C&: llvmContext,
1602 N: intAttr.getType().getIntOrFloatBitWidth()),
1603 V: intAttr.getValue()));
1604 })
1605 .Case(caseFn: [&](MDGlobalValueAttr a) -> FailureOr<llvm::Metadata *> {
1606 return convertSymbolRefToMetadata(name: a.getName(), emitError);
1607 })
1608 .Case(caseFn: [&](MDNullAttr a) -> FailureOr<llvm::Metadata *> {
1609 return llvm::ConstantAsMetadata::get(C: llvm::ConstantPointerNull::get(
1610 T: llvm::PointerType::get(C&: llvmContext, AddressSpace: a.getAddressSpace())));
1611 })
1612 .Case(caseFn: [&](MDAddrSpaceCastAttr a) -> FailureOr<llvm::Metadata *> {
1613 FailureOr<llvm::Metadata *> arg =
1614 convertMetadataAttr(attr: a.getArg(), emitError);
1615 if (failed(Result: arg))
1616 return failure();
1617 // The verifier restricts the operand to pointer-valued metadata
1618 // attributes, all of which translate to a ConstantAsMetadata.
1619 auto *argAsMD = cast<llvm::ConstantAsMetadata>(Val: *arg);
1620 return llvm::ConstantAsMetadata::get(
1621 C: llvm::ConstantExpr::getAddrSpaceCast(
1622 C: argAsMD->getValue(),
1623 Ty: llvm::PointerType::get(C&: llvmContext, AddressSpace: a.getAddressSpace())));
1624 })
1625 .Case(caseFn: [&](MDNodeAttr a) -> FailureOr<llvm::Metadata *> {
1626 SmallVector<llvm::Metadata *> operands;
1627 for (Attribute operand : a.getOperands()) {
1628 FailureOr<llvm::Metadata *> md =
1629 convertMetadataAttr(attr: operand, emitError);
1630 if (failed(Result: md))
1631 return failure();
1632 operands.push_back(Elt: *md);
1633 }
1634 return llvm::MDNode::get(Context&: llvmContext, MDs: operands);
1635 })
1636 .Default(defaultFn: [&](Attribute attr) -> FailureOr<llvm::Metadata *> {
1637 return emitError() << "unsupported LLVM metadata attribute " << attr;
1638 });
1639}
1640
1641FailureOr<llvm::Metadata *> ModuleTranslation::convertSymbolRefToMetadata(
1642 FlatSymbolRefAttr name, function_ref<InFlightDiagnostic()> emitError) {
1643 if (llvm::Function *fn = lookupFunction(name: name.getValue()))
1644 return llvm::ValueAsMetadata::get(V: fn);
1645 if (llvm::GlobalValue *global = lookupGlobal(name: name.getValue()))
1646 return llvm::ValueAsMetadata::get(V: global);
1647 Operation *symbol = symbolTable().lookupSymbolIn(symbolTableOp: mlirModule, name);
1648 if (auto alias = dyn_cast_if_present<LLVM::AliasOp>(Val: symbol)) {
1649 if (llvm::GlobalValue *global = lookupAlias(op: alias))
1650 return llvm::ValueAsMetadata::get(V: global);
1651 }
1652 if (auto ifunc = dyn_cast_if_present<LLVM::IFuncOp>(Val: symbol)) {
1653 if (llvm::GlobalValue *global = lookupIFunc(op: ifunc))
1654 return llvm::ValueAsMetadata::get(V: global);
1655 }
1656 return emitError() << "could not resolve metadata reference '" << name << "'";
1657}
1658
1659LogicalResult ModuleTranslation::convertFunctionMetadata() {
1660 for (auto function : getModuleBody(module: mlirModule).getOps<LLVMFuncOp>()) {
1661 ArrayAttr metadata = function.getFunctionMetadataAttr();
1662 if (!metadata)
1663 continue;
1664
1665 llvm::Function *llvmFunc = lookupFunction(name: function.getName());
1666 for (auto entry : metadata.getAsRange<LLVM::FunctionMetadataAttr>()) {
1667 StringRef metadataName = entry.getMetadataName().getValue();
1668
1669 FailureOr<llvm::Metadata *> md =
1670 convertMetadataAttr(attr: entry.getNode(), emitError: [&]() {
1671 return function.emitError()
1672 << "failed to convert function_metadata entry '"
1673 << metadataName << "': ";
1674 });
1675 if (failed(Result: md))
1676 return failure();
1677 llvm::MDNode *node = llvm::dyn_cast_if_present<llvm::MDNode>(Val: *md);
1678 if (!node) {
1679 return function.emitError()
1680 << "failed to convert function_metadata entry '" << metadataName
1681 << "'";
1682 }
1683 llvmFunc->addMetadata(Kind: metadataName, MD&: *node);
1684 }
1685 }
1686 return success();
1687}
1688
1689LogicalResult ModuleTranslation::convertOneFunction(LLVMFuncOp func) {
1690 // Clear the block, branch value mappings, they are only relevant within one
1691 // function.
1692 blockMapping.clear();
1693 valueMapping.clear();
1694 branchMapping.clear();
1695 llvm::Function *llvmFunc = lookupFunction(name: func.getName());
1696 llvm::LLVMContext &llvmContext = llvmFunc->getContext();
1697
1698 // Add function arguments to the value remapping table.
1699 for (auto [mlirArg, llvmArg] :
1700 llvm::zip(t: func.getArguments(), u: llvmFunc->args()))
1701 mapValue(mlir: mlirArg, llvm: &llvmArg);
1702
1703 // Check the personality and set it.
1704 if (func.getPersonality()) {
1705 llvm::Type *ty = llvm::PointerType::get(
1706 C&: llvmContext, AddressSpace: llvmModule->getDataLayout().getProgramAddressSpace());
1707 if (llvm::Constant *pfunc = getLLVMConstant(llvmType: ty, attr: func.getPersonalityAttr(),
1708 loc: func.getLoc(), moduleTranslation: *this))
1709 llvmFunc->setPersonalityFn(pfunc);
1710 }
1711
1712 if (std::optional<StringRef> section = func.getSection())
1713 llvmFunc->setSection(*section);
1714
1715 if (func.getArmStreaming())
1716 llvmFunc->addFnAttr(Kind: "aarch64_pstate_sm_enabled");
1717 else if (func.getArmLocallyStreaming())
1718 llvmFunc->addFnAttr(Kind: "aarch64_pstate_sm_body");
1719 else if (func.getArmStreamingCompatible())
1720 llvmFunc->addFnAttr(Kind: "aarch64_pstate_sm_compatible");
1721
1722 if (func.getArmNewZa())
1723 llvmFunc->addFnAttr(Kind: "aarch64_new_za");
1724 else if (func.getArmInZa())
1725 llvmFunc->addFnAttr(Kind: "aarch64_in_za");
1726 else if (func.getArmOutZa())
1727 llvmFunc->addFnAttr(Kind: "aarch64_out_za");
1728 else if (func.getArmInoutZa())
1729 llvmFunc->addFnAttr(Kind: "aarch64_inout_za");
1730 else if (func.getArmPreservesZa())
1731 llvmFunc->addFnAttr(Kind: "aarch64_preserves_za");
1732
1733 if (auto targetCpu = func.getTargetCpu())
1734 llvmFunc->addFnAttr(Kind: "target-cpu", Val: *targetCpu);
1735
1736 if (auto tuneCpu = func.getTuneCpu())
1737 llvmFunc->addFnAttr(Kind: "tune-cpu", Val: *tuneCpu);
1738
1739 if (auto reciprocalEstimates = func.getReciprocalEstimates())
1740 llvmFunc->addFnAttr(Kind: "reciprocal-estimates", Val: *reciprocalEstimates);
1741
1742 if (auto preferVectorWidth = func.getPreferVectorWidth())
1743 llvmFunc->addFnAttr(Kind: "prefer-vector-width", Val: *preferVectorWidth);
1744
1745 if (func.getUseSampleProfile())
1746 llvmFunc->addFnAttr(Kind: "use-sample-profile");
1747
1748 if (auto disableTailCalls = func.getDisableTailCalls())
1749 llvmFunc->addFnAttr(Kind: "disable-tail-calls",
1750 Val: llvm::toStringRef(B: *disableTailCalls));
1751
1752 if (auto sampleProfileSuffixElisionPolicy =
1753 func.getSampleProfileSuffixElisionPolicy())
1754 llvmFunc->addFnAttr(Kind: "sample-profile-suffix-elision-policy",
1755 Val: *sampleProfileSuffixElisionPolicy);
1756
1757 if (auto attr = func.getVscaleRange())
1758 llvmFunc->addFnAttr(Attr: llvm::Attribute::getWithVScaleRangeArgs(
1759 Context&: getLLVMContext(), MinValue: attr->getMinRange().getInt(),
1760 MaxValue: attr->getMaxRange().getInt()));
1761
1762 if (auto noSignedZerosFpMath = func.getNoSignedZerosFpMath())
1763 llvmFunc->addFnAttr(Kind: "no-signed-zeros-fp-math",
1764 Val: llvm::toStringRef(B: *noSignedZerosFpMath));
1765
1766 if (auto fpContract = func.getFpContract())
1767 llvmFunc->addFnAttr(Kind: "fp-contract", Val: *fpContract);
1768
1769 if (auto instrumentFunctionEntry = func.getInstrumentFunctionEntry())
1770 llvmFunc->addFnAttr(Kind: "instrument-function-entry", Val: *instrumentFunctionEntry);
1771
1772 if (auto instrumentFunctionExit = func.getInstrumentFunctionExit())
1773 llvmFunc->addFnAttr(Kind: "instrument-function-exit", Val: *instrumentFunctionExit);
1774
1775 // First, create all blocks so we can jump to them.
1776 for (auto &bb : func) {
1777 auto *llvmBB = llvm::BasicBlock::Create(Context&: llvmContext);
1778 llvmBB->insertInto(Parent: llvmFunc);
1779 mapBlock(mlir: &bb, llvm: llvmBB);
1780 }
1781
1782 // Then, convert blocks one by one in topological order to ensure defs are
1783 // converted before uses.
1784 auto blocks = getBlocksSortedByDominance(region&: func.getBody());
1785 for (Block *bb : blocks) {
1786 CapturingIRBuilder builder(*llvmModule,
1787 llvm::TargetFolder(llvmModule->getDataLayout()));
1788 builder.disableGEPCanonicalization();
1789 if (failed(Result: convertBlockImpl(bb&: *bb, ignoreArguments: bb->isEntryBlock(), builder,
1790 /*recordInsertions=*/true)))
1791 return failure();
1792 }
1793
1794 // After all blocks have been traversed and values mapped, connect the PHI
1795 // nodes to the results of preceding blocks.
1796 detail::connectPHINodes(region&: func.getBody(), state: *this);
1797
1798 // Finally, convert dialect attributes attached to the function.
1799 return convertDialectAttributes(op: func, instructions: {});
1800}
1801
1802LogicalResult ModuleTranslation::convertDialectAttributes(
1803 Operation *op, ArrayRef<llvm::Instruction *> instructions) {
1804 for (NamedAttribute attribute : op->getDialectAttrs())
1805 if (failed(Result: iface.amendOperation(op, instructions, attribute, moduleTranslation&: *this)))
1806 return failure();
1807 return success();
1808}
1809
1810/// Converts memory effect attributes from `func` and attaches them to
1811/// `llvmFunc`.
1812static void convertFunctionMemoryAttributes(LLVMFuncOp func,
1813 llvm::Function *llvmFunc) {
1814 if (!func.getMemoryEffects())
1815 return;
1816
1817 MemoryEffectsAttr memEffects = func.getMemoryEffectsAttr();
1818
1819 // Add memory effects incrementally.
1820 llvm::MemoryEffects newMemEffects =
1821 llvm::MemoryEffects(llvm::MemoryEffects::Location::ArgMem,
1822 convertModRefInfoToLLVM(value: memEffects.getArgMem()));
1823 newMemEffects |= llvm::MemoryEffects(
1824 llvm::MemoryEffects::Location::InaccessibleMem,
1825 convertModRefInfoToLLVM(value: memEffects.getInaccessibleMem()));
1826 newMemEffects |=
1827 llvm::MemoryEffects(llvm::MemoryEffects::Location::Other,
1828 convertModRefInfoToLLVM(value: memEffects.getOther()));
1829 newMemEffects |=
1830 llvm::MemoryEffects(llvm::MemoryEffects::Location::ErrnoMem,
1831 convertModRefInfoToLLVM(value: memEffects.getErrnoMem()));
1832 newMemEffects |=
1833 llvm::MemoryEffects(llvm::MemoryEffects::Location::TargetMem0,
1834 convertModRefInfoToLLVM(value: memEffects.getTargetMem0()));
1835 newMemEffects |=
1836 llvm::MemoryEffects(llvm::MemoryEffects::Location::TargetMem1,
1837 convertModRefInfoToLLVM(value: memEffects.getTargetMem1()));
1838 llvmFunc->setMemoryEffects(newMemEffects);
1839}
1840
1841llvm::Attribute
1842ModuleTranslation::convertAllocsizeAttr(DenseI32ArrayAttr allocSizeAttr) {
1843 if (!allocSizeAttr || allocSizeAttr.empty())
1844 return llvm::Attribute{};
1845
1846 unsigned elemSize = static_cast<unsigned>(allocSizeAttr[0]);
1847 std::optional<unsigned> numElems;
1848 if (allocSizeAttr.size() > 1)
1849 numElems = static_cast<unsigned>(allocSizeAttr[1]);
1850
1851 return llvm::Attribute::getWithAllocSizeArgs(Context&: getLLVMContext(), ElemSizeArg: elemSize,
1852 NumElemsArg: numElems);
1853}
1854static void convertDenormalFPEnvAttribute(LLVMFuncOp func,
1855 llvm::AttrBuilder &Attrs) {
1856 std::optional<DenormalFPEnvAttr> denormalFpEnv = func.getDenormalFpenv();
1857 if (!denormalFpEnv)
1858 return;
1859
1860 llvm::DenormalMode DefaultMode(
1861 convertDenormalModeKindToLLVM(value: denormalFpEnv->getDefaultOutputMode()),
1862 convertDenormalModeKindToLLVM(value: denormalFpEnv->getDefaultInputMode()));
1863 llvm::DenormalMode FloatMode(
1864 convertDenormalModeKindToLLVM(value: denormalFpEnv->getFloatOutputMode()),
1865 convertDenormalModeKindToLLVM(value: denormalFpEnv->getFloatInputMode()));
1866
1867 llvm::DenormalFPEnv FPEnv(DefaultMode, FloatMode);
1868 Attrs.addDenormalFPEnvAttr(Mode: FPEnv);
1869}
1870
1871/// Converts function attributes from `func` and attaches them to `llvmFunc`.
1872static void convertFunctionAttributes(ModuleTranslation &mod, LLVMFuncOp func,
1873 llvm::Function *llvmFunc) {
1874 // FIXME: Use AttrBuilder far all cases
1875 llvm::AttrBuilder AttrBuilder(llvmFunc->getContext());
1876
1877 if (func.getNoInlineAttr())
1878 llvmFunc->addFnAttr(Kind: llvm::Attribute::NoInline);
1879 if (func.getAlwaysInlineAttr())
1880 llvmFunc->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
1881 if (func.getInlineHintAttr())
1882 llvmFunc->addFnAttr(Kind: llvm::Attribute::InlineHint);
1883 if (func.getOptimizeNoneAttr())
1884 llvmFunc->addFnAttr(Kind: llvm::Attribute::OptimizeNone);
1885 if (func.getReturnsTwiceAttr())
1886 llvmFunc->addFnAttr(Kind: llvm::Attribute::ReturnsTwice);
1887 if (func.getColdAttr())
1888 llvmFunc->addFnAttr(Kind: llvm::Attribute::Cold);
1889 if (func.getHotAttr())
1890 llvmFunc->addFnAttr(Kind: llvm::Attribute::Hot);
1891 if (func.getNoduplicateAttr())
1892 llvmFunc->addFnAttr(Kind: llvm::Attribute::NoDuplicate);
1893 if (func.getConvergentAttr())
1894 llvmFunc->addFnAttr(Kind: llvm::Attribute::Convergent);
1895 if (func.getNoUnwindAttr())
1896 llvmFunc->addFnAttr(Kind: llvm::Attribute::NoUnwind);
1897 if (func.getWillReturnAttr())
1898 llvmFunc->addFnAttr(Kind: llvm::Attribute::WillReturn);
1899 if (func.getNoreturnAttr())
1900 llvmFunc->addFnAttr(Kind: llvm::Attribute::NoReturn);
1901 if (func.getOptsizeAttr())
1902 llvmFunc->addFnAttr(Kind: llvm::Attribute::OptimizeForSize);
1903 if (func.getMinsizeAttr())
1904 llvmFunc->addFnAttr(Kind: llvm::Attribute::MinSize);
1905 if (func.getSaveRegParamsAttr())
1906 llvmFunc->addFnAttr(Kind: "save-reg-params");
1907 if (func.getNoCallerSavedRegistersAttr())
1908 llvmFunc->addFnAttr(Kind: "no_caller_saved_registers");
1909 if (func.getNocallbackAttr())
1910 llvmFunc->addFnAttr(Kind: llvm::Attribute::NoCallback);
1911 if (StringAttr modFormat = func.getModularFormatAttr())
1912 llvmFunc->addFnAttr(Kind: "modular-format", Val: modFormat.getValue());
1913 if (TargetFeaturesAttr targetFeatAttr = func.getTargetFeaturesAttr())
1914 llvmFunc->addFnAttr(Kind: "target-features", Val: targetFeatAttr.getFeaturesString());
1915 if (FramePointerKindAttr fpAttr = func.getFramePointerAttr())
1916 llvmFunc->addFnAttr(Kind: "frame-pointer", Val: stringifyFramePointerKind(
1917 fpAttr.getFramePointerKind()));
1918 if (UWTableKindAttr uwTableKindAttr = func.getUwtableKindAttr())
1919 llvmFunc->setUWTableKind(
1920 convertUWTableKindToLLVM(value: uwTableKindAttr.getUwtableKind()));
1921 if (StringAttr zcsr = func.getZeroCallUsedRegsAttr())
1922 llvmFunc->addFnAttr(Kind: "zero-call-used-regs", Val: zcsr.getValue());
1923 if (func.getUniformWorkGroupSizeAttr())
1924 llvmFunc->addFnAttr(Kind: "uniform-work-group-size");
1925
1926 if (ArrayAttr noBuiltins = func.getNobuiltinsAttr()) {
1927 if (noBuiltins.empty())
1928 llvmFunc->addFnAttr(Kind: "no-builtins");
1929
1930 mod.convertFunctionAttrCollection(attrs: noBuiltins, op: llvmFunc,
1931 conv&: ModuleTranslation::convertNoBuiltin);
1932 }
1933
1934 mod.convertFunctionAttrCollection(attrs: func.getDefaultFuncAttrsAttr(), op: llvmFunc,
1935 conv&: ModuleTranslation::convertDefaultFuncAttr);
1936
1937 if (llvm::Attribute attr = mod.convertAllocsizeAttr(allocSizeAttr: func.getAllocsizeAttr());
1938 attr.isValid())
1939 llvmFunc->addFnAttr(Attr: attr);
1940
1941 convertFunctionMemoryAttributes(func, llvmFunc);
1942
1943 convertDenormalFPEnvAttribute(func, Attrs&: AttrBuilder);
1944 llvmFunc->addFnAttrs(Attrs: AttrBuilder);
1945}
1946
1947/// Converts function attributes from `func` and attaches them to `llvmFunc`.
1948static void convertFunctionKernelAttributes(LLVMFuncOp func,
1949 llvm::Function *llvmFunc,
1950 ModuleTranslation &translation) {
1951 llvm::LLVMContext &llvmContext = llvmFunc->getContext();
1952
1953 if (VecTypeHintAttr vecTypeHint = func.getVecTypeHintAttr()) {
1954 Type type = vecTypeHint.getHint().getValue();
1955 llvm::Type *llvmType = translation.convertType(type);
1956 bool isSigned = vecTypeHint.getIsSigned();
1957 llvmFunc->setMetadata(
1958 Kind: func.getVecTypeHintAttrName(),
1959 Node: convertVecTypeHintToMDNode(context&: llvmContext, type: llvmType, isSigned));
1960 }
1961
1962 if (std::optional<ArrayRef<int32_t>> workGroupSizeHint =
1963 func.getWorkGroupSizeHint()) {
1964 llvmFunc->setMetadata(
1965 Kind: func.getWorkGroupSizeHintAttrName(),
1966 Node: convertIntegerArrayToMDNode(context&: llvmContext, values: *workGroupSizeHint));
1967 }
1968
1969 if (std::optional<ArrayRef<int32_t>> reqdWorkGroupSize =
1970 func.getReqdWorkGroupSize()) {
1971 llvmFunc->setMetadata(
1972 Kind: func.getReqdWorkGroupSizeAttrName(),
1973 Node: convertIntegerArrayToMDNode(context&: llvmContext, values: *reqdWorkGroupSize));
1974 }
1975
1976 if (std::optional<uint32_t> intelReqdSubGroupSize =
1977 func.getIntelReqdSubGroupSize()) {
1978 llvmFunc->setMetadata(
1979 Kind: func.getIntelReqdSubGroupSizeAttrName(),
1980 Node: convertIntegerToMDNode(context&: llvmContext,
1981 value: llvm::APInt(32, *intelReqdSubGroupSize)));
1982 }
1983}
1984
1985static LogicalResult convertParameterAttr(llvm::AttrBuilder &attrBuilder,
1986 llvm::Attribute::AttrKind llvmKind,
1987 NamedAttribute namedAttr,
1988 ModuleTranslation &moduleTranslation,
1989 Location loc) {
1990 return llvm::TypeSwitch<Attribute, LogicalResult>(namedAttr.getValue())
1991 .Case(caseFn: [&](TypeAttr typeAttr) {
1992 attrBuilder.addTypeAttr(
1993 Kind: llvmKind, Ty: moduleTranslation.convertType(type: typeAttr.getValue()));
1994 return success();
1995 })
1996 .Case(caseFn: [&](IntegerAttr intAttr) {
1997 attrBuilder.addRawIntAttr(Kind: llvmKind, Value: intAttr.getInt());
1998 return success();
1999 })
2000 .Case(caseFn: [&](UnitAttr) {
2001 attrBuilder.addAttribute(Val: llvmKind);
2002 return success();
2003 })
2004 .Case(caseFn: [&](LLVM::ConstantRangeAttr rangeAttr) {
2005 attrBuilder.addConstantRangeAttr(
2006 Kind: llvmKind,
2007 CR: llvm::ConstantRange(rangeAttr.getLower(), rangeAttr.getUpper()));
2008 return success();
2009 })
2010 .Default(defaultFn: [loc](auto) {
2011 return emitError(loc, message: "unsupported parameter attribute type");
2012 });
2013}
2014
2015FailureOr<llvm::AttrBuilder>
2016ModuleTranslation::convertParameterAttrs(LLVMFuncOp func, int argIdx,
2017 DictionaryAttr paramAttrs) {
2018 llvm::AttrBuilder attrBuilder(llvmModule->getContext());
2019 auto attrNameToKindMapping = getAttrNameToKindMapping();
2020 Location loc = func.getLoc();
2021
2022 for (auto namedAttr : paramAttrs) {
2023 auto it = attrNameToKindMapping.find(Val: namedAttr.getName());
2024 if (it != attrNameToKindMapping.end()) {
2025 llvm::Attribute::AttrKind llvmKind = it->second;
2026 if (failed(Result: convertParameterAttr(attrBuilder, llvmKind, namedAttr, moduleTranslation&: *this,
2027 loc)))
2028 return failure();
2029 } else if (namedAttr.getNameDialect()) {
2030 if (failed(Result: iface.convertParameterAttr(function: func, argIdx, attribute: namedAttr, moduleTranslation&: *this)))
2031 return failure();
2032 }
2033 }
2034
2035 return attrBuilder;
2036}
2037
2038LogicalResult ModuleTranslation::convertArgAndResultAttrs(
2039 ArgAndResultAttrsOpInterface attrsOp, llvm::CallBase *call,
2040 ArrayRef<unsigned> immArgPositions) {
2041 // Convert the argument attributes.
2042 if (ArrayAttr argAttrsArray = attrsOp.getArgAttrsAttr()) {
2043 unsigned argAttrIdx = 0;
2044 llvm::SmallDenseSet<unsigned> immArgPositionsSet(immArgPositions.begin(),
2045 immArgPositions.end());
2046 for (unsigned argIdx : llvm::seq<unsigned>(Size: call->arg_size())) {
2047 if (argAttrIdx >= argAttrsArray.size())
2048 break;
2049 // Skip immediate arguments (they have no entries in argAttrsArray).
2050 if (immArgPositionsSet.contains(V: argIdx))
2051 continue;
2052 // Skip empty argument attributes.
2053 auto argAttrs = cast<DictionaryAttr>(Val: argAttrsArray[argAttrIdx++]);
2054 if (argAttrs.empty())
2055 continue;
2056 // Convert and add attributes to the call instruction.
2057 FailureOr<llvm::AttrBuilder> attrBuilder =
2058 convertParameterAttrs(loc: attrsOp->getLoc(), paramAttrs: argAttrs);
2059 if (failed(Result: attrBuilder))
2060 return failure();
2061 call->addParamAttrs(ArgNo: argIdx, B: *attrBuilder);
2062 }
2063 }
2064
2065 // Convert the result attributes.
2066 if (ArrayAttr resAttrsArray = attrsOp.getResAttrsAttr()) {
2067 if (!resAttrsArray.empty()) {
2068 auto resAttrs = cast<DictionaryAttr>(Val: resAttrsArray[0]);
2069 FailureOr<llvm::AttrBuilder> attrBuilder =
2070 convertParameterAttrs(loc: attrsOp->getLoc(), paramAttrs: resAttrs);
2071 if (failed(Result: attrBuilder))
2072 return failure();
2073 call->addRetAttrs(B: *attrBuilder);
2074 }
2075 }
2076
2077 return success();
2078}
2079
2080std::optional<llvm::Attribute>
2081ModuleTranslation::convertNoBuiltin(llvm::LLVMContext &ctx, mlir::Attribute a) {
2082 if (auto str = dyn_cast<StringAttr>(Val&: a))
2083 return llvm::Attribute::get(Context&: ctx, Kind: ("no-builtin-" + str.getValue()).str());
2084 return std::nullopt;
2085}
2086
2087std::optional<llvm::Attribute>
2088ModuleTranslation::convertDefaultFuncAttr(llvm::LLVMContext &ctx,
2089 mlir::NamedAttribute namedAttr) {
2090 StringAttr name = namedAttr.getName();
2091 Attribute value = namedAttr.getValue();
2092
2093 if (auto strVal = dyn_cast<StringAttr>(Val&: value))
2094 return llvm::Attribute::get(Context&: ctx, Kind: name.getValue(), Val: strVal.getValue());
2095 if (mlir::isa<UnitAttr>(Val: value))
2096 return llvm::Attribute::get(Context&: ctx, Kind: name.getValue());
2097 return std::nullopt;
2098}
2099
2100FailureOr<llvm::AttrBuilder>
2101ModuleTranslation::convertParameterAttrs(Location loc,
2102 DictionaryAttr paramAttrs) {
2103 llvm::AttrBuilder attrBuilder(llvmModule->getContext());
2104 auto attrNameToKindMapping = getAttrNameToKindMapping();
2105
2106 for (auto namedAttr : paramAttrs) {
2107 auto it = attrNameToKindMapping.find(Val: namedAttr.getName());
2108 if (it != attrNameToKindMapping.end()) {
2109 llvm::Attribute::AttrKind llvmKind = it->second;
2110 if (failed(Result: convertParameterAttr(attrBuilder, llvmKind, namedAttr, moduleTranslation&: *this,
2111 loc)))
2112 return failure();
2113 }
2114 }
2115
2116 return attrBuilder;
2117}
2118
2119LogicalResult ModuleTranslation::convertFunctionSignatures() {
2120 // Declare all functions first because there may be function calls that form a
2121 // call graph with cycles, global initializers that reference functions, or
2122 // metadata that references functions declared later in the module.
2123 for (auto function : getModuleBody(module: mlirModule).getOps<LLVMFuncOp>()) {
2124 llvm::FunctionCallee llvmFuncCst = llvmModule->getOrInsertFunction(
2125 Name: function.getName(),
2126 T: cast<llvm::FunctionType>(Val: convertType(type: function.getFunctionType())));
2127 llvm::Function *llvmFunc = cast<llvm::Function>(Val: llvmFuncCst.getCallee());
2128 mapFunction(name: function.getName(), func: llvmFunc);
2129 }
2130
2131 for (auto function : getModuleBody(module: mlirModule).getOps<LLVMFuncOp>()) {
2132 llvm::Function *llvmFunc = lookupFunction(name: function.getName());
2133 llvmFunc->setLinkage(convertLinkageToLLVM(value: function.getLinkage()));
2134 llvmFunc->setCallingConv(convertCConvToLLVM(value: function.getCConv()));
2135 addRuntimePreemptionSpecifier(dsoLocalRequested: function.getDsoLocal(), gv: llvmFunc);
2136
2137 // Convert function attributes.
2138 convertFunctionAttributes(mod&: *this, func: function, llvmFunc);
2139
2140 // Convert function kernel attributes to metadata.
2141 convertFunctionKernelAttributes(func: function, llvmFunc, translation&: *this);
2142
2143 // Convert function_entry_count attribute to metadata.
2144 if (auto entryCount = function.getFunctionEntryCountAttr()) {
2145 ArrayRef<uint64_t> imports = entryCount.getImports();
2146 llvm::DenseSet<llvm::GlobalValue::GUID> importGUIDs;
2147 if (!imports.empty())
2148 importGUIDs.insert(I: imports.begin(), E: imports.end());
2149 llvmFunc->setEntryCount(Count: entryCount.getEntryCount(),
2150 Imports: imports.empty() ? nullptr : &importGUIDs);
2151 }
2152
2153 // Convert result attributes.
2154 if (ArrayAttr allResultAttrs = function.getAllResultAttrs()) {
2155 DictionaryAttr resultAttrs = cast<DictionaryAttr>(Val: allResultAttrs[0]);
2156 FailureOr<llvm::AttrBuilder> attrBuilder =
2157 convertParameterAttrs(func: function, argIdx: -1, paramAttrs: resultAttrs);
2158 if (failed(Result: attrBuilder))
2159 return failure();
2160 llvmFunc->addRetAttrs(Attrs: *attrBuilder);
2161 }
2162
2163 // Convert argument attributes.
2164 for (auto [argIdx, llvmArg] : llvm::enumerate(First: llvmFunc->args())) {
2165 if (DictionaryAttr argAttrs = function.getArgAttrDict(index: argIdx)) {
2166 FailureOr<llvm::AttrBuilder> attrBuilder =
2167 convertParameterAttrs(func: function, argIdx, paramAttrs: argAttrs);
2168 if (failed(Result: attrBuilder))
2169 return failure();
2170 llvmArg.addAttrs(B&: *attrBuilder);
2171 }
2172 }
2173
2174 // Forward the pass-through attributes to LLVM.
2175 FailureOr<llvm::AttrBuilder> convertedPassthroughAttrs =
2176 convertMLIRAttributesToLLVM(loc: function.getLoc(), ctx&: llvmFunc->getContext(),
2177 arrayAttr: function.getPassthroughAttr(),
2178 arrayAttrName: function.getPassthroughAttrName());
2179 if (failed(Result: convertedPassthroughAttrs))
2180 return failure();
2181 llvmFunc->addFnAttrs(Attrs: *convertedPassthroughAttrs);
2182
2183 // Convert visibility attribute.
2184 llvmFunc->setVisibility(convertVisibilityToLLVM(value: function.getVisibility_()));
2185
2186 // Convert the comdat attribute.
2187 if (std::optional<mlir::SymbolRefAttr> comdat = function.getComdat()) {
2188 auto selectorOp = cast<ComdatSelectorOp>(
2189 Val: symbolTable().lookupNearestSymbolFrom(from: function, symbol: *comdat));
2190 llvmFunc->setComdat(comdatMapping.lookup(Val: selectorOp));
2191 }
2192
2193 if (auto gc = function.getGarbageCollector())
2194 llvmFunc->setGC(gc->str());
2195
2196 if (auto unnamedAddr = function.getUnnamedAddr())
2197 llvmFunc->setUnnamedAddr(convertUnnamedAddrToLLVM(value: *unnamedAddr));
2198
2199 if (auto alignment = function.getAlignment())
2200 llvmFunc->setAlignment(llvm::MaybeAlign(*alignment));
2201
2202 // Translate the debug information for this function.
2203 debugTranslation->translate(func: function, llvmFunc&: *llvmFunc);
2204 }
2205
2206 return success();
2207}
2208
2209LogicalResult ModuleTranslation::convertFunctions() {
2210 // Convert functions.
2211 for (auto function : getModuleBody(module: mlirModule).getOps<LLVMFuncOp>()) {
2212 // Do not convert external functions, but do process dialect attributes
2213 // attached to them.
2214 if (function.isExternal()) {
2215 if (failed(Result: convertDialectAttributes(op: function, instructions: {})))
2216 return failure();
2217 continue;
2218 }
2219
2220 if (failed(Result: convertOneFunction(func: function)))
2221 return failure();
2222 }
2223
2224 return success();
2225}
2226
2227LogicalResult ModuleTranslation::convertIFuncs() {
2228 for (auto op : getModuleBody(module: mlirModule).getOps<IFuncOp>()) {
2229 llvm::Type *type = convertType(type: op.getIFuncType());
2230 llvm::GlobalValue::LinkageTypes linkage =
2231 convertLinkageToLLVM(value: op.getLinkage());
2232 llvm::Constant *resolver;
2233 if (auto *resolverFn = lookupFunction(name: op.getResolver())) {
2234 resolver = cast<llvm::Constant>(Val: resolverFn);
2235 } else {
2236 Operation *aliasOp = symbolTable().lookupSymbolIn(symbolTableOp: parentLLVMModule(op),
2237 name: op.getResolverAttr());
2238 resolver = cast<llvm::Constant>(Val: lookupAlias(op: aliasOp));
2239 }
2240
2241 auto *ifunc =
2242 llvm::GlobalIFunc::create(Ty: type, AddressSpace: op.getAddressSpace(), Linkage: linkage,
2243 Name: op.getSymName(), Resolver: resolver, Parent: llvmModule.get());
2244 addRuntimePreemptionSpecifier(dsoLocalRequested: op.getDsoLocal(), gv: ifunc);
2245 ifunc->setUnnamedAddr(convertUnnamedAddrToLLVM(value: op.getUnnamedAddr()));
2246 ifunc->setVisibility(convertVisibilityToLLVM(value: op.getVisibility_()));
2247
2248 ifuncMapping.try_emplace(Key: op, Args&: ifunc);
2249 }
2250
2251 return success();
2252}
2253
2254// Attach global metadata after all globals, aliases, ifuncs, and function
2255// signatures exist so symbol references can be resolved.
2256LogicalResult ModuleTranslation::convertGlobalMetadata() {
2257 for (auto op : getModuleBody(module: mlirModule).getOps<LLVM::GlobalOp>()) {
2258 auto *var = cast<llvm::GlobalVariable>(Val: lookupGlobal(op));
2259 if (FlatSymbolRefAttr associated = op.getAssociatedAttr()) {
2260 FailureOr<llvm::Metadata *> md =
2261 convertSymbolRefToMetadata(name: associated, emitError: [&]() {
2262 return op.emitError(message: "failed to convert associated metadata");
2263 });
2264 if (failed(Result: md))
2265 return failure();
2266 var->setMetadata(KindID: llvm::LLVMContext::MD_associated,
2267 Node: llvm::MDNode::get(Context&: var->getContext(), MDs: *md));
2268 }
2269
2270 if (ArrayAttr absSym = op.getAbsoluteSymbolAttr()) {
2271 SmallVector<llvm::Metadata *> mdOps;
2272 llvm::LLVMContext &ctx = var->getContext();
2273 mdOps.reserve(N: absSym.size());
2274 for (Attribute attr : absSym) {
2275 auto intAttr = cast<IntegerAttr>(Val&: attr);
2276 llvm::IntegerType *ty = llvm::IntegerType::get(
2277 C&: ctx, NumBits: intAttr.getType().getIntOrFloatBitWidth());
2278 mdOps.push_back(Elt: llvm::ConstantAsMetadata::get(
2279 C: llvm::ConstantInt::get(Ty: ty, V: intAttr.getValue())));
2280 }
2281 var->setMetadata(KindID: llvm::LLVMContext::MD_absolute_symbol,
2282 Node: llvm::MDNode::get(Context&: ctx, MDs: mdOps));
2283 }
2284 }
2285
2286 return success();
2287}
2288
2289LogicalResult ModuleTranslation::convertComdats() {
2290 for (auto comdatOp : getModuleBody(module: mlirModule).getOps<ComdatOp>()) {
2291 for (auto selectorOp : comdatOp.getOps<ComdatSelectorOp>()) {
2292 llvm::Module *module = getLLVMModule();
2293 if (module->getComdatSymbolTable().contains(Key: selectorOp.getSymName()))
2294 return emitError(loc: selectorOp.getLoc())
2295 << "comdat selection symbols must be unique even in different "
2296 "comdat regions";
2297 llvm::Comdat *comdat = module->getOrInsertComdat(Name: selectorOp.getSymName());
2298 comdat->setSelectionKind(convertComdatToLLVM(value: selectorOp.getComdat()));
2299 comdatMapping.try_emplace(Key: selectorOp, Args&: comdat);
2300 }
2301 }
2302 return success();
2303}
2304
2305LogicalResult ModuleTranslation::convertUnresolvedBlockAddress() {
2306 for (auto &[blockAddressOp, llvmCst] : unresolvedBlockAddressMapping) {
2307 BlockAddressAttr blockAddressAttr = blockAddressOp.getBlockAddr();
2308 llvm::BasicBlock *llvmBlock = lookupBlockAddress(attr: blockAddressAttr);
2309 assert(llvmBlock && "expected LLVM blocks to be already translated");
2310
2311 // Update mapping with new block address constant.
2312 auto *llvmBlockAddr = llvm::BlockAddress::get(
2313 F: lookupFunction(name: blockAddressAttr.getFunction().getValue()), BB: llvmBlock);
2314 llvmCst->replaceAllUsesWith(V: llvmBlockAddr);
2315 assert(llvmCst->use_empty() && "expected all uses to be replaced");
2316 cast<llvm::GlobalVariable>(Val: llvmCst)->eraseFromParent();
2317 }
2318 unresolvedBlockAddressMapping.clear();
2319 return success();
2320}
2321
2322void ModuleTranslation::setAccessGroupsMetadata(AccessGroupOpInterface op,
2323 llvm::Instruction *inst) {
2324 if (llvm::MDNode *node = loopAnnotationTranslation->getAccessGroups(op))
2325 inst->setMetadata(KindID: llvm::LLVMContext::MD_access_group, Node: node);
2326}
2327
2328llvm::MDNode *
2329ModuleTranslation::getOrCreateAliasScope(AliasScopeAttr aliasScopeAttr) {
2330 auto [scopeIt, scopeInserted] =
2331 aliasScopeMetadataMapping.try_emplace(Key: aliasScopeAttr, Args: nullptr);
2332 if (!scopeInserted)
2333 return scopeIt->second;
2334 llvm::LLVMContext &ctx = llvmModule->getContext();
2335 auto dummy = llvm::MDNode::getTemporary(Context&: ctx, MDs: {});
2336 // Convert the domain metadata node if necessary.
2337 auto [domainIt, insertedDomain] = aliasDomainMetadataMapping.try_emplace(
2338 Key: aliasScopeAttr.getDomain(), Args: nullptr);
2339 if (insertedDomain) {
2340 llvm::SmallVector<llvm::Metadata *, 3> operands;
2341 // Placeholder for potential self-reference.
2342 operands.push_back(Elt: dummy.get());
2343 operands.push_back(
2344 Elt: llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::getFalse(Context&: ctx)));
2345 if (StringAttr description = aliasScopeAttr.getDomain().getDescription())
2346 operands.push_back(Elt: llvm::MDString::get(Context&: ctx, Str: description));
2347 domainIt->second = llvm::MDNode::get(Context&: ctx, MDs: operands);
2348 // Self-reference for uniqueness.
2349 llvm::Metadata *replacement;
2350 if (auto stringAttr =
2351 dyn_cast<StringAttr>(Val: aliasScopeAttr.getDomain().getId()))
2352 replacement = llvm::MDString::get(Context&: ctx, Str: stringAttr.getValue());
2353 else
2354 replacement = domainIt->second;
2355 domainIt->second->replaceOperandWith(I: 0, New: replacement);
2356 }
2357 // Convert the scope metadata node.
2358 assert(domainIt->second && "Scope's domain should already be valid");
2359 llvm::SmallVector<llvm::Metadata *, 3> operands;
2360 // Placeholder for potential self-reference.
2361 operands.push_back(Elt: dummy.get());
2362 operands.push_back(Elt: domainIt->second);
2363 if (StringAttr description = aliasScopeAttr.getDescription())
2364 operands.push_back(Elt: llvm::MDString::get(Context&: ctx, Str: description));
2365 scopeIt->second = llvm::MDNode::get(Context&: ctx, MDs: operands);
2366 // Self-reference for uniqueness.
2367 llvm::Metadata *replacement;
2368 if (auto stringAttr = dyn_cast<StringAttr>(Val: aliasScopeAttr.getId()))
2369 replacement = llvm::MDString::get(Context&: ctx, Str: stringAttr.getValue());
2370 else
2371 replacement = scopeIt->second;
2372 scopeIt->second->replaceOperandWith(I: 0, New: replacement);
2373 return scopeIt->second;
2374}
2375
2376llvm::MDNode *ModuleTranslation::getOrCreateAliasScopes(
2377 ArrayRef<AliasScopeAttr> aliasScopeAttrs) {
2378 SmallVector<llvm::Metadata *> nodes;
2379 nodes.reserve(N: aliasScopeAttrs.size());
2380 for (AliasScopeAttr aliasScopeAttr : aliasScopeAttrs)
2381 nodes.push_back(Elt: getOrCreateAliasScope(aliasScopeAttr));
2382 return llvm::MDNode::get(Context&: getLLVMContext(), MDs: nodes);
2383}
2384
2385void ModuleTranslation::setAliasScopeMetadata(AliasAnalysisOpInterface op,
2386 llvm::Instruction *inst) {
2387 auto populateScopeMetadata = [&](ArrayAttr aliasScopeAttrs, unsigned kind) {
2388 if (!aliasScopeAttrs || aliasScopeAttrs.empty())
2389 return;
2390 llvm::MDNode *node = getOrCreateAliasScopes(
2391 aliasScopeAttrs: llvm::to_vector(Range: aliasScopeAttrs.getAsRange<AliasScopeAttr>()));
2392 inst->setMetadata(KindID: kind, Node: node);
2393 };
2394
2395 populateScopeMetadata(op.getAliasScopesOrNull(),
2396 llvm::LLVMContext::MD_alias_scope);
2397 populateScopeMetadata(op.getNoAliasScopesOrNull(),
2398 llvm::LLVMContext::MD_noalias);
2399}
2400
2401llvm::MDNode *ModuleTranslation::getTBAANode(TBAATagAttr tbaaAttr) const {
2402 return tbaaMetadataMapping.lookup(Val: tbaaAttr);
2403}
2404
2405void ModuleTranslation::setTBAAMetadata(AliasAnalysisOpInterface op,
2406 llvm::Instruction *inst) {
2407 ArrayAttr tagRefs = op.getTBAATagsOrNull();
2408 if (!tagRefs || tagRefs.empty())
2409 return;
2410
2411 // LLVM IR currently does not support attaching more than one TBAA access tag
2412 // to a memory accessing instruction. It may be useful to support this in
2413 // future, but for the time being just ignore the metadata if MLIR operation
2414 // has multiple access tags.
2415 if (tagRefs.size() > 1) {
2416 op.emitWarning() << "TBAA access tags were not translated, because LLVM "
2417 "IR only supports a single tag per instruction";
2418 return;
2419 }
2420
2421 llvm::MDNode *node = getTBAANode(tbaaAttr: cast<TBAATagAttr>(Val: tagRefs[0]));
2422 inst->setMetadata(KindID: llvm::LLVMContext::MD_tbaa, Node: node);
2423}
2424
2425void ModuleTranslation::setTBAAStructMetadata(ArrayAttr fields,
2426 llvm::Instruction *inst) {
2427 if (!fields || fields.empty())
2428 return;
2429
2430 // The node is a flat list of (offset, size, tag) triples, not a list of
2431 // per-field nodes.
2432 SmallVector<llvm::MDBuilder::TBAAStructField> structFields;
2433 structFields.reserve(N: fields.size());
2434 for (Attribute field : fields) {
2435 auto fieldAttr = cast<TBAAStructFieldAttr>(Val&: field);
2436 structFields.emplace_back(Args: fieldAttr.getOffset(), Args: fieldAttr.getSize(),
2437 Args: getTBAANode(tbaaAttr: fieldAttr.getTag()));
2438 }
2439 inst->setMetadata(
2440 KindID: llvm::LLVMContext::MD_tbaa_struct,
2441 Node: llvm::MDBuilder(getLLVMContext()).createTBAAStructNode(Fields: structFields));
2442}
2443
2444void ModuleTranslation::setDereferenceableMetadata(
2445 DereferenceableOpInterface op, llvm::Instruction *inst) {
2446 DereferenceableAttr derefAttr = op.getDereferenceableOrNull();
2447 if (!derefAttr)
2448 return;
2449
2450 llvm::MDNode *derefSizeNode = llvm::MDNode::get(
2451 Context&: getLLVMContext(),
2452 MDs: llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
2453 Ty: llvm::IntegerType::get(C&: getLLVMContext(), NumBits: 64), V: derefAttr.getBytes())));
2454 unsigned kindId = derefAttr.getMayBeNull()
2455 ? llvm::LLVMContext::MD_dereferenceable_or_null
2456 : llvm::LLVMContext::MD_dereferenceable;
2457 inst->setMetadata(KindID: kindId, Node: derefSizeNode);
2458}
2459
2460void ModuleTranslation::setBranchWeightsMetadata(WeightedBranchOpInterface op) {
2461 SmallVector<uint32_t> weights;
2462 llvm::transform(Range: op.getWeights(), d_first: std::back_inserter(x&: weights),
2463 F: [](int32_t value) { return static_cast<uint32_t>(value); });
2464 if (weights.empty())
2465 return;
2466
2467 llvm::Instruction *inst = isa<CallOp>(Val: op) ? lookupCall(op) : lookupBranch(op);
2468 assert(inst && "expected the operation to have a mapping to an instruction");
2469 inst->setMetadata(
2470 KindID: llvm::LLVMContext::MD_prof,
2471 Node: llvm::MDBuilder(getLLVMContext()).createBranchWeights(Weights: weights));
2472}
2473
2474LogicalResult ModuleTranslation::createTBAAMetadata() {
2475 llvm::LLVMContext &ctx = llvmModule->getContext();
2476 llvm::IntegerType *offsetTy = llvm::IntegerType::get(C&: ctx, NumBits: 64);
2477
2478 // Walk the entire module and create all metadata nodes for the TBAA
2479 // attributes. The code below relies on two invariants of the
2480 // `AttrTypeWalker`:
2481 // 1. Attributes are visited in post-order: Since the attributes create a DAG,
2482 // this ensures that any lookups into `tbaaMetadataMapping` for child
2483 // attributes succeed.
2484 // 2. Attributes are only ever visited once: This way we don't leak any
2485 // LLVM metadata instances.
2486 AttrTypeWalker walker;
2487 walker.addWalk(callback: [&](TBAARootAttr root) {
2488 llvm::MDNode *node;
2489 if (StringAttr id = root.getId()) {
2490 node = llvm::MDNode::get(Context&: ctx, MDs: llvm::MDString::get(Context&: ctx, Str: id));
2491 } else {
2492 // Anonymous root nodes are self-referencing.
2493 auto selfRef = llvm::MDNode::getTemporary(Context&: ctx, MDs: {});
2494 node = llvm::MDNode::get(Context&: ctx, MDs: {selfRef.get()});
2495 node->replaceOperandWith(I: 0, New: node);
2496 }
2497 tbaaMetadataMapping.insert(KV: {root, node});
2498 });
2499
2500 walker.addWalk(callback: [&](TBAATypeDescriptorAttr descriptor) {
2501 SmallVector<llvm::Metadata *> operands;
2502 operands.push_back(Elt: llvm::MDString::get(Context&: ctx, Str: descriptor.getId()));
2503 for (TBAAMemberAttr member : descriptor.getMembers()) {
2504 operands.push_back(Elt: tbaaMetadataMapping.lookup(Val: member.getTypeDesc()));
2505 operands.push_back(Elt: llvm::ConstantAsMetadata::get(
2506 C: llvm::ConstantInt::get(Ty: offsetTy, V: member.getOffset())));
2507 }
2508
2509 tbaaMetadataMapping.insert(KV: {descriptor, llvm::MDNode::get(Context&: ctx, MDs: operands)});
2510 });
2511
2512 walker.addWalk(callback: [&](TBAATagAttr tag) {
2513 SmallVector<llvm::Metadata *> operands;
2514
2515 operands.push_back(Elt: tbaaMetadataMapping.lookup(Val: tag.getBaseType()));
2516 operands.push_back(Elt: tbaaMetadataMapping.lookup(Val: tag.getAccessType()));
2517
2518 operands.push_back(Elt: llvm::ConstantAsMetadata::get(
2519 C: llvm::ConstantInt::get(Ty: offsetTy, V: tag.getOffset())));
2520 if (tag.getConstant())
2521 operands.push_back(
2522 Elt: llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(Ty: offsetTy, V: 1)));
2523
2524 tbaaMetadataMapping.insert(KV: {tag, llvm::MDNode::get(Context&: ctx, MDs: operands)});
2525 });
2526
2527 mlirModule->walk(callback: [&](AliasAnalysisOpInterface analysisOpInterface) {
2528 if (auto attr = analysisOpInterface.getTBAATagsOrNull())
2529 walker.walk(element: attr);
2530 });
2531
2532 // The tags inside a `tbaa_struct` are reachable from nowhere else, so without
2533 // this walk they never get an MDNode and the exported node has null tags.
2534 auto walkStructFields = [&](ArrayAttr fields) {
2535 if (fields)
2536 walker.walk(element: fields);
2537 };
2538 mlirModule->walk(callback: [&](TBAAStructOpInterface tbaaStructOp) {
2539 walkStructFields(tbaaStructOp.getTBAAStructOrNull());
2540 });
2541
2542 return success();
2543}
2544
2545LogicalResult ModuleTranslation::createIdentMetadata() {
2546 if (auto attr = mlirModule->getDiscardableAttrOfType<StringAttr>(
2547 name: LLVMDialect::getIdentAttrName())) {
2548 StringRef ident = attr;
2549 llvm::LLVMContext &ctx = llvmModule->getContext();
2550 llvm::NamedMDNode *namedMd =
2551 llvmModule->getOrInsertNamedMetadata(Name: LLVMDialect::getIdentAttrName());
2552 llvm::MDNode *md = llvm::MDNode::get(Context&: ctx, MDs: llvm::MDString::get(Context&: ctx, Str: ident));
2553 namedMd->addOperand(M: md);
2554 }
2555
2556 return success();
2557}
2558
2559LogicalResult ModuleTranslation::createCommandlineMetadata() {
2560 if (auto attr = mlirModule->getDiscardableAttrOfType<StringAttr>(
2561 name: LLVMDialect::getCommandlineAttrName())) {
2562 StringRef cmdLine = attr;
2563 llvm::LLVMContext &ctx = llvmModule->getContext();
2564 llvm::NamedMDNode *nmd = llvmModule->getOrInsertNamedMetadata(
2565 Name: LLVMDialect::getCommandlineAttrName());
2566 llvm::MDNode *md =
2567 llvm::MDNode::get(Context&: ctx, MDs: llvm::MDString::get(Context&: ctx, Str: cmdLine));
2568 nmd->addOperand(M: md);
2569 }
2570
2571 return success();
2572}
2573
2574LogicalResult ModuleTranslation::createDependentLibrariesMetadata() {
2575 if (auto dependentLibrariesAttr = mlirModule->getDiscardableAttr(
2576 name: LLVM::LLVMDialect::getDependentLibrariesAttrName())) {
2577 auto *nmd =
2578 llvmModule->getOrInsertNamedMetadata(Name: "llvm.dependent-libraries");
2579 llvm::LLVMContext &ctx = llvmModule->getContext();
2580 for (auto libAttr :
2581 cast<ArrayAttr>(Val&: dependentLibrariesAttr).getAsRange<StringAttr>()) {
2582 auto *md =
2583 llvm::MDNode::get(Context&: ctx, MDs: llvm::MDString::get(Context&: ctx, Str: libAttr.getValue()));
2584 nmd->addOperand(M: md);
2585 }
2586 }
2587 return success();
2588}
2589
2590void ModuleTranslation::setLoopMetadata(Operation *op,
2591 llvm::Instruction *inst) {
2592 LoopAnnotationAttr attr =
2593 TypeSwitch<Operation *, LoopAnnotationAttr>(op)
2594 .Case<LLVM::BrOp, LLVM::CondBrOp>(
2595 caseFn: [](auto branchOp) { return branchOp.getLoopAnnotationAttr(); });
2596 if (!attr)
2597 return;
2598 llvm::MDNode *loopMD =
2599 loopAnnotationTranslation->translateLoopAnnotation(attr, op);
2600 inst->setMetadata(KindID: llvm::LLVMContext::MD_loop, Node: loopMD);
2601}
2602
2603void ModuleTranslation::setDisjointFlag(Operation *op, llvm::Value *value) {
2604 auto iface = cast<DisjointFlagInterface>(Val: op);
2605 // We do a dyn_cast here in case the value got folded into a constant.
2606 if (auto *disjointInst = dyn_cast<llvm::PossiblyDisjointInst>(Val: value))
2607 disjointInst->setIsDisjoint(iface.getIsDisjoint());
2608}
2609
2610llvm::Type *ModuleTranslation::convertType(Type type) {
2611 return typeTranslator.translateType(type);
2612}
2613
2614/// A helper to look up remapped operands in the value remapping table.
2615SmallVector<llvm::Value *> ModuleTranslation::lookupValues(ValueRange values) {
2616 SmallVector<llvm::Value *> remapped;
2617 remapped.reserve(N: values.size());
2618 for (Value v : values)
2619 remapped.push_back(Elt: lookupValue(value: v));
2620 return remapped;
2621}
2622
2623void ModuleTranslation::remapAllValuesWith(llvm::Value *oldValue,
2624 llvm::Value *newValue) {
2625 if (oldValue == newValue)
2626 return;
2627
2628 for (auto &entry : valueMapping)
2629 if (entry.second == oldValue)
2630 entry.second = newValue;
2631}
2632
2633llvm::vfs::FileSystem &ModuleTranslation::getFileSystem() {
2634 if (fileSystem)
2635 return *fileSystem;
2636 return *llvm::vfs::getRealFileSystem();
2637}
2638
2639llvm::DILocation *ModuleTranslation::translateLoc(Location loc,
2640 llvm::DILocalScope *scope) {
2641 return debugTranslation->translateLoc(loc, scope);
2642}
2643
2644llvm::DIExpression *
2645ModuleTranslation::translateExpression(LLVM::DIExpressionAttr attr) {
2646 return debugTranslation->translateExpression(attr);
2647}
2648
2649llvm::DIGlobalVariableExpression *
2650ModuleTranslation::translateGlobalVariableExpression(
2651 LLVM::DIGlobalVariableExpressionAttr attr) {
2652 return debugTranslation->translateGlobalVariableExpression(attr);
2653}
2654
2655llvm::Metadata *ModuleTranslation::translateDebugInfo(LLVM::DINodeAttr attr) {
2656 return debugTranslation->translate(attr);
2657}
2658
2659llvm::RoundingMode
2660ModuleTranslation::translateRoundingMode(LLVM::RoundingMode rounding) {
2661 return convertRoundingModeToLLVM(value: rounding);
2662}
2663
2664llvm::fp::ExceptionBehavior ModuleTranslation::translateFPExceptionBehavior(
2665 LLVM::FPExceptionBehavior exceptionBehavior) {
2666 return convertFPExceptionBehaviorToLLVM(value: exceptionBehavior);
2667}
2668
2669llvm::NamedMDNode *
2670ModuleTranslation::getOrInsertNamedModuleMetadata(StringRef name) {
2671 return llvmModule->getOrInsertNamedMetadata(Name: name);
2672}
2673
2674static std::unique_ptr<llvm::Module>
2675prepareLLVMModule(Operation *m, llvm::LLVMContext &llvmContext,
2676 StringRef name) {
2677 m->getContext()->getOrLoadDialect<LLVM::LLVMDialect>();
2678 auto llvmModule = std::make_unique<llvm::Module>(args&: name, args&: llvmContext);
2679 if (auto dataLayoutAttr =
2680 m->getDiscardableAttr(name: LLVM::LLVMDialect::getDataLayoutAttrName())) {
2681 llvmModule->setDataLayout(cast<StringAttr>(Val&: dataLayoutAttr).getValue());
2682 } else {
2683 FailureOr<llvm::DataLayout> llvmDataLayout(llvm::DataLayout(""));
2684 if (auto iface = dyn_cast<DataLayoutOpInterface>(Val: m)) {
2685 if (DataLayoutSpecInterface spec = iface.getDataLayoutSpec()) {
2686 llvmDataLayout =
2687 translateDataLayout(attribute: spec, dataLayout: DataLayout(iface), loc: m->getLoc());
2688 }
2689 } else if (auto mod = dyn_cast<ModuleOp>(Val: m)) {
2690 if (DataLayoutSpecInterface spec = mod.getDataLayoutSpec()) {
2691 llvmDataLayout =
2692 translateDataLayout(attribute: spec, dataLayout: DataLayout(mod), loc: m->getLoc());
2693 }
2694 }
2695 if (failed(Result: llvmDataLayout))
2696 return nullptr;
2697 llvmModule->setDataLayout(*llvmDataLayout);
2698 }
2699 if (auto targetTripleAttr =
2700 m->getDiscardableAttr(name: LLVM::LLVMDialect::getTargetTripleAttrName()))
2701 llvmModule->setTargetTriple(
2702 llvm::Triple(cast<StringAttr>(Val&: targetTripleAttr).getValue()));
2703
2704 if (auto asmAttr = m->getDiscardableAttr(
2705 name: LLVM::LLVMDialect::getModuleLevelAsmAttrName())) {
2706 auto asmArrayAttr = dyn_cast<ArrayAttr>(Val&: asmAttr);
2707 if (!asmArrayAttr) {
2708 m->emitError(message: "expected an array attribute for a module level asm");
2709 return nullptr;
2710 }
2711
2712 for (Attribute elt : asmArrayAttr) {
2713 auto asmStrAttr = dyn_cast<StringAttr>(Val&: elt);
2714 if (!asmStrAttr) {
2715 m->emitError(
2716 message: "expected a string attribute for each entry of a module level asm");
2717 return nullptr;
2718 }
2719 llvmModule->appendModuleInlineAsm(Fragment: asmStrAttr.getValue());
2720 }
2721 }
2722
2723 return llvmModule;
2724}
2725
2726std::unique_ptr<llvm::Module>
2727mlir::translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext,
2728 StringRef name, bool disableVerification,
2729 llvm::vfs::FileSystem *fs) {
2730 if (!satisfiesLLVMModule(op: module)) {
2731 module->emitOpError(message: "can not be translated to an LLVMIR module");
2732 return nullptr;
2733 }
2734
2735 std::unique_ptr<llvm::Module> llvmModule =
2736 prepareLLVMModule(m: module, llvmContext, name);
2737 if (!llvmModule)
2738 return nullptr;
2739
2740 LLVM::ensureDistinctSuccessors(op: module);
2741 LLVM::legalizeDIExpressionsRecursively(op: module);
2742
2743 ModuleTranslation translator(module, std::move(llvmModule), fs);
2744 llvm::IRBuilder<llvm::TargetFolder> llvmBuilder(
2745 *translator.getLLVMModule(),
2746 llvm::TargetFolder(translator.getLLVMModule()->getDataLayout()));
2747
2748 // Convert module before functions and operations inside, so dialect
2749 // attributes can be used to change dialect-specific global configurations via
2750 // `amendOperation()`. These configurations can then influence the translation
2751 // of operations afterwards.
2752 if (failed(Result: translator.convertOperation(op&: *module, builder&: llvmBuilder)))
2753 return nullptr;
2754
2755 if (failed(Result: translator.convertComdats()))
2756 return nullptr;
2757 if (failed(Result: translator.convertFunctionSignatures()))
2758 return nullptr;
2759 if (failed(Result: translator.convertGlobalsAndAliases()))
2760 return nullptr;
2761 if (failed(Result: translator.convertIFuncs()))
2762 return nullptr;
2763 if (failed(Result: translator.convertGlobalMetadata()))
2764 return nullptr;
2765 if (failed(Result: translator.convertFunctionMetadata()))
2766 return nullptr;
2767 if (failed(Result: translator.createTBAAMetadata()))
2768 return nullptr;
2769 if (failed(Result: translator.createIdentMetadata()))
2770 return nullptr;
2771 if (failed(Result: translator.createCommandlineMetadata()))
2772 return nullptr;
2773 if (failed(Result: translator.createDependentLibrariesMetadata()))
2774 return nullptr;
2775
2776 // Convert other top-level operations if possible.
2777 for (Operation &o : getModuleBody(module).getOperations()) {
2778 if (!isa<LLVM::LLVMFuncOp, LLVM::AliasOp, LLVM::GlobalOp,
2779 LLVM::GlobalCtorsOp, LLVM::GlobalDtorsOp, LLVM::ComdatOp,
2780 LLVM::IFuncOp>(Val: &o) &&
2781 !o.hasTrait<OpTrait::IsTerminator>() &&
2782 failed(Result: translator.convertOperation(op&: o, builder&: llvmBuilder))) {
2783 return nullptr;
2784 }
2785 }
2786
2787 // Operations in function bodies with symbolic references must be converted
2788 // after the top-level operations they refer to are declared, so we do it
2789 // last.
2790 if (failed(Result: translator.convertFunctions()))
2791 return nullptr;
2792
2793 // Now that all MLIR blocks are resolved into LLVM ones, patch block address
2794 // constants to point to the correct blocks.
2795 if (failed(Result: translator.convertUnresolvedBlockAddress()))
2796 return nullptr;
2797
2798 // Add the necessary debug info module flags, if they were not encoded in MLIR
2799 // beforehand.
2800 translator.debugTranslation->addModuleFlagsIfNotPresent();
2801
2802 // Let dialect translations complete their pending work (e.g. OpenMP region
2803 // outlining) prior to verifying the module. Index-based iteration tolerates
2804 // extensions created during finalization.
2805 for (size_t i = 0; i < translator.extensions.size(); ++i)
2806 if (failed(Result: translator.extensions.begin()[i].second->finalize()))
2807 return nullptr;
2808
2809 if (!disableVerification &&
2810 llvm::verifyModule(M: *translator.llvmModule, OS: &llvm::errs()))
2811 return nullptr;
2812
2813 return std::move(translator.llvmModule);
2814}
2815