| 1 | //===- ModuleTranslation.cpp - MLIR to LLVM conversion --------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements the translation between an MLIR LLVM dialect module and |
| 10 | // the corresponding LLVMIR module. It only handles core LLVM IR operations. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Target/LLVMIR/ModuleTranslation.h" |
| 15 | |
| 16 | #include "AttrKindDetail.h" |
| 17 | #include "DebugTranslation.h" |
| 18 | #include "LoopAnnotationTranslation.h" |
| 19 | #include "mlir/Analysis/TopologicalSortUtils.h" |
| 20 | #include "mlir/Dialect/DLTI/DLTI.h" |
| 21 | #include "mlir/Dialect/LLVMIR/LLVMCallOps.h" |
| 22 | #include "mlir/Dialect/LLVMIR/LLVMInterfaces.h" |
| 23 | #include "mlir/Dialect/LLVMIR/LLVMModuleOps.h" |
| 24 | #include "mlir/Dialect/LLVMIR/LLVMTerminatorOps.h" |
| 25 | #include "mlir/Dialect/LLVMIR/Transforms/DIExpressionLegalization.h" |
| 26 | #include "mlir/Dialect/LLVMIR/Transforms/LegalizeForExport.h" |
| 27 | #include "mlir/IR/AttrTypeSubElements.h" |
| 28 | #include "mlir/IR/Attributes.h" |
| 29 | #include "mlir/IR/BuiltinOps.h" |
| 30 | #include "mlir/IR/BuiltinTypes.h" |
| 31 | #include "mlir/IR/DialectResourceBlobManager.h" |
| 32 | #include "mlir/Support/LLVM.h" |
| 33 | #include "mlir/Target/LLVMIR/LLVMTranslationInterface.h" |
| 34 | #include "mlir/Target/LLVMIR/TypeToLLVM.h" |
| 35 | |
| 36 | #include "llvm/ADT/DenseSet.h" |
| 37 | #include "llvm/ADT/STLExtras.h" |
| 38 | #include "llvm/ADT/StringExtras.h" |
| 39 | #include "llvm/ADT/TypeSwitch.h" |
| 40 | #include "llvm/Analysis/TargetFolder.h" |
| 41 | #include "llvm/IR/BasicBlock.h" |
| 42 | #include "llvm/IR/CFG.h" |
| 43 | #include "llvm/IR/Constants.h" |
| 44 | #include "llvm/IR/DerivedTypes.h" |
| 45 | #include "llvm/IR/IRBuilder.h" |
| 46 | #include "llvm/IR/InlineAsm.h" |
| 47 | #include "llvm/IR/LLVMContext.h" |
| 48 | #include "llvm/IR/MDBuilder.h" |
| 49 | #include "llvm/IR/Metadata.h" |
| 50 | #include "llvm/IR/Module.h" |
| 51 | #include "llvm/IR/Verifier.h" |
| 52 | #include "llvm/Support/Debug.h" |
| 53 | #include "llvm/Support/ErrorHandling.h" |
| 54 | #include "llvm/Support/VirtualFileSystem.h" |
| 55 | #include "llvm/Support/raw_ostream.h" |
| 56 | #include "llvm/Transforms/Utils/BasicBlockUtils.h" |
| 57 | #include "llvm/Transforms/Utils/Cloning.h" |
| 58 | #include "llvm/Transforms/Utils/ModuleUtils.h" |
| 59 | #include <numeric> |
| 60 | #include <optional> |
| 61 | |
| 62 | #define DEBUG_TYPE "llvm-dialect-to-llvm-ir" |
| 63 | |
| 64 | using namespace mlir; |
| 65 | using namespace mlir::LLVM; |
| 66 | using namespace mlir::LLVM::detail; |
| 67 | |
| 68 | #include "mlir/Dialect/LLVMIR/LLVMConversionEnumsToLLVM.inc" |
| 69 | |
| 70 | namespace { |
| 71 | /// A customized inserter for LLVM's IRBuilder that captures all LLVM IR |
| 72 | /// instructions that are created for future reference. |
| 73 | /// |
| 74 | /// This is intended to be used with the `CollectionScope` RAII object: |
| 75 | /// |
| 76 | /// llvm::IRBuilder<..., InstructionCapturingInserter> builder; |
| 77 | /// { |
| 78 | /// InstructionCapturingInserter::CollectionScope scope(builder); |
| 79 | /// // Call IRBuilder methods as usual. |
| 80 | /// |
| 81 | /// // This will return a list of all instructions created by the builder, |
| 82 | /// // in order of creation. |
| 83 | /// builder.getInserter().getCapturedInstructions(); |
| 84 | /// } |
| 85 | /// // This will return an empty list. |
| 86 | /// builder.getInserter().getCapturedInstructions(); |
| 87 | /// |
| 88 | /// The capturing functionality is _disabled_ by default for performance |
| 89 | /// consideration. It needs to be explicitly enabled, which is achieved by |
| 90 | /// creating a `CollectionScope`. |
| 91 | class InstructionCapturingInserter : public llvm::IRBuilderCallbackInserter { |
| 92 | public: |
| 93 | /// Constructs the inserter. |
| 94 | InstructionCapturingInserter() |
| 95 | : llvm::IRBuilderCallbackInserter([this](llvm::Instruction *instruction) { |
| 96 | if (LLVM_LIKELY(enabled)) |
| 97 | capturedInstructions.push_back(Elt: instruction); |
| 98 | }) {} |
| 99 | |
| 100 | /// Returns the list of LLVM IR instructions captured since the last cleanup. |
| 101 | ArrayRef<llvm::Instruction *> getCapturedInstructions() const { |
| 102 | return capturedInstructions; |
| 103 | } |
| 104 | |
| 105 | /// Clears the list of captured LLVM IR instructions. |
| 106 | void clearCapturedInstructions() { capturedInstructions.clear(); } |
| 107 | |
| 108 | /// RAII object enabling the capture of created LLVM IR instructions. |
| 109 | class CollectionScope { |
| 110 | public: |
| 111 | /// Creates the scope for the given inserter. |
| 112 | CollectionScope(llvm::IRBuilderBase &irBuilder, bool isBuilderCapturing); |
| 113 | |
| 114 | /// Ends the scope. |
| 115 | ~CollectionScope(); |
| 116 | |
| 117 | ArrayRef<llvm::Instruction *> getCapturedInstructions() { |
| 118 | if (!inserter) |
| 119 | return {}; |
| 120 | return inserter->getCapturedInstructions(); |
| 121 | } |
| 122 | |
| 123 | private: |
| 124 | /// Back reference to the inserter. |
| 125 | InstructionCapturingInserter *inserter = nullptr; |
| 126 | |
| 127 | /// List of instructions in the inserter prior to this scope. |
| 128 | SmallVector<llvm::Instruction *> previouslyCollectedInstructions; |
| 129 | |
| 130 | /// Whether the inserter was enabled prior to this scope. |
| 131 | bool wasEnabled; |
| 132 | }; |
| 133 | |
| 134 | /// Enable or disable the capturing mechanism. |
| 135 | void setEnabled(bool enabled = true) { this->enabled = enabled; } |
| 136 | |
| 137 | private: |
| 138 | /// List of captured instructions. |
| 139 | SmallVector<llvm::Instruction *> capturedInstructions; |
| 140 | |
| 141 | /// Whether the collection is enabled. |
| 142 | bool enabled = false; |
| 143 | }; |
| 144 | |
| 145 | using CapturingIRBuilder = |
| 146 | llvm::IRBuilder<llvm::TargetFolder, InstructionCapturingInserter>; |
| 147 | } // namespace |
| 148 | |
| 149 | InstructionCapturingInserter::CollectionScope::CollectionScope( |
| 150 | llvm::IRBuilderBase &irBuilder, bool isBuilderCapturing) { |
| 151 | |
| 152 | if (!isBuilderCapturing) |
| 153 | return; |
| 154 | |
| 155 | auto &capturingIRBuilder = static_cast<CapturingIRBuilder &>(irBuilder); |
| 156 | inserter = &capturingIRBuilder.getInserter(); |
| 157 | wasEnabled = inserter->enabled; |
| 158 | if (wasEnabled) |
| 159 | previouslyCollectedInstructions.swap(RHS&: inserter->capturedInstructions); |
| 160 | inserter->setEnabled(true); |
| 161 | } |
| 162 | |
| 163 | InstructionCapturingInserter::CollectionScope::~CollectionScope() { |
| 164 | if (!inserter) |
| 165 | return; |
| 166 | |
| 167 | previouslyCollectedInstructions.swap(RHS&: inserter->capturedInstructions); |
| 168 | // If collection was enabled (likely in another, surrounding scope), keep |
| 169 | // the instructions collected in this scope. |
| 170 | if (wasEnabled) { |
| 171 | llvm::append_range(C&: inserter->capturedInstructions, |
| 172 | R&: previouslyCollectedInstructions); |
| 173 | } |
| 174 | inserter->setEnabled(wasEnabled); |
| 175 | } |
| 176 | |
| 177 | /// Translates the given data layout spec attribute to the LLVM IR data layout. |
| 178 | /// Only integer, float, pointer and endianness entries are currently supported. |
| 179 | static FailureOr<llvm::DataLayout> |
| 180 | translateDataLayout(DataLayoutSpecInterface attribute, |
| 181 | const DataLayout &dataLayout, |
| 182 | std::optional<Location> loc = std::nullopt) { |
| 183 | if (!loc) |
| 184 | loc = UnknownLoc::get(context: attribute.getContext()); |
| 185 | |
| 186 | // Translate the endianness attribute. |
| 187 | std::string llvmDataLayout; |
| 188 | llvm::raw_string_ostream layoutStream(llvmDataLayout); |
| 189 | for (DataLayoutEntryInterface entry : attribute.getEntries()) { |
| 190 | auto key = llvm::dyn_cast_if_present<StringAttr>(Val: entry.getKey()); |
| 191 | if (!key) |
| 192 | continue; |
| 193 | if (key.getValue() == DLTIDialect::kDataLayoutEndiannessKey) { |
| 194 | auto value = cast<StringAttr>(Val: entry.getValue()); |
| 195 | bool isLittleEndian = |
| 196 | value.getValue() == DLTIDialect::kDataLayoutEndiannessLittle; |
| 197 | layoutStream << "-" << (isLittleEndian ? "e" : "E" ); |
| 198 | continue; |
| 199 | } |
| 200 | if (key.getValue() == DLTIDialect::kDataLayoutManglingModeKey) { |
| 201 | auto value = cast<StringAttr>(Val: entry.getValue()); |
| 202 | layoutStream << "-m:" << value.getValue(); |
| 203 | continue; |
| 204 | } |
| 205 | if (key.getValue() == DLTIDialect::kDataLayoutProgramMemorySpaceKey) { |
| 206 | auto value = cast<IntegerAttr>(Val: entry.getValue()); |
| 207 | uint64_t space = value.getValue().getZExtValue(); |
| 208 | // Skip the default address space. |
| 209 | if (space == 0) |
| 210 | continue; |
| 211 | layoutStream << "-P" << space; |
| 212 | continue; |
| 213 | } |
| 214 | if (key.getValue() == DLTIDialect::kDataLayoutGlobalMemorySpaceKey) { |
| 215 | auto value = cast<IntegerAttr>(Val: entry.getValue()); |
| 216 | uint64_t space = value.getValue().getZExtValue(); |
| 217 | // Skip the default address space. |
| 218 | if (space == 0) |
| 219 | continue; |
| 220 | layoutStream << "-G" << space; |
| 221 | continue; |
| 222 | } |
| 223 | if (key.getValue() == DLTIDialect::kDataLayoutAllocaMemorySpaceKey) { |
| 224 | auto value = cast<IntegerAttr>(Val: entry.getValue()); |
| 225 | uint64_t space = value.getValue().getZExtValue(); |
| 226 | // Skip the default address space. |
| 227 | if (space == 0) |
| 228 | continue; |
| 229 | layoutStream << "-A" << space; |
| 230 | continue; |
| 231 | } |
| 232 | if (key.getValue() == DLTIDialect::kDataLayoutStackAlignmentKey) { |
| 233 | auto value = cast<IntegerAttr>(Val: entry.getValue()); |
| 234 | uint64_t alignment = value.getValue().getZExtValue(); |
| 235 | // Skip the default stack alignment. |
| 236 | if (alignment == 0) |
| 237 | continue; |
| 238 | layoutStream << "-S" << alignment; |
| 239 | continue; |
| 240 | } |
| 241 | if (key.getValue() == DLTIDialect::kDataLayoutFunctionPointerAlignmentKey) { |
| 242 | auto value = cast<FunctionPointerAlignmentAttr>(Val: entry.getValue()); |
| 243 | uint64_t alignment = value.getAlignment(); |
| 244 | // Skip the default function pointer alignment. |
| 245 | if (alignment == 0) |
| 246 | continue; |
| 247 | layoutStream << "-F" << (value.getFunctionDependent() ? "n" : "i" ) |
| 248 | << alignment; |
| 249 | continue; |
| 250 | } |
| 251 | if (key.getValue() == DLTIDialect::kDataLayoutLegalIntWidthsKey) { |
| 252 | layoutStream << "-n" ; |
| 253 | llvm::interleave( |
| 254 | c: cast<DenseI32ArrayAttr>(Val: entry.getValue()).asArrayRef(), os&: layoutStream, |
| 255 | each_fn: [&](int32_t val) { layoutStream << val; }, separator: ":" ); |
| 256 | continue; |
| 257 | } |
| 258 | emitError(loc: *loc) << "unsupported data layout key " << key; |
| 259 | return failure(); |
| 260 | } |
| 261 | |
| 262 | // Go through the list of entries to check which types are explicitly |
| 263 | // specified in entries. Where possible, data layout queries are used instead |
| 264 | // of directly inspecting the entries. |
| 265 | for (DataLayoutEntryInterface entry : attribute.getEntries()) { |
| 266 | auto type = llvm::dyn_cast_if_present<Type>(Val: entry.getKey()); |
| 267 | if (!type) |
| 268 | continue; |
| 269 | // Data layout for the index type is irrelevant at this point. |
| 270 | if (isa<IndexType>(Val: type)) |
| 271 | continue; |
| 272 | layoutStream << "-" ; |
| 273 | LogicalResult result = |
| 274 | llvm::TypeSwitch<Type, LogicalResult>(type) |
| 275 | .Case<IntegerType, Float16Type, Float32Type, Float64Type, |
| 276 | Float80Type, Float128Type>(caseFn: [&](Type type) -> LogicalResult { |
| 277 | if (auto intType = dyn_cast<IntegerType>(Val&: type)) { |
| 278 | if (intType.getSignedness() != IntegerType::Signless) |
| 279 | return emitError(loc: *loc) |
| 280 | << "unsupported data layout for non-signless integer " |
| 281 | << intType; |
| 282 | layoutStream << "i" ; |
| 283 | } else { |
| 284 | layoutStream << "f" ; |
| 285 | } |
| 286 | uint64_t size = dataLayout.getTypeSizeInBits(t: type); |
| 287 | uint64_t abi = dataLayout.getTypeABIAlignment(t: type) * 8u; |
| 288 | uint64_t preferred = |
| 289 | dataLayout.getTypePreferredAlignment(t: type) * 8u; |
| 290 | layoutStream << size << ":" << abi; |
| 291 | if (abi != preferred) |
| 292 | layoutStream << ":" << preferred; |
| 293 | return success(); |
| 294 | }) |
| 295 | .Case(caseFn: [&](LLVMPointerType type) { |
| 296 | layoutStream << "p" << type.getAddressSpace() << ":" ; |
| 297 | uint64_t size = dataLayout.getTypeSizeInBits(t: type); |
| 298 | uint64_t abi = dataLayout.getTypeABIAlignment(t: type) * 8u; |
| 299 | uint64_t preferred = |
| 300 | dataLayout.getTypePreferredAlignment(t: type) * 8u; |
| 301 | uint64_t index = *dataLayout.getTypeIndexBitwidth(t: type); |
| 302 | layoutStream << size << ":" << abi << ":" << preferred << ":" |
| 303 | << index; |
| 304 | return success(); |
| 305 | }) |
| 306 | .Default(defaultFn: [loc](Type type) { |
| 307 | return emitError(loc: *loc) |
| 308 | << "unsupported type in data layout: " << type; |
| 309 | }); |
| 310 | if (failed(Result: result)) |
| 311 | return failure(); |
| 312 | } |
| 313 | StringRef layoutSpec(llvmDataLayout); |
| 314 | layoutSpec.consume_front(Prefix: "-" ); |
| 315 | |
| 316 | return llvm::DataLayout(layoutSpec); |
| 317 | } |
| 318 | |
| 319 | /// Builds a constant of a sequential LLVM type `type`, potentially containing |
| 320 | /// other sequential types recursively, from the individual constant values |
| 321 | /// provided in `constants`. `shape` contains the number of elements in nested |
| 322 | /// sequential types. Reports errors at `loc` and returns nullptr on error. |
| 323 | static llvm::Constant * |
| 324 | buildSequentialConstant(ArrayRef<llvm::Constant *> &constants, |
| 325 | ArrayRef<int64_t> shape, llvm::Type *type, |
| 326 | Location loc) { |
| 327 | if (shape.empty()) { |
| 328 | llvm::Constant *result = constants.front(); |
| 329 | constants = constants.drop_front(); |
| 330 | return result; |
| 331 | } |
| 332 | |
| 333 | llvm::Type *elementType; |
| 334 | if (auto *arrayTy = dyn_cast<llvm::ArrayType>(Val: type)) { |
| 335 | elementType = arrayTy->getElementType(); |
| 336 | } else if (auto *vectorTy = dyn_cast<llvm::VectorType>(Val: type)) { |
| 337 | elementType = vectorTy->getElementType(); |
| 338 | } else { |
| 339 | emitError(loc) << "expected sequential LLVM types wrapping a scalar" ; |
| 340 | return nullptr; |
| 341 | } |
| 342 | |
| 343 | SmallVector<llvm::Constant *, 8> nested; |
| 344 | nested.reserve(N: shape.front()); |
| 345 | for (int64_t i = 0; i < shape.front(); ++i) { |
| 346 | nested.push_back(Elt: buildSequentialConstant(constants, shape: shape.drop_front(), |
| 347 | type: elementType, loc)); |
| 348 | if (!nested.back()) |
| 349 | return nullptr; |
| 350 | } |
| 351 | |
| 352 | if (shape.size() == 1 && type->isVectorTy()) |
| 353 | return llvm::ConstantVector::get(V: nested); |
| 354 | return llvm::ConstantArray::get( |
| 355 | T: llvm::ArrayType::get(ElementType: elementType, NumElements: shape.front()), V: nested); |
| 356 | } |
| 357 | |
| 358 | /// Returns the first non-sequential type nested in sequential types. |
| 359 | static llvm::Type *getInnermostElementType(llvm::Type *type) { |
| 360 | do { |
| 361 | if (auto *arrayTy = dyn_cast<llvm::ArrayType>(Val: type)) { |
| 362 | type = arrayTy->getElementType(); |
| 363 | } else if (auto *vectorTy = dyn_cast<llvm::VectorType>(Val: type)) { |
| 364 | type = vectorTy->getElementType(); |
| 365 | } else { |
| 366 | return type; |
| 367 | } |
| 368 | } while (true); |
| 369 | } |
| 370 | |
| 371 | /// Convert a dense elements attribute to an LLVM IR constant using its raw data |
| 372 | /// storage if possible. This supports elements attributes of tensor or vector |
| 373 | /// type and avoids constructing separate objects for individual values of the |
| 374 | /// innermost dimension. Constants for other dimensions are still constructed |
| 375 | /// recursively. Returns null if constructing from raw data is not supported for |
| 376 | /// this type, e.g., element type is not a power-of-two-sized primitive. Reports |
| 377 | /// other errors at `loc`. |
| 378 | static llvm::Constant * |
| 379 | convertDenseElementsAttr(Location loc, DenseElementsAttr denseElementsAttr, |
| 380 | llvm::Type *llvmType, |
| 381 | const ModuleTranslation &moduleTranslation) { |
| 382 | if (!denseElementsAttr) |
| 383 | return nullptr; |
| 384 | |
| 385 | llvm::Type *innermostLLVMType = getInnermostElementType(type: llvmType); |
| 386 | if (!llvm::ConstantDataSequential::isElementTypeCompatible(Ty: innermostLLVMType)) |
| 387 | return nullptr; |
| 388 | |
| 389 | ShapedType type = denseElementsAttr.getType(); |
| 390 | if (type.getNumElements() == 0) |
| 391 | return nullptr; |
| 392 | |
| 393 | // Check that the raw data size matches what is expected for the scalar size. |
| 394 | // TODO: in theory, we could repack the data here to keep constructing from |
| 395 | // raw data. |
| 396 | // TODO: we may also need to consider endianness when cross-compiling to an |
| 397 | // architecture where it is different. |
| 398 | int64_t elementByteSize = denseElementsAttr.getRawData().size() / |
| 399 | denseElementsAttr.getNumElements(); |
| 400 | if (8 * elementByteSize != innermostLLVMType->getScalarSizeInBits()) |
| 401 | return nullptr; |
| 402 | |
| 403 | // Compute the shape of all dimensions but the innermost. Note that the |
| 404 | // innermost dimension may be that of the vector element type. |
| 405 | bool hasVectorElementType = isa<VectorType>(Val: type.getElementType()); |
| 406 | int64_t numAggregates = |
| 407 | denseElementsAttr.getNumElements() / |
| 408 | (hasVectorElementType ? 1 |
| 409 | : denseElementsAttr.getType().getShape().back()); |
| 410 | ArrayRef<int64_t> outerShape = type.getShape(); |
| 411 | if (!hasVectorElementType) |
| 412 | outerShape = outerShape.drop_back(); |
| 413 | |
| 414 | // Handle the case of vector splat, LLVM has special support for it. |
| 415 | if (denseElementsAttr.isSplat() && |
| 416 | (isa<VectorType>(Val: type) || hasVectorElementType)) { |
| 417 | llvm::Constant *splatValue = LLVM::detail::getLLVMConstant( |
| 418 | llvmType: innermostLLVMType, attr: denseElementsAttr.getSplatValue<Attribute>(), loc, |
| 419 | moduleTranslation); |
| 420 | llvm::Constant *splatVector = |
| 421 | llvm::ConstantDataVector::getSplat(NumElts: 0, Elt: splatValue); |
| 422 | SmallVector<llvm::Constant *> constants(numAggregates, splatVector); |
| 423 | ArrayRef<llvm::Constant *> constantsRef = constants; |
| 424 | return buildSequentialConstant(constants&: constantsRef, shape: outerShape, type: llvmType, loc); |
| 425 | } |
| 426 | if (denseElementsAttr.isSplat()) |
| 427 | return nullptr; |
| 428 | |
| 429 | // In case of non-splat, create a constructor for the innermost constant from |
| 430 | // a piece of raw data. |
| 431 | std::function<llvm::Constant *(StringRef)> buildCstData; |
| 432 | if (isa<TensorType>(Val: type)) { |
| 433 | auto vectorElementType = dyn_cast<VectorType>(Val: type.getElementType()); |
| 434 | if (vectorElementType && vectorElementType.getRank() == 1) { |
| 435 | buildCstData = [&](StringRef data) { |
| 436 | return llvm::ConstantDataVector::getRaw( |
| 437 | Data: data, NumElements: vectorElementType.getShape().back(), ElementTy: innermostLLVMType); |
| 438 | }; |
| 439 | } else if (!vectorElementType) { |
| 440 | buildCstData = [&](StringRef data) { |
| 441 | return llvm::ConstantDataArray::getRaw(Data: data, NumElements: type.getShape().back(), |
| 442 | ElementTy: innermostLLVMType); |
| 443 | }; |
| 444 | } |
| 445 | } else if (isa<VectorType>(Val: type)) { |
| 446 | buildCstData = [&](StringRef data) { |
| 447 | return llvm::ConstantDataVector::getRaw(Data: data, NumElements: type.getShape().back(), |
| 448 | ElementTy: innermostLLVMType); |
| 449 | }; |
| 450 | } |
| 451 | if (!buildCstData) |
| 452 | return nullptr; |
| 453 | |
| 454 | // Create innermost constants and defer to the default constant creation |
| 455 | // mechanism for other dimensions. |
| 456 | SmallVector<llvm::Constant *> constants; |
| 457 | int64_t aggregateSize = denseElementsAttr.getType().getShape().back() * |
| 458 | (innermostLLVMType->getScalarSizeInBits() / 8); |
| 459 | constants.reserve(N: numAggregates); |
| 460 | for (unsigned i = 0; i < numAggregates; ++i) { |
| 461 | StringRef data(denseElementsAttr.getRawData().data() + i * aggregateSize, |
| 462 | aggregateSize); |
| 463 | constants.push_back(Elt: buildCstData(data)); |
| 464 | } |
| 465 | |
| 466 | ArrayRef<llvm::Constant *> constantsRef = constants; |
| 467 | return buildSequentialConstant(constants&: constantsRef, shape: outerShape, type: llvmType, loc); |
| 468 | } |
| 469 | |
| 470 | /// Convert a dense resource elements attribute to an LLVM IR constant using its |
| 471 | /// raw data storage if possible. This supports elements attributes of tensor or |
| 472 | /// vector type and avoids constructing separate objects for individual values |
| 473 | /// of the innermost dimension. Constants for other dimensions are still |
| 474 | /// constructed recursively. Returns nullptr on failure and emits errors at |
| 475 | /// `loc`. |
| 476 | static llvm::Constant *convertDenseResourceElementsAttr( |
| 477 | Location loc, DenseResourceElementsAttr denseResourceAttr, |
| 478 | llvm::Type *llvmType, const ModuleTranslation &moduleTranslation) { |
| 479 | assert(denseResourceAttr && "expected non-null attribute" ); |
| 480 | |
| 481 | llvm::Type *innermostLLVMType = getInnermostElementType(type: llvmType); |
| 482 | if (!llvm::ConstantDataSequential::isElementTypeCompatible( |
| 483 | Ty: innermostLLVMType)) { |
| 484 | emitError(loc, message: "no known conversion for innermost element type" ); |
| 485 | return nullptr; |
| 486 | } |
| 487 | |
| 488 | ShapedType type = denseResourceAttr.getType(); |
| 489 | assert(type.getNumElements() > 0 && "Expected non-empty elements attribute" ); |
| 490 | |
| 491 | AsmResourceBlob *blob = denseResourceAttr.getRawHandle().getBlob(); |
| 492 | if (!blob) { |
| 493 | emitError(loc, message: "resource does not exist" ); |
| 494 | return nullptr; |
| 495 | } |
| 496 | |
| 497 | ArrayRef<char> rawData = blob->getData(); |
| 498 | |
| 499 | // Check that the raw data size matches what is expected for the scalar size. |
| 500 | // TODO: in theory, we could repack the data here to keep constructing from |
| 501 | // raw data. |
| 502 | // TODO: we may also need to consider endianness when cross-compiling to an |
| 503 | // architecture where it is different. |
| 504 | int64_t numElements = denseResourceAttr.getType().getNumElements(); |
| 505 | int64_t elementByteSize = rawData.size() / numElements; |
| 506 | if (8 * elementByteSize != innermostLLVMType->getScalarSizeInBits()) { |
| 507 | emitError(loc, message: "raw data size does not match element type size" ); |
| 508 | return nullptr; |
| 509 | } |
| 510 | |
| 511 | // Compute the shape of all dimensions but the innermost. Note that the |
| 512 | // innermost dimension may be that of the vector element type. |
| 513 | bool hasVectorElementType = isa<VectorType>(Val: type.getElementType()); |
| 514 | int64_t numAggregates = |
| 515 | numElements / (hasVectorElementType |
| 516 | ? 1 |
| 517 | : denseResourceAttr.getType().getShape().back()); |
| 518 | ArrayRef<int64_t> outerShape = type.getShape(); |
| 519 | if (!hasVectorElementType) |
| 520 | outerShape = outerShape.drop_back(); |
| 521 | |
| 522 | // Create a constructor for the innermost constant from a piece of raw data. |
| 523 | std::function<llvm::Constant *(StringRef)> buildCstData; |
| 524 | if (isa<TensorType>(Val: type)) { |
| 525 | auto vectorElementType = dyn_cast<VectorType>(Val: type.getElementType()); |
| 526 | if (vectorElementType && vectorElementType.getRank() == 1) { |
| 527 | buildCstData = [&](StringRef data) { |
| 528 | return llvm::ConstantDataVector::getRaw( |
| 529 | Data: data, NumElements: vectorElementType.getShape().back(), ElementTy: innermostLLVMType); |
| 530 | }; |
| 531 | } else if (!vectorElementType) { |
| 532 | buildCstData = [&](StringRef data) { |
| 533 | return llvm::ConstantDataArray::getRaw(Data: data, NumElements: type.getShape().back(), |
| 534 | ElementTy: innermostLLVMType); |
| 535 | }; |
| 536 | } |
| 537 | } else if (isa<VectorType>(Val: type)) { |
| 538 | buildCstData = [&](StringRef data) { |
| 539 | return llvm::ConstantDataVector::getRaw(Data: data, NumElements: type.getShape().back(), |
| 540 | ElementTy: innermostLLVMType); |
| 541 | }; |
| 542 | } |
| 543 | if (!buildCstData) { |
| 544 | emitError(loc, message: "unsupported dense_resource type" ); |
| 545 | return nullptr; |
| 546 | } |
| 547 | |
| 548 | // Create innermost constants and defer to the default constant creation |
| 549 | // mechanism for other dimensions. |
| 550 | SmallVector<llvm::Constant *> constants; |
| 551 | int64_t aggregateSize = denseResourceAttr.getType().getShape().back() * |
| 552 | (innermostLLVMType->getScalarSizeInBits() / 8); |
| 553 | constants.reserve(N: numAggregates); |
| 554 | for (unsigned i = 0; i < numAggregates; ++i) { |
| 555 | StringRef data(rawData.data() + i * aggregateSize, aggregateSize); |
| 556 | constants.push_back(Elt: buildCstData(data)); |
| 557 | } |
| 558 | |
| 559 | ArrayRef<llvm::Constant *> constantsRef = constants; |
| 560 | return buildSequentialConstant(constants&: constantsRef, shape: outerShape, type: llvmType, loc); |
| 561 | } |
| 562 | |
| 563 | /// Create an LLVM IR constant of `llvmType` from the MLIR attribute `attr`. |
| 564 | /// This currently supports integer, floating point, splat and dense element |
| 565 | /// attributes and combinations thereof. Also, an array attribute with two |
| 566 | /// elements is supported to represent a complex constant. In case of error, |
| 567 | /// report it to `loc` and return nullptr. |
| 568 | llvm::Constant *mlir::LLVM::detail::getLLVMConstant( |
| 569 | llvm::Type *llvmType, Attribute attr, Location loc, |
| 570 | const ModuleTranslation &moduleTranslation) { |
| 571 | if (!attr || isa<UndefAttr>(Val: attr)) |
| 572 | return llvm::UndefValue::get(T: llvmType); |
| 573 | if (isa<ZeroAttr>(Val: attr)) |
| 574 | return llvm::Constant::getNullValue(Ty: llvmType); |
| 575 | if (auto *structType = dyn_cast<::llvm::StructType>(Val: llvmType)) { |
| 576 | auto arrayAttr = dyn_cast<ArrayAttr>(Val&: attr); |
| 577 | if (!arrayAttr) { |
| 578 | emitError(loc, message: "expected an array attribute for a struct constant" ); |
| 579 | return nullptr; |
| 580 | } |
| 581 | SmallVector<llvm::Constant *> structElements; |
| 582 | structElements.reserve(N: structType->getNumElements()); |
| 583 | for (auto [elemType, elemAttr] : |
| 584 | zip_equal(t: structType->elements(), u&: arrayAttr)) { |
| 585 | llvm::Constant *element = |
| 586 | getLLVMConstant(llvmType: elemType, attr: elemAttr, loc, moduleTranslation); |
| 587 | if (!element) |
| 588 | return nullptr; |
| 589 | structElements.push_back(Elt: element); |
| 590 | } |
| 591 | return llvm::ConstantStruct::get(T: structType, V: structElements); |
| 592 | } |
| 593 | // For integer types, we allow a mismatch in sizes as the index type in |
| 594 | // MLIR might have a different size than the index type in the LLVM module. |
| 595 | if (auto intAttr = dyn_cast<IntegerAttr>(Val&: attr)) { |
| 596 | // If the attribute is an unsigned integer or a 1-bit integer, zero-extend |
| 597 | // the value to the bit width of the LLVM type. Otherwise, sign-extend. |
| 598 | auto intTy = dyn_cast<IntegerType>(Val: intAttr.getType()); |
| 599 | APInt value; |
| 600 | if (intTy && (intTy.isUnsigned() || intTy.getWidth() == 1)) |
| 601 | value = intAttr.getValue().zextOrTrunc(width: llvmType->getIntegerBitWidth()); |
| 602 | else |
| 603 | value = intAttr.getValue().sextOrTrunc(width: llvmType->getIntegerBitWidth()); |
| 604 | return llvm::ConstantInt::get(Ty: llvmType, V: value); |
| 605 | } |
| 606 | if (auto floatAttr = dyn_cast<FloatAttr>(Val&: attr)) { |
| 607 | const llvm::fltSemantics &sem = floatAttr.getValue().getSemantics(); |
| 608 | // Special case for 8-bit floats, which are represented by integers due to |
| 609 | // the lack of native fp8 types in LLVM at the moment. Additionally, handle |
| 610 | // targets (like AMDGPU) that don't implement bfloat and convert all bfloats |
| 611 | // to i16. |
| 612 | unsigned floatWidth = APFloat::getSizeInBits(Sem: sem); |
| 613 | if (llvmType->isIntegerTy(BitWidth: floatWidth)) |
| 614 | return llvm::ConstantInt::get(Ty: llvmType, |
| 615 | V: floatAttr.getValue().bitcastToAPInt()); |
| 616 | if (llvmType != |
| 617 | llvm::Type::getFloatingPointTy(C&: llvmType->getContext(), |
| 618 | S: floatAttr.getValue().getSemantics())) { |
| 619 | emitError(loc, message: "FloatAttr does not match expected type of the constant" ); |
| 620 | return nullptr; |
| 621 | } |
| 622 | return llvm::ConstantFP::get(Ty: llvmType, V: floatAttr.getValue()); |
| 623 | } |
| 624 | if (auto symAttr = dyn_cast<FlatSymbolRefAttr>(Val&: attr)) { |
| 625 | StringRef name = symAttr.getValue(); |
| 626 | if (llvm::Function *func = moduleTranslation.lookupFunction(name)) |
| 627 | return llvm::ConstantExpr::getBitCast(C: func, Ty: llvmType); |
| 628 | if (llvm::GlobalValue *global = moduleTranslation.lookupGlobal(name)) |
| 629 | return llvm::ConstantExpr::getBitCast(C: global, Ty: llvmType); |
| 630 | emitError(loc, message: "unknown symbol reference '" ) << name << "' in constant" ; |
| 631 | return nullptr; |
| 632 | } |
| 633 | if (auto splatAttr = dyn_cast<SplatElementsAttr>(Val&: attr)) { |
| 634 | llvm::Type *elementType; |
| 635 | uint64_t numElements; |
| 636 | bool isScalable = false; |
| 637 | if (auto *arrayTy = dyn_cast<llvm::ArrayType>(Val: llvmType)) { |
| 638 | elementType = arrayTy->getElementType(); |
| 639 | numElements = arrayTy->getNumElements(); |
| 640 | } else if (auto *fVectorTy = dyn_cast<llvm::FixedVectorType>(Val: llvmType)) { |
| 641 | elementType = fVectorTy->getElementType(); |
| 642 | numElements = fVectorTy->getNumElements(); |
| 643 | } else if (auto *sVectorTy = dyn_cast<llvm::ScalableVectorType>(Val: llvmType)) { |
| 644 | elementType = sVectorTy->getElementType(); |
| 645 | numElements = sVectorTy->getMinNumElements(); |
| 646 | isScalable = true; |
| 647 | } else { |
| 648 | llvm_unreachable("unrecognized constant vector type" ); |
| 649 | } |
| 650 | // Splat value is a scalar. Extract it only if the element type is not |
| 651 | // another sequence type. The recursion terminates because each step removes |
| 652 | // one outer sequential type. |
| 653 | bool elementTypeSequential = |
| 654 | isa<llvm::ArrayType, llvm::VectorType>(Val: elementType); |
| 655 | llvm::Constant *child = getLLVMConstant( |
| 656 | llvmType: elementType, |
| 657 | attr: elementTypeSequential ? splatAttr |
| 658 | : splatAttr.getSplatValue<Attribute>(), |
| 659 | loc, moduleTranslation); |
| 660 | if (!child) |
| 661 | return nullptr; |
| 662 | if (llvmType->isVectorTy()) |
| 663 | return llvm::ConstantVector::getSplat( |
| 664 | EC: llvm::ElementCount::get(MinVal: numElements, /*Scalable=*/isScalable), Elt: child); |
| 665 | if (llvmType->isArrayTy()) { |
| 666 | auto *arrayType = llvm::ArrayType::get(ElementType: elementType, NumElements: numElements); |
| 667 | if (child->isNullValue() && !elementType->isFPOrFPVectorTy()) { |
| 668 | return llvm::ConstantAggregateZero::get(Ty: arrayType); |
| 669 | } |
| 670 | if (llvm::ConstantDataSequential::isElementTypeCompatible(Ty: elementType)) { |
| 671 | if (isa<llvm::IntegerType>(Val: elementType)) { |
| 672 | if (llvm::ConstantInt *ci = dyn_cast<llvm::ConstantInt>(Val: child)) { |
| 673 | if (ci->getBitWidth() == 8) { |
| 674 | SmallVector<int8_t> constants(numElements, ci->getZExtValue()); |
| 675 | return llvm::ConstantDataArray::get(Context&: elementType->getContext(), |
| 676 | Elts&: constants); |
| 677 | } |
| 678 | if (ci->getBitWidth() == 16) { |
| 679 | SmallVector<int16_t> constants(numElements, ci->getZExtValue()); |
| 680 | return llvm::ConstantDataArray::get(Context&: elementType->getContext(), |
| 681 | Elts&: constants); |
| 682 | } |
| 683 | if (ci->getBitWidth() == 32) { |
| 684 | SmallVector<int32_t> constants(numElements, ci->getZExtValue()); |
| 685 | return llvm::ConstantDataArray::get(Context&: elementType->getContext(), |
| 686 | Elts&: constants); |
| 687 | } |
| 688 | if (ci->getBitWidth() == 64) { |
| 689 | SmallVector<int64_t> constants(numElements, ci->getZExtValue()); |
| 690 | return llvm::ConstantDataArray::get(Context&: elementType->getContext(), |
| 691 | Elts&: constants); |
| 692 | } |
| 693 | } |
| 694 | } |
| 695 | if (elementType->isFloatingPointTy()) { |
| 696 | if (llvm::ConstantFP *cfp = dyn_cast<llvm::ConstantFP>(Val: child)) { |
| 697 | APInt bitPattern = cfp->getValueAPF().bitcastToAPInt(); |
| 698 | uint64_t value = bitPattern.getZExtValue(); |
| 699 | // TODO: This code only handles 16, 32, and 64 bit floats. Handle |
| 700 | // all compatible types, fp8, fp4, etc. |
| 701 | if (bitPattern.getBitWidth() == 16) { |
| 702 | SmallVector<uint16_t> constants(numElements, value); |
| 703 | return llvm::ConstantDataArray::getFP(ElementType: elementType, Elts: constants); |
| 704 | } |
| 705 | if (bitPattern.getBitWidth() == 32) { |
| 706 | SmallVector<uint32_t> constants(numElements, value); |
| 707 | return llvm::ConstantDataArray::getFP(ElementType: elementType, Elts: constants); |
| 708 | } |
| 709 | if (bitPattern.getBitWidth() == 64) { |
| 710 | SmallVector<uint64_t> constants(numElements, value); |
| 711 | return llvm::ConstantDataArray::getFP(ElementType: elementType, Elts: constants); |
| 712 | } |
| 713 | } |
| 714 | } |
| 715 | } |
| 716 | // std::vector is used here to accomodate large number of elements that |
| 717 | // exceed SmallVector capacity. |
| 718 | std::vector<llvm::Constant *> constants(numElements, child); |
| 719 | return llvm::ConstantArray::get(T: arrayType, V: constants); |
| 720 | } |
| 721 | } |
| 722 | |
| 723 | // Try using raw elements data if possible. |
| 724 | if (llvm::Constant *result = |
| 725 | convertDenseElementsAttr(loc, denseElementsAttr: dyn_cast<DenseElementsAttr>(Val&: attr), |
| 726 | llvmType, moduleTranslation)) { |
| 727 | return result; |
| 728 | } |
| 729 | |
| 730 | if (auto denseResourceAttr = dyn_cast<DenseResourceElementsAttr>(Val&: attr)) { |
| 731 | return convertDenseResourceElementsAttr(loc, denseResourceAttr, llvmType, |
| 732 | moduleTranslation); |
| 733 | } |
| 734 | |
| 735 | // Fall back to element-by-element construction otherwise. |
| 736 | if (auto elementsAttr = dyn_cast<ElementsAttr>(Val&: attr)) { |
| 737 | assert(elementsAttr.getShapedType().hasStaticShape()); |
| 738 | assert(!elementsAttr.getShapedType().getShape().empty() && |
| 739 | "unexpected empty elements attribute shape" ); |
| 740 | |
| 741 | SmallVector<llvm::Constant *, 8> constants; |
| 742 | constants.reserve(N: elementsAttr.getNumElements()); |
| 743 | llvm::Type *innermostType = getInnermostElementType(type: llvmType); |
| 744 | for (auto n : elementsAttr.getValues<Attribute>()) { |
| 745 | constants.push_back( |
| 746 | Elt: getLLVMConstant(llvmType: innermostType, attr: n, loc, moduleTranslation)); |
| 747 | if (!constants.back()) |
| 748 | return nullptr; |
| 749 | } |
| 750 | ArrayRef<llvm::Constant *> constantsRef = constants; |
| 751 | llvm::Constant *result = buildSequentialConstant( |
| 752 | constants&: constantsRef, shape: elementsAttr.getShapedType().getShape(), type: llvmType, loc); |
| 753 | assert(constantsRef.empty() && "did not consume all elemental constants" ); |
| 754 | return result; |
| 755 | } |
| 756 | |
| 757 | if (auto stringAttr = dyn_cast<StringAttr>(Val&: attr)) { |
| 758 | return llvm::ConstantDataArray::get(Context&: moduleTranslation.getLLVMContext(), |
| 759 | Elts: ArrayRef<char>{stringAttr.getValue()}); |
| 760 | } |
| 761 | |
| 762 | // Handle arrays of structs that cannot be represented as DenseElementsAttr |
| 763 | // in MLIR. |
| 764 | if (auto arrayAttr = dyn_cast<ArrayAttr>(Val&: attr)) { |
| 765 | if (auto *arrayTy = dyn_cast<llvm::ArrayType>(Val: llvmType)) { |
| 766 | llvm::Type *elementType = arrayTy->getElementType(); |
| 767 | Attribute previousElementAttr; |
| 768 | llvm::Constant *elementCst = nullptr; |
| 769 | SmallVector<llvm::Constant *> constants; |
| 770 | constants.reserve(N: arrayTy->getNumElements()); |
| 771 | for (Attribute elementAttr : arrayAttr) { |
| 772 | // Arrays with a single value or with repeating values are quite common. |
| 773 | // Short-circuit the translation when the element value is the same as |
| 774 | // the previous one. |
| 775 | if (!previousElementAttr || previousElementAttr != elementAttr) { |
| 776 | previousElementAttr = elementAttr; |
| 777 | elementCst = |
| 778 | getLLVMConstant(llvmType: elementType, attr: elementAttr, loc, moduleTranslation); |
| 779 | if (!elementCst) |
| 780 | return nullptr; |
| 781 | } |
| 782 | constants.push_back(Elt: elementCst); |
| 783 | } |
| 784 | return llvm::ConstantArray::get(T: arrayTy, V: constants); |
| 785 | } |
| 786 | } |
| 787 | |
| 788 | emitError(loc, message: "unsupported constant value" ); |
| 789 | return nullptr; |
| 790 | } |
| 791 | |
| 792 | ModuleTranslation::ModuleTranslation(Operation *module, |
| 793 | std::unique_ptr<llvm::Module> llvmModule, |
| 794 | llvm::vfs::FileSystem *fs) |
| 795 | : mlirModule(module), llvmModule(std::move(llvmModule)), |
| 796 | debugTranslation( |
| 797 | std::make_unique<DebugTranslation>(args&: module, args&: *this->llvmModule)), |
| 798 | loopAnnotationTranslation(std::make_unique<LoopAnnotationTranslation>( |
| 799 | args&: *this, args&: *this->llvmModule)), |
| 800 | fileSystem(fs), typeTranslator(this->llvmModule->getContext()), |
| 801 | iface(module->getContext()) { |
| 802 | assert(satisfiesLLVMModule(mlirModule) && |
| 803 | "mlirModule should honor LLVM's module semantics." ); |
| 804 | } |
| 805 | |
| 806 | ModuleTranslation::~ModuleTranslation() { |
| 807 | // Destroy extensions while the rest of the translation state is still alive. |
| 808 | extensions.clear(); |
| 809 | } |
| 810 | |
| 811 | ModuleTranslation::Extension::~Extension() = default; |
| 812 | |
| 813 | void ModuleTranslation::forgetMapping(Region ®ion) { |
| 814 | SmallVector<Region *> toProcess; |
| 815 | toProcess.push_back(Elt: ®ion); |
| 816 | while (!toProcess.empty()) { |
| 817 | Region *current = toProcess.pop_back_val(); |
| 818 | for (Block &block : *current) { |
| 819 | blockMapping.erase(Val: &block); |
| 820 | for (Value arg : block.getArguments()) |
| 821 | valueMapping.erase(Val: arg); |
| 822 | for (Operation &op : block) { |
| 823 | for (Value value : op.getResults()) |
| 824 | valueMapping.erase(Val: value); |
| 825 | if (op.hasSuccessors()) |
| 826 | branchMapping.erase(Val: &op); |
| 827 | if (isa<LLVM::GlobalOp>(Val: op)) |
| 828 | globalsMapping.erase(Val: &op); |
| 829 | if (isa<LLVM::AliasOp>(Val: op)) |
| 830 | aliasesMapping.erase(Val: &op); |
| 831 | if (isa<LLVM::IFuncOp>(Val: op)) |
| 832 | ifuncMapping.erase(Val: &op); |
| 833 | if (isa<LLVM::CallOp>(Val: op)) |
| 834 | callMapping.erase(Val: &op); |
| 835 | llvm::append_range( |
| 836 | C&: toProcess, |
| 837 | R: llvm::map_range(C: op.getRegions(), F: [](Region &r) { return &r; })); |
| 838 | } |
| 839 | } |
| 840 | } |
| 841 | } |
| 842 | |
| 843 | /// Get the SSA value passed to the current block from the terminator operation |
| 844 | /// of its predecessor. |
| 845 | static Value getPHISourceValue(Block *current, Block *pred, |
| 846 | unsigned numArguments, unsigned index) { |
| 847 | Operation &terminator = *pred->getTerminator(); |
| 848 | if (isa<LLVM::BrOp>(Val: terminator)) |
| 849 | return terminator.getOperand(idx: index); |
| 850 | |
| 851 | #ifndef NDEBUG |
| 852 | llvm::SmallPtrSet<Block *, 4> seenSuccessors; |
| 853 | for (unsigned i = 0, e = terminator.getNumSuccessors(); i < e; ++i) { |
| 854 | Block *successor = terminator.getSuccessor(i); |
| 855 | auto branch = cast<BranchOpInterface>(terminator); |
| 856 | SuccessorOperands successorOperands = branch.getSuccessorOperands(i); |
| 857 | assert( |
| 858 | (!seenSuccessors.contains(successor) || successorOperands.empty()) && |
| 859 | "successors with arguments in LLVM branches must be different blocks" ); |
| 860 | seenSuccessors.insert(successor); |
| 861 | } |
| 862 | #endif |
| 863 | |
| 864 | // For instructions that branch based on a condition value, we need to take |
| 865 | // the operands for the branch that was taken. |
| 866 | if (auto condBranchOp = dyn_cast<LLVM::CondBrOp>(Val&: terminator)) { |
| 867 | // For conditional branches, we take the operands from either the "true" or |
| 868 | // the "false" branch. |
| 869 | return condBranchOp.getSuccessor(i: 0) == current |
| 870 | ? condBranchOp.getTrueDestOperands()[index] |
| 871 | : condBranchOp.getFalseDestOperands()[index]; |
| 872 | } |
| 873 | |
| 874 | if (auto switchOp = dyn_cast<LLVM::SwitchOp>(Val&: terminator)) { |
| 875 | // For switches, we take the operands from either the default case, or from |
| 876 | // the case branch that was taken. |
| 877 | if (switchOp.getDefaultDestination() == current) |
| 878 | return switchOp.getDefaultOperands()[index]; |
| 879 | for (const auto &i : llvm::enumerate(First: switchOp.getCaseDestinations())) |
| 880 | if (i.value() == current) |
| 881 | return switchOp.getCaseOperands(index: i.index())[index]; |
| 882 | } |
| 883 | |
| 884 | if (auto indBrOp = dyn_cast<LLVM::IndirectBrOp>(Val&: terminator)) { |
| 885 | // For indirect branches we take operands for each successor. |
| 886 | for (const auto &i : llvm::enumerate(First: indBrOp->getSuccessors())) { |
| 887 | if (indBrOp->getSuccessor(index: i.index()) == current) |
| 888 | return indBrOp.getSuccessorOperands(index: i.index())[index]; |
| 889 | } |
| 890 | } |
| 891 | |
| 892 | if (auto invokeOp = dyn_cast<LLVM::InvokeOp>(Val&: terminator)) { |
| 893 | return invokeOp.getNormalDest() == current |
| 894 | ? invokeOp.getNormalDestOperands()[index] |
| 895 | : invokeOp.getUnwindDestOperands()[index]; |
| 896 | } |
| 897 | |
| 898 | llvm_unreachable( |
| 899 | "only branch, switch or invoke operations can be terminators " |
| 900 | "of a block that has successors" ); |
| 901 | } |
| 902 | |
| 903 | /// Connect the PHI nodes to the results of preceding blocks. |
| 904 | void mlir::LLVM::detail::connectPHINodes(Region ®ion, |
| 905 | const ModuleTranslation &state) { |
| 906 | // Skip the first block, it cannot be branched to and its arguments correspond |
| 907 | // to the arguments of the LLVM function. |
| 908 | for (Block &bb : llvm::drop_begin(RangeOrContainer&: region)) { |
| 909 | llvm::BasicBlock *llvmBB = state.lookupBlock(block: &bb); |
| 910 | auto phis = llvmBB->phis(); |
| 911 | unsigned numArguments = bb.getNumArguments(); |
| 912 | assert(numArguments == llvm::range_size(phis)); |
| 913 | for (auto [index, phiNode] : llvm::enumerate(First&: phis)) { |
| 914 | for (auto *pred : bb.getPredecessors()) { |
| 915 | // Find the LLVM IR block that contains the converted terminator |
| 916 | // instruction and use it in the PHI node. Note that this block is not |
| 917 | // necessarily the same as state.lookupBlock(pred), some operations |
| 918 | // (in particular, OpenMP operations using OpenMPIRBuilder) may have |
| 919 | // split the blocks. |
| 920 | llvm::Instruction *terminator = |
| 921 | state.lookupBranch(op: pred->getTerminator()); |
| 922 | assert(terminator && "missing the mapping for a terminator" ); |
| 923 | phiNode.addIncoming(V: state.lookupValue(value: getPHISourceValue( |
| 924 | current: &bb, pred, numArguments, index)), |
| 925 | BB: terminator->getParent()); |
| 926 | } |
| 927 | } |
| 928 | } |
| 929 | } |
| 930 | |
| 931 | llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall( |
| 932 | llvm::IRBuilderBase &builder, llvm::Intrinsic::ID intrinsic, |
| 933 | ArrayRef<llvm::Value *> args, ArrayRef<llvm::Type *> tys) { |
| 934 | return builder.CreateIntrinsicWithoutFolding(ID: intrinsic, OverloadTypes: tys, Args: args); |
| 935 | } |
| 936 | |
| 937 | llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall( |
| 938 | llvm::IRBuilderBase &builder, llvm::Intrinsic::ID intrinsic, |
| 939 | llvm::Type *retTy, ArrayRef<llvm::Value *> args) { |
| 940 | return builder.CreateIntrinsicWithoutFolding(RetTy: retTy, ID: intrinsic, Args: args); |
| 941 | } |
| 942 | |
| 943 | llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall( |
| 944 | llvm::IRBuilderBase &builder, ModuleTranslation &moduleTranslation, |
| 945 | Operation *intrOp, llvm::Intrinsic::ID intrinsic, unsigned numResults, |
| 946 | ArrayRef<unsigned> overloadedResults, ArrayRef<unsigned> overloadedOperands, |
| 947 | ArrayRef<unsigned> immArgPositions, |
| 948 | ArrayRef<StringLiteral> immArgAttrNames) { |
| 949 | assert(immArgPositions.size() == immArgAttrNames.size() && |
| 950 | "LLVM `immArgPositions` and MLIR `immArgAttrNames` should have equal " |
| 951 | "length" ); |
| 952 | |
| 953 | SmallVector<llvm::OperandBundleDef> opBundles; |
| 954 | size_t numOpBundleOperands = 0; |
| 955 | auto opBundleSizesAttr = cast_if_present<DenseI32ArrayAttr>( |
| 956 | Val: intrOp->getInherentAttr(name: LLVMDialect::getOpBundleSizesAttrName()) |
| 957 | .value_or(u: Attribute{})); |
| 958 | auto opBundleTagsAttr = cast_if_present<ArrayAttr>( |
| 959 | Val: intrOp->getInherentAttr(name: LLVMDialect::getOpBundleTagsAttrName()) |
| 960 | .value_or(u: Attribute{})); |
| 961 | |
| 962 | if (opBundleSizesAttr && opBundleTagsAttr) { |
| 963 | ArrayRef<int> opBundleSizes = opBundleSizesAttr.asArrayRef(); |
| 964 | assert(opBundleSizes.size() == opBundleTagsAttr.size() && |
| 965 | "operand bundles and tags do not match" ); |
| 966 | |
| 967 | numOpBundleOperands = llvm::sum_of(Range&: opBundleSizes); |
| 968 | assert(numOpBundleOperands <= intrOp->getNumOperands() && |
| 969 | "operand bundle operands is more than the number of operands" ); |
| 970 | |
| 971 | ValueRange operands = intrOp->getOperands().take_back(n: numOpBundleOperands); |
| 972 | size_t nextOperandIdx = 0; |
| 973 | opBundles.reserve(N: opBundleSizesAttr.size()); |
| 974 | |
| 975 | for (auto [opBundleTagAttr, bundleSize] : |
| 976 | llvm::zip(t&: opBundleTagsAttr, u&: opBundleSizes)) { |
| 977 | auto bundleTag = cast<StringAttr>(Val: opBundleTagAttr).str(); |
| 978 | auto bundleOperands = moduleTranslation.lookupValues( |
| 979 | values: operands.slice(n: nextOperandIdx, m: bundleSize)); |
| 980 | opBundles.emplace_back(Args: std::move(bundleTag), Args: std::move(bundleOperands)); |
| 981 | nextOperandIdx += bundleSize; |
| 982 | } |
| 983 | } |
| 984 | |
| 985 | // Map operands and attributes to LLVM values. |
| 986 | auto opOperands = intrOp->getOperands().drop_back(n: numOpBundleOperands); |
| 987 | auto operands = moduleTranslation.lookupValues(values: opOperands); |
| 988 | SmallVector<llvm::Value *> args(immArgPositions.size() + operands.size()); |
| 989 | for (auto [immArgPos, immArgName] : |
| 990 | llvm::zip(t&: immArgPositions, u&: immArgAttrNames)) { |
| 991 | Attribute attr = intrOp->getInherentAttr(name: immArgName).value_or(u: Attribute{}); |
| 992 | if (auto intrinsicIntegerAttr = |
| 993 | dyn_cast<LLVM::IntrinsicIntegerAttrInterface>(Val&: attr)) |
| 994 | attr = intrinsicIntegerAttr.getIntegerAttr(); |
| 995 | auto typedAttr = llvm::cast<TypedAttr>(Val&: attr); |
| 996 | assert(typedAttr.getType().isIntOrFloat() && |
| 997 | "expected int or float immarg" ); |
| 998 | auto *type = moduleTranslation.convertType(type: typedAttr.getType()); |
| 999 | args[immArgPos] = LLVM::detail::getLLVMConstant( |
| 1000 | llvmType: type, attr: typedAttr, loc: intrOp->getLoc(), moduleTranslation); |
| 1001 | } |
| 1002 | unsigned opArg = 0; |
| 1003 | for (auto &arg : args) { |
| 1004 | if (!arg) |
| 1005 | arg = operands[opArg++]; |
| 1006 | } |
| 1007 | |
| 1008 | // Resolve overloaded intrinsic declaration. |
| 1009 | SmallVector<llvm::Type *> overloadedTypes; |
| 1010 | for (unsigned overloadedResultIdx : overloadedResults) { |
| 1011 | if (numResults > 1) { |
| 1012 | // More than one result is mapped to an LLVM struct. |
| 1013 | overloadedTypes.push_back(Elt: moduleTranslation.convertType( |
| 1014 | type: llvm::cast<LLVM::LLVMStructType>(Val: intrOp->getResult(idx: 0).getType()) |
| 1015 | .getBody()[overloadedResultIdx])); |
| 1016 | } else { |
| 1017 | overloadedTypes.push_back( |
| 1018 | Elt: moduleTranslation.convertType(type: intrOp->getResult(idx: 0).getType())); |
| 1019 | } |
| 1020 | } |
| 1021 | for (unsigned overloadedOperandIdx : overloadedOperands) |
| 1022 | overloadedTypes.push_back(Elt: args[overloadedOperandIdx]->getType()); |
| 1023 | llvm::Module *module = builder.getModule(); |
| 1024 | llvm::Function *llvmIntr = llvm::Intrinsic::getOrInsertDeclaration( |
| 1025 | M: module, id: intrinsic, OverloadTys: overloadedTypes); |
| 1026 | |
| 1027 | return builder.CreateCall(Callee: llvmIntr, Args: args, OpBundles: opBundles); |
| 1028 | } |
| 1029 | |
| 1030 | /// Given a single MLIR operation, create the corresponding LLVM IR operation |
| 1031 | /// using the `builder`. |
| 1032 | LogicalResult ModuleTranslation::convertOperationImpl( |
| 1033 | Operation &op, llvm::IRBuilderBase &builder, bool recordInsertions) { |
| 1034 | const LLVMTranslationDialectInterface *opIface = iface.getInterfaceFor(obj: &op); |
| 1035 | if (!opIface) |
| 1036 | return op.emitError(message: "cannot be converted to LLVM IR: missing " |
| 1037 | "`LLVMTranslationDialectInterface` registration for " |
| 1038 | "dialect for op: " ) |
| 1039 | << op.getName(); |
| 1040 | |
| 1041 | InstructionCapturingInserter::CollectionScope scope(builder, |
| 1042 | recordInsertions); |
| 1043 | if (failed(Result: opIface->convertOperation(op: &op, builder, moduleTranslation&: *this))) |
| 1044 | return op.emitError(message: "LLVM Translation failed for operation: " ) |
| 1045 | << op.getName(); |
| 1046 | |
| 1047 | return convertDialectAttributes(op: &op, instructions: scope.getCapturedInstructions()); |
| 1048 | } |
| 1049 | |
| 1050 | /// Convert block to LLVM IR. Unless `ignoreArguments` is set, emit PHI nodes |
| 1051 | /// to define values corresponding to the MLIR block arguments. These nodes |
| 1052 | /// are not connected to the source basic blocks, which may not exist yet. Uses |
| 1053 | /// `builder` to construct the LLVM IR. Expects the LLVM IR basic block to have |
| 1054 | /// been created for `bb` and included in the block mapping. Inserts new |
| 1055 | /// instructions at the end of the block and leaves `builder` in a state |
| 1056 | /// suitable for further insertion into the end of the block. |
| 1057 | LogicalResult ModuleTranslation::convertBlockImpl(Block &bb, |
| 1058 | bool ignoreArguments, |
| 1059 | llvm::IRBuilderBase &builder, |
| 1060 | bool recordInsertions) { |
| 1061 | builder.SetInsertPoint(lookupBlock(block: &bb)); |
| 1062 | auto *subprogram = builder.GetInsertBlock()->getParent()->getSubprogram(); |
| 1063 | |
| 1064 | // Before traversing operations, make block arguments available through |
| 1065 | // value remapping and PHI nodes, but do not add incoming edges for the PHI |
| 1066 | // nodes just yet: those values may be defined by this or following blocks. |
| 1067 | // This step is omitted if "ignoreArguments" is set. The arguments of the |
| 1068 | // first block have been already made available through the remapping of |
| 1069 | // LLVM function arguments. |
| 1070 | if (!ignoreArguments) { |
| 1071 | auto predecessors = bb.getPredecessors(); |
| 1072 | unsigned numPredecessors = |
| 1073 | std::distance(first: predecessors.begin(), last: predecessors.end()); |
| 1074 | for (auto arg : bb.getArguments()) { |
| 1075 | auto wrappedType = arg.getType(); |
| 1076 | if (!isCompatibleType(type: wrappedType)) |
| 1077 | return emitError(loc: bb.front().getLoc(), |
| 1078 | message: "block argument does not have an LLVM type" ); |
| 1079 | builder.SetCurrentDebugLocation( |
| 1080 | debugTranslation->translateLoc(loc: arg.getLoc(), scope: subprogram)); |
| 1081 | llvm::Type *type = convertType(type: wrappedType); |
| 1082 | llvm::PHINode *phi = builder.CreatePHI(Ty: type, NumReservedValues: numPredecessors); |
| 1083 | mapValue(mlir: arg, llvm: phi); |
| 1084 | } |
| 1085 | } |
| 1086 | |
| 1087 | // Traverse operations. |
| 1088 | for (auto &op : bb) { |
| 1089 | // Set the current debug location within the builder. |
| 1090 | builder.SetCurrentDebugLocation( |
| 1091 | debugTranslation->translateLoc(loc: op.getLoc(), scope: subprogram)); |
| 1092 | |
| 1093 | if (failed(Result: convertOperationImpl(op, builder, recordInsertions))) |
| 1094 | return failure(); |
| 1095 | |
| 1096 | // Set the branch weight metadata on the translated instruction. |
| 1097 | if (auto iface = dyn_cast<WeightedBranchOpInterface>(Val&: op)) |
| 1098 | setBranchWeightsMetadata(iface); |
| 1099 | } |
| 1100 | |
| 1101 | return success(); |
| 1102 | } |
| 1103 | |
| 1104 | /// A helper method to get the single Block in an operation honoring LLVM's |
| 1105 | /// module requirements. |
| 1106 | static Block &getModuleBody(Operation *module) { |
| 1107 | return module->getRegion(index: 0).front(); |
| 1108 | } |
| 1109 | |
| 1110 | /// A helper method to decide if a constant must not be set as a global variable |
| 1111 | /// initializer. For an external linkage variable, the variable with an |
| 1112 | /// initializer is considered externally visible and defined in this module, the |
| 1113 | /// variable without an initializer is externally available and is defined |
| 1114 | /// elsewhere. |
| 1115 | static bool shouldDropGlobalInitializer(llvm::GlobalValue::LinkageTypes linkage, |
| 1116 | llvm::Constant *cst) { |
| 1117 | return (linkage == llvm::GlobalVariable::ExternalLinkage && !cst) || |
| 1118 | linkage == llvm::GlobalVariable::ExternalWeakLinkage; |
| 1119 | } |
| 1120 | |
| 1121 | /// Sets the runtime preemption specifier of `gv` to dso_local if |
| 1122 | /// `dsoLocalRequested` is true, otherwise it is left unchanged. |
| 1123 | static void addRuntimePreemptionSpecifier(bool dsoLocalRequested, |
| 1124 | llvm::GlobalValue *gv) { |
| 1125 | if (dsoLocalRequested) |
| 1126 | gv->setDSOLocal(true); |
| 1127 | } |
| 1128 | |
| 1129 | /// Attempts to translate an MLIR attribute identified by `key`, optionally with |
| 1130 | /// the given `value`, into an LLVM IR attribute. Reports errors at `loc` if |
| 1131 | /// any. If the attribute name corresponds to a known LLVM IR attribute kind, |
| 1132 | /// creates the LLVM attribute of that kind; otherwise, keeps it as a string |
| 1133 | /// attribute. Performs additional checks for attributes known to have or not |
| 1134 | /// have a value in order to avoid assertions inside LLVM upon construction. |
| 1135 | static FailureOr<llvm::Attribute> |
| 1136 | convertMLIRAttributeToLLVM(Location loc, llvm::LLVMContext &ctx, StringRef key, |
| 1137 | StringRef value = StringRef()) { |
| 1138 | auto kind = llvm::Attribute::getAttrKindFromName(AttrName: key); |
| 1139 | if (kind == llvm::Attribute::None) |
| 1140 | return llvm::Attribute::get(Context&: ctx, Kind: key, Val: value); |
| 1141 | |
| 1142 | if (llvm::Attribute::isIntAttrKind(Kind: kind)) { |
| 1143 | if (value.empty()) |
| 1144 | return emitError(loc) << "LLVM attribute '" << key << "' expects a value" ; |
| 1145 | |
| 1146 | int64_t result; |
| 1147 | if (!value.getAsInteger(/*Radix=*/0, Result&: result)) |
| 1148 | return llvm::Attribute::get(Context&: ctx, Kind: kind, Val: result); |
| 1149 | return llvm::Attribute::get(Context&: ctx, Kind: key, Val: value); |
| 1150 | } |
| 1151 | |
| 1152 | if (!value.empty()) |
| 1153 | return emitError(loc) << "LLVM attribute '" << key |
| 1154 | << "' does not expect a value, found '" << value |
| 1155 | << "'" ; |
| 1156 | |
| 1157 | return llvm::Attribute::get(Context&: ctx, Kind: kind); |
| 1158 | } |
| 1159 | |
| 1160 | /// Converts the MLIR attributes listed in the given array attribute into LLVM |
| 1161 | /// attributes. Returns an `AttrBuilder` containing the converted attributes. |
| 1162 | /// Reports error to `loc` if any and returns immediately. Expects `arrayAttr` |
| 1163 | /// to contain either string attributes, treated as value-less LLVM attributes, |
| 1164 | /// or array attributes containing two string attributes, with the first string |
| 1165 | /// being the name of the corresponding LLVM attribute and the second string |
| 1166 | /// beings its value. Note that even integer attributes are expected to have |
| 1167 | /// their values expressed as strings. |
| 1168 | static FailureOr<llvm::AttrBuilder> |
| 1169 | convertMLIRAttributesToLLVM(Location loc, llvm::LLVMContext &ctx, |
| 1170 | ArrayAttr arrayAttr, StringRef arrayAttrName) { |
| 1171 | llvm::AttrBuilder attrBuilder(ctx); |
| 1172 | if (!arrayAttr) |
| 1173 | return attrBuilder; |
| 1174 | |
| 1175 | for (Attribute attr : arrayAttr) { |
| 1176 | if (auto stringAttr = dyn_cast<StringAttr>(Val&: attr)) { |
| 1177 | FailureOr<llvm::Attribute> llvmAttr = |
| 1178 | convertMLIRAttributeToLLVM(loc, ctx, key: stringAttr.getValue()); |
| 1179 | if (failed(Result: llvmAttr)) |
| 1180 | return failure(); |
| 1181 | attrBuilder.addAttribute(A: *llvmAttr); |
| 1182 | continue; |
| 1183 | } |
| 1184 | |
| 1185 | auto arrayAttr = dyn_cast<ArrayAttr>(Val&: attr); |
| 1186 | if (!arrayAttr || arrayAttr.size() != 2) |
| 1187 | return emitError(loc) << "expected '" << arrayAttrName |
| 1188 | << "' to contain string or array attributes" ; |
| 1189 | |
| 1190 | auto keyAttr = dyn_cast<StringAttr>(Val: arrayAttr[0]); |
| 1191 | auto valueAttr = dyn_cast<StringAttr>(Val: arrayAttr[1]); |
| 1192 | if (!keyAttr || !valueAttr) |
| 1193 | return emitError(loc) << "expected arrays within '" << arrayAttrName |
| 1194 | << "' to contain two strings" ; |
| 1195 | |
| 1196 | FailureOr<llvm::Attribute> llvmAttr = convertMLIRAttributeToLLVM( |
| 1197 | loc, ctx, key: keyAttr.getValue(), value: valueAttr.getValue()); |
| 1198 | if (failed(Result: llvmAttr)) |
| 1199 | return failure(); |
| 1200 | attrBuilder.addAttribute(A: *llvmAttr); |
| 1201 | } |
| 1202 | |
| 1203 | return attrBuilder; |
| 1204 | } |
| 1205 | |
| 1206 | LogicalResult ModuleTranslation::convertGlobalsAndAliases() { |
| 1207 | // Mapping from compile unit to its respective set of global variables. |
| 1208 | DenseMap<llvm::DICompileUnit *, SmallVector<llvm::Metadata *>> globalGVars; |
| 1209 | // Mapping from subprogram to its respective set of static local variables. |
| 1210 | DenseMap<llvm::DISubprogram *, SmallVector<llvm::Metadata *>> staticLocals; |
| 1211 | |
| 1212 | // First, create all global variables and global aliases in LLVM IR. A global |
| 1213 | // or alias body may refer to another global/alias or itself, so all the |
| 1214 | // mapping needs to happen prior to body conversion. |
| 1215 | |
| 1216 | // Create all llvm::GlobalVariable |
| 1217 | for (auto op : getModuleBody(module: mlirModule).getOps<LLVM::GlobalOp>()) { |
| 1218 | llvm::Type *type = convertType(type: op.getType()); |
| 1219 | llvm::Constant *cst = nullptr; |
| 1220 | const bool deferValueAttrToPass2 = op.getValueOrNull() && |
| 1221 | !op.getInitializerBlock() && |
| 1222 | !isa<StringAttr>(Val: op.getValueOrNull()); |
| 1223 | if (op.getValueOrNull() && !deferValueAttrToPass2) { |
| 1224 | // String attributes are treated separately because they cannot appear as |
| 1225 | // in-function constants and are thus not supported by getLLVMConstant. |
| 1226 | if (auto strAttr = dyn_cast_or_null<StringAttr>(Val: op.getValueOrNull())) { |
| 1227 | cst = llvm::ConstantDataArray::getString( |
| 1228 | Context&: llvmModule->getContext(), Initializer: strAttr.getValue(), /*AddNull=*/false); |
| 1229 | type = cst->getType(); |
| 1230 | } |
| 1231 | } |
| 1232 | |
| 1233 | auto linkage = convertLinkageToLLVM(value: op.getLinkage()); |
| 1234 | |
| 1235 | // LLVM IR requires constant with linkage other than external or weak |
| 1236 | // external to have initializers. If MLIR does not provide an initializer, |
| 1237 | // default to undef. |
| 1238 | bool dropInitializer = shouldDropGlobalInitializer(linkage, cst); |
| 1239 | if (!deferValueAttrToPass2) { |
| 1240 | if (!dropInitializer && !cst) |
| 1241 | cst = llvm::UndefValue::get(T: type); |
| 1242 | else if (dropInitializer && cst) |
| 1243 | cst = nullptr; |
| 1244 | } else { |
| 1245 | cst = nullptr; |
| 1246 | } |
| 1247 | |
| 1248 | auto *var = new llvm::GlobalVariable( |
| 1249 | *llvmModule, type, op.getConstant(), linkage, cst, op.getSymName(), |
| 1250 | /*InsertBefore=*/nullptr, convertThreadLocalModeToLLVM(value: op.getTlsMode()), |
| 1251 | op.getAddrSpace(), op.getExternallyInitialized()); |
| 1252 | |
| 1253 | if (std::optional<mlir::SymbolRefAttr> comdat = op.getComdat()) { |
| 1254 | auto selectorOp = cast<ComdatSelectorOp>( |
| 1255 | Val: symbolTable().lookupNearestSymbolFrom(from: op, symbol: *comdat)); |
| 1256 | var->setComdat(comdatMapping.lookup(Val: selectorOp)); |
| 1257 | } |
| 1258 | |
| 1259 | if (op.getUnnamedAddr().has_value()) |
| 1260 | var->setUnnamedAddr(convertUnnamedAddrToLLVM(value: *op.getUnnamedAddr())); |
| 1261 | |
| 1262 | if (op.getSection().has_value()) |
| 1263 | var->setSection(*op.getSection()); |
| 1264 | |
| 1265 | addRuntimePreemptionSpecifier(dsoLocalRequested: op.getDsoLocal(), gv: var); |
| 1266 | |
| 1267 | std::optional<uint64_t> alignment = op.getAlignment(); |
| 1268 | if (alignment.has_value()) |
| 1269 | var->setAlignment(llvm::MaybeAlign(alignment.value())); |
| 1270 | |
| 1271 | var->setVisibility(convertVisibilityToLLVM(value: op.getVisibility_())); |
| 1272 | |
| 1273 | globalsMapping.try_emplace(Key: op, Args&: var); |
| 1274 | globalsByNameMapping.try_emplace(Key: op.getSymName(), Args&: var); |
| 1275 | |
| 1276 | // Add debug information if present. |
| 1277 | if (op.getDbgExprs()) { |
| 1278 | for (auto exprAttr : |
| 1279 | op.getDbgExprs()->getAsRange<DIGlobalVariableExpressionAttr>()) { |
| 1280 | llvm::DIGlobalVariableExpression *diGlobalExpr = |
| 1281 | debugTranslation->translateGlobalVariableExpression(attr: exprAttr); |
| 1282 | llvm::DIGlobalVariable *diGlobalVar = diGlobalExpr->getVariable(); |
| 1283 | var->addDebugInfo(GV: diGlobalExpr); |
| 1284 | |
| 1285 | // There is no `globals` field in DICompileUnitAttr which can be |
| 1286 | // directly assigned to DICompileUnit. We have to build the list by |
| 1287 | // looking at the dbgExpr of all the GlobalOps. The scope of the |
| 1288 | // variable is used to get the DICompileUnit in which to add it. But |
| 1289 | // there are cases where the scope of a global does not directly point |
| 1290 | // to the DICompileUnit and we have to do a bit more work to get to |
| 1291 | // it. Some of those cases are: |
| 1292 | // |
| 1293 | // 1. For the languages that support modules, the scope hierarchy can |
| 1294 | // be variable -> DIModule -> DICompileUnit |
| 1295 | // |
| 1296 | // 2. For the Fortran common block variable, the scope hierarchy can |
| 1297 | // be variable -> DICommonBlock -> DISubprogram -> DICompileUnit |
| 1298 | // |
| 1299 | // 3. For entities like static local variables in C or variable with |
| 1300 | // SAVE attribute in Fortran, the scope hierarchy can be |
| 1301 | // variable (-> DILocalScope)* -> DISubprogram |
| 1302 | llvm::DIScope *scope = diGlobalVar->getScope(); |
| 1303 | if (auto *mod = dyn_cast_if_present<llvm::DIModule>(Val: scope)) |
| 1304 | scope = mod->getScope(); |
| 1305 | else if (auto *cb = dyn_cast_if_present<llvm::DICommonBlock>(Val: scope)) { |
| 1306 | if (auto *sp = |
| 1307 | dyn_cast_if_present<llvm::DISubprogram>(Val: cb->getScope())) |
| 1308 | scope = sp->getUnit(); |
| 1309 | } else if (auto *lbb = |
| 1310 | dyn_cast_if_present<llvm::DILexicalBlockBase>(Val: scope)) { |
| 1311 | scope = lbb->getSubprogram(); |
| 1312 | } |
| 1313 | |
| 1314 | // Get the compile unit (scope) of the the global variable, or the |
| 1315 | // subprogram of the static local variable. |
| 1316 | if (llvm::DICompileUnit *compileUnit = |
| 1317 | dyn_cast_if_present<llvm::DICompileUnit>(Val: scope)) { |
| 1318 | // Update the compile unit with this incoming global variable |
| 1319 | // expression during the finalizing step later. |
| 1320 | globalGVars[compileUnit].push_back(Elt: diGlobalExpr); |
| 1321 | } else if (llvm::DISubprogram *sp = |
| 1322 | dyn_cast_if_present<llvm::DISubprogram>(Val: scope)) { |
| 1323 | // Update the subprogram with this incoming static local variable |
| 1324 | // expression during the finalizing step later. |
| 1325 | staticLocals[sp].push_back(Elt: diGlobalExpr); |
| 1326 | } |
| 1327 | } |
| 1328 | } |
| 1329 | |
| 1330 | // Forward the target-specific attributes to LLVM. |
| 1331 | FailureOr<llvm::AttrBuilder> convertedTargetSpecificAttrs = |
| 1332 | convertMLIRAttributesToLLVM(loc: op.getLoc(), ctx&: var->getContext(), |
| 1333 | arrayAttr: op.getTargetSpecificAttrsAttr(), |
| 1334 | arrayAttrName: op.getTargetSpecificAttrsAttrName()); |
| 1335 | if (failed(Result: convertedTargetSpecificAttrs)) |
| 1336 | return failure(); |
| 1337 | var->addAttributes(AttrBuilder: *convertedTargetSpecificAttrs); |
| 1338 | } |
| 1339 | |
| 1340 | // Value-attribute initializers may reference other globals by symbol name. |
| 1341 | // Register every global above before materializing those constants. |
| 1342 | for (auto op : getModuleBody(module: mlirModule).getOps<LLVM::GlobalOp>()) { |
| 1343 | if (!op.getValueOrNull() || op.getInitializerBlock() || |
| 1344 | isa<StringAttr>(Val: op.getValueOrNull())) |
| 1345 | continue; |
| 1346 | |
| 1347 | llvm::Type *type = convertType(type: op.getType()); |
| 1348 | llvm::Constant *cst = |
| 1349 | getLLVMConstant(llvmType: type, attr: op.getValueOrNull(), loc: op.getLoc(), moduleTranslation: *this); |
| 1350 | if (!cst) |
| 1351 | return failure(); |
| 1352 | |
| 1353 | auto linkage = convertLinkageToLLVM(value: op.getLinkage()); |
| 1354 | bool dropInitializer = shouldDropGlobalInitializer(linkage, cst); |
| 1355 | auto *var = cast<llvm::GlobalVariable>(Val: lookupGlobal(op)); |
| 1356 | if (dropInitializer) |
| 1357 | var->setInitializer(nullptr); |
| 1358 | else |
| 1359 | var->setInitializer(cst); |
| 1360 | } |
| 1361 | |
| 1362 | // Create all llvm::GlobalAlias |
| 1363 | for (auto op : getModuleBody(module: mlirModule).getOps<LLVM::AliasOp>()) { |
| 1364 | llvm::Type *type = convertType(type: op.getType()); |
| 1365 | llvm::Constant *cst = nullptr; |
| 1366 | llvm::GlobalValue::LinkageTypes linkage = |
| 1367 | convertLinkageToLLVM(value: op.getLinkage()); |
| 1368 | llvm::Module &llvmMod = *llvmModule; |
| 1369 | |
| 1370 | // Note address space and aliasee info isn't set just yet. |
| 1371 | llvm::GlobalAlias *var = llvm::GlobalAlias::create( |
| 1372 | Ty: type, AddressSpace: op.getAddrSpace(), Linkage: linkage, Name: op.getSymName(), /*placeholder*/ Aliasee: cst, |
| 1373 | Parent: &llvmMod); |
| 1374 | |
| 1375 | var->setThreadLocalMode(convertThreadLocalModeToLLVM(value: op.getTlsMode())); |
| 1376 | |
| 1377 | // Note there is no need to setup the comdat because GlobalAlias calls into |
| 1378 | // the aliasee comdat information automatically. |
| 1379 | |
| 1380 | if (op.getUnnamedAddr().has_value()) |
| 1381 | var->setUnnamedAddr(convertUnnamedAddrToLLVM(value: *op.getUnnamedAddr())); |
| 1382 | |
| 1383 | var->setVisibility(convertVisibilityToLLVM(value: op.getVisibility_())); |
| 1384 | addRuntimePreemptionSpecifier(dsoLocalRequested: op.getDsoLocal(), gv: var); |
| 1385 | |
| 1386 | aliasesMapping.try_emplace(Key: op, Args&: var); |
| 1387 | } |
| 1388 | |
| 1389 | // Convert global variable bodies. |
| 1390 | for (auto op : getModuleBody(module: mlirModule).getOps<LLVM::GlobalOp>()) { |
| 1391 | if (Block *initializer = op.getInitializerBlock()) { |
| 1392 | llvm::IRBuilder<llvm::TargetFolder> builder( |
| 1393 | *llvmModule, llvm::TargetFolder(llvmModule->getDataLayout())); |
| 1394 | |
| 1395 | [[maybe_unused]] int numConstantsHit = 0; |
| 1396 | [[maybe_unused]] int numConstantsErased = 0; |
| 1397 | DenseMap<llvm::ConstantAggregate *, int> constantAggregateUseMap; |
| 1398 | |
| 1399 | for (auto &op : initializer->without_terminator()) { |
| 1400 | if (failed(Result: convertOperation(op, builder))) |
| 1401 | return emitError(loc: op.getLoc(), message: "fail to convert global initializer" ); |
| 1402 | auto *cst = dyn_cast<llvm::Constant>(Val: lookupValue(value: op.getResult(idx: 0))); |
| 1403 | if (!cst) |
| 1404 | return emitError(loc: op.getLoc(), message: "unemittable constant value" ); |
| 1405 | |
| 1406 | // When emitting an LLVM constant, a new constant is created and the old |
| 1407 | // constant may become dangling and take space. We should remove the |
| 1408 | // dangling constants to avoid memory explosion especially for constant |
| 1409 | // arrays whose number of elements is large. |
| 1410 | // Because multiple operations may refer to the same constant, we need |
| 1411 | // to count the number of uses of each constant array and remove it only |
| 1412 | // when the count becomes zero. |
| 1413 | if (auto *agg = dyn_cast<llvm::ConstantAggregate>(Val: cst)) { |
| 1414 | numConstantsHit++; |
| 1415 | Value result = op.getResult(idx: 0); |
| 1416 | int numUsers = std::distance(first: result.use_begin(), last: result.use_end()); |
| 1417 | auto [iterator, inserted] = |
| 1418 | constantAggregateUseMap.try_emplace(Key: agg, Args&: numUsers); |
| 1419 | if (!inserted) { |
| 1420 | // Key already exists, update the value |
| 1421 | iterator->second += numUsers; |
| 1422 | } |
| 1423 | } |
| 1424 | // Scan the operands of the operation to decrement the use count of |
| 1425 | // constants. Erase the constant if the use count becomes zero. |
| 1426 | for (Value v : op.getOperands()) { |
| 1427 | auto *cst = dyn_cast<llvm::ConstantAggregate>(Val: lookupValue(value: v)); |
| 1428 | if (!cst) |
| 1429 | continue; |
| 1430 | auto iter = constantAggregateUseMap.find(Val: cst); |
| 1431 | assert(iter != constantAggregateUseMap.end() && "constant not found" ); |
| 1432 | iter->second--; |
| 1433 | if (iter->second == 0) { |
| 1434 | // NOTE: cannot call removeDeadConstantUsers() here because it |
| 1435 | // may remove the constant which has uses not be converted yet. |
| 1436 | if (cst->user_empty()) { |
| 1437 | cst->destroyConstant(); |
| 1438 | numConstantsErased++; |
| 1439 | } |
| 1440 | constantAggregateUseMap.erase(I: iter); |
| 1441 | } |
| 1442 | } |
| 1443 | } |
| 1444 | |
| 1445 | ReturnOp ret = cast<ReturnOp>(Val: initializer->getTerminator()); |
| 1446 | llvm::Constant *cst = |
| 1447 | cast<llvm::Constant>(Val: lookupValue(value: ret.getOperand(i: 0))); |
| 1448 | auto *global = cast<llvm::GlobalVariable>(Val: lookupGlobal(op)); |
| 1449 | if (!shouldDropGlobalInitializer(linkage: global->getLinkage(), cst)) |
| 1450 | global->setInitializer(cst); |
| 1451 | |
| 1452 | // Try to remove the dangling constants again after all operations are |
| 1453 | // converted. |
| 1454 | for (auto it : constantAggregateUseMap) { |
| 1455 | auto *cst = it.first; |
| 1456 | cst->removeDeadConstantUsers(); |
| 1457 | if (cst->user_empty()) { |
| 1458 | cst->destroyConstant(); |
| 1459 | numConstantsErased++; |
| 1460 | } |
| 1461 | } |
| 1462 | |
| 1463 | LLVM_DEBUG(llvm::dbgs() |
| 1464 | << "Convert initializer for " << op.getName() << "\n" ; |
| 1465 | llvm::dbgs() << numConstantsHit << " new constants hit\n" ; |
| 1466 | llvm::dbgs() |
| 1467 | << numConstantsErased << " dangling constants erased\n" ;); |
| 1468 | } |
| 1469 | } |
| 1470 | |
| 1471 | // Convert llvm.mlir.global_ctors and dtors. |
| 1472 | for (Operation &op : getModuleBody(module: mlirModule)) { |
| 1473 | auto ctorOp = dyn_cast<GlobalCtorsOp>(Val&: op); |
| 1474 | auto dtorOp = dyn_cast<GlobalDtorsOp>(Val&: op); |
| 1475 | if (!ctorOp && !dtorOp) |
| 1476 | continue; |
| 1477 | |
| 1478 | // The empty / zero initialized version of llvm.global_(c|d)tors cannot be |
| 1479 | // handled by appendGlobalFn logic below, which just ignores empty (c|d)tor |
| 1480 | // lists. Make sure it gets emitted. |
| 1481 | if ((ctorOp && ctorOp.getCtors().empty()) || |
| 1482 | (dtorOp && dtorOp.getDtors().empty())) { |
| 1483 | llvm::IRBuilder<llvm::TargetFolder> builder( |
| 1484 | *llvmModule, llvm::TargetFolder(llvmModule->getDataLayout())); |
| 1485 | llvm::Type *eltTy = llvm::StructType::get( |
| 1486 | elt1: builder.getInt32Ty(), elts: builder.getPtrTy(), elts: builder.getPtrTy()); |
| 1487 | llvm::ArrayType *at = llvm::ArrayType::get(ElementType: eltTy, NumElements: 0); |
| 1488 | llvm::Constant *zeroInit = llvm::Constant::getNullValue(Ty: at); |
| 1489 | (void)new llvm::GlobalVariable( |
| 1490 | *llvmModule, zeroInit->getType(), false, |
| 1491 | llvm::GlobalValue::AppendingLinkage, zeroInit, |
| 1492 | ctorOp ? "llvm.global_ctors" : "llvm.global_dtors" ); |
| 1493 | } else { |
| 1494 | auto range = ctorOp |
| 1495 | ? llvm::zip(t: ctorOp.getCtors(), u: ctorOp.getPriorities()) |
| 1496 | : llvm::zip(t: dtorOp.getDtors(), u: dtorOp.getPriorities()); |
| 1497 | auto appendGlobalFn = |
| 1498 | ctorOp ? llvm::appendToGlobalCtors : llvm::appendToGlobalDtors; |
| 1499 | for (const auto &[sym, prio] : range) { |
| 1500 | llvm::Function *f = |
| 1501 | lookupFunction(name: cast<FlatSymbolRefAttr>(Val: sym).getValue()); |
| 1502 | appendGlobalFn(*llvmModule, f, cast<IntegerAttr>(Val: prio).getInt(), |
| 1503 | /*Data=*/nullptr); |
| 1504 | } |
| 1505 | } |
| 1506 | } |
| 1507 | |
| 1508 | for (auto op : getModuleBody(module: mlirModule).getOps<LLVM::GlobalOp>()) |
| 1509 | if (failed(Result: convertDialectAttributes(op, instructions: {}))) |
| 1510 | return failure(); |
| 1511 | |
| 1512 | // Finally, update the compile units their respective sets of global variables |
| 1513 | // created earlier. |
| 1514 | for (const auto &[compileUnit, globals] : globalGVars) |
| 1515 | compileUnit->replaceGlobalVariables( |
| 1516 | N: llvm::MDTuple::get(Context&: getLLVMContext(), MDs: globals)); |
| 1517 | |
| 1518 | // And update the subprograms with their respective sets of static local |
| 1519 | // variables. |
| 1520 | for (const auto &[sp, globals] : staticLocals) |
| 1521 | sp->retainNodes(NodesBegin: globals.begin(), NodesEnd: globals.end()); |
| 1522 | |
| 1523 | // Convert global alias bodies. |
| 1524 | for (auto op : getModuleBody(module: mlirModule).getOps<LLVM::AliasOp>()) { |
| 1525 | Block &initializer = op.getInitializerBlock(); |
| 1526 | llvm::IRBuilder<llvm::TargetFolder> builder( |
| 1527 | *llvmModule, llvm::TargetFolder(llvmModule->getDataLayout())); |
| 1528 | |
| 1529 | for (mlir::Operation &op : initializer.without_terminator()) { |
| 1530 | if (failed(Result: convertOperation(op, builder))) |
| 1531 | return emitError(loc: op.getLoc(), message: "fail to convert alias initializer" ); |
| 1532 | if (!isa<llvm::Constant>(Val: lookupValue(value: op.getResult(idx: 0)))) |
| 1533 | return emitError(loc: op.getLoc(), message: "unemittable constant value" ); |
| 1534 | } |
| 1535 | |
| 1536 | auto ret = cast<ReturnOp>(Val: initializer.getTerminator()); |
| 1537 | auto *cst = cast<llvm::Constant>(Val: lookupValue(value: ret.getOperand(i: 0))); |
| 1538 | assert(aliasesMapping.count(op)); |
| 1539 | auto *alias = cast<llvm::GlobalAlias>(Val: aliasesMapping[op]); |
| 1540 | alias->setAliasee(cst); |
| 1541 | } |
| 1542 | |
| 1543 | for (auto op : getModuleBody(module: mlirModule).getOps<LLVM::AliasOp>()) |
| 1544 | if (failed(Result: convertDialectAttributes(op, instructions: {}))) |
| 1545 | return failure(); |
| 1546 | |
| 1547 | return success(); |
| 1548 | } |
| 1549 | |
| 1550 | /// Return a representation of `value` as metadata. |
| 1551 | static llvm::Metadata *convertIntegerToMetadata(llvm::LLVMContext &context, |
| 1552 | const llvm::APInt &value) { |
| 1553 | llvm::Constant *constant = llvm::ConstantInt::get(Context&: context, V: value); |
| 1554 | return llvm::ConstantAsMetadata::get(C: constant); |
| 1555 | } |
| 1556 | |
| 1557 | /// Return a representation of `value` as an MDNode. |
| 1558 | static llvm::MDNode *convertIntegerToMDNode(llvm::LLVMContext &context, |
| 1559 | const llvm::APInt &value) { |
| 1560 | return llvm::MDNode::get(Context&: context, MDs: convertIntegerToMetadata(context, value)); |
| 1561 | } |
| 1562 | |
| 1563 | /// Return an MDNode encoding `vec_type_hint` metadata. |
| 1564 | static llvm::MDNode *convertVecTypeHintToMDNode(llvm::LLVMContext &context, |
| 1565 | llvm::Type *type, |
| 1566 | bool isSigned) { |
| 1567 | llvm::Metadata *typeMD = |
| 1568 | llvm::ConstantAsMetadata::get(C: llvm::UndefValue::get(T: type)); |
| 1569 | llvm::Metadata *isSignedMD = |
| 1570 | convertIntegerToMetadata(context, value: llvm::APInt(32, isSigned ? 1 : 0)); |
| 1571 | return llvm::MDNode::get(Context&: context, MDs: {typeMD, isSignedMD}); |
| 1572 | } |
| 1573 | |
| 1574 | /// Return an MDNode with a tuple given by the values in `values`. |
| 1575 | static llvm::MDNode *convertIntegerArrayToMDNode(llvm::LLVMContext &context, |
| 1576 | ArrayRef<int32_t> values) { |
| 1577 | SmallVector<llvm::Metadata *> mdValues; |
| 1578 | llvm::transform(Range&: values, d_first: std::back_inserter(x&: mdValues), |
| 1579 | F: [&context](int32_t value) { |
| 1580 | return convertIntegerToMetadata( |
| 1581 | context, value: llvm::APInt(32, value, /*isSigned=*/true)); |
| 1582 | }); |
| 1583 | return llvm::MDNode::get(Context&: context, MDs: mdValues); |
| 1584 | } |
| 1585 | |
| 1586 | FailureOr<llvm::Metadata *> ModuleTranslation::convertMetadataAttr( |
| 1587 | Attribute attr, function_ref<InFlightDiagnostic()> emitError) { |
| 1588 | llvm::LLVMContext &llvmContext = getLLVMContext(); |
| 1589 | |
| 1590 | return llvm::TypeSwitch<Attribute, FailureOr<llvm::Metadata *>>(attr) |
| 1591 | .Case(caseFn: [&](MDStringAttr a) -> FailureOr<llvm::Metadata *> { |
| 1592 | return llvm::MDString::get(Context&: llvmContext, Str: a.getValue().getValue()); |
| 1593 | }) |
| 1594 | .Case(caseFn: [&](MDConstantAttr a) -> FailureOr<llvm::Metadata *> { |
| 1595 | IntegerAttr intAttr = llvm::dyn_cast<IntegerAttr>(Val: a.getValue()); |
| 1596 | if (!intAttr) { |
| 1597 | return emitError() |
| 1598 | << "expected integer attribute in metadata constant" ; |
| 1599 | } |
| 1600 | return llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get( |
| 1601 | Ty: llvm::Type::getIntNTy(C&: llvmContext, |
| 1602 | N: intAttr.getType().getIntOrFloatBitWidth()), |
| 1603 | V: intAttr.getValue())); |
| 1604 | }) |
| 1605 | .Case(caseFn: [&](MDGlobalValueAttr a) -> FailureOr<llvm::Metadata *> { |
| 1606 | return convertSymbolRefToMetadata(name: a.getName(), emitError); |
| 1607 | }) |
| 1608 | .Case(caseFn: [&](MDNullAttr a) -> FailureOr<llvm::Metadata *> { |
| 1609 | return llvm::ConstantAsMetadata::get(C: llvm::ConstantPointerNull::get( |
| 1610 | T: llvm::PointerType::get(C&: llvmContext, AddressSpace: a.getAddressSpace()))); |
| 1611 | }) |
| 1612 | .Case(caseFn: [&](MDAddrSpaceCastAttr a) -> FailureOr<llvm::Metadata *> { |
| 1613 | FailureOr<llvm::Metadata *> arg = |
| 1614 | convertMetadataAttr(attr: a.getArg(), emitError); |
| 1615 | if (failed(Result: arg)) |
| 1616 | return failure(); |
| 1617 | // The verifier restricts the operand to pointer-valued metadata |
| 1618 | // attributes, all of which translate to a ConstantAsMetadata. |
| 1619 | auto *argAsMD = cast<llvm::ConstantAsMetadata>(Val: *arg); |
| 1620 | return llvm::ConstantAsMetadata::get( |
| 1621 | C: llvm::ConstantExpr::getAddrSpaceCast( |
| 1622 | C: argAsMD->getValue(), |
| 1623 | Ty: llvm::PointerType::get(C&: llvmContext, AddressSpace: a.getAddressSpace()))); |
| 1624 | }) |
| 1625 | .Case(caseFn: [&](MDNodeAttr a) -> FailureOr<llvm::Metadata *> { |
| 1626 | SmallVector<llvm::Metadata *> operands; |
| 1627 | for (Attribute operand : a.getOperands()) { |
| 1628 | FailureOr<llvm::Metadata *> md = |
| 1629 | convertMetadataAttr(attr: operand, emitError); |
| 1630 | if (failed(Result: md)) |
| 1631 | return failure(); |
| 1632 | operands.push_back(Elt: *md); |
| 1633 | } |
| 1634 | return llvm::MDNode::get(Context&: llvmContext, MDs: operands); |
| 1635 | }) |
| 1636 | .Default(defaultFn: [&](Attribute attr) -> FailureOr<llvm::Metadata *> { |
| 1637 | return emitError() << "unsupported LLVM metadata attribute " << attr; |
| 1638 | }); |
| 1639 | } |
| 1640 | |
| 1641 | FailureOr<llvm::Metadata *> ModuleTranslation::convertSymbolRefToMetadata( |
| 1642 | FlatSymbolRefAttr name, function_ref<InFlightDiagnostic()> emitError) { |
| 1643 | if (llvm::Function *fn = lookupFunction(name: name.getValue())) |
| 1644 | return llvm::ValueAsMetadata::get(V: fn); |
| 1645 | if (llvm::GlobalValue *global = lookupGlobal(name: name.getValue())) |
| 1646 | return llvm::ValueAsMetadata::get(V: global); |
| 1647 | Operation *symbol = symbolTable().lookupSymbolIn(symbolTableOp: mlirModule, name); |
| 1648 | if (auto alias = dyn_cast_if_present<LLVM::AliasOp>(Val: symbol)) { |
| 1649 | if (llvm::GlobalValue *global = lookupAlias(op: alias)) |
| 1650 | return llvm::ValueAsMetadata::get(V: global); |
| 1651 | } |
| 1652 | if (auto ifunc = dyn_cast_if_present<LLVM::IFuncOp>(Val: symbol)) { |
| 1653 | if (llvm::GlobalValue *global = lookupIFunc(op: ifunc)) |
| 1654 | return llvm::ValueAsMetadata::get(V: global); |
| 1655 | } |
| 1656 | return emitError() << "could not resolve metadata reference '" << name << "'" ; |
| 1657 | } |
| 1658 | |
| 1659 | LogicalResult ModuleTranslation::convertFunctionMetadata() { |
| 1660 | for (auto function : getModuleBody(module: mlirModule).getOps<LLVMFuncOp>()) { |
| 1661 | ArrayAttr metadata = function.getFunctionMetadataAttr(); |
| 1662 | if (!metadata) |
| 1663 | continue; |
| 1664 | |
| 1665 | llvm::Function *llvmFunc = lookupFunction(name: function.getName()); |
| 1666 | for (auto entry : metadata.getAsRange<LLVM::FunctionMetadataAttr>()) { |
| 1667 | StringRef metadataName = entry.getMetadataName().getValue(); |
| 1668 | |
| 1669 | FailureOr<llvm::Metadata *> md = |
| 1670 | convertMetadataAttr(attr: entry.getNode(), emitError: [&]() { |
| 1671 | return function.emitError() |
| 1672 | << "failed to convert function_metadata entry '" |
| 1673 | << metadataName << "': " ; |
| 1674 | }); |
| 1675 | if (failed(Result: md)) |
| 1676 | return failure(); |
| 1677 | llvm::MDNode *node = llvm::dyn_cast_if_present<llvm::MDNode>(Val: *md); |
| 1678 | if (!node) { |
| 1679 | return function.emitError() |
| 1680 | << "failed to convert function_metadata entry '" << metadataName |
| 1681 | << "'" ; |
| 1682 | } |
| 1683 | llvmFunc->addMetadata(Kind: metadataName, MD&: *node); |
| 1684 | } |
| 1685 | } |
| 1686 | return success(); |
| 1687 | } |
| 1688 | |
| 1689 | LogicalResult ModuleTranslation::convertOneFunction(LLVMFuncOp func) { |
| 1690 | // Clear the block, branch value mappings, they are only relevant within one |
| 1691 | // function. |
| 1692 | blockMapping.clear(); |
| 1693 | valueMapping.clear(); |
| 1694 | branchMapping.clear(); |
| 1695 | llvm::Function *llvmFunc = lookupFunction(name: func.getName()); |
| 1696 | llvm::LLVMContext &llvmContext = llvmFunc->getContext(); |
| 1697 | |
| 1698 | // Add function arguments to the value remapping table. |
| 1699 | for (auto [mlirArg, llvmArg] : |
| 1700 | llvm::zip(t: func.getArguments(), u: llvmFunc->args())) |
| 1701 | mapValue(mlir: mlirArg, llvm: &llvmArg); |
| 1702 | |
| 1703 | // Check the personality and set it. |
| 1704 | if (func.getPersonality()) { |
| 1705 | llvm::Type *ty = llvm::PointerType::get( |
| 1706 | C&: llvmContext, AddressSpace: llvmModule->getDataLayout().getProgramAddressSpace()); |
| 1707 | if (llvm::Constant *pfunc = getLLVMConstant(llvmType: ty, attr: func.getPersonalityAttr(), |
| 1708 | loc: func.getLoc(), moduleTranslation: *this)) |
| 1709 | llvmFunc->setPersonalityFn(pfunc); |
| 1710 | } |
| 1711 | |
| 1712 | if (std::optional<StringRef> section = func.getSection()) |
| 1713 | llvmFunc->setSection(*section); |
| 1714 | |
| 1715 | if (func.getArmStreaming()) |
| 1716 | llvmFunc->addFnAttr(Kind: "aarch64_pstate_sm_enabled" ); |
| 1717 | else if (func.getArmLocallyStreaming()) |
| 1718 | llvmFunc->addFnAttr(Kind: "aarch64_pstate_sm_body" ); |
| 1719 | else if (func.getArmStreamingCompatible()) |
| 1720 | llvmFunc->addFnAttr(Kind: "aarch64_pstate_sm_compatible" ); |
| 1721 | |
| 1722 | if (func.getArmNewZa()) |
| 1723 | llvmFunc->addFnAttr(Kind: "aarch64_new_za" ); |
| 1724 | else if (func.getArmInZa()) |
| 1725 | llvmFunc->addFnAttr(Kind: "aarch64_in_za" ); |
| 1726 | else if (func.getArmOutZa()) |
| 1727 | llvmFunc->addFnAttr(Kind: "aarch64_out_za" ); |
| 1728 | else if (func.getArmInoutZa()) |
| 1729 | llvmFunc->addFnAttr(Kind: "aarch64_inout_za" ); |
| 1730 | else if (func.getArmPreservesZa()) |
| 1731 | llvmFunc->addFnAttr(Kind: "aarch64_preserves_za" ); |
| 1732 | |
| 1733 | if (auto targetCpu = func.getTargetCpu()) |
| 1734 | llvmFunc->addFnAttr(Kind: "target-cpu" , Val: *targetCpu); |
| 1735 | |
| 1736 | if (auto tuneCpu = func.getTuneCpu()) |
| 1737 | llvmFunc->addFnAttr(Kind: "tune-cpu" , Val: *tuneCpu); |
| 1738 | |
| 1739 | if (auto reciprocalEstimates = func.getReciprocalEstimates()) |
| 1740 | llvmFunc->addFnAttr(Kind: "reciprocal-estimates" , Val: *reciprocalEstimates); |
| 1741 | |
| 1742 | if (auto preferVectorWidth = func.getPreferVectorWidth()) |
| 1743 | llvmFunc->addFnAttr(Kind: "prefer-vector-width" , Val: *preferVectorWidth); |
| 1744 | |
| 1745 | if (func.getUseSampleProfile()) |
| 1746 | llvmFunc->addFnAttr(Kind: "use-sample-profile" ); |
| 1747 | |
| 1748 | if (auto disableTailCalls = func.getDisableTailCalls()) |
| 1749 | llvmFunc->addFnAttr(Kind: "disable-tail-calls" , |
| 1750 | Val: llvm::toStringRef(B: *disableTailCalls)); |
| 1751 | |
| 1752 | if (auto sampleProfileSuffixElisionPolicy = |
| 1753 | func.getSampleProfileSuffixElisionPolicy()) |
| 1754 | llvmFunc->addFnAttr(Kind: "sample-profile-suffix-elision-policy" , |
| 1755 | Val: *sampleProfileSuffixElisionPolicy); |
| 1756 | |
| 1757 | if (auto attr = func.getVscaleRange()) |
| 1758 | llvmFunc->addFnAttr(Attr: llvm::Attribute::getWithVScaleRangeArgs( |
| 1759 | Context&: getLLVMContext(), MinValue: attr->getMinRange().getInt(), |
| 1760 | MaxValue: attr->getMaxRange().getInt())); |
| 1761 | |
| 1762 | if (auto noSignedZerosFpMath = func.getNoSignedZerosFpMath()) |
| 1763 | llvmFunc->addFnAttr(Kind: "no-signed-zeros-fp-math" , |
| 1764 | Val: llvm::toStringRef(B: *noSignedZerosFpMath)); |
| 1765 | |
| 1766 | if (auto fpContract = func.getFpContract()) |
| 1767 | llvmFunc->addFnAttr(Kind: "fp-contract" , Val: *fpContract); |
| 1768 | |
| 1769 | if (auto instrumentFunctionEntry = func.getInstrumentFunctionEntry()) |
| 1770 | llvmFunc->addFnAttr(Kind: "instrument-function-entry" , Val: *instrumentFunctionEntry); |
| 1771 | |
| 1772 | if (auto instrumentFunctionExit = func.getInstrumentFunctionExit()) |
| 1773 | llvmFunc->addFnAttr(Kind: "instrument-function-exit" , Val: *instrumentFunctionExit); |
| 1774 | |
| 1775 | // First, create all blocks so we can jump to them. |
| 1776 | for (auto &bb : func) { |
| 1777 | auto *llvmBB = llvm::BasicBlock::Create(Context&: llvmContext); |
| 1778 | llvmBB->insertInto(Parent: llvmFunc); |
| 1779 | mapBlock(mlir: &bb, llvm: llvmBB); |
| 1780 | } |
| 1781 | |
| 1782 | // Then, convert blocks one by one in topological order to ensure defs are |
| 1783 | // converted before uses. |
| 1784 | auto blocks = getBlocksSortedByDominance(region&: func.getBody()); |
| 1785 | for (Block *bb : blocks) { |
| 1786 | CapturingIRBuilder builder(*llvmModule, |
| 1787 | llvm::TargetFolder(llvmModule->getDataLayout())); |
| 1788 | builder.disableGEPCanonicalization(); |
| 1789 | if (failed(Result: convertBlockImpl(bb&: *bb, ignoreArguments: bb->isEntryBlock(), builder, |
| 1790 | /*recordInsertions=*/true))) |
| 1791 | return failure(); |
| 1792 | } |
| 1793 | |
| 1794 | // After all blocks have been traversed and values mapped, connect the PHI |
| 1795 | // nodes to the results of preceding blocks. |
| 1796 | detail::connectPHINodes(region&: func.getBody(), state: *this); |
| 1797 | |
| 1798 | // Finally, convert dialect attributes attached to the function. |
| 1799 | return convertDialectAttributes(op: func, instructions: {}); |
| 1800 | } |
| 1801 | |
| 1802 | LogicalResult ModuleTranslation::convertDialectAttributes( |
| 1803 | Operation *op, ArrayRef<llvm::Instruction *> instructions) { |
| 1804 | for (NamedAttribute attribute : op->getDialectAttrs()) |
| 1805 | if (failed(Result: iface.amendOperation(op, instructions, attribute, moduleTranslation&: *this))) |
| 1806 | return failure(); |
| 1807 | return success(); |
| 1808 | } |
| 1809 | |
| 1810 | /// Converts memory effect attributes from `func` and attaches them to |
| 1811 | /// `llvmFunc`. |
| 1812 | static void convertFunctionMemoryAttributes(LLVMFuncOp func, |
| 1813 | llvm::Function *llvmFunc) { |
| 1814 | if (!func.getMemoryEffects()) |
| 1815 | return; |
| 1816 | |
| 1817 | MemoryEffectsAttr memEffects = func.getMemoryEffectsAttr(); |
| 1818 | |
| 1819 | // Add memory effects incrementally. |
| 1820 | llvm::MemoryEffects newMemEffects = |
| 1821 | llvm::MemoryEffects(llvm::MemoryEffects::Location::ArgMem, |
| 1822 | convertModRefInfoToLLVM(value: memEffects.getArgMem())); |
| 1823 | newMemEffects |= llvm::MemoryEffects( |
| 1824 | llvm::MemoryEffects::Location::InaccessibleMem, |
| 1825 | convertModRefInfoToLLVM(value: memEffects.getInaccessibleMem())); |
| 1826 | newMemEffects |= |
| 1827 | llvm::MemoryEffects(llvm::MemoryEffects::Location::Other, |
| 1828 | convertModRefInfoToLLVM(value: memEffects.getOther())); |
| 1829 | newMemEffects |= |
| 1830 | llvm::MemoryEffects(llvm::MemoryEffects::Location::ErrnoMem, |
| 1831 | convertModRefInfoToLLVM(value: memEffects.getErrnoMem())); |
| 1832 | newMemEffects |= |
| 1833 | llvm::MemoryEffects(llvm::MemoryEffects::Location::TargetMem0, |
| 1834 | convertModRefInfoToLLVM(value: memEffects.getTargetMem0())); |
| 1835 | newMemEffects |= |
| 1836 | llvm::MemoryEffects(llvm::MemoryEffects::Location::TargetMem1, |
| 1837 | convertModRefInfoToLLVM(value: memEffects.getTargetMem1())); |
| 1838 | llvmFunc->setMemoryEffects(newMemEffects); |
| 1839 | } |
| 1840 | |
| 1841 | llvm::Attribute |
| 1842 | ModuleTranslation::convertAllocsizeAttr(DenseI32ArrayAttr allocSizeAttr) { |
| 1843 | if (!allocSizeAttr || allocSizeAttr.empty()) |
| 1844 | return llvm::Attribute{}; |
| 1845 | |
| 1846 | unsigned elemSize = static_cast<unsigned>(allocSizeAttr[0]); |
| 1847 | std::optional<unsigned> numElems; |
| 1848 | if (allocSizeAttr.size() > 1) |
| 1849 | numElems = static_cast<unsigned>(allocSizeAttr[1]); |
| 1850 | |
| 1851 | return llvm::Attribute::getWithAllocSizeArgs(Context&: getLLVMContext(), ElemSizeArg: elemSize, |
| 1852 | NumElemsArg: numElems); |
| 1853 | } |
| 1854 | static void convertDenormalFPEnvAttribute(LLVMFuncOp func, |
| 1855 | llvm::AttrBuilder &Attrs) { |
| 1856 | std::optional<DenormalFPEnvAttr> denormalFpEnv = func.getDenormalFpenv(); |
| 1857 | if (!denormalFpEnv) |
| 1858 | return; |
| 1859 | |
| 1860 | llvm::DenormalMode DefaultMode( |
| 1861 | convertDenormalModeKindToLLVM(value: denormalFpEnv->getDefaultOutputMode()), |
| 1862 | convertDenormalModeKindToLLVM(value: denormalFpEnv->getDefaultInputMode())); |
| 1863 | llvm::DenormalMode FloatMode( |
| 1864 | convertDenormalModeKindToLLVM(value: denormalFpEnv->getFloatOutputMode()), |
| 1865 | convertDenormalModeKindToLLVM(value: denormalFpEnv->getFloatInputMode())); |
| 1866 | |
| 1867 | llvm::DenormalFPEnv FPEnv(DefaultMode, FloatMode); |
| 1868 | Attrs.addDenormalFPEnvAttr(Mode: FPEnv); |
| 1869 | } |
| 1870 | |
| 1871 | /// Converts function attributes from `func` and attaches them to `llvmFunc`. |
| 1872 | static void convertFunctionAttributes(ModuleTranslation &mod, LLVMFuncOp func, |
| 1873 | llvm::Function *llvmFunc) { |
| 1874 | // FIXME: Use AttrBuilder far all cases |
| 1875 | llvm::AttrBuilder AttrBuilder(llvmFunc->getContext()); |
| 1876 | |
| 1877 | if (func.getNoInlineAttr()) |
| 1878 | llvmFunc->addFnAttr(Kind: llvm::Attribute::NoInline); |
| 1879 | if (func.getAlwaysInlineAttr()) |
| 1880 | llvmFunc->addFnAttr(Kind: llvm::Attribute::AlwaysInline); |
| 1881 | if (func.getInlineHintAttr()) |
| 1882 | llvmFunc->addFnAttr(Kind: llvm::Attribute::InlineHint); |
| 1883 | if (func.getOptimizeNoneAttr()) |
| 1884 | llvmFunc->addFnAttr(Kind: llvm::Attribute::OptimizeNone); |
| 1885 | if (func.getReturnsTwiceAttr()) |
| 1886 | llvmFunc->addFnAttr(Kind: llvm::Attribute::ReturnsTwice); |
| 1887 | if (func.getColdAttr()) |
| 1888 | llvmFunc->addFnAttr(Kind: llvm::Attribute::Cold); |
| 1889 | if (func.getHotAttr()) |
| 1890 | llvmFunc->addFnAttr(Kind: llvm::Attribute::Hot); |
| 1891 | if (func.getNoduplicateAttr()) |
| 1892 | llvmFunc->addFnAttr(Kind: llvm::Attribute::NoDuplicate); |
| 1893 | if (func.getConvergentAttr()) |
| 1894 | llvmFunc->addFnAttr(Kind: llvm::Attribute::Convergent); |
| 1895 | if (func.getNoUnwindAttr()) |
| 1896 | llvmFunc->addFnAttr(Kind: llvm::Attribute::NoUnwind); |
| 1897 | if (func.getWillReturnAttr()) |
| 1898 | llvmFunc->addFnAttr(Kind: llvm::Attribute::WillReturn); |
| 1899 | if (func.getNoreturnAttr()) |
| 1900 | llvmFunc->addFnAttr(Kind: llvm::Attribute::NoReturn); |
| 1901 | if (func.getOptsizeAttr()) |
| 1902 | llvmFunc->addFnAttr(Kind: llvm::Attribute::OptimizeForSize); |
| 1903 | if (func.getMinsizeAttr()) |
| 1904 | llvmFunc->addFnAttr(Kind: llvm::Attribute::MinSize); |
| 1905 | if (func.getSaveRegParamsAttr()) |
| 1906 | llvmFunc->addFnAttr(Kind: "save-reg-params" ); |
| 1907 | if (func.getNoCallerSavedRegistersAttr()) |
| 1908 | llvmFunc->addFnAttr(Kind: "no_caller_saved_registers" ); |
| 1909 | if (func.getNocallbackAttr()) |
| 1910 | llvmFunc->addFnAttr(Kind: llvm::Attribute::NoCallback); |
| 1911 | if (StringAttr modFormat = func.getModularFormatAttr()) |
| 1912 | llvmFunc->addFnAttr(Kind: "modular-format" , Val: modFormat.getValue()); |
| 1913 | if (TargetFeaturesAttr targetFeatAttr = func.getTargetFeaturesAttr()) |
| 1914 | llvmFunc->addFnAttr(Kind: "target-features" , Val: targetFeatAttr.getFeaturesString()); |
| 1915 | if (FramePointerKindAttr fpAttr = func.getFramePointerAttr()) |
| 1916 | llvmFunc->addFnAttr(Kind: "frame-pointer" , Val: stringifyFramePointerKind( |
| 1917 | fpAttr.getFramePointerKind())); |
| 1918 | if (UWTableKindAttr uwTableKindAttr = func.getUwtableKindAttr()) |
| 1919 | llvmFunc->setUWTableKind( |
| 1920 | convertUWTableKindToLLVM(value: uwTableKindAttr.getUwtableKind())); |
| 1921 | if (StringAttr zcsr = func.getZeroCallUsedRegsAttr()) |
| 1922 | llvmFunc->addFnAttr(Kind: "zero-call-used-regs" , Val: zcsr.getValue()); |
| 1923 | if (func.getUniformWorkGroupSizeAttr()) |
| 1924 | llvmFunc->addFnAttr(Kind: "uniform-work-group-size" ); |
| 1925 | |
| 1926 | if (ArrayAttr noBuiltins = func.getNobuiltinsAttr()) { |
| 1927 | if (noBuiltins.empty()) |
| 1928 | llvmFunc->addFnAttr(Kind: "no-builtins" ); |
| 1929 | |
| 1930 | mod.convertFunctionAttrCollection(attrs: noBuiltins, op: llvmFunc, |
| 1931 | conv&: ModuleTranslation::convertNoBuiltin); |
| 1932 | } |
| 1933 | |
| 1934 | mod.convertFunctionAttrCollection(attrs: func.getDefaultFuncAttrsAttr(), op: llvmFunc, |
| 1935 | conv&: ModuleTranslation::convertDefaultFuncAttr); |
| 1936 | |
| 1937 | if (llvm::Attribute attr = mod.convertAllocsizeAttr(allocSizeAttr: func.getAllocsizeAttr()); |
| 1938 | attr.isValid()) |
| 1939 | llvmFunc->addFnAttr(Attr: attr); |
| 1940 | |
| 1941 | convertFunctionMemoryAttributes(func, llvmFunc); |
| 1942 | |
| 1943 | convertDenormalFPEnvAttribute(func, Attrs&: AttrBuilder); |
| 1944 | llvmFunc->addFnAttrs(Attrs: AttrBuilder); |
| 1945 | } |
| 1946 | |
| 1947 | /// Converts function attributes from `func` and attaches them to `llvmFunc`. |
| 1948 | static void convertFunctionKernelAttributes(LLVMFuncOp func, |
| 1949 | llvm::Function *llvmFunc, |
| 1950 | ModuleTranslation &translation) { |
| 1951 | llvm::LLVMContext &llvmContext = llvmFunc->getContext(); |
| 1952 | |
| 1953 | if (VecTypeHintAttr vecTypeHint = func.getVecTypeHintAttr()) { |
| 1954 | Type type = vecTypeHint.getHint().getValue(); |
| 1955 | llvm::Type *llvmType = translation.convertType(type); |
| 1956 | bool isSigned = vecTypeHint.getIsSigned(); |
| 1957 | llvmFunc->setMetadata( |
| 1958 | Kind: func.getVecTypeHintAttrName(), |
| 1959 | Node: convertVecTypeHintToMDNode(context&: llvmContext, type: llvmType, isSigned)); |
| 1960 | } |
| 1961 | |
| 1962 | if (std::optional<ArrayRef<int32_t>> workGroupSizeHint = |
| 1963 | func.getWorkGroupSizeHint()) { |
| 1964 | llvmFunc->setMetadata( |
| 1965 | Kind: func.getWorkGroupSizeHintAttrName(), |
| 1966 | Node: convertIntegerArrayToMDNode(context&: llvmContext, values: *workGroupSizeHint)); |
| 1967 | } |
| 1968 | |
| 1969 | if (std::optional<ArrayRef<int32_t>> reqdWorkGroupSize = |
| 1970 | func.getReqdWorkGroupSize()) { |
| 1971 | llvmFunc->setMetadata( |
| 1972 | Kind: func.getReqdWorkGroupSizeAttrName(), |
| 1973 | Node: convertIntegerArrayToMDNode(context&: llvmContext, values: *reqdWorkGroupSize)); |
| 1974 | } |
| 1975 | |
| 1976 | if (std::optional<uint32_t> intelReqdSubGroupSize = |
| 1977 | func.getIntelReqdSubGroupSize()) { |
| 1978 | llvmFunc->setMetadata( |
| 1979 | Kind: func.getIntelReqdSubGroupSizeAttrName(), |
| 1980 | Node: convertIntegerToMDNode(context&: llvmContext, |
| 1981 | value: llvm::APInt(32, *intelReqdSubGroupSize))); |
| 1982 | } |
| 1983 | } |
| 1984 | |
| 1985 | static LogicalResult convertParameterAttr(llvm::AttrBuilder &attrBuilder, |
| 1986 | llvm::Attribute::AttrKind llvmKind, |
| 1987 | NamedAttribute namedAttr, |
| 1988 | ModuleTranslation &moduleTranslation, |
| 1989 | Location loc) { |
| 1990 | return llvm::TypeSwitch<Attribute, LogicalResult>(namedAttr.getValue()) |
| 1991 | .Case(caseFn: [&](TypeAttr typeAttr) { |
| 1992 | attrBuilder.addTypeAttr( |
| 1993 | Kind: llvmKind, Ty: moduleTranslation.convertType(type: typeAttr.getValue())); |
| 1994 | return success(); |
| 1995 | }) |
| 1996 | .Case(caseFn: [&](IntegerAttr intAttr) { |
| 1997 | attrBuilder.addRawIntAttr(Kind: llvmKind, Value: intAttr.getInt()); |
| 1998 | return success(); |
| 1999 | }) |
| 2000 | .Case(caseFn: [&](UnitAttr) { |
| 2001 | attrBuilder.addAttribute(Val: llvmKind); |
| 2002 | return success(); |
| 2003 | }) |
| 2004 | .Case(caseFn: [&](LLVM::ConstantRangeAttr rangeAttr) { |
| 2005 | attrBuilder.addConstantRangeAttr( |
| 2006 | Kind: llvmKind, |
| 2007 | CR: llvm::ConstantRange(rangeAttr.getLower(), rangeAttr.getUpper())); |
| 2008 | return success(); |
| 2009 | }) |
| 2010 | .Default(defaultFn: [loc](auto) { |
| 2011 | return emitError(loc, message: "unsupported parameter attribute type" ); |
| 2012 | }); |
| 2013 | } |
| 2014 | |
| 2015 | FailureOr<llvm::AttrBuilder> |
| 2016 | ModuleTranslation::convertParameterAttrs(LLVMFuncOp func, int argIdx, |
| 2017 | DictionaryAttr paramAttrs) { |
| 2018 | llvm::AttrBuilder attrBuilder(llvmModule->getContext()); |
| 2019 | auto attrNameToKindMapping = getAttrNameToKindMapping(); |
| 2020 | Location loc = func.getLoc(); |
| 2021 | |
| 2022 | for (auto namedAttr : paramAttrs) { |
| 2023 | auto it = attrNameToKindMapping.find(Val: namedAttr.getName()); |
| 2024 | if (it != attrNameToKindMapping.end()) { |
| 2025 | llvm::Attribute::AttrKind llvmKind = it->second; |
| 2026 | if (failed(Result: convertParameterAttr(attrBuilder, llvmKind, namedAttr, moduleTranslation&: *this, |
| 2027 | loc))) |
| 2028 | return failure(); |
| 2029 | } else if (namedAttr.getNameDialect()) { |
| 2030 | if (failed(Result: iface.convertParameterAttr(function: func, argIdx, attribute: namedAttr, moduleTranslation&: *this))) |
| 2031 | return failure(); |
| 2032 | } |
| 2033 | } |
| 2034 | |
| 2035 | return attrBuilder; |
| 2036 | } |
| 2037 | |
| 2038 | LogicalResult ModuleTranslation::convertArgAndResultAttrs( |
| 2039 | ArgAndResultAttrsOpInterface attrsOp, llvm::CallBase *call, |
| 2040 | ArrayRef<unsigned> immArgPositions) { |
| 2041 | // Convert the argument attributes. |
| 2042 | if (ArrayAttr argAttrsArray = attrsOp.getArgAttrsAttr()) { |
| 2043 | unsigned argAttrIdx = 0; |
| 2044 | llvm::SmallDenseSet<unsigned> immArgPositionsSet(immArgPositions.begin(), |
| 2045 | immArgPositions.end()); |
| 2046 | for (unsigned argIdx : llvm::seq<unsigned>(Size: call->arg_size())) { |
| 2047 | if (argAttrIdx >= argAttrsArray.size()) |
| 2048 | break; |
| 2049 | // Skip immediate arguments (they have no entries in argAttrsArray). |
| 2050 | if (immArgPositionsSet.contains(V: argIdx)) |
| 2051 | continue; |
| 2052 | // Skip empty argument attributes. |
| 2053 | auto argAttrs = cast<DictionaryAttr>(Val: argAttrsArray[argAttrIdx++]); |
| 2054 | if (argAttrs.empty()) |
| 2055 | continue; |
| 2056 | // Convert and add attributes to the call instruction. |
| 2057 | FailureOr<llvm::AttrBuilder> attrBuilder = |
| 2058 | convertParameterAttrs(loc: attrsOp->getLoc(), paramAttrs: argAttrs); |
| 2059 | if (failed(Result: attrBuilder)) |
| 2060 | return failure(); |
| 2061 | call->addParamAttrs(ArgNo: argIdx, B: *attrBuilder); |
| 2062 | } |
| 2063 | } |
| 2064 | |
| 2065 | // Convert the result attributes. |
| 2066 | if (ArrayAttr resAttrsArray = attrsOp.getResAttrsAttr()) { |
| 2067 | if (!resAttrsArray.empty()) { |
| 2068 | auto resAttrs = cast<DictionaryAttr>(Val: resAttrsArray[0]); |
| 2069 | FailureOr<llvm::AttrBuilder> attrBuilder = |
| 2070 | convertParameterAttrs(loc: attrsOp->getLoc(), paramAttrs: resAttrs); |
| 2071 | if (failed(Result: attrBuilder)) |
| 2072 | return failure(); |
| 2073 | call->addRetAttrs(B: *attrBuilder); |
| 2074 | } |
| 2075 | } |
| 2076 | |
| 2077 | return success(); |
| 2078 | } |
| 2079 | |
| 2080 | std::optional<llvm::Attribute> |
| 2081 | ModuleTranslation::convertNoBuiltin(llvm::LLVMContext &ctx, mlir::Attribute a) { |
| 2082 | if (auto str = dyn_cast<StringAttr>(Val&: a)) |
| 2083 | return llvm::Attribute::get(Context&: ctx, Kind: ("no-builtin-" + str.getValue()).str()); |
| 2084 | return std::nullopt; |
| 2085 | } |
| 2086 | |
| 2087 | std::optional<llvm::Attribute> |
| 2088 | ModuleTranslation::convertDefaultFuncAttr(llvm::LLVMContext &ctx, |
| 2089 | mlir::NamedAttribute namedAttr) { |
| 2090 | StringAttr name = namedAttr.getName(); |
| 2091 | Attribute value = namedAttr.getValue(); |
| 2092 | |
| 2093 | if (auto strVal = dyn_cast<StringAttr>(Val&: value)) |
| 2094 | return llvm::Attribute::get(Context&: ctx, Kind: name.getValue(), Val: strVal.getValue()); |
| 2095 | if (mlir::isa<UnitAttr>(Val: value)) |
| 2096 | return llvm::Attribute::get(Context&: ctx, Kind: name.getValue()); |
| 2097 | return std::nullopt; |
| 2098 | } |
| 2099 | |
| 2100 | FailureOr<llvm::AttrBuilder> |
| 2101 | ModuleTranslation::convertParameterAttrs(Location loc, |
| 2102 | DictionaryAttr paramAttrs) { |
| 2103 | llvm::AttrBuilder attrBuilder(llvmModule->getContext()); |
| 2104 | auto attrNameToKindMapping = getAttrNameToKindMapping(); |
| 2105 | |
| 2106 | for (auto namedAttr : paramAttrs) { |
| 2107 | auto it = attrNameToKindMapping.find(Val: namedAttr.getName()); |
| 2108 | if (it != attrNameToKindMapping.end()) { |
| 2109 | llvm::Attribute::AttrKind llvmKind = it->second; |
| 2110 | if (failed(Result: convertParameterAttr(attrBuilder, llvmKind, namedAttr, moduleTranslation&: *this, |
| 2111 | loc))) |
| 2112 | return failure(); |
| 2113 | } |
| 2114 | } |
| 2115 | |
| 2116 | return attrBuilder; |
| 2117 | } |
| 2118 | |
| 2119 | LogicalResult ModuleTranslation::convertFunctionSignatures() { |
| 2120 | // Declare all functions first because there may be function calls that form a |
| 2121 | // call graph with cycles, global initializers that reference functions, or |
| 2122 | // metadata that references functions declared later in the module. |
| 2123 | for (auto function : getModuleBody(module: mlirModule).getOps<LLVMFuncOp>()) { |
| 2124 | llvm::FunctionCallee llvmFuncCst = llvmModule->getOrInsertFunction( |
| 2125 | Name: function.getName(), |
| 2126 | T: cast<llvm::FunctionType>(Val: convertType(type: function.getFunctionType()))); |
| 2127 | llvm::Function *llvmFunc = cast<llvm::Function>(Val: llvmFuncCst.getCallee()); |
| 2128 | mapFunction(name: function.getName(), func: llvmFunc); |
| 2129 | } |
| 2130 | |
| 2131 | for (auto function : getModuleBody(module: mlirModule).getOps<LLVMFuncOp>()) { |
| 2132 | llvm::Function *llvmFunc = lookupFunction(name: function.getName()); |
| 2133 | llvmFunc->setLinkage(convertLinkageToLLVM(value: function.getLinkage())); |
| 2134 | llvmFunc->setCallingConv(convertCConvToLLVM(value: function.getCConv())); |
| 2135 | addRuntimePreemptionSpecifier(dsoLocalRequested: function.getDsoLocal(), gv: llvmFunc); |
| 2136 | |
| 2137 | // Convert function attributes. |
| 2138 | convertFunctionAttributes(mod&: *this, func: function, llvmFunc); |
| 2139 | |
| 2140 | // Convert function kernel attributes to metadata. |
| 2141 | convertFunctionKernelAttributes(func: function, llvmFunc, translation&: *this); |
| 2142 | |
| 2143 | // Convert function_entry_count attribute to metadata. |
| 2144 | if (auto entryCount = function.getFunctionEntryCountAttr()) { |
| 2145 | ArrayRef<uint64_t> imports = entryCount.getImports(); |
| 2146 | llvm::DenseSet<llvm::GlobalValue::GUID> importGUIDs; |
| 2147 | if (!imports.empty()) |
| 2148 | importGUIDs.insert(I: imports.begin(), E: imports.end()); |
| 2149 | llvmFunc->setEntryCount(Count: entryCount.getEntryCount(), |
| 2150 | Imports: imports.empty() ? nullptr : &importGUIDs); |
| 2151 | } |
| 2152 | |
| 2153 | // Convert result attributes. |
| 2154 | if (ArrayAttr allResultAttrs = function.getAllResultAttrs()) { |
| 2155 | DictionaryAttr resultAttrs = cast<DictionaryAttr>(Val: allResultAttrs[0]); |
| 2156 | FailureOr<llvm::AttrBuilder> attrBuilder = |
| 2157 | convertParameterAttrs(func: function, argIdx: -1, paramAttrs: resultAttrs); |
| 2158 | if (failed(Result: attrBuilder)) |
| 2159 | return failure(); |
| 2160 | llvmFunc->addRetAttrs(Attrs: *attrBuilder); |
| 2161 | } |
| 2162 | |
| 2163 | // Convert argument attributes. |
| 2164 | for (auto [argIdx, llvmArg] : llvm::enumerate(First: llvmFunc->args())) { |
| 2165 | if (DictionaryAttr argAttrs = function.getArgAttrDict(index: argIdx)) { |
| 2166 | FailureOr<llvm::AttrBuilder> attrBuilder = |
| 2167 | convertParameterAttrs(func: function, argIdx, paramAttrs: argAttrs); |
| 2168 | if (failed(Result: attrBuilder)) |
| 2169 | return failure(); |
| 2170 | llvmArg.addAttrs(B&: *attrBuilder); |
| 2171 | } |
| 2172 | } |
| 2173 | |
| 2174 | // Forward the pass-through attributes to LLVM. |
| 2175 | FailureOr<llvm::AttrBuilder> convertedPassthroughAttrs = |
| 2176 | convertMLIRAttributesToLLVM(loc: function.getLoc(), ctx&: llvmFunc->getContext(), |
| 2177 | arrayAttr: function.getPassthroughAttr(), |
| 2178 | arrayAttrName: function.getPassthroughAttrName()); |
| 2179 | if (failed(Result: convertedPassthroughAttrs)) |
| 2180 | return failure(); |
| 2181 | llvmFunc->addFnAttrs(Attrs: *convertedPassthroughAttrs); |
| 2182 | |
| 2183 | // Convert visibility attribute. |
| 2184 | llvmFunc->setVisibility(convertVisibilityToLLVM(value: function.getVisibility_())); |
| 2185 | |
| 2186 | // Convert the comdat attribute. |
| 2187 | if (std::optional<mlir::SymbolRefAttr> comdat = function.getComdat()) { |
| 2188 | auto selectorOp = cast<ComdatSelectorOp>( |
| 2189 | Val: symbolTable().lookupNearestSymbolFrom(from: function, symbol: *comdat)); |
| 2190 | llvmFunc->setComdat(comdatMapping.lookup(Val: selectorOp)); |
| 2191 | } |
| 2192 | |
| 2193 | if (auto gc = function.getGarbageCollector()) |
| 2194 | llvmFunc->setGC(gc->str()); |
| 2195 | |
| 2196 | if (auto unnamedAddr = function.getUnnamedAddr()) |
| 2197 | llvmFunc->setUnnamedAddr(convertUnnamedAddrToLLVM(value: *unnamedAddr)); |
| 2198 | |
| 2199 | if (auto alignment = function.getAlignment()) |
| 2200 | llvmFunc->setAlignment(llvm::MaybeAlign(*alignment)); |
| 2201 | |
| 2202 | // Translate the debug information for this function. |
| 2203 | debugTranslation->translate(func: function, llvmFunc&: *llvmFunc); |
| 2204 | } |
| 2205 | |
| 2206 | return success(); |
| 2207 | } |
| 2208 | |
| 2209 | LogicalResult ModuleTranslation::convertFunctions() { |
| 2210 | // Convert functions. |
| 2211 | for (auto function : getModuleBody(module: mlirModule).getOps<LLVMFuncOp>()) { |
| 2212 | // Do not convert external functions, but do process dialect attributes |
| 2213 | // attached to them. |
| 2214 | if (function.isExternal()) { |
| 2215 | if (failed(Result: convertDialectAttributes(op: function, instructions: {}))) |
| 2216 | return failure(); |
| 2217 | continue; |
| 2218 | } |
| 2219 | |
| 2220 | if (failed(Result: convertOneFunction(func: function))) |
| 2221 | return failure(); |
| 2222 | } |
| 2223 | |
| 2224 | return success(); |
| 2225 | } |
| 2226 | |
| 2227 | LogicalResult ModuleTranslation::convertIFuncs() { |
| 2228 | for (auto op : getModuleBody(module: mlirModule).getOps<IFuncOp>()) { |
| 2229 | llvm::Type *type = convertType(type: op.getIFuncType()); |
| 2230 | llvm::GlobalValue::LinkageTypes linkage = |
| 2231 | convertLinkageToLLVM(value: op.getLinkage()); |
| 2232 | llvm::Constant *resolver; |
| 2233 | if (auto *resolverFn = lookupFunction(name: op.getResolver())) { |
| 2234 | resolver = cast<llvm::Constant>(Val: resolverFn); |
| 2235 | } else { |
| 2236 | Operation *aliasOp = symbolTable().lookupSymbolIn(symbolTableOp: parentLLVMModule(op), |
| 2237 | name: op.getResolverAttr()); |
| 2238 | resolver = cast<llvm::Constant>(Val: lookupAlias(op: aliasOp)); |
| 2239 | } |
| 2240 | |
| 2241 | auto *ifunc = |
| 2242 | llvm::GlobalIFunc::create(Ty: type, AddressSpace: op.getAddressSpace(), Linkage: linkage, |
| 2243 | Name: op.getSymName(), Resolver: resolver, Parent: llvmModule.get()); |
| 2244 | addRuntimePreemptionSpecifier(dsoLocalRequested: op.getDsoLocal(), gv: ifunc); |
| 2245 | ifunc->setUnnamedAddr(convertUnnamedAddrToLLVM(value: op.getUnnamedAddr())); |
| 2246 | ifunc->setVisibility(convertVisibilityToLLVM(value: op.getVisibility_())); |
| 2247 | |
| 2248 | ifuncMapping.try_emplace(Key: op, Args&: ifunc); |
| 2249 | } |
| 2250 | |
| 2251 | return success(); |
| 2252 | } |
| 2253 | |
| 2254 | // Attach global metadata after all globals, aliases, ifuncs, and function |
| 2255 | // signatures exist so symbol references can be resolved. |
| 2256 | LogicalResult ModuleTranslation::convertGlobalMetadata() { |
| 2257 | for (auto op : getModuleBody(module: mlirModule).getOps<LLVM::GlobalOp>()) { |
| 2258 | auto *var = cast<llvm::GlobalVariable>(Val: lookupGlobal(op)); |
| 2259 | if (FlatSymbolRefAttr associated = op.getAssociatedAttr()) { |
| 2260 | FailureOr<llvm::Metadata *> md = |
| 2261 | convertSymbolRefToMetadata(name: associated, emitError: [&]() { |
| 2262 | return op.emitError(message: "failed to convert associated metadata" ); |
| 2263 | }); |
| 2264 | if (failed(Result: md)) |
| 2265 | return failure(); |
| 2266 | var->setMetadata(KindID: llvm::LLVMContext::MD_associated, |
| 2267 | Node: llvm::MDNode::get(Context&: var->getContext(), MDs: *md)); |
| 2268 | } |
| 2269 | |
| 2270 | if (ArrayAttr absSym = op.getAbsoluteSymbolAttr()) { |
| 2271 | SmallVector<llvm::Metadata *> mdOps; |
| 2272 | llvm::LLVMContext &ctx = var->getContext(); |
| 2273 | mdOps.reserve(N: absSym.size()); |
| 2274 | for (Attribute attr : absSym) { |
| 2275 | auto intAttr = cast<IntegerAttr>(Val&: attr); |
| 2276 | llvm::IntegerType *ty = llvm::IntegerType::get( |
| 2277 | C&: ctx, NumBits: intAttr.getType().getIntOrFloatBitWidth()); |
| 2278 | mdOps.push_back(Elt: llvm::ConstantAsMetadata::get( |
| 2279 | C: llvm::ConstantInt::get(Ty: ty, V: intAttr.getValue()))); |
| 2280 | } |
| 2281 | var->setMetadata(KindID: llvm::LLVMContext::MD_absolute_symbol, |
| 2282 | Node: llvm::MDNode::get(Context&: ctx, MDs: mdOps)); |
| 2283 | } |
| 2284 | } |
| 2285 | |
| 2286 | return success(); |
| 2287 | } |
| 2288 | |
| 2289 | LogicalResult ModuleTranslation::convertComdats() { |
| 2290 | for (auto comdatOp : getModuleBody(module: mlirModule).getOps<ComdatOp>()) { |
| 2291 | for (auto selectorOp : comdatOp.getOps<ComdatSelectorOp>()) { |
| 2292 | llvm::Module *module = getLLVMModule(); |
| 2293 | if (module->getComdatSymbolTable().contains(Key: selectorOp.getSymName())) |
| 2294 | return emitError(loc: selectorOp.getLoc()) |
| 2295 | << "comdat selection symbols must be unique even in different " |
| 2296 | "comdat regions" ; |
| 2297 | llvm::Comdat *comdat = module->getOrInsertComdat(Name: selectorOp.getSymName()); |
| 2298 | comdat->setSelectionKind(convertComdatToLLVM(value: selectorOp.getComdat())); |
| 2299 | comdatMapping.try_emplace(Key: selectorOp, Args&: comdat); |
| 2300 | } |
| 2301 | } |
| 2302 | return success(); |
| 2303 | } |
| 2304 | |
| 2305 | LogicalResult ModuleTranslation::convertUnresolvedBlockAddress() { |
| 2306 | for (auto &[blockAddressOp, llvmCst] : unresolvedBlockAddressMapping) { |
| 2307 | BlockAddressAttr blockAddressAttr = blockAddressOp.getBlockAddr(); |
| 2308 | llvm::BasicBlock *llvmBlock = lookupBlockAddress(attr: blockAddressAttr); |
| 2309 | assert(llvmBlock && "expected LLVM blocks to be already translated" ); |
| 2310 | |
| 2311 | // Update mapping with new block address constant. |
| 2312 | auto *llvmBlockAddr = llvm::BlockAddress::get( |
| 2313 | F: lookupFunction(name: blockAddressAttr.getFunction().getValue()), BB: llvmBlock); |
| 2314 | llvmCst->replaceAllUsesWith(V: llvmBlockAddr); |
| 2315 | assert(llvmCst->use_empty() && "expected all uses to be replaced" ); |
| 2316 | cast<llvm::GlobalVariable>(Val: llvmCst)->eraseFromParent(); |
| 2317 | } |
| 2318 | unresolvedBlockAddressMapping.clear(); |
| 2319 | return success(); |
| 2320 | } |
| 2321 | |
| 2322 | void ModuleTranslation::setAccessGroupsMetadata(AccessGroupOpInterface op, |
| 2323 | llvm::Instruction *inst) { |
| 2324 | if (llvm::MDNode *node = loopAnnotationTranslation->getAccessGroups(op)) |
| 2325 | inst->setMetadata(KindID: llvm::LLVMContext::MD_access_group, Node: node); |
| 2326 | } |
| 2327 | |
| 2328 | llvm::MDNode * |
| 2329 | ModuleTranslation::getOrCreateAliasScope(AliasScopeAttr aliasScopeAttr) { |
| 2330 | auto [scopeIt, scopeInserted] = |
| 2331 | aliasScopeMetadataMapping.try_emplace(Key: aliasScopeAttr, Args: nullptr); |
| 2332 | if (!scopeInserted) |
| 2333 | return scopeIt->second; |
| 2334 | llvm::LLVMContext &ctx = llvmModule->getContext(); |
| 2335 | auto dummy = llvm::MDNode::getTemporary(Context&: ctx, MDs: {}); |
| 2336 | // Convert the domain metadata node if necessary. |
| 2337 | auto [domainIt, insertedDomain] = aliasDomainMetadataMapping.try_emplace( |
| 2338 | Key: aliasScopeAttr.getDomain(), Args: nullptr); |
| 2339 | if (insertedDomain) { |
| 2340 | llvm::SmallVector<llvm::Metadata *, 3> operands; |
| 2341 | // Placeholder for potential self-reference. |
| 2342 | operands.push_back(Elt: dummy.get()); |
| 2343 | operands.push_back( |
| 2344 | Elt: llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::getFalse(Context&: ctx))); |
| 2345 | if (StringAttr description = aliasScopeAttr.getDomain().getDescription()) |
| 2346 | operands.push_back(Elt: llvm::MDString::get(Context&: ctx, Str: description)); |
| 2347 | domainIt->second = llvm::MDNode::get(Context&: ctx, MDs: operands); |
| 2348 | // Self-reference for uniqueness. |
| 2349 | llvm::Metadata *replacement; |
| 2350 | if (auto stringAttr = |
| 2351 | dyn_cast<StringAttr>(Val: aliasScopeAttr.getDomain().getId())) |
| 2352 | replacement = llvm::MDString::get(Context&: ctx, Str: stringAttr.getValue()); |
| 2353 | else |
| 2354 | replacement = domainIt->second; |
| 2355 | domainIt->second->replaceOperandWith(I: 0, New: replacement); |
| 2356 | } |
| 2357 | // Convert the scope metadata node. |
| 2358 | assert(domainIt->second && "Scope's domain should already be valid" ); |
| 2359 | llvm::SmallVector<llvm::Metadata *, 3> operands; |
| 2360 | // Placeholder for potential self-reference. |
| 2361 | operands.push_back(Elt: dummy.get()); |
| 2362 | operands.push_back(Elt: domainIt->second); |
| 2363 | if (StringAttr description = aliasScopeAttr.getDescription()) |
| 2364 | operands.push_back(Elt: llvm::MDString::get(Context&: ctx, Str: description)); |
| 2365 | scopeIt->second = llvm::MDNode::get(Context&: ctx, MDs: operands); |
| 2366 | // Self-reference for uniqueness. |
| 2367 | llvm::Metadata *replacement; |
| 2368 | if (auto stringAttr = dyn_cast<StringAttr>(Val: aliasScopeAttr.getId())) |
| 2369 | replacement = llvm::MDString::get(Context&: ctx, Str: stringAttr.getValue()); |
| 2370 | else |
| 2371 | replacement = scopeIt->second; |
| 2372 | scopeIt->second->replaceOperandWith(I: 0, New: replacement); |
| 2373 | return scopeIt->second; |
| 2374 | } |
| 2375 | |
| 2376 | llvm::MDNode *ModuleTranslation::getOrCreateAliasScopes( |
| 2377 | ArrayRef<AliasScopeAttr> aliasScopeAttrs) { |
| 2378 | SmallVector<llvm::Metadata *> nodes; |
| 2379 | nodes.reserve(N: aliasScopeAttrs.size()); |
| 2380 | for (AliasScopeAttr aliasScopeAttr : aliasScopeAttrs) |
| 2381 | nodes.push_back(Elt: getOrCreateAliasScope(aliasScopeAttr)); |
| 2382 | return llvm::MDNode::get(Context&: getLLVMContext(), MDs: nodes); |
| 2383 | } |
| 2384 | |
| 2385 | void ModuleTranslation::setAliasScopeMetadata(AliasAnalysisOpInterface op, |
| 2386 | llvm::Instruction *inst) { |
| 2387 | auto populateScopeMetadata = [&](ArrayAttr aliasScopeAttrs, unsigned kind) { |
| 2388 | if (!aliasScopeAttrs || aliasScopeAttrs.empty()) |
| 2389 | return; |
| 2390 | llvm::MDNode *node = getOrCreateAliasScopes( |
| 2391 | aliasScopeAttrs: llvm::to_vector(Range: aliasScopeAttrs.getAsRange<AliasScopeAttr>())); |
| 2392 | inst->setMetadata(KindID: kind, Node: node); |
| 2393 | }; |
| 2394 | |
| 2395 | populateScopeMetadata(op.getAliasScopesOrNull(), |
| 2396 | llvm::LLVMContext::MD_alias_scope); |
| 2397 | populateScopeMetadata(op.getNoAliasScopesOrNull(), |
| 2398 | llvm::LLVMContext::MD_noalias); |
| 2399 | } |
| 2400 | |
| 2401 | llvm::MDNode *ModuleTranslation::getTBAANode(TBAATagAttr tbaaAttr) const { |
| 2402 | return tbaaMetadataMapping.lookup(Val: tbaaAttr); |
| 2403 | } |
| 2404 | |
| 2405 | void ModuleTranslation::setTBAAMetadata(AliasAnalysisOpInterface op, |
| 2406 | llvm::Instruction *inst) { |
| 2407 | ArrayAttr tagRefs = op.getTBAATagsOrNull(); |
| 2408 | if (!tagRefs || tagRefs.empty()) |
| 2409 | return; |
| 2410 | |
| 2411 | // LLVM IR currently does not support attaching more than one TBAA access tag |
| 2412 | // to a memory accessing instruction. It may be useful to support this in |
| 2413 | // future, but for the time being just ignore the metadata if MLIR operation |
| 2414 | // has multiple access tags. |
| 2415 | if (tagRefs.size() > 1) { |
| 2416 | op.emitWarning() << "TBAA access tags were not translated, because LLVM " |
| 2417 | "IR only supports a single tag per instruction" ; |
| 2418 | return; |
| 2419 | } |
| 2420 | |
| 2421 | llvm::MDNode *node = getTBAANode(tbaaAttr: cast<TBAATagAttr>(Val: tagRefs[0])); |
| 2422 | inst->setMetadata(KindID: llvm::LLVMContext::MD_tbaa, Node: node); |
| 2423 | } |
| 2424 | |
| 2425 | void ModuleTranslation::setTBAAStructMetadata(ArrayAttr fields, |
| 2426 | llvm::Instruction *inst) { |
| 2427 | if (!fields || fields.empty()) |
| 2428 | return; |
| 2429 | |
| 2430 | // The node is a flat list of (offset, size, tag) triples, not a list of |
| 2431 | // per-field nodes. |
| 2432 | SmallVector<llvm::MDBuilder::TBAAStructField> structFields; |
| 2433 | structFields.reserve(N: fields.size()); |
| 2434 | for (Attribute field : fields) { |
| 2435 | auto fieldAttr = cast<TBAAStructFieldAttr>(Val&: field); |
| 2436 | structFields.emplace_back(Args: fieldAttr.getOffset(), Args: fieldAttr.getSize(), |
| 2437 | Args: getTBAANode(tbaaAttr: fieldAttr.getTag())); |
| 2438 | } |
| 2439 | inst->setMetadata( |
| 2440 | KindID: llvm::LLVMContext::MD_tbaa_struct, |
| 2441 | Node: llvm::MDBuilder(getLLVMContext()).createTBAAStructNode(Fields: structFields)); |
| 2442 | } |
| 2443 | |
| 2444 | void ModuleTranslation::setDereferenceableMetadata( |
| 2445 | DereferenceableOpInterface op, llvm::Instruction *inst) { |
| 2446 | DereferenceableAttr derefAttr = op.getDereferenceableOrNull(); |
| 2447 | if (!derefAttr) |
| 2448 | return; |
| 2449 | |
| 2450 | llvm::MDNode *derefSizeNode = llvm::MDNode::get( |
| 2451 | Context&: getLLVMContext(), |
| 2452 | MDs: llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get( |
| 2453 | Ty: llvm::IntegerType::get(C&: getLLVMContext(), NumBits: 64), V: derefAttr.getBytes()))); |
| 2454 | unsigned kindId = derefAttr.getMayBeNull() |
| 2455 | ? llvm::LLVMContext::MD_dereferenceable_or_null |
| 2456 | : llvm::LLVMContext::MD_dereferenceable; |
| 2457 | inst->setMetadata(KindID: kindId, Node: derefSizeNode); |
| 2458 | } |
| 2459 | |
| 2460 | void ModuleTranslation::setBranchWeightsMetadata(WeightedBranchOpInterface op) { |
| 2461 | SmallVector<uint32_t> weights; |
| 2462 | llvm::transform(Range: op.getWeights(), d_first: std::back_inserter(x&: weights), |
| 2463 | F: [](int32_t value) { return static_cast<uint32_t>(value); }); |
| 2464 | if (weights.empty()) |
| 2465 | return; |
| 2466 | |
| 2467 | llvm::Instruction *inst = isa<CallOp>(Val: op) ? lookupCall(op) : lookupBranch(op); |
| 2468 | assert(inst && "expected the operation to have a mapping to an instruction" ); |
| 2469 | inst->setMetadata( |
| 2470 | KindID: llvm::LLVMContext::MD_prof, |
| 2471 | Node: llvm::MDBuilder(getLLVMContext()).createBranchWeights(Weights: weights)); |
| 2472 | } |
| 2473 | |
| 2474 | LogicalResult ModuleTranslation::createTBAAMetadata() { |
| 2475 | llvm::LLVMContext &ctx = llvmModule->getContext(); |
| 2476 | llvm::IntegerType *offsetTy = llvm::IntegerType::get(C&: ctx, NumBits: 64); |
| 2477 | |
| 2478 | // Walk the entire module and create all metadata nodes for the TBAA |
| 2479 | // attributes. The code below relies on two invariants of the |
| 2480 | // `AttrTypeWalker`: |
| 2481 | // 1. Attributes are visited in post-order: Since the attributes create a DAG, |
| 2482 | // this ensures that any lookups into `tbaaMetadataMapping` for child |
| 2483 | // attributes succeed. |
| 2484 | // 2. Attributes are only ever visited once: This way we don't leak any |
| 2485 | // LLVM metadata instances. |
| 2486 | AttrTypeWalker walker; |
| 2487 | walker.addWalk(callback: [&](TBAARootAttr root) { |
| 2488 | llvm::MDNode *node; |
| 2489 | if (StringAttr id = root.getId()) { |
| 2490 | node = llvm::MDNode::get(Context&: ctx, MDs: llvm::MDString::get(Context&: ctx, Str: id)); |
| 2491 | } else { |
| 2492 | // Anonymous root nodes are self-referencing. |
| 2493 | auto selfRef = llvm::MDNode::getTemporary(Context&: ctx, MDs: {}); |
| 2494 | node = llvm::MDNode::get(Context&: ctx, MDs: {selfRef.get()}); |
| 2495 | node->replaceOperandWith(I: 0, New: node); |
| 2496 | } |
| 2497 | tbaaMetadataMapping.insert(KV: {root, node}); |
| 2498 | }); |
| 2499 | |
| 2500 | walker.addWalk(callback: [&](TBAATypeDescriptorAttr descriptor) { |
| 2501 | SmallVector<llvm::Metadata *> operands; |
| 2502 | operands.push_back(Elt: llvm::MDString::get(Context&: ctx, Str: descriptor.getId())); |
| 2503 | for (TBAAMemberAttr member : descriptor.getMembers()) { |
| 2504 | operands.push_back(Elt: tbaaMetadataMapping.lookup(Val: member.getTypeDesc())); |
| 2505 | operands.push_back(Elt: llvm::ConstantAsMetadata::get( |
| 2506 | C: llvm::ConstantInt::get(Ty: offsetTy, V: member.getOffset()))); |
| 2507 | } |
| 2508 | |
| 2509 | tbaaMetadataMapping.insert(KV: {descriptor, llvm::MDNode::get(Context&: ctx, MDs: operands)}); |
| 2510 | }); |
| 2511 | |
| 2512 | walker.addWalk(callback: [&](TBAATagAttr tag) { |
| 2513 | SmallVector<llvm::Metadata *> operands; |
| 2514 | |
| 2515 | operands.push_back(Elt: tbaaMetadataMapping.lookup(Val: tag.getBaseType())); |
| 2516 | operands.push_back(Elt: tbaaMetadataMapping.lookup(Val: tag.getAccessType())); |
| 2517 | |
| 2518 | operands.push_back(Elt: llvm::ConstantAsMetadata::get( |
| 2519 | C: llvm::ConstantInt::get(Ty: offsetTy, V: tag.getOffset()))); |
| 2520 | if (tag.getConstant()) |
| 2521 | operands.push_back( |
| 2522 | Elt: llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(Ty: offsetTy, V: 1))); |
| 2523 | |
| 2524 | tbaaMetadataMapping.insert(KV: {tag, llvm::MDNode::get(Context&: ctx, MDs: operands)}); |
| 2525 | }); |
| 2526 | |
| 2527 | mlirModule->walk(callback: [&](AliasAnalysisOpInterface analysisOpInterface) { |
| 2528 | if (auto attr = analysisOpInterface.getTBAATagsOrNull()) |
| 2529 | walker.walk(element: attr); |
| 2530 | }); |
| 2531 | |
| 2532 | // The tags inside a `tbaa_struct` are reachable from nowhere else, so without |
| 2533 | // this walk they never get an MDNode and the exported node has null tags. |
| 2534 | auto walkStructFields = [&](ArrayAttr fields) { |
| 2535 | if (fields) |
| 2536 | walker.walk(element: fields); |
| 2537 | }; |
| 2538 | mlirModule->walk(callback: [&](TBAAStructOpInterface tbaaStructOp) { |
| 2539 | walkStructFields(tbaaStructOp.getTBAAStructOrNull()); |
| 2540 | }); |
| 2541 | |
| 2542 | return success(); |
| 2543 | } |
| 2544 | |
| 2545 | LogicalResult ModuleTranslation::createIdentMetadata() { |
| 2546 | if (auto attr = mlirModule->getDiscardableAttrOfType<StringAttr>( |
| 2547 | name: LLVMDialect::getIdentAttrName())) { |
| 2548 | StringRef ident = attr; |
| 2549 | llvm::LLVMContext &ctx = llvmModule->getContext(); |
| 2550 | llvm::NamedMDNode *namedMd = |
| 2551 | llvmModule->getOrInsertNamedMetadata(Name: LLVMDialect::getIdentAttrName()); |
| 2552 | llvm::MDNode *md = llvm::MDNode::get(Context&: ctx, MDs: llvm::MDString::get(Context&: ctx, Str: ident)); |
| 2553 | namedMd->addOperand(M: md); |
| 2554 | } |
| 2555 | |
| 2556 | return success(); |
| 2557 | } |
| 2558 | |
| 2559 | LogicalResult ModuleTranslation::createCommandlineMetadata() { |
| 2560 | if (auto attr = mlirModule->getDiscardableAttrOfType<StringAttr>( |
| 2561 | name: LLVMDialect::getCommandlineAttrName())) { |
| 2562 | StringRef cmdLine = attr; |
| 2563 | llvm::LLVMContext &ctx = llvmModule->getContext(); |
| 2564 | llvm::NamedMDNode *nmd = llvmModule->getOrInsertNamedMetadata( |
| 2565 | Name: LLVMDialect::getCommandlineAttrName()); |
| 2566 | llvm::MDNode *md = |
| 2567 | llvm::MDNode::get(Context&: ctx, MDs: llvm::MDString::get(Context&: ctx, Str: cmdLine)); |
| 2568 | nmd->addOperand(M: md); |
| 2569 | } |
| 2570 | |
| 2571 | return success(); |
| 2572 | } |
| 2573 | |
| 2574 | LogicalResult ModuleTranslation::createDependentLibrariesMetadata() { |
| 2575 | if (auto dependentLibrariesAttr = mlirModule->getDiscardableAttr( |
| 2576 | name: LLVM::LLVMDialect::getDependentLibrariesAttrName())) { |
| 2577 | auto *nmd = |
| 2578 | llvmModule->getOrInsertNamedMetadata(Name: "llvm.dependent-libraries" ); |
| 2579 | llvm::LLVMContext &ctx = llvmModule->getContext(); |
| 2580 | for (auto libAttr : |
| 2581 | cast<ArrayAttr>(Val&: dependentLibrariesAttr).getAsRange<StringAttr>()) { |
| 2582 | auto *md = |
| 2583 | llvm::MDNode::get(Context&: ctx, MDs: llvm::MDString::get(Context&: ctx, Str: libAttr.getValue())); |
| 2584 | nmd->addOperand(M: md); |
| 2585 | } |
| 2586 | } |
| 2587 | return success(); |
| 2588 | } |
| 2589 | |
| 2590 | void ModuleTranslation::setLoopMetadata(Operation *op, |
| 2591 | llvm::Instruction *inst) { |
| 2592 | LoopAnnotationAttr attr = |
| 2593 | TypeSwitch<Operation *, LoopAnnotationAttr>(op) |
| 2594 | .Case<LLVM::BrOp, LLVM::CondBrOp>( |
| 2595 | caseFn: [](auto branchOp) { return branchOp.getLoopAnnotationAttr(); }); |
| 2596 | if (!attr) |
| 2597 | return; |
| 2598 | llvm::MDNode *loopMD = |
| 2599 | loopAnnotationTranslation->translateLoopAnnotation(attr, op); |
| 2600 | inst->setMetadata(KindID: llvm::LLVMContext::MD_loop, Node: loopMD); |
| 2601 | } |
| 2602 | |
| 2603 | void ModuleTranslation::setDisjointFlag(Operation *op, llvm::Value *value) { |
| 2604 | auto iface = cast<DisjointFlagInterface>(Val: op); |
| 2605 | // We do a dyn_cast here in case the value got folded into a constant. |
| 2606 | if (auto *disjointInst = dyn_cast<llvm::PossiblyDisjointInst>(Val: value)) |
| 2607 | disjointInst->setIsDisjoint(iface.getIsDisjoint()); |
| 2608 | } |
| 2609 | |
| 2610 | llvm::Type *ModuleTranslation::convertType(Type type) { |
| 2611 | return typeTranslator.translateType(type); |
| 2612 | } |
| 2613 | |
| 2614 | /// A helper to look up remapped operands in the value remapping table. |
| 2615 | SmallVector<llvm::Value *> ModuleTranslation::lookupValues(ValueRange values) { |
| 2616 | SmallVector<llvm::Value *> remapped; |
| 2617 | remapped.reserve(N: values.size()); |
| 2618 | for (Value v : values) |
| 2619 | remapped.push_back(Elt: lookupValue(value: v)); |
| 2620 | return remapped; |
| 2621 | } |
| 2622 | |
| 2623 | void ModuleTranslation::remapAllValuesWith(llvm::Value *oldValue, |
| 2624 | llvm::Value *newValue) { |
| 2625 | if (oldValue == newValue) |
| 2626 | return; |
| 2627 | |
| 2628 | for (auto &entry : valueMapping) |
| 2629 | if (entry.second == oldValue) |
| 2630 | entry.second = newValue; |
| 2631 | } |
| 2632 | |
| 2633 | llvm::vfs::FileSystem &ModuleTranslation::getFileSystem() { |
| 2634 | if (fileSystem) |
| 2635 | return *fileSystem; |
| 2636 | return *llvm::vfs::getRealFileSystem(); |
| 2637 | } |
| 2638 | |
| 2639 | llvm::DILocation *ModuleTranslation::translateLoc(Location loc, |
| 2640 | llvm::DILocalScope *scope) { |
| 2641 | return debugTranslation->translateLoc(loc, scope); |
| 2642 | } |
| 2643 | |
| 2644 | llvm::DIExpression * |
| 2645 | ModuleTranslation::translateExpression(LLVM::DIExpressionAttr attr) { |
| 2646 | return debugTranslation->translateExpression(attr); |
| 2647 | } |
| 2648 | |
| 2649 | llvm::DIGlobalVariableExpression * |
| 2650 | ModuleTranslation::translateGlobalVariableExpression( |
| 2651 | LLVM::DIGlobalVariableExpressionAttr attr) { |
| 2652 | return debugTranslation->translateGlobalVariableExpression(attr); |
| 2653 | } |
| 2654 | |
| 2655 | llvm::Metadata *ModuleTranslation::translateDebugInfo(LLVM::DINodeAttr attr) { |
| 2656 | return debugTranslation->translate(attr); |
| 2657 | } |
| 2658 | |
| 2659 | llvm::RoundingMode |
| 2660 | ModuleTranslation::translateRoundingMode(LLVM::RoundingMode rounding) { |
| 2661 | return convertRoundingModeToLLVM(value: rounding); |
| 2662 | } |
| 2663 | |
| 2664 | llvm::fp::ExceptionBehavior ModuleTranslation::translateFPExceptionBehavior( |
| 2665 | LLVM::FPExceptionBehavior exceptionBehavior) { |
| 2666 | return convertFPExceptionBehaviorToLLVM(value: exceptionBehavior); |
| 2667 | } |
| 2668 | |
| 2669 | llvm::NamedMDNode * |
| 2670 | ModuleTranslation::getOrInsertNamedModuleMetadata(StringRef name) { |
| 2671 | return llvmModule->getOrInsertNamedMetadata(Name: name); |
| 2672 | } |
| 2673 | |
| 2674 | static std::unique_ptr<llvm::Module> |
| 2675 | prepareLLVMModule(Operation *m, llvm::LLVMContext &llvmContext, |
| 2676 | StringRef name) { |
| 2677 | m->getContext()->getOrLoadDialect<LLVM::LLVMDialect>(); |
| 2678 | auto llvmModule = std::make_unique<llvm::Module>(args&: name, args&: llvmContext); |
| 2679 | if (auto dataLayoutAttr = |
| 2680 | m->getDiscardableAttr(name: LLVM::LLVMDialect::getDataLayoutAttrName())) { |
| 2681 | llvmModule->setDataLayout(cast<StringAttr>(Val&: dataLayoutAttr).getValue()); |
| 2682 | } else { |
| 2683 | FailureOr<llvm::DataLayout> llvmDataLayout(llvm::DataLayout("" )); |
| 2684 | if (auto iface = dyn_cast<DataLayoutOpInterface>(Val: m)) { |
| 2685 | if (DataLayoutSpecInterface spec = iface.getDataLayoutSpec()) { |
| 2686 | llvmDataLayout = |
| 2687 | translateDataLayout(attribute: spec, dataLayout: DataLayout(iface), loc: m->getLoc()); |
| 2688 | } |
| 2689 | } else if (auto mod = dyn_cast<ModuleOp>(Val: m)) { |
| 2690 | if (DataLayoutSpecInterface spec = mod.getDataLayoutSpec()) { |
| 2691 | llvmDataLayout = |
| 2692 | translateDataLayout(attribute: spec, dataLayout: DataLayout(mod), loc: m->getLoc()); |
| 2693 | } |
| 2694 | } |
| 2695 | if (failed(Result: llvmDataLayout)) |
| 2696 | return nullptr; |
| 2697 | llvmModule->setDataLayout(*llvmDataLayout); |
| 2698 | } |
| 2699 | if (auto targetTripleAttr = |
| 2700 | m->getDiscardableAttr(name: LLVM::LLVMDialect::getTargetTripleAttrName())) |
| 2701 | llvmModule->setTargetTriple( |
| 2702 | llvm::Triple(cast<StringAttr>(Val&: targetTripleAttr).getValue())); |
| 2703 | |
| 2704 | if (auto asmAttr = m->getDiscardableAttr( |
| 2705 | name: LLVM::LLVMDialect::getModuleLevelAsmAttrName())) { |
| 2706 | auto asmArrayAttr = dyn_cast<ArrayAttr>(Val&: asmAttr); |
| 2707 | if (!asmArrayAttr) { |
| 2708 | m->emitError(message: "expected an array attribute for a module level asm" ); |
| 2709 | return nullptr; |
| 2710 | } |
| 2711 | |
| 2712 | for (Attribute elt : asmArrayAttr) { |
| 2713 | auto asmStrAttr = dyn_cast<StringAttr>(Val&: elt); |
| 2714 | if (!asmStrAttr) { |
| 2715 | m->emitError( |
| 2716 | message: "expected a string attribute for each entry of a module level asm" ); |
| 2717 | return nullptr; |
| 2718 | } |
| 2719 | llvmModule->appendModuleInlineAsm(Fragment: asmStrAttr.getValue()); |
| 2720 | } |
| 2721 | } |
| 2722 | |
| 2723 | return llvmModule; |
| 2724 | } |
| 2725 | |
| 2726 | std::unique_ptr<llvm::Module> |
| 2727 | mlir::translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext, |
| 2728 | StringRef name, bool disableVerification, |
| 2729 | llvm::vfs::FileSystem *fs) { |
| 2730 | if (!satisfiesLLVMModule(op: module)) { |
| 2731 | module->emitOpError(message: "can not be translated to an LLVMIR module" ); |
| 2732 | return nullptr; |
| 2733 | } |
| 2734 | |
| 2735 | std::unique_ptr<llvm::Module> llvmModule = |
| 2736 | prepareLLVMModule(m: module, llvmContext, name); |
| 2737 | if (!llvmModule) |
| 2738 | return nullptr; |
| 2739 | |
| 2740 | LLVM::ensureDistinctSuccessors(op: module); |
| 2741 | LLVM::legalizeDIExpressionsRecursively(op: module); |
| 2742 | |
| 2743 | ModuleTranslation translator(module, std::move(llvmModule), fs); |
| 2744 | llvm::IRBuilder<llvm::TargetFolder> llvmBuilder( |
| 2745 | *translator.getLLVMModule(), |
| 2746 | llvm::TargetFolder(translator.getLLVMModule()->getDataLayout())); |
| 2747 | |
| 2748 | // Convert module before functions and operations inside, so dialect |
| 2749 | // attributes can be used to change dialect-specific global configurations via |
| 2750 | // `amendOperation()`. These configurations can then influence the translation |
| 2751 | // of operations afterwards. |
| 2752 | if (failed(Result: translator.convertOperation(op&: *module, builder&: llvmBuilder))) |
| 2753 | return nullptr; |
| 2754 | |
| 2755 | if (failed(Result: translator.convertComdats())) |
| 2756 | return nullptr; |
| 2757 | if (failed(Result: translator.convertFunctionSignatures())) |
| 2758 | return nullptr; |
| 2759 | if (failed(Result: translator.convertGlobalsAndAliases())) |
| 2760 | return nullptr; |
| 2761 | if (failed(Result: translator.convertIFuncs())) |
| 2762 | return nullptr; |
| 2763 | if (failed(Result: translator.convertGlobalMetadata())) |
| 2764 | return nullptr; |
| 2765 | if (failed(Result: translator.convertFunctionMetadata())) |
| 2766 | return nullptr; |
| 2767 | if (failed(Result: translator.createTBAAMetadata())) |
| 2768 | return nullptr; |
| 2769 | if (failed(Result: translator.createIdentMetadata())) |
| 2770 | return nullptr; |
| 2771 | if (failed(Result: translator.createCommandlineMetadata())) |
| 2772 | return nullptr; |
| 2773 | if (failed(Result: translator.createDependentLibrariesMetadata())) |
| 2774 | return nullptr; |
| 2775 | |
| 2776 | // Convert other top-level operations if possible. |
| 2777 | for (Operation &o : getModuleBody(module).getOperations()) { |
| 2778 | if (!isa<LLVM::LLVMFuncOp, LLVM::AliasOp, LLVM::GlobalOp, |
| 2779 | LLVM::GlobalCtorsOp, LLVM::GlobalDtorsOp, LLVM::ComdatOp, |
| 2780 | LLVM::IFuncOp>(Val: &o) && |
| 2781 | !o.hasTrait<OpTrait::IsTerminator>() && |
| 2782 | failed(Result: translator.convertOperation(op&: o, builder&: llvmBuilder))) { |
| 2783 | return nullptr; |
| 2784 | } |
| 2785 | } |
| 2786 | |
| 2787 | // Operations in function bodies with symbolic references must be converted |
| 2788 | // after the top-level operations they refer to are declared, so we do it |
| 2789 | // last. |
| 2790 | if (failed(Result: translator.convertFunctions())) |
| 2791 | return nullptr; |
| 2792 | |
| 2793 | // Now that all MLIR blocks are resolved into LLVM ones, patch block address |
| 2794 | // constants to point to the correct blocks. |
| 2795 | if (failed(Result: translator.convertUnresolvedBlockAddress())) |
| 2796 | return nullptr; |
| 2797 | |
| 2798 | // Add the necessary debug info module flags, if they were not encoded in MLIR |
| 2799 | // beforehand. |
| 2800 | translator.debugTranslation->addModuleFlagsIfNotPresent(); |
| 2801 | |
| 2802 | // Let dialect translations complete their pending work (e.g. OpenMP region |
| 2803 | // outlining) prior to verifying the module. Index-based iteration tolerates |
| 2804 | // extensions created during finalization. |
| 2805 | for (size_t i = 0; i < translator.extensions.size(); ++i) |
| 2806 | if (failed(Result: translator.extensions.begin()[i].second->finalize())) |
| 2807 | return nullptr; |
| 2808 | |
| 2809 | if (!disableVerification && |
| 2810 | llvm::verifyModule(M: *translator.llvmModule, OS: &llvm::errs())) |
| 2811 | return nullptr; |
| 2812 | |
| 2813 | return std::move(translator.llvmModule); |
| 2814 | } |
| 2815 | |