| 1 | //====- LoweringHelpers.cpp - Lowering helper functions -------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file contains helper functions for lowering from CIR to LLVM or MLIR. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "clang/CIR/LoweringHelpers.h" |
| 14 | #include "mlir/Dialect/LLVMIR/LLVMArithOps.h" |
| 15 | #include "mlir/Dialect/LLVMIR/LLVMAttrs.h" |
| 16 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 17 | #include "mlir/Dialect/LLVMIR/LLVMModuleOps.h" |
| 18 | #include "mlir/Dialect/LLVMIR/LLVMTypes.h" |
| 19 | #include "mlir/IR/BuiltinTypes.h" |
| 20 | #include "mlir/IR/SymbolTable.h" |
| 21 | #include "mlir/Interfaces/DataLayoutInterfaces.h" |
| 22 | #include "clang/CIR/MissingFeatures.h" |
| 23 | |
| 24 | namespace { |
| 25 | /// A _BitInt(N) whose padded storage integer iM has a larger alloc size than |
| 26 | /// its M/8 store size is laid out by clang as a byte array, not a plain |
| 27 | /// integer (e.g. _BitInt(129) -> i192 with alloc size 32 != store size 24). |
| 28 | /// That "split" storage form is not yet implemented; lowerings must detect |
| 29 | /// it and report errorNYI rather than emit the wrong-sized integer. |
| 30 | bool isSplitStorageBitInt(cir::IntType ty, const mlir::DataLayout &dataLayout) { |
| 31 | if (!ty.isBitInt()) |
| 32 | return false; |
| 33 | unsigned storageBits = ty.getStorageTypeWidth(dataLayout); |
| 34 | auto storageTy = mlir::IntegerType::get(context: ty.getContext(), width: storageBits); |
| 35 | uint64_t storeSize = storageBits / 8; |
| 36 | uint64_t allocSize = |
| 37 | llvm::alignTo(Value: storeSize, Align: dataLayout.getTypeABIAlignment(t: storageTy)); |
| 38 | return allocSize != storeSize; |
| 39 | } |
| 40 | |
| 41 | /// Checks if `dataLayout` describes a big endian layout. |
| 42 | bool isBigEndian(const mlir::DataLayout &dataLayout) { |
| 43 | auto endiannessStr = |
| 44 | mlir::dyn_cast_or_null<mlir::StringAttr>(Val: dataLayout.getEndianness()); |
| 45 | return endiannessStr && endiannessStr == "big" ; |
| 46 | } |
| 47 | } // namespace |
| 48 | |
| 49 | mlir::Attribute getBitIntStorageAttr(mlir::ConversionPatternRewriter &rewriter, |
| 50 | cir::IntAttr attr, |
| 51 | const mlir::DataLayout &dataLayout) { |
| 52 | auto intTy = mlir::cast<cir::IntType>(Val: attr.getType()); |
| 53 | unsigned storageBits = intTy.getStorageTypeWidth(dataLayout); |
| 54 | llvm::APInt val = attr.getValue(); |
| 55 | val = intTy.isSigned() ? val.sext(width: storageBits) : val.zext(width: storageBits); |
| 56 | |
| 57 | if (!isSplitStorageBitInt(ty: intTy, dataLayout)) |
| 58 | return rewriter.getIntegerAttr( |
| 59 | type: mlir::IntegerType::get(context: intTy.getContext(), width: storageBits), value: val); |
| 60 | |
| 61 | if (isBigEndian(dataLayout)) |
| 62 | val = val.byteSwap(); |
| 63 | |
| 64 | // If we have to do split storage, we are an array of bytes. Split this up |
| 65 | // into the array that matches convertTypeForMemory. |
| 66 | unsigned numBytes = storageBits / 8; |
| 67 | llvm::SmallVector<mlir::APInt> bytes; |
| 68 | bytes.reserve(N: numBytes); |
| 69 | for (unsigned i = 0; i != numBytes; ++i) |
| 70 | bytes.emplace_back(Args: 8, Args: val.extractBitsAsZExtValue(numBits: 8, bitPosition: i * 8)); |
| 71 | |
| 72 | auto i8Ty = mlir::IntegerType::get(context: intTy.getContext(), width: 8); |
| 73 | return mlir::DenseElementsAttr::get( |
| 74 | type: mlir::RankedTensorType::get(shape: {numBytes}, elementType: i8Ty), values: bytes); |
| 75 | } |
| 76 | |
| 77 | mlir::Type convertTypeForMemory(const mlir::TypeConverter &converter, |
| 78 | mlir::DataLayout const &dataLayout, |
| 79 | mlir::Type type, bool isForHLSL) { |
| 80 | // TODO(cir): Handle other types similarly to clang's codegen |
| 81 | // convertTypeForMemory |
| 82 | if (mlir::isa<cir::BoolType>(Val: type)) { |
| 83 | return mlir::IntegerType::get(context: type.getContext(), |
| 84 | width: dataLayout.getTypeSizeInBits(t: type)); |
| 85 | } |
| 86 | |
| 87 | if (auto matrixTy = mlir::dyn_cast<cir::MatrixType>(Val&: type)) { |
| 88 | if (isForHLSL) { |
| 89 | unsigned vecLen = matrixTy.getNumRows(); |
| 90 | unsigned arrayLen = matrixTy.getNumColumns(); |
| 91 | mlir::Type elementType = converter.convertType(t: matrixTy.getElementType()); |
| 92 | auto vecTy = mlir::VectorType::get(shape: vecLen, elementType); |
| 93 | return mlir::LLVM::LLVMArrayType::get(elementType: vecTy, numElements: arrayLen); |
| 94 | } |
| 95 | |
| 96 | uint64_t size = matrixTy.getNumRows() * matrixTy.getNumColumns(); |
| 97 | mlir::Type elementType = converter.convertType(t: matrixTy.getElementType()); |
| 98 | return mlir::LLVM::LLVMArrayType::get(elementType, numElements: size); |
| 99 | } |
| 100 | |
| 101 | if (auto vecTy = mlir::dyn_cast<cir::VectorType>(Val&: type)) { |
| 102 | if (mlir::isa<cir::BoolType>(Val: vecTy.getElementType())) { |
| 103 | assert(!cir::MissingFeatures::hlsl()); |
| 104 | return mlir::IntegerType::get(context: type.getContext(), |
| 105 | width: vecTy.getBoolStorageWidth()); |
| 106 | } |
| 107 | } |
| 108 | |
| 109 | // _BitInt(N) keeps its literal width as a value but is stored in a padded |
| 110 | // integer iM in memory, the same way bool is i1 as a value and i8 in |
| 111 | // memory. When iM's own LLVM alloc size would overshoot the AST-exact |
| 112 | // M/8 byte count (isSplitStorageBitInt), use a byte array instead so the |
| 113 | // size stays exact. |
| 114 | if (auto intTy = mlir::dyn_cast<cir::IntType>(Val&: type); |
| 115 | intTy && intTy.isBitInt()) { |
| 116 | if (isSplitStorageBitInt(ty: intTy, dataLayout)) |
| 117 | return mlir::LLVM::LLVMArrayType::get( |
| 118 | elementType: mlir::IntegerType::get(context: type.getContext(), width: 8), |
| 119 | numElements: intTy.getStorageTypeWidth(dataLayout) / 8); |
| 120 | return mlir::IntegerType::get(context: type.getContext(), |
| 121 | width: intTy.getStorageTypeWidth(dataLayout)); |
| 122 | } |
| 123 | |
| 124 | return converter.convertType(t: type); |
| 125 | } |
| 126 | |
| 127 | mlir::Type convertTypeForLoadStore(const mlir::TypeConverter &converter, |
| 128 | mlir::DataLayout const &dataLayout, |
| 129 | mlir::Type type) { |
| 130 | // A split-storage _BitInt's memory type is a byte array (see |
| 131 | // convertTypeForMemory), but a load/store must still access the whole |
| 132 | // integer as-is. |
| 133 | if (auto intTy = mlir::dyn_cast<cir::IntType>(Val&: type); |
| 134 | intTy && intTy.isBitInt()) |
| 135 | return mlir::IntegerType::get(context: type.getContext(), |
| 136 | width: intTy.getStorageTypeWidth(dataLayout)); |
| 137 | |
| 138 | // Convert the Matrix type to a vector type (the value type of |
| 139 | // MatrixType), if it points to a array (the memory type of MatrixType). |
| 140 | if (auto matrixTy = mlir::dyn_cast<cir::MatrixType>(Val&: type)) { |
| 141 | uint64_t size = matrixTy.getNumRows() * matrixTy.getNumColumns(); |
| 142 | mlir::Type elemTy = converter.convertType(t: matrixTy.getElementType()); |
| 143 | return mlir::VectorType::get(shape: size, elementType: elemTy); |
| 144 | } |
| 145 | |
| 146 | return convertTypeForMemory(converter, dataLayout, type); |
| 147 | } |
| 148 | |
| 149 | static unsigned getIntOrBoolBitWidth(mlir::Type ty) { |
| 150 | if (auto intTy = mlir::dyn_cast<cir::IntType>(Val&: ty)) |
| 151 | return intTy.getWidth(); |
| 152 | assert(mlir::isa<cir::BoolType>(ty) && |
| 153 | "expected CIR integer or bool element type" ); |
| 154 | return 1; |
| 155 | } |
| 156 | |
| 157 | mlir::DenseElementsAttr |
| 158 | convertStringAttrToDenseElementsAttr(cir::ConstArrayAttr attr, |
| 159 | mlir::Type type) { |
| 160 | const auto stringAttr = mlir::cast<mlir::StringAttr>(Val: attr.getElts()); |
| 161 | const auto arrayTy = mlir::cast<cir::ArrayType>(Val: attr.getType()); |
| 162 | const unsigned totalSize = arrayTy.getSize(); |
| 163 | const unsigned trailingZeros = attr.getTrailingZerosNum(); |
| 164 | assert(stringAttr.size() + trailingZeros == totalSize && |
| 165 | "string const_array size must match explicit elements plus " |
| 166 | "trailing_zeros" ); |
| 167 | |
| 168 | const unsigned bitWidth = getIntOrBoolBitWidth(ty: arrayTy.getElementType()); |
| 169 | llvm::SmallVector<mlir::APInt> values; |
| 170 | values.reserve(N: totalSize); |
| 171 | |
| 172 | // String bytes are raw values; interpret each as an unsigned byte so a |
| 173 | // high-bit char (>= 0x80) does not sign-extend to a value that overflows |
| 174 | // the element bit width when constructing the APInt. |
| 175 | for (const char element : stringAttr) |
| 176 | values.emplace_back(Args: bitWidth, Args: static_cast<unsigned char>(element)); |
| 177 | |
| 178 | values.insert(I: values.end(), NumToInsert: trailingZeros, Elt: mlir::APInt::getZero(numBits: bitWidth)); |
| 179 | |
| 180 | return mlir::DenseElementsAttr::get( |
| 181 | type: mlir::RankedTensorType::get(shape: {totalSize}, elementType: type), values: llvm::ArrayRef(values)); |
| 182 | } |
| 183 | |
| 184 | template <> mlir::APInt getZeroInitFromType(mlir::Type ty) { |
| 185 | const auto intTy = mlir::cast<mlir::IntegerType>(Val&: ty); |
| 186 | return mlir::APInt::getZero(numBits: intTy.getWidth()); |
| 187 | } |
| 188 | |
| 189 | template <> mlir::APFloat getZeroInitFromType(mlir::Type ty) { |
| 190 | auto fpTy = mlir::cast<mlir::FloatType>(Val&: ty); |
| 191 | return mlir::APFloat::getZero(Sem: fpTy.getFloatSemantics()); |
| 192 | } |
| 193 | |
| 194 | /// \param attr the ConstArrayAttr to convert |
| 195 | /// \param values the output parameter, the values array to fill |
| 196 | /// \param currentDims the shpae of tensor we're going to convert to |
| 197 | /// \param dimIndex the current dimension we're processing |
| 198 | /// \param currentIndex the current index in the values array |
| 199 | /// \param convertedElementType the lowered storage type of each element |
| 200 | template <typename AttrTy, typename StorageTy> |
| 201 | bool convertToDenseElementsAttrImpl( |
| 202 | cir::ConstArrayAttr attr, llvm::SmallVectorImpl<StorageTy> &values, |
| 203 | const llvm::SmallVectorImpl<int64_t> ¤tDims, int64_t dimIndex, |
| 204 | int64_t currentIndex, mlir::Type convertedElementType) { |
| 205 | if (auto stringAttr = mlir::dyn_cast<mlir::StringAttr>(Val: attr.getElts())) { |
| 206 | if (auto arrayType = mlir::dyn_cast<cir::ArrayType>(Val: attr.getType())) { |
| 207 | for (auto element : stringAttr) { |
| 208 | auto intAttr = cir::IntAttr::get(type: arrayType.getElementType(), value: element); |
| 209 | values[currentIndex++] = mlir::dyn_cast<AttrTy>(intAttr).getValue(); |
| 210 | } |
| 211 | // Remaining slots are trailing zeros; values was zero-initialized. |
| 212 | currentIndex += attr.getTrailingZerosNum(); |
| 213 | return true; |
| 214 | } |
| 215 | } |
| 216 | |
| 217 | dimIndex++; |
| 218 | std::size_t elementsSizeInCurrentDim = 1; |
| 219 | for (std::size_t i = dimIndex; i < currentDims.size(); i++) |
| 220 | elementsSizeInCurrentDim *= currentDims[i]; |
| 221 | |
| 222 | auto arrayAttr = mlir::cast<mlir::ArrayAttr>(Val: attr.getElts()); |
| 223 | for (auto eltAttr : arrayAttr) { |
| 224 | if constexpr (std::is_same_v<StorageTy, mlir::APInt>) { |
| 225 | if (auto boolAttr = mlir::dyn_cast<cir::BoolAttr>(Val&: eltAttr)) { |
| 226 | unsigned bitWidth = |
| 227 | mlir::cast<mlir::IntegerType>(Val&: convertedElementType).getWidth(); |
| 228 | values[currentIndex++] = |
| 229 | llvm::APInt(bitWidth, static_cast<uint64_t>(boolAttr.getValue())); |
| 230 | continue; |
| 231 | } |
| 232 | } |
| 233 | if (auto valueAttr = mlir::dyn_cast<AttrTy>(eltAttr)) { |
| 234 | values[currentIndex++] = valueAttr.getValue(); |
| 235 | continue; |
| 236 | } |
| 237 | |
| 238 | if (auto subArrayAttr = mlir::dyn_cast<cir::ConstArrayAttr>(Val&: eltAttr)) { |
| 239 | convertToDenseElementsAttrImpl<AttrTy>(subArrayAttr, values, currentDims, |
| 240 | dimIndex, currentIndex, |
| 241 | convertedElementType); |
| 242 | currentIndex += elementsSizeInCurrentDim; |
| 243 | continue; |
| 244 | } |
| 245 | |
| 246 | if (mlir::isa<cir::ZeroAttr, cir::UndefAttr>(Val: eltAttr)) { |
| 247 | currentIndex += elementsSizeInCurrentDim; |
| 248 | continue; |
| 249 | } |
| 250 | |
| 251 | // A global view can be the result of pointer conversions, so we can't |
| 252 | // represent them as an APInt/APFloat. So give up if we see one. |
| 253 | if (auto global = mlir::dyn_cast<cir::GlobalViewAttr>(Val&: eltAttr)) |
| 254 | return false; |
| 255 | |
| 256 | llvm_unreachable("unknown element in ConstArrayAttr" ); |
| 257 | } |
| 258 | return true; |
| 259 | } |
| 260 | |
| 261 | template <typename AttrTy, typename StorageTy> |
| 262 | std::optional<mlir::DenseElementsAttr> |
| 263 | convertToDenseElementsAttr(cir::ConstArrayAttr attr, |
| 264 | const llvm::SmallVectorImpl<int64_t> &dims, |
| 265 | mlir::Type convertedElementType) { |
| 266 | unsigned vectorSize = 1; |
| 267 | for (auto dim : dims) |
| 268 | vectorSize *= dim; |
| 269 | llvm::SmallVector<StorageTy> values( |
| 270 | vectorSize, getZeroInitFromType<StorageTy>(convertedElementType)); |
| 271 | |
| 272 | if (!convertToDenseElementsAttrImpl<AttrTy>(attr, values, dims, |
| 273 | /*dimIndex=*/0, |
| 274 | /*currentIndex=*/0, |
| 275 | convertedElementType)) |
| 276 | return std::nullopt; |
| 277 | |
| 278 | return mlir::DenseElementsAttr::get( |
| 279 | mlir::RankedTensorType::get(shape: dims, elementType: convertedElementType), |
| 280 | llvm::ArrayRef(values)); |
| 281 | } |
| 282 | |
| 283 | /// Return true when \p gv can be lowered to a \c FlatSymbolRefAttr leaf without |
| 284 | /// addrspacecast or bitcast (mirrors \c CIRAttrToValue::visitCirAttr). |
| 285 | static bool globalViewMatchesPointerLeaf( |
| 286 | cir::GlobalViewAttr gv, mlir::ModuleOp moduleOp, |
| 287 | mlir::SymbolTableCollection &symbolTables, |
| 288 | const mlir::TypeConverter *converter, |
| 289 | const CIRPreConversionTypeMap &preConversionTypes) { |
| 290 | if (gv.getIndices() || mlir::isa<cir::IntType, cir::VPtrType>(Val: gv.getType())) |
| 291 | return false; |
| 292 | |
| 293 | auto ptrTy = mlir::dyn_cast<cir::PointerType>(Val: gv.getType()); |
| 294 | if (!ptrTy) |
| 295 | return false; |
| 296 | |
| 297 | unsigned sourceAddrSpace = 0; |
| 298 | mlir::Type sourceType; |
| 299 | auto sourceSymbol = symbolTables.lookupSymbolIn(symbolTableOp: moduleOp, name: gv.getSymbol()); |
| 300 | if (auto llvmSymbol = mlir::dyn_cast<mlir::LLVM::GlobalOp>(Val: sourceSymbol)) { |
| 301 | if (mlir::Type preType = preConversionTypes.lookup(Key: llvmSymbol.getSymName())) |
| 302 | sourceType = converter->convertType(t: preType); |
| 303 | else |
| 304 | sourceType = llvmSymbol.getType(); |
| 305 | sourceAddrSpace = llvmSymbol.getAddrSpace(); |
| 306 | } else if (auto cirSymbol = mlir::dyn_cast<cir::GlobalOp>(Val: sourceSymbol)) { |
| 307 | sourceType = converter->convertType(t: cirSymbol.getSymType()); |
| 308 | if (auto targetAS = mlir::dyn_cast_if_present<cir::TargetAddressSpaceAttr>( |
| 309 | Val: cirSymbol.getAddrSpaceAttr())) |
| 310 | sourceAddrSpace = targetAS.getValue(); |
| 311 | } else { |
| 312 | // cir.func and other symbols not yet lowered to globals cannot be used as |
| 313 | // bulk constant leaves; those cases keep the insertvalue fallback. |
| 314 | return false; |
| 315 | } |
| 316 | |
| 317 | auto llvmDstTy = converter->convertType<mlir::LLVM::LLVMPointerType>(t: ptrTy); |
| 318 | if (llvmDstTy.getAddressSpace() != sourceAddrSpace) |
| 319 | return false; |
| 320 | |
| 321 | mlir::Type llvmEltTy = converter->convertType(t: ptrTy.getPointee()); |
| 322 | if (llvmEltTy == sourceType) |
| 323 | return true; |
| 324 | if (auto arrTy = mlir::dyn_cast<mlir::LLVM::LLVMArrayType>(Val&: sourceType)) |
| 325 | return llvmEltTy == arrTy.getElementType(); |
| 326 | return false; |
| 327 | } |
| 328 | |
| 329 | /// Lower a single pointer-element of a \c cir.const_array to an LLVM-dialect |
| 330 | /// constant leaf suitable for a bulk \c llvm.mlir.constant. Only handles |
| 331 | /// address-of-global without indices and null pointers; indexed global views |
| 332 | /// must use the per-element \c llvm.insertvalue fallback. |
| 333 | static std::optional<mlir::Attribute> |
| 334 | lowerPointerElementAttr(mlir::Attribute elt, mlir::MLIRContext *ctx, |
| 335 | mlir::ModuleOp moduleOp, |
| 336 | mlir::SymbolTableCollection &symbolTables, |
| 337 | const mlir::TypeConverter *converter, |
| 338 | const CIRPreConversionTypeMap &preConversionTypes) { |
| 339 | if (auto gv = mlir::dyn_cast<cir::GlobalViewAttr>(Val&: elt)) { |
| 340 | if (!moduleOp || |
| 341 | !globalViewMatchesPointerLeaf(gv, moduleOp, symbolTables, converter, |
| 342 | preConversionTypes)) |
| 343 | return std::nullopt; |
| 344 | return gv.getSymbol(); |
| 345 | } |
| 346 | if (auto nullPtr = mlir::dyn_cast<cir::ConstPtrAttr>(Val&: elt)) { |
| 347 | if (nullPtr.isNullValue()) |
| 348 | return mlir::LLVM::ZeroAttr::get(ctx); |
| 349 | return std::nullopt; |
| 350 | } |
| 351 | return std::nullopt; |
| 352 | } |
| 353 | |
| 354 | static bool containsPoison(mlir::Attribute attr) { |
| 355 | if (mlir::isa<cir::PoisonAttr>(Val: attr)) |
| 356 | return true; |
| 357 | if (auto elts = mlir::dyn_cast<mlir::ArrayAttr>(Val&: attr)) |
| 358 | return llvm::any_of(Range&: elts, P: containsPoison); |
| 359 | if (auto constArr = mlir::dyn_cast<cir::ConstArrayAttr>(Val&: attr)) { |
| 360 | if (mlir::isa<mlir::StringAttr>(Val: constArr.getElts())) |
| 361 | return false; |
| 362 | if (auto elts = mlir::dyn_cast<mlir::ArrayAttr>(Val: constArr.getElts())) |
| 363 | return llvm::any_of(Range&: elts, P: containsPoison); |
| 364 | } |
| 365 | return false; |
| 366 | } |
| 367 | |
| 368 | /// Block-address attributes (address-of-label and label differences) are |
| 369 | /// lowered to relocation expressions that cannot be materialized as part of a |
| 370 | /// dense/aggregate constant attribute; they require the per-element |
| 371 | /// insertvalue region lowering. Return true if \p attr contains any such |
| 372 | /// element. |
| 373 | static bool containsBlockAddress(mlir::Attribute attr) { |
| 374 | if (mlir::isa<cir::BlockAddrInfoAttr, cir::BlockAddrDiffAttr>(Val: attr)) |
| 375 | return true; |
| 376 | if (auto elts = mlir::dyn_cast<mlir::ArrayAttr>(Val&: attr)) |
| 377 | return llvm::any_of(Range&: elts, P: containsBlockAddress); |
| 378 | if (auto constArr = mlir::dyn_cast<cir::ConstArrayAttr>(Val&: attr)) { |
| 379 | if (mlir::isa<mlir::StringAttr>(Val: constArr.getElts())) |
| 380 | return false; |
| 381 | if (auto elts = mlir::dyn_cast<mlir::ArrayAttr>(Val: constArr.getElts())) |
| 382 | return llvm::any_of(Range&: elts, P: containsBlockAddress); |
| 383 | } |
| 384 | return false; |
| 385 | } |
| 386 | |
| 387 | /// The fewest trailing zeros that adjustGlobalArrayTypeForInit splits off an |
| 388 | /// array constant into a zero-initialized tail, as in classic codegen. |
| 389 | static constexpr int minSplitTrailingZeros = 8; |
| 390 | |
| 391 | static bool shouldSplitTrailingZeros(cir::ConstArrayAttr attr) { |
| 392 | auto elts = mlir::dyn_cast<mlir::ArrayAttr>(Val: attr.getElts()); |
| 393 | return elts && !elts.empty() && |
| 394 | attr.getTrailingZerosNum() >= minSplitTrailingZeros; |
| 395 | } |
| 396 | |
| 397 | static bool containsSplitTrailingZeros(mlir::Attribute attr) { |
| 398 | if (auto record = mlir::dyn_cast<cir::ConstRecordAttr>(Val&: attr)) |
| 399 | return llvm::any_of(Range: record.getMembers(), P: containsSplitTrailingZeros); |
| 400 | auto array = mlir::dyn_cast<cir::ConstArrayAttr>(Val&: attr); |
| 401 | if (!array) |
| 402 | return false; |
| 403 | if (shouldSplitTrailingZeros(attr: array)) |
| 404 | return true; |
| 405 | auto elts = mlir::dyn_cast<mlir::ArrayAttr>(Val: array.getElts()); |
| 406 | return elts && llvm::any_of(Range&: elts, P: containsSplitTrailingZeros); |
| 407 | } |
| 408 | |
| 409 | mlir::Attribute spellOutSplitTrailingZeros(mlir::Attribute attr) { |
| 410 | if (!containsSplitTrailingZeros(attr)) |
| 411 | return attr; |
| 412 | |
| 413 | if (auto record = mlir::dyn_cast<cir::ConstRecordAttr>(Val&: attr)) { |
| 414 | auto members = |
| 415 | llvm::map_to_vector(C: record.getMembers(), F&: spellOutSplitTrailingZeros); |
| 416 | return cir::ConstRecordAttr::get( |
| 417 | type: mlir::cast<cir::RecordType>(Val: record.getType()), |
| 418 | members: mlir::ArrayAttr::get(context: attr.getContext(), value: members)); |
| 419 | } |
| 420 | |
| 421 | auto array = mlir::cast<cir::ConstArrayAttr>(Val&: attr); |
| 422 | auto arrayTy = mlir::cast<cir::ArrayType>(Val: array.getType()); |
| 423 | auto elts = llvm::map_to_vector(C: mlir::cast<mlir::ArrayAttr>(Val: array.getElts()), |
| 424 | F&: spellOutSplitTrailingZeros); |
| 425 | if (!shouldSplitTrailingZeros(attr: array)) |
| 426 | return cir::ConstArrayAttr::get( |
| 427 | type: arrayTy, elts: mlir::ArrayAttr::get(context: attr.getContext(), value: elts), |
| 428 | trailingZerosNum: array.getTrailingZerosNum()); |
| 429 | elts.append(NumInputs: array.getTrailingZerosNum(), |
| 430 | Elt: cir::ZeroAttr::get(type: arrayTy.getElementType())); |
| 431 | return cir::ConstArrayAttr::get(type: arrayTy, |
| 432 | elts: mlir::ArrayAttr::get(context: attr.getContext(), value: elts), |
| 433 | /*trailingZerosNum=*/0); |
| 434 | } |
| 435 | |
| 436 | static std::optional<mlir::Attribute> lowerConstRecordMemberAttr( |
| 437 | mlir::Attribute attr, mlir::SymbolTableCollection &symbolTables, |
| 438 | const mlir::TypeConverter *converter, |
| 439 | const CIRPreConversionTypeMap &preConversionTypes, mlir::ModuleOp moduleOp); |
| 440 | |
| 441 | /// Lower \p constArr, which is or contains an array split by |
| 442 | /// adjustGlobalArrayTypeForInit, element by element into the shape of its |
| 443 | /// adjusted type. A dense attribute would span the split-off zeros. |
| 444 | static std::optional<mlir::Attribute> |
| 445 | lowerConstArrayAttrByElement(cir::ConstArrayAttr constArr, |
| 446 | mlir::SymbolTableCollection &symbolTables, |
| 447 | const mlir::TypeConverter *converter, |
| 448 | const CIRPreConversionTypeMap &preConversionTypes, |
| 449 | mlir::ModuleOp moduleOp) { |
| 450 | mlir::MLIRContext *ctx = constArr.getContext(); |
| 451 | auto arrayTy = mlir::cast<cir::ArrayType>(Val: constArr.getType()); |
| 452 | auto elts = mlir::cast<mlir::ArrayAttr>(Val: constArr.getElts()); |
| 453 | mlir::Type adjustedTy = |
| 454 | adjustGlobalTypeForInit(llvmType: converter->convertType(t: arrayTy), init: constArr, |
| 455 | converter: *converter, dataLayout: mlir::DataLayout(moduleOp)); |
| 456 | |
| 457 | llvm::SmallVector<mlir::Attribute> lowered; |
| 458 | // Fewer members than elements: the elements were grouped into one array. |
| 459 | auto structTy = mlir::dyn_cast<mlir::LLVM::LLVMStructType>(Val&: adjustedTy); |
| 460 | if (structTy && structTy.getBody().size() < elts.size()) { |
| 461 | auto prefix = cir::ConstArrayAttr::get( |
| 462 | type: cir::ArrayType::get(elementType: arrayTy.getElementType(), size: elts.size()), elts); |
| 463 | std::optional<mlir::Attribute> loweredPrefix = lowerConstArrayAttr( |
| 464 | constArr: prefix, symbolTables, converter, preConversionTypes, moduleOp); |
| 465 | if (!loweredPrefix) |
| 466 | return std::nullopt; |
| 467 | lowered.push_back(Elt: *loweredPrefix); |
| 468 | } else { |
| 469 | for (mlir::Attribute elt : elts) { |
| 470 | std::optional<mlir::Attribute> loweredElt = lowerConstRecordMemberAttr( |
| 471 | attr: elt, symbolTables, converter, preConversionTypes, moduleOp); |
| 472 | if (!loweredElt) |
| 473 | return std::nullopt; |
| 474 | lowered.push_back(Elt: *loweredElt); |
| 475 | } |
| 476 | } |
| 477 | |
| 478 | // A split array ends in one zero-initialized member; otherwise its few |
| 479 | // trailing zeros are spelled out. |
| 480 | lowered.append( |
| 481 | NumInputs: shouldSplitTrailingZeros(attr: constArr) ? 1 : constArr.getTrailingZerosNum(), |
| 482 | Elt: mlir::LLVM::ZeroAttr::get(ctx)); |
| 483 | return mlir::ArrayAttr::get(context: ctx, value: lowered); |
| 484 | } |
| 485 | |
| 486 | std::optional<mlir::Attribute> |
| 487 | lowerConstArrayAttr(cir::ConstArrayAttr constArr, |
| 488 | mlir::SymbolTableCollection &symbolTables, |
| 489 | const mlir::TypeConverter *converter, |
| 490 | const CIRPreConversionTypeMap &preConversionTypes, |
| 491 | mlir::ModuleOp moduleOp) { |
| 492 | // Ensure ConstArrayAttr has a type. |
| 493 | const auto typedConstArr = mlir::cast<mlir::TypedAttr>(Val&: constArr); |
| 494 | |
| 495 | // Ensure ConstArrayAttr type is a ArrayType. |
| 496 | const auto cirArrayType = mlir::cast<cir::ArrayType>(Val: typedConstArr.getType()); |
| 497 | |
| 498 | // Is a ConstArrayAttr with an cir::ArrayType: fetch element type. |
| 499 | mlir::Type type = cirArrayType; |
| 500 | auto dims = llvm::SmallVector<int64_t, 2>{}; |
| 501 | while (auto arrayType = mlir::dyn_cast<cir::ArrayType>(Val&: type)) { |
| 502 | dims.push_back(Elt: arrayType.getSize()); |
| 503 | type = arrayType.getElementType(); |
| 504 | } |
| 505 | |
| 506 | if (containsPoison(attr: constArr)) |
| 507 | return std::nullopt; |
| 508 | |
| 509 | // Block-address initializers cannot be represented as a dense/aggregate |
| 510 | // constant attribute; fall back to the per-element insertvalue lowering. |
| 511 | if (containsBlockAddress(attr: constArr)) |
| 512 | return std::nullopt; |
| 513 | |
| 514 | if (containsSplitTrailingZeros(attr: constArr)) |
| 515 | return lowerConstArrayAttrByElement(constArr, symbolTables, converter, |
| 516 | preConversionTypes, moduleOp); |
| 517 | |
| 518 | if (mlir::isa<mlir::StringAttr>(Val: constArr.getElts())) |
| 519 | return convertStringAttrToDenseElementsAttr(attr: constArr, |
| 520 | type: converter->convertType(t: type)); |
| 521 | if (mlir::isa<cir::IntType>(Val: type)) { |
| 522 | // A _BitInt array element is stored in a padded integer in memory; the |
| 523 | // dense-attribute path here cannot express that widening, so fall back to |
| 524 | // the insertvalue region, which widens each element via emitToMemory. |
| 525 | if (mlir::cast<cir::IntType>(Val&: type).isBitInt()) |
| 526 | return std::nullopt; |
| 527 | return convertToDenseElementsAttr<cir::IntAttr, mlir::APInt>( |
| 528 | attr: constArr, dims, convertedElementType: converter->convertType(t: type)); |
| 529 | } |
| 530 | |
| 531 | if (mlir::isa<cir::BoolType>(Val: type)) { |
| 532 | mlir::Type storageType = |
| 533 | convertTypeForMemory(converter: *converter, dataLayout: mlir::DataLayout(moduleOp), type); |
| 534 | return convertToDenseElementsAttr<cir::IntAttr, mlir::APInt>(attr: constArr, dims, |
| 535 | convertedElementType: storageType); |
| 536 | } |
| 537 | |
| 538 | if (mlir::isa<cir::FPTypeInterface>(Val: type)) |
| 539 | return convertToDenseElementsAttr<cir::FPAttr, mlir::APFloat>( |
| 540 | attr: constArr, dims, convertedElementType: converter->convertType(t: type)); |
| 541 | |
| 542 | if (mlir::isa<cir::PointerType>(Val: type)) { |
| 543 | // FIXME: Pointer arrays with trailing_zeros (null-sentinel tables) fall |
| 544 | // through to the insertvalue path for now. |
| 545 | if (constArr.getTrailingZerosNum() > 0) |
| 546 | return std::nullopt; |
| 547 | auto eltsArr = mlir::dyn_cast<mlir::ArrayAttr>(Val: constArr.getElts()); |
| 548 | if (!eltsArr) |
| 549 | return std::nullopt; |
| 550 | llvm::SmallVector<mlir::Attribute> lowered; |
| 551 | lowered.reserve(N: eltsArr.size()); |
| 552 | mlir::MLIRContext *ctx = constArr.getContext(); |
| 553 | for (mlir::Attribute elt : eltsArr) { |
| 554 | std::optional<mlir::Attribute> llvmElt = lowerPointerElementAttr( |
| 555 | elt, ctx, moduleOp, symbolTables, converter, preConversionTypes); |
| 556 | if (!llvmElt) |
| 557 | return std::nullopt; |
| 558 | lowered.push_back(Elt: *llvmElt); |
| 559 | } |
| 560 | return mlir::ArrayAttr::get(context: ctx, value: lowered); |
| 561 | } |
| 562 | |
| 563 | if (mlir::isa<cir::RecordType>(Val: type)) { |
| 564 | // A record type is just an array of the element values. |
| 565 | auto eltsArr = mlir::dyn_cast<mlir::ArrayAttr>(Val: constArr.getElts()); |
| 566 | if (!eltsArr) |
| 567 | return std::nullopt; |
| 568 | |
| 569 | llvm::SmallVector<mlir::Attribute> loweredElts; |
| 570 | loweredElts.reserve(N: cirArrayType.getSize()); |
| 571 | |
| 572 | for (mlir::Attribute elt : eltsArr) { |
| 573 | std::optional<mlir::Attribute> llvmElt = lowerConstRecordMemberAttr( |
| 574 | attr: elt, symbolTables, converter, preConversionTypes, moduleOp); |
| 575 | if (!llvmElt) |
| 576 | return std::nullopt; |
| 577 | loweredElts.push_back(Elt: *llvmElt); |
| 578 | } |
| 579 | |
| 580 | // Remaining elts are either going to be padding (and should be undef), or |
| 581 | // trailing-zeros. We can't really tell the difference as CIR lowering |
| 582 | // doesn't differentiate anyway, so just zero-fill them. |
| 583 | while (loweredElts.size() < cirArrayType.getSize()) |
| 584 | loweredElts.push_back(Elt: mlir::LLVM::ZeroAttr::get(ctx: constArr.getContext())); |
| 585 | return mlir::ArrayAttr::get(context: constArr.getContext(), value: loweredElts); |
| 586 | } |
| 587 | |
| 588 | return std::nullopt; |
| 589 | } |
| 590 | |
| 591 | /// Lower a constant attribute that initializes a single member of a record (or |
| 592 | /// a leaf of a nested aggregate) to an LLVM-dialect attribute that can be |
| 593 | /// attached directly to an \c llvm.mlir.global, avoiding an insertvalue |
| 594 | /// initializer region. Returns \c std::nullopt when the attribute cannot be |
| 595 | /// represented as a single constant attribute (e.g. an indexed |
| 596 | /// \c GlobalViewAttr), in which case the caller falls back to the region-based |
| 597 | /// lowering. |
| 598 | static std::optional<mlir::Attribute> |
| 599 | lowerConstRecordMemberAttr(mlir::Attribute attr, |
| 600 | mlir::SymbolTableCollection &symbolTables, |
| 601 | const mlir::TypeConverter *converter, |
| 602 | const CIRPreConversionTypeMap &preConversionTypes, |
| 603 | mlir::ModuleOp moduleOp) { |
| 604 | mlir::MLIRContext *ctx = attr.getContext(); |
| 605 | |
| 606 | if (auto arrayAttr = mlir::dyn_cast<cir::ConstArrayAttr>(Val&: attr)) |
| 607 | return lowerConstArrayAttr(constArr: arrayAttr, symbolTables, converter, |
| 608 | preConversionTypes, moduleOp); |
| 609 | |
| 610 | if (auto recordAttr = mlir::dyn_cast<cir::ConstRecordAttr>(Val&: attr)) |
| 611 | return lowerConstRecordAttr(constRecord: recordAttr, symbolTables, converter, |
| 612 | preConversionTypes, moduleOp); |
| 613 | |
| 614 | if (mlir::isa<cir::ZeroAttr>(Val: attr)) |
| 615 | return mlir::LLVM::ZeroAttr::get(ctx); |
| 616 | |
| 617 | if (mlir::isa<cir::UndefAttr>(Val: attr)) |
| 618 | return mlir::LLVM::UndefAttr::get(ctx); |
| 619 | |
| 620 | if (auto intAttr = mlir::dyn_cast<cir::IntAttr>(Val&: attr)) { |
| 621 | // A _BitInt member is stored in a padded integer in memory; defer to the |
| 622 | // insertvalue region (CIRAttrToValue), which performs the extension. |
| 623 | if (mlir::cast<cir::IntType>(Val: intAttr.getType()).isBitInt()) |
| 624 | return std::nullopt; |
| 625 | return mlir::IntegerAttr::get(type: converter->convertType(t: intAttr.getType()), |
| 626 | value: intAttr.getValue()); |
| 627 | } |
| 628 | |
| 629 | if (auto boolAttr = mlir::dyn_cast<cir::BoolAttr>(Val&: attr)) { |
| 630 | mlir::Type storageType = convertTypeForMemory( |
| 631 | converter: *converter, dataLayout: mlir::DataLayout(moduleOp), type: boolAttr.getType()); |
| 632 | return mlir::IntegerAttr::get(type: storageType, value: boolAttr.getValue() ? 1 : 0); |
| 633 | } |
| 634 | |
| 635 | if (auto fpAttr = mlir::dyn_cast<cir::FPAttr>(Val&: attr)) |
| 636 | return mlir::FloatAttr::get(type: converter->convertType(t: fpAttr.getType()), |
| 637 | value: fpAttr.getValue()); |
| 638 | |
| 639 | // Null pointers and simple address-of-global references can be represented |
| 640 | // as constant attributes; anything more complex uses the region fallback. |
| 641 | return lowerPointerElementAttr(elt: attr, ctx, moduleOp, symbolTables, converter, |
| 642 | preConversionTypes); |
| 643 | } |
| 644 | |
| 645 | // Figure out if we want mark the new struct 'packed' if it isn't already. IF |
| 646 | // it is already, we have to keep that behavior. We pack it with logic similar |
| 647 | // to classic codegen, though will end up missing cases, since we don't want to |
| 648 | // change the type other than the FAM. |
| 649 | // We can do so if: |
| 650 | // 1- Packing it won't change any of the field offsets. |
| 651 | // 2- the non-padded struct would add padding beyond the |
| 652 | // flexible array member. We don't pack if the flexible array member manages |
| 653 | // to not cause trailing padding. |
| 654 | static bool shouldPackFAMStruct(const mlir::DataLayout &dataLayout, |
| 655 | llvm::ArrayRef<mlir::Type> members) { |
| 656 | uint64_t maxAlign = 1; |
| 657 | uint64_t totalSize = 0; |
| 658 | for (mlir::Type member : members) { |
| 659 | uint64_t align = dataLayout.getTypeABIAlignment(t: member); |
| 660 | maxAlign = std::max(a: maxAlign, b: align); |
| 661 | uint64_t size = dataLayout.getTypeSize(t: member).getFixedValue(); |
| 662 | |
| 663 | if (llvm::alignTo(Value: totalSize, Align: align) != totalSize) |
| 664 | return false; |
| 665 | |
| 666 | totalSize += size; |
| 667 | } |
| 668 | return llvm::alignTo(Value: totalSize, Align: maxAlign) != totalSize; |
| 669 | } |
| 670 | |
| 671 | // CIR supports flexible-array-members in its struct types. That is, a |
| 672 | // zero-length array as the last element, which can be initialized with an |
| 673 | // arbitrary number of elements. A ConstRecordAttr can be created with one of |
| 674 | // these, and our verifier allows it. However, the LLVM implementation does NOT |
| 675 | // permit this. So we have to replace this type in LLVM with special struct for |
| 676 | // this value. |
| 677 | // |
| 678 | // Additionally, the struct itself could contain a struct with a FAM or a union |
| 679 | // that needed adjustment, so it recurses to check those. If no such type has |
| 680 | // been found/no adjustment needed, this returns the type unchanged. |
| 681 | // |
| 682 | // Additionally, a union having an active member of not-the-largest |
| 683 | // alignment can cause the need for a small padding array. We also capture the |
| 684 | // original indices of the fields that had this padding prepended, so the |
| 685 | // lowerConstRecordAttr can later put in a 'zero' init there. |
| 686 | static mlir::Type adjustGlobalStructTypeForInit( |
| 687 | mlir::LLVM::LLVMStructType structTy, cir::ConstRecordAttr constRecord, |
| 688 | const mlir::TypeConverter &converter, const mlir::DataLayout &dataLayout, |
| 689 | llvm::SmallVectorImpl<unsigned> &paddingAddedIndexes, |
| 690 | mlir::Type *failedType) { |
| 691 | assert(paddingAddedIndexes.empty() && |
| 692 | "Not for accumulation, just single depth" ); |
| 693 | llvm::ArrayRef<mlir::Attribute> initMembers = |
| 694 | constRecord.getMembers().getValue(); |
| 695 | llvm::SmallVector<mlir::Type> origBody{structTy.getBody()}; |
| 696 | llvm::SmallVector<mlir::Type> newBody{}; |
| 697 | bool packed = structTy.isPacked(); |
| 698 | uint64_t curOffset = 0; |
| 699 | bool changed = false; |
| 700 | |
| 701 | // Recursively adjust each member. A member that is itself a union (or a |
| 702 | // struct containing one) lowers to a type that differs from its declared |
| 703 | // field type, and this struct has to adopt that adjusted type so the |
| 704 | // enclosing insertvalue chain type-checks. |
| 705 | for (auto [idx, member] : llvm::enumerate(First&: initMembers)) { |
| 706 | if (idx >= origBody.size()) |
| 707 | break; |
| 708 | mlir::Type adjusted = adjustGlobalTypeForInit( |
| 709 | llvmType: origBody[idx], init: member, converter, dataLayout, failedType); |
| 710 | if (!adjusted) |
| 711 | return {}; |
| 712 | unsigned adjustedAlign = dataLayout.getTypeABIAlignment(t: adjusted); |
| 713 | |
| 714 | if (adjusted != origBody[idx]) { |
| 715 | // We're always going to 'change' the layout if it has changed, but we |
| 716 | // need to see if there is new 'padding' that won't happen automatically |
| 717 | // here based on alignment. |
| 718 | unsigned origAlign = dataLayout.getTypeABIAlignment(t: origBody[idx]); |
| 719 | |
| 720 | uint64_t origOffset = |
| 721 | packed ? curOffset : llvm::alignTo(Value: curOffset, Align: origAlign); |
| 722 | uint64_t adjustedOffset = |
| 723 | packed ? curOffset : llvm::alignTo(Value: curOffset, Align: adjustedAlign); |
| 724 | |
| 725 | if (adjustedOffset != origOffset) { |
| 726 | // If the offset would change, we have to insert padding to make up for |
| 727 | // it. This should only happen since alignment will decrease with |
| 728 | // unions, so we should be able to assume adjusted-offset < origOffset? |
| 729 | assert(adjustedOffset < origOffset); |
| 730 | // Rather than just pad the difference between the offsets, we have to |
| 731 | // fill in since the end of the last field, else we leave room thanks to |
| 732 | // alignment between this field and the padding. |
| 733 | uint64_t difference = origOffset - curOffset; |
| 734 | newBody.push_back(Elt: mlir::LLVM::LLVMArrayType::get( |
| 735 | elementType: mlir::IntegerType::get(context: structTy.getContext(), width: 8), numElements: difference)); |
| 736 | paddingAddedIndexes.push_back(Elt: idx); |
| 737 | curOffset = origOffset; |
| 738 | } |
| 739 | changed = true; |
| 740 | } |
| 741 | newBody.push_back(Elt: adjusted); |
| 742 | |
| 743 | if (!packed) |
| 744 | curOffset = llvm::alignTo(Value: curOffset, Align: adjustedAlign); |
| 745 | curOffset += dataLayout.getTypeSize(t: adjusted).getFixedValue(); |
| 746 | } |
| 747 | |
| 748 | // CIR supports flexible-array-members in its struct types. That is, a |
| 749 | // zero-length array as the last element, which can be initialized with an |
| 750 | // arbitrary number of elements. A ConstRecordAttr can be created with one of |
| 751 | // these, and our verifier allows it. However, the LLVM implementation does |
| 752 | // NOT permit this, so we widen that trailing member to the initializer's |
| 753 | // array type (packing the struct if that changes the layout). |
| 754 | bool widenedFAM = false; |
| 755 | if (auto fam = |
| 756 | mlir::dyn_cast<mlir::LLVM::LLVMArrayType>(Val: structTy.getBody().back()); |
| 757 | fam && fam.getNumElements() == 0) { |
| 758 | mlir::Type lastInitType = |
| 759 | mlir::cast<mlir::TypedAttr>(Val: initMembers.back()).getType(); |
| 760 | if (mlir::cast<cir::ArrayType>(Val&: lastInitType).getSize() != 0) { |
| 761 | newBody.back() = adjustGlobalTypeForInit( |
| 762 | llvmType: converter.convertType(t: lastInitType), init: initMembers.back(), converter, |
| 763 | dataLayout, failedType); |
| 764 | if (!newBody.back()) |
| 765 | return {}; |
| 766 | packed = packed || shouldPackFAMStruct(dataLayout, members: newBody); |
| 767 | widenedFAM = true; |
| 768 | changed = true; |
| 769 | } |
| 770 | } |
| 771 | |
| 772 | if (!changed) |
| 773 | return structTy; |
| 774 | |
| 775 | // We've likely reduced the alignment, so make sure we put padding 'behind' |
| 776 | // it. We can skip this in the FAM case, since a Flexible array member is not |
| 777 | // allowed to be initialized unless it is the 'last' element. So it doesn't |
| 778 | // need to be padded out. |
| 779 | if (!widenedFAM) { |
| 780 | uint64_t declaredSize = dataLayout.getTypeSize(t: structTy).getFixedValue(); |
| 781 | assert(curOffset <= declaredSize && "body bigger than type?" ); |
| 782 | if (curOffset < declaredSize) |
| 783 | newBody.push_back(Elt: mlir::LLVM::LLVMArrayType::get( |
| 784 | elementType: mlir::IntegerType::get(context: structTy.getContext(), width: 8), |
| 785 | numElements: declaredSize - curOffset)); |
| 786 | } |
| 787 | |
| 788 | return mlir::LLVM::LLVMStructType::getLiteral(context: structTy.getContext(), types: newBody, |
| 789 | isPacked: packed); |
| 790 | } |
| 791 | |
| 792 | // A union constant only initializes its active member, but a union is lowered |
| 793 | // to its most-aligned member (its 'storage' type), which need not be the |
| 794 | // initialized one -- e.g. `union { char buf[16]; long cap; }` stores as `long` |
| 795 | // (higher alignment) but may be initialized through `buf`. So the storage type |
| 796 | // (and thus structTy) generally can't hold the active member's value. Rebuild |
| 797 | // an anonymous struct `{ <active member>, [pad x i8] }` that holds the active |
| 798 | // member followed by enough byte padding to span the union's full allocated |
| 799 | // size, mirroring classic codegen. |
| 800 | static mlir::Type adjustGlobalUnionTypeForInit( |
| 801 | mlir::LLVM::LLVMStructType structTy, cir::ConstRecordAttr constRecord, |
| 802 | const mlir::TypeConverter &converter, const mlir::DataLayout &dataLayout, |
| 803 | mlir::Type *failedType) { |
| 804 | |
| 805 | auto unionTy = mlir::cast<cir::UnionType>(Val: constRecord.getType()); |
| 806 | |
| 807 | // Unions can only initialize one field, so this has to be sizeof-one. |
| 808 | assert(constRecord.getMembers().size() == 1); |
| 809 | mlir::Attribute member = constRecord.getMembers()[0]; |
| 810 | mlir::Type memberTy = convertTypeForMemory( |
| 811 | converter, dataLayout, type: mlir::cast<mlir::TypedAttr>(Val&: member).getType()); |
| 812 | |
| 813 | // The active member may itself need adjusting (e.g. it is a nested union, or |
| 814 | // a struct containing one), so recurse before using its type below. |
| 815 | memberTy = adjustGlobalTypeForInit(llvmType: memberTy, init: member, converter, dataLayout, |
| 816 | failedType); |
| 817 | if (!memberTy) |
| 818 | return {}; |
| 819 | |
| 820 | // The converted union type is { storage, [padding] }, where storage is the |
| 821 | // union's most-aligned member. When the active member IS that storage type, |
| 822 | // the converted type already describes this initializer exactly (the pad we |
| 823 | // would compute equals the union's declared pad), so there is nothing to do. |
| 824 | if (memberTy == structTy.getBody().front()) |
| 825 | return structTy; |
| 826 | |
| 827 | uint64_t unionSize = dataLayout.getTypeSize(t: unionTy).getFixedValue(); |
| 828 | uint64_t initSize = dataLayout.getTypeSize(t: memberTy).getFixedValue(); |
| 829 | llvm::SmallVector<mlir::Type> newBody; |
| 830 | newBody.push_back(Elt: memberTy); |
| 831 | |
| 832 | // Pad to the union size unless a flexible array initializer is larger. |
| 833 | if (initSize < unionSize) |
| 834 | newBody.push_back(Elt: mlir::LLVM::LLVMArrayType::get( |
| 835 | elementType: mlir::IntegerType::get(context: structTy.getContext(), width: 8), |
| 836 | numElements: unionSize - initSize)); |
| 837 | |
| 838 | return mlir::LLVM::LLVMStructType::getLiteral(context: structTy.getContext(), types: newBody, |
| 839 | isPacked: unionTy.getPacked()); |
| 840 | } |
| 841 | |
| 842 | // Unions in an array can cause individual elements to be of different types. |
| 843 | // This function adjusts the array type and replaces it with a struct type if |
| 844 | // necessary, OR leaves it as a 'new' array type if necessary. |
| 845 | // |
| 846 | // Like classic codegen, an array ending in many zeros becomes a packed struct |
| 847 | // of its explicit elements and one zero-initialized array, so the zeros are |
| 848 | // never spelled out: |
| 849 | // <{ T0, T1, [N x T] }> |
| 850 | // or, when 8 or more elements share a type, |
| 851 | // <{ [M x T], [N x T] }> |
| 852 | static mlir::Type adjustGlobalArrayTypeForInit( |
| 853 | mlir::LLVM::LLVMArrayType arrayTy, cir::ConstArrayAttr arrayInit, |
| 854 | const mlir::TypeConverter &converter, const mlir::DataLayout &dataLayout, |
| 855 | mlir::Type *failedType) { |
| 856 | auto elts = mlir::dyn_cast<mlir::ArrayAttr>(Val: arrayInit.getElts()); |
| 857 | |
| 858 | if (!elts) |
| 859 | return arrayTy; |
| 860 | |
| 861 | mlir::Type origEltTy = arrayTy.getElementType(); |
| 862 | llvm::SmallVector<mlir::Type> adjustedElts; |
| 863 | adjustedElts.reserve(N: elts.size()); |
| 864 | bool changed = elts.size() > arrayTy.getNumElements(); |
| 865 | |
| 866 | for (mlir::Attribute elt : elts) { |
| 867 | mlir::Type adjusted = adjustGlobalTypeForInit(llvmType: origEltTy, init: elt, converter, |
| 868 | dataLayout, failedType); |
| 869 | if (!adjusted) |
| 870 | return {}; |
| 871 | changed |= adjusted != origEltTy; |
| 872 | adjustedElts.push_back(Elt: adjusted); |
| 873 | } |
| 874 | |
| 875 | if (shouldSplitTrailingZeros(attr: arrayInit)) { |
| 876 | mlir::Type commonEltTy = |
| 877 | llvm::all_equal(Range&: adjustedElts) ? adjustedElts.front() : mlir::Type{}; |
| 878 | llvm::SmallVector<mlir::Type> body; |
| 879 | if (commonEltTy && elts.size() >= minSplitTrailingZeros) |
| 880 | body.push_back( |
| 881 | Elt: mlir::LLVM::LLVMArrayType::get(elementType: commonEltTy, numElements: adjustedElts.size())); |
| 882 | else |
| 883 | body.append(RHS: adjustedElts); |
| 884 | body.push_back( |
| 885 | Elt: mlir::LLVM::LLVMArrayType::get(elementType: commonEltTy ? commonEltTy : origEltTy, |
| 886 | numElements: arrayInit.getTrailingZerosNum())); |
| 887 | return mlir::LLVM::LLVMStructType::getLiteral(context: arrayTy.getContext(), types: body, |
| 888 | /*isPacked=*/true); |
| 889 | } |
| 890 | |
| 891 | if (!changed) |
| 892 | return arrayTy; |
| 893 | |
| 894 | adjustedElts.resize( |
| 895 | N: std::max<size_t>(a: adjustedElts.size(), b: arrayTy.getNumElements()), |
| 896 | NV: origEltTy); |
| 897 | |
| 898 | if (llvm::all_equal(Range&: adjustedElts)) |
| 899 | return mlir::LLVM::LLVMArrayType::get(elementType: adjustedElts.front(), |
| 900 | numElements: adjustedElts.size()); |
| 901 | |
| 902 | // Packed, because arrays shouldn't be subject to padding. |
| 903 | return mlir::LLVM::LLVMStructType::getLiteral( |
| 904 | context: arrayTy.getContext(), types: adjustedElts, /*isPacked=*/true); |
| 905 | } |
| 906 | |
| 907 | // Apply various adjustments required for struct/union types. |
| 908 | mlir::Type |
| 909 | adjustGlobalTypeForInit(mlir::Type llvmType, mlir::Attribute init, |
| 910 | const mlir::TypeConverter &converter, |
| 911 | const mlir::DataLayout &dataLayout, |
| 912 | llvm::SmallVectorImpl<unsigned> &paddingAddedIndexes, |
| 913 | mlir::Type *failedType) { |
| 914 | if (!llvmType) { |
| 915 | if (failedType) |
| 916 | if (auto typedInit = mlir::dyn_cast_if_present<mlir::TypedAttr>(Val&: init)) |
| 917 | *failedType = typedInit.getType(); |
| 918 | return {}; |
| 919 | } |
| 920 | if (auto arrayInit = mlir::dyn_cast_if_present<cir::ConstArrayAttr>(Val&: init)) { |
| 921 | auto arrayTy = mlir::dyn_cast<mlir::LLVM::LLVMArrayType>(Val&: llvmType); |
| 922 | if (!arrayTy) |
| 923 | return llvmType; |
| 924 | return adjustGlobalArrayTypeForInit(arrayTy, arrayInit, converter, |
| 925 | dataLayout, failedType); |
| 926 | } |
| 927 | |
| 928 | // Conversions for both only happen if we have a record init. |
| 929 | auto constRecord = mlir::dyn_cast_if_present<cir::ConstRecordAttr>(Val&: init); |
| 930 | if (!constRecord) |
| 931 | return llvmType; |
| 932 | |
| 933 | // If this isn't of struct-type, or doesn't have any members, there is nothing |
| 934 | // to do. |
| 935 | auto structTy = mlir::dyn_cast<mlir::LLVM::LLVMStructType>(Val&: llvmType); |
| 936 | if (!structTy || structTy.getBody().empty()) |
| 937 | return llvmType; |
| 938 | |
| 939 | // Structs can have a flexible array member, adjust that. |
| 940 | if (mlir::isa<cir::StructType>(Val: constRecord.getType())) |
| 941 | return adjustGlobalStructTypeForInit(structTy, constRecord, converter, |
| 942 | dataLayout, paddingAddedIndexes, |
| 943 | failedType); |
| 944 | if (mlir::isa<cir::UnionType>(Val: constRecord.getType())) |
| 945 | return adjustGlobalUnionTypeForInit(structTy, constRecord, converter, |
| 946 | dataLayout, failedType); |
| 947 | return llvmType; |
| 948 | } |
| 949 | |
| 950 | mlir::Type adjustGlobalTypeForInit(mlir::Type llvmType, mlir::Attribute init, |
| 951 | const mlir::TypeConverter &converter, |
| 952 | const mlir::DataLayout &dataLayout, |
| 953 | mlir::Type *failedType) { |
| 954 | llvm::SmallVector<unsigned> ignoredAddedIndexes; |
| 955 | return adjustGlobalTypeForInit(llvmType, init, converter, dataLayout, |
| 956 | paddingAddedIndexes&: ignoredAddedIndexes, failedType); |
| 957 | } |
| 958 | |
| 959 | std::optional<mlir::Attribute> |
| 960 | lowerConstRecordAttr(cir::ConstRecordAttr constRecord, |
| 961 | mlir::SymbolTableCollection &symbolTables, |
| 962 | const mlir::TypeConverter *converter, |
| 963 | const CIRPreConversionTypeMap &preConversionTypes, |
| 964 | mlir::ModuleOp moduleOp) { |
| 965 | |
| 966 | // Build one constant attribute per record member. The LLVM dialect global |
| 967 | // translation accepts an ArrayAttr (one element per struct field) and emits |
| 968 | // an llvm::ConstantStruct, so the whole initializer can be a single |
| 969 | // attribute on the global instead of an insertvalue region. |
| 970 | mlir::ArrayAttr memberAttrs = constRecord.getMembers(); |
| 971 | llvm::SmallVector<mlir::Attribute> loweredMembers; |
| 972 | loweredMembers.reserve(N: memberAttrs.size()); |
| 973 | for (mlir::Attribute member : memberAttrs) { |
| 974 | std::optional<mlir::Attribute> lowered = lowerConstRecordMemberAttr( |
| 975 | attr: member, symbolTables, converter, preConversionTypes, moduleOp); |
| 976 | if (!lowered) |
| 977 | return std::nullopt; |
| 978 | loweredMembers.push_back(Elt: *lowered); |
| 979 | } |
| 980 | |
| 981 | // The lowered LLVM type may have more fields than the CIR record has members |
| 982 | // for a few reasons: |
| 983 | // 1- a union lowers to { active-member, [pad x i8]). |
| 984 | // 2- A struct that contains such a union can have its alignment changed too, |
| 985 | // so it needs tail padding to fill that in. |
| 986 | // 3- A struct containing a union whose initializer doesn't use the |
| 987 | // highest-aligned field will have to prepend a bit of padding, such as struct |
| 988 | // { i32, union { i64, i32 } }. Typically the union gets lowered to a struct |
| 989 | // { i64 } (as i64 has the greatest alignment), but if the init causes it to |
| 990 | // be the i32(or any such smaller field) we have to prepend it with padding: |
| 991 | // struct { i32, [4 x i8], struct { i32 }} |
| 992 | // instead of (with no init): |
| 993 | // struct { i32, struct { i64 }} |
| 994 | llvm::SmallVector<unsigned> paddingAddedIndexes; |
| 995 | mlir::Type adjustedTy = adjustGlobalTypeForInit( |
| 996 | llvmType: converter->convertType(t: constRecord.getType()), init: constRecord, converter: *converter, |
| 997 | dataLayout: mlir::DataLayout(moduleOp), paddingAddedIndexes); |
| 998 | if (!adjustedTy) |
| 999 | return std::nullopt; |
| 1000 | |
| 1001 | // This handles #3 from above. adjustGlobalTypeForInit ensures the |
| 1002 | // indexes are in increasing order, so we can insert 'backwards' without |
| 1003 | // causing problems. |
| 1004 | for (unsigned paddedElt : llvm::reverse(C&: paddingAddedIndexes)) |
| 1005 | loweredMembers.insert(I: loweredMembers.begin() + paddedElt, |
| 1006 | Elt: mlir::LLVM::ZeroAttr::get(ctx: constRecord.getContext())); |
| 1007 | |
| 1008 | // Any remaining difference will be the union/struct padding case. We don't |
| 1009 | // have a great handle/way to tell when to zero-vs-undef init, so always |
| 1010 | // zero init, as it is always safe to do so. |
| 1011 | if (auto structTy = mlir::dyn_cast<mlir::LLVM::LLVMStructType>(Val&: adjustedTy)) |
| 1012 | while (loweredMembers.size() < structTy.getBody().size()) |
| 1013 | loweredMembers.push_back( |
| 1014 | Elt: mlir::LLVM::ZeroAttr::get(ctx: constRecord.getContext())); |
| 1015 | |
| 1016 | return mlir::ArrayAttr::get(context: constRecord.getContext(), value: loweredMembers); |
| 1017 | } |
| 1018 | |
| 1019 | mlir::Value getConstAPInt(mlir::OpBuilder &bld, mlir::Location loc, |
| 1020 | mlir::Type typ, const llvm::APInt &val) { |
| 1021 | return mlir::LLVM::ConstantOp::create(builder&: bld, location: loc, type: typ, value: val); |
| 1022 | } |
| 1023 | |
| 1024 | mlir::Value getConst(mlir::OpBuilder &bld, mlir::Location loc, mlir::Type typ, |
| 1025 | unsigned val) { |
| 1026 | return mlir::LLVM::ConstantOp::create(builder&: bld, location: loc, type: typ, value: val); |
| 1027 | } |
| 1028 | |
| 1029 | mlir::Value createShL(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs) { |
| 1030 | if (!rhs) |
| 1031 | return lhs; |
| 1032 | mlir::Value rhsVal = getConst(bld, loc: lhs.getLoc(), typ: lhs.getType(), val: rhs); |
| 1033 | return mlir::LLVM::ShlOp::create(builder&: bld, location: lhs.getLoc(), lhs, rhs: rhsVal); |
| 1034 | } |
| 1035 | |
| 1036 | mlir::Value createAShR(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs) { |
| 1037 | if (!rhs) |
| 1038 | return lhs; |
| 1039 | mlir::Value rhsVal = getConst(bld, loc: lhs.getLoc(), typ: lhs.getType(), val: rhs); |
| 1040 | return mlir::LLVM::AShrOp::create(builder&: bld, location: lhs.getLoc(), lhs, rhs: rhsVal); |
| 1041 | } |
| 1042 | |
| 1043 | mlir::Value createAnd(mlir::OpBuilder &bld, mlir::Value lhs, |
| 1044 | const llvm::APInt &rhs) { |
| 1045 | mlir::Value rhsVal = getConstAPInt(bld, loc: lhs.getLoc(), typ: lhs.getType(), val: rhs); |
| 1046 | return mlir::LLVM::AndOp::create(builder&: bld, location: lhs.getLoc(), lhs, rhs: rhsVal); |
| 1047 | } |
| 1048 | |
| 1049 | mlir::Value createLShR(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs) { |
| 1050 | if (!rhs) |
| 1051 | return lhs; |
| 1052 | mlir::Value rhsVal = getConst(bld, loc: lhs.getLoc(), typ: lhs.getType(), val: rhs); |
| 1053 | return mlir::LLVM::LShrOp::create(builder&: bld, location: lhs.getLoc(), lhs, rhs: rhsVal); |
| 1054 | } |
| 1055 | |