1//====- LoweringHelpers.cpp - Lowering helper functions -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains helper functions for lowering from CIR to LLVM or MLIR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "clang/CIR/LoweringHelpers.h"
14#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
15#include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
16#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
17#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
18#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
19#include "mlir/IR/BuiltinTypes.h"
20#include "mlir/IR/SymbolTable.h"
21#include "mlir/Interfaces/DataLayoutInterfaces.h"
22#include "clang/CIR/MissingFeatures.h"
23
24namespace {
25/// A _BitInt(N) whose padded storage integer iM has a larger alloc size than
26/// its M/8 store size is laid out by clang as a byte array, not a plain
27/// integer (e.g. _BitInt(129) -> i192 with alloc size 32 != store size 24).
28/// That "split" storage form is not yet implemented; lowerings must detect
29/// it and report errorNYI rather than emit the wrong-sized integer.
30bool isSplitStorageBitInt(cir::IntType ty, const mlir::DataLayout &dataLayout) {
31 if (!ty.isBitInt())
32 return false;
33 unsigned storageBits = ty.getStorageTypeWidth(dataLayout);
34 auto storageTy = mlir::IntegerType::get(context: ty.getContext(), width: storageBits);
35 uint64_t storeSize = storageBits / 8;
36 uint64_t allocSize =
37 llvm::alignTo(Value: storeSize, Align: dataLayout.getTypeABIAlignment(t: storageTy));
38 return allocSize != storeSize;
39}
40
41/// Checks if `dataLayout` describes a big endian layout.
42bool isBigEndian(const mlir::DataLayout &dataLayout) {
43 auto endiannessStr =
44 mlir::dyn_cast_or_null<mlir::StringAttr>(Val: dataLayout.getEndianness());
45 return endiannessStr && endiannessStr == "big";
46}
47} // namespace
48
49mlir::Attribute getBitIntStorageAttr(mlir::ConversionPatternRewriter &rewriter,
50 cir::IntAttr attr,
51 const mlir::DataLayout &dataLayout) {
52 auto intTy = mlir::cast<cir::IntType>(Val: attr.getType());
53 unsigned storageBits = intTy.getStorageTypeWidth(dataLayout);
54 llvm::APInt val = attr.getValue();
55 val = intTy.isSigned() ? val.sext(width: storageBits) : val.zext(width: storageBits);
56
57 if (!isSplitStorageBitInt(ty: intTy, dataLayout))
58 return rewriter.getIntegerAttr(
59 type: mlir::IntegerType::get(context: intTy.getContext(), width: storageBits), value: val);
60
61 if (isBigEndian(dataLayout))
62 val = val.byteSwap();
63
64 // If we have to do split storage, we are an array of bytes. Split this up
65 // into the array that matches convertTypeForMemory.
66 unsigned numBytes = storageBits / 8;
67 llvm::SmallVector<mlir::APInt> bytes;
68 bytes.reserve(N: numBytes);
69 for (unsigned i = 0; i != numBytes; ++i)
70 bytes.emplace_back(Args: 8, Args: val.extractBitsAsZExtValue(numBits: 8, bitPosition: i * 8));
71
72 auto i8Ty = mlir::IntegerType::get(context: intTy.getContext(), width: 8);
73 return mlir::DenseElementsAttr::get(
74 type: mlir::RankedTensorType::get(shape: {numBytes}, elementType: i8Ty), values: bytes);
75}
76
77mlir::Type convertTypeForMemory(const mlir::TypeConverter &converter,
78 mlir::DataLayout const &dataLayout,
79 mlir::Type type, bool isForHLSL) {
80 // TODO(cir): Handle other types similarly to clang's codegen
81 // convertTypeForMemory
82 if (mlir::isa<cir::BoolType>(Val: type)) {
83 return mlir::IntegerType::get(context: type.getContext(),
84 width: dataLayout.getTypeSizeInBits(t: type));
85 }
86
87 if (auto matrixTy = mlir::dyn_cast<cir::MatrixType>(Val&: type)) {
88 if (isForHLSL) {
89 unsigned vecLen = matrixTy.getNumRows();
90 unsigned arrayLen = matrixTy.getNumColumns();
91 mlir::Type elementType = converter.convertType(t: matrixTy.getElementType());
92 auto vecTy = mlir::VectorType::get(shape: vecLen, elementType);
93 return mlir::LLVM::LLVMArrayType::get(elementType: vecTy, numElements: arrayLen);
94 }
95
96 uint64_t size = matrixTy.getNumRows() * matrixTy.getNumColumns();
97 mlir::Type elementType = converter.convertType(t: matrixTy.getElementType());
98 return mlir::LLVM::LLVMArrayType::get(elementType, numElements: size);
99 }
100
101 if (auto vecTy = mlir::dyn_cast<cir::VectorType>(Val&: type)) {
102 if (mlir::isa<cir::BoolType>(Val: vecTy.getElementType())) {
103 assert(!cir::MissingFeatures::hlsl());
104 return mlir::IntegerType::get(context: type.getContext(),
105 width: vecTy.getBoolStorageWidth());
106 }
107 }
108
109 // _BitInt(N) keeps its literal width as a value but is stored in a padded
110 // integer iM in memory, the same way bool is i1 as a value and i8 in
111 // memory. When iM's own LLVM alloc size would overshoot the AST-exact
112 // M/8 byte count (isSplitStorageBitInt), use a byte array instead so the
113 // size stays exact.
114 if (auto intTy = mlir::dyn_cast<cir::IntType>(Val&: type);
115 intTy && intTy.isBitInt()) {
116 if (isSplitStorageBitInt(ty: intTy, dataLayout))
117 return mlir::LLVM::LLVMArrayType::get(
118 elementType: mlir::IntegerType::get(context: type.getContext(), width: 8),
119 numElements: intTy.getStorageTypeWidth(dataLayout) / 8);
120 return mlir::IntegerType::get(context: type.getContext(),
121 width: intTy.getStorageTypeWidth(dataLayout));
122 }
123
124 return converter.convertType(t: type);
125}
126
127mlir::Type convertTypeForLoadStore(const mlir::TypeConverter &converter,
128 mlir::DataLayout const &dataLayout,
129 mlir::Type type) {
130 // A split-storage _BitInt's memory type is a byte array (see
131 // convertTypeForMemory), but a load/store must still access the whole
132 // integer as-is.
133 if (auto intTy = mlir::dyn_cast<cir::IntType>(Val&: type);
134 intTy && intTy.isBitInt())
135 return mlir::IntegerType::get(context: type.getContext(),
136 width: intTy.getStorageTypeWidth(dataLayout));
137
138 // Convert the Matrix type to a vector type (the value type of
139 // MatrixType), if it points to a array (the memory type of MatrixType).
140 if (auto matrixTy = mlir::dyn_cast<cir::MatrixType>(Val&: type)) {
141 uint64_t size = matrixTy.getNumRows() * matrixTy.getNumColumns();
142 mlir::Type elemTy = converter.convertType(t: matrixTy.getElementType());
143 return mlir::VectorType::get(shape: size, elementType: elemTy);
144 }
145
146 return convertTypeForMemory(converter, dataLayout, type);
147}
148
149static unsigned getIntOrBoolBitWidth(mlir::Type ty) {
150 if (auto intTy = mlir::dyn_cast<cir::IntType>(Val&: ty))
151 return intTy.getWidth();
152 assert(mlir::isa<cir::BoolType>(ty) &&
153 "expected CIR integer or bool element type");
154 return 1;
155}
156
157mlir::DenseElementsAttr
158convertStringAttrToDenseElementsAttr(cir::ConstArrayAttr attr,
159 mlir::Type type) {
160 const auto stringAttr = mlir::cast<mlir::StringAttr>(Val: attr.getElts());
161 const auto arrayTy = mlir::cast<cir::ArrayType>(Val: attr.getType());
162 const unsigned totalSize = arrayTy.getSize();
163 const unsigned trailingZeros = attr.getTrailingZerosNum();
164 assert(stringAttr.size() + trailingZeros == totalSize &&
165 "string const_array size must match explicit elements plus "
166 "trailing_zeros");
167
168 const unsigned bitWidth = getIntOrBoolBitWidth(ty: arrayTy.getElementType());
169 llvm::SmallVector<mlir::APInt> values;
170 values.reserve(N: totalSize);
171
172 // String bytes are raw values; interpret each as an unsigned byte so a
173 // high-bit char (>= 0x80) does not sign-extend to a value that overflows
174 // the element bit width when constructing the APInt.
175 for (const char element : stringAttr)
176 values.emplace_back(Args: bitWidth, Args: static_cast<unsigned char>(element));
177
178 values.insert(I: values.end(), NumToInsert: trailingZeros, Elt: mlir::APInt::getZero(numBits: bitWidth));
179
180 return mlir::DenseElementsAttr::get(
181 type: mlir::RankedTensorType::get(shape: {totalSize}, elementType: type), values: llvm::ArrayRef(values));
182}
183
184template <> mlir::APInt getZeroInitFromType(mlir::Type ty) {
185 const auto intTy = mlir::cast<mlir::IntegerType>(Val&: ty);
186 return mlir::APInt::getZero(numBits: intTy.getWidth());
187}
188
189template <> mlir::APFloat getZeroInitFromType(mlir::Type ty) {
190 auto fpTy = mlir::cast<mlir::FloatType>(Val&: ty);
191 return mlir::APFloat::getZero(Sem: fpTy.getFloatSemantics());
192}
193
194/// \param attr the ConstArrayAttr to convert
195/// \param values the output parameter, the values array to fill
196/// \param currentDims the shpae of tensor we're going to convert to
197/// \param dimIndex the current dimension we're processing
198/// \param currentIndex the current index in the values array
199/// \param convertedElementType the lowered storage type of each element
200template <typename AttrTy, typename StorageTy>
201bool convertToDenseElementsAttrImpl(
202 cir::ConstArrayAttr attr, llvm::SmallVectorImpl<StorageTy> &values,
203 const llvm::SmallVectorImpl<int64_t> &currentDims, int64_t dimIndex,
204 int64_t currentIndex, mlir::Type convertedElementType) {
205 if (auto stringAttr = mlir::dyn_cast<mlir::StringAttr>(Val: attr.getElts())) {
206 if (auto arrayType = mlir::dyn_cast<cir::ArrayType>(Val: attr.getType())) {
207 for (auto element : stringAttr) {
208 auto intAttr = cir::IntAttr::get(type: arrayType.getElementType(), value: element);
209 values[currentIndex++] = mlir::dyn_cast<AttrTy>(intAttr).getValue();
210 }
211 // Remaining slots are trailing zeros; values was zero-initialized.
212 currentIndex += attr.getTrailingZerosNum();
213 return true;
214 }
215 }
216
217 dimIndex++;
218 std::size_t elementsSizeInCurrentDim = 1;
219 for (std::size_t i = dimIndex; i < currentDims.size(); i++)
220 elementsSizeInCurrentDim *= currentDims[i];
221
222 auto arrayAttr = mlir::cast<mlir::ArrayAttr>(Val: attr.getElts());
223 for (auto eltAttr : arrayAttr) {
224 if constexpr (std::is_same_v<StorageTy, mlir::APInt>) {
225 if (auto boolAttr = mlir::dyn_cast<cir::BoolAttr>(Val&: eltAttr)) {
226 unsigned bitWidth =
227 mlir::cast<mlir::IntegerType>(Val&: convertedElementType).getWidth();
228 values[currentIndex++] =
229 llvm::APInt(bitWidth, static_cast<uint64_t>(boolAttr.getValue()));
230 continue;
231 }
232 }
233 if (auto valueAttr = mlir::dyn_cast<AttrTy>(eltAttr)) {
234 values[currentIndex++] = valueAttr.getValue();
235 continue;
236 }
237
238 if (auto subArrayAttr = mlir::dyn_cast<cir::ConstArrayAttr>(Val&: eltAttr)) {
239 convertToDenseElementsAttrImpl<AttrTy>(subArrayAttr, values, currentDims,
240 dimIndex, currentIndex,
241 convertedElementType);
242 currentIndex += elementsSizeInCurrentDim;
243 continue;
244 }
245
246 if (mlir::isa<cir::ZeroAttr, cir::UndefAttr>(Val: eltAttr)) {
247 currentIndex += elementsSizeInCurrentDim;
248 continue;
249 }
250
251 // A global view can be the result of pointer conversions, so we can't
252 // represent them as an APInt/APFloat. So give up if we see one.
253 if (auto global = mlir::dyn_cast<cir::GlobalViewAttr>(Val&: eltAttr))
254 return false;
255
256 llvm_unreachable("unknown element in ConstArrayAttr");
257 }
258 return true;
259}
260
261template <typename AttrTy, typename StorageTy>
262std::optional<mlir::DenseElementsAttr>
263convertToDenseElementsAttr(cir::ConstArrayAttr attr,
264 const llvm::SmallVectorImpl<int64_t> &dims,
265 mlir::Type convertedElementType) {
266 unsigned vectorSize = 1;
267 for (auto dim : dims)
268 vectorSize *= dim;
269 llvm::SmallVector<StorageTy> values(
270 vectorSize, getZeroInitFromType<StorageTy>(convertedElementType));
271
272 if (!convertToDenseElementsAttrImpl<AttrTy>(attr, values, dims,
273 /*dimIndex=*/0,
274 /*currentIndex=*/0,
275 convertedElementType))
276 return std::nullopt;
277
278 return mlir::DenseElementsAttr::get(
279 mlir::RankedTensorType::get(shape: dims, elementType: convertedElementType),
280 llvm::ArrayRef(values));
281}
282
283/// Return true when \p gv can be lowered to a \c FlatSymbolRefAttr leaf without
284/// addrspacecast or bitcast (mirrors \c CIRAttrToValue::visitCirAttr).
285static bool globalViewMatchesPointerLeaf(
286 cir::GlobalViewAttr gv, mlir::ModuleOp moduleOp,
287 mlir::SymbolTableCollection &symbolTables,
288 const mlir::TypeConverter *converter,
289 const CIRPreConversionTypeMap &preConversionTypes) {
290 if (gv.getIndices() || mlir::isa<cir::IntType, cir::VPtrType>(Val: gv.getType()))
291 return false;
292
293 auto ptrTy = mlir::dyn_cast<cir::PointerType>(Val: gv.getType());
294 if (!ptrTy)
295 return false;
296
297 unsigned sourceAddrSpace = 0;
298 mlir::Type sourceType;
299 auto sourceSymbol = symbolTables.lookupSymbolIn(symbolTableOp: moduleOp, name: gv.getSymbol());
300 if (auto llvmSymbol = mlir::dyn_cast<mlir::LLVM::GlobalOp>(Val: sourceSymbol)) {
301 if (mlir::Type preType = preConversionTypes.lookup(Key: llvmSymbol.getSymName()))
302 sourceType = converter->convertType(t: preType);
303 else
304 sourceType = llvmSymbol.getType();
305 sourceAddrSpace = llvmSymbol.getAddrSpace();
306 } else if (auto cirSymbol = mlir::dyn_cast<cir::GlobalOp>(Val: sourceSymbol)) {
307 sourceType = converter->convertType(t: cirSymbol.getSymType());
308 if (auto targetAS = mlir::dyn_cast_if_present<cir::TargetAddressSpaceAttr>(
309 Val: cirSymbol.getAddrSpaceAttr()))
310 sourceAddrSpace = targetAS.getValue();
311 } else {
312 // cir.func and other symbols not yet lowered to globals cannot be used as
313 // bulk constant leaves; those cases keep the insertvalue fallback.
314 return false;
315 }
316
317 auto llvmDstTy = converter->convertType<mlir::LLVM::LLVMPointerType>(t: ptrTy);
318 if (llvmDstTy.getAddressSpace() != sourceAddrSpace)
319 return false;
320
321 mlir::Type llvmEltTy = converter->convertType(t: ptrTy.getPointee());
322 if (llvmEltTy == sourceType)
323 return true;
324 if (auto arrTy = mlir::dyn_cast<mlir::LLVM::LLVMArrayType>(Val&: sourceType))
325 return llvmEltTy == arrTy.getElementType();
326 return false;
327}
328
329/// Lower a single pointer-element of a \c cir.const_array to an LLVM-dialect
330/// constant leaf suitable for a bulk \c llvm.mlir.constant. Only handles
331/// address-of-global without indices and null pointers; indexed global views
332/// must use the per-element \c llvm.insertvalue fallback.
333static std::optional<mlir::Attribute>
334lowerPointerElementAttr(mlir::Attribute elt, mlir::MLIRContext *ctx,
335 mlir::ModuleOp moduleOp,
336 mlir::SymbolTableCollection &symbolTables,
337 const mlir::TypeConverter *converter,
338 const CIRPreConversionTypeMap &preConversionTypes) {
339 if (auto gv = mlir::dyn_cast<cir::GlobalViewAttr>(Val&: elt)) {
340 if (!moduleOp ||
341 !globalViewMatchesPointerLeaf(gv, moduleOp, symbolTables, converter,
342 preConversionTypes))
343 return std::nullopt;
344 return gv.getSymbol();
345 }
346 if (auto nullPtr = mlir::dyn_cast<cir::ConstPtrAttr>(Val&: elt)) {
347 if (nullPtr.isNullValue())
348 return mlir::LLVM::ZeroAttr::get(ctx);
349 return std::nullopt;
350 }
351 return std::nullopt;
352}
353
354static bool containsPoison(mlir::Attribute attr) {
355 if (mlir::isa<cir::PoisonAttr>(Val: attr))
356 return true;
357 if (auto elts = mlir::dyn_cast<mlir::ArrayAttr>(Val&: attr))
358 return llvm::any_of(Range&: elts, P: containsPoison);
359 if (auto constArr = mlir::dyn_cast<cir::ConstArrayAttr>(Val&: attr)) {
360 if (mlir::isa<mlir::StringAttr>(Val: constArr.getElts()))
361 return false;
362 if (auto elts = mlir::dyn_cast<mlir::ArrayAttr>(Val: constArr.getElts()))
363 return llvm::any_of(Range&: elts, P: containsPoison);
364 }
365 return false;
366}
367
368/// Block-address attributes (address-of-label and label differences) are
369/// lowered to relocation expressions that cannot be materialized as part of a
370/// dense/aggregate constant attribute; they require the per-element
371/// insertvalue region lowering. Return true if \p attr contains any such
372/// element.
373static bool containsBlockAddress(mlir::Attribute attr) {
374 if (mlir::isa<cir::BlockAddrInfoAttr, cir::BlockAddrDiffAttr>(Val: attr))
375 return true;
376 if (auto elts = mlir::dyn_cast<mlir::ArrayAttr>(Val&: attr))
377 return llvm::any_of(Range&: elts, P: containsBlockAddress);
378 if (auto constArr = mlir::dyn_cast<cir::ConstArrayAttr>(Val&: attr)) {
379 if (mlir::isa<mlir::StringAttr>(Val: constArr.getElts()))
380 return false;
381 if (auto elts = mlir::dyn_cast<mlir::ArrayAttr>(Val: constArr.getElts()))
382 return llvm::any_of(Range&: elts, P: containsBlockAddress);
383 }
384 return false;
385}
386
387/// The fewest trailing zeros that adjustGlobalArrayTypeForInit splits off an
388/// array constant into a zero-initialized tail, as in classic codegen.
389static constexpr int minSplitTrailingZeros = 8;
390
391static bool shouldSplitTrailingZeros(cir::ConstArrayAttr attr) {
392 auto elts = mlir::dyn_cast<mlir::ArrayAttr>(Val: attr.getElts());
393 return elts && !elts.empty() &&
394 attr.getTrailingZerosNum() >= minSplitTrailingZeros;
395}
396
397static bool containsSplitTrailingZeros(mlir::Attribute attr) {
398 if (auto record = mlir::dyn_cast<cir::ConstRecordAttr>(Val&: attr))
399 return llvm::any_of(Range: record.getMembers(), P: containsSplitTrailingZeros);
400 auto array = mlir::dyn_cast<cir::ConstArrayAttr>(Val&: attr);
401 if (!array)
402 return false;
403 if (shouldSplitTrailingZeros(attr: array))
404 return true;
405 auto elts = mlir::dyn_cast<mlir::ArrayAttr>(Val: array.getElts());
406 return elts && llvm::any_of(Range&: elts, P: containsSplitTrailingZeros);
407}
408
409mlir::Attribute spellOutSplitTrailingZeros(mlir::Attribute attr) {
410 if (!containsSplitTrailingZeros(attr))
411 return attr;
412
413 if (auto record = mlir::dyn_cast<cir::ConstRecordAttr>(Val&: attr)) {
414 auto members =
415 llvm::map_to_vector(C: record.getMembers(), F&: spellOutSplitTrailingZeros);
416 return cir::ConstRecordAttr::get(
417 type: mlir::cast<cir::RecordType>(Val: record.getType()),
418 members: mlir::ArrayAttr::get(context: attr.getContext(), value: members));
419 }
420
421 auto array = mlir::cast<cir::ConstArrayAttr>(Val&: attr);
422 auto arrayTy = mlir::cast<cir::ArrayType>(Val: array.getType());
423 auto elts = llvm::map_to_vector(C: mlir::cast<mlir::ArrayAttr>(Val: array.getElts()),
424 F&: spellOutSplitTrailingZeros);
425 if (!shouldSplitTrailingZeros(attr: array))
426 return cir::ConstArrayAttr::get(
427 type: arrayTy, elts: mlir::ArrayAttr::get(context: attr.getContext(), value: elts),
428 trailingZerosNum: array.getTrailingZerosNum());
429 elts.append(NumInputs: array.getTrailingZerosNum(),
430 Elt: cir::ZeroAttr::get(type: arrayTy.getElementType()));
431 return cir::ConstArrayAttr::get(type: arrayTy,
432 elts: mlir::ArrayAttr::get(context: attr.getContext(), value: elts),
433 /*trailingZerosNum=*/0);
434}
435
436static std::optional<mlir::Attribute> lowerConstRecordMemberAttr(
437 mlir::Attribute attr, mlir::SymbolTableCollection &symbolTables,
438 const mlir::TypeConverter *converter,
439 const CIRPreConversionTypeMap &preConversionTypes, mlir::ModuleOp moduleOp);
440
441/// Lower \p constArr, which is or contains an array split by
442/// adjustGlobalArrayTypeForInit, element by element into the shape of its
443/// adjusted type. A dense attribute would span the split-off zeros.
444static std::optional<mlir::Attribute>
445lowerConstArrayAttrByElement(cir::ConstArrayAttr constArr,
446 mlir::SymbolTableCollection &symbolTables,
447 const mlir::TypeConverter *converter,
448 const CIRPreConversionTypeMap &preConversionTypes,
449 mlir::ModuleOp moduleOp) {
450 mlir::MLIRContext *ctx = constArr.getContext();
451 auto arrayTy = mlir::cast<cir::ArrayType>(Val: constArr.getType());
452 auto elts = mlir::cast<mlir::ArrayAttr>(Val: constArr.getElts());
453 mlir::Type adjustedTy =
454 adjustGlobalTypeForInit(llvmType: converter->convertType(t: arrayTy), init: constArr,
455 converter: *converter, dataLayout: mlir::DataLayout(moduleOp));
456
457 llvm::SmallVector<mlir::Attribute> lowered;
458 // Fewer members than elements: the elements were grouped into one array.
459 auto structTy = mlir::dyn_cast<mlir::LLVM::LLVMStructType>(Val&: adjustedTy);
460 if (structTy && structTy.getBody().size() < elts.size()) {
461 auto prefix = cir::ConstArrayAttr::get(
462 type: cir::ArrayType::get(elementType: arrayTy.getElementType(), size: elts.size()), elts);
463 std::optional<mlir::Attribute> loweredPrefix = lowerConstArrayAttr(
464 constArr: prefix, symbolTables, converter, preConversionTypes, moduleOp);
465 if (!loweredPrefix)
466 return std::nullopt;
467 lowered.push_back(Elt: *loweredPrefix);
468 } else {
469 for (mlir::Attribute elt : elts) {
470 std::optional<mlir::Attribute> loweredElt = lowerConstRecordMemberAttr(
471 attr: elt, symbolTables, converter, preConversionTypes, moduleOp);
472 if (!loweredElt)
473 return std::nullopt;
474 lowered.push_back(Elt: *loweredElt);
475 }
476 }
477
478 // A split array ends in one zero-initialized member; otherwise its few
479 // trailing zeros are spelled out.
480 lowered.append(
481 NumInputs: shouldSplitTrailingZeros(attr: constArr) ? 1 : constArr.getTrailingZerosNum(),
482 Elt: mlir::LLVM::ZeroAttr::get(ctx));
483 return mlir::ArrayAttr::get(context: ctx, value: lowered);
484}
485
486std::optional<mlir::Attribute>
487lowerConstArrayAttr(cir::ConstArrayAttr constArr,
488 mlir::SymbolTableCollection &symbolTables,
489 const mlir::TypeConverter *converter,
490 const CIRPreConversionTypeMap &preConversionTypes,
491 mlir::ModuleOp moduleOp) {
492 // Ensure ConstArrayAttr has a type.
493 const auto typedConstArr = mlir::cast<mlir::TypedAttr>(Val&: constArr);
494
495 // Ensure ConstArrayAttr type is a ArrayType.
496 const auto cirArrayType = mlir::cast<cir::ArrayType>(Val: typedConstArr.getType());
497
498 // Is a ConstArrayAttr with an cir::ArrayType: fetch element type.
499 mlir::Type type = cirArrayType;
500 auto dims = llvm::SmallVector<int64_t, 2>{};
501 while (auto arrayType = mlir::dyn_cast<cir::ArrayType>(Val&: type)) {
502 dims.push_back(Elt: arrayType.getSize());
503 type = arrayType.getElementType();
504 }
505
506 if (containsPoison(attr: constArr))
507 return std::nullopt;
508
509 // Block-address initializers cannot be represented as a dense/aggregate
510 // constant attribute; fall back to the per-element insertvalue lowering.
511 if (containsBlockAddress(attr: constArr))
512 return std::nullopt;
513
514 if (containsSplitTrailingZeros(attr: constArr))
515 return lowerConstArrayAttrByElement(constArr, symbolTables, converter,
516 preConversionTypes, moduleOp);
517
518 if (mlir::isa<mlir::StringAttr>(Val: constArr.getElts()))
519 return convertStringAttrToDenseElementsAttr(attr: constArr,
520 type: converter->convertType(t: type));
521 if (mlir::isa<cir::IntType>(Val: type)) {
522 // A _BitInt array element is stored in a padded integer in memory; the
523 // dense-attribute path here cannot express that widening, so fall back to
524 // the insertvalue region, which widens each element via emitToMemory.
525 if (mlir::cast<cir::IntType>(Val&: type).isBitInt())
526 return std::nullopt;
527 return convertToDenseElementsAttr<cir::IntAttr, mlir::APInt>(
528 attr: constArr, dims, convertedElementType: converter->convertType(t: type));
529 }
530
531 if (mlir::isa<cir::BoolType>(Val: type)) {
532 mlir::Type storageType =
533 convertTypeForMemory(converter: *converter, dataLayout: mlir::DataLayout(moduleOp), type);
534 return convertToDenseElementsAttr<cir::IntAttr, mlir::APInt>(attr: constArr, dims,
535 convertedElementType: storageType);
536 }
537
538 if (mlir::isa<cir::FPTypeInterface>(Val: type))
539 return convertToDenseElementsAttr<cir::FPAttr, mlir::APFloat>(
540 attr: constArr, dims, convertedElementType: converter->convertType(t: type));
541
542 if (mlir::isa<cir::PointerType>(Val: type)) {
543 // FIXME: Pointer arrays with trailing_zeros (null-sentinel tables) fall
544 // through to the insertvalue path for now.
545 if (constArr.getTrailingZerosNum() > 0)
546 return std::nullopt;
547 auto eltsArr = mlir::dyn_cast<mlir::ArrayAttr>(Val: constArr.getElts());
548 if (!eltsArr)
549 return std::nullopt;
550 llvm::SmallVector<mlir::Attribute> lowered;
551 lowered.reserve(N: eltsArr.size());
552 mlir::MLIRContext *ctx = constArr.getContext();
553 for (mlir::Attribute elt : eltsArr) {
554 std::optional<mlir::Attribute> llvmElt = lowerPointerElementAttr(
555 elt, ctx, moduleOp, symbolTables, converter, preConversionTypes);
556 if (!llvmElt)
557 return std::nullopt;
558 lowered.push_back(Elt: *llvmElt);
559 }
560 return mlir::ArrayAttr::get(context: ctx, value: lowered);
561 }
562
563 if (mlir::isa<cir::RecordType>(Val: type)) {
564 // A record type is just an array of the element values.
565 auto eltsArr = mlir::dyn_cast<mlir::ArrayAttr>(Val: constArr.getElts());
566 if (!eltsArr)
567 return std::nullopt;
568
569 llvm::SmallVector<mlir::Attribute> loweredElts;
570 loweredElts.reserve(N: cirArrayType.getSize());
571
572 for (mlir::Attribute elt : eltsArr) {
573 std::optional<mlir::Attribute> llvmElt = lowerConstRecordMemberAttr(
574 attr: elt, symbolTables, converter, preConversionTypes, moduleOp);
575 if (!llvmElt)
576 return std::nullopt;
577 loweredElts.push_back(Elt: *llvmElt);
578 }
579
580 // Remaining elts are either going to be padding (and should be undef), or
581 // trailing-zeros. We can't really tell the difference as CIR lowering
582 // doesn't differentiate anyway, so just zero-fill them.
583 while (loweredElts.size() < cirArrayType.getSize())
584 loweredElts.push_back(Elt: mlir::LLVM::ZeroAttr::get(ctx: constArr.getContext()));
585 return mlir::ArrayAttr::get(context: constArr.getContext(), value: loweredElts);
586 }
587
588 return std::nullopt;
589}
590
591/// Lower a constant attribute that initializes a single member of a record (or
592/// a leaf of a nested aggregate) to an LLVM-dialect attribute that can be
593/// attached directly to an \c llvm.mlir.global, avoiding an insertvalue
594/// initializer region. Returns \c std::nullopt when the attribute cannot be
595/// represented as a single constant attribute (e.g. an indexed
596/// \c GlobalViewAttr), in which case the caller falls back to the region-based
597/// lowering.
598static std::optional<mlir::Attribute>
599lowerConstRecordMemberAttr(mlir::Attribute attr,
600 mlir::SymbolTableCollection &symbolTables,
601 const mlir::TypeConverter *converter,
602 const CIRPreConversionTypeMap &preConversionTypes,
603 mlir::ModuleOp moduleOp) {
604 mlir::MLIRContext *ctx = attr.getContext();
605
606 if (auto arrayAttr = mlir::dyn_cast<cir::ConstArrayAttr>(Val&: attr))
607 return lowerConstArrayAttr(constArr: arrayAttr, symbolTables, converter,
608 preConversionTypes, moduleOp);
609
610 if (auto recordAttr = mlir::dyn_cast<cir::ConstRecordAttr>(Val&: attr))
611 return lowerConstRecordAttr(constRecord: recordAttr, symbolTables, converter,
612 preConversionTypes, moduleOp);
613
614 if (mlir::isa<cir::ZeroAttr>(Val: attr))
615 return mlir::LLVM::ZeroAttr::get(ctx);
616
617 if (mlir::isa<cir::UndefAttr>(Val: attr))
618 return mlir::LLVM::UndefAttr::get(ctx);
619
620 if (auto intAttr = mlir::dyn_cast<cir::IntAttr>(Val&: attr)) {
621 // A _BitInt member is stored in a padded integer in memory; defer to the
622 // insertvalue region (CIRAttrToValue), which performs the extension.
623 if (mlir::cast<cir::IntType>(Val: intAttr.getType()).isBitInt())
624 return std::nullopt;
625 return mlir::IntegerAttr::get(type: converter->convertType(t: intAttr.getType()),
626 value: intAttr.getValue());
627 }
628
629 if (auto boolAttr = mlir::dyn_cast<cir::BoolAttr>(Val&: attr)) {
630 mlir::Type storageType = convertTypeForMemory(
631 converter: *converter, dataLayout: mlir::DataLayout(moduleOp), type: boolAttr.getType());
632 return mlir::IntegerAttr::get(type: storageType, value: boolAttr.getValue() ? 1 : 0);
633 }
634
635 if (auto fpAttr = mlir::dyn_cast<cir::FPAttr>(Val&: attr))
636 return mlir::FloatAttr::get(type: converter->convertType(t: fpAttr.getType()),
637 value: fpAttr.getValue());
638
639 // Null pointers and simple address-of-global references can be represented
640 // as constant attributes; anything more complex uses the region fallback.
641 return lowerPointerElementAttr(elt: attr, ctx, moduleOp, symbolTables, converter,
642 preConversionTypes);
643}
644
645// Figure out if we want mark the new struct 'packed' if it isn't already. IF
646// it is already, we have to keep that behavior. We pack it with logic similar
647// to classic codegen, though will end up missing cases, since we don't want to
648// change the type other than the FAM.
649// We can do so if:
650// 1- Packing it won't change any of the field offsets.
651// 2- the non-padded struct would add padding beyond the
652// flexible array member. We don't pack if the flexible array member manages
653// to not cause trailing padding.
654static bool shouldPackFAMStruct(const mlir::DataLayout &dataLayout,
655 llvm::ArrayRef<mlir::Type> members) {
656 uint64_t maxAlign = 1;
657 uint64_t totalSize = 0;
658 for (mlir::Type member : members) {
659 uint64_t align = dataLayout.getTypeABIAlignment(t: member);
660 maxAlign = std::max(a: maxAlign, b: align);
661 uint64_t size = dataLayout.getTypeSize(t: member).getFixedValue();
662
663 if (llvm::alignTo(Value: totalSize, Align: align) != totalSize)
664 return false;
665
666 totalSize += size;
667 }
668 return llvm::alignTo(Value: totalSize, Align: maxAlign) != totalSize;
669}
670
671// CIR supports flexible-array-members in its struct types. That is, a
672// zero-length array as the last element, which can be initialized with an
673// arbitrary number of elements. A ConstRecordAttr can be created with one of
674// these, and our verifier allows it. However, the LLVM implementation does NOT
675// permit this. So we have to replace this type in LLVM with special struct for
676// this value.
677//
678// Additionally, the struct itself could contain a struct with a FAM or a union
679// that needed adjustment, so it recurses to check those. If no such type has
680// been found/no adjustment needed, this returns the type unchanged.
681//
682// Additionally, a union having an active member of not-the-largest
683// alignment can cause the need for a small padding array. We also capture the
684// original indices of the fields that had this padding prepended, so the
685// lowerConstRecordAttr can later put in a 'zero' init there.
686static mlir::Type adjustGlobalStructTypeForInit(
687 mlir::LLVM::LLVMStructType structTy, cir::ConstRecordAttr constRecord,
688 const mlir::TypeConverter &converter, const mlir::DataLayout &dataLayout,
689 llvm::SmallVectorImpl<unsigned> &paddingAddedIndexes,
690 mlir::Type *failedType) {
691 assert(paddingAddedIndexes.empty() &&
692 "Not for accumulation, just single depth");
693 llvm::ArrayRef<mlir::Attribute> initMembers =
694 constRecord.getMembers().getValue();
695 llvm::SmallVector<mlir::Type> origBody{structTy.getBody()};
696 llvm::SmallVector<mlir::Type> newBody{};
697 bool packed = structTy.isPacked();
698 uint64_t curOffset = 0;
699 bool changed = false;
700
701 // Recursively adjust each member. A member that is itself a union (or a
702 // struct containing one) lowers to a type that differs from its declared
703 // field type, and this struct has to adopt that adjusted type so the
704 // enclosing insertvalue chain type-checks.
705 for (auto [idx, member] : llvm::enumerate(First&: initMembers)) {
706 if (idx >= origBody.size())
707 break;
708 mlir::Type adjusted = adjustGlobalTypeForInit(
709 llvmType: origBody[idx], init: member, converter, dataLayout, failedType);
710 if (!adjusted)
711 return {};
712 unsigned adjustedAlign = dataLayout.getTypeABIAlignment(t: adjusted);
713
714 if (adjusted != origBody[idx]) {
715 // We're always going to 'change' the layout if it has changed, but we
716 // need to see if there is new 'padding' that won't happen automatically
717 // here based on alignment.
718 unsigned origAlign = dataLayout.getTypeABIAlignment(t: origBody[idx]);
719
720 uint64_t origOffset =
721 packed ? curOffset : llvm::alignTo(Value: curOffset, Align: origAlign);
722 uint64_t adjustedOffset =
723 packed ? curOffset : llvm::alignTo(Value: curOffset, Align: adjustedAlign);
724
725 if (adjustedOffset != origOffset) {
726 // If the offset would change, we have to insert padding to make up for
727 // it. This should only happen since alignment will decrease with
728 // unions, so we should be able to assume adjusted-offset < origOffset?
729 assert(adjustedOffset < origOffset);
730 // Rather than just pad the difference between the offsets, we have to
731 // fill in since the end of the last field, else we leave room thanks to
732 // alignment between this field and the padding.
733 uint64_t difference = origOffset - curOffset;
734 newBody.push_back(Elt: mlir::LLVM::LLVMArrayType::get(
735 elementType: mlir::IntegerType::get(context: structTy.getContext(), width: 8), numElements: difference));
736 paddingAddedIndexes.push_back(Elt: idx);
737 curOffset = origOffset;
738 }
739 changed = true;
740 }
741 newBody.push_back(Elt: adjusted);
742
743 if (!packed)
744 curOffset = llvm::alignTo(Value: curOffset, Align: adjustedAlign);
745 curOffset += dataLayout.getTypeSize(t: adjusted).getFixedValue();
746 }
747
748 // CIR supports flexible-array-members in its struct types. That is, a
749 // zero-length array as the last element, which can be initialized with an
750 // arbitrary number of elements. A ConstRecordAttr can be created with one of
751 // these, and our verifier allows it. However, the LLVM implementation does
752 // NOT permit this, so we widen that trailing member to the initializer's
753 // array type (packing the struct if that changes the layout).
754 bool widenedFAM = false;
755 if (auto fam =
756 mlir::dyn_cast<mlir::LLVM::LLVMArrayType>(Val: structTy.getBody().back());
757 fam && fam.getNumElements() == 0) {
758 mlir::Type lastInitType =
759 mlir::cast<mlir::TypedAttr>(Val: initMembers.back()).getType();
760 if (mlir::cast<cir::ArrayType>(Val&: lastInitType).getSize() != 0) {
761 newBody.back() = adjustGlobalTypeForInit(
762 llvmType: converter.convertType(t: lastInitType), init: initMembers.back(), converter,
763 dataLayout, failedType);
764 if (!newBody.back())
765 return {};
766 packed = packed || shouldPackFAMStruct(dataLayout, members: newBody);
767 widenedFAM = true;
768 changed = true;
769 }
770 }
771
772 if (!changed)
773 return structTy;
774
775 // We've likely reduced the alignment, so make sure we put padding 'behind'
776 // it. We can skip this in the FAM case, since a Flexible array member is not
777 // allowed to be initialized unless it is the 'last' element. So it doesn't
778 // need to be padded out.
779 if (!widenedFAM) {
780 uint64_t declaredSize = dataLayout.getTypeSize(t: structTy).getFixedValue();
781 assert(curOffset <= declaredSize && "body bigger than type?");
782 if (curOffset < declaredSize)
783 newBody.push_back(Elt: mlir::LLVM::LLVMArrayType::get(
784 elementType: mlir::IntegerType::get(context: structTy.getContext(), width: 8),
785 numElements: declaredSize - curOffset));
786 }
787
788 return mlir::LLVM::LLVMStructType::getLiteral(context: structTy.getContext(), types: newBody,
789 isPacked: packed);
790}
791
792// A union constant only initializes its active member, but a union is lowered
793// to its most-aligned member (its 'storage' type), which need not be the
794// initialized one -- e.g. `union { char buf[16]; long cap; }` stores as `long`
795// (higher alignment) but may be initialized through `buf`. So the storage type
796// (and thus structTy) generally can't hold the active member's value. Rebuild
797// an anonymous struct `{ <active member>, [pad x i8] }` that holds the active
798// member followed by enough byte padding to span the union's full allocated
799// size, mirroring classic codegen.
800static mlir::Type adjustGlobalUnionTypeForInit(
801 mlir::LLVM::LLVMStructType structTy, cir::ConstRecordAttr constRecord,
802 const mlir::TypeConverter &converter, const mlir::DataLayout &dataLayout,
803 mlir::Type *failedType) {
804
805 auto unionTy = mlir::cast<cir::UnionType>(Val: constRecord.getType());
806
807 // Unions can only initialize one field, so this has to be sizeof-one.
808 assert(constRecord.getMembers().size() == 1);
809 mlir::Attribute member = constRecord.getMembers()[0];
810 mlir::Type memberTy = convertTypeForMemory(
811 converter, dataLayout, type: mlir::cast<mlir::TypedAttr>(Val&: member).getType());
812
813 // The active member may itself need adjusting (e.g. it is a nested union, or
814 // a struct containing one), so recurse before using its type below.
815 memberTy = adjustGlobalTypeForInit(llvmType: memberTy, init: member, converter, dataLayout,
816 failedType);
817 if (!memberTy)
818 return {};
819
820 // The converted union type is { storage, [padding] }, where storage is the
821 // union's most-aligned member. When the active member IS that storage type,
822 // the converted type already describes this initializer exactly (the pad we
823 // would compute equals the union's declared pad), so there is nothing to do.
824 if (memberTy == structTy.getBody().front())
825 return structTy;
826
827 uint64_t unionSize = dataLayout.getTypeSize(t: unionTy).getFixedValue();
828 uint64_t initSize = dataLayout.getTypeSize(t: memberTy).getFixedValue();
829 llvm::SmallVector<mlir::Type> newBody;
830 newBody.push_back(Elt: memberTy);
831
832 // Pad to the union size unless a flexible array initializer is larger.
833 if (initSize < unionSize)
834 newBody.push_back(Elt: mlir::LLVM::LLVMArrayType::get(
835 elementType: mlir::IntegerType::get(context: structTy.getContext(), width: 8),
836 numElements: unionSize - initSize));
837
838 return mlir::LLVM::LLVMStructType::getLiteral(context: structTy.getContext(), types: newBody,
839 isPacked: unionTy.getPacked());
840}
841
842// Unions in an array can cause individual elements to be of different types.
843// This function adjusts the array type and replaces it with a struct type if
844// necessary, OR leaves it as a 'new' array type if necessary.
845//
846// Like classic codegen, an array ending in many zeros becomes a packed struct
847// of its explicit elements and one zero-initialized array, so the zeros are
848// never spelled out:
849// <{ T0, T1, [N x T] }>
850// or, when 8 or more elements share a type,
851// <{ [M x T], [N x T] }>
852static mlir::Type adjustGlobalArrayTypeForInit(
853 mlir::LLVM::LLVMArrayType arrayTy, cir::ConstArrayAttr arrayInit,
854 const mlir::TypeConverter &converter, const mlir::DataLayout &dataLayout,
855 mlir::Type *failedType) {
856 auto elts = mlir::dyn_cast<mlir::ArrayAttr>(Val: arrayInit.getElts());
857
858 if (!elts)
859 return arrayTy;
860
861 mlir::Type origEltTy = arrayTy.getElementType();
862 llvm::SmallVector<mlir::Type> adjustedElts;
863 adjustedElts.reserve(N: elts.size());
864 bool changed = elts.size() > arrayTy.getNumElements();
865
866 for (mlir::Attribute elt : elts) {
867 mlir::Type adjusted = adjustGlobalTypeForInit(llvmType: origEltTy, init: elt, converter,
868 dataLayout, failedType);
869 if (!adjusted)
870 return {};
871 changed |= adjusted != origEltTy;
872 adjustedElts.push_back(Elt: adjusted);
873 }
874
875 if (shouldSplitTrailingZeros(attr: arrayInit)) {
876 mlir::Type commonEltTy =
877 llvm::all_equal(Range&: adjustedElts) ? adjustedElts.front() : mlir::Type{};
878 llvm::SmallVector<mlir::Type> body;
879 if (commonEltTy && elts.size() >= minSplitTrailingZeros)
880 body.push_back(
881 Elt: mlir::LLVM::LLVMArrayType::get(elementType: commonEltTy, numElements: adjustedElts.size()));
882 else
883 body.append(RHS: adjustedElts);
884 body.push_back(
885 Elt: mlir::LLVM::LLVMArrayType::get(elementType: commonEltTy ? commonEltTy : origEltTy,
886 numElements: arrayInit.getTrailingZerosNum()));
887 return mlir::LLVM::LLVMStructType::getLiteral(context: arrayTy.getContext(), types: body,
888 /*isPacked=*/true);
889 }
890
891 if (!changed)
892 return arrayTy;
893
894 adjustedElts.resize(
895 N: std::max<size_t>(a: adjustedElts.size(), b: arrayTy.getNumElements()),
896 NV: origEltTy);
897
898 if (llvm::all_equal(Range&: adjustedElts))
899 return mlir::LLVM::LLVMArrayType::get(elementType: adjustedElts.front(),
900 numElements: adjustedElts.size());
901
902 // Packed, because arrays shouldn't be subject to padding.
903 return mlir::LLVM::LLVMStructType::getLiteral(
904 context: arrayTy.getContext(), types: adjustedElts, /*isPacked=*/true);
905}
906
907// Apply various adjustments required for struct/union types.
908mlir::Type
909adjustGlobalTypeForInit(mlir::Type llvmType, mlir::Attribute init,
910 const mlir::TypeConverter &converter,
911 const mlir::DataLayout &dataLayout,
912 llvm::SmallVectorImpl<unsigned> &paddingAddedIndexes,
913 mlir::Type *failedType) {
914 if (!llvmType) {
915 if (failedType)
916 if (auto typedInit = mlir::dyn_cast_if_present<mlir::TypedAttr>(Val&: init))
917 *failedType = typedInit.getType();
918 return {};
919 }
920 if (auto arrayInit = mlir::dyn_cast_if_present<cir::ConstArrayAttr>(Val&: init)) {
921 auto arrayTy = mlir::dyn_cast<mlir::LLVM::LLVMArrayType>(Val&: llvmType);
922 if (!arrayTy)
923 return llvmType;
924 return adjustGlobalArrayTypeForInit(arrayTy, arrayInit, converter,
925 dataLayout, failedType);
926 }
927
928 // Conversions for both only happen if we have a record init.
929 auto constRecord = mlir::dyn_cast_if_present<cir::ConstRecordAttr>(Val&: init);
930 if (!constRecord)
931 return llvmType;
932
933 // If this isn't of struct-type, or doesn't have any members, there is nothing
934 // to do.
935 auto structTy = mlir::dyn_cast<mlir::LLVM::LLVMStructType>(Val&: llvmType);
936 if (!structTy || structTy.getBody().empty())
937 return llvmType;
938
939 // Structs can have a flexible array member, adjust that.
940 if (mlir::isa<cir::StructType>(Val: constRecord.getType()))
941 return adjustGlobalStructTypeForInit(structTy, constRecord, converter,
942 dataLayout, paddingAddedIndexes,
943 failedType);
944 if (mlir::isa<cir::UnionType>(Val: constRecord.getType()))
945 return adjustGlobalUnionTypeForInit(structTy, constRecord, converter,
946 dataLayout, failedType);
947 return llvmType;
948}
949
950mlir::Type adjustGlobalTypeForInit(mlir::Type llvmType, mlir::Attribute init,
951 const mlir::TypeConverter &converter,
952 const mlir::DataLayout &dataLayout,
953 mlir::Type *failedType) {
954 llvm::SmallVector<unsigned> ignoredAddedIndexes;
955 return adjustGlobalTypeForInit(llvmType, init, converter, dataLayout,
956 paddingAddedIndexes&: ignoredAddedIndexes, failedType);
957}
958
959std::optional<mlir::Attribute>
960lowerConstRecordAttr(cir::ConstRecordAttr constRecord,
961 mlir::SymbolTableCollection &symbolTables,
962 const mlir::TypeConverter *converter,
963 const CIRPreConversionTypeMap &preConversionTypes,
964 mlir::ModuleOp moduleOp) {
965
966 // Build one constant attribute per record member. The LLVM dialect global
967 // translation accepts an ArrayAttr (one element per struct field) and emits
968 // an llvm::ConstantStruct, so the whole initializer can be a single
969 // attribute on the global instead of an insertvalue region.
970 mlir::ArrayAttr memberAttrs = constRecord.getMembers();
971 llvm::SmallVector<mlir::Attribute> loweredMembers;
972 loweredMembers.reserve(N: memberAttrs.size());
973 for (mlir::Attribute member : memberAttrs) {
974 std::optional<mlir::Attribute> lowered = lowerConstRecordMemberAttr(
975 attr: member, symbolTables, converter, preConversionTypes, moduleOp);
976 if (!lowered)
977 return std::nullopt;
978 loweredMembers.push_back(Elt: *lowered);
979 }
980
981 // The lowered LLVM type may have more fields than the CIR record has members
982 // for a few reasons:
983 // 1- a union lowers to { active-member, [pad x i8]).
984 // 2- A struct that contains such a union can have its alignment changed too,
985 // so it needs tail padding to fill that in.
986 // 3- A struct containing a union whose initializer doesn't use the
987 // highest-aligned field will have to prepend a bit of padding, such as struct
988 // { i32, union { i64, i32 } }. Typically the union gets lowered to a struct
989 // { i64 } (as i64 has the greatest alignment), but if the init causes it to
990 // be the i32(or any such smaller field) we have to prepend it with padding:
991 // struct { i32, [4 x i8], struct { i32 }}
992 // instead of (with no init):
993 // struct { i32, struct { i64 }}
994 llvm::SmallVector<unsigned> paddingAddedIndexes;
995 mlir::Type adjustedTy = adjustGlobalTypeForInit(
996 llvmType: converter->convertType(t: constRecord.getType()), init: constRecord, converter: *converter,
997 dataLayout: mlir::DataLayout(moduleOp), paddingAddedIndexes);
998 if (!adjustedTy)
999 return std::nullopt;
1000
1001 // This handles #3 from above. adjustGlobalTypeForInit ensures the
1002 // indexes are in increasing order, so we can insert 'backwards' without
1003 // causing problems.
1004 for (unsigned paddedElt : llvm::reverse(C&: paddingAddedIndexes))
1005 loweredMembers.insert(I: loweredMembers.begin() + paddedElt,
1006 Elt: mlir::LLVM::ZeroAttr::get(ctx: constRecord.getContext()));
1007
1008 // Any remaining difference will be the union/struct padding case. We don't
1009 // have a great handle/way to tell when to zero-vs-undef init, so always
1010 // zero init, as it is always safe to do so.
1011 if (auto structTy = mlir::dyn_cast<mlir::LLVM::LLVMStructType>(Val&: adjustedTy))
1012 while (loweredMembers.size() < structTy.getBody().size())
1013 loweredMembers.push_back(
1014 Elt: mlir::LLVM::ZeroAttr::get(ctx: constRecord.getContext()));
1015
1016 return mlir::ArrayAttr::get(context: constRecord.getContext(), value: loweredMembers);
1017}
1018
1019mlir::Value getConstAPInt(mlir::OpBuilder &bld, mlir::Location loc,
1020 mlir::Type typ, const llvm::APInt &val) {
1021 return mlir::LLVM::ConstantOp::create(builder&: bld, location: loc, type: typ, value: val);
1022}
1023
1024mlir::Value getConst(mlir::OpBuilder &bld, mlir::Location loc, mlir::Type typ,
1025 unsigned val) {
1026 return mlir::LLVM::ConstantOp::create(builder&: bld, location: loc, type: typ, value: val);
1027}
1028
1029mlir::Value createShL(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs) {
1030 if (!rhs)
1031 return lhs;
1032 mlir::Value rhsVal = getConst(bld, loc: lhs.getLoc(), typ: lhs.getType(), val: rhs);
1033 return mlir::LLVM::ShlOp::create(builder&: bld, location: lhs.getLoc(), lhs, rhs: rhsVal);
1034}
1035
1036mlir::Value createAShR(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs) {
1037 if (!rhs)
1038 return lhs;
1039 mlir::Value rhsVal = getConst(bld, loc: lhs.getLoc(), typ: lhs.getType(), val: rhs);
1040 return mlir::LLVM::AShrOp::create(builder&: bld, location: lhs.getLoc(), lhs, rhs: rhsVal);
1041}
1042
1043mlir::Value createAnd(mlir::OpBuilder &bld, mlir::Value lhs,
1044 const llvm::APInt &rhs) {
1045 mlir::Value rhsVal = getConstAPInt(bld, loc: lhs.getLoc(), typ: lhs.getType(), val: rhs);
1046 return mlir::LLVM::AndOp::create(builder&: bld, location: lhs.getLoc(), lhs, rhs: rhsVal);
1047}
1048
1049mlir::Value createLShR(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs) {
1050 if (!rhs)
1051 return lhs;
1052 mlir::Value rhsVal = getConst(bld, loc: lhs.getLoc(), typ: lhs.getType(), val: rhs);
1053 return mlir::LLVM::LShrOp::create(builder&: bld, location: lhs.getLoc(), lhs, rhs: rhsVal);
1054}
1055