1//===--- CGExprConstant.cpp - Emit LLVM Code from Constant Expressions ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Constant Expr nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ABIInfoImpl.h"
14#include "CGCXXABI.h"
15#include "CGObjCRuntime.h"
16#include "CGRecordLayout.h"
17#include "CodeGenFunction.h"
18#include "CodeGenModule.h"
19#include "ConstantEmitter.h"
20#include "TargetInfo.h"
21#include "clang/AST/APValue.h"
22#include "clang/AST/ASTContext.h"
23#include "clang/AST/Attr.h"
24#include "clang/AST/MatrixUtils.h"
25#include "clang/AST/NSAPI.h"
26#include "clang/AST/RecordLayout.h"
27#include "clang/AST/StmtVisitor.h"
28#include "clang/Basic/Builtins.h"
29#include "clang/CodeGenUtils/RecordLayoutUtils.h"
30#include "llvm/ADT/STLExtras.h"
31#include "llvm/ADT/Sequence.h"
32#include "llvm/Analysis/ConstantFolding.h"
33#include "llvm/IR/Constants.h"
34#include "llvm/IR/DataLayout.h"
35#include "llvm/IR/Function.h"
36#include "llvm/IR/GlobalVariable.h"
37#include "llvm/Support/SipHash.h"
38#include <optional>
39using namespace clang;
40using namespace CodeGen;
41
42//===----------------------------------------------------------------------===//
43// ConstantAggregateBuilder
44//===----------------------------------------------------------------------===//
45
46namespace {
47class ConstExprEmitter;
48
49llvm::Constant *getPadding(const CodeGenModule &CGM, CharUnits PadSize) {
50 llvm::Type *Ty = CGM.CharTy;
51 if (PadSize > CharUnits::One())
52 Ty = llvm::ArrayType::get(ElementType: Ty, NumElements: PadSize.getQuantity());
53 if (CGM.shouldZeroInitPadding()) {
54 return llvm::Constant::getNullValue(Ty);
55 }
56 return llvm::UndefValue::get(T: Ty);
57}
58
59struct ConstantAggregateBuilderUtils {
60 CodeGenModule &CGM;
61
62 ConstantAggregateBuilderUtils(CodeGenModule &CGM) : CGM(CGM) {}
63
64 CharUnits getAlignment(const llvm::Constant *C) const {
65 return CharUnits::fromQuantity(
66 Quantity: CGM.getDataLayout().getABITypeAlign(Ty: C->getType()));
67 }
68
69 CharUnits getSize(llvm::Type *Ty) const {
70 return CharUnits::fromQuantity(Quantity: CGM.getDataLayout().getTypeAllocSize(Ty));
71 }
72
73 CharUnits getSize(const llvm::Constant *C) const {
74 return getSize(Ty: C->getType());
75 }
76
77 llvm::Constant *getPadding(CharUnits PadSize) const {
78 return ::getPadding(CGM, PadSize);
79 }
80
81 llvm::Constant *getZeroes(CharUnits ZeroSize) const {
82 llvm::Type *Ty = llvm::ArrayType::get(ElementType: CGM.CharTy, NumElements: ZeroSize.getQuantity());
83 return llvm::ConstantAggregateZero::get(Ty);
84 }
85};
86
87/// Incremental builder for an llvm::Constant* holding a struct or array
88/// constant.
89class ConstantAggregateBuilder : private ConstantAggregateBuilderUtils {
90 /// The elements of the constant. These two arrays must have the same size;
91 /// Offsets[i] describes the offset of Elems[i] within the constant. The
92 /// elements are kept in increasing offset order, and we ensure that there
93 /// is no overlap: Offsets[i+1] >= Offsets[i] + getSize(Elemes[i]).
94 ///
95 /// This may contain explicit padding elements (in order to create a
96 /// natural layout), but need not. Gaps between elements are implicitly
97 /// considered to be filled with undef.
98 llvm::SmallVector<llvm::Constant*, 32> Elems;
99 llvm::SmallVector<CharUnits, 32> Offsets;
100
101 /// The size of the constant (the maximum end offset of any added element).
102 /// May be larger than the end of Elems.back() if we split the last element
103 /// and removed some trailing undefs.
104 CharUnits Size = CharUnits::Zero();
105
106 /// This is true only if laying out Elems in order as the elements of a
107 /// non-packed LLVM struct will give the correct layout.
108 bool NaturalLayout = true;
109
110 bool split(size_t Index, CharUnits Hint);
111 std::optional<size_t> splitAt(CharUnits Pos);
112
113 static llvm::Constant *buildFrom(CodeGenModule &CGM,
114 ArrayRef<llvm::Constant *> Elems,
115 ArrayRef<CharUnits> Offsets,
116 CharUnits StartOffset, CharUnits Size,
117 bool NaturalLayout, llvm::Type *DesiredTy,
118 bool AllowOversized);
119
120public:
121 ConstantAggregateBuilder(CodeGenModule &CGM)
122 : ConstantAggregateBuilderUtils(CGM) {}
123
124 /// Update or overwrite the value starting at \p Offset with \c C.
125 ///
126 /// \param AllowOverwrite If \c true, this constant might overwrite (part of)
127 /// a constant that has already been added. This flag is only used to
128 /// detect bugs.
129 bool add(llvm::Constant *C, CharUnits Offset, bool AllowOverwrite);
130
131 /// Update or overwrite the bits starting at \p OffsetInBits with \p Bits.
132 bool addBits(llvm::APInt Bits, uint64_t OffsetInBits, bool AllowOverwrite);
133
134 /// Attempt to condense the value starting at \p Offset to a constant of type
135 /// \p DesiredTy.
136 void condense(CharUnits Offset, llvm::Type *DesiredTy);
137
138 /// Produce a constant representing the entire accumulated value, ideally of
139 /// the specified type. If \p AllowOversized, the constant might be larger
140 /// than implied by \p DesiredTy (eg, if there is a flexible array member).
141 /// Otherwise, the constant will be of exactly the same size as \p DesiredTy
142 /// even if we can't represent it as that type.
143 llvm::Constant *build(llvm::Type *DesiredTy, bool AllowOversized) const {
144 return buildFrom(CGM, Elems, Offsets, StartOffset: CharUnits::Zero(), Size,
145 NaturalLayout, DesiredTy, AllowOversized);
146 }
147};
148
149template<typename Container, typename Range = std::initializer_list<
150 typename Container::value_type>>
151static void replace(Container &C, size_t BeginOff, size_t EndOff, Range Vals) {
152 assert(BeginOff <= EndOff && "invalid replacement range");
153 llvm::replace(C, C.begin() + BeginOff, C.begin() + EndOff, Vals);
154}
155
156bool ConstantAggregateBuilder::add(llvm::Constant *C, CharUnits Offset,
157 bool AllowOverwrite) {
158 // Common case: appending to a layout.
159 if (Offset >= Size) {
160 CharUnits Align = getAlignment(C);
161 CharUnits AlignedSize = Size.alignTo(Align);
162 if (AlignedSize > Offset || Offset.alignTo(Align) != Offset)
163 NaturalLayout = false;
164 else if (AlignedSize < Offset) {
165 Elems.push_back(Elt: getPadding(PadSize: Offset - Size));
166 Offsets.push_back(Elt: Size);
167 }
168 Elems.push_back(Elt: C);
169 Offsets.push_back(Elt: Offset);
170 Size = Offset + getSize(C);
171 return true;
172 }
173
174 // Uncommon case: constant overlaps what we've already created.
175 std::optional<size_t> FirstElemToReplace = splitAt(Pos: Offset);
176 if (!FirstElemToReplace)
177 return false;
178
179 CharUnits CSize = getSize(C);
180 std::optional<size_t> LastElemToReplace = splitAt(Pos: Offset + CSize);
181 if (!LastElemToReplace)
182 return false;
183
184 assert((FirstElemToReplace == LastElemToReplace || AllowOverwrite) &&
185 "unexpectedly overwriting field");
186
187 replace(C&: Elems, BeginOff: *FirstElemToReplace, EndOff: *LastElemToReplace, Vals: {C});
188 replace(C&: Offsets, BeginOff: *FirstElemToReplace, EndOff: *LastElemToReplace, Vals: {Offset});
189 Size = std::max(a: Size, b: Offset + CSize);
190 NaturalLayout = false;
191 return true;
192}
193
194bool ConstantAggregateBuilder::addBits(llvm::APInt Bits, uint64_t OffsetInBits,
195 bool AllowOverwrite) {
196 const ASTContext &Context = CGM.getContext();
197 const uint64_t CharWidth = CGM.getContext().getCharWidth();
198
199 // Offset of where we want the first bit to go within the bits of the
200 // current char.
201 unsigned OffsetWithinChar = OffsetInBits % CharWidth;
202
203 // We split bit-fields up into individual bytes. Walk over the bytes and
204 // update them.
205 for (CharUnits OffsetInChars =
206 Context.toCharUnitsFromBits(BitSize: OffsetInBits - OffsetWithinChar);
207 /**/; ++OffsetInChars) {
208 // Number of bits we want to fill in this char.
209 unsigned WantedBits =
210 std::min(a: (uint64_t)Bits.getBitWidth(), b: CharWidth - OffsetWithinChar);
211
212 // Get a char containing the bits we want in the right places. The other
213 // bits have unspecified values.
214 llvm::APInt BitsThisChar = Bits;
215 if (BitsThisChar.getBitWidth() < CharWidth)
216 BitsThisChar = BitsThisChar.zext(width: CharWidth);
217 if (CGM.getDataLayout().isBigEndian()) {
218 // Figure out how much to shift by. We may need to left-shift if we have
219 // less than one byte of Bits left.
220 int Shift = Bits.getBitWidth() - CharWidth + OffsetWithinChar;
221 if (Shift > 0)
222 BitsThisChar.lshrInPlace(ShiftAmt: Shift);
223 else if (Shift < 0)
224 BitsThisChar = BitsThisChar.shl(shiftAmt: -Shift);
225 } else {
226 BitsThisChar = BitsThisChar.shl(shiftAmt: OffsetWithinChar);
227 }
228 if (BitsThisChar.getBitWidth() > CharWidth)
229 BitsThisChar = BitsThisChar.trunc(width: CharWidth);
230
231 if (WantedBits == CharWidth) {
232 // Got a full byte: just add it directly.
233 add(C: llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: BitsThisChar),
234 Offset: OffsetInChars, AllowOverwrite);
235 } else {
236 // Partial byte: update the existing integer if there is one. If we
237 // can't split out a 1-CharUnit range to update, then we can't add
238 // these bits and fail the entire constant emission.
239 std::optional<size_t> FirstElemToUpdate = splitAt(Pos: OffsetInChars);
240 if (!FirstElemToUpdate)
241 return false;
242 std::optional<size_t> LastElemToUpdate =
243 splitAt(Pos: OffsetInChars + CharUnits::One());
244 if (!LastElemToUpdate)
245 return false;
246 assert(*LastElemToUpdate - *FirstElemToUpdate < 2 &&
247 "should have at most one element covering one byte");
248
249 // Figure out which bits we want and discard the rest.
250 llvm::APInt UpdateMask(CharWidth, 0);
251 if (CGM.getDataLayout().isBigEndian())
252 UpdateMask.setBits(loBit: CharWidth - OffsetWithinChar - WantedBits,
253 hiBit: CharWidth - OffsetWithinChar);
254 else
255 UpdateMask.setBits(loBit: OffsetWithinChar, hiBit: OffsetWithinChar + WantedBits);
256 BitsThisChar &= UpdateMask;
257
258 if (*FirstElemToUpdate == *LastElemToUpdate ||
259 Elems[*FirstElemToUpdate]->isNullValue() ||
260 isa<llvm::UndefValue>(Val: Elems[*FirstElemToUpdate])) {
261 // All existing bits are either zero or undef.
262 add(C: llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: BitsThisChar),
263 Offset: OffsetInChars, /*AllowOverwrite*/ true);
264 } else {
265 llvm::Constant *&ToUpdate = Elems[*FirstElemToUpdate];
266 // In order to perform a partial update, we need the existing bitwise
267 // value, which we can only extract for a constant int.
268 auto *CI = dyn_cast<llvm::ConstantInt>(Val: ToUpdate);
269 if (!CI)
270 return false;
271 // Because this is a 1-CharUnit range, the constant occupying it must
272 // be exactly one CharUnit wide.
273 assert(CI->getBitWidth() == CharWidth && "splitAt failed");
274 assert((!(CI->getValue() & UpdateMask) || AllowOverwrite) &&
275 "unexpectedly overwriting bitfield");
276 BitsThisChar |= (CI->getValue() & ~UpdateMask);
277 ToUpdate = llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: BitsThisChar);
278 }
279 }
280
281 // Stop if we've added all the bits.
282 if (WantedBits == Bits.getBitWidth())
283 break;
284
285 // Remove the consumed bits from Bits.
286 if (!CGM.getDataLayout().isBigEndian())
287 Bits.lshrInPlace(ShiftAmt: WantedBits);
288 Bits = Bits.trunc(width: Bits.getBitWidth() - WantedBits);
289
290 // The remanining bits go at the start of the following bytes.
291 OffsetWithinChar = 0;
292 }
293
294 return true;
295}
296
297/// Returns a position within Elems and Offsets such that all elements
298/// before the returned index end before Pos and all elements at or after
299/// the returned index begin at or after Pos. Splits elements as necessary
300/// to ensure this. Returns std::nullopt if we find something we can't split.
301std::optional<size_t> ConstantAggregateBuilder::splitAt(CharUnits Pos) {
302 if (Pos >= Size)
303 return Offsets.size();
304
305 while (true) {
306 auto FirstAfterPos = llvm::upper_bound(Range&: Offsets, Value&: Pos);
307 if (FirstAfterPos == Offsets.begin())
308 return 0;
309
310 // If we already have an element starting at Pos, we're done.
311 size_t LastAtOrBeforePosIndex = FirstAfterPos - Offsets.begin() - 1;
312 if (Offsets[LastAtOrBeforePosIndex] == Pos)
313 return LastAtOrBeforePosIndex;
314
315 // We found an element starting before Pos. Check for overlap.
316 if (Offsets[LastAtOrBeforePosIndex] +
317 getSize(C: Elems[LastAtOrBeforePosIndex]) <= Pos)
318 return LastAtOrBeforePosIndex + 1;
319
320 // Try to decompose it into smaller constants.
321 if (!split(Index: LastAtOrBeforePosIndex, Hint: Pos))
322 return std::nullopt;
323 }
324}
325
326/// Split the constant at index Index, if possible. Return true if we did.
327/// Hint indicates the location at which we'd like to split, but may be
328/// ignored.
329bool ConstantAggregateBuilder::split(size_t Index, CharUnits Hint) {
330 NaturalLayout = false;
331 llvm::Constant *C = Elems[Index];
332 CharUnits Offset = Offsets[Index];
333
334 if (auto *CA = dyn_cast<llvm::ConstantAggregate>(Val: C)) {
335 // Expand the sequence into its contained elements.
336 // FIXME: This assumes vector elements are byte-sized.
337 replace(C&: Elems, BeginOff: Index, EndOff: Index + 1,
338 Vals: llvm::map_range(C: llvm::seq(Begin: 0u, End: CA->getNumOperands()),
339 F: [&](unsigned Op) { return CA->getOperand(i_nocapture: Op); }));
340 if (isa<llvm::ArrayType>(Val: CA->getType()) ||
341 isa<llvm::VectorType>(Val: CA->getType())) {
342 // Array or vector.
343 llvm::Type *ElemTy =
344 llvm::GetElementPtrInst::getTypeAtIndex(Ty: CA->getType(), Idx: (uint64_t)0);
345 CharUnits ElemSize = getSize(Ty: ElemTy);
346 replace(
347 C&: Offsets, BeginOff: Index, EndOff: Index + 1,
348 Vals: llvm::map_range(C: llvm::seq(Begin: 0u, End: CA->getNumOperands()),
349 F: [&](unsigned Op) { return Offset + Op * ElemSize; }));
350 } else {
351 // Must be a struct.
352 auto *ST = cast<llvm::StructType>(Val: CA->getType());
353 const llvm::StructLayout *Layout =
354 CGM.getDataLayout().getStructLayout(Ty: ST);
355 replace(C&: Offsets, BeginOff: Index, EndOff: Index + 1,
356 Vals: llvm::map_range(
357 C: llvm::seq(Begin: 0u, End: CA->getNumOperands()), F: [&](unsigned Op) {
358 return Offset + CharUnits::fromQuantity(
359 Quantity: Layout->getElementOffset(Idx: Op));
360 }));
361 }
362 return true;
363 }
364
365 if (auto *CDS = dyn_cast<llvm::ConstantDataSequential>(Val: C)) {
366 // Expand the sequence into its contained elements.
367 // FIXME: This assumes vector elements are byte-sized.
368 // FIXME: If possible, split into two ConstantDataSequentials at Hint.
369 CharUnits ElemSize = getSize(Ty: CDS->getElementType());
370 replace(C&: Elems, BeginOff: Index, EndOff: Index + 1,
371 Vals: llvm::map_range(C: llvm::seq(Begin: uint64_t(0u), End: CDS->getNumElements()),
372 F: [&](uint64_t Elem) {
373 return CDS->getElementAsConstant(i: Elem);
374 }));
375 replace(C&: Offsets, BeginOff: Index, EndOff: Index + 1,
376 Vals: llvm::map_range(
377 C: llvm::seq(Begin: uint64_t(0u), End: CDS->getNumElements()),
378 F: [&](uint64_t Elem) { return Offset + Elem * ElemSize; }));
379 return true;
380 }
381
382 if (isa<llvm::ConstantAggregateZero>(Val: C)) {
383 // Split into two zeros at the hinted offset.
384 CharUnits ElemSize = getSize(C);
385 assert(Hint > Offset && Hint < Offset + ElemSize && "nothing to split");
386 replace(C&: Elems, BeginOff: Index, EndOff: Index + 1,
387 Vals: {getZeroes(ZeroSize: Hint - Offset), getZeroes(ZeroSize: Offset + ElemSize - Hint)});
388 replace(C&: Offsets, BeginOff: Index, EndOff: Index + 1, Vals: {Offset, Hint});
389 return true;
390 }
391
392 if (isa<llvm::UndefValue>(Val: C)) {
393 // Drop undef; it doesn't contribute to the final layout.
394 replace(C&: Elems, BeginOff: Index, EndOff: Index + 1, Vals: {});
395 replace(C&: Offsets, BeginOff: Index, EndOff: Index + 1, Vals: {});
396 return true;
397 }
398
399 // FIXME: We could split a ConstantInt if the need ever arose.
400 // We don't need to do this to handle bit-fields because we always eagerly
401 // split them into 1-byte chunks.
402
403 return false;
404}
405
406static llvm::Constant *
407EmitArrayConstant(CodeGenModule &CGM, llvm::ArrayType *DesiredType,
408 llvm::Type *CommonElementType, uint64_t ArrayBound,
409 SmallVectorImpl<llvm::Constant *> &Elements,
410 llvm::Constant *Filler);
411
412llvm::Constant *ConstantAggregateBuilder::buildFrom(
413 CodeGenModule &CGM, ArrayRef<llvm::Constant *> Elems,
414 ArrayRef<CharUnits> Offsets, CharUnits StartOffset, CharUnits Size,
415 bool NaturalLayout, llvm::Type *DesiredTy, bool AllowOversized) {
416 ConstantAggregateBuilderUtils Utils(CGM);
417
418 if (Elems.empty())
419 return llvm::UndefValue::get(T: DesiredTy);
420
421 auto Offset = [&](size_t I) { return Offsets[I] - StartOffset; };
422
423 // If we want an array type, see if all the elements are the same type and
424 // appropriately spaced.
425 if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(Val: DesiredTy)) {
426 assert(!AllowOversized && "oversized array emission not supported");
427
428 bool CanEmitArray = true;
429 llvm::Type *CommonType = Elems[0]->getType();
430 llvm::Constant *Filler = llvm::Constant::getNullValue(Ty: CommonType);
431 CharUnits ElemSize = Utils.getSize(Ty: ATy->getElementType());
432 SmallVector<llvm::Constant*, 32> ArrayElements;
433 for (size_t I = 0; I != Elems.size(); ++I) {
434 // Skip zeroes; we'll use a zero value as our array filler.
435 if (Elems[I]->isNullValue())
436 continue;
437
438 // All remaining elements must be the same type.
439 if (Elems[I]->getType() != CommonType ||
440 !Offset(I).isMultipleOf(N: ElemSize)) {
441 CanEmitArray = false;
442 break;
443 }
444 ArrayElements.resize(N: Offset(I) / ElemSize + 1, NV: Filler);
445 ArrayElements.back() = Elems[I];
446 }
447
448 if (CanEmitArray) {
449 return EmitArrayConstant(CGM, DesiredType: ATy, CommonElementType: CommonType, ArrayBound: ATy->getNumElements(),
450 Elements&: ArrayElements, Filler);
451 }
452
453 // Can't emit as an array, carry on to emit as a struct.
454 }
455
456 // The size of the constant we plan to generate. This is usually just
457 // the size of the initialized type, but in AllowOversized mode (i.e.
458 // flexible array init), it can be larger.
459 CharUnits DesiredSize = Utils.getSize(Ty: DesiredTy);
460 if (Size > DesiredSize) {
461 assert(AllowOversized && "Elems are oversized");
462 DesiredSize = Size;
463 }
464
465 // The natural alignment of an unpacked LLVM struct with the given elements.
466 CharUnits Align = CharUnits::One();
467 for (llvm::Constant *C : Elems)
468 Align = std::max(a: Align, b: Utils.getAlignment(C));
469
470 // The natural size of an unpacked LLVM struct with the given elements.
471 CharUnits AlignedSize = Size.alignTo(Align);
472
473 bool Packed = false;
474 ArrayRef<llvm::Constant*> UnpackedElems = Elems;
475 llvm::SmallVector<llvm::Constant*, 32> UnpackedElemStorage;
476 if (DesiredSize < AlignedSize || DesiredSize.alignTo(Align) != DesiredSize) {
477 // The natural layout would be too big; force use of a packed layout.
478 NaturalLayout = false;
479 Packed = true;
480 } else if (DesiredSize > AlignedSize) {
481 // The natural layout would be too small. Add padding to fix it. (This
482 // is ignored if we choose a packed layout.)
483 UnpackedElemStorage.assign(in_start: Elems.begin(), in_end: Elems.end());
484 UnpackedElemStorage.push_back(Elt: Utils.getPadding(PadSize: DesiredSize - Size));
485 UnpackedElems = UnpackedElemStorage;
486 }
487
488 // If we don't have a natural layout, insert padding as necessary.
489 // As we go, double-check to see if we can actually just emit Elems
490 // as a non-packed struct and do so opportunistically if possible.
491 llvm::SmallVector<llvm::Constant*, 32> PackedElems;
492 if (!NaturalLayout) {
493 CharUnits SizeSoFar = CharUnits::Zero();
494 for (size_t I = 0; I != Elems.size(); ++I) {
495 CharUnits Align = Utils.getAlignment(C: Elems[I]);
496 CharUnits NaturalOffset = SizeSoFar.alignTo(Align);
497 CharUnits DesiredOffset = Offset(I);
498 assert(DesiredOffset >= SizeSoFar && "elements out of order");
499
500 if (DesiredOffset != NaturalOffset)
501 Packed = true;
502 if (DesiredOffset != SizeSoFar)
503 PackedElems.push_back(Elt: Utils.getPadding(PadSize: DesiredOffset - SizeSoFar));
504 PackedElems.push_back(Elt: Elems[I]);
505 SizeSoFar = DesiredOffset + Utils.getSize(C: Elems[I]);
506 }
507 // If we're using the packed layout, pad it out to the desired size if
508 // necessary.
509 if (Packed) {
510 assert(SizeSoFar <= DesiredSize &&
511 "requested size is too small for contents");
512 if (SizeSoFar < DesiredSize)
513 PackedElems.push_back(Elt: Utils.getPadding(PadSize: DesiredSize - SizeSoFar));
514 }
515 }
516
517 llvm::StructType *STy = llvm::ConstantStruct::getTypeForElements(
518 Ctx&: CGM.getLLVMContext(), V: Packed ? PackedElems : UnpackedElems, Packed);
519
520 // Pick the type to use. If the type is layout identical to the desired
521 // type then use it, otherwise use whatever the builder produced for us.
522 if (llvm::StructType *DesiredSTy = dyn_cast<llvm::StructType>(Val: DesiredTy)) {
523 if (DesiredSTy->isLayoutIdentical(Other: STy))
524 STy = DesiredSTy;
525 }
526
527 return llvm::ConstantStruct::get(T: STy, V: Packed ? PackedElems : UnpackedElems);
528}
529
530void ConstantAggregateBuilder::condense(CharUnits Offset,
531 llvm::Type *DesiredTy) {
532 CharUnits Size = getSize(Ty: DesiredTy);
533
534 std::optional<size_t> FirstElemToReplace = splitAt(Pos: Offset);
535 if (!FirstElemToReplace)
536 return;
537 size_t First = *FirstElemToReplace;
538
539 std::optional<size_t> LastElemToReplace = splitAt(Pos: Offset + Size);
540 if (!LastElemToReplace)
541 return;
542 size_t Last = *LastElemToReplace;
543
544 size_t Length = Last - First;
545 if (Length == 0)
546 return;
547
548 if (Length == 1 && Offsets[First] == Offset &&
549 getSize(C: Elems[First]) == Size) {
550 // Re-wrap single element structs if necessary. Otherwise, leave any single
551 // element constant of the right size alone even if it has the wrong type.
552 auto *STy = dyn_cast<llvm::StructType>(Val: DesiredTy);
553 if (STy && STy->getNumElements() == 1 &&
554 STy->getElementType(N: 0) == Elems[First]->getType())
555 Elems[First] = llvm::ConstantStruct::get(T: STy, Vs: Elems[First]);
556 return;
557 }
558
559 llvm::Constant *Replacement = buildFrom(
560 CGM, Elems: ArrayRef(Elems).slice(N: First, M: Length),
561 Offsets: ArrayRef(Offsets).slice(N: First, M: Length), StartOffset: Offset, Size: getSize(Ty: DesiredTy),
562 /*known to have natural layout=*/NaturalLayout: false, DesiredTy, AllowOversized: false);
563 replace(C&: Elems, BeginOff: First, EndOff: Last, Vals: {Replacement});
564 replace(C&: Offsets, BeginOff: First, EndOff: Last, Vals: {Offset});
565}
566
567//===----------------------------------------------------------------------===//
568// ConstStructBuilder
569//===----------------------------------------------------------------------===//
570
571class ConstStructBuilder {
572 CodeGenModule &CGM;
573 ConstantEmitter &Emitter;
574 ConstantAggregateBuilder &Builder;
575 CharUnits StartOffset;
576
577public:
578 static llvm::Constant *BuildStruct(ConstantEmitter &Emitter,
579 const InitListExpr *ILE,
580 QualType StructTy);
581 static llvm::Constant *BuildStruct(ConstantEmitter &Emitter,
582 const APValue &Value, QualType ValTy);
583 static bool UpdateStruct(ConstantEmitter &Emitter,
584 ConstantAggregateBuilder &Const, CharUnits Offset,
585 const InitListExpr *Updater);
586
587private:
588 ConstStructBuilder(ConstantEmitter &Emitter,
589 ConstantAggregateBuilder &Builder, CharUnits StartOffset)
590 : CGM(Emitter.CGM), Emitter(Emitter), Builder(Builder),
591 StartOffset(StartOffset) {}
592
593 bool AppendField(const FieldDecl *Field, uint64_t FieldOffset,
594 llvm::Constant *InitExpr, bool AllowOverwrite = false);
595
596 bool AppendBytes(CharUnits FieldOffsetInChars, llvm::Constant *InitCst,
597 bool AllowOverwrite = false);
598
599 bool AppendBitField(const FieldDecl *Field, uint64_t FieldOffset,
600 llvm::Constant *InitExpr, bool AllowOverwrite = false);
601
602 bool Build(const InitListExpr *ILE, bool AllowOverwrite);
603 bool Build(const APValue &Val, const RecordDecl *RD, bool IsPrimaryBase,
604 const CXXRecordDecl *VTableClass, CharUnits BaseOffset,
605 bool IsCompleteClass = true);
606 bool DoZeroInitPadding(const ASTRecordLayout &Layout, unsigned FieldNo,
607 const FieldDecl &Field, bool AllowOverwrite,
608 CharUnits &SizeSoFar, bool &ZeroFieldSize);
609 bool DoZeroInitPadding(const ASTRecordLayout &Layout, bool AllowOverwrite,
610 CharUnits SizeSoFar);
611 llvm::Constant *Finalize(QualType Ty);
612};
613
614bool ConstStructBuilder::AppendField(
615 const FieldDecl *Field, uint64_t FieldOffset, llvm::Constant *InitCst,
616 bool AllowOverwrite) {
617 const ASTContext &Context = CGM.getContext();
618
619 CharUnits FieldOffsetInChars = Context.toCharUnitsFromBits(BitSize: FieldOffset);
620
621 return AppendBytes(FieldOffsetInChars, InitCst, AllowOverwrite);
622}
623
624bool ConstStructBuilder::AppendBytes(CharUnits FieldOffsetInChars,
625 llvm::Constant *InitCst,
626 bool AllowOverwrite) {
627 return Builder.add(C: InitCst, Offset: StartOffset + FieldOffsetInChars, AllowOverwrite);
628}
629
630bool ConstStructBuilder::AppendBitField(const FieldDecl *Field,
631 uint64_t FieldOffset, llvm::Constant *C,
632 bool AllowOverwrite) {
633
634 llvm::ConstantInt *CI = dyn_cast<llvm::ConstantInt>(Val: C);
635 if (!CI) {
636 // Constants for long _BitInt types are sometimes split into individual
637 // bytes. Try to fold these back into an integer constant. If that doesn't
638 // work out, then we are trying to initialize a bitfield with a non-trivial
639 // constant, this must require run-time code.
640 llvm::Type *LoadType =
641 CGM.getTypes().convertTypeForLoadStore(T: Field->getType(), LLVMTy: C->getType());
642 llvm::Constant *FoldedConstant = llvm::ConstantFoldLoadFromConst(
643 C, Ty: LoadType, Offset: llvm::APInt::getZero(numBits: 32), DL: CGM.getDataLayout());
644 CI = dyn_cast_if_present<llvm::ConstantInt>(Val: FoldedConstant);
645 if (!CI)
646 return false;
647 }
648
649 const CGRecordLayout &RL =
650 CGM.getTypes().getCGRecordLayout(Field->getParent());
651 const CGBitFieldInfo &Info = RL.getBitFieldInfo(FD: Field);
652 llvm::APInt FieldValue = CI->getValue();
653
654 // Promote the size of FieldValue if necessary
655 // FIXME: This should never occur, but currently it can because initializer
656 // constants are cast to bool, and because clang is not enforcing bitfield
657 // width limits.
658 if (Info.Size > FieldValue.getBitWidth())
659 FieldValue = FieldValue.zext(width: Info.Size);
660
661 // Truncate the size of FieldValue to the bit field size.
662 if (Info.Size < FieldValue.getBitWidth())
663 FieldValue = FieldValue.trunc(width: Info.Size);
664
665 return Builder.addBits(Bits: FieldValue,
666 OffsetInBits: CGM.getContext().toBits(CharSize: StartOffset) + FieldOffset,
667 AllowOverwrite);
668}
669
670static bool EmitDesignatedInitUpdater(ConstantEmitter &Emitter,
671 ConstantAggregateBuilder &Const,
672 CharUnits Offset, QualType Type,
673 const InitListExpr *Updater) {
674 if (Type->isRecordType())
675 return ConstStructBuilder::UpdateStruct(Emitter, Const, Offset, Updater);
676
677 auto CAT = Emitter.CGM.getContext().getAsConstantArrayType(T: Type);
678 if (!CAT)
679 return false;
680 QualType ElemType = CAT->getElementType();
681 CharUnits ElemSize = Emitter.CGM.getContext().getTypeSizeInChars(T: ElemType);
682 llvm::Type *ElemTy = Emitter.CGM.getTypes().ConvertTypeForMem(T: ElemType);
683
684 llvm::Constant *FillC = nullptr;
685 if (const Expr *Filler = Updater->getArrayFiller()) {
686 if (!isa<NoInitExpr>(Val: Filler)) {
687 FillC = Emitter.tryEmitAbstractForMemory(E: Filler, T: ElemType);
688 if (!FillC)
689 return false;
690 }
691 }
692
693 unsigned NumElementsToUpdate =
694 FillC ? CAT->getZExtSize() : Updater->getNumInits();
695 for (unsigned I = 0; I != NumElementsToUpdate; ++I, Offset += ElemSize) {
696 const Expr *Init = nullptr;
697 if (I < Updater->getNumInits())
698 Init = Updater->getInit(Init: I);
699
700 if (!Init && FillC) {
701 if (!Const.add(C: FillC, Offset, AllowOverwrite: true))
702 return false;
703 } else if (!Init || isa<NoInitExpr>(Val: Init)) {
704 continue;
705 } else if (const auto *ChildILE = dyn_cast<InitListExpr>(Val: Init)) {
706 if (!EmitDesignatedInitUpdater(Emitter, Const, Offset, Type: ElemType,
707 Updater: ChildILE))
708 return false;
709 // Attempt to reduce the array element to a single constant if necessary.
710 Const.condense(Offset, DesiredTy: ElemTy);
711 } else {
712 llvm::Constant *Val = Emitter.tryEmitPrivateForMemory(E: Init, T: ElemType);
713 if (!Const.add(C: Val, Offset, AllowOverwrite: true))
714 return false;
715 }
716 }
717
718 return true;
719}
720
721bool ConstStructBuilder::Build(const InitListExpr *ILE, bool AllowOverwrite) {
722 auto *RD = ILE->getType()->castAsRecordDecl();
723 const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(D: RD);
724
725 unsigned FieldNo = -1;
726 unsigned ElementNo = 0;
727
728 // Bail out if we have base classes. We could support these, but they only
729 // arise in C++1z where we will have already constant folded most interesting
730 // cases. FIXME: There are still a few more cases we can handle this way.
731 if (auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD))
732 if (CXXRD->getNumBases())
733 return false;
734
735 const bool ZeroInitPadding = CGM.shouldZeroInitPadding();
736 bool ZeroFieldSize = false;
737 CharUnits SizeSoFar = CharUnits::Zero();
738
739 for (FieldDecl *Field : RD->fields()) {
740 ++FieldNo;
741
742 // If this is a union, skip all the fields that aren't being initialized.
743 if (RD->isUnion() &&
744 !declaresSameEntity(D1: ILE->getInitializedFieldInUnion(), D2: Field))
745 continue;
746
747 // Don't emit anonymous bitfields.
748 if (Field->isUnnamedBitField())
749 continue;
750
751 // Get the initializer. A struct can include fields without initializers,
752 // we just use explicit null values for them.
753 const Expr *Init = nullptr;
754 if (ElementNo < ILE->getNumInits())
755 Init = ILE->getInit(Init: ElementNo++);
756 if (isa_and_nonnull<NoInitExpr>(Val: Init)) {
757 if (ZeroInitPadding &&
758 !DoZeroInitPadding(Layout, FieldNo, Field: *Field, AllowOverwrite, SizeSoFar,
759 ZeroFieldSize))
760 return false;
761 continue;
762 }
763
764 // Zero-sized fields are not emitted, but their initializers may still
765 // prevent emission of this struct as a constant.
766 if (CodeGenUtils::isEmptyFieldForLayout(Ctx: CGM.getContext(), FD: Field)) {
767 if (Init && Init->HasSideEffects(Ctx: CGM.getContext()))
768 return false;
769 continue;
770 }
771
772 if (ZeroInitPadding &&
773 !DoZeroInitPadding(Layout, FieldNo, Field: *Field, AllowOverwrite, SizeSoFar,
774 ZeroFieldSize))
775 return false;
776
777 // When emitting a DesignatedInitUpdateExpr, a nested InitListExpr
778 // represents additional overwriting of our current constant value, and not
779 // a new constant to emit independently.
780 if (AllowOverwrite &&
781 (Field->getType()->isArrayType() || Field->getType()->isRecordType())) {
782 if (auto *SubILE = dyn_cast<InitListExpr>(Val: Init)) {
783 CharUnits Offset = CGM.getContext().toCharUnitsFromBits(
784 BitSize: Layout.getFieldOffset(FieldNo));
785 if (!EmitDesignatedInitUpdater(Emitter, Const&: Builder, Offset: StartOffset + Offset,
786 Type: Field->getType(), Updater: SubILE))
787 return false;
788 // If we split apart the field's value, try to collapse it down to a
789 // single value now.
790 Builder.condense(Offset: StartOffset + Offset,
791 DesiredTy: CGM.getTypes().ConvertTypeForMem(T: Field->getType()));
792 continue;
793 }
794 }
795
796 llvm::Constant *EltInit =
797 Init ? Emitter.tryEmitPrivateForMemory(E: Init, T: Field->getType())
798 : Emitter.emitNullForMemory(T: Field->getType());
799 if (!EltInit)
800 return false;
801
802 if (ZeroInitPadding && ZeroFieldSize)
803 SizeSoFar += CharUnits::fromQuantity(
804 Quantity: CGM.getDataLayout().getTypeAllocSize(Ty: EltInit->getType()));
805
806 if (!Field->isBitField()) {
807 // Handle non-bitfield members.
808 if (!AppendField(Field, FieldOffset: Layout.getFieldOffset(FieldNo), InitCst: EltInit,
809 AllowOverwrite))
810 return false;
811 // After emitting a non-empty field with [[no_unique_address]], we may
812 // need to overwrite its tail padding.
813 if (Field->hasAttr<NoUniqueAddressAttr>())
814 AllowOverwrite = true;
815 } else {
816 // Otherwise we have a bitfield.
817 if (!AppendBitField(Field, FieldOffset: Layout.getFieldOffset(FieldNo), C: EltInit,
818 AllowOverwrite))
819 return false;
820 }
821 }
822
823 if (ZeroInitPadding && !DoZeroInitPadding(Layout, AllowOverwrite, SizeSoFar))
824 return false;
825
826 return true;
827}
828
829namespace {
830struct BaseInfo {
831 BaseInfo(const CXXRecordDecl *Decl, CharUnits Offset, unsigned Index)
832 : Decl(Decl), Offset(Offset), Index(Index) {
833 }
834
835 const CXXRecordDecl *Decl;
836 CharUnits Offset;
837 unsigned Index;
838
839 bool operator<(const BaseInfo &O) const { return Offset < O.Offset; }
840};
841}
842
843bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD,
844 bool IsPrimaryBase,
845 const CXXRecordDecl *VTableClass,
846 CharUnits Offset, bool IsCompleteClass) {
847 assert(Val.isStruct() || Val.isUnion());
848
849 const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(D: RD);
850
851 if (Val.isStruct()) {
852 if (const CXXRecordDecl *CD = dyn_cast<CXXRecordDecl>(Val: RD)) {
853 // Add a vtable pointer, if we need one and it hasn't already been added.
854 if (Layout.hasOwnVFPtr()) {
855 llvm::Constant *VTableAddressPoint =
856 CGM.getCXXABI().getVTableAddressPoint(Base: BaseSubobject(CD, Offset),
857 VTableClass);
858 if (auto Authentication =
859 CGM.getVTablePointerAuthentication(thisClass: CD,
860 /*IsVTTEntry=*/false)) {
861 VTableAddressPoint = Emitter.tryEmitConstantSignedPointer(
862 Ptr: VTableAddressPoint, Auth: *Authentication);
863 if (!VTableAddressPoint)
864 return false;
865 }
866 if (!AppendBytes(FieldOffsetInChars: Offset, InitCst: VTableAddressPoint))
867 return false;
868 }
869
870 // Accumulate and sort bases, in order to visit them in address order,
871 // which may not be the same as declaration order.
872 SmallVector<BaseInfo, 8> Bases;
873 Bases.reserve(N: Val.getStructNumBases());
874 unsigned BaseNo = 0;
875 for (const CXXBaseSpecifier &Base : CD->bases()) {
876 if (Base.isVirtual())
877 continue;
878 const CXXRecordDecl *BD = Base.getType()->getAsCXXRecordDecl();
879 CharUnits BaseOffset = Layout.getBaseClassOffset(Base: BD);
880 Bases.push_back(Elt: BaseInfo(BD, BaseOffset, BaseNo));
881 ++BaseNo;
882 }
883 llvm::stable_sort(Range&: Bases);
884
885 for (const BaseInfo &Base : Bases) {
886 bool IsPrimaryBase = Layout.getPrimaryBase() == Base.Decl;
887 if (!Build(Val: Val.getStructBase(i: Base.Index), RD: Base.Decl, IsPrimaryBase,
888 VTableClass, Offset: Offset + Base.Offset, IsCompleteClass: false))
889 return false;
890 }
891
892 if (IsCompleteClass) {
893 Bases.clear();
894 BaseNo = 0;
895 Bases.reserve(N: Val.getStructNumVirtualBases());
896 for (const CXXBaseSpecifier &Base : CD->vbases()) {
897 const CXXRecordDecl *BD = Base.getType()->getAsCXXRecordDecl();
898 CharUnits BaseOffset = Layout.getVBaseClassOffset(VBase: BD);
899 Bases.push_back(Elt: BaseInfo(BD, BaseOffset, BaseNo));
900 ++BaseNo;
901 }
902 llvm::stable_sort(Range&: Bases);
903
904 for (const BaseInfo &Base : Bases) {
905 bool IsPrimaryBase = Layout.getPrimaryBase() == Base.Decl;
906 if (!Build(Val: Val.getStructVirtualBase(i: Base.Index), RD: Base.Decl,
907 IsPrimaryBase, VTableClass, Offset: Offset + Base.Offset, IsCompleteClass: false))
908 return false;
909 }
910 }
911 }
912 }
913
914 unsigned FieldNo = 0;
915 uint64_t OffsetBits = CGM.getContext().toBits(CharSize: Offset);
916 const bool ZeroInitPadding = CGM.shouldZeroInitPadding();
917 bool ZeroFieldSize = false;
918 CharUnits SizeSoFar = CharUnits::Zero();
919
920 bool AllowOverwrite = false;
921 for (RecordDecl::field_iterator Field = RD->field_begin(),
922 FieldEnd = RD->field_end(); Field != FieldEnd; ++Field, ++FieldNo) {
923 // If this is a union, skip all the fields that aren't being initialized.
924 if (RD->isUnion() && !declaresSameEntity(D1: Val.getUnionField(), D2: *Field))
925 continue;
926
927 // Don't emit anonymous bitfields or zero-sized fields.
928 if (Field->isUnnamedBitField() ||
929 CodeGenUtils::isEmptyFieldForLayout(Ctx: CGM.getContext(), FD: *Field))
930 continue;
931
932 // Emit the value of the initializer.
933 const APValue &FieldValue =
934 RD->isUnion() ? Val.getUnionValue() : Val.getStructField(i: FieldNo);
935 llvm::Constant *EltInit =
936 Emitter.tryEmitPrivateForMemory(value: FieldValue, T: Field->getType());
937 if (!EltInit)
938 return false;
939
940 if (CGM.getContext().isPFPField(Field: *Field)) {
941 llvm::ConstantInt *Disc;
942 llvm::Constant *AddrDisc;
943 if (CGM.getContext().arePFPFieldsTriviallyCopyable(RD)) {
944 uint64_t FieldSignature =
945 llvm::getPointerAuthStableSipHash(S: CGM.getPFPFieldName(FD: *Field));
946 Disc = llvm::ConstantInt::get(Ty: CGM.Int64Ty, V: FieldSignature);
947 AddrDisc = llvm::ConstantPointerNull::get(T: CGM.VoidPtrTy);
948 } else if (Emitter.isAbstract()) {
949 // isAbstract means that we don't know the global's address. Since we
950 // can only form a pointer without knowing the address if the fields are
951 // trivially copyable, we need to return false otherwise.
952 return false;
953 } else {
954 Disc = llvm::ConstantInt::get(Ty: CGM.Int64Ty,
955 V: -(Layout.getFieldOffset(FieldNo) / 8));
956 AddrDisc = Emitter.getCurrentAddrPrivate();
957 }
958 EltInit = llvm::ConstantPtrAuth::get(
959 Ptr: EltInit, Key: llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 2), Disc, AddrDisc,
960 DeactivationSymbol: CGM.getPFPDeactivationSymbol(FD: *Field));
961 if (!CGM.getContext().arePFPFieldsTriviallyCopyable(RD))
962 Emitter.registerCurrentAddrPrivate(signal: EltInit,
963 placeholder: cast<llvm::GlobalValue>(Val: AddrDisc));
964 }
965
966 if (ZeroInitPadding) {
967 if (!DoZeroInitPadding(Layout, FieldNo, Field: **Field, AllowOverwrite,
968 SizeSoFar, ZeroFieldSize))
969 return false;
970 if (ZeroFieldSize)
971 SizeSoFar += CharUnits::fromQuantity(
972 Quantity: CGM.getDataLayout().getTypeAllocSize(Ty: EltInit->getType()));
973 }
974
975 if (!Field->isBitField()) {
976 // Handle non-bitfield members.
977 if (!AppendField(Field: *Field, FieldOffset: Layout.getFieldOffset(FieldNo) + OffsetBits,
978 InitCst: EltInit, AllowOverwrite))
979 return false;
980 // After emitting a non-empty field with [[no_unique_address]], we may
981 // need to overwrite its tail padding.
982 if (Field->hasAttr<NoUniqueAddressAttr>())
983 AllowOverwrite = true;
984 } else {
985 // Otherwise we have a bitfield.
986 if (!AppendBitField(Field: *Field, FieldOffset: Layout.getFieldOffset(FieldNo) + OffsetBits,
987 C: EltInit, AllowOverwrite))
988 return false;
989 }
990 }
991 if (ZeroInitPadding && !DoZeroInitPadding(Layout, AllowOverwrite, SizeSoFar))
992 return false;
993
994 return true;
995}
996
997bool ConstStructBuilder::DoZeroInitPadding(
998 const ASTRecordLayout &Layout, unsigned FieldNo, const FieldDecl &Field,
999 bool AllowOverwrite, CharUnits &SizeSoFar, bool &ZeroFieldSize) {
1000 uint64_t StartBitOffset = Layout.getFieldOffset(FieldNo);
1001 CharUnits StartOffset = CGM.getContext().toCharUnitsFromBits(BitSize: StartBitOffset);
1002 if (SizeSoFar < StartOffset)
1003 if (!AppendBytes(FieldOffsetInChars: SizeSoFar, InitCst: getPadding(CGM, PadSize: StartOffset - SizeSoFar),
1004 AllowOverwrite))
1005 return false;
1006
1007 if (!Field.isBitField()) {
1008 CharUnits FieldSize = CGM.getContext().getTypeSizeInChars(T: Field.getType());
1009 SizeSoFar = StartOffset + FieldSize;
1010 ZeroFieldSize = FieldSize.isZero();
1011 } else {
1012 const CGRecordLayout &RL =
1013 CGM.getTypes().getCGRecordLayout(Field.getParent());
1014 const CGBitFieldInfo &Info = RL.getBitFieldInfo(FD: &Field);
1015 uint64_t EndBitOffset = StartBitOffset + Info.Size;
1016 SizeSoFar = CGM.getContext().toCharUnitsFromBits(BitSize: EndBitOffset);
1017 if (EndBitOffset % CGM.getContext().getCharWidth() != 0) {
1018 SizeSoFar++;
1019 }
1020 ZeroFieldSize = Info.Size == 0;
1021 }
1022 return true;
1023}
1024
1025bool ConstStructBuilder::DoZeroInitPadding(const ASTRecordLayout &Layout,
1026 bool AllowOverwrite,
1027 CharUnits SizeSoFar) {
1028 CharUnits TotalSize = Layout.getSize();
1029 if (SizeSoFar < TotalSize)
1030 if (!AppendBytes(FieldOffsetInChars: SizeSoFar, InitCst: getPadding(CGM, PadSize: TotalSize - SizeSoFar),
1031 AllowOverwrite))
1032 return false;
1033 SizeSoFar = TotalSize;
1034 return true;
1035}
1036
1037llvm::Constant *ConstStructBuilder::Finalize(QualType Type) {
1038 Type = Type.getNonReferenceType();
1039 auto *RD = Type->castAsRecordDecl();
1040 llvm::Type *ValTy = CGM.getTypes().ConvertType(T: Type);
1041 return Builder.build(DesiredTy: ValTy, AllowOversized: RD->hasFlexibleArrayMember());
1042}
1043
1044llvm::Constant *ConstStructBuilder::BuildStruct(ConstantEmitter &Emitter,
1045 const InitListExpr *ILE,
1046 QualType ValTy) {
1047 ConstantAggregateBuilder Const(Emitter.CGM);
1048 ConstStructBuilder Builder(Emitter, Const, CharUnits::Zero());
1049
1050 if (!Builder.Build(ILE, /*AllowOverwrite*/false))
1051 return nullptr;
1052
1053 return Builder.Finalize(Type: ValTy);
1054}
1055
1056llvm::Constant *ConstStructBuilder::BuildStruct(ConstantEmitter &Emitter,
1057 const APValue &Val,
1058 QualType ValTy) {
1059 ConstantAggregateBuilder Const(Emitter.CGM);
1060 ConstStructBuilder Builder(Emitter, Const, CharUnits::Zero());
1061
1062 const auto *RD = ValTy->castAsRecordDecl();
1063 const CXXRecordDecl *CD = dyn_cast<CXXRecordDecl>(Val: RD);
1064 if (!Builder.Build(Val, RD, IsPrimaryBase: false, VTableClass: CD, Offset: CharUnits::Zero()))
1065 return nullptr;
1066
1067 return Builder.Finalize(Type: ValTy);
1068}
1069
1070bool ConstStructBuilder::UpdateStruct(ConstantEmitter &Emitter,
1071 ConstantAggregateBuilder &Const,
1072 CharUnits Offset,
1073 const InitListExpr *Updater) {
1074 return ConstStructBuilder(Emitter, Const, Offset)
1075 .Build(ILE: Updater, /*AllowOverwrite*/ true);
1076}
1077
1078//===----------------------------------------------------------------------===//
1079// ConstExprEmitter
1080//===----------------------------------------------------------------------===//
1081
1082static ConstantAddress
1083tryEmitGlobalCompoundLiteral(ConstantEmitter &emitter,
1084 const CompoundLiteralExpr *E) {
1085 CodeGenModule &CGM = emitter.CGM;
1086 CharUnits Align = CGM.getContext().getTypeAlignInChars(T: E->getType());
1087 if (llvm::GlobalVariable *Addr =
1088 CGM.getAddrOfConstantCompoundLiteralIfEmitted(E))
1089 return ConstantAddress(Addr, Addr->getValueType(), Align);
1090
1091 LangAS addressSpace = E->getType().getAddressSpace();
1092 llvm::Constant *C = emitter.tryEmitForInitializer(E: E->getInitializer(),
1093 destAddrSpace: addressSpace, destType: E->getType());
1094 if (!C) {
1095 assert(!E->isFileScope() &&
1096 "file-scope compound literal did not have constant initializer!");
1097 return ConstantAddress::invalid();
1098 }
1099
1100 auto GV = new llvm::GlobalVariable(
1101 CGM.getModule(), C->getType(),
1102 E->getType().isConstantStorage(Ctx: CGM.getContext(), ExcludeCtor: true, ExcludeDtor: false),
1103 llvm::GlobalValue::InternalLinkage, C, ".compoundliteral", nullptr,
1104 llvm::GlobalVariable::NotThreadLocal,
1105 CGM.getContext().getTargetAddressSpace(AS: addressSpace));
1106 emitter.finalize(global: GV);
1107 GV->setAlignment(Align.getAsAlign());
1108 CGM.setAddrOfConstantCompoundLiteral(CLE: E, GV);
1109 return ConstantAddress(GV, GV->getValueType(), Align);
1110}
1111
1112static llvm::Constant *
1113EmitArrayConstant(CodeGenModule &CGM, llvm::ArrayType *DesiredType,
1114 llvm::Type *CommonElementType, uint64_t ArrayBound,
1115 SmallVectorImpl<llvm::Constant *> &Elements,
1116 llvm::Constant *Filler) {
1117 // Figure out how long the initial prefix of non-zero elements is.
1118 uint64_t NonzeroLength = ArrayBound;
1119 if (Elements.size() < NonzeroLength && Filler->isNullValue())
1120 NonzeroLength = Elements.size();
1121 if (NonzeroLength == Elements.size()) {
1122 while (NonzeroLength > 0 && Elements[NonzeroLength - 1]->isNullValue())
1123 --NonzeroLength;
1124 }
1125
1126 if (NonzeroLength == 0)
1127 return llvm::ConstantAggregateZero::get(Ty: DesiredType);
1128
1129 // Add a zeroinitializer array filler if we have lots of trailing zeroes.
1130 uint64_t TrailingZeroes = ArrayBound - NonzeroLength;
1131 if (TrailingZeroes >= 8) {
1132 assert(Elements.size() >= NonzeroLength &&
1133 "missing initializer for non-zero element");
1134
1135 // If all the elements had the same type up to the trailing zeroes, emit a
1136 // struct of two arrays (the nonzero data and the zeroinitializer).
1137 if (CommonElementType && NonzeroLength >= 8) {
1138 llvm::Constant *Initial = llvm::ConstantArray::get(
1139 T: llvm::ArrayType::get(ElementType: CommonElementType, NumElements: NonzeroLength),
1140 V: ArrayRef(Elements).take_front(N: NonzeroLength));
1141 Elements.resize(N: 2);
1142 Elements[0] = Initial;
1143 } else {
1144 Elements.resize(N: NonzeroLength + 1);
1145 }
1146
1147 auto *FillerType =
1148 CommonElementType ? CommonElementType : DesiredType->getElementType();
1149 FillerType = llvm::ArrayType::get(ElementType: FillerType, NumElements: TrailingZeroes);
1150 Elements.back() = llvm::ConstantAggregateZero::get(Ty: FillerType);
1151 CommonElementType = nullptr;
1152 } else if (Elements.size() != ArrayBound) {
1153 // Otherwise pad to the right size with the filler if necessary.
1154 Elements.resize(N: ArrayBound, NV: Filler);
1155 if (Filler->getType() != CommonElementType)
1156 CommonElementType = nullptr;
1157 }
1158
1159 // If all elements have the same type, just emit an array constant.
1160 if (CommonElementType)
1161 return llvm::ConstantArray::get(
1162 T: llvm::ArrayType::get(ElementType: CommonElementType, NumElements: ArrayBound), V: Elements);
1163
1164 // We have mixed types. Use a packed struct.
1165 llvm::SmallVector<llvm::Type *, 16> Types;
1166 Types.reserve(N: Elements.size());
1167 for (llvm::Constant *Elt : Elements)
1168 Types.push_back(Elt: Elt->getType());
1169 llvm::StructType *SType =
1170 llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: Types, isPacked: true);
1171 return llvm::ConstantStruct::get(T: SType, V: Elements);
1172}
1173
1174// This class only needs to handle arrays, structs and unions. Outside C++11
1175// mode, we don't currently constant fold those types. All other types are
1176// handled by constant folding.
1177//
1178// Constant folding is currently missing support for a few features supported
1179// here: CK_ReinterpretMemberPointer, and DesignatedInitUpdateExpr.
1180class ConstExprEmitter
1181 : public ConstStmtVisitor<ConstExprEmitter, llvm::Constant *, QualType> {
1182 CodeGenModule &CGM;
1183 ConstantEmitter &Emitter;
1184 llvm::LLVMContext &VMContext;
1185public:
1186 ConstExprEmitter(ConstantEmitter &emitter)
1187 : CGM(emitter.CGM), Emitter(emitter), VMContext(CGM.getLLVMContext()) {
1188 }
1189
1190 //===--------------------------------------------------------------------===//
1191 // Visitor Methods
1192 //===--------------------------------------------------------------------===//
1193
1194 llvm::Constant *VisitStmt(const Stmt *S, QualType T) { return nullptr; }
1195
1196 llvm::Constant *VisitConstantExpr(const ConstantExpr *CE, QualType T) {
1197 if (llvm::Constant *Result = Emitter.tryEmitConstantExpr(CE))
1198 return Result;
1199 return Visit(S: CE->getSubExpr(), P: T);
1200 }
1201
1202 llvm::Constant *VisitParenExpr(const ParenExpr *PE, QualType T) {
1203 return Visit(S: PE->getSubExpr(), P: T);
1204 }
1205
1206 llvm::Constant *
1207 VisitSubstNonTypeTemplateParmExpr(const SubstNonTypeTemplateParmExpr *PE,
1208 QualType T) {
1209 return Visit(S: PE->getReplacement(), P: T);
1210 }
1211
1212 llvm::Constant *VisitGenericSelectionExpr(const GenericSelectionExpr *GE,
1213 QualType T) {
1214 return Visit(S: GE->getResultExpr(), P: T);
1215 }
1216
1217 llvm::Constant *VisitChooseExpr(const ChooseExpr *CE, QualType T) {
1218 return Visit(S: CE->getChosenSubExpr(), P: T);
1219 }
1220
1221 llvm::Constant *VisitCompoundLiteralExpr(const CompoundLiteralExpr *E,
1222 QualType T) {
1223 return Visit(S: E->getInitializer(), P: T);
1224 }
1225
1226 llvm::Constant *ProduceIntToIntCast(const Expr *E, QualType DestType) {
1227 QualType FromType = E->getType();
1228 // See also HandleIntToIntCast in ExprConstant.cpp
1229 if (FromType->isIntegerType())
1230 if (llvm::Constant *C = Visit(S: E, P: FromType))
1231 if (auto *CI = dyn_cast<llvm::ConstantInt>(Val: C)) {
1232 unsigned SrcWidth = CGM.getContext().getIntWidth(T: FromType);
1233 unsigned DstWidth = CGM.getContext().getIntWidth(T: DestType);
1234 if (DstWidth == SrcWidth)
1235 return CI;
1236 llvm::APInt A = FromType->isSignedIntegerType()
1237 ? CI->getValue().sextOrTrunc(width: DstWidth)
1238 : CI->getValue().zextOrTrunc(width: DstWidth);
1239 return llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: A);
1240 }
1241 return nullptr;
1242 }
1243
1244 llvm::Constant *VisitCastExpr(const CastExpr *E, QualType destType) {
1245 if (const auto *ECE = dyn_cast<ExplicitCastExpr>(Val: E))
1246 CGM.EmitExplicitCastExprType(E: ECE, CGF: Emitter.CGF);
1247 const Expr *subExpr = E->getSubExpr();
1248
1249 switch (E->getCastKind()) {
1250 case CK_ToUnion: {
1251 // GCC cast to union extension
1252 assert(E->getType()->isUnionType() &&
1253 "Destination type is not union type!");
1254
1255 auto field = E->getTargetUnionField();
1256
1257 auto C = Emitter.tryEmitPrivateForMemory(E: subExpr, T: field->getType());
1258 if (!C) return nullptr;
1259
1260 auto destTy = ConvertType(T: destType);
1261 if (C->getType() == destTy) return C;
1262
1263 // Build a struct with the union sub-element as the first member,
1264 // and padded to the appropriate size.
1265 SmallVector<llvm::Constant*, 2> Elts;
1266 SmallVector<llvm::Type*, 2> Types;
1267 Elts.push_back(Elt: C);
1268 Types.push_back(Elt: C->getType());
1269 unsigned CurSize = CGM.getDataLayout().getTypeAllocSize(Ty: C->getType());
1270 unsigned TotalSize = CGM.getDataLayout().getTypeAllocSize(Ty: destTy);
1271
1272 assert(CurSize <= TotalSize && "Union size mismatch!");
1273 if (unsigned NumPadBytes = TotalSize - CurSize) {
1274 llvm::Constant *Padding =
1275 getPadding(CGM, PadSize: CharUnits::fromQuantity(Quantity: NumPadBytes));
1276 Elts.push_back(Elt: Padding);
1277 Types.push_back(Elt: Padding->getType());
1278 }
1279
1280 llvm::StructType *STy = llvm::StructType::get(Context&: VMContext, Elements: Types, isPacked: false);
1281 return llvm::ConstantStruct::get(T: STy, V: Elts);
1282 }
1283
1284 case CK_AddressSpaceConversion: {
1285 llvm::Constant *C = Emitter.tryEmitPrivate(E: subExpr, T: subExpr->getType());
1286 if (!C)
1287 return nullptr;
1288 llvm::Type *destTy = ConvertType(T: E->getType());
1289 return CGM.performAddrSpaceCast(Src: C, DestTy: destTy);
1290 }
1291
1292 case CK_LValueToRValue: {
1293 // We don't really support doing lvalue-to-rvalue conversions here; any
1294 // interesting conversions should be done in Evaluate(). But as a
1295 // special case, allow compound literals to support the gcc extension
1296 // allowing "struct x {int x;} x = (struct x) {};".
1297 if (const auto *E =
1298 dyn_cast<CompoundLiteralExpr>(Val: subExpr->IgnoreParens()))
1299 return Visit(S: E->getInitializer(), P: destType);
1300 return nullptr;
1301 }
1302
1303 case CK_AtomicToNonAtomic:
1304 case CK_NonAtomicToAtomic:
1305 case CK_NoOp:
1306 case CK_ConstructorConversion:
1307 return Visit(S: subExpr, P: destType);
1308
1309 case CK_ArrayToPointerDecay:
1310 if (const auto *S = dyn_cast<StringLiteral>(Val: subExpr))
1311 return CGM.GetAddrOfConstantStringFromLiteral(S).getPointer();
1312 return nullptr;
1313 case CK_NullToPointer:
1314 if (Visit(S: subExpr, P: destType))
1315 return CGM.EmitNullConstant(T: destType);
1316 return nullptr;
1317
1318 case CK_IntToOCLSampler:
1319 llvm_unreachable("global sampler variables are not generated");
1320
1321 case CK_IntegralCast:
1322 return ProduceIntToIntCast(E: subExpr, DestType: destType);
1323
1324 case CK_Dependent: llvm_unreachable("saw dependent cast!");
1325
1326 case CK_BuiltinFnToFnPtr:
1327 llvm_unreachable("builtin functions are handled elsewhere");
1328
1329 case CK_ReinterpretMemberPointer:
1330 case CK_DerivedToBaseMemberPointer:
1331 case CK_BaseToDerivedMemberPointer: {
1332 auto C = Emitter.tryEmitPrivate(E: subExpr, T: subExpr->getType());
1333 if (!C) return nullptr;
1334 return CGM.getCXXABI().EmitMemberPointerConversion(E, Src: C);
1335 }
1336
1337 // These will never be supported.
1338 case CK_ObjCObjectLValueCast:
1339 case CK_ARCProduceObject:
1340 case CK_ARCConsumeObject:
1341 case CK_ARCReclaimReturnedObject:
1342 case CK_ARCExtendBlockObject:
1343 case CK_CopyAndAutoreleaseBlockObject:
1344 return nullptr;
1345
1346 // These don't need to be handled here because Evaluate knows how to
1347 // evaluate them in the cases where they can be folded.
1348 case CK_BitCast:
1349 case CK_ToVoid:
1350 case CK_Dynamic:
1351 case CK_LValueBitCast:
1352 case CK_LValueToRValueBitCast:
1353 case CK_NullToMemberPointer:
1354 case CK_UserDefinedConversion:
1355 case CK_CPointerToObjCPointerCast:
1356 case CK_BlockPointerToObjCPointerCast:
1357 case CK_AnyPointerToBlockPointerCast:
1358 case CK_FunctionToPointerDecay:
1359 case CK_BaseToDerived:
1360 case CK_DerivedToBase:
1361 case CK_UncheckedDerivedToBase:
1362 case CK_MemberPointerToBoolean:
1363 case CK_VectorSplat:
1364 case CK_FloatingRealToComplex:
1365 case CK_FloatingComplexToReal:
1366 case CK_FloatingComplexToBoolean:
1367 case CK_FloatingComplexCast:
1368 case CK_FloatingComplexToIntegralComplex:
1369 case CK_IntegralRealToComplex:
1370 case CK_IntegralComplexToReal:
1371 case CK_IntegralComplexToBoolean:
1372 case CK_IntegralComplexCast:
1373 case CK_IntegralComplexToFloatingComplex:
1374 case CK_PointerToIntegral:
1375 case CK_PointerToBoolean:
1376 case CK_BooleanToSignedIntegral:
1377 case CK_IntegralToPointer:
1378 case CK_IntegralToBoolean:
1379 case CK_IntegralToFloating:
1380 case CK_FloatingToIntegral:
1381 case CK_FloatingToBoolean:
1382 case CK_FloatingCast:
1383 case CK_FloatingToFixedPoint:
1384 case CK_FixedPointToFloating:
1385 case CK_FixedPointCast:
1386 case CK_FixedPointToBoolean:
1387 case CK_FixedPointToIntegral:
1388 case CK_IntegralToFixedPoint:
1389 case CK_ZeroToOCLOpaqueType:
1390 case CK_MatrixCast:
1391 case CK_HLSLVectorTruncation:
1392 case CK_HLSLMatrixTruncation:
1393 case CK_HLSLArrayRValue:
1394 case CK_HLSLElementwiseCast:
1395 case CK_HLSLAggregateSplatCast:
1396 return nullptr;
1397 }
1398 llvm_unreachable("Invalid CastKind");
1399 }
1400
1401 llvm::Constant *VisitCXXDefaultInitExpr(const CXXDefaultInitExpr *DIE,
1402 QualType T) {
1403 // No need for a DefaultInitExprScope: we don't handle 'this' in a
1404 // constant expression.
1405 return Visit(S: DIE->getExpr(), P: T);
1406 }
1407
1408 llvm::Constant *VisitExprWithCleanups(const ExprWithCleanups *E, QualType T) {
1409 return Visit(S: E->getSubExpr(), P: T);
1410 }
1411
1412 llvm::Constant *VisitIntegerLiteral(const IntegerLiteral *I, QualType T) {
1413 return llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: I->getValue());
1414 }
1415
1416 static APValue withDestType(ASTContext &Ctx, const Expr *E, QualType SrcType,
1417 QualType DestType, const llvm::APSInt &Value) {
1418 if (!Ctx.hasSameType(T1: SrcType, T2: DestType)) {
1419 if (DestType->isFloatingType()) {
1420 llvm::APFloat Result =
1421 llvm::APFloat(Ctx.getFloatTypeSemantics(T: DestType), 1);
1422 llvm::RoundingMode RM =
1423 E->getFPFeaturesInEffect(LO: Ctx.getLangOpts()).getRoundingMode();
1424 if (RM == llvm::RoundingMode::Dynamic)
1425 RM = llvm::RoundingMode::NearestTiesToEven;
1426 Result.convertFromAPInt(Input: Value, IsSigned: Value.isSigned(), RM);
1427 return APValue(Result);
1428 }
1429 }
1430 return APValue(Value);
1431 }
1432
1433 llvm::Constant *EmitArrayInitialization(const InitListExpr *ILE, QualType T) {
1434 auto *CAT = CGM.getContext().getAsConstantArrayType(T: ILE->getType());
1435 assert(CAT && "can't emit array init for non-constant-bound array");
1436 uint64_t NumInitElements = ILE->getNumInits();
1437 const uint64_t NumElements = CAT->getZExtSize();
1438 for (const auto *Init : ILE->inits()) {
1439 if (const auto *Embed =
1440 dyn_cast<EmbedExpr>(Val: Init->IgnoreParenImpCasts())) {
1441 NumInitElements += Embed->getDataElementCount() - 1;
1442 if (NumInitElements > NumElements) {
1443 NumInitElements = NumElements;
1444 break;
1445 }
1446 }
1447 }
1448
1449 // Initialising an array requires us to automatically
1450 // initialise any elements that have not been initialised explicitly
1451 uint64_t NumInitableElts = std::min<uint64_t>(a: NumInitElements, b: NumElements);
1452
1453 QualType EltType = CAT->getElementType();
1454
1455 // Initialize remaining array elements.
1456 llvm::Constant *fillC = nullptr;
1457 if (const Expr *filler = ILE->getArrayFiller()) {
1458 fillC = Emitter.tryEmitAbstractForMemory(E: filler, T: EltType);
1459 if (!fillC)
1460 return nullptr;
1461 }
1462
1463 // Copy initializer elements.
1464 SmallVector<llvm::Constant *, 16> Elts;
1465 if (fillC && fillC->isNullValue())
1466 Elts.reserve(N: NumInitableElts + 1);
1467 else
1468 Elts.reserve(N: NumElements);
1469
1470 llvm::Type *CommonElementType = nullptr;
1471 auto Emit = [&](const Expr *Init, unsigned ArrayIndex) {
1472 llvm::Constant *C = nullptr;
1473 C = Emitter.tryEmitPrivateForMemory(E: Init, T: EltType);
1474 if (!C)
1475 return false;
1476 if (ArrayIndex == 0)
1477 CommonElementType = C->getType();
1478 else if (C->getType() != CommonElementType)
1479 CommonElementType = nullptr;
1480 Elts.push_back(Elt: C);
1481 return true;
1482 };
1483
1484 unsigned ArrayIndex = 0;
1485 QualType DestTy = CAT->getElementType();
1486 for (unsigned i = 0; i < ILE->getNumInits(); ++i) {
1487 const Expr *Init = ILE->getInit(Init: i);
1488 if (auto *EmbedS = dyn_cast<EmbedExpr>(Val: Init->IgnoreParenImpCasts())) {
1489 StringLiteral *SL = EmbedS->getDataStringLiteral();
1490 llvm::APSInt Value(CGM.getContext().getTypeSize(T: DestTy),
1491 DestTy->isUnsignedIntegerType());
1492 llvm::Constant *C;
1493 for (unsigned I = EmbedS->getStartingElementPos(),
1494 N = EmbedS->getDataElementCount();
1495 I != EmbedS->getStartingElementPos() + N; ++I) {
1496 Value = SL->getCodeUnit(I);
1497 if (DestTy->isIntegerType()) {
1498 C = llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: Value);
1499 } else {
1500 C = Emitter.tryEmitPrivateForMemory(
1501 value: withDestType(Ctx&: CGM.getContext(), E: Init, SrcType: EmbedS->getType(), DestType: DestTy,
1502 Value),
1503 T: EltType);
1504 }
1505 if (!C)
1506 return nullptr;
1507 Elts.push_back(Elt: C);
1508 ArrayIndex++;
1509 }
1510 if ((ArrayIndex - EmbedS->getDataElementCount()) == 0)
1511 CommonElementType = C->getType();
1512 else if (C->getType() != CommonElementType)
1513 CommonElementType = nullptr;
1514 } else {
1515 if (!Emit(Init, ArrayIndex))
1516 return nullptr;
1517 ArrayIndex++;
1518 }
1519 }
1520
1521 llvm::ArrayType *Desired =
1522 cast<llvm::ArrayType>(Val: CGM.getTypes().ConvertType(T: ILE->getType()));
1523 return EmitArrayConstant(CGM, DesiredType: Desired, CommonElementType, ArrayBound: NumElements, Elements&: Elts,
1524 Filler: fillC);
1525 }
1526
1527 llvm::Constant *EmitRecordInitialization(const InitListExpr *ILE,
1528 QualType T) {
1529 return ConstStructBuilder::BuildStruct(Emitter, ILE, ValTy: T);
1530 }
1531
1532 llvm::Constant *VisitImplicitValueInitExpr(const ImplicitValueInitExpr *E,
1533 QualType T) {
1534 return CGM.EmitNullConstant(T);
1535 }
1536
1537 llvm::Constant *VisitInitListExpr(const InitListExpr *ILE, QualType T) {
1538 if (ILE->isTransparent())
1539 return Visit(S: ILE->getInit(Init: 0), P: T);
1540
1541 if (ILE->getType()->isArrayType())
1542 return EmitArrayInitialization(ILE, T);
1543
1544 if (ILE->getType()->isRecordType())
1545 return EmitRecordInitialization(ILE, T);
1546
1547 return nullptr;
1548 }
1549
1550 llvm::Constant *
1551 VisitDesignatedInitUpdateExpr(const DesignatedInitUpdateExpr *E,
1552 QualType destType) {
1553 auto C = Visit(S: E->getBase(), P: destType);
1554 if (!C)
1555 return nullptr;
1556
1557 ConstantAggregateBuilder Const(CGM);
1558 Const.add(C, Offset: CharUnits::Zero(), AllowOverwrite: false);
1559
1560 if (!EmitDesignatedInitUpdater(Emitter, Const, Offset: CharUnits::Zero(), Type: destType,
1561 Updater: E->getUpdater()))
1562 return nullptr;
1563
1564 llvm::Type *ValTy = CGM.getTypes().ConvertType(T: destType);
1565 bool HasFlexibleArray = false;
1566 if (const auto *RD = destType->getAsRecordDecl())
1567 HasFlexibleArray = RD->hasFlexibleArrayMember();
1568 return Const.build(DesiredTy: ValTy, AllowOversized: HasFlexibleArray);
1569 }
1570
1571 llvm::Constant *VisitCXXConstructExpr(const CXXConstructExpr *E,
1572 QualType Ty) {
1573 if (!E->getConstructor()->isTrivial())
1574 return nullptr;
1575
1576 // Only default and copy/move constructors can be trivial.
1577 if (E->getNumArgs()) {
1578 assert(E->getNumArgs() == 1 && "trivial ctor with > 1 argument");
1579 assert(E->getConstructor()->isCopyOrMoveConstructor() &&
1580 "trivial ctor has argument but isn't a copy/move ctor");
1581
1582 const Expr *Arg = E->getArg(Arg: 0);
1583 assert(CGM.getContext().hasSameUnqualifiedType(Ty, Arg->getType()) &&
1584 "argument to copy ctor is of wrong type");
1585
1586 // Look through the temporary; it's just converting the value to an
1587 // lvalue to pass it to the constructor.
1588 if (const auto *MTE = dyn_cast<MaterializeTemporaryExpr>(Val: Arg))
1589 return Visit(S: MTE->getSubExpr(), P: Ty);
1590 // Don't try to support arbitrary lvalue-to-rvalue conversions for now.
1591 return nullptr;
1592 }
1593
1594 return CGM.EmitNullConstant(T: Ty);
1595 }
1596
1597 llvm::Constant *VisitStringLiteral(const StringLiteral *E, QualType T) {
1598 // This is a string literal initializing an array in an initializer.
1599 return CGM.GetConstantArrayFromStringLiteral(E);
1600 }
1601
1602 llvm::Constant *VisitObjCEncodeExpr(const ObjCEncodeExpr *E, QualType T) {
1603 // This must be an @encode initializing an array in a static initializer.
1604 // Don't emit it as the address of the string, emit the string data itself
1605 // as an inline array.
1606 std::string Str;
1607 CGM.getContext().getObjCEncodingForType(T: E->getEncodedType(), S&: Str);
1608 const ConstantArrayType *CAT = CGM.getContext().getAsConstantArrayType(T);
1609 assert(CAT && "String data not of constant array type!");
1610
1611 // Resize the string to the right size, adding zeros at the end, or
1612 // truncating as needed.
1613 Str.resize(n: CAT->getZExtSize(), c: '\0');
1614 return llvm::ConstantDataArray::getString(Context&: VMContext, Initializer: Str, AddNull: false);
1615 }
1616
1617 llvm::Constant *VisitUnaryExtension(const UnaryOperator *E, QualType T) {
1618 return Visit(S: E->getSubExpr(), P: T);
1619 }
1620
1621 llvm::Constant *VisitUnaryMinus(const UnaryOperator *U, QualType T) {
1622 if (llvm::Constant *C = Visit(S: U->getSubExpr(), P: T))
1623 if (auto *CI = dyn_cast<llvm::ConstantInt>(Val: C))
1624 return llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: -CI->getValue());
1625 return nullptr;
1626 }
1627
1628 llvm::Constant *VisitPackIndexingExpr(const PackIndexingExpr *E, QualType T) {
1629 return Visit(S: E->getSelectedExpr(), P: T);
1630 }
1631
1632 // Utility methods
1633 llvm::Type *ConvertType(QualType T) {
1634 return CGM.getTypes().ConvertType(T);
1635 }
1636};
1637
1638} // end anonymous namespace.
1639
1640llvm::Constant *ConstantEmitter::validateAndPopAbstract(llvm::Constant *C,
1641 AbstractState saved) {
1642 Abstract = saved.OldValue;
1643
1644 assert(saved.OldPlaceholdersSize == PlaceholderAddresses.size() &&
1645 "created a placeholder while doing an abstract emission?");
1646
1647 // No validation necessary for now.
1648 // No cleanup to do for now.
1649 return C;
1650}
1651
1652llvm::Constant *
1653ConstantEmitter::tryEmitAbstractForInitializer(const VarDecl &D) {
1654 auto state = pushAbstract();
1655 auto C = tryEmitPrivateForVarInit(D);
1656 return validateAndPopAbstract(C, saved: state);
1657}
1658
1659llvm::Constant *
1660ConstantEmitter::tryEmitAbstract(const Expr *E, QualType destType) {
1661 auto state = pushAbstract();
1662 auto C = tryEmitPrivate(E, T: destType);
1663 return validateAndPopAbstract(C, saved: state);
1664}
1665
1666llvm::Constant *
1667ConstantEmitter::tryEmitAbstract(const APValue &value, QualType destType) {
1668 auto state = pushAbstract();
1669 auto C = tryEmitPrivate(value, T: destType);
1670 return validateAndPopAbstract(C, saved: state);
1671}
1672
1673llvm::Constant *ConstantEmitter::tryEmitConstantExpr(const ConstantExpr *CE) {
1674 if (!CE->hasAPValueResult())
1675 return nullptr;
1676
1677 QualType RetType = CE->getType();
1678 if (CE->isGLValue())
1679 RetType = CGM.getContext().getLValueReferenceType(T: RetType);
1680
1681 return tryEmitAbstract(value: CE->getAPValueResult(), destType: RetType);
1682}
1683
1684llvm::Constant *
1685ConstantEmitter::emitAbstract(const Expr *E, QualType destType) {
1686 auto state = pushAbstract();
1687 auto C = tryEmitPrivate(E, T: destType);
1688 C = validateAndPopAbstract(C, saved: state);
1689 if (!C) {
1690 CGM.Error(loc: E->getExprLoc(),
1691 error: "internal error: could not emit constant value \"abstractly\"");
1692 C = CGM.EmitNullConstant(T: destType);
1693 }
1694 return C;
1695}
1696
1697llvm::Constant *
1698ConstantEmitter::emitAbstract(SourceLocation loc, const APValue &value,
1699 QualType destType,
1700 bool EnablePtrAuthFunctionTypeDiscrimination) {
1701 auto state = pushAbstract();
1702 auto C =
1703 tryEmitPrivate(value, T: destType, EnablePtrAuthFunctionTypeDiscrimination);
1704 C = validateAndPopAbstract(C, saved: state);
1705 if (!C) {
1706 CGM.Error(loc,
1707 error: "internal error: could not emit constant value \"abstractly\"");
1708 C = CGM.EmitNullConstant(T: destType);
1709 }
1710 return C;
1711}
1712
1713llvm::Constant *ConstantEmitter::tryEmitForInitializer(const VarDecl &D) {
1714 initializeNonAbstract(destAS: D.getType().getAddressSpace());
1715 llvm::Constant *Init = tryEmitPrivateForVarInit(D);
1716
1717 // If a placeholder address was needed for a TLS variable, implying that the
1718 // initializer's value depends on its address, then the object may not be
1719 // initialized in .tdata because the initializer will be memcpy'd to the
1720 // thread's TLS. Instead the initialization must be done in code.
1721 if (!PlaceholderAddresses.empty() && D.getTLSKind() != VarDecl::TLS_None) {
1722 for (auto [_, GV] : PlaceholderAddresses)
1723 GV->eraseFromParent();
1724 PlaceholderAddresses.clear();
1725 Init = nullptr;
1726 }
1727
1728 return markIfFailed(init: Init);
1729}
1730
1731llvm::Constant *ConstantEmitter::tryEmitForInitializer(const Expr *E,
1732 LangAS destAddrSpace,
1733 QualType destType) {
1734 initializeNonAbstract(destAS: destAddrSpace);
1735 return markIfFailed(init: tryEmitPrivateForMemory(E, T: destType));
1736}
1737
1738llvm::Constant *ConstantEmitter::emitForInitializer(const APValue &value,
1739 LangAS destAddrSpace,
1740 QualType destType) {
1741 initializeNonAbstract(destAS: destAddrSpace);
1742 auto C = tryEmitPrivateForMemory(value, T: destType);
1743 assert(C && "couldn't emit constant value non-abstractly?");
1744 return C;
1745}
1746
1747llvm::GlobalValue *ConstantEmitter::getCurrentAddrPrivate() {
1748 assert(!Abstract && "cannot get current address for abstract constant");
1749
1750
1751
1752 // Make an obviously ill-formed global that should blow up compilation
1753 // if it survives.
1754 auto global = new llvm::GlobalVariable(CGM.getModule(), CGM.Int8Ty, true,
1755 llvm::GlobalValue::PrivateLinkage,
1756 /*init*/ nullptr,
1757 /*name*/ "",
1758 /*before*/ nullptr,
1759 llvm::GlobalVariable::NotThreadLocal,
1760 CGM.getContext().getTargetAddressSpace(AS: DestAddressSpace));
1761
1762 PlaceholderAddresses.push_back(Elt: std::make_pair(x: nullptr, y&: global));
1763
1764 return global;
1765}
1766
1767void ConstantEmitter::registerCurrentAddrPrivate(llvm::Constant *signal,
1768 llvm::GlobalValue *placeholder) {
1769 assert(!PlaceholderAddresses.empty());
1770 assert(PlaceholderAddresses.back().first == nullptr);
1771 assert(PlaceholderAddresses.back().second == placeholder);
1772 PlaceholderAddresses.back().first = signal;
1773}
1774
1775namespace {
1776 struct ReplacePlaceholders {
1777 CodeGenModule &CGM;
1778
1779 /// The base address of the global.
1780 llvm::Constant *Base;
1781 llvm::Type *BaseValueTy = nullptr;
1782
1783 /// The placeholder addresses that were registered during emission.
1784 llvm::DenseMap<llvm::Constant*, llvm::GlobalVariable*> PlaceholderAddresses;
1785
1786 /// The locations of the placeholder signals.
1787 llvm::DenseMap<llvm::GlobalVariable*, llvm::Constant*> Locations;
1788
1789 /// The current index stack. We use a simple unsigned stack because
1790 /// we assume that placeholders will be relatively sparse in the
1791 /// initializer, but we cache the index values we find just in case.
1792 llvm::SmallVector<unsigned, 8> Indices;
1793 llvm::SmallVector<llvm::Constant*, 8> IndexValues;
1794
1795 ReplacePlaceholders(CodeGenModule &CGM, llvm::Constant *base,
1796 ArrayRef<std::pair<llvm::Constant*,
1797 llvm::GlobalVariable*>> addresses)
1798 : CGM(CGM), Base(base),
1799 PlaceholderAddresses(addresses.begin(), addresses.end()) {
1800 }
1801
1802 void replaceInInitializer(llvm::Constant *init) {
1803 // Remember the type of the top-most initializer.
1804 BaseValueTy = init->getType();
1805
1806 // Initialize the stack.
1807 Indices.push_back(Elt: 0);
1808 IndexValues.push_back(Elt: nullptr);
1809
1810 // Recurse into the initializer.
1811 findLocations(init);
1812
1813 // Check invariants.
1814 assert(IndexValues.size() == Indices.size() && "mismatch");
1815 assert(Indices.size() == 1 && "didn't pop all indices");
1816
1817 // Do the replacement; this basically invalidates 'init'.
1818 assert(Locations.size() == PlaceholderAddresses.size() &&
1819 "missed a placeholder?");
1820
1821 // We're iterating over a hashtable, so this would be a source of
1822 // non-determinism in compiler output *except* that we're just
1823 // messing around with llvm::Constant structures, which never itself
1824 // does anything that should be visible in compiler output.
1825 for (auto &entry : Locations) {
1826 assert(entry.first->getName() == "" && "not a placeholder!");
1827 entry.first->replaceAllUsesWith(V: entry.second);
1828 entry.first->eraseFromParent();
1829 }
1830 }
1831
1832 private:
1833 void findLocations(llvm::Constant *init) {
1834 // Recurse into aggregates.
1835 if (auto agg = dyn_cast<llvm::ConstantAggregate>(Val: init)) {
1836 for (unsigned i = 0, e = agg->getNumOperands(); i != e; ++i) {
1837 Indices.push_back(Elt: i);
1838 IndexValues.push_back(Elt: nullptr);
1839
1840 findLocations(init: agg->getOperand(i_nocapture: i));
1841
1842 IndexValues.pop_back();
1843 Indices.pop_back();
1844 }
1845 return;
1846 }
1847
1848 // Otherwise, check for registered constants.
1849 while (true) {
1850 auto it = PlaceholderAddresses.find(Val: init);
1851 if (it != PlaceholderAddresses.end()) {
1852 setLocation(it->second);
1853 break;
1854 }
1855
1856 // Look through bitcasts or other expressions.
1857 if (auto expr = dyn_cast<llvm::ConstantExpr>(Val: init)) {
1858 init = expr->getOperand(i_nocapture: 0);
1859 } else {
1860 break;
1861 }
1862 }
1863 }
1864
1865 void setLocation(llvm::GlobalVariable *placeholder) {
1866 assert(!Locations.contains(placeholder) &&
1867 "already found location for placeholder!");
1868
1869 // Lazily fill in IndexValues with the values from Indices.
1870 // We do this in reverse because we should always have a strict
1871 // prefix of indices from the start.
1872 assert(Indices.size() == IndexValues.size());
1873 for (size_t i = Indices.size() - 1; i != size_t(-1); --i) {
1874 if (IndexValues[i]) {
1875#ifndef NDEBUG
1876 for (size_t j = 0; j != i + 1; ++j) {
1877 assert(IndexValues[j] &&
1878 isa<llvm::ConstantInt>(IndexValues[j]) &&
1879 cast<llvm::ConstantInt>(IndexValues[j])->getZExtValue()
1880 == Indices[j]);
1881 }
1882#endif
1883 break;
1884 }
1885
1886 IndexValues[i] = llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: Indices[i]);
1887 }
1888
1889 llvm::Constant *location = llvm::ConstantExpr::getGetElementPtr(
1890 DL: CGM.getDataLayout(), Ty: BaseValueTy, C: Base, IdxList: IndexValues,
1891 NW: llvm::GEPNoWrapFlags::inBounds());
1892
1893 Locations.insert(KV: {placeholder, location});
1894 }
1895 };
1896}
1897
1898void ConstantEmitter::finalize(llvm::GlobalVariable *global) {
1899 assert(InitializedNonAbstract &&
1900 "finalizing emitter that was used for abstract emission?");
1901 assert(!Finalized && "finalizing emitter multiple times");
1902 assert(global->getInitializer());
1903
1904 // Note that we might also be Failed.
1905 Finalized = true;
1906
1907 if (!PlaceholderAddresses.empty()) {
1908 ReplacePlaceholders(CGM, global, PlaceholderAddresses)
1909 .replaceInInitializer(init: global->getInitializer());
1910 PlaceholderAddresses.clear(); // satisfy
1911 }
1912}
1913
1914ConstantEmitter::~ConstantEmitter() {
1915 assert((!InitializedNonAbstract || Finalized || Failed) &&
1916 "not finalized after being initialized for non-abstract emission");
1917 assert(PlaceholderAddresses.empty() && "unhandled placeholders");
1918}
1919
1920static QualType getNonMemoryType(CodeGenModule &CGM, QualType type) {
1921 if (auto AT = type->getAs<AtomicType>()) {
1922 return CGM.getContext().getQualifiedType(T: AT->getValueType(),
1923 Qs: type.getQualifiers());
1924 }
1925 return type;
1926}
1927
1928llvm::Constant *ConstantEmitter::tryEmitPrivateForVarInit(const VarDecl &D) {
1929 // Make a quick check if variable can be default NULL initialized
1930 // and avoid going through rest of code which may do, for c++11,
1931 // initialization of memory to all NULLs.
1932 if (!D.hasLocalStorage()) {
1933 QualType Ty = CGM.getContext().getBaseElementType(QT: D.getType());
1934 if (Ty->isRecordType())
1935 if (const CXXConstructExpr *E =
1936 dyn_cast_or_null<CXXConstructExpr>(Val: D.getInit())) {
1937 const CXXConstructorDecl *CD = E->getConstructor();
1938 if (CD->isTrivial() && CD->isDefaultConstructor())
1939 return CGM.EmitNullConstant(T: D.getType());
1940 }
1941 }
1942 InConstantContext = D.hasConstantInitialization();
1943
1944 QualType destType = D.getType();
1945 const Expr *E = D.getInit();
1946 assert(E && "No initializer to emit");
1947
1948 if (!destType->isReferenceType()) {
1949 QualType nonMemoryDestType = getNonMemoryType(CGM, type: destType);
1950 if (llvm::Constant *C = ConstExprEmitter(*this).Visit(S: E, P: nonMemoryDestType))
1951 return emitForMemory(C, T: destType);
1952 }
1953
1954 // Try to emit the initializer. Note that this can allow some things that
1955 // are not allowed by tryEmitPrivateForMemory alone.
1956 if (const APValue *value = D.evaluateValue()) {
1957 assert(!value->allowConstexprUnknown() &&
1958 "Constexpr unknown values are not allowed in CodeGen");
1959 return tryEmitPrivateForMemory(value: *value, T: destType);
1960 }
1961
1962 return nullptr;
1963}
1964
1965llvm::Constant *
1966ConstantEmitter::tryEmitAbstractForMemory(const Expr *E, QualType destType) {
1967 auto nonMemoryDestType = getNonMemoryType(CGM, type: destType);
1968 auto C = tryEmitAbstract(E, destType: nonMemoryDestType);
1969 return (C ? emitForMemory(C, T: destType) : nullptr);
1970}
1971
1972llvm::Constant *
1973ConstantEmitter::tryEmitAbstractForMemory(const APValue &value,
1974 QualType destType) {
1975 auto nonMemoryDestType = getNonMemoryType(CGM, type: destType);
1976 auto C = tryEmitAbstract(value, destType: nonMemoryDestType);
1977 return (C ? emitForMemory(C, T: destType) : nullptr);
1978}
1979
1980llvm::Constant *ConstantEmitter::tryEmitPrivateForMemory(const Expr *E,
1981 QualType destType) {
1982 auto nonMemoryDestType = getNonMemoryType(CGM, type: destType);
1983 llvm::Constant *C = tryEmitPrivate(E, T: nonMemoryDestType);
1984 return (C ? emitForMemory(C, T: destType) : nullptr);
1985}
1986
1987llvm::Constant *ConstantEmitter::tryEmitPrivateForMemory(const APValue &value,
1988 QualType destType) {
1989 auto nonMemoryDestType = getNonMemoryType(CGM, type: destType);
1990 auto C = tryEmitPrivate(value, T: nonMemoryDestType);
1991 return (C ? emitForMemory(C, T: destType) : nullptr);
1992}
1993
1994/// Try to emit a constant signed pointer, given a raw pointer and the
1995/// destination ptrauth qualifier.
1996///
1997/// This can fail if the qualifier needs address discrimination and the
1998/// emitter is in an abstract mode.
1999llvm::Constant *
2000ConstantEmitter::tryEmitConstantSignedPointer(llvm::Constant *UnsignedPointer,
2001 PointerAuthQualifier Schema) {
2002 assert(Schema && "applying trivial ptrauth schema");
2003
2004 if (Schema.hasKeyNone())
2005 return UnsignedPointer;
2006
2007 unsigned Key = Schema.getKey();
2008
2009 // Create an address placeholder if we're using address discrimination.
2010 llvm::GlobalValue *StorageAddress = nullptr;
2011 if (Schema.isAddressDiscriminated()) {
2012 // We can't do this if the emitter is in an abstract state.
2013 if (isAbstract())
2014 return nullptr;
2015
2016 StorageAddress = getCurrentAddrPrivate();
2017 }
2018
2019 llvm::ConstantInt *Discriminator =
2020 llvm::ConstantInt::get(Ty: CGM.IntPtrTy, V: Schema.getExtraDiscriminator());
2021
2022 llvm::Constant *SignedPointer = CGM.getConstantSignedPointer(
2023 Pointer: UnsignedPointer, Key, StorageAddress, OtherDiscriminator: Discriminator);
2024
2025 if (Schema.isAddressDiscriminated())
2026 registerCurrentAddrPrivate(signal: SignedPointer, placeholder: StorageAddress);
2027
2028 return SignedPointer;
2029}
2030
2031llvm::Constant *ConstantEmitter::emitForMemory(CodeGenModule &CGM,
2032 llvm::Constant *C,
2033 QualType destType) {
2034 // For an _Atomic-qualified constant, we may need to add tail padding.
2035 if (auto AT = destType->getAs<AtomicType>()) {
2036 QualType destValueType = AT->getValueType();
2037 C = emitForMemory(CGM, C, destType: destValueType);
2038
2039 uint64_t innerSize = CGM.getContext().getTypeSize(T: destValueType);
2040 uint64_t outerSize = CGM.getContext().getTypeSize(T: destType);
2041 if (innerSize == outerSize)
2042 return C;
2043
2044 assert(innerSize < outerSize && "emitted over-large constant for atomic");
2045 llvm::Constant *elts[] = {
2046 C,
2047 llvm::ConstantAggregateZero::get(
2048 Ty: llvm::ArrayType::get(ElementType: CGM.Int8Ty, NumElements: (outerSize - innerSize) / 8))
2049 };
2050 return llvm::ConstantStruct::getAnon(V: elts);
2051 }
2052
2053 // Zero-extend bool.
2054 // In HLSL bool vectors are stored in memory as a vector of i32
2055 if ((C->getType()->isIntegerTy(BitWidth: 1) && !destType->isBitIntType()) ||
2056 (destType->isExtVectorBoolType() &&
2057 !destType->isPackedVectorBoolType(ctx: CGM.getContext()))) {
2058 llvm::Type *boolTy = CGM.getTypes().ConvertTypeForMem(T: destType);
2059 llvm::Constant *Res = llvm::ConstantFoldCastOperand(
2060 Opcode: llvm::Instruction::ZExt, C, DestTy: boolTy, DL: CGM.getDataLayout());
2061 assert(Res && "Constant folding must succeed");
2062 return Res;
2063 }
2064
2065 if (destType->isBitIntType()) {
2066 llvm::Type *MemTy = CGM.getTypes().ConvertTypeForMem(T: destType);
2067 if (C->getType() != MemTy) {
2068 ConstantAggregateBuilder Builder(CGM);
2069 llvm::Type *LoadStoreTy =
2070 CGM.getTypes().convertTypeForLoadStore(T: destType);
2071 // ptrtoint/inttoptr should not involve _BitInt in constant expressions,
2072 // so casting to ConstantInt is safe here.
2073 auto *CI = cast<llvm::ConstantInt>(Val: C);
2074 llvm::Constant *Res = llvm::ConstantFoldCastOperand(
2075 Opcode: destType->isSignedIntegerOrEnumerationType()
2076 ? llvm::Instruction::SExt
2077 : llvm::Instruction::ZExt,
2078 C: CI, DestTy: LoadStoreTy, DL: CGM.getDataLayout());
2079 if (CGM.getTypes().typeRequiresSplitIntoByteArray(ASTTy: destType,
2080 LLVMTy: C->getType())) {
2081 // Long _BitInt has array of bytes as in-memory type.
2082 // So, split constant into individual bytes.
2083 llvm::APInt Value = cast<llvm::ConstantInt>(Val: Res)->getValue();
2084 Builder.addBits(Bits: Value, /*OffsetInBits=*/0, /*AllowOverwrite=*/false);
2085 return Builder.build(DesiredTy: MemTy, /*AllowOversized*/ false);
2086 }
2087 return Res;
2088 }
2089 }
2090
2091 if (destType->isConstantMatrixType() &&
2092 isMatrixRowMajor(LangOpts: CGM.getLangOpts(), T: destType)) {
2093 const auto *MT = destType->castAs<ConstantMatrixType>();
2094 SmallVector<llvm::Constant *, 16> Inits(MT->getNumElementsFlattened());
2095 for (unsigned Row = 0; Row != MT->getNumRows(); ++Row)
2096 for (unsigned Col = 0; Col != MT->getNumColumns(); ++Col)
2097 Inits[MT->getRowMajorFlattenedIndex(Row, Column: Col)] =
2098 C->getAggregateElement(Elt: MT->getColumnMajorFlattenedIndex(Row, Column: Col));
2099 llvm::Constant *MemoryValue = llvm::ConstantVector::get(V: Inits);
2100 if (destType->isConstantMatrixBoolType()) {
2101 llvm::Constant *Res = llvm::ConstantFoldCastOperand(
2102 Opcode: llvm::Instruction::ZExt, C: MemoryValue,
2103 DestTy: CGM.getTypes().convertTypeForLoadStore(T: destType),
2104 DL: CGM.getDataLayout());
2105 assert(Res && "Constant folding must succeed");
2106 return Res;
2107 }
2108 return MemoryValue;
2109 }
2110
2111 return C;
2112}
2113
2114llvm::Constant *ConstantEmitter::tryEmitPrivate(const Expr *E,
2115 QualType destType) {
2116 assert(!destType->isVoidType() && "can't emit a void constant");
2117
2118 if (!destType->isReferenceType())
2119 if (llvm::Constant *C = ConstExprEmitter(*this).Visit(S: E, P: destType))
2120 return C;
2121
2122 Expr::EvalResult Result;
2123
2124 bool Success = false;
2125
2126 if (destType->isReferenceType())
2127 Success = E->EvaluateAsLValue(Result, Ctx: CGM.getContext());
2128 else
2129 Success = E->EvaluateAsRValue(Result, Ctx: CGM.getContext(), InConstantContext);
2130
2131 if (Success && !Result.HasSideEffects)
2132 return tryEmitPrivate(value: Result.Val, T: destType);
2133
2134 return nullptr;
2135}
2136
2137llvm::Constant *CodeGenModule::getNullPointer(llvm::PointerType *T, QualType QT) {
2138 return getTargetCodeGenInfo().getNullPointer(CGM: *this, T, QT);
2139}
2140
2141namespace {
2142/// A struct which can be used to peephole certain kinds of finalization
2143/// that normally happen during l-value emission.
2144struct ConstantLValue {
2145 llvm::Constant *Value;
2146 bool HasOffsetApplied;
2147 bool HasDestPointerAuth;
2148
2149 /*implicit*/ ConstantLValue(llvm::Constant *value,
2150 bool hasOffsetApplied = false,
2151 bool hasDestPointerAuth = false)
2152 : Value(value), HasOffsetApplied(hasOffsetApplied),
2153 HasDestPointerAuth(hasDestPointerAuth) {}
2154
2155 /*implicit*/ ConstantLValue(ConstantAddress address)
2156 : ConstantLValue(address.getPointer()) {}
2157};
2158
2159/// A helper class for emitting constant l-values.
2160class ConstantLValueEmitter : public ConstStmtVisitor<ConstantLValueEmitter,
2161 ConstantLValue> {
2162 CodeGenModule &CGM;
2163 ConstantEmitter &Emitter;
2164 const APValue &Value;
2165 QualType DestType;
2166 bool EnablePtrAuthFunctionTypeDiscrimination;
2167
2168 // Befriend StmtVisitorBase so that we don't have to expose Visit*.
2169 friend StmtVisitorBase;
2170
2171public:
2172 ConstantLValueEmitter(ConstantEmitter &emitter, const APValue &value,
2173 QualType destType,
2174 bool EnablePtrAuthFunctionTypeDiscrimination = true)
2175 : CGM(emitter.CGM), Emitter(emitter), Value(value), DestType(destType),
2176 EnablePtrAuthFunctionTypeDiscrimination(
2177 EnablePtrAuthFunctionTypeDiscrimination) {}
2178
2179 llvm::Constant *tryEmit();
2180
2181private:
2182 llvm::Constant *tryEmitAbsolute(llvm::Type *destTy);
2183 ConstantLValue tryEmitBase(const APValue::LValueBase &base);
2184
2185 ConstantLValue VisitStmt(const Stmt *S) { return nullptr; }
2186 ConstantLValue VisitConstantExpr(const ConstantExpr *E);
2187 ConstantLValue VisitCompoundLiteralExpr(const CompoundLiteralExpr *E);
2188 ConstantLValue VisitStringLiteral(const StringLiteral *E);
2189 ConstantLValue VisitObjCBoxedExpr(const ObjCBoxedExpr *E);
2190 ConstantLValue VisitObjCEncodeExpr(const ObjCEncodeExpr *E);
2191 ConstantLValue VisitObjCStringLiteral(const ObjCStringLiteral *E);
2192 llvm::Constant *VisitObjCCollectionElement(const Expr *E);
2193 ConstantLValue VisitObjCArrayLiteral(const ObjCArrayLiteral *E);
2194 ConstantLValue VisitObjCDictionaryLiteral(const ObjCDictionaryLiteral *E);
2195 ConstantLValue VisitPredefinedExpr(const PredefinedExpr *E);
2196 ConstantLValue VisitAddrLabelExpr(const AddrLabelExpr *E);
2197 ConstantLValue VisitCallExpr(const CallExpr *E);
2198 ConstantLValue VisitBlockExpr(const BlockExpr *E);
2199 ConstantLValue VisitCXXTypeidExpr(const CXXTypeidExpr *E);
2200 ConstantLValue VisitMaterializeTemporaryExpr(
2201 const MaterializeTemporaryExpr *E);
2202
2203 ConstantLValue emitPointerAuthSignConstant(const CallExpr *E);
2204 llvm::Constant *emitPointerAuthPointer(const Expr *E);
2205 unsigned emitPointerAuthKey(const Expr *E);
2206 std::pair<llvm::Constant *, llvm::ConstantInt *>
2207 emitPointerAuthDiscriminator(const Expr *E);
2208
2209 bool hasNonZeroOffset() const {
2210 return !Value.getLValueOffset().isZero();
2211 }
2212
2213 /// Return the value offset.
2214 llvm::Constant *getOffset() {
2215 return llvm::ConstantInt::get(Ty: CGM.Int64Ty,
2216 V: Value.getLValueOffset().getQuantity());
2217 }
2218
2219 /// Apply the value offset to the given constant.
2220 llvm::Constant *applyOffset(llvm::Constant *C) {
2221 if (!hasNonZeroOffset())
2222 return C;
2223
2224 return llvm::ConstantExpr::getPtrAdd(Ptr: C, Offset: getOffset());
2225 }
2226};
2227
2228}
2229
2230llvm::Constant *ConstantLValueEmitter::tryEmit() {
2231 const APValue::LValueBase &base = Value.getLValueBase();
2232
2233 // The destination type should be a pointer or reference
2234 // type, but it might also be a cast thereof.
2235 //
2236 // FIXME: the chain of casts required should be reflected in the APValue.
2237 // We need this in order to correctly handle things like a ptrtoint of a
2238 // non-zero null pointer and addrspace casts that aren't trivially
2239 // represented in LLVM IR.
2240 auto destTy = CGM.getTypes().ConvertTypeForMem(T: DestType);
2241 assert(isa<llvm::IntegerType>(destTy) || isa<llvm::PointerType>(destTy));
2242
2243 // If there's no base at all, this is a null or absolute pointer,
2244 // possibly cast back to an integer type.
2245 if (!base) {
2246 return tryEmitAbsolute(destTy);
2247 }
2248
2249 // Otherwise, try to emit the base.
2250 ConstantLValue result = tryEmitBase(base);
2251
2252 // If that failed, we're done.
2253 llvm::Constant *value = result.Value;
2254 if (!value) return nullptr;
2255
2256 // Apply the offset if necessary and not already done.
2257 if (!result.HasOffsetApplied) {
2258 value = applyOffset(C: value);
2259 }
2260
2261 // Apply pointer-auth signing from the destination type.
2262 if (PointerAuthQualifier PointerAuth = DestType.getPointerAuth();
2263 PointerAuth && !result.HasDestPointerAuth) {
2264 value = Emitter.tryEmitConstantSignedPointer(UnsignedPointer: value, Schema: PointerAuth);
2265 if (!value)
2266 return nullptr;
2267 }
2268
2269 // Convert to the appropriate type; this could be an lvalue for
2270 // an integer. FIXME: performAddrSpaceCast
2271 if (isa<llvm::PointerType>(Val: destTy))
2272 return llvm::ConstantExpr::getPointerCast(C: value, Ty: destTy);
2273
2274 return llvm::ConstantExpr::getPtrToInt(C: value, Ty: destTy);
2275}
2276
2277/// Try to emit an absolute l-value, such as a null pointer or an integer
2278/// bitcast to pointer type.
2279llvm::Constant *
2280ConstantLValueEmitter::tryEmitAbsolute(llvm::Type *destTy) {
2281 // If we're producing a pointer, this is easy.
2282 auto destPtrTy = cast<llvm::PointerType>(Val: destTy);
2283 if (Value.isNullPointer()) {
2284 // FIXME: integer offsets from non-zero null pointers.
2285 return CGM.getNullPointer(T: destPtrTy, QT: DestType);
2286 }
2287
2288 // Convert the integer to a pointer-sized integer before converting it
2289 // to a pointer.
2290 // FIXME: signedness depends on the original integer type.
2291 auto intptrTy = CGM.getDataLayout().getIntPtrType(destPtrTy);
2292 llvm::Constant *C;
2293 C = llvm::ConstantFoldIntegerCast(C: getOffset(), DestTy: intptrTy, /*isSigned*/ IsSigned: false,
2294 DL: CGM.getDataLayout());
2295 assert(C && "Must have folded, as Offset is a ConstantInt");
2296 C = llvm::ConstantExpr::getIntToPtr(C, Ty: destPtrTy);
2297 return C;
2298}
2299
2300ConstantLValue
2301ConstantLValueEmitter::tryEmitBase(const APValue::LValueBase &base) {
2302 // Handle values.
2303 if (const ValueDecl *D = base.dyn_cast<const ValueDecl*>()) {
2304 // The constant always points to the canonical declaration. We want to look
2305 // at properties of the most recent declaration at the point of emission.
2306 D = cast<ValueDecl>(Val: D->getMostRecentDecl());
2307
2308 if (D->hasAttr<WeakRefAttr>())
2309 return CGM.GetWeakRefReference(VD: D).getPointer();
2310
2311 auto PtrAuthSign = [&](llvm::Constant *C) {
2312 if (PointerAuthQualifier PointerAuth = DestType.getPointerAuth()) {
2313 C = applyOffset(C);
2314 C = Emitter.tryEmitConstantSignedPointer(UnsignedPointer: C, Schema: PointerAuth);
2315 return ConstantLValue(C, /*applied offset*/ true, /*signed*/ true);
2316 }
2317
2318 CGPointerAuthInfo AuthInfo;
2319
2320 if (EnablePtrAuthFunctionTypeDiscrimination)
2321 AuthInfo = CGM.getFunctionPointerAuthInfo(T: DestType);
2322
2323 if (AuthInfo) {
2324 if (hasNonZeroOffset())
2325 return ConstantLValue(nullptr);
2326
2327 C = applyOffset(C);
2328 C = CGM.getConstantSignedPointer(
2329 Pointer: C, Key: AuthInfo.getKey(), StorageAddress: nullptr,
2330 OtherDiscriminator: cast_or_null<llvm::ConstantInt>(Val: AuthInfo.getDiscriminator()));
2331 return ConstantLValue(C, /*applied offset*/ true, /*signed*/ true);
2332 }
2333
2334 return ConstantLValue(C);
2335 };
2336
2337 if (const auto *FD = dyn_cast<FunctionDecl>(Val: D)) {
2338 llvm::Constant *C = CGM.getRawFunctionPointer(GD: FD);
2339 if (FD->getType()->isCFIUncheckedCalleeFunctionType())
2340 C = llvm::NoCFIValue::get(GV: cast<llvm::GlobalValue>(Val: C));
2341 return PtrAuthSign(C);
2342 }
2343
2344 if (const auto *VD = dyn_cast<VarDecl>(Val: D)) {
2345 // We can never refer to a variable with local storage.
2346 if (!VD->hasLocalStorage()) {
2347 if (VD->isFileVarDecl() || VD->hasExternalStorage())
2348 return CGM.GetAddrOfGlobalVar(D: VD);
2349
2350 if (VD->isLocalVarDecl()) {
2351 return CGM.getOrCreateStaticVarDecl(
2352 D: *VD, Linkage: CGM.getLLVMLinkageVarDefinition(VD));
2353 }
2354 }
2355 }
2356
2357 if (const auto *GD = dyn_cast<MSGuidDecl>(Val: D))
2358 return CGM.GetAddrOfMSGuidDecl(GD);
2359
2360 if (const auto *GCD = dyn_cast<UnnamedGlobalConstantDecl>(Val: D))
2361 return CGM.GetAddrOfUnnamedGlobalConstantDecl(GCD);
2362
2363 if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(Val: D))
2364 return CGM.GetAddrOfTemplateParamObject(TPO);
2365
2366 return nullptr;
2367 }
2368
2369 // Handle typeid(T).
2370 if (TypeInfoLValue TI = base.dyn_cast<TypeInfoLValue>())
2371 return CGM.GetAddrOfRTTIDescriptor(Ty: QualType(TI.getType(), 0));
2372
2373 // Otherwise, it must be an expression.
2374 return Visit(S: base.get<const Expr*>());
2375}
2376
2377ConstantLValue
2378ConstantLValueEmitter::VisitConstantExpr(const ConstantExpr *E) {
2379 if (llvm::Constant *Result = Emitter.tryEmitConstantExpr(CE: E))
2380 return Result;
2381 return Visit(S: E->getSubExpr());
2382}
2383
2384ConstantLValue
2385ConstantLValueEmitter::VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) {
2386 ConstantEmitter CompoundLiteralEmitter(CGM, Emitter.CGF);
2387 CompoundLiteralEmitter.setInConstantContext(Emitter.isInConstantContext());
2388 return tryEmitGlobalCompoundLiteral(emitter&: CompoundLiteralEmitter, E);
2389}
2390
2391ConstantLValue
2392ConstantLValueEmitter::VisitStringLiteral(const StringLiteral *E) {
2393 return CGM.GetAddrOfConstantStringFromLiteral(S: E);
2394}
2395
2396ConstantLValue
2397ConstantLValueEmitter::VisitObjCEncodeExpr(const ObjCEncodeExpr *E) {
2398 return CGM.GetAddrOfConstantStringFromObjCEncode(E);
2399}
2400
2401static ConstantLValue emitConstantObjCStringLiteral(const StringLiteral *S,
2402 QualType T,
2403 CodeGenModule &CGM) {
2404 auto C = CGM.getObjCRuntime().GenerateConstantString(S);
2405 return C.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T));
2406}
2407
2408ConstantLValue
2409ConstantLValueEmitter::VisitObjCStringLiteral(const ObjCStringLiteral *E) {
2410 return emitConstantObjCStringLiteral(S: E->getString(), T: E->getType(), CGM);
2411}
2412
2413ConstantLValue
2414ConstantLValueEmitter::VisitObjCBoxedExpr(const ObjCBoxedExpr *E) {
2415 ASTContext &Context = CGM.getContext();
2416 CGObjCRuntime &Runtime = CGM.getObjCRuntime();
2417 const Expr *SubExpr = E->getSubExpr();
2418 const QualType &Ty = SubExpr->IgnoreParens()->getType();
2419
2420 assert(SubExpr->isEvaluatable(Context) &&
2421 "Non const NSNumber is being emitted as a constant");
2422
2423 if (const auto *SL = dyn_cast<StringLiteral>(Val: SubExpr->IgnoreParenCasts()))
2424 return emitConstantObjCStringLiteral(S: SL, T: E->getType(), CGM);
2425
2426 // Note `@YES` `@NO` need to be handled explicitly
2427 // to meet existing plist encoding / decoding expectations
2428 const bool IsBoolType =
2429 (Ty->isBooleanType() || NSAPI(Context).isObjCBOOLType(T: Ty));
2430 bool BoolValue = false;
2431 if (IsBoolType && SubExpr->EvaluateAsBooleanCondition(Result&: BoolValue, Ctx: Context)) {
2432 ConstantAddress C = Runtime.GenerateConstantNumber(Value: BoolValue, Ty);
2433 return C.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: E->getType()));
2434 }
2435
2436 Expr::EvalResult IntResult{};
2437 if (SubExpr->EvaluateAsInt(Result&: IntResult, Ctx: Context)) {
2438 ConstantAddress C =
2439 Runtime.GenerateConstantNumber(Value: IntResult.Val.getInt(), Ty);
2440 return C.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: E->getType()));
2441 }
2442
2443 llvm::APFloat FloatValue(0.0);
2444 if (SubExpr->EvaluateAsFloat(Result&: FloatValue, Ctx: Context)) {
2445 ConstantAddress C = Runtime.GenerateConstantNumber(Value: FloatValue, Ty);
2446 return C.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: E->getType()));
2447 }
2448
2449 llvm_unreachable("SubExpr is expected to be evaluated as a numeric type");
2450}
2451
2452llvm::Constant *
2453ConstantLValueEmitter::VisitObjCCollectionElement(const Expr *E) {
2454 auto CE = cast<CastExpr>(Val: E);
2455 const Expr *Elm = CE->getSubExpr();
2456 QualType DestTy = CE->getType();
2457
2458 assert(CE->getCastKind() == CK_BitCast &&
2459 "Expected a CK_BitCast type for valid items in constant objc "
2460 "collection literals");
2461
2462 llvm::Type *DstTy = CGM.getTypes().ConvertType(T: DestTy);
2463 ConstantLValue LV = Visit(S: Elm);
2464 llvm::Constant *ConstVal = cast<llvm::Constant>(Val: LV.Value);
2465 llvm::Constant *Val = llvm::ConstantExpr::getBitCast(C: ConstVal, Ty: DstTy);
2466 return Val;
2467}
2468
2469ConstantLValue
2470ConstantLValueEmitter::VisitObjCArrayLiteral(const ObjCArrayLiteral *E) {
2471 SmallVector<llvm::Constant *, 16> ObjectExpressions;
2472 uint64_t NumElements = E->getNumElements();
2473 ObjectExpressions.reserve(N: NumElements);
2474
2475 for (uint64_t i = 0; i < NumElements; i++) {
2476 llvm::Constant *Val = VisitObjCCollectionElement(E: E->getElement(Index: i));
2477 ObjectExpressions.push_back(Elt: Val);
2478 }
2479 ConstantAddress C =
2480 CGM.getObjCRuntime().GenerateConstantArray(Objects: ObjectExpressions);
2481 return C.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: E->getType()));
2482}
2483
2484ConstantLValue ConstantLValueEmitter::VisitObjCDictionaryLiteral(
2485 const ObjCDictionaryLiteral *E) {
2486 SmallVector<std::pair<llvm::Constant *, llvm::Constant *>, 16> KeysAndObjects;
2487 uint64_t NumElements = E->getNumElements();
2488 KeysAndObjects.reserve(N: NumElements);
2489
2490 for (uint64_t i = 0; i < NumElements; i++) {
2491 llvm::Constant *Key =
2492 VisitObjCCollectionElement(E: E->getKeyValueElement(Index: i).Key);
2493 llvm::Constant *Val =
2494 VisitObjCCollectionElement(E: E->getKeyValueElement(Index: i).Value);
2495 KeysAndObjects.push_back(Elt: {Key, Val});
2496 }
2497 ConstantAddress C =
2498 CGM.getObjCRuntime().GenerateConstantDictionary(E, KeysAndObjects);
2499 return C.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: E->getType()));
2500}
2501
2502ConstantLValue
2503ConstantLValueEmitter::VisitPredefinedExpr(const PredefinedExpr *E) {
2504 return CGM.GetAddrOfConstantStringFromLiteral(S: E->getFunctionName());
2505}
2506
2507ConstantLValue
2508ConstantLValueEmitter::VisitAddrLabelExpr(const AddrLabelExpr *E) {
2509 assert(Emitter.CGF && "Invalid address of label expression outside function");
2510 llvm::Constant *Ptr = Emitter.CGF->GetAddrOfLabel(L: E->getLabel());
2511 return Ptr;
2512}
2513
2514ConstantLValue
2515ConstantLValueEmitter::VisitCallExpr(const CallExpr *E) {
2516 unsigned builtin = E->getBuiltinCallee();
2517 if (builtin == Builtin::BI__builtin_function_start)
2518 return CGM.GetFunctionStart(
2519 Decl: E->getArg(Arg: 0)->getAsBuiltinConstantDeclRef(Context: CGM.getContext()));
2520
2521 if (builtin == Builtin::BI__builtin_ptrauth_sign_constant)
2522 return emitPointerAuthSignConstant(E);
2523
2524 if (builtin != Builtin::BI__builtin___CFStringMakeConstantString &&
2525 builtin != Builtin::BI__builtin___NSStringMakeConstantString)
2526 return nullptr;
2527
2528 const auto *Literal = cast<StringLiteral>(Val: E->getArg(Arg: 0)->IgnoreParenCasts());
2529 if (builtin == Builtin::BI__builtin___NSStringMakeConstantString) {
2530 return CGM.getObjCRuntime().GenerateConstantString(Literal);
2531 } else {
2532 // FIXME: need to deal with UCN conversion issues.
2533 return CGM.GetAddrOfConstantCFString(Literal);
2534 }
2535}
2536
2537ConstantLValue
2538ConstantLValueEmitter::emitPointerAuthSignConstant(const CallExpr *E) {
2539 llvm::Constant *UnsignedPointer = emitPointerAuthPointer(E: E->getArg(Arg: 0));
2540 unsigned Key = emitPointerAuthKey(E: E->getArg(Arg: 1));
2541 auto [StorageAddress, OtherDiscriminator] =
2542 emitPointerAuthDiscriminator(E: E->getArg(Arg: 2));
2543
2544 llvm::Constant *SignedPointer = CGM.getConstantSignedPointer(
2545 Pointer: UnsignedPointer, Key, StorageAddress, OtherDiscriminator);
2546 return SignedPointer;
2547}
2548
2549llvm::Constant *ConstantLValueEmitter::emitPointerAuthPointer(const Expr *E) {
2550 Expr::EvalResult Result;
2551 bool Succeeded = E->EvaluateAsRValue(Result, Ctx: CGM.getContext());
2552 assert(Succeeded);
2553 (void)Succeeded;
2554
2555 // The assertions here are all checked by Sema.
2556 assert(Result.Val.isLValue());
2557 if (isa<FunctionDecl>(Val: Result.Val.getLValueBase().get<const ValueDecl *>()))
2558 assert(Result.Val.getLValueOffset().isZero());
2559 return ConstantEmitter(CGM, Emitter.CGF)
2560 .emitAbstract(loc: E->getExprLoc(), value: Result.Val, destType: E->getType(), EnablePtrAuthFunctionTypeDiscrimination: false);
2561}
2562
2563unsigned ConstantLValueEmitter::emitPointerAuthKey(const Expr *E) {
2564 return E->EvaluateKnownConstInt(Ctx: CGM.getContext()).getZExtValue();
2565}
2566
2567std::pair<llvm::Constant *, llvm::ConstantInt *>
2568ConstantLValueEmitter::emitPointerAuthDiscriminator(const Expr *E) {
2569 E = E->IgnoreParens();
2570
2571 if (const auto *Call = dyn_cast<CallExpr>(Val: E)) {
2572 if (Call->getBuiltinCallee() ==
2573 Builtin::BI__builtin_ptrauth_blend_discriminator) {
2574 llvm::Constant *Pointer = ConstantEmitter(CGM).emitAbstract(
2575 E: Call->getArg(Arg: 0), destType: Call->getArg(Arg: 0)->getType());
2576 auto *Extra = cast<llvm::ConstantInt>(Val: ConstantEmitter(CGM).emitAbstract(
2577 E: Call->getArg(Arg: 1), destType: Call->getArg(Arg: 1)->getType()));
2578 return {Pointer, Extra};
2579 }
2580 }
2581
2582 llvm::Constant *Result = ConstantEmitter(CGM).emitAbstract(E, destType: E->getType());
2583 if (Result->getType()->isPointerTy())
2584 return {Result, nullptr};
2585 return {nullptr, cast<llvm::ConstantInt>(Val: Result)};
2586}
2587
2588ConstantLValue
2589ConstantLValueEmitter::VisitBlockExpr(const BlockExpr *E) {
2590 StringRef functionName;
2591 if (auto CGF = Emitter.CGF)
2592 functionName = CGF->CurFn->getName();
2593 else
2594 functionName = "global";
2595
2596 return CGM.GetAddrOfGlobalBlock(BE: E, Name: functionName);
2597}
2598
2599ConstantLValue
2600ConstantLValueEmitter::VisitCXXTypeidExpr(const CXXTypeidExpr *E) {
2601 QualType T;
2602 if (E->isTypeOperand())
2603 T = E->getTypeOperand(Context: CGM.getContext());
2604 else
2605 T = E->getExprOperand()->getType();
2606 return CGM.GetAddrOfRTTIDescriptor(Ty: T);
2607}
2608
2609ConstantLValue
2610ConstantLValueEmitter::VisitMaterializeTemporaryExpr(
2611 const MaterializeTemporaryExpr *E) {
2612 assert(E->getStorageDuration() == SD_Static);
2613 const Expr *Inner = E->getSubExpr()->skipRValueSubobjectAdjustments();
2614 return CGM.GetAddrOfGlobalTemporary(E, Inner);
2615}
2616
2617llvm::Constant *
2618ConstantEmitter::tryEmitPrivate(const APValue &Value, QualType DestType,
2619 bool EnablePtrAuthFunctionTypeDiscrimination) {
2620 switch (Value.getKind()) {
2621 case APValue::None:
2622 case APValue::Indeterminate:
2623 // Out-of-lifetime and indeterminate values can be modeled as 'undef'.
2624 return llvm::UndefValue::get(T: CGM.getTypes().ConvertType(T: DestType));
2625 case APValue::LValue:
2626 return ConstantLValueEmitter(*this, Value, DestType,
2627 EnablePtrAuthFunctionTypeDiscrimination)
2628 .tryEmit();
2629 case APValue::Int:
2630 if (PointerAuthQualifier PointerAuth = DestType.getPointerAuth();
2631 PointerAuth &&
2632 (PointerAuth.authenticatesNullValues() || Value.getInt() != 0))
2633 return nullptr;
2634 return llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: Value.getInt());
2635 case APValue::FixedPoint:
2636 return llvm::ConstantInt::get(Context&: CGM.getLLVMContext(),
2637 V: Value.getFixedPoint().getValue());
2638 case APValue::ComplexInt: {
2639 llvm::Constant *Complex[2];
2640
2641 Complex[0] = llvm::ConstantInt::get(Context&: CGM.getLLVMContext(),
2642 V: Value.getComplexIntReal());
2643 Complex[1] = llvm::ConstantInt::get(Context&: CGM.getLLVMContext(),
2644 V: Value.getComplexIntImag());
2645
2646 // FIXME: the target may want to specify that this is packed.
2647 llvm::StructType *STy =
2648 llvm::StructType::get(elt1: Complex[0]->getType(), elts: Complex[1]->getType());
2649 return llvm::ConstantStruct::get(T: STy, V: Complex);
2650 }
2651 case APValue::Float:
2652 return llvm::ConstantFP::get(Context&: CGM.getLLVMContext(), V: Value.getFloat());
2653 case APValue::ComplexFloat: {
2654 llvm::Constant *Complex[2];
2655
2656 Complex[0] = llvm::ConstantFP::get(Context&: CGM.getLLVMContext(),
2657 V: Value.getComplexFloatReal());
2658 Complex[1] = llvm::ConstantFP::get(Context&: CGM.getLLVMContext(),
2659 V: Value.getComplexFloatImag());
2660
2661 // FIXME: the target may want to specify that this is packed.
2662 llvm::StructType *STy =
2663 llvm::StructType::get(elt1: Complex[0]->getType(), elts: Complex[1]->getType());
2664 return llvm::ConstantStruct::get(T: STy, V: Complex);
2665 }
2666 case APValue::Vector: {
2667 unsigned NumElts = Value.getVectorLength();
2668 SmallVector<llvm::Constant *, 4> Inits(NumElts);
2669
2670 for (unsigned I = 0; I != NumElts; ++I) {
2671 const APValue &Elt = Value.getVectorElt(I);
2672 if (Elt.isInt())
2673 Inits[I] = llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: Elt.getInt());
2674 else if (Elt.isFloat())
2675 Inits[I] = llvm::ConstantFP::get(Context&: CGM.getLLVMContext(), V: Elt.getFloat());
2676 else if (Elt.isIndeterminate())
2677 Inits[I] = llvm::UndefValue::get(T: CGM.getTypes().ConvertType(
2678 T: DestType->castAs<VectorType>()->getElementType()));
2679 else
2680 llvm_unreachable("unsupported vector element type");
2681 }
2682 return llvm::ConstantVector::get(V: Inits);
2683 }
2684 case APValue::Matrix: {
2685 const auto *MT = DestType->castAs<ConstantMatrixType>();
2686 unsigned NumRows = Value.getMatrixNumRows();
2687 unsigned NumCols = Value.getMatrixNumColumns();
2688 unsigned NumElts = NumRows * NumCols;
2689 SmallVector<llvm::Constant *, 16> Inits(NumElts);
2690
2691 for (unsigned Row = 0; Row != NumRows; ++Row) {
2692 for (unsigned Col = 0; Col != NumCols; ++Col) {
2693 const APValue &Elt = Value.getMatrixElt(Row, Col);
2694 unsigned Idx = MT->getColumnMajorFlattenedIndex(Row, Column: Col);
2695 if (Elt.isInt())
2696 Inits[Idx] =
2697 llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: Elt.getInt());
2698 else if (Elt.isFloat())
2699 Inits[Idx] =
2700 llvm::ConstantFP::get(Context&: CGM.getLLVMContext(), V: Elt.getFloat());
2701 else if (Elt.isIndeterminate())
2702 Inits[Idx] = llvm::PoisonValue::get(
2703 T: CGM.getTypes().ConvertType(T: MT->getElementType()));
2704 else
2705 llvm_unreachable("unsupported matrix element type");
2706 }
2707 }
2708 return llvm::ConstantVector::get(V: Inits);
2709 }
2710 case APValue::AddrLabelDiff: {
2711 const AddrLabelExpr *LHSExpr = Value.getAddrLabelDiffLHS();
2712 const AddrLabelExpr *RHSExpr = Value.getAddrLabelDiffRHS();
2713 llvm::Constant *LHS = tryEmitPrivate(E: LHSExpr, destType: LHSExpr->getType());
2714 llvm::Constant *RHS = tryEmitPrivate(E: RHSExpr, destType: RHSExpr->getType());
2715 if (!LHS || !RHS) return nullptr;
2716
2717 // Compute difference
2718 llvm::Type *ResultType = CGM.getTypes().ConvertType(T: DestType);
2719 LHS = llvm::ConstantExpr::getPtrToInt(C: LHS, Ty: CGM.IntPtrTy);
2720 RHS = llvm::ConstantExpr::getPtrToInt(C: RHS, Ty: CGM.IntPtrTy);
2721 llvm::Constant *AddrLabelDiff = llvm::ConstantExpr::getSub(C1: LHS, C2: RHS);
2722
2723 // LLVM is a bit sensitive about the exact format of the
2724 // address-of-label difference; make sure to truncate after
2725 // the subtraction.
2726 return llvm::ConstantExpr::getTruncOrBitCast(C: AddrLabelDiff, Ty: ResultType);
2727 }
2728 case APValue::Struct:
2729 case APValue::Union:
2730 return ConstStructBuilder::BuildStruct(Emitter&: *this, Val: Value, ValTy: DestType);
2731 case APValue::Array: {
2732 const ArrayType *ArrayTy = CGM.getContext().getAsArrayType(T: DestType);
2733 unsigned NumElements = Value.getArraySize();
2734 unsigned NumInitElts = Value.getArrayInitializedElts();
2735
2736 // Emit array filler, if there is one.
2737 llvm::Constant *Filler = nullptr;
2738 if (Value.hasArrayFiller()) {
2739 Filler = tryEmitAbstractForMemory(value: Value.getArrayFiller(),
2740 destType: ArrayTy->getElementType());
2741 if (!Filler)
2742 return nullptr;
2743 }
2744
2745 // Emit initializer elements.
2746 SmallVector<llvm::Constant*, 16> Elts;
2747 if (Filler && Filler->isNullValue())
2748 Elts.reserve(N: NumInitElts + 1);
2749 else
2750 Elts.reserve(N: NumElements);
2751
2752 llvm::Type *CommonElementType = nullptr;
2753 for (unsigned I = 0; I < NumInitElts; ++I) {
2754 llvm::Constant *C = tryEmitPrivateForMemory(
2755 value: Value.getArrayInitializedElt(I), destType: ArrayTy->getElementType());
2756 if (!C) return nullptr;
2757
2758 if (I == 0)
2759 CommonElementType = C->getType();
2760 else if (C->getType() != CommonElementType)
2761 CommonElementType = nullptr;
2762 Elts.push_back(Elt: C);
2763 }
2764
2765 llvm::ArrayType *Desired =
2766 cast<llvm::ArrayType>(Val: CGM.getTypes().ConvertType(T: DestType));
2767
2768 // Fix the type of incomplete arrays if the initializer isn't empty.
2769 if (DestType->isIncompleteArrayType() && !Elts.empty())
2770 Desired = llvm::ArrayType::get(ElementType: Desired->getElementType(), NumElements: Elts.size());
2771
2772 return EmitArrayConstant(CGM, DesiredType: Desired, CommonElementType, ArrayBound: NumElements, Elements&: Elts,
2773 Filler);
2774 }
2775 case APValue::MemberPointer:
2776 return CGM.getCXXABI().EmitMemberPointer(MP: Value, MPT: DestType);
2777 case APValue::Reflection:
2778 llvm_unreachable("std::meta::info is consteval-only type");
2779 }
2780 llvm_unreachable("Unknown APValue kind");
2781}
2782
2783llvm::GlobalVariable *CodeGenModule::getAddrOfConstantCompoundLiteralIfEmitted(
2784 const CompoundLiteralExpr *E) {
2785 return EmittedCompoundLiterals.lookup(Val: E);
2786}
2787
2788void CodeGenModule::setAddrOfConstantCompoundLiteral(
2789 const CompoundLiteralExpr *CLE, llvm::GlobalVariable *GV) {
2790 bool Ok = EmittedCompoundLiterals.insert(KV: std::make_pair(x&: CLE, y&: GV)).second;
2791 (void)Ok;
2792 assert(Ok && "CLE has already been emitted!");
2793}
2794
2795ConstantAddress
2796CodeGenModule::GetAddrOfConstantCompoundLiteral(const CompoundLiteralExpr *E) {
2797 assert(E->isFileScope() && "not a file-scope compound literal expr");
2798 ConstantEmitter emitter(*this);
2799 return tryEmitGlobalCompoundLiteral(emitter, E);
2800}
2801
2802llvm::Constant *
2803CodeGenModule::getMemberPointerConstant(const UnaryOperator *uo) {
2804 // Member pointer constants always have a very particular form.
2805 const MemberPointerType *type = cast<MemberPointerType>(Val: uo->getType());
2806 const ValueDecl *decl = cast<DeclRefExpr>(Val: uo->getSubExpr())->getDecl();
2807
2808 // A member function pointer.
2809 if (const CXXMethodDecl *method = dyn_cast<CXXMethodDecl>(Val: decl))
2810 return getCXXABI().EmitMemberFunctionPointer(MD: method);
2811
2812 // Otherwise, a member data pointer.
2813 getContext().recordMemberDataPointerEvaluation(VD: decl);
2814 uint64_t fieldOffset = getContext().getFieldOffset(FD: decl);
2815 CharUnits chars = getContext().toCharUnitsFromBits(BitSize: (int64_t) fieldOffset);
2816 return getCXXABI().EmitMemberDataPointer(MPT: type, offset: chars);
2817}
2818
2819static llvm::Constant *EmitNullConstantForBase(CodeGenModule &CGM,
2820 llvm::Type *baseType,
2821 const CXXRecordDecl *base);
2822
2823static llvm::Constant *EmitNullConstant(CodeGenModule &CGM,
2824 const RecordDecl *record,
2825 bool asCompleteObject) {
2826 const CGRecordLayout &layout = CGM.getTypes().getCGRecordLayout(record);
2827 llvm::StructType *structure =
2828 (asCompleteObject ? layout.getLLVMType()
2829 : layout.getBaseSubobjectLLVMType());
2830
2831 unsigned numElements = structure->getNumElements();
2832 std::vector<llvm::Constant *> elements(numElements);
2833
2834 auto CXXR = dyn_cast<CXXRecordDecl>(Val: record);
2835 // Fill in all the bases.
2836 if (CXXR) {
2837 for (const auto &I : CXXR->bases()) {
2838 if (I.isVirtual()) {
2839 // Ignore virtual bases; if we're laying out for a complete
2840 // object, we'll lay these out later.
2841 continue;
2842 }
2843
2844 const auto *base = I.getType()->castAsCXXRecordDecl();
2845 // Ignore empty bases.
2846 if (CodeGenUtils::isEmptyRecordForLayout(Ctx: CGM.getContext(), T: I.getType()) ||
2847 CGM.getContext()
2848 .getASTRecordLayout(D: base)
2849 .getNonVirtualSize()
2850 .isZero())
2851 continue;
2852
2853 unsigned fieldIndex = layout.getNonVirtualBaseLLVMFieldNo(RD: base);
2854 llvm::Type *baseType = structure->getElementType(N: fieldIndex);
2855 elements[fieldIndex] = EmitNullConstantForBase(CGM, baseType, base);
2856 }
2857 }
2858
2859 // Fill in all the fields.
2860 for (const auto *Field : record->fields()) {
2861 // Fill in non-bitfields. (Bitfields always use a zero pattern, which we
2862 // will fill in later.)
2863 if (!Field->isBitField() &&
2864 !CodeGenUtils::isEmptyFieldForLayout(Ctx: CGM.getContext(), FD: Field)) {
2865 unsigned fieldIndex = layout.getLLVMFieldNo(FD: Field);
2866 elements[fieldIndex] = CGM.EmitNullConstant(T: Field->getType());
2867 }
2868
2869 // For unions, stop after the first named field.
2870 if (record->isUnion()) {
2871 if (Field->getIdentifier())
2872 break;
2873 if (const auto *FieldRD = Field->getType()->getAsRecordDecl())
2874 if (FieldRD->findFirstNamedDataMember())
2875 break;
2876 }
2877 }
2878
2879 // Fill in the virtual bases, if we're working with the complete object.
2880 if (CXXR && asCompleteObject) {
2881 for (const auto &I : CXXR->vbases()) {
2882 const auto *base = I.getType()->castAsCXXRecordDecl();
2883 // Ignore empty bases.
2884 if (CodeGenUtils::isEmptyRecordForLayout(Ctx: CGM.getContext(), T: I.getType()))
2885 continue;
2886
2887 unsigned fieldIndex = layout.getVirtualBaseIndex(base);
2888
2889 // We might have already laid this field out.
2890 if (elements[fieldIndex]) continue;
2891
2892 llvm::Type *baseType = structure->getElementType(N: fieldIndex);
2893 elements[fieldIndex] = EmitNullConstantForBase(CGM, baseType, base);
2894 }
2895 }
2896
2897 // Now go through all other fields and zero them out.
2898 for (unsigned i = 0; i != numElements; ++i) {
2899 if (!elements[i])
2900 elements[i] = llvm::Constant::getNullValue(Ty: structure->getElementType(N: i));
2901 }
2902
2903 return llvm::ConstantStruct::get(T: structure, V: elements);
2904}
2905
2906/// Emit the null constant for a base subobject.
2907static llvm::Constant *EmitNullConstantForBase(CodeGenModule &CGM,
2908 llvm::Type *baseType,
2909 const CXXRecordDecl *base) {
2910 const CGRecordLayout &baseLayout = CGM.getTypes().getCGRecordLayout(base);
2911
2912 // Just zero out bases that don't have any pointer to data members.
2913 if (baseLayout.isZeroInitializableAsBase())
2914 return llvm::Constant::getNullValue(Ty: baseType);
2915
2916 // Otherwise, we can just use its null constant.
2917 return EmitNullConstant(CGM, record: base, /*asCompleteObject=*/false);
2918}
2919
2920llvm::Constant *ConstantEmitter::emitNullForMemory(CodeGenModule &CGM,
2921 QualType T) {
2922 return emitForMemory(CGM, C: CGM.EmitNullConstant(T), destType: T);
2923}
2924
2925llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) {
2926 if (T->getAs<PointerType>()) {
2927 llvm::Type *LT = getTypes().ConvertTypeForMem(T);
2928 if (auto *PT = dyn_cast<llvm::PointerType>(Val: LT))
2929 return getNullPointer(T: PT, QT: T);
2930 // Some pointer types do not lower to an LLVM pointer (e.g. a WebAssembly
2931 // funcref, which is an opaque reference type). Use the type's zero value.
2932 return llvm::Constant::getNullValue(Ty: LT);
2933 }
2934
2935 if (getTypes().isZeroInitializable(T))
2936 return llvm::Constant::getNullValue(Ty: getTypes().ConvertTypeForMem(T));
2937
2938 if (const ConstantArrayType *CAT = Context.getAsConstantArrayType(T)) {
2939 llvm::ArrayType *ATy =
2940 cast<llvm::ArrayType>(Val: getTypes().ConvertTypeForMem(T));
2941
2942 QualType ElementTy = CAT->getElementType();
2943
2944 llvm::Constant *Element =
2945 ConstantEmitter::emitNullForMemory(CGM&: *this, T: ElementTy);
2946 unsigned NumElements = CAT->getZExtSize();
2947 SmallVector<llvm::Constant *, 8> Array(NumElements, Element);
2948 return llvm::ConstantArray::get(T: ATy, V: Array);
2949 }
2950
2951 if (const auto *RD = T->getAsRecordDecl())
2952 return ::EmitNullConstant(CGM&: *this, record: RD,
2953 /*asCompleteObject=*/true);
2954
2955 assert(T->isMemberDataPointerType() &&
2956 "Should only see pointers to data members here!");
2957
2958 return getCXXABI().EmitNullMemberPointer(MPT: T->castAs<MemberPointerType>());
2959}
2960
2961llvm::Constant *
2962CodeGenModule::EmitNullConstantForBase(const CXXRecordDecl *Record) {
2963 return ::EmitNullConstant(CGM&: *this, record: Record, asCompleteObject: false);
2964}
2965