1//===--- CGExprConstant.cpp - Emit LLVM Code from Constant Expressions ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Constant Expr nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ABIInfoImpl.h"
14#include "CGCXXABI.h"
15#include "CGObjCRuntime.h"
16#include "CGRecordLayout.h"
17#include "CodeGenFunction.h"
18#include "CodeGenModule.h"
19#include "ConstantEmitter.h"
20#include "TargetInfo.h"
21#include "clang/AST/APValue.h"
22#include "clang/AST/ASTContext.h"
23#include "clang/AST/Attr.h"
24#include "clang/AST/MatrixUtils.h"
25#include "clang/AST/NSAPI.h"
26#include "clang/AST/RecordLayout.h"
27#include "clang/AST/StmtVisitor.h"
28#include "clang/Basic/Builtins.h"
29#include "llvm/ADT/STLExtras.h"
30#include "llvm/ADT/Sequence.h"
31#include "llvm/Analysis/ConstantFolding.h"
32#include "llvm/IR/Constants.h"
33#include "llvm/IR/DataLayout.h"
34#include "llvm/IR/Function.h"
35#include "llvm/IR/GlobalVariable.h"
36#include "llvm/Support/SipHash.h"
37#include <optional>
38using namespace clang;
39using namespace CodeGen;
40
41//===----------------------------------------------------------------------===//
42// ConstantAggregateBuilder
43//===----------------------------------------------------------------------===//
44
45namespace {
46class ConstExprEmitter;
47
48llvm::Constant *getPadding(const CodeGenModule &CGM, CharUnits PadSize) {
49 llvm::Type *Ty = CGM.CharTy;
50 if (PadSize > CharUnits::One())
51 Ty = llvm::ArrayType::get(ElementType: Ty, NumElements: PadSize.getQuantity());
52 if (CGM.shouldZeroInitPadding()) {
53 return llvm::Constant::getNullValue(Ty);
54 }
55 return llvm::UndefValue::get(T: Ty);
56}
57
58struct ConstantAggregateBuilderUtils {
59 CodeGenModule &CGM;
60
61 ConstantAggregateBuilderUtils(CodeGenModule &CGM) : CGM(CGM) {}
62
63 CharUnits getAlignment(const llvm::Constant *C) const {
64 return CharUnits::fromQuantity(
65 Quantity: CGM.getDataLayout().getABITypeAlign(Ty: C->getType()));
66 }
67
68 CharUnits getSize(llvm::Type *Ty) const {
69 return CharUnits::fromQuantity(Quantity: CGM.getDataLayout().getTypeAllocSize(Ty));
70 }
71
72 CharUnits getSize(const llvm::Constant *C) const {
73 return getSize(Ty: C->getType());
74 }
75
76 llvm::Constant *getPadding(CharUnits PadSize) const {
77 return ::getPadding(CGM, PadSize);
78 }
79
80 llvm::Constant *getZeroes(CharUnits ZeroSize) const {
81 llvm::Type *Ty = llvm::ArrayType::get(ElementType: CGM.CharTy, NumElements: ZeroSize.getQuantity());
82 return llvm::ConstantAggregateZero::get(Ty);
83 }
84};
85
86/// Incremental builder for an llvm::Constant* holding a struct or array
87/// constant.
88class ConstantAggregateBuilder : private ConstantAggregateBuilderUtils {
89 /// The elements of the constant. These two arrays must have the same size;
90 /// Offsets[i] describes the offset of Elems[i] within the constant. The
91 /// elements are kept in increasing offset order, and we ensure that there
92 /// is no overlap: Offsets[i+1] >= Offsets[i] + getSize(Elemes[i]).
93 ///
94 /// This may contain explicit padding elements (in order to create a
95 /// natural layout), but need not. Gaps between elements are implicitly
96 /// considered to be filled with undef.
97 llvm::SmallVector<llvm::Constant*, 32> Elems;
98 llvm::SmallVector<CharUnits, 32> Offsets;
99
100 /// The size of the constant (the maximum end offset of any added element).
101 /// May be larger than the end of Elems.back() if we split the last element
102 /// and removed some trailing undefs.
103 CharUnits Size = CharUnits::Zero();
104
105 /// This is true only if laying out Elems in order as the elements of a
106 /// non-packed LLVM struct will give the correct layout.
107 bool NaturalLayout = true;
108
109 bool split(size_t Index, CharUnits Hint);
110 std::optional<size_t> splitAt(CharUnits Pos);
111
112 static llvm::Constant *buildFrom(CodeGenModule &CGM,
113 ArrayRef<llvm::Constant *> Elems,
114 ArrayRef<CharUnits> Offsets,
115 CharUnits StartOffset, CharUnits Size,
116 bool NaturalLayout, llvm::Type *DesiredTy,
117 bool AllowOversized);
118
119public:
120 ConstantAggregateBuilder(CodeGenModule &CGM)
121 : ConstantAggregateBuilderUtils(CGM) {}
122
123 /// Update or overwrite the value starting at \p Offset with \c C.
124 ///
125 /// \param AllowOverwrite If \c true, this constant might overwrite (part of)
126 /// a constant that has already been added. This flag is only used to
127 /// detect bugs.
128 bool add(llvm::Constant *C, CharUnits Offset, bool AllowOverwrite);
129
130 /// Update or overwrite the bits starting at \p OffsetInBits with \p Bits.
131 bool addBits(llvm::APInt Bits, uint64_t OffsetInBits, bool AllowOverwrite);
132
133 /// Attempt to condense the value starting at \p Offset to a constant of type
134 /// \p DesiredTy.
135 void condense(CharUnits Offset, llvm::Type *DesiredTy);
136
137 /// Produce a constant representing the entire accumulated value, ideally of
138 /// the specified type. If \p AllowOversized, the constant might be larger
139 /// than implied by \p DesiredTy (eg, if there is a flexible array member).
140 /// Otherwise, the constant will be of exactly the same size as \p DesiredTy
141 /// even if we can't represent it as that type.
142 llvm::Constant *build(llvm::Type *DesiredTy, bool AllowOversized) const {
143 return buildFrom(CGM, Elems, Offsets, StartOffset: CharUnits::Zero(), Size,
144 NaturalLayout, DesiredTy, AllowOversized);
145 }
146};
147
148template<typename Container, typename Range = std::initializer_list<
149 typename Container::value_type>>
150static void replace(Container &C, size_t BeginOff, size_t EndOff, Range Vals) {
151 assert(BeginOff <= EndOff && "invalid replacement range");
152 llvm::replace(C, C.begin() + BeginOff, C.begin() + EndOff, Vals);
153}
154
155bool ConstantAggregateBuilder::add(llvm::Constant *C, CharUnits Offset,
156 bool AllowOverwrite) {
157 // Common case: appending to a layout.
158 if (Offset >= Size) {
159 CharUnits Align = getAlignment(C);
160 CharUnits AlignedSize = Size.alignTo(Align);
161 if (AlignedSize > Offset || Offset.alignTo(Align) != Offset)
162 NaturalLayout = false;
163 else if (AlignedSize < Offset) {
164 Elems.push_back(Elt: getPadding(PadSize: Offset - Size));
165 Offsets.push_back(Elt: Size);
166 }
167 Elems.push_back(Elt: C);
168 Offsets.push_back(Elt: Offset);
169 Size = Offset + getSize(C);
170 return true;
171 }
172
173 // Uncommon case: constant overlaps what we've already created.
174 std::optional<size_t> FirstElemToReplace = splitAt(Pos: Offset);
175 if (!FirstElemToReplace)
176 return false;
177
178 CharUnits CSize = getSize(C);
179 std::optional<size_t> LastElemToReplace = splitAt(Pos: Offset + CSize);
180 if (!LastElemToReplace)
181 return false;
182
183 assert((FirstElemToReplace == LastElemToReplace || AllowOverwrite) &&
184 "unexpectedly overwriting field");
185
186 replace(C&: Elems, BeginOff: *FirstElemToReplace, EndOff: *LastElemToReplace, Vals: {C});
187 replace(C&: Offsets, BeginOff: *FirstElemToReplace, EndOff: *LastElemToReplace, Vals: {Offset});
188 Size = std::max(a: Size, b: Offset + CSize);
189 NaturalLayout = false;
190 return true;
191}
192
193bool ConstantAggregateBuilder::addBits(llvm::APInt Bits, uint64_t OffsetInBits,
194 bool AllowOverwrite) {
195 const ASTContext &Context = CGM.getContext();
196 const uint64_t CharWidth = CGM.getContext().getCharWidth();
197
198 // Offset of where we want the first bit to go within the bits of the
199 // current char.
200 unsigned OffsetWithinChar = OffsetInBits % CharWidth;
201
202 // We split bit-fields up into individual bytes. Walk over the bytes and
203 // update them.
204 for (CharUnits OffsetInChars =
205 Context.toCharUnitsFromBits(BitSize: OffsetInBits - OffsetWithinChar);
206 /**/; ++OffsetInChars) {
207 // Number of bits we want to fill in this char.
208 unsigned WantedBits =
209 std::min(a: (uint64_t)Bits.getBitWidth(), b: CharWidth - OffsetWithinChar);
210
211 // Get a char containing the bits we want in the right places. The other
212 // bits have unspecified values.
213 llvm::APInt BitsThisChar = Bits;
214 if (BitsThisChar.getBitWidth() < CharWidth)
215 BitsThisChar = BitsThisChar.zext(width: CharWidth);
216 if (CGM.getDataLayout().isBigEndian()) {
217 // Figure out how much to shift by. We may need to left-shift if we have
218 // less than one byte of Bits left.
219 int Shift = Bits.getBitWidth() - CharWidth + OffsetWithinChar;
220 if (Shift > 0)
221 BitsThisChar.lshrInPlace(ShiftAmt: Shift);
222 else if (Shift < 0)
223 BitsThisChar = BitsThisChar.shl(shiftAmt: -Shift);
224 } else {
225 BitsThisChar = BitsThisChar.shl(shiftAmt: OffsetWithinChar);
226 }
227 if (BitsThisChar.getBitWidth() > CharWidth)
228 BitsThisChar = BitsThisChar.trunc(width: CharWidth);
229
230 if (WantedBits == CharWidth) {
231 // Got a full byte: just add it directly.
232 add(C: llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: BitsThisChar),
233 Offset: OffsetInChars, AllowOverwrite);
234 } else {
235 // Partial byte: update the existing integer if there is one. If we
236 // can't split out a 1-CharUnit range to update, then we can't add
237 // these bits and fail the entire constant emission.
238 std::optional<size_t> FirstElemToUpdate = splitAt(Pos: OffsetInChars);
239 if (!FirstElemToUpdate)
240 return false;
241 std::optional<size_t> LastElemToUpdate =
242 splitAt(Pos: OffsetInChars + CharUnits::One());
243 if (!LastElemToUpdate)
244 return false;
245 assert(*LastElemToUpdate - *FirstElemToUpdate < 2 &&
246 "should have at most one element covering one byte");
247
248 // Figure out which bits we want and discard the rest.
249 llvm::APInt UpdateMask(CharWidth, 0);
250 if (CGM.getDataLayout().isBigEndian())
251 UpdateMask.setBits(loBit: CharWidth - OffsetWithinChar - WantedBits,
252 hiBit: CharWidth - OffsetWithinChar);
253 else
254 UpdateMask.setBits(loBit: OffsetWithinChar, hiBit: OffsetWithinChar + WantedBits);
255 BitsThisChar &= UpdateMask;
256
257 if (*FirstElemToUpdate == *LastElemToUpdate ||
258 Elems[*FirstElemToUpdate]->isNullValue() ||
259 isa<llvm::UndefValue>(Val: Elems[*FirstElemToUpdate])) {
260 // All existing bits are either zero or undef.
261 add(C: llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: BitsThisChar),
262 Offset: OffsetInChars, /*AllowOverwrite*/ true);
263 } else {
264 llvm::Constant *&ToUpdate = Elems[*FirstElemToUpdate];
265 // In order to perform a partial update, we need the existing bitwise
266 // value, which we can only extract for a constant int.
267 auto *CI = dyn_cast<llvm::ConstantInt>(Val: ToUpdate);
268 if (!CI)
269 return false;
270 // Because this is a 1-CharUnit range, the constant occupying it must
271 // be exactly one CharUnit wide.
272 assert(CI->getBitWidth() == CharWidth && "splitAt failed");
273 assert((!(CI->getValue() & UpdateMask) || AllowOverwrite) &&
274 "unexpectedly overwriting bitfield");
275 BitsThisChar |= (CI->getValue() & ~UpdateMask);
276 ToUpdate = llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: BitsThisChar);
277 }
278 }
279
280 // Stop if we've added all the bits.
281 if (WantedBits == Bits.getBitWidth())
282 break;
283
284 // Remove the consumed bits from Bits.
285 if (!CGM.getDataLayout().isBigEndian())
286 Bits.lshrInPlace(ShiftAmt: WantedBits);
287 Bits = Bits.trunc(width: Bits.getBitWidth() - WantedBits);
288
289 // The remanining bits go at the start of the following bytes.
290 OffsetWithinChar = 0;
291 }
292
293 return true;
294}
295
296/// Returns a position within Elems and Offsets such that all elements
297/// before the returned index end before Pos and all elements at or after
298/// the returned index begin at or after Pos. Splits elements as necessary
299/// to ensure this. Returns std::nullopt if we find something we can't split.
300std::optional<size_t> ConstantAggregateBuilder::splitAt(CharUnits Pos) {
301 if (Pos >= Size)
302 return Offsets.size();
303
304 while (true) {
305 auto FirstAfterPos = llvm::upper_bound(Range&: Offsets, Value&: Pos);
306 if (FirstAfterPos == Offsets.begin())
307 return 0;
308
309 // If we already have an element starting at Pos, we're done.
310 size_t LastAtOrBeforePosIndex = FirstAfterPos - Offsets.begin() - 1;
311 if (Offsets[LastAtOrBeforePosIndex] == Pos)
312 return LastAtOrBeforePosIndex;
313
314 // We found an element starting before Pos. Check for overlap.
315 if (Offsets[LastAtOrBeforePosIndex] +
316 getSize(C: Elems[LastAtOrBeforePosIndex]) <= Pos)
317 return LastAtOrBeforePosIndex + 1;
318
319 // Try to decompose it into smaller constants.
320 if (!split(Index: LastAtOrBeforePosIndex, Hint: Pos))
321 return std::nullopt;
322 }
323}
324
325/// Split the constant at index Index, if possible. Return true if we did.
326/// Hint indicates the location at which we'd like to split, but may be
327/// ignored.
328bool ConstantAggregateBuilder::split(size_t Index, CharUnits Hint) {
329 NaturalLayout = false;
330 llvm::Constant *C = Elems[Index];
331 CharUnits Offset = Offsets[Index];
332
333 if (auto *CA = dyn_cast<llvm::ConstantAggregate>(Val: C)) {
334 // Expand the sequence into its contained elements.
335 // FIXME: This assumes vector elements are byte-sized.
336 replace(C&: Elems, BeginOff: Index, EndOff: Index + 1,
337 Vals: llvm::map_range(C: llvm::seq(Begin: 0u, End: CA->getNumOperands()),
338 F: [&](unsigned Op) { return CA->getOperand(i_nocapture: Op); }));
339 if (isa<llvm::ArrayType>(Val: CA->getType()) ||
340 isa<llvm::VectorType>(Val: CA->getType())) {
341 // Array or vector.
342 llvm::Type *ElemTy =
343 llvm::GetElementPtrInst::getTypeAtIndex(Ty: CA->getType(), Idx: (uint64_t)0);
344 CharUnits ElemSize = getSize(Ty: ElemTy);
345 replace(
346 C&: Offsets, BeginOff: Index, EndOff: Index + 1,
347 Vals: llvm::map_range(C: llvm::seq(Begin: 0u, End: CA->getNumOperands()),
348 F: [&](unsigned Op) { return Offset + Op * ElemSize; }));
349 } else {
350 // Must be a struct.
351 auto *ST = cast<llvm::StructType>(Val: CA->getType());
352 const llvm::StructLayout *Layout =
353 CGM.getDataLayout().getStructLayout(Ty: ST);
354 replace(C&: Offsets, BeginOff: Index, EndOff: Index + 1,
355 Vals: llvm::map_range(
356 C: llvm::seq(Begin: 0u, End: CA->getNumOperands()), F: [&](unsigned Op) {
357 return Offset + CharUnits::fromQuantity(
358 Quantity: Layout->getElementOffset(Idx: Op));
359 }));
360 }
361 return true;
362 }
363
364 if (auto *CDS = dyn_cast<llvm::ConstantDataSequential>(Val: C)) {
365 // Expand the sequence into its contained elements.
366 // FIXME: This assumes vector elements are byte-sized.
367 // FIXME: If possible, split into two ConstantDataSequentials at Hint.
368 CharUnits ElemSize = getSize(Ty: CDS->getElementType());
369 replace(C&: Elems, BeginOff: Index, EndOff: Index + 1,
370 Vals: llvm::map_range(C: llvm::seq(Begin: uint64_t(0u), End: CDS->getNumElements()),
371 F: [&](uint64_t Elem) {
372 return CDS->getElementAsConstant(i: Elem);
373 }));
374 replace(C&: Offsets, BeginOff: Index, EndOff: Index + 1,
375 Vals: llvm::map_range(
376 C: llvm::seq(Begin: uint64_t(0u), End: CDS->getNumElements()),
377 F: [&](uint64_t Elem) { return Offset + Elem * ElemSize; }));
378 return true;
379 }
380
381 if (isa<llvm::ConstantAggregateZero>(Val: C)) {
382 // Split into two zeros at the hinted offset.
383 CharUnits ElemSize = getSize(C);
384 assert(Hint > Offset && Hint < Offset + ElemSize && "nothing to split");
385 replace(C&: Elems, BeginOff: Index, EndOff: Index + 1,
386 Vals: {getZeroes(ZeroSize: Hint - Offset), getZeroes(ZeroSize: Offset + ElemSize - Hint)});
387 replace(C&: Offsets, BeginOff: Index, EndOff: Index + 1, Vals: {Offset, Hint});
388 return true;
389 }
390
391 if (isa<llvm::UndefValue>(Val: C)) {
392 // Drop undef; it doesn't contribute to the final layout.
393 replace(C&: Elems, BeginOff: Index, EndOff: Index + 1, Vals: {});
394 replace(C&: Offsets, BeginOff: Index, EndOff: Index + 1, Vals: {});
395 return true;
396 }
397
398 // FIXME: We could split a ConstantInt if the need ever arose.
399 // We don't need to do this to handle bit-fields because we always eagerly
400 // split them into 1-byte chunks.
401
402 return false;
403}
404
405static llvm::Constant *
406EmitArrayConstant(CodeGenModule &CGM, llvm::ArrayType *DesiredType,
407 llvm::Type *CommonElementType, uint64_t ArrayBound,
408 SmallVectorImpl<llvm::Constant *> &Elements,
409 llvm::Constant *Filler);
410
411llvm::Constant *ConstantAggregateBuilder::buildFrom(
412 CodeGenModule &CGM, ArrayRef<llvm::Constant *> Elems,
413 ArrayRef<CharUnits> Offsets, CharUnits StartOffset, CharUnits Size,
414 bool NaturalLayout, llvm::Type *DesiredTy, bool AllowOversized) {
415 ConstantAggregateBuilderUtils Utils(CGM);
416
417 if (Elems.empty())
418 return llvm::UndefValue::get(T: DesiredTy);
419
420 auto Offset = [&](size_t I) { return Offsets[I] - StartOffset; };
421
422 // If we want an array type, see if all the elements are the same type and
423 // appropriately spaced.
424 if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(Val: DesiredTy)) {
425 assert(!AllowOversized && "oversized array emission not supported");
426
427 bool CanEmitArray = true;
428 llvm::Type *CommonType = Elems[0]->getType();
429 llvm::Constant *Filler = llvm::Constant::getNullValue(Ty: CommonType);
430 CharUnits ElemSize = Utils.getSize(Ty: ATy->getElementType());
431 SmallVector<llvm::Constant*, 32> ArrayElements;
432 for (size_t I = 0; I != Elems.size(); ++I) {
433 // Skip zeroes; we'll use a zero value as our array filler.
434 if (Elems[I]->isNullValue())
435 continue;
436
437 // All remaining elements must be the same type.
438 if (Elems[I]->getType() != CommonType ||
439 !Offset(I).isMultipleOf(N: ElemSize)) {
440 CanEmitArray = false;
441 break;
442 }
443 ArrayElements.resize(N: Offset(I) / ElemSize + 1, NV: Filler);
444 ArrayElements.back() = Elems[I];
445 }
446
447 if (CanEmitArray) {
448 return EmitArrayConstant(CGM, DesiredType: ATy, CommonElementType: CommonType, ArrayBound: ATy->getNumElements(),
449 Elements&: ArrayElements, Filler);
450 }
451
452 // Can't emit as an array, carry on to emit as a struct.
453 }
454
455 // The size of the constant we plan to generate. This is usually just
456 // the size of the initialized type, but in AllowOversized mode (i.e.
457 // flexible array init), it can be larger.
458 CharUnits DesiredSize = Utils.getSize(Ty: DesiredTy);
459 if (Size > DesiredSize) {
460 assert(AllowOversized && "Elems are oversized");
461 DesiredSize = Size;
462 }
463
464 // The natural alignment of an unpacked LLVM struct with the given elements.
465 CharUnits Align = CharUnits::One();
466 for (llvm::Constant *C : Elems)
467 Align = std::max(a: Align, b: Utils.getAlignment(C));
468
469 // The natural size of an unpacked LLVM struct with the given elements.
470 CharUnits AlignedSize = Size.alignTo(Align);
471
472 bool Packed = false;
473 ArrayRef<llvm::Constant*> UnpackedElems = Elems;
474 llvm::SmallVector<llvm::Constant*, 32> UnpackedElemStorage;
475 if (DesiredSize < AlignedSize || DesiredSize.alignTo(Align) != DesiredSize) {
476 // The natural layout would be too big; force use of a packed layout.
477 NaturalLayout = false;
478 Packed = true;
479 } else if (DesiredSize > AlignedSize) {
480 // The natural layout would be too small. Add padding to fix it. (This
481 // is ignored if we choose a packed layout.)
482 UnpackedElemStorage.assign(in_start: Elems.begin(), in_end: Elems.end());
483 UnpackedElemStorage.push_back(Elt: Utils.getPadding(PadSize: DesiredSize - Size));
484 UnpackedElems = UnpackedElemStorage;
485 }
486
487 // If we don't have a natural layout, insert padding as necessary.
488 // As we go, double-check to see if we can actually just emit Elems
489 // as a non-packed struct and do so opportunistically if possible.
490 llvm::SmallVector<llvm::Constant*, 32> PackedElems;
491 if (!NaturalLayout) {
492 CharUnits SizeSoFar = CharUnits::Zero();
493 for (size_t I = 0; I != Elems.size(); ++I) {
494 CharUnits Align = Utils.getAlignment(C: Elems[I]);
495 CharUnits NaturalOffset = SizeSoFar.alignTo(Align);
496 CharUnits DesiredOffset = Offset(I);
497 assert(DesiredOffset >= SizeSoFar && "elements out of order");
498
499 if (DesiredOffset != NaturalOffset)
500 Packed = true;
501 if (DesiredOffset != SizeSoFar)
502 PackedElems.push_back(Elt: Utils.getPadding(PadSize: DesiredOffset - SizeSoFar));
503 PackedElems.push_back(Elt: Elems[I]);
504 SizeSoFar = DesiredOffset + Utils.getSize(C: Elems[I]);
505 }
506 // If we're using the packed layout, pad it out to the desired size if
507 // necessary.
508 if (Packed) {
509 assert(SizeSoFar <= DesiredSize &&
510 "requested size is too small for contents");
511 if (SizeSoFar < DesiredSize)
512 PackedElems.push_back(Elt: Utils.getPadding(PadSize: DesiredSize - SizeSoFar));
513 }
514 }
515
516 llvm::StructType *STy = llvm::ConstantStruct::getTypeForElements(
517 Ctx&: CGM.getLLVMContext(), V: Packed ? PackedElems : UnpackedElems, Packed);
518
519 // Pick the type to use. If the type is layout identical to the desired
520 // type then use it, otherwise use whatever the builder produced for us.
521 if (llvm::StructType *DesiredSTy = dyn_cast<llvm::StructType>(Val: DesiredTy)) {
522 if (DesiredSTy->isLayoutIdentical(Other: STy))
523 STy = DesiredSTy;
524 }
525
526 return llvm::ConstantStruct::get(T: STy, V: Packed ? PackedElems : UnpackedElems);
527}
528
529void ConstantAggregateBuilder::condense(CharUnits Offset,
530 llvm::Type *DesiredTy) {
531 CharUnits Size = getSize(Ty: DesiredTy);
532
533 std::optional<size_t> FirstElemToReplace = splitAt(Pos: Offset);
534 if (!FirstElemToReplace)
535 return;
536 size_t First = *FirstElemToReplace;
537
538 std::optional<size_t> LastElemToReplace = splitAt(Pos: Offset + Size);
539 if (!LastElemToReplace)
540 return;
541 size_t Last = *LastElemToReplace;
542
543 size_t Length = Last - First;
544 if (Length == 0)
545 return;
546
547 if (Length == 1 && Offsets[First] == Offset &&
548 getSize(C: Elems[First]) == Size) {
549 // Re-wrap single element structs if necessary. Otherwise, leave any single
550 // element constant of the right size alone even if it has the wrong type.
551 auto *STy = dyn_cast<llvm::StructType>(Val: DesiredTy);
552 if (STy && STy->getNumElements() == 1 &&
553 STy->getElementType(N: 0) == Elems[First]->getType())
554 Elems[First] = llvm::ConstantStruct::get(T: STy, Vs: Elems[First]);
555 return;
556 }
557
558 llvm::Constant *Replacement = buildFrom(
559 CGM, Elems: ArrayRef(Elems).slice(N: First, M: Length),
560 Offsets: ArrayRef(Offsets).slice(N: First, M: Length), StartOffset: Offset, Size: getSize(Ty: DesiredTy),
561 /*known to have natural layout=*/NaturalLayout: false, DesiredTy, AllowOversized: false);
562 replace(C&: Elems, BeginOff: First, EndOff: Last, Vals: {Replacement});
563 replace(C&: Offsets, BeginOff: First, EndOff: Last, Vals: {Offset});
564}
565
566//===----------------------------------------------------------------------===//
567// ConstStructBuilder
568//===----------------------------------------------------------------------===//
569
570class ConstStructBuilder {
571 CodeGenModule &CGM;
572 ConstantEmitter &Emitter;
573 ConstantAggregateBuilder &Builder;
574 CharUnits StartOffset;
575
576public:
577 static llvm::Constant *BuildStruct(ConstantEmitter &Emitter,
578 const InitListExpr *ILE,
579 QualType StructTy);
580 static llvm::Constant *BuildStruct(ConstantEmitter &Emitter,
581 const APValue &Value, QualType ValTy);
582 static bool UpdateStruct(ConstantEmitter &Emitter,
583 ConstantAggregateBuilder &Const, CharUnits Offset,
584 const InitListExpr *Updater);
585
586private:
587 ConstStructBuilder(ConstantEmitter &Emitter,
588 ConstantAggregateBuilder &Builder, CharUnits StartOffset)
589 : CGM(Emitter.CGM), Emitter(Emitter), Builder(Builder),
590 StartOffset(StartOffset) {}
591
592 bool AppendField(const FieldDecl *Field, uint64_t FieldOffset,
593 llvm::Constant *InitExpr, bool AllowOverwrite = false);
594
595 bool AppendBytes(CharUnits FieldOffsetInChars, llvm::Constant *InitCst,
596 bool AllowOverwrite = false);
597
598 bool AppendBitField(const FieldDecl *Field, uint64_t FieldOffset,
599 llvm::Constant *InitExpr, bool AllowOverwrite = false);
600
601 bool Build(const InitListExpr *ILE, bool AllowOverwrite);
602 bool Build(const APValue &Val, const RecordDecl *RD, bool IsPrimaryBase,
603 const CXXRecordDecl *VTableClass, CharUnits BaseOffset,
604 bool IsCompleteClass = true);
605 bool DoZeroInitPadding(const ASTRecordLayout &Layout, unsigned FieldNo,
606 const FieldDecl &Field, bool AllowOverwrite,
607 CharUnits &SizeSoFar, bool &ZeroFieldSize);
608 bool DoZeroInitPadding(const ASTRecordLayout &Layout, bool AllowOverwrite,
609 CharUnits SizeSoFar);
610 llvm::Constant *Finalize(QualType Ty);
611};
612
613bool ConstStructBuilder::AppendField(
614 const FieldDecl *Field, uint64_t FieldOffset, llvm::Constant *InitCst,
615 bool AllowOverwrite) {
616 const ASTContext &Context = CGM.getContext();
617
618 CharUnits FieldOffsetInChars = Context.toCharUnitsFromBits(BitSize: FieldOffset);
619
620 return AppendBytes(FieldOffsetInChars, InitCst, AllowOverwrite);
621}
622
623bool ConstStructBuilder::AppendBytes(CharUnits FieldOffsetInChars,
624 llvm::Constant *InitCst,
625 bool AllowOverwrite) {
626 return Builder.add(C: InitCst, Offset: StartOffset + FieldOffsetInChars, AllowOverwrite);
627}
628
629bool ConstStructBuilder::AppendBitField(const FieldDecl *Field,
630 uint64_t FieldOffset, llvm::Constant *C,
631 bool AllowOverwrite) {
632
633 llvm::ConstantInt *CI = dyn_cast<llvm::ConstantInt>(Val: C);
634 if (!CI) {
635 // Constants for long _BitInt types are sometimes split into individual
636 // bytes. Try to fold these back into an integer constant. If that doesn't
637 // work out, then we are trying to initialize a bitfield with a non-trivial
638 // constant, this must require run-time code.
639 llvm::Type *LoadType =
640 CGM.getTypes().convertTypeForLoadStore(T: Field->getType(), LLVMTy: C->getType());
641 llvm::Constant *FoldedConstant = llvm::ConstantFoldLoadFromConst(
642 C, Ty: LoadType, Offset: llvm::APInt::getZero(numBits: 32), DL: CGM.getDataLayout());
643 CI = dyn_cast_if_present<llvm::ConstantInt>(Val: FoldedConstant);
644 if (!CI)
645 return false;
646 }
647
648 const CGRecordLayout &RL =
649 CGM.getTypes().getCGRecordLayout(Field->getParent());
650 const CGBitFieldInfo &Info = RL.getBitFieldInfo(FD: Field);
651 llvm::APInt FieldValue = CI->getValue();
652
653 // Promote the size of FieldValue if necessary
654 // FIXME: This should never occur, but currently it can because initializer
655 // constants are cast to bool, and because clang is not enforcing bitfield
656 // width limits.
657 if (Info.Size > FieldValue.getBitWidth())
658 FieldValue = FieldValue.zext(width: Info.Size);
659
660 // Truncate the size of FieldValue to the bit field size.
661 if (Info.Size < FieldValue.getBitWidth())
662 FieldValue = FieldValue.trunc(width: Info.Size);
663
664 return Builder.addBits(Bits: FieldValue,
665 OffsetInBits: CGM.getContext().toBits(CharSize: StartOffset) + FieldOffset,
666 AllowOverwrite);
667}
668
669static bool EmitDesignatedInitUpdater(ConstantEmitter &Emitter,
670 ConstantAggregateBuilder &Const,
671 CharUnits Offset, QualType Type,
672 const InitListExpr *Updater) {
673 if (Type->isRecordType())
674 return ConstStructBuilder::UpdateStruct(Emitter, Const, Offset, Updater);
675
676 auto CAT = Emitter.CGM.getContext().getAsConstantArrayType(T: Type);
677 if (!CAT)
678 return false;
679 QualType ElemType = CAT->getElementType();
680 CharUnits ElemSize = Emitter.CGM.getContext().getTypeSizeInChars(T: ElemType);
681 llvm::Type *ElemTy = Emitter.CGM.getTypes().ConvertTypeForMem(T: ElemType);
682
683 llvm::Constant *FillC = nullptr;
684 if (const Expr *Filler = Updater->getArrayFiller()) {
685 if (!isa<NoInitExpr>(Val: Filler)) {
686 FillC = Emitter.tryEmitAbstractForMemory(E: Filler, T: ElemType);
687 if (!FillC)
688 return false;
689 }
690 }
691
692 unsigned NumElementsToUpdate =
693 FillC ? CAT->getZExtSize() : Updater->getNumInits();
694 for (unsigned I = 0; I != NumElementsToUpdate; ++I, Offset += ElemSize) {
695 const Expr *Init = nullptr;
696 if (I < Updater->getNumInits())
697 Init = Updater->getInit(Init: I);
698
699 if (!Init && FillC) {
700 if (!Const.add(C: FillC, Offset, AllowOverwrite: true))
701 return false;
702 } else if (!Init || isa<NoInitExpr>(Val: Init)) {
703 continue;
704 } else if (const auto *ChildILE = dyn_cast<InitListExpr>(Val: Init)) {
705 if (!EmitDesignatedInitUpdater(Emitter, Const, Offset, Type: ElemType,
706 Updater: ChildILE))
707 return false;
708 // Attempt to reduce the array element to a single constant if necessary.
709 Const.condense(Offset, DesiredTy: ElemTy);
710 } else {
711 llvm::Constant *Val = Emitter.tryEmitPrivateForMemory(E: Init, T: ElemType);
712 if (!Const.add(C: Val, Offset, AllowOverwrite: true))
713 return false;
714 }
715 }
716
717 return true;
718}
719
720bool ConstStructBuilder::Build(const InitListExpr *ILE, bool AllowOverwrite) {
721 auto *RD = ILE->getType()->castAsRecordDecl();
722 const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(D: RD);
723
724 unsigned FieldNo = -1;
725 unsigned ElementNo = 0;
726
727 // Bail out if we have base classes. We could support these, but they only
728 // arise in C++1z where we will have already constant folded most interesting
729 // cases. FIXME: There are still a few more cases we can handle this way.
730 if (auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD))
731 if (CXXRD->getNumBases())
732 return false;
733
734 const bool ZeroInitPadding = CGM.shouldZeroInitPadding();
735 bool ZeroFieldSize = false;
736 CharUnits SizeSoFar = CharUnits::Zero();
737
738 for (FieldDecl *Field : RD->fields()) {
739 ++FieldNo;
740
741 // If this is a union, skip all the fields that aren't being initialized.
742 if (RD->isUnion() &&
743 !declaresSameEntity(D1: ILE->getInitializedFieldInUnion(), D2: Field))
744 continue;
745
746 // Don't emit anonymous bitfields.
747 if (Field->isUnnamedBitField())
748 continue;
749
750 // Get the initializer. A struct can include fields without initializers,
751 // we just use explicit null values for them.
752 const Expr *Init = nullptr;
753 if (ElementNo < ILE->getNumInits())
754 Init = ILE->getInit(Init: ElementNo++);
755 if (isa_and_nonnull<NoInitExpr>(Val: Init)) {
756 if (ZeroInitPadding &&
757 !DoZeroInitPadding(Layout, FieldNo, Field: *Field, AllowOverwrite, SizeSoFar,
758 ZeroFieldSize))
759 return false;
760 continue;
761 }
762
763 // Zero-sized fields are not emitted, but their initializers may still
764 // prevent emission of this struct as a constant.
765 if (isEmptyFieldForLayout(Context: CGM.getContext(), FD: Field)) {
766 if (Init && Init->HasSideEffects(Ctx: CGM.getContext()))
767 return false;
768 continue;
769 }
770
771 if (ZeroInitPadding &&
772 !DoZeroInitPadding(Layout, FieldNo, Field: *Field, AllowOverwrite, SizeSoFar,
773 ZeroFieldSize))
774 return false;
775
776 // When emitting a DesignatedInitUpdateExpr, a nested InitListExpr
777 // represents additional overwriting of our current constant value, and not
778 // a new constant to emit independently.
779 if (AllowOverwrite &&
780 (Field->getType()->isArrayType() || Field->getType()->isRecordType())) {
781 if (auto *SubILE = dyn_cast<InitListExpr>(Val: Init)) {
782 CharUnits Offset = CGM.getContext().toCharUnitsFromBits(
783 BitSize: Layout.getFieldOffset(FieldNo));
784 if (!EmitDesignatedInitUpdater(Emitter, Const&: Builder, Offset: StartOffset + Offset,
785 Type: Field->getType(), Updater: SubILE))
786 return false;
787 // If we split apart the field's value, try to collapse it down to a
788 // single value now.
789 Builder.condense(Offset: StartOffset + Offset,
790 DesiredTy: CGM.getTypes().ConvertTypeForMem(T: Field->getType()));
791 continue;
792 }
793 }
794
795 llvm::Constant *EltInit =
796 Init ? Emitter.tryEmitPrivateForMemory(E: Init, T: Field->getType())
797 : Emitter.emitNullForMemory(T: Field->getType());
798 if (!EltInit)
799 return false;
800
801 if (ZeroInitPadding && ZeroFieldSize)
802 SizeSoFar += CharUnits::fromQuantity(
803 Quantity: CGM.getDataLayout().getTypeAllocSize(Ty: EltInit->getType()));
804
805 if (!Field->isBitField()) {
806 // Handle non-bitfield members.
807 if (!AppendField(Field, FieldOffset: Layout.getFieldOffset(FieldNo), InitCst: EltInit,
808 AllowOverwrite))
809 return false;
810 // After emitting a non-empty field with [[no_unique_address]], we may
811 // need to overwrite its tail padding.
812 if (Field->hasAttr<NoUniqueAddressAttr>())
813 AllowOverwrite = true;
814 } else {
815 // Otherwise we have a bitfield.
816 if (!AppendBitField(Field, FieldOffset: Layout.getFieldOffset(FieldNo), C: EltInit,
817 AllowOverwrite))
818 return false;
819 }
820 }
821
822 if (ZeroInitPadding && !DoZeroInitPadding(Layout, AllowOverwrite, SizeSoFar))
823 return false;
824
825 return true;
826}
827
828namespace {
829struct BaseInfo {
830 BaseInfo(const CXXRecordDecl *Decl, CharUnits Offset, unsigned Index)
831 : Decl(Decl), Offset(Offset), Index(Index) {
832 }
833
834 const CXXRecordDecl *Decl;
835 CharUnits Offset;
836 unsigned Index;
837
838 bool operator<(const BaseInfo &O) const { return Offset < O.Offset; }
839};
840}
841
842bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD,
843 bool IsPrimaryBase,
844 const CXXRecordDecl *VTableClass,
845 CharUnits Offset, bool IsCompleteClass) {
846 assert(Val.isStruct() || Val.isUnion());
847
848 const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(D: RD);
849
850 if (Val.isStruct()) {
851 if (const CXXRecordDecl *CD = dyn_cast<CXXRecordDecl>(Val: RD)) {
852 // Add a vtable pointer, if we need one and it hasn't already been added.
853 if (Layout.hasOwnVFPtr()) {
854 llvm::Constant *VTableAddressPoint =
855 CGM.getCXXABI().getVTableAddressPoint(Base: BaseSubobject(CD, Offset),
856 VTableClass);
857 if (auto Authentication =
858 CGM.getVTablePointerAuthentication(thisClass: CD,
859 /*IsVTTEntry=*/false)) {
860 VTableAddressPoint = Emitter.tryEmitConstantSignedPointer(
861 Ptr: VTableAddressPoint, Auth: *Authentication);
862 if (!VTableAddressPoint)
863 return false;
864 }
865 if (!AppendBytes(FieldOffsetInChars: Offset, InitCst: VTableAddressPoint))
866 return false;
867 }
868
869 // Accumulate and sort bases, in order to visit them in address order,
870 // which may not be the same as declaration order.
871 SmallVector<BaseInfo, 8> Bases;
872 Bases.reserve(N: Val.getStructNumBases());
873 unsigned BaseNo = 0;
874 for (const CXXBaseSpecifier &Base : CD->bases()) {
875 if (Base.isVirtual())
876 continue;
877 const CXXRecordDecl *BD = Base.getType()->getAsCXXRecordDecl();
878 CharUnits BaseOffset = Layout.getBaseClassOffset(Base: BD);
879 Bases.push_back(Elt: BaseInfo(BD, BaseOffset, BaseNo));
880 ++BaseNo;
881 }
882 llvm::stable_sort(Range&: Bases);
883
884 for (const BaseInfo &Base : Bases) {
885 bool IsPrimaryBase = Layout.getPrimaryBase() == Base.Decl;
886 if (!Build(Val: Val.getStructBase(i: Base.Index), RD: Base.Decl, IsPrimaryBase,
887 VTableClass, Offset: Offset + Base.Offset, IsCompleteClass: false))
888 return false;
889 }
890
891 if (IsCompleteClass) {
892 Bases.clear();
893 BaseNo = 0;
894 Bases.reserve(N: Val.getStructNumVirtualBases());
895 for (const CXXBaseSpecifier &Base : CD->vbases()) {
896 const CXXRecordDecl *BD = Base.getType()->getAsCXXRecordDecl();
897 CharUnits BaseOffset = Layout.getVBaseClassOffset(VBase: BD);
898 Bases.push_back(Elt: BaseInfo(BD, BaseOffset, BaseNo));
899 ++BaseNo;
900 }
901 llvm::stable_sort(Range&: Bases);
902
903 for (const BaseInfo &Base : Bases) {
904 bool IsPrimaryBase = Layout.getPrimaryBase() == Base.Decl;
905 if (!Build(Val: Val.getStructVirtualBase(i: Base.Index), RD: Base.Decl,
906 IsPrimaryBase, VTableClass, Offset: Offset + Base.Offset, IsCompleteClass: false))
907 return false;
908 }
909 }
910 }
911 }
912
913 unsigned FieldNo = 0;
914 uint64_t OffsetBits = CGM.getContext().toBits(CharSize: Offset);
915 const bool ZeroInitPadding = CGM.shouldZeroInitPadding();
916 bool ZeroFieldSize = false;
917 CharUnits SizeSoFar = CharUnits::Zero();
918
919 bool AllowOverwrite = false;
920 for (RecordDecl::field_iterator Field = RD->field_begin(),
921 FieldEnd = RD->field_end(); Field != FieldEnd; ++Field, ++FieldNo) {
922 // If this is a union, skip all the fields that aren't being initialized.
923 if (RD->isUnion() && !declaresSameEntity(D1: Val.getUnionField(), D2: *Field))
924 continue;
925
926 // Don't emit anonymous bitfields or zero-sized fields.
927 if (Field->isUnnamedBitField() ||
928 isEmptyFieldForLayout(Context: CGM.getContext(), FD: *Field))
929 continue;
930
931 // Emit the value of the initializer.
932 const APValue &FieldValue =
933 RD->isUnion() ? Val.getUnionValue() : Val.getStructField(i: FieldNo);
934 llvm::Constant *EltInit =
935 Emitter.tryEmitPrivateForMemory(value: FieldValue, T: Field->getType());
936 if (!EltInit)
937 return false;
938
939 if (CGM.getContext().isPFPField(Field: *Field)) {
940 llvm::ConstantInt *Disc;
941 llvm::Constant *AddrDisc;
942 if (CGM.getContext().arePFPFieldsTriviallyCopyable(RD)) {
943 uint64_t FieldSignature =
944 llvm::getPointerAuthStableSipHash(S: CGM.getPFPFieldName(FD: *Field));
945 Disc = llvm::ConstantInt::get(Ty: CGM.Int64Ty, V: FieldSignature);
946 AddrDisc = llvm::ConstantPointerNull::get(T: CGM.VoidPtrTy);
947 } else if (Emitter.isAbstract()) {
948 // isAbstract means that we don't know the global's address. Since we
949 // can only form a pointer without knowing the address if the fields are
950 // trivially copyable, we need to return false otherwise.
951 return false;
952 } else {
953 Disc = llvm::ConstantInt::get(Ty: CGM.Int64Ty,
954 V: -(Layout.getFieldOffset(FieldNo) / 8));
955 AddrDisc = Emitter.getCurrentAddrPrivate();
956 }
957 EltInit = llvm::ConstantPtrAuth::get(
958 Ptr: EltInit, Key: llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 2), Disc, AddrDisc,
959 DeactivationSymbol: CGM.getPFPDeactivationSymbol(FD: *Field));
960 if (!CGM.getContext().arePFPFieldsTriviallyCopyable(RD))
961 Emitter.registerCurrentAddrPrivate(signal: EltInit,
962 placeholder: cast<llvm::GlobalValue>(Val: AddrDisc));
963 }
964
965 if (ZeroInitPadding) {
966 if (!DoZeroInitPadding(Layout, FieldNo, Field: **Field, AllowOverwrite,
967 SizeSoFar, ZeroFieldSize))
968 return false;
969 if (ZeroFieldSize)
970 SizeSoFar += CharUnits::fromQuantity(
971 Quantity: CGM.getDataLayout().getTypeAllocSize(Ty: EltInit->getType()));
972 }
973
974 if (!Field->isBitField()) {
975 // Handle non-bitfield members.
976 if (!AppendField(Field: *Field, FieldOffset: Layout.getFieldOffset(FieldNo) + OffsetBits,
977 InitCst: EltInit, AllowOverwrite))
978 return false;
979 // After emitting a non-empty field with [[no_unique_address]], we may
980 // need to overwrite its tail padding.
981 if (Field->hasAttr<NoUniqueAddressAttr>())
982 AllowOverwrite = true;
983 } else {
984 // Otherwise we have a bitfield.
985 if (!AppendBitField(Field: *Field, FieldOffset: Layout.getFieldOffset(FieldNo) + OffsetBits,
986 C: EltInit, AllowOverwrite))
987 return false;
988 }
989 }
990 if (ZeroInitPadding && !DoZeroInitPadding(Layout, AllowOverwrite, SizeSoFar))
991 return false;
992
993 return true;
994}
995
996bool ConstStructBuilder::DoZeroInitPadding(
997 const ASTRecordLayout &Layout, unsigned FieldNo, const FieldDecl &Field,
998 bool AllowOverwrite, CharUnits &SizeSoFar, bool &ZeroFieldSize) {
999 uint64_t StartBitOffset = Layout.getFieldOffset(FieldNo);
1000 CharUnits StartOffset = CGM.getContext().toCharUnitsFromBits(BitSize: StartBitOffset);
1001 if (SizeSoFar < StartOffset)
1002 if (!AppendBytes(FieldOffsetInChars: SizeSoFar, InitCst: getPadding(CGM, PadSize: StartOffset - SizeSoFar),
1003 AllowOverwrite))
1004 return false;
1005
1006 if (!Field.isBitField()) {
1007 CharUnits FieldSize = CGM.getContext().getTypeSizeInChars(T: Field.getType());
1008 SizeSoFar = StartOffset + FieldSize;
1009 ZeroFieldSize = FieldSize.isZero();
1010 } else {
1011 const CGRecordLayout &RL =
1012 CGM.getTypes().getCGRecordLayout(Field.getParent());
1013 const CGBitFieldInfo &Info = RL.getBitFieldInfo(FD: &Field);
1014 uint64_t EndBitOffset = StartBitOffset + Info.Size;
1015 SizeSoFar = CGM.getContext().toCharUnitsFromBits(BitSize: EndBitOffset);
1016 if (EndBitOffset % CGM.getContext().getCharWidth() != 0) {
1017 SizeSoFar++;
1018 }
1019 ZeroFieldSize = Info.Size == 0;
1020 }
1021 return true;
1022}
1023
1024bool ConstStructBuilder::DoZeroInitPadding(const ASTRecordLayout &Layout,
1025 bool AllowOverwrite,
1026 CharUnits SizeSoFar) {
1027 CharUnits TotalSize = Layout.getSize();
1028 if (SizeSoFar < TotalSize)
1029 if (!AppendBytes(FieldOffsetInChars: SizeSoFar, InitCst: getPadding(CGM, PadSize: TotalSize - SizeSoFar),
1030 AllowOverwrite))
1031 return false;
1032 SizeSoFar = TotalSize;
1033 return true;
1034}
1035
1036llvm::Constant *ConstStructBuilder::Finalize(QualType Type) {
1037 Type = Type.getNonReferenceType();
1038 auto *RD = Type->castAsRecordDecl();
1039 llvm::Type *ValTy = CGM.getTypes().ConvertType(T: Type);
1040 return Builder.build(DesiredTy: ValTy, AllowOversized: RD->hasFlexibleArrayMember());
1041}
1042
1043llvm::Constant *ConstStructBuilder::BuildStruct(ConstantEmitter &Emitter,
1044 const InitListExpr *ILE,
1045 QualType ValTy) {
1046 ConstantAggregateBuilder Const(Emitter.CGM);
1047 ConstStructBuilder Builder(Emitter, Const, CharUnits::Zero());
1048
1049 if (!Builder.Build(ILE, /*AllowOverwrite*/false))
1050 return nullptr;
1051
1052 return Builder.Finalize(Type: ValTy);
1053}
1054
1055llvm::Constant *ConstStructBuilder::BuildStruct(ConstantEmitter &Emitter,
1056 const APValue &Val,
1057 QualType ValTy) {
1058 ConstantAggregateBuilder Const(Emitter.CGM);
1059 ConstStructBuilder Builder(Emitter, Const, CharUnits::Zero());
1060
1061 const auto *RD = ValTy->castAsRecordDecl();
1062 const CXXRecordDecl *CD = dyn_cast<CXXRecordDecl>(Val: RD);
1063 if (!Builder.Build(Val, RD, IsPrimaryBase: false, VTableClass: CD, Offset: CharUnits::Zero()))
1064 return nullptr;
1065
1066 return Builder.Finalize(Type: ValTy);
1067}
1068
1069bool ConstStructBuilder::UpdateStruct(ConstantEmitter &Emitter,
1070 ConstantAggregateBuilder &Const,
1071 CharUnits Offset,
1072 const InitListExpr *Updater) {
1073 return ConstStructBuilder(Emitter, Const, Offset)
1074 .Build(ILE: Updater, /*AllowOverwrite*/ true);
1075}
1076
1077//===----------------------------------------------------------------------===//
1078// ConstExprEmitter
1079//===----------------------------------------------------------------------===//
1080
1081static ConstantAddress
1082tryEmitGlobalCompoundLiteral(ConstantEmitter &emitter,
1083 const CompoundLiteralExpr *E) {
1084 CodeGenModule &CGM = emitter.CGM;
1085 CharUnits Align = CGM.getContext().getTypeAlignInChars(T: E->getType());
1086 if (llvm::GlobalVariable *Addr =
1087 CGM.getAddrOfConstantCompoundLiteralIfEmitted(E))
1088 return ConstantAddress(Addr, Addr->getValueType(), Align);
1089
1090 LangAS addressSpace = E->getType().getAddressSpace();
1091 llvm::Constant *C = emitter.tryEmitForInitializer(E: E->getInitializer(),
1092 destAddrSpace: addressSpace, destType: E->getType());
1093 if (!C) {
1094 assert(!E->isFileScope() &&
1095 "file-scope compound literal did not have constant initializer!");
1096 return ConstantAddress::invalid();
1097 }
1098
1099 auto GV = new llvm::GlobalVariable(
1100 CGM.getModule(), C->getType(),
1101 E->getType().isConstantStorage(Ctx: CGM.getContext(), ExcludeCtor: true, ExcludeDtor: false),
1102 llvm::GlobalValue::InternalLinkage, C, ".compoundliteral", nullptr,
1103 llvm::GlobalVariable::NotThreadLocal,
1104 CGM.getContext().getTargetAddressSpace(AS: addressSpace));
1105 emitter.finalize(global: GV);
1106 GV->setAlignment(Align.getAsAlign());
1107 CGM.setAddrOfConstantCompoundLiteral(CLE: E, GV);
1108 return ConstantAddress(GV, GV->getValueType(), Align);
1109}
1110
1111static llvm::Constant *
1112EmitArrayConstant(CodeGenModule &CGM, llvm::ArrayType *DesiredType,
1113 llvm::Type *CommonElementType, uint64_t ArrayBound,
1114 SmallVectorImpl<llvm::Constant *> &Elements,
1115 llvm::Constant *Filler) {
1116 // Figure out how long the initial prefix of non-zero elements is.
1117 uint64_t NonzeroLength = ArrayBound;
1118 if (Elements.size() < NonzeroLength && Filler->isNullValue())
1119 NonzeroLength = Elements.size();
1120 if (NonzeroLength == Elements.size()) {
1121 while (NonzeroLength > 0 && Elements[NonzeroLength - 1]->isNullValue())
1122 --NonzeroLength;
1123 }
1124
1125 if (NonzeroLength == 0)
1126 return llvm::ConstantAggregateZero::get(Ty: DesiredType);
1127
1128 // Add a zeroinitializer array filler if we have lots of trailing zeroes.
1129 uint64_t TrailingZeroes = ArrayBound - NonzeroLength;
1130 if (TrailingZeroes >= 8) {
1131 assert(Elements.size() >= NonzeroLength &&
1132 "missing initializer for non-zero element");
1133
1134 // If all the elements had the same type up to the trailing zeroes, emit a
1135 // struct of two arrays (the nonzero data and the zeroinitializer).
1136 if (CommonElementType && NonzeroLength >= 8) {
1137 llvm::Constant *Initial = llvm::ConstantArray::get(
1138 T: llvm::ArrayType::get(ElementType: CommonElementType, NumElements: NonzeroLength),
1139 V: ArrayRef(Elements).take_front(N: NonzeroLength));
1140 Elements.resize(N: 2);
1141 Elements[0] = Initial;
1142 } else {
1143 Elements.resize(N: NonzeroLength + 1);
1144 }
1145
1146 auto *FillerType =
1147 CommonElementType ? CommonElementType : DesiredType->getElementType();
1148 FillerType = llvm::ArrayType::get(ElementType: FillerType, NumElements: TrailingZeroes);
1149 Elements.back() = llvm::ConstantAggregateZero::get(Ty: FillerType);
1150 CommonElementType = nullptr;
1151 } else if (Elements.size() != ArrayBound) {
1152 // Otherwise pad to the right size with the filler if necessary.
1153 Elements.resize(N: ArrayBound, NV: Filler);
1154 if (Filler->getType() != CommonElementType)
1155 CommonElementType = nullptr;
1156 }
1157
1158 // If all elements have the same type, just emit an array constant.
1159 if (CommonElementType)
1160 return llvm::ConstantArray::get(
1161 T: llvm::ArrayType::get(ElementType: CommonElementType, NumElements: ArrayBound), V: Elements);
1162
1163 // We have mixed types. Use a packed struct.
1164 llvm::SmallVector<llvm::Type *, 16> Types;
1165 Types.reserve(N: Elements.size());
1166 for (llvm::Constant *Elt : Elements)
1167 Types.push_back(Elt: Elt->getType());
1168 llvm::StructType *SType =
1169 llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: Types, isPacked: true);
1170 return llvm::ConstantStruct::get(T: SType, V: Elements);
1171}
1172
1173// This class only needs to handle arrays, structs and unions. Outside C++11
1174// mode, we don't currently constant fold those types. All other types are
1175// handled by constant folding.
1176//
1177// Constant folding is currently missing support for a few features supported
1178// here: CK_ReinterpretMemberPointer, and DesignatedInitUpdateExpr.
1179class ConstExprEmitter
1180 : public ConstStmtVisitor<ConstExprEmitter, llvm::Constant *, QualType> {
1181 CodeGenModule &CGM;
1182 ConstantEmitter &Emitter;
1183 llvm::LLVMContext &VMContext;
1184public:
1185 ConstExprEmitter(ConstantEmitter &emitter)
1186 : CGM(emitter.CGM), Emitter(emitter), VMContext(CGM.getLLVMContext()) {
1187 }
1188
1189 //===--------------------------------------------------------------------===//
1190 // Visitor Methods
1191 //===--------------------------------------------------------------------===//
1192
1193 llvm::Constant *VisitStmt(const Stmt *S, QualType T) { return nullptr; }
1194
1195 llvm::Constant *VisitConstantExpr(const ConstantExpr *CE, QualType T) {
1196 if (llvm::Constant *Result = Emitter.tryEmitConstantExpr(CE))
1197 return Result;
1198 return Visit(S: CE->getSubExpr(), P: T);
1199 }
1200
1201 llvm::Constant *VisitParenExpr(const ParenExpr *PE, QualType T) {
1202 return Visit(S: PE->getSubExpr(), P: T);
1203 }
1204
1205 llvm::Constant *
1206 VisitSubstNonTypeTemplateParmExpr(const SubstNonTypeTemplateParmExpr *PE,
1207 QualType T) {
1208 return Visit(S: PE->getReplacement(), P: T);
1209 }
1210
1211 llvm::Constant *VisitGenericSelectionExpr(const GenericSelectionExpr *GE,
1212 QualType T) {
1213 return Visit(S: GE->getResultExpr(), P: T);
1214 }
1215
1216 llvm::Constant *VisitChooseExpr(const ChooseExpr *CE, QualType T) {
1217 return Visit(S: CE->getChosenSubExpr(), P: T);
1218 }
1219
1220 llvm::Constant *VisitCompoundLiteralExpr(const CompoundLiteralExpr *E,
1221 QualType T) {
1222 return Visit(S: E->getInitializer(), P: T);
1223 }
1224
1225 llvm::Constant *ProduceIntToIntCast(const Expr *E, QualType DestType) {
1226 QualType FromType = E->getType();
1227 // See also HandleIntToIntCast in ExprConstant.cpp
1228 if (FromType->isIntegerType())
1229 if (llvm::Constant *C = Visit(S: E, P: FromType))
1230 if (auto *CI = dyn_cast<llvm::ConstantInt>(Val: C)) {
1231 unsigned SrcWidth = CGM.getContext().getIntWidth(T: FromType);
1232 unsigned DstWidth = CGM.getContext().getIntWidth(T: DestType);
1233 if (DstWidth == SrcWidth)
1234 return CI;
1235 llvm::APInt A = FromType->isSignedIntegerType()
1236 ? CI->getValue().sextOrTrunc(width: DstWidth)
1237 : CI->getValue().zextOrTrunc(width: DstWidth);
1238 return llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: A);
1239 }
1240 return nullptr;
1241 }
1242
1243 llvm::Constant *VisitCastExpr(const CastExpr *E, QualType destType) {
1244 if (const auto *ECE = dyn_cast<ExplicitCastExpr>(Val: E))
1245 CGM.EmitExplicitCastExprType(E: ECE, CGF: Emitter.CGF);
1246 const Expr *subExpr = E->getSubExpr();
1247
1248 switch (E->getCastKind()) {
1249 case CK_ToUnion: {
1250 // GCC cast to union extension
1251 assert(E->getType()->isUnionType() &&
1252 "Destination type is not union type!");
1253
1254 auto field = E->getTargetUnionField();
1255
1256 auto C = Emitter.tryEmitPrivateForMemory(E: subExpr, T: field->getType());
1257 if (!C) return nullptr;
1258
1259 auto destTy = ConvertType(T: destType);
1260 if (C->getType() == destTy) return C;
1261
1262 // Build a struct with the union sub-element as the first member,
1263 // and padded to the appropriate size.
1264 SmallVector<llvm::Constant*, 2> Elts;
1265 SmallVector<llvm::Type*, 2> Types;
1266 Elts.push_back(Elt: C);
1267 Types.push_back(Elt: C->getType());
1268 unsigned CurSize = CGM.getDataLayout().getTypeAllocSize(Ty: C->getType());
1269 unsigned TotalSize = CGM.getDataLayout().getTypeAllocSize(Ty: destTy);
1270
1271 assert(CurSize <= TotalSize && "Union size mismatch!");
1272 if (unsigned NumPadBytes = TotalSize - CurSize) {
1273 llvm::Constant *Padding =
1274 getPadding(CGM, PadSize: CharUnits::fromQuantity(Quantity: NumPadBytes));
1275 Elts.push_back(Elt: Padding);
1276 Types.push_back(Elt: Padding->getType());
1277 }
1278
1279 llvm::StructType *STy = llvm::StructType::get(Context&: VMContext, Elements: Types, isPacked: false);
1280 return llvm::ConstantStruct::get(T: STy, V: Elts);
1281 }
1282
1283 case CK_AddressSpaceConversion: {
1284 llvm::Constant *C = Emitter.tryEmitPrivate(E: subExpr, T: subExpr->getType());
1285 if (!C)
1286 return nullptr;
1287 llvm::Type *destTy = ConvertType(T: E->getType());
1288 return CGM.performAddrSpaceCast(Src: C, DestTy: destTy);
1289 }
1290
1291 case CK_LValueToRValue: {
1292 // We don't really support doing lvalue-to-rvalue conversions here; any
1293 // interesting conversions should be done in Evaluate(). But as a
1294 // special case, allow compound literals to support the gcc extension
1295 // allowing "struct x {int x;} x = (struct x) {};".
1296 if (const auto *E =
1297 dyn_cast<CompoundLiteralExpr>(Val: subExpr->IgnoreParens()))
1298 return Visit(S: E->getInitializer(), P: destType);
1299 return nullptr;
1300 }
1301
1302 case CK_AtomicToNonAtomic:
1303 case CK_NonAtomicToAtomic:
1304 case CK_NoOp:
1305 case CK_ConstructorConversion:
1306 return Visit(S: subExpr, P: destType);
1307
1308 case CK_ArrayToPointerDecay:
1309 if (const auto *S = dyn_cast<StringLiteral>(Val: subExpr))
1310 return CGM.GetAddrOfConstantStringFromLiteral(S).getPointer();
1311 return nullptr;
1312 case CK_NullToPointer:
1313 if (Visit(S: subExpr, P: destType))
1314 return CGM.EmitNullConstant(T: destType);
1315 return nullptr;
1316
1317 case CK_IntToOCLSampler:
1318 llvm_unreachable("global sampler variables are not generated");
1319
1320 case CK_IntegralCast:
1321 return ProduceIntToIntCast(E: subExpr, DestType: destType);
1322
1323 case CK_Dependent: llvm_unreachable("saw dependent cast!");
1324
1325 case CK_BuiltinFnToFnPtr:
1326 llvm_unreachable("builtin functions are handled elsewhere");
1327
1328 case CK_ReinterpretMemberPointer:
1329 case CK_DerivedToBaseMemberPointer:
1330 case CK_BaseToDerivedMemberPointer: {
1331 auto C = Emitter.tryEmitPrivate(E: subExpr, T: subExpr->getType());
1332 if (!C) return nullptr;
1333 return CGM.getCXXABI().EmitMemberPointerConversion(E, Src: C);
1334 }
1335
1336 // These will never be supported.
1337 case CK_ObjCObjectLValueCast:
1338 case CK_ARCProduceObject:
1339 case CK_ARCConsumeObject:
1340 case CK_ARCReclaimReturnedObject:
1341 case CK_ARCExtendBlockObject:
1342 case CK_CopyAndAutoreleaseBlockObject:
1343 return nullptr;
1344
1345 // These don't need to be handled here because Evaluate knows how to
1346 // evaluate them in the cases where they can be folded.
1347 case CK_BitCast:
1348 case CK_ToVoid:
1349 case CK_Dynamic:
1350 case CK_LValueBitCast:
1351 case CK_LValueToRValueBitCast:
1352 case CK_NullToMemberPointer:
1353 case CK_UserDefinedConversion:
1354 case CK_CPointerToObjCPointerCast:
1355 case CK_BlockPointerToObjCPointerCast:
1356 case CK_AnyPointerToBlockPointerCast:
1357 case CK_FunctionToPointerDecay:
1358 case CK_BaseToDerived:
1359 case CK_DerivedToBase:
1360 case CK_UncheckedDerivedToBase:
1361 case CK_MemberPointerToBoolean:
1362 case CK_VectorSplat:
1363 case CK_FloatingRealToComplex:
1364 case CK_FloatingComplexToReal:
1365 case CK_FloatingComplexToBoolean:
1366 case CK_FloatingComplexCast:
1367 case CK_FloatingComplexToIntegralComplex:
1368 case CK_IntegralRealToComplex:
1369 case CK_IntegralComplexToReal:
1370 case CK_IntegralComplexToBoolean:
1371 case CK_IntegralComplexCast:
1372 case CK_IntegralComplexToFloatingComplex:
1373 case CK_PointerToIntegral:
1374 case CK_PointerToBoolean:
1375 case CK_BooleanToSignedIntegral:
1376 case CK_IntegralToPointer:
1377 case CK_IntegralToBoolean:
1378 case CK_IntegralToFloating:
1379 case CK_FloatingToIntegral:
1380 case CK_FloatingToBoolean:
1381 case CK_FloatingCast:
1382 case CK_FloatingToFixedPoint:
1383 case CK_FixedPointToFloating:
1384 case CK_FixedPointCast:
1385 case CK_FixedPointToBoolean:
1386 case CK_FixedPointToIntegral:
1387 case CK_IntegralToFixedPoint:
1388 case CK_ZeroToOCLOpaqueType:
1389 case CK_MatrixCast:
1390 case CK_HLSLVectorTruncation:
1391 case CK_HLSLMatrixTruncation:
1392 case CK_HLSLArrayRValue:
1393 case CK_HLSLElementwiseCast:
1394 case CK_HLSLAggregateSplatCast:
1395 return nullptr;
1396 }
1397 llvm_unreachable("Invalid CastKind");
1398 }
1399
1400 llvm::Constant *VisitCXXDefaultInitExpr(const CXXDefaultInitExpr *DIE,
1401 QualType T) {
1402 // No need for a DefaultInitExprScope: we don't handle 'this' in a
1403 // constant expression.
1404 return Visit(S: DIE->getExpr(), P: T);
1405 }
1406
1407 llvm::Constant *VisitExprWithCleanups(const ExprWithCleanups *E, QualType T) {
1408 return Visit(S: E->getSubExpr(), P: T);
1409 }
1410
1411 llvm::Constant *VisitIntegerLiteral(const IntegerLiteral *I, QualType T) {
1412 return llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: I->getValue());
1413 }
1414
1415 static APValue withDestType(ASTContext &Ctx, const Expr *E, QualType SrcType,
1416 QualType DestType, const llvm::APSInt &Value) {
1417 if (!Ctx.hasSameType(T1: SrcType, T2: DestType)) {
1418 if (DestType->isFloatingType()) {
1419 llvm::APFloat Result =
1420 llvm::APFloat(Ctx.getFloatTypeSemantics(T: DestType), 1);
1421 llvm::RoundingMode RM =
1422 E->getFPFeaturesInEffect(LO: Ctx.getLangOpts()).getRoundingMode();
1423 if (RM == llvm::RoundingMode::Dynamic)
1424 RM = llvm::RoundingMode::NearestTiesToEven;
1425 Result.convertFromAPInt(Input: Value, IsSigned: Value.isSigned(), RM);
1426 return APValue(Result);
1427 }
1428 }
1429 return APValue(Value);
1430 }
1431
1432 llvm::Constant *EmitArrayInitialization(const InitListExpr *ILE, QualType T) {
1433 auto *CAT = CGM.getContext().getAsConstantArrayType(T: ILE->getType());
1434 assert(CAT && "can't emit array init for non-constant-bound array");
1435 uint64_t NumInitElements = ILE->getNumInits();
1436 const uint64_t NumElements = CAT->getZExtSize();
1437 for (const auto *Init : ILE->inits()) {
1438 if (const auto *Embed =
1439 dyn_cast<EmbedExpr>(Val: Init->IgnoreParenImpCasts())) {
1440 NumInitElements += Embed->getDataElementCount() - 1;
1441 if (NumInitElements > NumElements) {
1442 NumInitElements = NumElements;
1443 break;
1444 }
1445 }
1446 }
1447
1448 // Initialising an array requires us to automatically
1449 // initialise any elements that have not been initialised explicitly
1450 uint64_t NumInitableElts = std::min<uint64_t>(a: NumInitElements, b: NumElements);
1451
1452 QualType EltType = CAT->getElementType();
1453
1454 // Initialize remaining array elements.
1455 llvm::Constant *fillC = nullptr;
1456 if (const Expr *filler = ILE->getArrayFiller()) {
1457 fillC = Emitter.tryEmitAbstractForMemory(E: filler, T: EltType);
1458 if (!fillC)
1459 return nullptr;
1460 }
1461
1462 // Copy initializer elements.
1463 SmallVector<llvm::Constant *, 16> Elts;
1464 if (fillC && fillC->isNullValue())
1465 Elts.reserve(N: NumInitableElts + 1);
1466 else
1467 Elts.reserve(N: NumElements);
1468
1469 llvm::Type *CommonElementType = nullptr;
1470 auto Emit = [&](const Expr *Init, unsigned ArrayIndex) {
1471 llvm::Constant *C = nullptr;
1472 C = Emitter.tryEmitPrivateForMemory(E: Init, T: EltType);
1473 if (!C)
1474 return false;
1475 if (ArrayIndex == 0)
1476 CommonElementType = C->getType();
1477 else if (C->getType() != CommonElementType)
1478 CommonElementType = nullptr;
1479 Elts.push_back(Elt: C);
1480 return true;
1481 };
1482
1483 unsigned ArrayIndex = 0;
1484 QualType DestTy = CAT->getElementType();
1485 for (unsigned i = 0; i < ILE->getNumInits(); ++i) {
1486 const Expr *Init = ILE->getInit(Init: i);
1487 if (auto *EmbedS = dyn_cast<EmbedExpr>(Val: Init->IgnoreParenImpCasts())) {
1488 StringLiteral *SL = EmbedS->getDataStringLiteral();
1489 llvm::APSInt Value(CGM.getContext().getTypeSize(T: DestTy),
1490 DestTy->isUnsignedIntegerType());
1491 llvm::Constant *C;
1492 for (unsigned I = EmbedS->getStartingElementPos(),
1493 N = EmbedS->getDataElementCount();
1494 I != EmbedS->getStartingElementPos() + N; ++I) {
1495 Value = SL->getCodeUnit(I);
1496 if (DestTy->isIntegerType()) {
1497 C = llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: Value);
1498 } else {
1499 C = Emitter.tryEmitPrivateForMemory(
1500 value: withDestType(Ctx&: CGM.getContext(), E: Init, SrcType: EmbedS->getType(), DestType: DestTy,
1501 Value),
1502 T: EltType);
1503 }
1504 if (!C)
1505 return nullptr;
1506 Elts.push_back(Elt: C);
1507 ArrayIndex++;
1508 }
1509 if ((ArrayIndex - EmbedS->getDataElementCount()) == 0)
1510 CommonElementType = C->getType();
1511 else if (C->getType() != CommonElementType)
1512 CommonElementType = nullptr;
1513 } else {
1514 if (!Emit(Init, ArrayIndex))
1515 return nullptr;
1516 ArrayIndex++;
1517 }
1518 }
1519
1520 llvm::ArrayType *Desired =
1521 cast<llvm::ArrayType>(Val: CGM.getTypes().ConvertType(T: ILE->getType()));
1522 return EmitArrayConstant(CGM, DesiredType: Desired, CommonElementType, ArrayBound: NumElements, Elements&: Elts,
1523 Filler: fillC);
1524 }
1525
1526 llvm::Constant *EmitRecordInitialization(const InitListExpr *ILE,
1527 QualType T) {
1528 return ConstStructBuilder::BuildStruct(Emitter, ILE, ValTy: T);
1529 }
1530
1531 llvm::Constant *VisitImplicitValueInitExpr(const ImplicitValueInitExpr *E,
1532 QualType T) {
1533 return CGM.EmitNullConstant(T);
1534 }
1535
1536 llvm::Constant *VisitInitListExpr(const InitListExpr *ILE, QualType T) {
1537 if (ILE->isTransparent())
1538 return Visit(S: ILE->getInit(Init: 0), P: T);
1539
1540 if (ILE->getType()->isArrayType())
1541 return EmitArrayInitialization(ILE, T);
1542
1543 if (ILE->getType()->isRecordType())
1544 return EmitRecordInitialization(ILE, T);
1545
1546 return nullptr;
1547 }
1548
1549 llvm::Constant *
1550 VisitDesignatedInitUpdateExpr(const DesignatedInitUpdateExpr *E,
1551 QualType destType) {
1552 auto C = Visit(S: E->getBase(), P: destType);
1553 if (!C)
1554 return nullptr;
1555
1556 ConstantAggregateBuilder Const(CGM);
1557 Const.add(C, Offset: CharUnits::Zero(), AllowOverwrite: false);
1558
1559 if (!EmitDesignatedInitUpdater(Emitter, Const, Offset: CharUnits::Zero(), Type: destType,
1560 Updater: E->getUpdater()))
1561 return nullptr;
1562
1563 llvm::Type *ValTy = CGM.getTypes().ConvertType(T: destType);
1564 bool HasFlexibleArray = false;
1565 if (const auto *RD = destType->getAsRecordDecl())
1566 HasFlexibleArray = RD->hasFlexibleArrayMember();
1567 return Const.build(DesiredTy: ValTy, AllowOversized: HasFlexibleArray);
1568 }
1569
1570 llvm::Constant *VisitCXXConstructExpr(const CXXConstructExpr *E,
1571 QualType Ty) {
1572 if (!E->getConstructor()->isTrivial())
1573 return nullptr;
1574
1575 // Only default and copy/move constructors can be trivial.
1576 if (E->getNumArgs()) {
1577 assert(E->getNumArgs() == 1 && "trivial ctor with > 1 argument");
1578 assert(E->getConstructor()->isCopyOrMoveConstructor() &&
1579 "trivial ctor has argument but isn't a copy/move ctor");
1580
1581 const Expr *Arg = E->getArg(Arg: 0);
1582 assert(CGM.getContext().hasSameUnqualifiedType(Ty, Arg->getType()) &&
1583 "argument to copy ctor is of wrong type");
1584
1585 // Look through the temporary; it's just converting the value to an
1586 // lvalue to pass it to the constructor.
1587 if (const auto *MTE = dyn_cast<MaterializeTemporaryExpr>(Val: Arg))
1588 return Visit(S: MTE->getSubExpr(), P: Ty);
1589 // Don't try to support arbitrary lvalue-to-rvalue conversions for now.
1590 return nullptr;
1591 }
1592
1593 return CGM.EmitNullConstant(T: Ty);
1594 }
1595
1596 llvm::Constant *VisitStringLiteral(const StringLiteral *E, QualType T) {
1597 // This is a string literal initializing an array in an initializer.
1598 return CGM.GetConstantArrayFromStringLiteral(E);
1599 }
1600
1601 llvm::Constant *VisitObjCEncodeExpr(const ObjCEncodeExpr *E, QualType T) {
1602 // This must be an @encode initializing an array in a static initializer.
1603 // Don't emit it as the address of the string, emit the string data itself
1604 // as an inline array.
1605 std::string Str;
1606 CGM.getContext().getObjCEncodingForType(T: E->getEncodedType(), S&: Str);
1607 const ConstantArrayType *CAT = CGM.getContext().getAsConstantArrayType(T);
1608 assert(CAT && "String data not of constant array type!");
1609
1610 // Resize the string to the right size, adding zeros at the end, or
1611 // truncating as needed.
1612 Str.resize(n: CAT->getZExtSize(), c: '\0');
1613 return llvm::ConstantDataArray::getString(Context&: VMContext, Initializer: Str, AddNull: false);
1614 }
1615
1616 llvm::Constant *VisitUnaryExtension(const UnaryOperator *E, QualType T) {
1617 return Visit(S: E->getSubExpr(), P: T);
1618 }
1619
1620 llvm::Constant *VisitUnaryMinus(const UnaryOperator *U, QualType T) {
1621 if (llvm::Constant *C = Visit(S: U->getSubExpr(), P: T))
1622 if (auto *CI = dyn_cast<llvm::ConstantInt>(Val: C))
1623 return llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: -CI->getValue());
1624 return nullptr;
1625 }
1626
1627 llvm::Constant *VisitPackIndexingExpr(const PackIndexingExpr *E, QualType T) {
1628 return Visit(S: E->getSelectedExpr(), P: T);
1629 }
1630
1631 // Utility methods
1632 llvm::Type *ConvertType(QualType T) {
1633 return CGM.getTypes().ConvertType(T);
1634 }
1635};
1636
1637} // end anonymous namespace.
1638
1639llvm::Constant *ConstantEmitter::validateAndPopAbstract(llvm::Constant *C,
1640 AbstractState saved) {
1641 Abstract = saved.OldValue;
1642
1643 assert(saved.OldPlaceholdersSize == PlaceholderAddresses.size() &&
1644 "created a placeholder while doing an abstract emission?");
1645
1646 // No validation necessary for now.
1647 // No cleanup to do for now.
1648 return C;
1649}
1650
1651llvm::Constant *
1652ConstantEmitter::tryEmitAbstractForInitializer(const VarDecl &D) {
1653 auto state = pushAbstract();
1654 auto C = tryEmitPrivateForVarInit(D);
1655 return validateAndPopAbstract(C, saved: state);
1656}
1657
1658llvm::Constant *
1659ConstantEmitter::tryEmitAbstract(const Expr *E, QualType destType) {
1660 auto state = pushAbstract();
1661 auto C = tryEmitPrivate(E, T: destType);
1662 return validateAndPopAbstract(C, saved: state);
1663}
1664
1665llvm::Constant *
1666ConstantEmitter::tryEmitAbstract(const APValue &value, QualType destType) {
1667 auto state = pushAbstract();
1668 auto C = tryEmitPrivate(value, T: destType);
1669 return validateAndPopAbstract(C, saved: state);
1670}
1671
1672llvm::Constant *ConstantEmitter::tryEmitConstantExpr(const ConstantExpr *CE) {
1673 if (!CE->hasAPValueResult())
1674 return nullptr;
1675
1676 QualType RetType = CE->getType();
1677 if (CE->isGLValue())
1678 RetType = CGM.getContext().getLValueReferenceType(T: RetType);
1679
1680 return tryEmitAbstract(value: CE->getAPValueResult(), destType: RetType);
1681}
1682
1683llvm::Constant *
1684ConstantEmitter::emitAbstract(const Expr *E, QualType destType) {
1685 auto state = pushAbstract();
1686 auto C = tryEmitPrivate(E, T: destType);
1687 C = validateAndPopAbstract(C, saved: state);
1688 if (!C) {
1689 CGM.Error(loc: E->getExprLoc(),
1690 error: "internal error: could not emit constant value \"abstractly\"");
1691 C = CGM.EmitNullConstant(T: destType);
1692 }
1693 return C;
1694}
1695
1696llvm::Constant *
1697ConstantEmitter::emitAbstract(SourceLocation loc, const APValue &value,
1698 QualType destType,
1699 bool EnablePtrAuthFunctionTypeDiscrimination) {
1700 auto state = pushAbstract();
1701 auto C =
1702 tryEmitPrivate(value, T: destType, EnablePtrAuthFunctionTypeDiscrimination);
1703 C = validateAndPopAbstract(C, saved: state);
1704 if (!C) {
1705 CGM.Error(loc,
1706 error: "internal error: could not emit constant value \"abstractly\"");
1707 C = CGM.EmitNullConstant(T: destType);
1708 }
1709 return C;
1710}
1711
1712llvm::Constant *ConstantEmitter::tryEmitForInitializer(const VarDecl &D) {
1713 initializeNonAbstract(destAS: D.getType().getAddressSpace());
1714 llvm::Constant *Init = tryEmitPrivateForVarInit(D);
1715
1716 // If a placeholder address was needed for a TLS variable, implying that the
1717 // initializer's value depends on its address, then the object may not be
1718 // initialized in .tdata because the initializer will be memcpy'd to the
1719 // thread's TLS. Instead the initialization must be done in code.
1720 if (!PlaceholderAddresses.empty() && D.getTLSKind() != VarDecl::TLS_None) {
1721 for (auto [_, GV] : PlaceholderAddresses)
1722 GV->eraseFromParent();
1723 PlaceholderAddresses.clear();
1724 Init = nullptr;
1725 }
1726
1727 return markIfFailed(init: Init);
1728}
1729
1730llvm::Constant *ConstantEmitter::tryEmitForInitializer(const Expr *E,
1731 LangAS destAddrSpace,
1732 QualType destType) {
1733 initializeNonAbstract(destAS: destAddrSpace);
1734 return markIfFailed(init: tryEmitPrivateForMemory(E, T: destType));
1735}
1736
1737llvm::Constant *ConstantEmitter::emitForInitializer(const APValue &value,
1738 LangAS destAddrSpace,
1739 QualType destType) {
1740 initializeNonAbstract(destAS: destAddrSpace);
1741 auto C = tryEmitPrivateForMemory(value, T: destType);
1742 assert(C && "couldn't emit constant value non-abstractly?");
1743 return C;
1744}
1745
1746llvm::GlobalValue *ConstantEmitter::getCurrentAddrPrivate() {
1747 assert(!Abstract && "cannot get current address for abstract constant");
1748
1749
1750
1751 // Make an obviously ill-formed global that should blow up compilation
1752 // if it survives.
1753 auto global = new llvm::GlobalVariable(CGM.getModule(), CGM.Int8Ty, true,
1754 llvm::GlobalValue::PrivateLinkage,
1755 /*init*/ nullptr,
1756 /*name*/ "",
1757 /*before*/ nullptr,
1758 llvm::GlobalVariable::NotThreadLocal,
1759 CGM.getContext().getTargetAddressSpace(AS: DestAddressSpace));
1760
1761 PlaceholderAddresses.push_back(Elt: std::make_pair(x: nullptr, y&: global));
1762
1763 return global;
1764}
1765
1766void ConstantEmitter::registerCurrentAddrPrivate(llvm::Constant *signal,
1767 llvm::GlobalValue *placeholder) {
1768 assert(!PlaceholderAddresses.empty());
1769 assert(PlaceholderAddresses.back().first == nullptr);
1770 assert(PlaceholderAddresses.back().second == placeholder);
1771 PlaceholderAddresses.back().first = signal;
1772}
1773
1774namespace {
1775 struct ReplacePlaceholders {
1776 CodeGenModule &CGM;
1777
1778 /// The base address of the global.
1779 llvm::Constant *Base;
1780 llvm::Type *BaseValueTy = nullptr;
1781
1782 /// The placeholder addresses that were registered during emission.
1783 llvm::DenseMap<llvm::Constant*, llvm::GlobalVariable*> PlaceholderAddresses;
1784
1785 /// The locations of the placeholder signals.
1786 llvm::DenseMap<llvm::GlobalVariable*, llvm::Constant*> Locations;
1787
1788 /// The current index stack. We use a simple unsigned stack because
1789 /// we assume that placeholders will be relatively sparse in the
1790 /// initializer, but we cache the index values we find just in case.
1791 llvm::SmallVector<unsigned, 8> Indices;
1792 llvm::SmallVector<llvm::Constant*, 8> IndexValues;
1793
1794 ReplacePlaceholders(CodeGenModule &CGM, llvm::Constant *base,
1795 ArrayRef<std::pair<llvm::Constant*,
1796 llvm::GlobalVariable*>> addresses)
1797 : CGM(CGM), Base(base),
1798 PlaceholderAddresses(addresses.begin(), addresses.end()) {
1799 }
1800
1801 void replaceInInitializer(llvm::Constant *init) {
1802 // Remember the type of the top-most initializer.
1803 BaseValueTy = init->getType();
1804
1805 // Initialize the stack.
1806 Indices.push_back(Elt: 0);
1807 IndexValues.push_back(Elt: nullptr);
1808
1809 // Recurse into the initializer.
1810 findLocations(init);
1811
1812 // Check invariants.
1813 assert(IndexValues.size() == Indices.size() && "mismatch");
1814 assert(Indices.size() == 1 && "didn't pop all indices");
1815
1816 // Do the replacement; this basically invalidates 'init'.
1817 assert(Locations.size() == PlaceholderAddresses.size() &&
1818 "missed a placeholder?");
1819
1820 // We're iterating over a hashtable, so this would be a source of
1821 // non-determinism in compiler output *except* that we're just
1822 // messing around with llvm::Constant structures, which never itself
1823 // does anything that should be visible in compiler output.
1824 for (auto &entry : Locations) {
1825 assert(entry.first->getName() == "" && "not a placeholder!");
1826 entry.first->replaceAllUsesWith(V: entry.second);
1827 entry.first->eraseFromParent();
1828 }
1829 }
1830
1831 private:
1832 void findLocations(llvm::Constant *init) {
1833 // Recurse into aggregates.
1834 if (auto agg = dyn_cast<llvm::ConstantAggregate>(Val: init)) {
1835 for (unsigned i = 0, e = agg->getNumOperands(); i != e; ++i) {
1836 Indices.push_back(Elt: i);
1837 IndexValues.push_back(Elt: nullptr);
1838
1839 findLocations(init: agg->getOperand(i_nocapture: i));
1840
1841 IndexValues.pop_back();
1842 Indices.pop_back();
1843 }
1844 return;
1845 }
1846
1847 // Otherwise, check for registered constants.
1848 while (true) {
1849 auto it = PlaceholderAddresses.find(Val: init);
1850 if (it != PlaceholderAddresses.end()) {
1851 setLocation(it->second);
1852 break;
1853 }
1854
1855 // Look through bitcasts or other expressions.
1856 if (auto expr = dyn_cast<llvm::ConstantExpr>(Val: init)) {
1857 init = expr->getOperand(i_nocapture: 0);
1858 } else {
1859 break;
1860 }
1861 }
1862 }
1863
1864 void setLocation(llvm::GlobalVariable *placeholder) {
1865 assert(!Locations.contains(placeholder) &&
1866 "already found location for placeholder!");
1867
1868 // Lazily fill in IndexValues with the values from Indices.
1869 // We do this in reverse because we should always have a strict
1870 // prefix of indices from the start.
1871 assert(Indices.size() == IndexValues.size());
1872 for (size_t i = Indices.size() - 1; i != size_t(-1); --i) {
1873 if (IndexValues[i]) {
1874#ifndef NDEBUG
1875 for (size_t j = 0; j != i + 1; ++j) {
1876 assert(IndexValues[j] &&
1877 isa<llvm::ConstantInt>(IndexValues[j]) &&
1878 cast<llvm::ConstantInt>(IndexValues[j])->getZExtValue()
1879 == Indices[j]);
1880 }
1881#endif
1882 break;
1883 }
1884
1885 IndexValues[i] = llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: Indices[i]);
1886 }
1887
1888 llvm::Constant *location = llvm::ConstantExpr::getInBoundsGetElementPtr(
1889 Ty: BaseValueTy, C: Base, IdxList: IndexValues);
1890
1891 Locations.insert(KV: {placeholder, location});
1892 }
1893 };
1894}
1895
1896void ConstantEmitter::finalize(llvm::GlobalVariable *global) {
1897 assert(InitializedNonAbstract &&
1898 "finalizing emitter that was used for abstract emission?");
1899 assert(!Finalized && "finalizing emitter multiple times");
1900 assert(global->getInitializer());
1901
1902 // Note that we might also be Failed.
1903 Finalized = true;
1904
1905 if (!PlaceholderAddresses.empty()) {
1906 ReplacePlaceholders(CGM, global, PlaceholderAddresses)
1907 .replaceInInitializer(init: global->getInitializer());
1908 PlaceholderAddresses.clear(); // satisfy
1909 }
1910}
1911
1912ConstantEmitter::~ConstantEmitter() {
1913 assert((!InitializedNonAbstract || Finalized || Failed) &&
1914 "not finalized after being initialized for non-abstract emission");
1915 assert(PlaceholderAddresses.empty() && "unhandled placeholders");
1916}
1917
1918static QualType getNonMemoryType(CodeGenModule &CGM, QualType type) {
1919 if (auto AT = type->getAs<AtomicType>()) {
1920 return CGM.getContext().getQualifiedType(T: AT->getValueType(),
1921 Qs: type.getQualifiers());
1922 }
1923 return type;
1924}
1925
1926llvm::Constant *ConstantEmitter::tryEmitPrivateForVarInit(const VarDecl &D) {
1927 // Make a quick check if variable can be default NULL initialized
1928 // and avoid going through rest of code which may do, for c++11,
1929 // initialization of memory to all NULLs.
1930 if (!D.hasLocalStorage()) {
1931 QualType Ty = CGM.getContext().getBaseElementType(QT: D.getType());
1932 if (Ty->isRecordType())
1933 if (const CXXConstructExpr *E =
1934 dyn_cast_or_null<CXXConstructExpr>(Val: D.getInit())) {
1935 const CXXConstructorDecl *CD = E->getConstructor();
1936 if (CD->isTrivial() && CD->isDefaultConstructor())
1937 return CGM.EmitNullConstant(T: D.getType());
1938 }
1939 }
1940 InConstantContext = D.hasConstantInitialization();
1941
1942 QualType destType = D.getType();
1943 const Expr *E = D.getInit();
1944 assert(E && "No initializer to emit");
1945
1946 if (!destType->isReferenceType()) {
1947 QualType nonMemoryDestType = getNonMemoryType(CGM, type: destType);
1948 if (llvm::Constant *C = ConstExprEmitter(*this).Visit(S: E, P: nonMemoryDestType))
1949 return emitForMemory(C, T: destType);
1950 }
1951
1952 // Try to emit the initializer. Note that this can allow some things that
1953 // are not allowed by tryEmitPrivateForMemory alone.
1954 if (const APValue *value = D.evaluateValue()) {
1955 assert(!value->allowConstexprUnknown() &&
1956 "Constexpr unknown values are not allowed in CodeGen");
1957 return tryEmitPrivateForMemory(value: *value, T: destType);
1958 }
1959
1960 return nullptr;
1961}
1962
1963llvm::Constant *
1964ConstantEmitter::tryEmitAbstractForMemory(const Expr *E, QualType destType) {
1965 auto nonMemoryDestType = getNonMemoryType(CGM, type: destType);
1966 auto C = tryEmitAbstract(E, destType: nonMemoryDestType);
1967 return (C ? emitForMemory(C, T: destType) : nullptr);
1968}
1969
1970llvm::Constant *
1971ConstantEmitter::tryEmitAbstractForMemory(const APValue &value,
1972 QualType destType) {
1973 auto nonMemoryDestType = getNonMemoryType(CGM, type: destType);
1974 auto C = tryEmitAbstract(value, destType: nonMemoryDestType);
1975 return (C ? emitForMemory(C, T: destType) : nullptr);
1976}
1977
1978llvm::Constant *ConstantEmitter::tryEmitPrivateForMemory(const Expr *E,
1979 QualType destType) {
1980 auto nonMemoryDestType = getNonMemoryType(CGM, type: destType);
1981 llvm::Constant *C = tryEmitPrivate(E, T: nonMemoryDestType);
1982 return (C ? emitForMemory(C, T: destType) : nullptr);
1983}
1984
1985llvm::Constant *ConstantEmitter::tryEmitPrivateForMemory(const APValue &value,
1986 QualType destType) {
1987 auto nonMemoryDestType = getNonMemoryType(CGM, type: destType);
1988 auto C = tryEmitPrivate(value, T: nonMemoryDestType);
1989 return (C ? emitForMemory(C, T: destType) : nullptr);
1990}
1991
1992/// Try to emit a constant signed pointer, given a raw pointer and the
1993/// destination ptrauth qualifier.
1994///
1995/// This can fail if the qualifier needs address discrimination and the
1996/// emitter is in an abstract mode.
1997llvm::Constant *
1998ConstantEmitter::tryEmitConstantSignedPointer(llvm::Constant *UnsignedPointer,
1999 PointerAuthQualifier Schema) {
2000 assert(Schema && "applying trivial ptrauth schema");
2001
2002 if (Schema.hasKeyNone())
2003 return UnsignedPointer;
2004
2005 unsigned Key = Schema.getKey();
2006
2007 // Create an address placeholder if we're using address discrimination.
2008 llvm::GlobalValue *StorageAddress = nullptr;
2009 if (Schema.isAddressDiscriminated()) {
2010 // We can't do this if the emitter is in an abstract state.
2011 if (isAbstract())
2012 return nullptr;
2013
2014 StorageAddress = getCurrentAddrPrivate();
2015 }
2016
2017 llvm::ConstantInt *Discriminator =
2018 llvm::ConstantInt::get(Ty: CGM.IntPtrTy, V: Schema.getExtraDiscriminator());
2019
2020 llvm::Constant *SignedPointer = CGM.getConstantSignedPointer(
2021 Pointer: UnsignedPointer, Key, StorageAddress, OtherDiscriminator: Discriminator);
2022
2023 if (Schema.isAddressDiscriminated())
2024 registerCurrentAddrPrivate(signal: SignedPointer, placeholder: StorageAddress);
2025
2026 return SignedPointer;
2027}
2028
2029llvm::Constant *ConstantEmitter::emitForMemory(CodeGenModule &CGM,
2030 llvm::Constant *C,
2031 QualType destType) {
2032 // For an _Atomic-qualified constant, we may need to add tail padding.
2033 if (auto AT = destType->getAs<AtomicType>()) {
2034 QualType destValueType = AT->getValueType();
2035 C = emitForMemory(CGM, C, destType: destValueType);
2036
2037 uint64_t innerSize = CGM.getContext().getTypeSize(T: destValueType);
2038 uint64_t outerSize = CGM.getContext().getTypeSize(T: destType);
2039 if (innerSize == outerSize)
2040 return C;
2041
2042 assert(innerSize < outerSize && "emitted over-large constant for atomic");
2043 llvm::Constant *elts[] = {
2044 C,
2045 llvm::ConstantAggregateZero::get(
2046 Ty: llvm::ArrayType::get(ElementType: CGM.Int8Ty, NumElements: (outerSize - innerSize) / 8))
2047 };
2048 return llvm::ConstantStruct::getAnon(V: elts);
2049 }
2050
2051 // Zero-extend bool.
2052 // In HLSL bool vectors are stored in memory as a vector of i32
2053 if ((C->getType()->isIntegerTy(BitWidth: 1) && !destType->isBitIntType()) ||
2054 (destType->isExtVectorBoolType() &&
2055 !destType->isPackedVectorBoolType(ctx: CGM.getContext()))) {
2056 llvm::Type *boolTy = CGM.getTypes().ConvertTypeForMem(T: destType);
2057 llvm::Constant *Res = llvm::ConstantFoldCastOperand(
2058 Opcode: llvm::Instruction::ZExt, C, DestTy: boolTy, DL: CGM.getDataLayout());
2059 assert(Res && "Constant folding must succeed");
2060 return Res;
2061 }
2062
2063 if (destType->isBitIntType()) {
2064 llvm::Type *MemTy = CGM.getTypes().ConvertTypeForMem(T: destType);
2065 if (C->getType() != MemTy) {
2066 ConstantAggregateBuilder Builder(CGM);
2067 llvm::Type *LoadStoreTy =
2068 CGM.getTypes().convertTypeForLoadStore(T: destType);
2069 // ptrtoint/inttoptr should not involve _BitInt in constant expressions,
2070 // so casting to ConstantInt is safe here.
2071 auto *CI = cast<llvm::ConstantInt>(Val: C);
2072 llvm::Constant *Res = llvm::ConstantFoldCastOperand(
2073 Opcode: destType->isSignedIntegerOrEnumerationType()
2074 ? llvm::Instruction::SExt
2075 : llvm::Instruction::ZExt,
2076 C: CI, DestTy: LoadStoreTy, DL: CGM.getDataLayout());
2077 if (CGM.getTypes().typeRequiresSplitIntoByteArray(ASTTy: destType,
2078 LLVMTy: C->getType())) {
2079 // Long _BitInt has array of bytes as in-memory type.
2080 // So, split constant into individual bytes.
2081 llvm::APInt Value = cast<llvm::ConstantInt>(Val: Res)->getValue();
2082 Builder.addBits(Bits: Value, /*OffsetInBits=*/0, /*AllowOverwrite=*/false);
2083 return Builder.build(DesiredTy: MemTy, /*AllowOversized*/ false);
2084 }
2085 return Res;
2086 }
2087 }
2088
2089 return C;
2090}
2091
2092llvm::Constant *ConstantEmitter::tryEmitPrivate(const Expr *E,
2093 QualType destType) {
2094 assert(!destType->isVoidType() && "can't emit a void constant");
2095
2096 if (!destType->isReferenceType())
2097 if (llvm::Constant *C = ConstExprEmitter(*this).Visit(S: E, P: destType))
2098 return C;
2099
2100 Expr::EvalResult Result;
2101
2102 bool Success = false;
2103
2104 if (destType->isReferenceType())
2105 Success = E->EvaluateAsLValue(Result, Ctx: CGM.getContext());
2106 else
2107 Success = E->EvaluateAsRValue(Result, Ctx: CGM.getContext(), InConstantContext);
2108
2109 if (Success && !Result.HasSideEffects)
2110 return tryEmitPrivate(value: Result.Val, T: destType);
2111
2112 return nullptr;
2113}
2114
2115llvm::Constant *CodeGenModule::getNullPointer(llvm::PointerType *T, QualType QT) {
2116 return getTargetCodeGenInfo().getNullPointer(CGM: *this, T, QT);
2117}
2118
2119namespace {
2120/// A struct which can be used to peephole certain kinds of finalization
2121/// that normally happen during l-value emission.
2122struct ConstantLValue {
2123 llvm::Constant *Value;
2124 bool HasOffsetApplied;
2125 bool HasDestPointerAuth;
2126
2127 /*implicit*/ ConstantLValue(llvm::Constant *value,
2128 bool hasOffsetApplied = false,
2129 bool hasDestPointerAuth = false)
2130 : Value(value), HasOffsetApplied(hasOffsetApplied),
2131 HasDestPointerAuth(hasDestPointerAuth) {}
2132
2133 /*implicit*/ ConstantLValue(ConstantAddress address)
2134 : ConstantLValue(address.getPointer()) {}
2135};
2136
2137/// A helper class for emitting constant l-values.
2138class ConstantLValueEmitter : public ConstStmtVisitor<ConstantLValueEmitter,
2139 ConstantLValue> {
2140 CodeGenModule &CGM;
2141 ConstantEmitter &Emitter;
2142 const APValue &Value;
2143 QualType DestType;
2144 bool EnablePtrAuthFunctionTypeDiscrimination;
2145
2146 // Befriend StmtVisitorBase so that we don't have to expose Visit*.
2147 friend StmtVisitorBase;
2148
2149public:
2150 ConstantLValueEmitter(ConstantEmitter &emitter, const APValue &value,
2151 QualType destType,
2152 bool EnablePtrAuthFunctionTypeDiscrimination = true)
2153 : CGM(emitter.CGM), Emitter(emitter), Value(value), DestType(destType),
2154 EnablePtrAuthFunctionTypeDiscrimination(
2155 EnablePtrAuthFunctionTypeDiscrimination) {}
2156
2157 llvm::Constant *tryEmit();
2158
2159private:
2160 llvm::Constant *tryEmitAbsolute(llvm::Type *destTy);
2161 ConstantLValue tryEmitBase(const APValue::LValueBase &base);
2162
2163 ConstantLValue VisitStmt(const Stmt *S) { return nullptr; }
2164 ConstantLValue VisitConstantExpr(const ConstantExpr *E);
2165 ConstantLValue VisitCompoundLiteralExpr(const CompoundLiteralExpr *E);
2166 ConstantLValue VisitStringLiteral(const StringLiteral *E);
2167 ConstantLValue VisitObjCBoxedExpr(const ObjCBoxedExpr *E);
2168 ConstantLValue VisitObjCEncodeExpr(const ObjCEncodeExpr *E);
2169 ConstantLValue VisitObjCStringLiteral(const ObjCStringLiteral *E);
2170 llvm::Constant *VisitObjCCollectionElement(const Expr *E);
2171 ConstantLValue VisitObjCArrayLiteral(const ObjCArrayLiteral *E);
2172 ConstantLValue VisitObjCDictionaryLiteral(const ObjCDictionaryLiteral *E);
2173 ConstantLValue VisitPredefinedExpr(const PredefinedExpr *E);
2174 ConstantLValue VisitAddrLabelExpr(const AddrLabelExpr *E);
2175 ConstantLValue VisitCallExpr(const CallExpr *E);
2176 ConstantLValue VisitBlockExpr(const BlockExpr *E);
2177 ConstantLValue VisitCXXTypeidExpr(const CXXTypeidExpr *E);
2178 ConstantLValue VisitMaterializeTemporaryExpr(
2179 const MaterializeTemporaryExpr *E);
2180
2181 ConstantLValue emitPointerAuthSignConstant(const CallExpr *E);
2182 llvm::Constant *emitPointerAuthPointer(const Expr *E);
2183 unsigned emitPointerAuthKey(const Expr *E);
2184 std::pair<llvm::Constant *, llvm::ConstantInt *>
2185 emitPointerAuthDiscriminator(const Expr *E);
2186
2187 bool hasNonZeroOffset() const {
2188 return !Value.getLValueOffset().isZero();
2189 }
2190
2191 /// Return the value offset.
2192 llvm::Constant *getOffset() {
2193 return llvm::ConstantInt::get(Ty: CGM.Int64Ty,
2194 V: Value.getLValueOffset().getQuantity());
2195 }
2196
2197 /// Apply the value offset to the given constant.
2198 llvm::Constant *applyOffset(llvm::Constant *C) {
2199 if (!hasNonZeroOffset())
2200 return C;
2201
2202 return llvm::ConstantExpr::getPtrAdd(Ptr: C, Offset: getOffset());
2203 }
2204};
2205
2206}
2207
2208llvm::Constant *ConstantLValueEmitter::tryEmit() {
2209 const APValue::LValueBase &base = Value.getLValueBase();
2210
2211 // The destination type should be a pointer or reference
2212 // type, but it might also be a cast thereof.
2213 //
2214 // FIXME: the chain of casts required should be reflected in the APValue.
2215 // We need this in order to correctly handle things like a ptrtoint of a
2216 // non-zero null pointer and addrspace casts that aren't trivially
2217 // represented in LLVM IR.
2218 auto destTy = CGM.getTypes().ConvertTypeForMem(T: DestType);
2219 assert(isa<llvm::IntegerType>(destTy) || isa<llvm::PointerType>(destTy));
2220
2221 // If there's no base at all, this is a null or absolute pointer,
2222 // possibly cast back to an integer type.
2223 if (!base) {
2224 return tryEmitAbsolute(destTy);
2225 }
2226
2227 // Otherwise, try to emit the base.
2228 ConstantLValue result = tryEmitBase(base);
2229
2230 // If that failed, we're done.
2231 llvm::Constant *value = result.Value;
2232 if (!value) return nullptr;
2233
2234 // Apply the offset if necessary and not already done.
2235 if (!result.HasOffsetApplied) {
2236 value = applyOffset(C: value);
2237 }
2238
2239 // Apply pointer-auth signing from the destination type.
2240 if (PointerAuthQualifier PointerAuth = DestType.getPointerAuth();
2241 PointerAuth && !result.HasDestPointerAuth) {
2242 value = Emitter.tryEmitConstantSignedPointer(UnsignedPointer: value, Schema: PointerAuth);
2243 if (!value)
2244 return nullptr;
2245 }
2246
2247 // Convert to the appropriate type; this could be an lvalue for
2248 // an integer. FIXME: performAddrSpaceCast
2249 if (isa<llvm::PointerType>(Val: destTy))
2250 return llvm::ConstantExpr::getPointerCast(C: value, Ty: destTy);
2251
2252 return llvm::ConstantExpr::getPtrToInt(C: value, Ty: destTy);
2253}
2254
2255/// Try to emit an absolute l-value, such as a null pointer or an integer
2256/// bitcast to pointer type.
2257llvm::Constant *
2258ConstantLValueEmitter::tryEmitAbsolute(llvm::Type *destTy) {
2259 // If we're producing a pointer, this is easy.
2260 auto destPtrTy = cast<llvm::PointerType>(Val: destTy);
2261 if (Value.isNullPointer()) {
2262 // FIXME: integer offsets from non-zero null pointers.
2263 return CGM.getNullPointer(T: destPtrTy, QT: DestType);
2264 }
2265
2266 // Convert the integer to a pointer-sized integer before converting it
2267 // to a pointer.
2268 // FIXME: signedness depends on the original integer type.
2269 auto intptrTy = CGM.getDataLayout().getIntPtrType(destPtrTy);
2270 llvm::Constant *C;
2271 C = llvm::ConstantFoldIntegerCast(C: getOffset(), DestTy: intptrTy, /*isSigned*/ IsSigned: false,
2272 DL: CGM.getDataLayout());
2273 assert(C && "Must have folded, as Offset is a ConstantInt");
2274 C = llvm::ConstantExpr::getIntToPtr(C, Ty: destPtrTy);
2275 return C;
2276}
2277
2278ConstantLValue
2279ConstantLValueEmitter::tryEmitBase(const APValue::LValueBase &base) {
2280 // Handle values.
2281 if (const ValueDecl *D = base.dyn_cast<const ValueDecl*>()) {
2282 // The constant always points to the canonical declaration. We want to look
2283 // at properties of the most recent declaration at the point of emission.
2284 D = cast<ValueDecl>(Val: D->getMostRecentDecl());
2285
2286 if (D->hasAttr<WeakRefAttr>())
2287 return CGM.GetWeakRefReference(VD: D).getPointer();
2288
2289 auto PtrAuthSign = [&](llvm::Constant *C) {
2290 if (PointerAuthQualifier PointerAuth = DestType.getPointerAuth()) {
2291 C = applyOffset(C);
2292 C = Emitter.tryEmitConstantSignedPointer(UnsignedPointer: C, Schema: PointerAuth);
2293 return ConstantLValue(C, /*applied offset*/ true, /*signed*/ true);
2294 }
2295
2296 CGPointerAuthInfo AuthInfo;
2297
2298 if (EnablePtrAuthFunctionTypeDiscrimination)
2299 AuthInfo = CGM.getFunctionPointerAuthInfo(T: DestType);
2300
2301 if (AuthInfo) {
2302 if (hasNonZeroOffset())
2303 return ConstantLValue(nullptr);
2304
2305 C = applyOffset(C);
2306 C = CGM.getConstantSignedPointer(
2307 Pointer: C, Key: AuthInfo.getKey(), StorageAddress: nullptr,
2308 OtherDiscriminator: cast_or_null<llvm::ConstantInt>(Val: AuthInfo.getDiscriminator()));
2309 return ConstantLValue(C, /*applied offset*/ true, /*signed*/ true);
2310 }
2311
2312 return ConstantLValue(C);
2313 };
2314
2315 if (const auto *FD = dyn_cast<FunctionDecl>(Val: D)) {
2316 llvm::Constant *C = CGM.getRawFunctionPointer(GD: FD);
2317 if (FD->getType()->isCFIUncheckedCalleeFunctionType())
2318 C = llvm::NoCFIValue::get(GV: cast<llvm::GlobalValue>(Val: C));
2319 return PtrAuthSign(C);
2320 }
2321
2322 if (const auto *VD = dyn_cast<VarDecl>(Val: D)) {
2323 // We can never refer to a variable with local storage.
2324 if (!VD->hasLocalStorage()) {
2325 if (VD->isFileVarDecl() || VD->hasExternalStorage())
2326 return CGM.GetAddrOfGlobalVar(D: VD);
2327
2328 if (VD->isLocalVarDecl()) {
2329 return CGM.getOrCreateStaticVarDecl(
2330 D: *VD, Linkage: CGM.getLLVMLinkageVarDefinition(VD));
2331 }
2332 }
2333 }
2334
2335 if (const auto *GD = dyn_cast<MSGuidDecl>(Val: D))
2336 return CGM.GetAddrOfMSGuidDecl(GD);
2337
2338 if (const auto *GCD = dyn_cast<UnnamedGlobalConstantDecl>(Val: D))
2339 return CGM.GetAddrOfUnnamedGlobalConstantDecl(GCD);
2340
2341 if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(Val: D))
2342 return CGM.GetAddrOfTemplateParamObject(TPO);
2343
2344 return nullptr;
2345 }
2346
2347 // Handle typeid(T).
2348 if (TypeInfoLValue TI = base.dyn_cast<TypeInfoLValue>())
2349 return CGM.GetAddrOfRTTIDescriptor(Ty: QualType(TI.getType(), 0));
2350
2351 // Otherwise, it must be an expression.
2352 return Visit(S: base.get<const Expr*>());
2353}
2354
2355ConstantLValue
2356ConstantLValueEmitter::VisitConstantExpr(const ConstantExpr *E) {
2357 if (llvm::Constant *Result = Emitter.tryEmitConstantExpr(CE: E))
2358 return Result;
2359 return Visit(S: E->getSubExpr());
2360}
2361
2362ConstantLValue
2363ConstantLValueEmitter::VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) {
2364 ConstantEmitter CompoundLiteralEmitter(CGM, Emitter.CGF);
2365 CompoundLiteralEmitter.setInConstantContext(Emitter.isInConstantContext());
2366 return tryEmitGlobalCompoundLiteral(emitter&: CompoundLiteralEmitter, E);
2367}
2368
2369ConstantLValue
2370ConstantLValueEmitter::VisitStringLiteral(const StringLiteral *E) {
2371 return CGM.GetAddrOfConstantStringFromLiteral(S: E);
2372}
2373
2374ConstantLValue
2375ConstantLValueEmitter::VisitObjCEncodeExpr(const ObjCEncodeExpr *E) {
2376 return CGM.GetAddrOfConstantStringFromObjCEncode(E);
2377}
2378
2379static ConstantLValue emitConstantObjCStringLiteral(const StringLiteral *S,
2380 QualType T,
2381 CodeGenModule &CGM) {
2382 auto C = CGM.getObjCRuntime().GenerateConstantString(S);
2383 return C.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T));
2384}
2385
2386ConstantLValue
2387ConstantLValueEmitter::VisitObjCStringLiteral(const ObjCStringLiteral *E) {
2388 return emitConstantObjCStringLiteral(S: E->getString(), T: E->getType(), CGM);
2389}
2390
2391ConstantLValue
2392ConstantLValueEmitter::VisitObjCBoxedExpr(const ObjCBoxedExpr *E) {
2393 ASTContext &Context = CGM.getContext();
2394 CGObjCRuntime &Runtime = CGM.getObjCRuntime();
2395 const Expr *SubExpr = E->getSubExpr();
2396 const QualType &Ty = SubExpr->IgnoreParens()->getType();
2397
2398 assert(SubExpr->isEvaluatable(Context) &&
2399 "Non const NSNumber is being emitted as a constant");
2400
2401 if (const auto *SL = dyn_cast<StringLiteral>(Val: SubExpr->IgnoreParenCasts()))
2402 return emitConstantObjCStringLiteral(S: SL, T: E->getType(), CGM);
2403
2404 // Note `@YES` `@NO` need to be handled explicitly
2405 // to meet existing plist encoding / decoding expectations
2406 const bool IsBoolType =
2407 (Ty->isBooleanType() || NSAPI(Context).isObjCBOOLType(T: Ty));
2408 bool BoolValue = false;
2409 if (IsBoolType && SubExpr->EvaluateAsBooleanCondition(Result&: BoolValue, Ctx: Context)) {
2410 ConstantAddress C = Runtime.GenerateConstantNumber(Value: BoolValue, Ty);
2411 return C.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: E->getType()));
2412 }
2413
2414 Expr::EvalResult IntResult{};
2415 if (SubExpr->EvaluateAsInt(Result&: IntResult, Ctx: Context)) {
2416 ConstantAddress C =
2417 Runtime.GenerateConstantNumber(Value: IntResult.Val.getInt(), Ty);
2418 return C.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: E->getType()));
2419 }
2420
2421 llvm::APFloat FloatValue(0.0);
2422 if (SubExpr->EvaluateAsFloat(Result&: FloatValue, Ctx: Context)) {
2423 ConstantAddress C = Runtime.GenerateConstantNumber(Value: FloatValue, Ty);
2424 return C.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: E->getType()));
2425 }
2426
2427 llvm_unreachable("SubExpr is expected to be evaluated as a numeric type");
2428}
2429
2430llvm::Constant *
2431ConstantLValueEmitter::VisitObjCCollectionElement(const Expr *E) {
2432 auto CE = cast<CastExpr>(Val: E);
2433 const Expr *Elm = CE->getSubExpr();
2434 QualType DestTy = CE->getType();
2435
2436 assert(CE->getCastKind() == CK_BitCast &&
2437 "Expected a CK_BitCast type for valid items in constant objc "
2438 "collection literals");
2439
2440 llvm::Type *DstTy = CGM.getTypes().ConvertType(T: DestTy);
2441 ConstantLValue LV = Visit(S: Elm);
2442 llvm::Constant *ConstVal = cast<llvm::Constant>(Val: LV.Value);
2443 llvm::Constant *Val = llvm::ConstantExpr::getBitCast(C: ConstVal, Ty: DstTy);
2444 return Val;
2445}
2446
2447ConstantLValue
2448ConstantLValueEmitter::VisitObjCArrayLiteral(const ObjCArrayLiteral *E) {
2449 SmallVector<llvm::Constant *, 16> ObjectExpressions;
2450 uint64_t NumElements = E->getNumElements();
2451 ObjectExpressions.reserve(N: NumElements);
2452
2453 for (uint64_t i = 0; i < NumElements; i++) {
2454 llvm::Constant *Val = VisitObjCCollectionElement(E: E->getElement(Index: i));
2455 ObjectExpressions.push_back(Elt: Val);
2456 }
2457 ConstantAddress C =
2458 CGM.getObjCRuntime().GenerateConstantArray(Objects: ObjectExpressions);
2459 return C.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: E->getType()));
2460}
2461
2462ConstantLValue ConstantLValueEmitter::VisitObjCDictionaryLiteral(
2463 const ObjCDictionaryLiteral *E) {
2464 SmallVector<std::pair<llvm::Constant *, llvm::Constant *>, 16> KeysAndObjects;
2465 uint64_t NumElements = E->getNumElements();
2466 KeysAndObjects.reserve(N: NumElements);
2467
2468 for (uint64_t i = 0; i < NumElements; i++) {
2469 llvm::Constant *Key =
2470 VisitObjCCollectionElement(E: E->getKeyValueElement(Index: i).Key);
2471 llvm::Constant *Val =
2472 VisitObjCCollectionElement(E: E->getKeyValueElement(Index: i).Value);
2473 KeysAndObjects.push_back(Elt: {Key, Val});
2474 }
2475 ConstantAddress C =
2476 CGM.getObjCRuntime().GenerateConstantDictionary(E, KeysAndObjects);
2477 return C.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: E->getType()));
2478}
2479
2480ConstantLValue
2481ConstantLValueEmitter::VisitPredefinedExpr(const PredefinedExpr *E) {
2482 return CGM.GetAddrOfConstantStringFromLiteral(S: E->getFunctionName());
2483}
2484
2485ConstantLValue
2486ConstantLValueEmitter::VisitAddrLabelExpr(const AddrLabelExpr *E) {
2487 assert(Emitter.CGF && "Invalid address of label expression outside function");
2488 llvm::Constant *Ptr = Emitter.CGF->GetAddrOfLabel(L: E->getLabel());
2489 return Ptr;
2490}
2491
2492ConstantLValue
2493ConstantLValueEmitter::VisitCallExpr(const CallExpr *E) {
2494 unsigned builtin = E->getBuiltinCallee();
2495 if (builtin == Builtin::BI__builtin_function_start)
2496 return CGM.GetFunctionStart(
2497 Decl: E->getArg(Arg: 0)->getAsBuiltinConstantDeclRef(Context: CGM.getContext()));
2498
2499 if (builtin == Builtin::BI__builtin_ptrauth_sign_constant)
2500 return emitPointerAuthSignConstant(E);
2501
2502 if (builtin != Builtin::BI__builtin___CFStringMakeConstantString &&
2503 builtin != Builtin::BI__builtin___NSStringMakeConstantString)
2504 return nullptr;
2505
2506 const auto *Literal = cast<StringLiteral>(Val: E->getArg(Arg: 0)->IgnoreParenCasts());
2507 if (builtin == Builtin::BI__builtin___NSStringMakeConstantString) {
2508 return CGM.getObjCRuntime().GenerateConstantString(Literal);
2509 } else {
2510 // FIXME: need to deal with UCN conversion issues.
2511 return CGM.GetAddrOfConstantCFString(Literal);
2512 }
2513}
2514
2515ConstantLValue
2516ConstantLValueEmitter::emitPointerAuthSignConstant(const CallExpr *E) {
2517 llvm::Constant *UnsignedPointer = emitPointerAuthPointer(E: E->getArg(Arg: 0));
2518 unsigned Key = emitPointerAuthKey(E: E->getArg(Arg: 1));
2519 auto [StorageAddress, OtherDiscriminator] =
2520 emitPointerAuthDiscriminator(E: E->getArg(Arg: 2));
2521
2522 llvm::Constant *SignedPointer = CGM.getConstantSignedPointer(
2523 Pointer: UnsignedPointer, Key, StorageAddress, OtherDiscriminator);
2524 return SignedPointer;
2525}
2526
2527llvm::Constant *ConstantLValueEmitter::emitPointerAuthPointer(const Expr *E) {
2528 Expr::EvalResult Result;
2529 bool Succeeded = E->EvaluateAsRValue(Result, Ctx: CGM.getContext());
2530 assert(Succeeded);
2531 (void)Succeeded;
2532
2533 // The assertions here are all checked by Sema.
2534 assert(Result.Val.isLValue());
2535 if (isa<FunctionDecl>(Val: Result.Val.getLValueBase().get<const ValueDecl *>()))
2536 assert(Result.Val.getLValueOffset().isZero());
2537 return ConstantEmitter(CGM, Emitter.CGF)
2538 .emitAbstract(loc: E->getExprLoc(), value: Result.Val, destType: E->getType(), EnablePtrAuthFunctionTypeDiscrimination: false);
2539}
2540
2541unsigned ConstantLValueEmitter::emitPointerAuthKey(const Expr *E) {
2542 return E->EvaluateKnownConstInt(Ctx: CGM.getContext()).getZExtValue();
2543}
2544
2545std::pair<llvm::Constant *, llvm::ConstantInt *>
2546ConstantLValueEmitter::emitPointerAuthDiscriminator(const Expr *E) {
2547 E = E->IgnoreParens();
2548
2549 if (const auto *Call = dyn_cast<CallExpr>(Val: E)) {
2550 if (Call->getBuiltinCallee() ==
2551 Builtin::BI__builtin_ptrauth_blend_discriminator) {
2552 llvm::Constant *Pointer = ConstantEmitter(CGM).emitAbstract(
2553 E: Call->getArg(Arg: 0), destType: Call->getArg(Arg: 0)->getType());
2554 auto *Extra = cast<llvm::ConstantInt>(Val: ConstantEmitter(CGM).emitAbstract(
2555 E: Call->getArg(Arg: 1), destType: Call->getArg(Arg: 1)->getType()));
2556 return {Pointer, Extra};
2557 }
2558 }
2559
2560 llvm::Constant *Result = ConstantEmitter(CGM).emitAbstract(E, destType: E->getType());
2561 if (Result->getType()->isPointerTy())
2562 return {Result, nullptr};
2563 return {nullptr, cast<llvm::ConstantInt>(Val: Result)};
2564}
2565
2566ConstantLValue
2567ConstantLValueEmitter::VisitBlockExpr(const BlockExpr *E) {
2568 StringRef functionName;
2569 if (auto CGF = Emitter.CGF)
2570 functionName = CGF->CurFn->getName();
2571 else
2572 functionName = "global";
2573
2574 return CGM.GetAddrOfGlobalBlock(BE: E, Name: functionName);
2575}
2576
2577ConstantLValue
2578ConstantLValueEmitter::VisitCXXTypeidExpr(const CXXTypeidExpr *E) {
2579 QualType T;
2580 if (E->isTypeOperand())
2581 T = E->getTypeOperand(Context: CGM.getContext());
2582 else
2583 T = E->getExprOperand()->getType();
2584 return CGM.GetAddrOfRTTIDescriptor(Ty: T);
2585}
2586
2587ConstantLValue
2588ConstantLValueEmitter::VisitMaterializeTemporaryExpr(
2589 const MaterializeTemporaryExpr *E) {
2590 assert(E->getStorageDuration() == SD_Static);
2591 const Expr *Inner = E->getSubExpr()->skipRValueSubobjectAdjustments();
2592 return CGM.GetAddrOfGlobalTemporary(E, Inner);
2593}
2594
2595llvm::Constant *
2596ConstantEmitter::tryEmitPrivate(const APValue &Value, QualType DestType,
2597 bool EnablePtrAuthFunctionTypeDiscrimination) {
2598 switch (Value.getKind()) {
2599 case APValue::None:
2600 case APValue::Indeterminate:
2601 // Out-of-lifetime and indeterminate values can be modeled as 'undef'.
2602 return llvm::UndefValue::get(T: CGM.getTypes().ConvertType(T: DestType));
2603 case APValue::LValue:
2604 return ConstantLValueEmitter(*this, Value, DestType,
2605 EnablePtrAuthFunctionTypeDiscrimination)
2606 .tryEmit();
2607 case APValue::Int:
2608 if (PointerAuthQualifier PointerAuth = DestType.getPointerAuth();
2609 PointerAuth &&
2610 (PointerAuth.authenticatesNullValues() || Value.getInt() != 0))
2611 return nullptr;
2612 return llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: Value.getInt());
2613 case APValue::FixedPoint:
2614 return llvm::ConstantInt::get(Context&: CGM.getLLVMContext(),
2615 V: Value.getFixedPoint().getValue());
2616 case APValue::ComplexInt: {
2617 llvm::Constant *Complex[2];
2618
2619 Complex[0] = llvm::ConstantInt::get(Context&: CGM.getLLVMContext(),
2620 V: Value.getComplexIntReal());
2621 Complex[1] = llvm::ConstantInt::get(Context&: CGM.getLLVMContext(),
2622 V: Value.getComplexIntImag());
2623
2624 // FIXME: the target may want to specify that this is packed.
2625 llvm::StructType *STy =
2626 llvm::StructType::get(elt1: Complex[0]->getType(), elts: Complex[1]->getType());
2627 return llvm::ConstantStruct::get(T: STy, V: Complex);
2628 }
2629 case APValue::Float:
2630 return llvm::ConstantFP::get(Context&: CGM.getLLVMContext(), V: Value.getFloat());
2631 case APValue::ComplexFloat: {
2632 llvm::Constant *Complex[2];
2633
2634 Complex[0] = llvm::ConstantFP::get(Context&: CGM.getLLVMContext(),
2635 V: Value.getComplexFloatReal());
2636 Complex[1] = llvm::ConstantFP::get(Context&: CGM.getLLVMContext(),
2637 V: Value.getComplexFloatImag());
2638
2639 // FIXME: the target may want to specify that this is packed.
2640 llvm::StructType *STy =
2641 llvm::StructType::get(elt1: Complex[0]->getType(), elts: Complex[1]->getType());
2642 return llvm::ConstantStruct::get(T: STy, V: Complex);
2643 }
2644 case APValue::Vector: {
2645 unsigned NumElts = Value.getVectorLength();
2646 SmallVector<llvm::Constant *, 4> Inits(NumElts);
2647
2648 for (unsigned I = 0; I != NumElts; ++I) {
2649 const APValue &Elt = Value.getVectorElt(I);
2650 if (Elt.isInt())
2651 Inits[I] = llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: Elt.getInt());
2652 else if (Elt.isFloat())
2653 Inits[I] = llvm::ConstantFP::get(Context&: CGM.getLLVMContext(), V: Elt.getFloat());
2654 else if (Elt.isIndeterminate())
2655 Inits[I] = llvm::UndefValue::get(T: CGM.getTypes().ConvertType(
2656 T: DestType->castAs<VectorType>()->getElementType()));
2657 else
2658 llvm_unreachable("unsupported vector element type");
2659 }
2660 return llvm::ConstantVector::get(V: Inits);
2661 }
2662 case APValue::Matrix: {
2663 const auto *MT = DestType->castAs<ConstantMatrixType>();
2664 unsigned NumRows = Value.getMatrixNumRows();
2665 unsigned NumCols = Value.getMatrixNumColumns();
2666 unsigned NumElts = NumRows * NumCols;
2667 SmallVector<llvm::Constant *, 16> Inits(NumElts);
2668
2669 bool IsRowMajor = isMatrixRowMajor(LangOpts: CGM.getLangOpts(), T: DestType);
2670
2671 for (unsigned Row = 0; Row != NumRows; ++Row) {
2672 for (unsigned Col = 0; Col != NumCols; ++Col) {
2673 const APValue &Elt = Value.getMatrixElt(Row, Col);
2674 unsigned Idx = MT->getFlattenedIndex(Row, Column: Col, IsRowMajor);
2675 if (Elt.isInt())
2676 Inits[Idx] =
2677 llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: Elt.getInt());
2678 else if (Elt.isFloat())
2679 Inits[Idx] =
2680 llvm::ConstantFP::get(Context&: CGM.getLLVMContext(), V: Elt.getFloat());
2681 else if (Elt.isIndeterminate())
2682 Inits[Idx] = llvm::PoisonValue::get(
2683 T: CGM.getTypes().ConvertType(T: MT->getElementType()));
2684 else
2685 llvm_unreachable("unsupported matrix element type");
2686 }
2687 }
2688 return llvm::ConstantVector::get(V: Inits);
2689 }
2690 case APValue::AddrLabelDiff: {
2691 const AddrLabelExpr *LHSExpr = Value.getAddrLabelDiffLHS();
2692 const AddrLabelExpr *RHSExpr = Value.getAddrLabelDiffRHS();
2693 llvm::Constant *LHS = tryEmitPrivate(E: LHSExpr, destType: LHSExpr->getType());
2694 llvm::Constant *RHS = tryEmitPrivate(E: RHSExpr, destType: RHSExpr->getType());
2695 if (!LHS || !RHS) return nullptr;
2696
2697 // Compute difference
2698 llvm::Type *ResultType = CGM.getTypes().ConvertType(T: DestType);
2699 LHS = llvm::ConstantExpr::getPtrToInt(C: LHS, Ty: CGM.IntPtrTy);
2700 RHS = llvm::ConstantExpr::getPtrToInt(C: RHS, Ty: CGM.IntPtrTy);
2701 llvm::Constant *AddrLabelDiff = llvm::ConstantExpr::getSub(C1: LHS, C2: RHS);
2702
2703 // LLVM is a bit sensitive about the exact format of the
2704 // address-of-label difference; make sure to truncate after
2705 // the subtraction.
2706 return llvm::ConstantExpr::getTruncOrBitCast(C: AddrLabelDiff, Ty: ResultType);
2707 }
2708 case APValue::Struct:
2709 case APValue::Union:
2710 return ConstStructBuilder::BuildStruct(Emitter&: *this, Val: Value, ValTy: DestType);
2711 case APValue::Array: {
2712 const ArrayType *ArrayTy = CGM.getContext().getAsArrayType(T: DestType);
2713 unsigned NumElements = Value.getArraySize();
2714 unsigned NumInitElts = Value.getArrayInitializedElts();
2715
2716 // Emit array filler, if there is one.
2717 llvm::Constant *Filler = nullptr;
2718 if (Value.hasArrayFiller()) {
2719 Filler = tryEmitAbstractForMemory(value: Value.getArrayFiller(),
2720 destType: ArrayTy->getElementType());
2721 if (!Filler)
2722 return nullptr;
2723 }
2724
2725 // Emit initializer elements.
2726 SmallVector<llvm::Constant*, 16> Elts;
2727 if (Filler && Filler->isNullValue())
2728 Elts.reserve(N: NumInitElts + 1);
2729 else
2730 Elts.reserve(N: NumElements);
2731
2732 llvm::Type *CommonElementType = nullptr;
2733 for (unsigned I = 0; I < NumInitElts; ++I) {
2734 llvm::Constant *C = tryEmitPrivateForMemory(
2735 value: Value.getArrayInitializedElt(I), destType: ArrayTy->getElementType());
2736 if (!C) return nullptr;
2737
2738 if (I == 0)
2739 CommonElementType = C->getType();
2740 else if (C->getType() != CommonElementType)
2741 CommonElementType = nullptr;
2742 Elts.push_back(Elt: C);
2743 }
2744
2745 llvm::ArrayType *Desired =
2746 cast<llvm::ArrayType>(Val: CGM.getTypes().ConvertType(T: DestType));
2747
2748 // Fix the type of incomplete arrays if the initializer isn't empty.
2749 if (DestType->isIncompleteArrayType() && !Elts.empty())
2750 Desired = llvm::ArrayType::get(ElementType: Desired->getElementType(), NumElements: Elts.size());
2751
2752 return EmitArrayConstant(CGM, DesiredType: Desired, CommonElementType, ArrayBound: NumElements, Elements&: Elts,
2753 Filler);
2754 }
2755 case APValue::MemberPointer:
2756 return CGM.getCXXABI().EmitMemberPointer(MP: Value, MPT: DestType);
2757 }
2758 llvm_unreachable("Unknown APValue kind");
2759}
2760
2761llvm::GlobalVariable *CodeGenModule::getAddrOfConstantCompoundLiteralIfEmitted(
2762 const CompoundLiteralExpr *E) {
2763 return EmittedCompoundLiterals.lookup(Val: E);
2764}
2765
2766void CodeGenModule::setAddrOfConstantCompoundLiteral(
2767 const CompoundLiteralExpr *CLE, llvm::GlobalVariable *GV) {
2768 bool Ok = EmittedCompoundLiterals.insert(KV: std::make_pair(x&: CLE, y&: GV)).second;
2769 (void)Ok;
2770 assert(Ok && "CLE has already been emitted!");
2771}
2772
2773ConstantAddress
2774CodeGenModule::GetAddrOfConstantCompoundLiteral(const CompoundLiteralExpr *E) {
2775 assert(E->isFileScope() && "not a file-scope compound literal expr");
2776 ConstantEmitter emitter(*this);
2777 return tryEmitGlobalCompoundLiteral(emitter, E);
2778}
2779
2780llvm::Constant *
2781CodeGenModule::getMemberPointerConstant(const UnaryOperator *uo) {
2782 // Member pointer constants always have a very particular form.
2783 const MemberPointerType *type = cast<MemberPointerType>(Val: uo->getType());
2784 const ValueDecl *decl = cast<DeclRefExpr>(Val: uo->getSubExpr())->getDecl();
2785
2786 // A member function pointer.
2787 if (const CXXMethodDecl *method = dyn_cast<CXXMethodDecl>(Val: decl))
2788 return getCXXABI().EmitMemberFunctionPointer(MD: method);
2789
2790 // Otherwise, a member data pointer.
2791 getContext().recordMemberDataPointerEvaluation(VD: decl);
2792 uint64_t fieldOffset = getContext().getFieldOffset(FD: decl);
2793 CharUnits chars = getContext().toCharUnitsFromBits(BitSize: (int64_t) fieldOffset);
2794 return getCXXABI().EmitMemberDataPointer(MPT: type, offset: chars);
2795}
2796
2797static llvm::Constant *EmitNullConstantForBase(CodeGenModule &CGM,
2798 llvm::Type *baseType,
2799 const CXXRecordDecl *base);
2800
2801static llvm::Constant *EmitNullConstant(CodeGenModule &CGM,
2802 const RecordDecl *record,
2803 bool asCompleteObject) {
2804 const CGRecordLayout &layout = CGM.getTypes().getCGRecordLayout(record);
2805 llvm::StructType *structure =
2806 (asCompleteObject ? layout.getLLVMType()
2807 : layout.getBaseSubobjectLLVMType());
2808
2809 unsigned numElements = structure->getNumElements();
2810 std::vector<llvm::Constant *> elements(numElements);
2811
2812 auto CXXR = dyn_cast<CXXRecordDecl>(Val: record);
2813 // Fill in all the bases.
2814 if (CXXR) {
2815 for (const auto &I : CXXR->bases()) {
2816 if (I.isVirtual()) {
2817 // Ignore virtual bases; if we're laying out for a complete
2818 // object, we'll lay these out later.
2819 continue;
2820 }
2821
2822 const auto *base = I.getType()->castAsCXXRecordDecl();
2823 // Ignore empty bases.
2824 if (isEmptyRecordForLayout(Context: CGM.getContext(), T: I.getType()) ||
2825 CGM.getContext()
2826 .getASTRecordLayout(D: base)
2827 .getNonVirtualSize()
2828 .isZero())
2829 continue;
2830
2831 unsigned fieldIndex = layout.getNonVirtualBaseLLVMFieldNo(RD: base);
2832 llvm::Type *baseType = structure->getElementType(N: fieldIndex);
2833 elements[fieldIndex] = EmitNullConstantForBase(CGM, baseType, base);
2834 }
2835 }
2836
2837 // Fill in all the fields.
2838 for (const auto *Field : record->fields()) {
2839 // Fill in non-bitfields. (Bitfields always use a zero pattern, which we
2840 // will fill in later.)
2841 if (!Field->isBitField() &&
2842 !isEmptyFieldForLayout(Context: CGM.getContext(), FD: Field)) {
2843 unsigned fieldIndex = layout.getLLVMFieldNo(FD: Field);
2844 elements[fieldIndex] = CGM.EmitNullConstant(T: Field->getType());
2845 }
2846
2847 // For unions, stop after the first named field.
2848 if (record->isUnion()) {
2849 if (Field->getIdentifier())
2850 break;
2851 if (const auto *FieldRD = Field->getType()->getAsRecordDecl())
2852 if (FieldRD->findFirstNamedDataMember())
2853 break;
2854 }
2855 }
2856
2857 // Fill in the virtual bases, if we're working with the complete object.
2858 if (CXXR && asCompleteObject) {
2859 for (const auto &I : CXXR->vbases()) {
2860 const auto *base = I.getType()->castAsCXXRecordDecl();
2861 // Ignore empty bases.
2862 if (isEmptyRecordForLayout(Context: CGM.getContext(), T: I.getType()))
2863 continue;
2864
2865 unsigned fieldIndex = layout.getVirtualBaseIndex(base);
2866
2867 // We might have already laid this field out.
2868 if (elements[fieldIndex]) continue;
2869
2870 llvm::Type *baseType = structure->getElementType(N: fieldIndex);
2871 elements[fieldIndex] = EmitNullConstantForBase(CGM, baseType, base);
2872 }
2873 }
2874
2875 // Now go through all other fields and zero them out.
2876 for (unsigned i = 0; i != numElements; ++i) {
2877 if (!elements[i])
2878 elements[i] = llvm::Constant::getNullValue(Ty: structure->getElementType(N: i));
2879 }
2880
2881 return llvm::ConstantStruct::get(T: structure, V: elements);
2882}
2883
2884/// Emit the null constant for a base subobject.
2885static llvm::Constant *EmitNullConstantForBase(CodeGenModule &CGM,
2886 llvm::Type *baseType,
2887 const CXXRecordDecl *base) {
2888 const CGRecordLayout &baseLayout = CGM.getTypes().getCGRecordLayout(base);
2889
2890 // Just zero out bases that don't have any pointer to data members.
2891 if (baseLayout.isZeroInitializableAsBase())
2892 return llvm::Constant::getNullValue(Ty: baseType);
2893
2894 // Otherwise, we can just use its null constant.
2895 return EmitNullConstant(CGM, record: base, /*asCompleteObject=*/false);
2896}
2897
2898llvm::Constant *ConstantEmitter::emitNullForMemory(CodeGenModule &CGM,
2899 QualType T) {
2900 return emitForMemory(CGM, C: CGM.EmitNullConstant(T), destType: T);
2901}
2902
2903llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) {
2904 if (T->getAs<PointerType>()) {
2905 llvm::Type *LT = getTypes().ConvertTypeForMem(T);
2906 if (auto *PT = dyn_cast<llvm::PointerType>(Val: LT))
2907 return getNullPointer(T: PT, QT: T);
2908 // Some pointer types do not lower to an LLVM pointer (e.g. a WebAssembly
2909 // funcref, which is an opaque reference type). Use the type's zero value.
2910 return llvm::Constant::getNullValue(Ty: LT);
2911 }
2912
2913 if (getTypes().isZeroInitializable(T))
2914 return llvm::Constant::getNullValue(Ty: getTypes().ConvertTypeForMem(T));
2915
2916 if (const ConstantArrayType *CAT = Context.getAsConstantArrayType(T)) {
2917 llvm::ArrayType *ATy =
2918 cast<llvm::ArrayType>(Val: getTypes().ConvertTypeForMem(T));
2919
2920 QualType ElementTy = CAT->getElementType();
2921
2922 llvm::Constant *Element =
2923 ConstantEmitter::emitNullForMemory(CGM&: *this, T: ElementTy);
2924 unsigned NumElements = CAT->getZExtSize();
2925 SmallVector<llvm::Constant *, 8> Array(NumElements, Element);
2926 return llvm::ConstantArray::get(T: ATy, V: Array);
2927 }
2928
2929 if (const auto *RD = T->getAsRecordDecl())
2930 return ::EmitNullConstant(CGM&: *this, record: RD,
2931 /*asCompleteObject=*/true);
2932
2933 assert(T->isMemberDataPointerType() &&
2934 "Should only see pointers to data members here!");
2935
2936 return getCXXABI().EmitNullMemberPointer(MPT: T->castAs<MemberPointerType>());
2937}
2938
2939llvm::Constant *
2940CodeGenModule::EmitNullConstantForBase(const CXXRecordDecl *Record) {
2941 return ::EmitNullConstant(CGM&: *this, record: Record, asCompleteObject: false);
2942}
2943