1//===--- CGExprConstant.cpp - Emit LLVM Code from Constant Expressions ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Constant Expr nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ABIInfoImpl.h"
14#include "CGCXXABI.h"
15#include "CGObjCRuntime.h"
16#include "CGRecordLayout.h"
17#include "CodeGenFunction.h"
18#include "CodeGenModule.h"
19#include "ConstantEmitter.h"
20#include "TargetInfo.h"
21#include "clang/AST/APValue.h"
22#include "clang/AST/ASTContext.h"
23#include "clang/AST/Attr.h"
24#include "clang/AST/MatrixUtils.h"
25#include "clang/AST/NSAPI.h"
26#include "clang/AST/RecordLayout.h"
27#include "clang/AST/StmtVisitor.h"
28#include "clang/Basic/Builtins.h"
29#include "llvm/ADT/STLExtras.h"
30#include "llvm/ADT/Sequence.h"
31#include "llvm/Analysis/ConstantFolding.h"
32#include "llvm/IR/Constants.h"
33#include "llvm/IR/DataLayout.h"
34#include "llvm/IR/Function.h"
35#include "llvm/IR/GlobalVariable.h"
36#include "llvm/Support/SipHash.h"
37#include <optional>
38using namespace clang;
39using namespace CodeGen;
40
41//===----------------------------------------------------------------------===//
42// ConstantAggregateBuilder
43//===----------------------------------------------------------------------===//
44
45namespace {
46class ConstExprEmitter;
47
48llvm::Constant *getPadding(const CodeGenModule &CGM, CharUnits PadSize) {
49 llvm::Type *Ty = CGM.CharTy;
50 if (PadSize > CharUnits::One())
51 Ty = llvm::ArrayType::get(ElementType: Ty, NumElements: PadSize.getQuantity());
52 if (CGM.shouldZeroInitPadding()) {
53 return llvm::Constant::getNullValue(Ty);
54 }
55 return llvm::UndefValue::get(T: Ty);
56}
57
58struct ConstantAggregateBuilderUtils {
59 CodeGenModule &CGM;
60
61 ConstantAggregateBuilderUtils(CodeGenModule &CGM) : CGM(CGM) {}
62
63 CharUnits getAlignment(const llvm::Constant *C) const {
64 return CharUnits::fromQuantity(
65 Quantity: CGM.getDataLayout().getABITypeAlign(Ty: C->getType()));
66 }
67
68 CharUnits getSize(llvm::Type *Ty) const {
69 return CharUnits::fromQuantity(Quantity: CGM.getDataLayout().getTypeAllocSize(Ty));
70 }
71
72 CharUnits getSize(const llvm::Constant *C) const {
73 return getSize(Ty: C->getType());
74 }
75
76 llvm::Constant *getPadding(CharUnits PadSize) const {
77 return ::getPadding(CGM, PadSize);
78 }
79
80 llvm::Constant *getZeroes(CharUnits ZeroSize) const {
81 llvm::Type *Ty = llvm::ArrayType::get(ElementType: CGM.CharTy, NumElements: ZeroSize.getQuantity());
82 return llvm::ConstantAggregateZero::get(Ty);
83 }
84};
85
86/// Incremental builder for an llvm::Constant* holding a struct or array
87/// constant.
88class ConstantAggregateBuilder : private ConstantAggregateBuilderUtils {
89 /// The elements of the constant. These two arrays must have the same size;
90 /// Offsets[i] describes the offset of Elems[i] within the constant. The
91 /// elements are kept in increasing offset order, and we ensure that there
92 /// is no overlap: Offsets[i+1] >= Offsets[i] + getSize(Elemes[i]).
93 ///
94 /// This may contain explicit padding elements (in order to create a
95 /// natural layout), but need not. Gaps between elements are implicitly
96 /// considered to be filled with undef.
97 llvm::SmallVector<llvm::Constant*, 32> Elems;
98 llvm::SmallVector<CharUnits, 32> Offsets;
99
100 /// The size of the constant (the maximum end offset of any added element).
101 /// May be larger than the end of Elems.back() if we split the last element
102 /// and removed some trailing undefs.
103 CharUnits Size = CharUnits::Zero();
104
105 /// This is true only if laying out Elems in order as the elements of a
106 /// non-packed LLVM struct will give the correct layout.
107 bool NaturalLayout = true;
108
109 bool split(size_t Index, CharUnits Hint);
110 std::optional<size_t> splitAt(CharUnits Pos);
111
112 static llvm::Constant *buildFrom(CodeGenModule &CGM,
113 ArrayRef<llvm::Constant *> Elems,
114 ArrayRef<CharUnits> Offsets,
115 CharUnits StartOffset, CharUnits Size,
116 bool NaturalLayout, llvm::Type *DesiredTy,
117 bool AllowOversized);
118
119public:
120 ConstantAggregateBuilder(CodeGenModule &CGM)
121 : ConstantAggregateBuilderUtils(CGM) {}
122
123 /// Update or overwrite the value starting at \p Offset with \c C.
124 ///
125 /// \param AllowOverwrite If \c true, this constant might overwrite (part of)
126 /// a constant that has already been added. This flag is only used to
127 /// detect bugs.
128 bool add(llvm::Constant *C, CharUnits Offset, bool AllowOverwrite);
129
130 /// Update or overwrite the bits starting at \p OffsetInBits with \p Bits.
131 bool addBits(llvm::APInt Bits, uint64_t OffsetInBits, bool AllowOverwrite);
132
133 /// Attempt to condense the value starting at \p Offset to a constant of type
134 /// \p DesiredTy.
135 void condense(CharUnits Offset, llvm::Type *DesiredTy);
136
137 /// Produce a constant representing the entire accumulated value, ideally of
138 /// the specified type. If \p AllowOversized, the constant might be larger
139 /// than implied by \p DesiredTy (eg, if there is a flexible array member).
140 /// Otherwise, the constant will be of exactly the same size as \p DesiredTy
141 /// even if we can't represent it as that type.
142 llvm::Constant *build(llvm::Type *DesiredTy, bool AllowOversized) const {
143 return buildFrom(CGM, Elems, Offsets, StartOffset: CharUnits::Zero(), Size,
144 NaturalLayout, DesiredTy, AllowOversized);
145 }
146};
147
148template<typename Container, typename Range = std::initializer_list<
149 typename Container::value_type>>
150static void replace(Container &C, size_t BeginOff, size_t EndOff, Range Vals) {
151 assert(BeginOff <= EndOff && "invalid replacement range");
152 llvm::replace(C, C.begin() + BeginOff, C.begin() + EndOff, Vals);
153}
154
155bool ConstantAggregateBuilder::add(llvm::Constant *C, CharUnits Offset,
156 bool AllowOverwrite) {
157 // Common case: appending to a layout.
158 if (Offset >= Size) {
159 CharUnits Align = getAlignment(C);
160 CharUnits AlignedSize = Size.alignTo(Align);
161 if (AlignedSize > Offset || Offset.alignTo(Align) != Offset)
162 NaturalLayout = false;
163 else if (AlignedSize < Offset) {
164 Elems.push_back(Elt: getPadding(PadSize: Offset - Size));
165 Offsets.push_back(Elt: Size);
166 }
167 Elems.push_back(Elt: C);
168 Offsets.push_back(Elt: Offset);
169 Size = Offset + getSize(C);
170 return true;
171 }
172
173 // Uncommon case: constant overlaps what we've already created.
174 std::optional<size_t> FirstElemToReplace = splitAt(Pos: Offset);
175 if (!FirstElemToReplace)
176 return false;
177
178 CharUnits CSize = getSize(C);
179 std::optional<size_t> LastElemToReplace = splitAt(Pos: Offset + CSize);
180 if (!LastElemToReplace)
181 return false;
182
183 assert((FirstElemToReplace == LastElemToReplace || AllowOverwrite) &&
184 "unexpectedly overwriting field");
185
186 replace(C&: Elems, BeginOff: *FirstElemToReplace, EndOff: *LastElemToReplace, Vals: {C});
187 replace(C&: Offsets, BeginOff: *FirstElemToReplace, EndOff: *LastElemToReplace, Vals: {Offset});
188 Size = std::max(a: Size, b: Offset + CSize);
189 NaturalLayout = false;
190 return true;
191}
192
193bool ConstantAggregateBuilder::addBits(llvm::APInt Bits, uint64_t OffsetInBits,
194 bool AllowOverwrite) {
195 const ASTContext &Context = CGM.getContext();
196 const uint64_t CharWidth = CGM.getContext().getCharWidth();
197
198 // Offset of where we want the first bit to go within the bits of the
199 // current char.
200 unsigned OffsetWithinChar = OffsetInBits % CharWidth;
201
202 // We split bit-fields up into individual bytes. Walk over the bytes and
203 // update them.
204 for (CharUnits OffsetInChars =
205 Context.toCharUnitsFromBits(BitSize: OffsetInBits - OffsetWithinChar);
206 /**/; ++OffsetInChars) {
207 // Number of bits we want to fill in this char.
208 unsigned WantedBits =
209 std::min(a: (uint64_t)Bits.getBitWidth(), b: CharWidth - OffsetWithinChar);
210
211 // Get a char containing the bits we want in the right places. The other
212 // bits have unspecified values.
213 llvm::APInt BitsThisChar = Bits;
214 if (BitsThisChar.getBitWidth() < CharWidth)
215 BitsThisChar = BitsThisChar.zext(width: CharWidth);
216 if (CGM.getDataLayout().isBigEndian()) {
217 // Figure out how much to shift by. We may need to left-shift if we have
218 // less than one byte of Bits left.
219 int Shift = Bits.getBitWidth() - CharWidth + OffsetWithinChar;
220 if (Shift > 0)
221 BitsThisChar.lshrInPlace(ShiftAmt: Shift);
222 else if (Shift < 0)
223 BitsThisChar = BitsThisChar.shl(shiftAmt: -Shift);
224 } else {
225 BitsThisChar = BitsThisChar.shl(shiftAmt: OffsetWithinChar);
226 }
227 if (BitsThisChar.getBitWidth() > CharWidth)
228 BitsThisChar = BitsThisChar.trunc(width: CharWidth);
229
230 if (WantedBits == CharWidth) {
231 // Got a full byte: just add it directly.
232 add(C: llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: BitsThisChar),
233 Offset: OffsetInChars, AllowOverwrite);
234 } else {
235 // Partial byte: update the existing integer if there is one. If we
236 // can't split out a 1-CharUnit range to update, then we can't add
237 // these bits and fail the entire constant emission.
238 std::optional<size_t> FirstElemToUpdate = splitAt(Pos: OffsetInChars);
239 if (!FirstElemToUpdate)
240 return false;
241 std::optional<size_t> LastElemToUpdate =
242 splitAt(Pos: OffsetInChars + CharUnits::One());
243 if (!LastElemToUpdate)
244 return false;
245 assert(*LastElemToUpdate - *FirstElemToUpdate < 2 &&
246 "should have at most one element covering one byte");
247
248 // Figure out which bits we want and discard the rest.
249 llvm::APInt UpdateMask(CharWidth, 0);
250 if (CGM.getDataLayout().isBigEndian())
251 UpdateMask.setBits(loBit: CharWidth - OffsetWithinChar - WantedBits,
252 hiBit: CharWidth - OffsetWithinChar);
253 else
254 UpdateMask.setBits(loBit: OffsetWithinChar, hiBit: OffsetWithinChar + WantedBits);
255 BitsThisChar &= UpdateMask;
256
257 if (*FirstElemToUpdate == *LastElemToUpdate ||
258 Elems[*FirstElemToUpdate]->isNullValue() ||
259 isa<llvm::UndefValue>(Val: Elems[*FirstElemToUpdate])) {
260 // All existing bits are either zero or undef.
261 add(C: llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: BitsThisChar),
262 Offset: OffsetInChars, /*AllowOverwrite*/ true);
263 } else {
264 llvm::Constant *&ToUpdate = Elems[*FirstElemToUpdate];
265 // In order to perform a partial update, we need the existing bitwise
266 // value, which we can only extract for a constant int.
267 auto *CI = dyn_cast<llvm::ConstantInt>(Val: ToUpdate);
268 if (!CI)
269 return false;
270 // Because this is a 1-CharUnit range, the constant occupying it must
271 // be exactly one CharUnit wide.
272 assert(CI->getBitWidth() == CharWidth && "splitAt failed");
273 assert((!(CI->getValue() & UpdateMask) || AllowOverwrite) &&
274 "unexpectedly overwriting bitfield");
275 BitsThisChar |= (CI->getValue() & ~UpdateMask);
276 ToUpdate = llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: BitsThisChar);
277 }
278 }
279
280 // Stop if we've added all the bits.
281 if (WantedBits == Bits.getBitWidth())
282 break;
283
284 // Remove the consumed bits from Bits.
285 if (!CGM.getDataLayout().isBigEndian())
286 Bits.lshrInPlace(ShiftAmt: WantedBits);
287 Bits = Bits.trunc(width: Bits.getBitWidth() - WantedBits);
288
289 // The remanining bits go at the start of the following bytes.
290 OffsetWithinChar = 0;
291 }
292
293 return true;
294}
295
296/// Returns a position within Elems and Offsets such that all elements
297/// before the returned index end before Pos and all elements at or after
298/// the returned index begin at or after Pos. Splits elements as necessary
299/// to ensure this. Returns std::nullopt if we find something we can't split.
300std::optional<size_t> ConstantAggregateBuilder::splitAt(CharUnits Pos) {
301 if (Pos >= Size)
302 return Offsets.size();
303
304 while (true) {
305 auto FirstAfterPos = llvm::upper_bound(Range&: Offsets, Value&: Pos);
306 if (FirstAfterPos == Offsets.begin())
307 return 0;
308
309 // If we already have an element starting at Pos, we're done.
310 size_t LastAtOrBeforePosIndex = FirstAfterPos - Offsets.begin() - 1;
311 if (Offsets[LastAtOrBeforePosIndex] == Pos)
312 return LastAtOrBeforePosIndex;
313
314 // We found an element starting before Pos. Check for overlap.
315 if (Offsets[LastAtOrBeforePosIndex] +
316 getSize(C: Elems[LastAtOrBeforePosIndex]) <= Pos)
317 return LastAtOrBeforePosIndex + 1;
318
319 // Try to decompose it into smaller constants.
320 if (!split(Index: LastAtOrBeforePosIndex, Hint: Pos))
321 return std::nullopt;
322 }
323}
324
325/// Split the constant at index Index, if possible. Return true if we did.
326/// Hint indicates the location at which we'd like to split, but may be
327/// ignored.
328bool ConstantAggregateBuilder::split(size_t Index, CharUnits Hint) {
329 NaturalLayout = false;
330 llvm::Constant *C = Elems[Index];
331 CharUnits Offset = Offsets[Index];
332
333 if (auto *CA = dyn_cast<llvm::ConstantAggregate>(Val: C)) {
334 // Expand the sequence into its contained elements.
335 // FIXME: This assumes vector elements are byte-sized.
336 replace(C&: Elems, BeginOff: Index, EndOff: Index + 1,
337 Vals: llvm::map_range(C: llvm::seq(Begin: 0u, End: CA->getNumOperands()),
338 F: [&](unsigned Op) { return CA->getOperand(i_nocapture: Op); }));
339 if (isa<llvm::ArrayType>(Val: CA->getType()) ||
340 isa<llvm::VectorType>(Val: CA->getType())) {
341 // Array or vector.
342 llvm::Type *ElemTy =
343 llvm::GetElementPtrInst::getTypeAtIndex(Ty: CA->getType(), Idx: (uint64_t)0);
344 CharUnits ElemSize = getSize(Ty: ElemTy);
345 replace(
346 C&: Offsets, BeginOff: Index, EndOff: Index + 1,
347 Vals: llvm::map_range(C: llvm::seq(Begin: 0u, End: CA->getNumOperands()),
348 F: [&](unsigned Op) { return Offset + Op * ElemSize; }));
349 } else {
350 // Must be a struct.
351 auto *ST = cast<llvm::StructType>(Val: CA->getType());
352 const llvm::StructLayout *Layout =
353 CGM.getDataLayout().getStructLayout(Ty: ST);
354 replace(C&: Offsets, BeginOff: Index, EndOff: Index + 1,
355 Vals: llvm::map_range(
356 C: llvm::seq(Begin: 0u, End: CA->getNumOperands()), F: [&](unsigned Op) {
357 return Offset + CharUnits::fromQuantity(
358 Quantity: Layout->getElementOffset(Idx: Op));
359 }));
360 }
361 return true;
362 }
363
364 if (auto *CDS = dyn_cast<llvm::ConstantDataSequential>(Val: C)) {
365 // Expand the sequence into its contained elements.
366 // FIXME: This assumes vector elements are byte-sized.
367 // FIXME: If possible, split into two ConstantDataSequentials at Hint.
368 CharUnits ElemSize = getSize(Ty: CDS->getElementType());
369 replace(C&: Elems, BeginOff: Index, EndOff: Index + 1,
370 Vals: llvm::map_range(C: llvm::seq(Begin: uint64_t(0u), End: CDS->getNumElements()),
371 F: [&](uint64_t Elem) {
372 return CDS->getElementAsConstant(i: Elem);
373 }));
374 replace(C&: Offsets, BeginOff: Index, EndOff: Index + 1,
375 Vals: llvm::map_range(
376 C: llvm::seq(Begin: uint64_t(0u), End: CDS->getNumElements()),
377 F: [&](uint64_t Elem) { return Offset + Elem * ElemSize; }));
378 return true;
379 }
380
381 if (isa<llvm::ConstantAggregateZero>(Val: C)) {
382 // Split into two zeros at the hinted offset.
383 CharUnits ElemSize = getSize(C);
384 assert(Hint > Offset && Hint < Offset + ElemSize && "nothing to split");
385 replace(C&: Elems, BeginOff: Index, EndOff: Index + 1,
386 Vals: {getZeroes(ZeroSize: Hint - Offset), getZeroes(ZeroSize: Offset + ElemSize - Hint)});
387 replace(C&: Offsets, BeginOff: Index, EndOff: Index + 1, Vals: {Offset, Hint});
388 return true;
389 }
390
391 if (isa<llvm::UndefValue>(Val: C)) {
392 // Drop undef; it doesn't contribute to the final layout.
393 replace(C&: Elems, BeginOff: Index, EndOff: Index + 1, Vals: {});
394 replace(C&: Offsets, BeginOff: Index, EndOff: Index + 1, Vals: {});
395 return true;
396 }
397
398 // FIXME: We could split a ConstantInt if the need ever arose.
399 // We don't need to do this to handle bit-fields because we always eagerly
400 // split them into 1-byte chunks.
401
402 return false;
403}
404
405static llvm::Constant *
406EmitArrayConstant(CodeGenModule &CGM, llvm::ArrayType *DesiredType,
407 llvm::Type *CommonElementType, uint64_t ArrayBound,
408 SmallVectorImpl<llvm::Constant *> &Elements,
409 llvm::Constant *Filler);
410
411llvm::Constant *ConstantAggregateBuilder::buildFrom(
412 CodeGenModule &CGM, ArrayRef<llvm::Constant *> Elems,
413 ArrayRef<CharUnits> Offsets, CharUnits StartOffset, CharUnits Size,
414 bool NaturalLayout, llvm::Type *DesiredTy, bool AllowOversized) {
415 ConstantAggregateBuilderUtils Utils(CGM);
416
417 if (Elems.empty())
418 return llvm::UndefValue::get(T: DesiredTy);
419
420 auto Offset = [&](size_t I) { return Offsets[I] - StartOffset; };
421
422 // If we want an array type, see if all the elements are the same type and
423 // appropriately spaced.
424 if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(Val: DesiredTy)) {
425 assert(!AllowOversized && "oversized array emission not supported");
426
427 bool CanEmitArray = true;
428 llvm::Type *CommonType = Elems[0]->getType();
429 llvm::Constant *Filler = llvm::Constant::getNullValue(Ty: CommonType);
430 CharUnits ElemSize = Utils.getSize(Ty: ATy->getElementType());
431 SmallVector<llvm::Constant*, 32> ArrayElements;
432 for (size_t I = 0; I != Elems.size(); ++I) {
433 // Skip zeroes; we'll use a zero value as our array filler.
434 if (Elems[I]->isNullValue())
435 continue;
436
437 // All remaining elements must be the same type.
438 if (Elems[I]->getType() != CommonType ||
439 !Offset(I).isMultipleOf(N: ElemSize)) {
440 CanEmitArray = false;
441 break;
442 }
443 ArrayElements.resize(N: Offset(I) / ElemSize + 1, NV: Filler);
444 ArrayElements.back() = Elems[I];
445 }
446
447 if (CanEmitArray) {
448 return EmitArrayConstant(CGM, DesiredType: ATy, CommonElementType: CommonType, ArrayBound: ATy->getNumElements(),
449 Elements&: ArrayElements, Filler);
450 }
451
452 // Can't emit as an array, carry on to emit as a struct.
453 }
454
455 // The size of the constant we plan to generate. This is usually just
456 // the size of the initialized type, but in AllowOversized mode (i.e.
457 // flexible array init), it can be larger.
458 CharUnits DesiredSize = Utils.getSize(Ty: DesiredTy);
459 if (Size > DesiredSize) {
460 assert(AllowOversized && "Elems are oversized");
461 DesiredSize = Size;
462 }
463
464 // The natural alignment of an unpacked LLVM struct with the given elements.
465 CharUnits Align = CharUnits::One();
466 for (llvm::Constant *C : Elems)
467 Align = std::max(a: Align, b: Utils.getAlignment(C));
468
469 // The natural size of an unpacked LLVM struct with the given elements.
470 CharUnits AlignedSize = Size.alignTo(Align);
471
472 bool Packed = false;
473 ArrayRef<llvm::Constant*> UnpackedElems = Elems;
474 llvm::SmallVector<llvm::Constant*, 32> UnpackedElemStorage;
475 if (DesiredSize < AlignedSize || DesiredSize.alignTo(Align) != DesiredSize) {
476 // The natural layout would be too big; force use of a packed layout.
477 NaturalLayout = false;
478 Packed = true;
479 } else if (DesiredSize > AlignedSize) {
480 // The natural layout would be too small. Add padding to fix it. (This
481 // is ignored if we choose a packed layout.)
482 UnpackedElemStorage.assign(in_start: Elems.begin(), in_end: Elems.end());
483 UnpackedElemStorage.push_back(Elt: Utils.getPadding(PadSize: DesiredSize - Size));
484 UnpackedElems = UnpackedElemStorage;
485 }
486
487 // If we don't have a natural layout, insert padding as necessary.
488 // As we go, double-check to see if we can actually just emit Elems
489 // as a non-packed struct and do so opportunistically if possible.
490 llvm::SmallVector<llvm::Constant*, 32> PackedElems;
491 if (!NaturalLayout) {
492 CharUnits SizeSoFar = CharUnits::Zero();
493 for (size_t I = 0; I != Elems.size(); ++I) {
494 CharUnits Align = Utils.getAlignment(C: Elems[I]);
495 CharUnits NaturalOffset = SizeSoFar.alignTo(Align);
496 CharUnits DesiredOffset = Offset(I);
497 assert(DesiredOffset >= SizeSoFar && "elements out of order");
498
499 if (DesiredOffset != NaturalOffset)
500 Packed = true;
501 if (DesiredOffset != SizeSoFar)
502 PackedElems.push_back(Elt: Utils.getPadding(PadSize: DesiredOffset - SizeSoFar));
503 PackedElems.push_back(Elt: Elems[I]);
504 SizeSoFar = DesiredOffset + Utils.getSize(C: Elems[I]);
505 }
506 // If we're using the packed layout, pad it out to the desired size if
507 // necessary.
508 if (Packed) {
509 assert(SizeSoFar <= DesiredSize &&
510 "requested size is too small for contents");
511 if (SizeSoFar < DesiredSize)
512 PackedElems.push_back(Elt: Utils.getPadding(PadSize: DesiredSize - SizeSoFar));
513 }
514 }
515
516 llvm::StructType *STy = llvm::ConstantStruct::getTypeForElements(
517 Ctx&: CGM.getLLVMContext(), V: Packed ? PackedElems : UnpackedElems, Packed);
518
519 // Pick the type to use. If the type is layout identical to the desired
520 // type then use it, otherwise use whatever the builder produced for us.
521 if (llvm::StructType *DesiredSTy = dyn_cast<llvm::StructType>(Val: DesiredTy)) {
522 if (DesiredSTy->isLayoutIdentical(Other: STy))
523 STy = DesiredSTy;
524 }
525
526 return llvm::ConstantStruct::get(T: STy, V: Packed ? PackedElems : UnpackedElems);
527}
528
529void ConstantAggregateBuilder::condense(CharUnits Offset,
530 llvm::Type *DesiredTy) {
531 CharUnits Size = getSize(Ty: DesiredTy);
532
533 std::optional<size_t> FirstElemToReplace = splitAt(Pos: Offset);
534 if (!FirstElemToReplace)
535 return;
536 size_t First = *FirstElemToReplace;
537
538 std::optional<size_t> LastElemToReplace = splitAt(Pos: Offset + Size);
539 if (!LastElemToReplace)
540 return;
541 size_t Last = *LastElemToReplace;
542
543 size_t Length = Last - First;
544 if (Length == 0)
545 return;
546
547 if (Length == 1 && Offsets[First] == Offset &&
548 getSize(C: Elems[First]) == Size) {
549 // Re-wrap single element structs if necessary. Otherwise, leave any single
550 // element constant of the right size alone even if it has the wrong type.
551 auto *STy = dyn_cast<llvm::StructType>(Val: DesiredTy);
552 if (STy && STy->getNumElements() == 1 &&
553 STy->getElementType(N: 0) == Elems[First]->getType())
554 Elems[First] = llvm::ConstantStruct::get(T: STy, Vs: Elems[First]);
555 return;
556 }
557
558 llvm::Constant *Replacement = buildFrom(
559 CGM, Elems: ArrayRef(Elems).slice(N: First, M: Length),
560 Offsets: ArrayRef(Offsets).slice(N: First, M: Length), StartOffset: Offset, Size: getSize(Ty: DesiredTy),
561 /*known to have natural layout=*/NaturalLayout: false, DesiredTy, AllowOversized: false);
562 replace(C&: Elems, BeginOff: First, EndOff: Last, Vals: {Replacement});
563 replace(C&: Offsets, BeginOff: First, EndOff: Last, Vals: {Offset});
564}
565
566//===----------------------------------------------------------------------===//
567// ConstStructBuilder
568//===----------------------------------------------------------------------===//
569
570class ConstStructBuilder {
571 CodeGenModule &CGM;
572 ConstantEmitter &Emitter;
573 ConstantAggregateBuilder &Builder;
574 CharUnits StartOffset;
575
576public:
577 static llvm::Constant *BuildStruct(ConstantEmitter &Emitter,
578 const InitListExpr *ILE,
579 QualType StructTy);
580 static llvm::Constant *BuildStruct(ConstantEmitter &Emitter,
581 const APValue &Value, QualType ValTy);
582 static bool UpdateStruct(ConstantEmitter &Emitter,
583 ConstantAggregateBuilder &Const, CharUnits Offset,
584 const InitListExpr *Updater);
585
586private:
587 ConstStructBuilder(ConstantEmitter &Emitter,
588 ConstantAggregateBuilder &Builder, CharUnits StartOffset)
589 : CGM(Emitter.CGM), Emitter(Emitter), Builder(Builder),
590 StartOffset(StartOffset) {}
591
592 bool AppendField(const FieldDecl *Field, uint64_t FieldOffset,
593 llvm::Constant *InitExpr, bool AllowOverwrite = false);
594
595 bool AppendBytes(CharUnits FieldOffsetInChars, llvm::Constant *InitCst,
596 bool AllowOverwrite = false);
597
598 bool AppendBitField(const FieldDecl *Field, uint64_t FieldOffset,
599 llvm::Constant *InitExpr, bool AllowOverwrite = false);
600
601 bool Build(const InitListExpr *ILE, bool AllowOverwrite);
602 bool Build(const APValue &Val, const RecordDecl *RD, bool IsPrimaryBase,
603 const CXXRecordDecl *VTableClass, CharUnits BaseOffset);
604 bool DoZeroInitPadding(const ASTRecordLayout &Layout, unsigned FieldNo,
605 const FieldDecl &Field, bool AllowOverwrite,
606 CharUnits &SizeSoFar, bool &ZeroFieldSize);
607 bool DoZeroInitPadding(const ASTRecordLayout &Layout, bool AllowOverwrite,
608 CharUnits SizeSoFar);
609 llvm::Constant *Finalize(QualType Ty);
610};
611
612bool ConstStructBuilder::AppendField(
613 const FieldDecl *Field, uint64_t FieldOffset, llvm::Constant *InitCst,
614 bool AllowOverwrite) {
615 const ASTContext &Context = CGM.getContext();
616
617 CharUnits FieldOffsetInChars = Context.toCharUnitsFromBits(BitSize: FieldOffset);
618
619 return AppendBytes(FieldOffsetInChars, InitCst, AllowOverwrite);
620}
621
622bool ConstStructBuilder::AppendBytes(CharUnits FieldOffsetInChars,
623 llvm::Constant *InitCst,
624 bool AllowOverwrite) {
625 return Builder.add(C: InitCst, Offset: StartOffset + FieldOffsetInChars, AllowOverwrite);
626}
627
628bool ConstStructBuilder::AppendBitField(const FieldDecl *Field,
629 uint64_t FieldOffset, llvm::Constant *C,
630 bool AllowOverwrite) {
631
632 llvm::ConstantInt *CI = dyn_cast<llvm::ConstantInt>(Val: C);
633 if (!CI) {
634 // Constants for long _BitInt types are sometimes split into individual
635 // bytes. Try to fold these back into an integer constant. If that doesn't
636 // work out, then we are trying to initialize a bitfield with a non-trivial
637 // constant, this must require run-time code.
638 llvm::Type *LoadType =
639 CGM.getTypes().convertTypeForLoadStore(T: Field->getType(), LLVMTy: C->getType());
640 llvm::Constant *FoldedConstant = llvm::ConstantFoldLoadFromConst(
641 C, Ty: LoadType, Offset: llvm::APInt::getZero(numBits: 32), DL: CGM.getDataLayout());
642 CI = dyn_cast_if_present<llvm::ConstantInt>(Val: FoldedConstant);
643 if (!CI)
644 return false;
645 }
646
647 const CGRecordLayout &RL =
648 CGM.getTypes().getCGRecordLayout(Field->getParent());
649 const CGBitFieldInfo &Info = RL.getBitFieldInfo(FD: Field);
650 llvm::APInt FieldValue = CI->getValue();
651
652 // Promote the size of FieldValue if necessary
653 // FIXME: This should never occur, but currently it can because initializer
654 // constants are cast to bool, and because clang is not enforcing bitfield
655 // width limits.
656 if (Info.Size > FieldValue.getBitWidth())
657 FieldValue = FieldValue.zext(width: Info.Size);
658
659 // Truncate the size of FieldValue to the bit field size.
660 if (Info.Size < FieldValue.getBitWidth())
661 FieldValue = FieldValue.trunc(width: Info.Size);
662
663 return Builder.addBits(Bits: FieldValue,
664 OffsetInBits: CGM.getContext().toBits(CharSize: StartOffset) + FieldOffset,
665 AllowOverwrite);
666}
667
668static bool EmitDesignatedInitUpdater(ConstantEmitter &Emitter,
669 ConstantAggregateBuilder &Const,
670 CharUnits Offset, QualType Type,
671 const InitListExpr *Updater) {
672 if (Type->isRecordType())
673 return ConstStructBuilder::UpdateStruct(Emitter, Const, Offset, Updater);
674
675 auto CAT = Emitter.CGM.getContext().getAsConstantArrayType(T: Type);
676 if (!CAT)
677 return false;
678 QualType ElemType = CAT->getElementType();
679 CharUnits ElemSize = Emitter.CGM.getContext().getTypeSizeInChars(T: ElemType);
680 llvm::Type *ElemTy = Emitter.CGM.getTypes().ConvertTypeForMem(T: ElemType);
681
682 llvm::Constant *FillC = nullptr;
683 if (const Expr *Filler = Updater->getArrayFiller()) {
684 if (!isa<NoInitExpr>(Val: Filler)) {
685 FillC = Emitter.tryEmitAbstractForMemory(E: Filler, T: ElemType);
686 if (!FillC)
687 return false;
688 }
689 }
690
691 unsigned NumElementsToUpdate =
692 FillC ? CAT->getZExtSize() : Updater->getNumInits();
693 for (unsigned I = 0; I != NumElementsToUpdate; ++I, Offset += ElemSize) {
694 const Expr *Init = nullptr;
695 if (I < Updater->getNumInits())
696 Init = Updater->getInit(Init: I);
697
698 if (!Init && FillC) {
699 if (!Const.add(C: FillC, Offset, AllowOverwrite: true))
700 return false;
701 } else if (!Init || isa<NoInitExpr>(Val: Init)) {
702 continue;
703 } else if (const auto *ChildILE = dyn_cast<InitListExpr>(Val: Init)) {
704 if (!EmitDesignatedInitUpdater(Emitter, Const, Offset, Type: ElemType,
705 Updater: ChildILE))
706 return false;
707 // Attempt to reduce the array element to a single constant if necessary.
708 Const.condense(Offset, DesiredTy: ElemTy);
709 } else {
710 llvm::Constant *Val = Emitter.tryEmitPrivateForMemory(E: Init, T: ElemType);
711 if (!Const.add(C: Val, Offset, AllowOverwrite: true))
712 return false;
713 }
714 }
715
716 return true;
717}
718
719bool ConstStructBuilder::Build(const InitListExpr *ILE, bool AllowOverwrite) {
720 auto *RD = ILE->getType()->castAsRecordDecl();
721 const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(D: RD);
722
723 unsigned FieldNo = -1;
724 unsigned ElementNo = 0;
725
726 // Bail out if we have base classes. We could support these, but they only
727 // arise in C++1z where we will have already constant folded most interesting
728 // cases. FIXME: There are still a few more cases we can handle this way.
729 if (auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD))
730 if (CXXRD->getNumBases())
731 return false;
732
733 const bool ZeroInitPadding = CGM.shouldZeroInitPadding();
734 bool ZeroFieldSize = false;
735 CharUnits SizeSoFar = CharUnits::Zero();
736
737 for (FieldDecl *Field : RD->fields()) {
738 ++FieldNo;
739
740 // If this is a union, skip all the fields that aren't being initialized.
741 if (RD->isUnion() &&
742 !declaresSameEntity(D1: ILE->getInitializedFieldInUnion(), D2: Field))
743 continue;
744
745 // Don't emit anonymous bitfields.
746 if (Field->isUnnamedBitField())
747 continue;
748
749 // Get the initializer. A struct can include fields without initializers,
750 // we just use explicit null values for them.
751 const Expr *Init = nullptr;
752 if (ElementNo < ILE->getNumInits())
753 Init = ILE->getInit(Init: ElementNo++);
754 if (isa_and_nonnull<NoInitExpr>(Val: Init)) {
755 if (ZeroInitPadding &&
756 !DoZeroInitPadding(Layout, FieldNo, Field: *Field, AllowOverwrite, SizeSoFar,
757 ZeroFieldSize))
758 return false;
759 continue;
760 }
761
762 // Zero-sized fields are not emitted, but their initializers may still
763 // prevent emission of this struct as a constant.
764 if (isEmptyFieldForLayout(Context: CGM.getContext(), FD: Field)) {
765 if (Init && Init->HasSideEffects(Ctx: CGM.getContext()))
766 return false;
767 continue;
768 }
769
770 if (ZeroInitPadding &&
771 !DoZeroInitPadding(Layout, FieldNo, Field: *Field, AllowOverwrite, SizeSoFar,
772 ZeroFieldSize))
773 return false;
774
775 // When emitting a DesignatedInitUpdateExpr, a nested InitListExpr
776 // represents additional overwriting of our current constant value, and not
777 // a new constant to emit independently.
778 if (AllowOverwrite &&
779 (Field->getType()->isArrayType() || Field->getType()->isRecordType())) {
780 if (auto *SubILE = dyn_cast<InitListExpr>(Val: Init)) {
781 CharUnits Offset = CGM.getContext().toCharUnitsFromBits(
782 BitSize: Layout.getFieldOffset(FieldNo));
783 if (!EmitDesignatedInitUpdater(Emitter, Const&: Builder, Offset: StartOffset + Offset,
784 Type: Field->getType(), Updater: SubILE))
785 return false;
786 // If we split apart the field's value, try to collapse it down to a
787 // single value now.
788 Builder.condense(Offset: StartOffset + Offset,
789 DesiredTy: CGM.getTypes().ConvertTypeForMem(T: Field->getType()));
790 continue;
791 }
792 }
793
794 llvm::Constant *EltInit =
795 Init ? Emitter.tryEmitPrivateForMemory(E: Init, T: Field->getType())
796 : Emitter.emitNullForMemory(T: Field->getType());
797 if (!EltInit)
798 return false;
799
800 if (ZeroInitPadding && ZeroFieldSize)
801 SizeSoFar += CharUnits::fromQuantity(
802 Quantity: CGM.getDataLayout().getTypeAllocSize(Ty: EltInit->getType()));
803
804 if (!Field->isBitField()) {
805 // Handle non-bitfield members.
806 if (!AppendField(Field, FieldOffset: Layout.getFieldOffset(FieldNo), InitCst: EltInit,
807 AllowOverwrite))
808 return false;
809 // After emitting a non-empty field with [[no_unique_address]], we may
810 // need to overwrite its tail padding.
811 if (Field->hasAttr<NoUniqueAddressAttr>())
812 AllowOverwrite = true;
813 } else {
814 // Otherwise we have a bitfield.
815 if (!AppendBitField(Field, FieldOffset: Layout.getFieldOffset(FieldNo), C: EltInit,
816 AllowOverwrite))
817 return false;
818 }
819 }
820
821 if (ZeroInitPadding && !DoZeroInitPadding(Layout, AllowOverwrite, SizeSoFar))
822 return false;
823
824 return true;
825}
826
827namespace {
828struct BaseInfo {
829 BaseInfo(const CXXRecordDecl *Decl, CharUnits Offset, unsigned Index)
830 : Decl(Decl), Offset(Offset), Index(Index) {
831 }
832
833 const CXXRecordDecl *Decl;
834 CharUnits Offset;
835 unsigned Index;
836
837 bool operator<(const BaseInfo &O) const { return Offset < O.Offset; }
838};
839}
840
841bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD,
842 bool IsPrimaryBase,
843 const CXXRecordDecl *VTableClass,
844 CharUnits Offset) {
845 const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(D: RD);
846
847 if (const CXXRecordDecl *CD = dyn_cast<CXXRecordDecl>(Val: RD)) {
848 // Add a vtable pointer, if we need one and it hasn't already been added.
849 if (Layout.hasOwnVFPtr()) {
850 llvm::Constant *VTableAddressPoint =
851 CGM.getCXXABI().getVTableAddressPoint(Base: BaseSubobject(CD, Offset),
852 VTableClass);
853 if (auto Authentication = CGM.getVTablePointerAuthentication(thisClass: CD)) {
854 VTableAddressPoint = Emitter.tryEmitConstantSignedPointer(
855 Ptr: VTableAddressPoint, Auth: *Authentication);
856 if (!VTableAddressPoint)
857 return false;
858 }
859 if (!AppendBytes(FieldOffsetInChars: Offset, InitCst: VTableAddressPoint))
860 return false;
861 }
862
863 // Accumulate and sort bases, in order to visit them in address order, which
864 // may not be the same as declaration order.
865 SmallVector<BaseInfo, 8> Bases;
866 Bases.reserve(N: CD->getNumBases());
867 unsigned BaseNo = 0;
868 for (CXXRecordDecl::base_class_const_iterator Base = CD->bases_begin(),
869 BaseEnd = CD->bases_end(); Base != BaseEnd; ++Base, ++BaseNo) {
870 assert(!Base->isVirtual() && "should not have virtual bases here");
871 const CXXRecordDecl *BD = Base->getType()->getAsCXXRecordDecl();
872 CharUnits BaseOffset = Layout.getBaseClassOffset(Base: BD);
873 Bases.push_back(Elt: BaseInfo(BD, BaseOffset, BaseNo));
874 }
875 llvm::stable_sort(Range&: Bases);
876
877 for (const BaseInfo &Base : Bases) {
878 bool IsPrimaryBase = Layout.getPrimaryBase() == Base.Decl;
879 if (!Build(Val: Val.getStructBase(i: Base.Index), RD: Base.Decl, IsPrimaryBase,
880 VTableClass, Offset: Offset + Base.Offset))
881 return false;
882 }
883 }
884
885 unsigned FieldNo = 0;
886 uint64_t OffsetBits = CGM.getContext().toBits(CharSize: Offset);
887 const bool ZeroInitPadding = CGM.shouldZeroInitPadding();
888 bool ZeroFieldSize = false;
889 CharUnits SizeSoFar = CharUnits::Zero();
890
891 bool AllowOverwrite = false;
892 for (RecordDecl::field_iterator Field = RD->field_begin(),
893 FieldEnd = RD->field_end(); Field != FieldEnd; ++Field, ++FieldNo) {
894 // If this is a union, skip all the fields that aren't being initialized.
895 if (RD->isUnion() && !declaresSameEntity(D1: Val.getUnionField(), D2: *Field))
896 continue;
897
898 // Don't emit anonymous bitfields or zero-sized fields.
899 if (Field->isUnnamedBitField() ||
900 isEmptyFieldForLayout(Context: CGM.getContext(), FD: *Field))
901 continue;
902
903 // Emit the value of the initializer.
904 const APValue &FieldValue =
905 RD->isUnion() ? Val.getUnionValue() : Val.getStructField(i: FieldNo);
906 llvm::Constant *EltInit =
907 Emitter.tryEmitPrivateForMemory(value: FieldValue, T: Field->getType());
908 if (!EltInit)
909 return false;
910
911 if (CGM.getContext().isPFPField(Field: *Field)) {
912 llvm::ConstantInt *Disc;
913 llvm::Constant *AddrDisc;
914 if (CGM.getContext().arePFPFieldsTriviallyCopyable(RD)) {
915 uint64_t FieldSignature =
916 llvm::getPointerAuthStableSipHash(S: CGM.getPFPFieldName(FD: *Field));
917 Disc = llvm::ConstantInt::get(Ty: CGM.Int64Ty, V: FieldSignature);
918 AddrDisc = llvm::ConstantPointerNull::get(T: CGM.VoidPtrTy);
919 } else if (Emitter.isAbstract()) {
920 // isAbstract means that we don't know the global's address. Since we
921 // can only form a pointer without knowing the address if the fields are
922 // trivially copyable, we need to return false otherwise.
923 return false;
924 } else {
925 Disc = llvm::ConstantInt::get(Ty: CGM.Int64Ty,
926 V: -(Layout.getFieldOffset(FieldNo) / 8));
927 AddrDisc = Emitter.getCurrentAddrPrivate();
928 }
929 EltInit = llvm::ConstantPtrAuth::get(
930 Ptr: EltInit, Key: llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 2), Disc, AddrDisc,
931 DeactivationSymbol: CGM.getPFPDeactivationSymbol(FD: *Field));
932 if (!CGM.getContext().arePFPFieldsTriviallyCopyable(RD))
933 Emitter.registerCurrentAddrPrivate(signal: EltInit,
934 placeholder: cast<llvm::GlobalValue>(Val: AddrDisc));
935 }
936
937 if (ZeroInitPadding) {
938 if (!DoZeroInitPadding(Layout, FieldNo, Field: **Field, AllowOverwrite,
939 SizeSoFar, ZeroFieldSize))
940 return false;
941 if (ZeroFieldSize)
942 SizeSoFar += CharUnits::fromQuantity(
943 Quantity: CGM.getDataLayout().getTypeAllocSize(Ty: EltInit->getType()));
944 }
945
946 if (!Field->isBitField()) {
947 // Handle non-bitfield members.
948 if (!AppendField(Field: *Field, FieldOffset: Layout.getFieldOffset(FieldNo) + OffsetBits,
949 InitCst: EltInit, AllowOverwrite))
950 return false;
951 // After emitting a non-empty field with [[no_unique_address]], we may
952 // need to overwrite its tail padding.
953 if (Field->hasAttr<NoUniqueAddressAttr>())
954 AllowOverwrite = true;
955 } else {
956 // Otherwise we have a bitfield.
957 if (!AppendBitField(Field: *Field, FieldOffset: Layout.getFieldOffset(FieldNo) + OffsetBits,
958 C: EltInit, AllowOverwrite))
959 return false;
960 }
961 }
962 if (ZeroInitPadding && !DoZeroInitPadding(Layout, AllowOverwrite, SizeSoFar))
963 return false;
964
965 return true;
966}
967
968bool ConstStructBuilder::DoZeroInitPadding(
969 const ASTRecordLayout &Layout, unsigned FieldNo, const FieldDecl &Field,
970 bool AllowOverwrite, CharUnits &SizeSoFar, bool &ZeroFieldSize) {
971 uint64_t StartBitOffset = Layout.getFieldOffset(FieldNo);
972 CharUnits StartOffset = CGM.getContext().toCharUnitsFromBits(BitSize: StartBitOffset);
973 if (SizeSoFar < StartOffset)
974 if (!AppendBytes(FieldOffsetInChars: SizeSoFar, InitCst: getPadding(CGM, PadSize: StartOffset - SizeSoFar),
975 AllowOverwrite))
976 return false;
977
978 if (!Field.isBitField()) {
979 CharUnits FieldSize = CGM.getContext().getTypeSizeInChars(T: Field.getType());
980 SizeSoFar = StartOffset + FieldSize;
981 ZeroFieldSize = FieldSize.isZero();
982 } else {
983 const CGRecordLayout &RL =
984 CGM.getTypes().getCGRecordLayout(Field.getParent());
985 const CGBitFieldInfo &Info = RL.getBitFieldInfo(FD: &Field);
986 uint64_t EndBitOffset = StartBitOffset + Info.Size;
987 SizeSoFar = CGM.getContext().toCharUnitsFromBits(BitSize: EndBitOffset);
988 if (EndBitOffset % CGM.getContext().getCharWidth() != 0) {
989 SizeSoFar++;
990 }
991 ZeroFieldSize = Info.Size == 0;
992 }
993 return true;
994}
995
996bool ConstStructBuilder::DoZeroInitPadding(const ASTRecordLayout &Layout,
997 bool AllowOverwrite,
998 CharUnits SizeSoFar) {
999 CharUnits TotalSize = Layout.getSize();
1000 if (SizeSoFar < TotalSize)
1001 if (!AppendBytes(FieldOffsetInChars: SizeSoFar, InitCst: getPadding(CGM, PadSize: TotalSize - SizeSoFar),
1002 AllowOverwrite))
1003 return false;
1004 SizeSoFar = TotalSize;
1005 return true;
1006}
1007
1008llvm::Constant *ConstStructBuilder::Finalize(QualType Type) {
1009 Type = Type.getNonReferenceType();
1010 auto *RD = Type->castAsRecordDecl();
1011 llvm::Type *ValTy = CGM.getTypes().ConvertType(T: Type);
1012 return Builder.build(DesiredTy: ValTy, AllowOversized: RD->hasFlexibleArrayMember());
1013}
1014
1015llvm::Constant *ConstStructBuilder::BuildStruct(ConstantEmitter &Emitter,
1016 const InitListExpr *ILE,
1017 QualType ValTy) {
1018 ConstantAggregateBuilder Const(Emitter.CGM);
1019 ConstStructBuilder Builder(Emitter, Const, CharUnits::Zero());
1020
1021 if (!Builder.Build(ILE, /*AllowOverwrite*/false))
1022 return nullptr;
1023
1024 return Builder.Finalize(Type: ValTy);
1025}
1026
1027llvm::Constant *ConstStructBuilder::BuildStruct(ConstantEmitter &Emitter,
1028 const APValue &Val,
1029 QualType ValTy) {
1030 ConstantAggregateBuilder Const(Emitter.CGM);
1031 ConstStructBuilder Builder(Emitter, Const, CharUnits::Zero());
1032
1033 const auto *RD = ValTy->castAsRecordDecl();
1034 const CXXRecordDecl *CD = dyn_cast<CXXRecordDecl>(Val: RD);
1035 if (!Builder.Build(Val, RD, IsPrimaryBase: false, VTableClass: CD, Offset: CharUnits::Zero()))
1036 return nullptr;
1037
1038 return Builder.Finalize(Type: ValTy);
1039}
1040
1041bool ConstStructBuilder::UpdateStruct(ConstantEmitter &Emitter,
1042 ConstantAggregateBuilder &Const,
1043 CharUnits Offset,
1044 const InitListExpr *Updater) {
1045 return ConstStructBuilder(Emitter, Const, Offset)
1046 .Build(ILE: Updater, /*AllowOverwrite*/ true);
1047}
1048
1049//===----------------------------------------------------------------------===//
1050// ConstExprEmitter
1051//===----------------------------------------------------------------------===//
1052
1053static ConstantAddress
1054tryEmitGlobalCompoundLiteral(ConstantEmitter &emitter,
1055 const CompoundLiteralExpr *E) {
1056 CodeGenModule &CGM = emitter.CGM;
1057 CharUnits Align = CGM.getContext().getTypeAlignInChars(T: E->getType());
1058 if (llvm::GlobalVariable *Addr =
1059 CGM.getAddrOfConstantCompoundLiteralIfEmitted(E))
1060 return ConstantAddress(Addr, Addr->getValueType(), Align);
1061
1062 LangAS addressSpace = E->getType().getAddressSpace();
1063 llvm::Constant *C = emitter.tryEmitForInitializer(E: E->getInitializer(),
1064 destAddrSpace: addressSpace, destType: E->getType());
1065 if (!C) {
1066 assert(!E->isFileScope() &&
1067 "file-scope compound literal did not have constant initializer!");
1068 return ConstantAddress::invalid();
1069 }
1070
1071 auto GV = new llvm::GlobalVariable(
1072 CGM.getModule(), C->getType(),
1073 E->getType().isConstantStorage(Ctx: CGM.getContext(), ExcludeCtor: true, ExcludeDtor: false),
1074 llvm::GlobalValue::InternalLinkage, C, ".compoundliteral", nullptr,
1075 llvm::GlobalVariable::NotThreadLocal,
1076 CGM.getContext().getTargetAddressSpace(AS: addressSpace));
1077 emitter.finalize(global: GV);
1078 GV->setAlignment(Align.getAsAlign());
1079 CGM.setAddrOfConstantCompoundLiteral(CLE: E, GV);
1080 return ConstantAddress(GV, GV->getValueType(), Align);
1081}
1082
1083static llvm::Constant *
1084EmitArrayConstant(CodeGenModule &CGM, llvm::ArrayType *DesiredType,
1085 llvm::Type *CommonElementType, uint64_t ArrayBound,
1086 SmallVectorImpl<llvm::Constant *> &Elements,
1087 llvm::Constant *Filler) {
1088 // Figure out how long the initial prefix of non-zero elements is.
1089 uint64_t NonzeroLength = ArrayBound;
1090 if (Elements.size() < NonzeroLength && Filler->isNullValue())
1091 NonzeroLength = Elements.size();
1092 if (NonzeroLength == Elements.size()) {
1093 while (NonzeroLength > 0 && Elements[NonzeroLength - 1]->isNullValue())
1094 --NonzeroLength;
1095 }
1096
1097 if (NonzeroLength == 0)
1098 return llvm::ConstantAggregateZero::get(Ty: DesiredType);
1099
1100 // Add a zeroinitializer array filler if we have lots of trailing zeroes.
1101 uint64_t TrailingZeroes = ArrayBound - NonzeroLength;
1102 if (TrailingZeroes >= 8) {
1103 assert(Elements.size() >= NonzeroLength &&
1104 "missing initializer for non-zero element");
1105
1106 // If all the elements had the same type up to the trailing zeroes, emit a
1107 // struct of two arrays (the nonzero data and the zeroinitializer).
1108 if (CommonElementType && NonzeroLength >= 8) {
1109 llvm::Constant *Initial = llvm::ConstantArray::get(
1110 T: llvm::ArrayType::get(ElementType: CommonElementType, NumElements: NonzeroLength),
1111 V: ArrayRef(Elements).take_front(N: NonzeroLength));
1112 Elements.resize(N: 2);
1113 Elements[0] = Initial;
1114 } else {
1115 Elements.resize(N: NonzeroLength + 1);
1116 }
1117
1118 auto *FillerType =
1119 CommonElementType ? CommonElementType : DesiredType->getElementType();
1120 FillerType = llvm::ArrayType::get(ElementType: FillerType, NumElements: TrailingZeroes);
1121 Elements.back() = llvm::ConstantAggregateZero::get(Ty: FillerType);
1122 CommonElementType = nullptr;
1123 } else if (Elements.size() != ArrayBound) {
1124 // Otherwise pad to the right size with the filler if necessary.
1125 Elements.resize(N: ArrayBound, NV: Filler);
1126 if (Filler->getType() != CommonElementType)
1127 CommonElementType = nullptr;
1128 }
1129
1130 // If all elements have the same type, just emit an array constant.
1131 if (CommonElementType)
1132 return llvm::ConstantArray::get(
1133 T: llvm::ArrayType::get(ElementType: CommonElementType, NumElements: ArrayBound), V: Elements);
1134
1135 // We have mixed types. Use a packed struct.
1136 llvm::SmallVector<llvm::Type *, 16> Types;
1137 Types.reserve(N: Elements.size());
1138 for (llvm::Constant *Elt : Elements)
1139 Types.push_back(Elt: Elt->getType());
1140 llvm::StructType *SType =
1141 llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: Types, isPacked: true);
1142 return llvm::ConstantStruct::get(T: SType, V: Elements);
1143}
1144
1145// This class only needs to handle arrays, structs and unions. Outside C++11
1146// mode, we don't currently constant fold those types. All other types are
1147// handled by constant folding.
1148//
1149// Constant folding is currently missing support for a few features supported
1150// here: CK_ReinterpretMemberPointer, and DesignatedInitUpdateExpr.
1151class ConstExprEmitter
1152 : public ConstStmtVisitor<ConstExprEmitter, llvm::Constant *, QualType> {
1153 CodeGenModule &CGM;
1154 ConstantEmitter &Emitter;
1155 llvm::LLVMContext &VMContext;
1156public:
1157 ConstExprEmitter(ConstantEmitter &emitter)
1158 : CGM(emitter.CGM), Emitter(emitter), VMContext(CGM.getLLVMContext()) {
1159 }
1160
1161 //===--------------------------------------------------------------------===//
1162 // Visitor Methods
1163 //===--------------------------------------------------------------------===//
1164
1165 llvm::Constant *VisitStmt(const Stmt *S, QualType T) { return nullptr; }
1166
1167 llvm::Constant *VisitConstantExpr(const ConstantExpr *CE, QualType T) {
1168 if (llvm::Constant *Result = Emitter.tryEmitConstantExpr(CE))
1169 return Result;
1170 return Visit(S: CE->getSubExpr(), P: T);
1171 }
1172
1173 llvm::Constant *VisitParenExpr(const ParenExpr *PE, QualType T) {
1174 return Visit(S: PE->getSubExpr(), P: T);
1175 }
1176
1177 llvm::Constant *
1178 VisitSubstNonTypeTemplateParmExpr(const SubstNonTypeTemplateParmExpr *PE,
1179 QualType T) {
1180 return Visit(S: PE->getReplacement(), P: T);
1181 }
1182
1183 llvm::Constant *VisitGenericSelectionExpr(const GenericSelectionExpr *GE,
1184 QualType T) {
1185 return Visit(S: GE->getResultExpr(), P: T);
1186 }
1187
1188 llvm::Constant *VisitChooseExpr(const ChooseExpr *CE, QualType T) {
1189 return Visit(S: CE->getChosenSubExpr(), P: T);
1190 }
1191
1192 llvm::Constant *VisitCompoundLiteralExpr(const CompoundLiteralExpr *E,
1193 QualType T) {
1194 return Visit(S: E->getInitializer(), P: T);
1195 }
1196
1197 llvm::Constant *ProduceIntToIntCast(const Expr *E, QualType DestType) {
1198 QualType FromType = E->getType();
1199 // See also HandleIntToIntCast in ExprConstant.cpp
1200 if (FromType->isIntegerType())
1201 if (llvm::Constant *C = Visit(S: E, P: FromType))
1202 if (auto *CI = dyn_cast<llvm::ConstantInt>(Val: C)) {
1203 unsigned SrcWidth = CGM.getContext().getIntWidth(T: FromType);
1204 unsigned DstWidth = CGM.getContext().getIntWidth(T: DestType);
1205 if (DstWidth == SrcWidth)
1206 return CI;
1207 llvm::APInt A = FromType->isSignedIntegerType()
1208 ? CI->getValue().sextOrTrunc(width: DstWidth)
1209 : CI->getValue().zextOrTrunc(width: DstWidth);
1210 return llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: A);
1211 }
1212 return nullptr;
1213 }
1214
1215 llvm::Constant *VisitCastExpr(const CastExpr *E, QualType destType) {
1216 if (const auto *ECE = dyn_cast<ExplicitCastExpr>(Val: E))
1217 CGM.EmitExplicitCastExprType(E: ECE, CGF: Emitter.CGF);
1218 const Expr *subExpr = E->getSubExpr();
1219
1220 switch (E->getCastKind()) {
1221 case CK_ToUnion: {
1222 // GCC cast to union extension
1223 assert(E->getType()->isUnionType() &&
1224 "Destination type is not union type!");
1225
1226 auto field = E->getTargetUnionField();
1227
1228 auto C = Emitter.tryEmitPrivateForMemory(E: subExpr, T: field->getType());
1229 if (!C) return nullptr;
1230
1231 auto destTy = ConvertType(T: destType);
1232 if (C->getType() == destTy) return C;
1233
1234 // Build a struct with the union sub-element as the first member,
1235 // and padded to the appropriate size.
1236 SmallVector<llvm::Constant*, 2> Elts;
1237 SmallVector<llvm::Type*, 2> Types;
1238 Elts.push_back(Elt: C);
1239 Types.push_back(Elt: C->getType());
1240 unsigned CurSize = CGM.getDataLayout().getTypeAllocSize(Ty: C->getType());
1241 unsigned TotalSize = CGM.getDataLayout().getTypeAllocSize(Ty: destTy);
1242
1243 assert(CurSize <= TotalSize && "Union size mismatch!");
1244 if (unsigned NumPadBytes = TotalSize - CurSize) {
1245 llvm::Constant *Padding =
1246 getPadding(CGM, PadSize: CharUnits::fromQuantity(Quantity: NumPadBytes));
1247 Elts.push_back(Elt: Padding);
1248 Types.push_back(Elt: Padding->getType());
1249 }
1250
1251 llvm::StructType *STy = llvm::StructType::get(Context&: VMContext, Elements: Types, isPacked: false);
1252 return llvm::ConstantStruct::get(T: STy, V: Elts);
1253 }
1254
1255 case CK_AddressSpaceConversion: {
1256 llvm::Constant *C = Emitter.tryEmitPrivate(E: subExpr, T: subExpr->getType());
1257 if (!C)
1258 return nullptr;
1259 llvm::Type *destTy = ConvertType(T: E->getType());
1260 return CGM.performAddrSpaceCast(Src: C, DestTy: destTy);
1261 }
1262
1263 case CK_LValueToRValue: {
1264 // We don't really support doing lvalue-to-rvalue conversions here; any
1265 // interesting conversions should be done in Evaluate(). But as a
1266 // special case, allow compound literals to support the gcc extension
1267 // allowing "struct x {int x;} x = (struct x) {};".
1268 if (const auto *E =
1269 dyn_cast<CompoundLiteralExpr>(Val: subExpr->IgnoreParens()))
1270 return Visit(S: E->getInitializer(), P: destType);
1271 return nullptr;
1272 }
1273
1274 case CK_AtomicToNonAtomic:
1275 case CK_NonAtomicToAtomic:
1276 case CK_NoOp:
1277 case CK_ConstructorConversion:
1278 return Visit(S: subExpr, P: destType);
1279
1280 case CK_ArrayToPointerDecay:
1281 if (const auto *S = dyn_cast<StringLiteral>(Val: subExpr))
1282 return CGM.GetAddrOfConstantStringFromLiteral(S).getPointer();
1283 return nullptr;
1284 case CK_NullToPointer:
1285 if (Visit(S: subExpr, P: destType))
1286 return CGM.EmitNullConstant(T: destType);
1287 return nullptr;
1288
1289 case CK_IntToOCLSampler:
1290 llvm_unreachable("global sampler variables are not generated");
1291
1292 case CK_IntegralCast:
1293 return ProduceIntToIntCast(E: subExpr, DestType: destType);
1294
1295 case CK_Dependent: llvm_unreachable("saw dependent cast!");
1296
1297 case CK_BuiltinFnToFnPtr:
1298 llvm_unreachable("builtin functions are handled elsewhere");
1299
1300 case CK_ReinterpretMemberPointer:
1301 case CK_DerivedToBaseMemberPointer:
1302 case CK_BaseToDerivedMemberPointer: {
1303 auto C = Emitter.tryEmitPrivate(E: subExpr, T: subExpr->getType());
1304 if (!C) return nullptr;
1305 return CGM.getCXXABI().EmitMemberPointerConversion(E, Src: C);
1306 }
1307
1308 // These will never be supported.
1309 case CK_ObjCObjectLValueCast:
1310 case CK_ARCProduceObject:
1311 case CK_ARCConsumeObject:
1312 case CK_ARCReclaimReturnedObject:
1313 case CK_ARCExtendBlockObject:
1314 case CK_CopyAndAutoreleaseBlockObject:
1315 return nullptr;
1316
1317 // These don't need to be handled here because Evaluate knows how to
1318 // evaluate them in the cases where they can be folded.
1319 case CK_BitCast:
1320 case CK_ToVoid:
1321 case CK_Dynamic:
1322 case CK_LValueBitCast:
1323 case CK_LValueToRValueBitCast:
1324 case CK_NullToMemberPointer:
1325 case CK_UserDefinedConversion:
1326 case CK_CPointerToObjCPointerCast:
1327 case CK_BlockPointerToObjCPointerCast:
1328 case CK_AnyPointerToBlockPointerCast:
1329 case CK_FunctionToPointerDecay:
1330 case CK_BaseToDerived:
1331 case CK_DerivedToBase:
1332 case CK_UncheckedDerivedToBase:
1333 case CK_MemberPointerToBoolean:
1334 case CK_VectorSplat:
1335 case CK_FloatingRealToComplex:
1336 case CK_FloatingComplexToReal:
1337 case CK_FloatingComplexToBoolean:
1338 case CK_FloatingComplexCast:
1339 case CK_FloatingComplexToIntegralComplex:
1340 case CK_IntegralRealToComplex:
1341 case CK_IntegralComplexToReal:
1342 case CK_IntegralComplexToBoolean:
1343 case CK_IntegralComplexCast:
1344 case CK_IntegralComplexToFloatingComplex:
1345 case CK_PointerToIntegral:
1346 case CK_PointerToBoolean:
1347 case CK_BooleanToSignedIntegral:
1348 case CK_IntegralToPointer:
1349 case CK_IntegralToBoolean:
1350 case CK_IntegralToFloating:
1351 case CK_FloatingToIntegral:
1352 case CK_FloatingToBoolean:
1353 case CK_FloatingCast:
1354 case CK_FloatingToFixedPoint:
1355 case CK_FixedPointToFloating:
1356 case CK_FixedPointCast:
1357 case CK_FixedPointToBoolean:
1358 case CK_FixedPointToIntegral:
1359 case CK_IntegralToFixedPoint:
1360 case CK_ZeroToOCLOpaqueType:
1361 case CK_MatrixCast:
1362 case CK_HLSLVectorTruncation:
1363 case CK_HLSLMatrixTruncation:
1364 case CK_HLSLArrayRValue:
1365 case CK_HLSLElementwiseCast:
1366 case CK_HLSLAggregateSplatCast:
1367 return nullptr;
1368 }
1369 llvm_unreachable("Invalid CastKind");
1370 }
1371
1372 llvm::Constant *VisitCXXDefaultInitExpr(const CXXDefaultInitExpr *DIE,
1373 QualType T) {
1374 // No need for a DefaultInitExprScope: we don't handle 'this' in a
1375 // constant expression.
1376 return Visit(S: DIE->getExpr(), P: T);
1377 }
1378
1379 llvm::Constant *VisitExprWithCleanups(const ExprWithCleanups *E, QualType T) {
1380 return Visit(S: E->getSubExpr(), P: T);
1381 }
1382
1383 llvm::Constant *VisitIntegerLiteral(const IntegerLiteral *I, QualType T) {
1384 return llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: I->getValue());
1385 }
1386
1387 static APValue withDestType(ASTContext &Ctx, const Expr *E, QualType SrcType,
1388 QualType DestType, const llvm::APSInt &Value) {
1389 if (!Ctx.hasSameType(T1: SrcType, T2: DestType)) {
1390 if (DestType->isFloatingType()) {
1391 llvm::APFloat Result =
1392 llvm::APFloat(Ctx.getFloatTypeSemantics(T: DestType), 1);
1393 llvm::RoundingMode RM =
1394 E->getFPFeaturesInEffect(LO: Ctx.getLangOpts()).getRoundingMode();
1395 if (RM == llvm::RoundingMode::Dynamic)
1396 RM = llvm::RoundingMode::NearestTiesToEven;
1397 Result.convertFromAPInt(Input: Value, IsSigned: Value.isSigned(), RM);
1398 return APValue(Result);
1399 }
1400 }
1401 return APValue(Value);
1402 }
1403
1404 llvm::Constant *EmitArrayInitialization(const InitListExpr *ILE, QualType T) {
1405 auto *CAT = CGM.getContext().getAsConstantArrayType(T: ILE->getType());
1406 assert(CAT && "can't emit array init for non-constant-bound array");
1407 uint64_t NumInitElements = ILE->getNumInits();
1408 const uint64_t NumElements = CAT->getZExtSize();
1409 for (const auto *Init : ILE->inits()) {
1410 if (const auto *Embed =
1411 dyn_cast<EmbedExpr>(Val: Init->IgnoreParenImpCasts())) {
1412 NumInitElements += Embed->getDataElementCount() - 1;
1413 if (NumInitElements > NumElements) {
1414 NumInitElements = NumElements;
1415 break;
1416 }
1417 }
1418 }
1419
1420 // Initialising an array requires us to automatically
1421 // initialise any elements that have not been initialised explicitly
1422 uint64_t NumInitableElts = std::min<uint64_t>(a: NumInitElements, b: NumElements);
1423
1424 QualType EltType = CAT->getElementType();
1425
1426 // Initialize remaining array elements.
1427 llvm::Constant *fillC = nullptr;
1428 if (const Expr *filler = ILE->getArrayFiller()) {
1429 fillC = Emitter.tryEmitAbstractForMemory(E: filler, T: EltType);
1430 if (!fillC)
1431 return nullptr;
1432 }
1433
1434 // Copy initializer elements.
1435 SmallVector<llvm::Constant *, 16> Elts;
1436 if (fillC && fillC->isNullValue())
1437 Elts.reserve(N: NumInitableElts + 1);
1438 else
1439 Elts.reserve(N: NumElements);
1440
1441 llvm::Type *CommonElementType = nullptr;
1442 auto Emit = [&](const Expr *Init, unsigned ArrayIndex) {
1443 llvm::Constant *C = nullptr;
1444 C = Emitter.tryEmitPrivateForMemory(E: Init, T: EltType);
1445 if (!C)
1446 return false;
1447 if (ArrayIndex == 0)
1448 CommonElementType = C->getType();
1449 else if (C->getType() != CommonElementType)
1450 CommonElementType = nullptr;
1451 Elts.push_back(Elt: C);
1452 return true;
1453 };
1454
1455 unsigned ArrayIndex = 0;
1456 QualType DestTy = CAT->getElementType();
1457 for (unsigned i = 0; i < ILE->getNumInits(); ++i) {
1458 const Expr *Init = ILE->getInit(Init: i);
1459 if (auto *EmbedS = dyn_cast<EmbedExpr>(Val: Init->IgnoreParenImpCasts())) {
1460 StringLiteral *SL = EmbedS->getDataStringLiteral();
1461 llvm::APSInt Value(CGM.getContext().getTypeSize(T: DestTy),
1462 DestTy->isUnsignedIntegerType());
1463 llvm::Constant *C;
1464 for (unsigned I = EmbedS->getStartingElementPos(),
1465 N = EmbedS->getDataElementCount();
1466 I != EmbedS->getStartingElementPos() + N; ++I) {
1467 Value = SL->getCodeUnit(i: I);
1468 if (DestTy->isIntegerType()) {
1469 C = llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: Value);
1470 } else {
1471 C = Emitter.tryEmitPrivateForMemory(
1472 value: withDestType(Ctx&: CGM.getContext(), E: Init, SrcType: EmbedS->getType(), DestType: DestTy,
1473 Value),
1474 T: EltType);
1475 }
1476 if (!C)
1477 return nullptr;
1478 Elts.push_back(Elt: C);
1479 ArrayIndex++;
1480 }
1481 if ((ArrayIndex - EmbedS->getDataElementCount()) == 0)
1482 CommonElementType = C->getType();
1483 else if (C->getType() != CommonElementType)
1484 CommonElementType = nullptr;
1485 } else {
1486 if (!Emit(Init, ArrayIndex))
1487 return nullptr;
1488 ArrayIndex++;
1489 }
1490 }
1491
1492 llvm::ArrayType *Desired =
1493 cast<llvm::ArrayType>(Val: CGM.getTypes().ConvertType(T: ILE->getType()));
1494 return EmitArrayConstant(CGM, DesiredType: Desired, CommonElementType, ArrayBound: NumElements, Elements&: Elts,
1495 Filler: fillC);
1496 }
1497
1498 llvm::Constant *EmitRecordInitialization(const InitListExpr *ILE,
1499 QualType T) {
1500 return ConstStructBuilder::BuildStruct(Emitter, ILE, ValTy: T);
1501 }
1502
1503 llvm::Constant *VisitImplicitValueInitExpr(const ImplicitValueInitExpr *E,
1504 QualType T) {
1505 return CGM.EmitNullConstant(T);
1506 }
1507
1508 llvm::Constant *VisitInitListExpr(const InitListExpr *ILE, QualType T) {
1509 if (ILE->isTransparent())
1510 return Visit(S: ILE->getInit(Init: 0), P: T);
1511
1512 if (ILE->getType()->isArrayType())
1513 return EmitArrayInitialization(ILE, T);
1514
1515 if (ILE->getType()->isRecordType())
1516 return EmitRecordInitialization(ILE, T);
1517
1518 return nullptr;
1519 }
1520
1521 llvm::Constant *
1522 VisitDesignatedInitUpdateExpr(const DesignatedInitUpdateExpr *E,
1523 QualType destType) {
1524 auto C = Visit(S: E->getBase(), P: destType);
1525 if (!C)
1526 return nullptr;
1527
1528 ConstantAggregateBuilder Const(CGM);
1529 Const.add(C, Offset: CharUnits::Zero(), AllowOverwrite: false);
1530
1531 if (!EmitDesignatedInitUpdater(Emitter, Const, Offset: CharUnits::Zero(), Type: destType,
1532 Updater: E->getUpdater()))
1533 return nullptr;
1534
1535 llvm::Type *ValTy = CGM.getTypes().ConvertType(T: destType);
1536 bool HasFlexibleArray = false;
1537 if (const auto *RD = destType->getAsRecordDecl())
1538 HasFlexibleArray = RD->hasFlexibleArrayMember();
1539 return Const.build(DesiredTy: ValTy, AllowOversized: HasFlexibleArray);
1540 }
1541
1542 llvm::Constant *VisitCXXConstructExpr(const CXXConstructExpr *E,
1543 QualType Ty) {
1544 if (!E->getConstructor()->isTrivial())
1545 return nullptr;
1546
1547 // Only default and copy/move constructors can be trivial.
1548 if (E->getNumArgs()) {
1549 assert(E->getNumArgs() == 1 && "trivial ctor with > 1 argument");
1550 assert(E->getConstructor()->isCopyOrMoveConstructor() &&
1551 "trivial ctor has argument but isn't a copy/move ctor");
1552
1553 const Expr *Arg = E->getArg(Arg: 0);
1554 assert(CGM.getContext().hasSameUnqualifiedType(Ty, Arg->getType()) &&
1555 "argument to copy ctor is of wrong type");
1556
1557 // Look through the temporary; it's just converting the value to an
1558 // lvalue to pass it to the constructor.
1559 if (const auto *MTE = dyn_cast<MaterializeTemporaryExpr>(Val: Arg))
1560 return Visit(S: MTE->getSubExpr(), P: Ty);
1561 // Don't try to support arbitrary lvalue-to-rvalue conversions for now.
1562 return nullptr;
1563 }
1564
1565 return CGM.EmitNullConstant(T: Ty);
1566 }
1567
1568 llvm::Constant *VisitStringLiteral(const StringLiteral *E, QualType T) {
1569 // This is a string literal initializing an array in an initializer.
1570 return CGM.GetConstantArrayFromStringLiteral(E);
1571 }
1572
1573 llvm::Constant *VisitObjCEncodeExpr(const ObjCEncodeExpr *E, QualType T) {
1574 // This must be an @encode initializing an array in a static initializer.
1575 // Don't emit it as the address of the string, emit the string data itself
1576 // as an inline array.
1577 std::string Str;
1578 CGM.getContext().getObjCEncodingForType(T: E->getEncodedType(), S&: Str);
1579 const ConstantArrayType *CAT = CGM.getContext().getAsConstantArrayType(T);
1580 assert(CAT && "String data not of constant array type!");
1581
1582 // Resize the string to the right size, adding zeros at the end, or
1583 // truncating as needed.
1584 Str.resize(n: CAT->getZExtSize(), c: '\0');
1585 return llvm::ConstantDataArray::getString(Context&: VMContext, Initializer: Str, AddNull: false);
1586 }
1587
1588 llvm::Constant *VisitUnaryExtension(const UnaryOperator *E, QualType T) {
1589 return Visit(S: E->getSubExpr(), P: T);
1590 }
1591
1592 llvm::Constant *VisitUnaryMinus(const UnaryOperator *U, QualType T) {
1593 if (llvm::Constant *C = Visit(S: U->getSubExpr(), P: T))
1594 if (auto *CI = dyn_cast<llvm::ConstantInt>(Val: C))
1595 return llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: -CI->getValue());
1596 return nullptr;
1597 }
1598
1599 llvm::Constant *VisitPackIndexingExpr(const PackIndexingExpr *E, QualType T) {
1600 return Visit(S: E->getSelectedExpr(), P: T);
1601 }
1602
1603 // Utility methods
1604 llvm::Type *ConvertType(QualType T) {
1605 return CGM.getTypes().ConvertType(T);
1606 }
1607};
1608
1609} // end anonymous namespace.
1610
1611llvm::Constant *ConstantEmitter::validateAndPopAbstract(llvm::Constant *C,
1612 AbstractState saved) {
1613 Abstract = saved.OldValue;
1614
1615 assert(saved.OldPlaceholdersSize == PlaceholderAddresses.size() &&
1616 "created a placeholder while doing an abstract emission?");
1617
1618 // No validation necessary for now.
1619 // No cleanup to do for now.
1620 return C;
1621}
1622
1623llvm::Constant *
1624ConstantEmitter::tryEmitAbstractForInitializer(const VarDecl &D) {
1625 auto state = pushAbstract();
1626 auto C = tryEmitPrivateForVarInit(D);
1627 return validateAndPopAbstract(C, saved: state);
1628}
1629
1630llvm::Constant *
1631ConstantEmitter::tryEmitAbstract(const Expr *E, QualType destType) {
1632 auto state = pushAbstract();
1633 auto C = tryEmitPrivate(E, T: destType);
1634 return validateAndPopAbstract(C, saved: state);
1635}
1636
1637llvm::Constant *
1638ConstantEmitter::tryEmitAbstract(const APValue &value, QualType destType) {
1639 auto state = pushAbstract();
1640 auto C = tryEmitPrivate(value, T: destType);
1641 return validateAndPopAbstract(C, saved: state);
1642}
1643
1644llvm::Constant *ConstantEmitter::tryEmitConstantExpr(const ConstantExpr *CE) {
1645 if (!CE->hasAPValueResult())
1646 return nullptr;
1647
1648 QualType RetType = CE->getType();
1649 if (CE->isGLValue())
1650 RetType = CGM.getContext().getLValueReferenceType(T: RetType);
1651
1652 return tryEmitAbstract(value: CE->getAPValueResult(), destType: RetType);
1653}
1654
1655llvm::Constant *
1656ConstantEmitter::emitAbstract(const Expr *E, QualType destType) {
1657 auto state = pushAbstract();
1658 auto C = tryEmitPrivate(E, T: destType);
1659 C = validateAndPopAbstract(C, saved: state);
1660 if (!C) {
1661 CGM.Error(loc: E->getExprLoc(),
1662 error: "internal error: could not emit constant value \"abstractly\"");
1663 C = CGM.EmitNullConstant(T: destType);
1664 }
1665 return C;
1666}
1667
1668llvm::Constant *
1669ConstantEmitter::emitAbstract(SourceLocation loc, const APValue &value,
1670 QualType destType,
1671 bool EnablePtrAuthFunctionTypeDiscrimination) {
1672 auto state = pushAbstract();
1673 auto C =
1674 tryEmitPrivate(value, T: destType, EnablePtrAuthFunctionTypeDiscrimination);
1675 C = validateAndPopAbstract(C, saved: state);
1676 if (!C) {
1677 CGM.Error(loc,
1678 error: "internal error: could not emit constant value \"abstractly\"");
1679 C = CGM.EmitNullConstant(T: destType);
1680 }
1681 return C;
1682}
1683
1684llvm::Constant *ConstantEmitter::tryEmitForInitializer(const VarDecl &D) {
1685 initializeNonAbstract(destAS: D.getType().getAddressSpace());
1686 llvm::Constant *Init = tryEmitPrivateForVarInit(D);
1687
1688 // If a placeholder address was needed for a TLS variable, implying that the
1689 // initializer's value depends on its address, then the object may not be
1690 // initialized in .tdata because the initializer will be memcpy'd to the
1691 // thread's TLS. Instead the initialization must be done in code.
1692 if (!PlaceholderAddresses.empty() && D.getTLSKind() != VarDecl::TLS_None) {
1693 for (auto [_, GV] : PlaceholderAddresses)
1694 GV->eraseFromParent();
1695 PlaceholderAddresses.clear();
1696 Init = nullptr;
1697 }
1698
1699 return markIfFailed(init: Init);
1700}
1701
1702llvm::Constant *ConstantEmitter::tryEmitForInitializer(const Expr *E,
1703 LangAS destAddrSpace,
1704 QualType destType) {
1705 initializeNonAbstract(destAS: destAddrSpace);
1706 return markIfFailed(init: tryEmitPrivateForMemory(E, T: destType));
1707}
1708
1709llvm::Constant *ConstantEmitter::emitForInitializer(const APValue &value,
1710 LangAS destAddrSpace,
1711 QualType destType) {
1712 initializeNonAbstract(destAS: destAddrSpace);
1713 auto C = tryEmitPrivateForMemory(value, T: destType);
1714 assert(C && "couldn't emit constant value non-abstractly?");
1715 return C;
1716}
1717
1718llvm::GlobalValue *ConstantEmitter::getCurrentAddrPrivate() {
1719 assert(!Abstract && "cannot get current address for abstract constant");
1720
1721
1722
1723 // Make an obviously ill-formed global that should blow up compilation
1724 // if it survives.
1725 auto global = new llvm::GlobalVariable(CGM.getModule(), CGM.Int8Ty, true,
1726 llvm::GlobalValue::PrivateLinkage,
1727 /*init*/ nullptr,
1728 /*name*/ "",
1729 /*before*/ nullptr,
1730 llvm::GlobalVariable::NotThreadLocal,
1731 CGM.getContext().getTargetAddressSpace(AS: DestAddressSpace));
1732
1733 PlaceholderAddresses.push_back(Elt: std::make_pair(x: nullptr, y&: global));
1734
1735 return global;
1736}
1737
1738void ConstantEmitter::registerCurrentAddrPrivate(llvm::Constant *signal,
1739 llvm::GlobalValue *placeholder) {
1740 assert(!PlaceholderAddresses.empty());
1741 assert(PlaceholderAddresses.back().first == nullptr);
1742 assert(PlaceholderAddresses.back().second == placeholder);
1743 PlaceholderAddresses.back().first = signal;
1744}
1745
1746namespace {
1747 struct ReplacePlaceholders {
1748 CodeGenModule &CGM;
1749
1750 /// The base address of the global.
1751 llvm::Constant *Base;
1752 llvm::Type *BaseValueTy = nullptr;
1753
1754 /// The placeholder addresses that were registered during emission.
1755 llvm::DenseMap<llvm::Constant*, llvm::GlobalVariable*> PlaceholderAddresses;
1756
1757 /// The locations of the placeholder signals.
1758 llvm::DenseMap<llvm::GlobalVariable*, llvm::Constant*> Locations;
1759
1760 /// The current index stack. We use a simple unsigned stack because
1761 /// we assume that placeholders will be relatively sparse in the
1762 /// initializer, but we cache the index values we find just in case.
1763 llvm::SmallVector<unsigned, 8> Indices;
1764 llvm::SmallVector<llvm::Constant*, 8> IndexValues;
1765
1766 ReplacePlaceholders(CodeGenModule &CGM, llvm::Constant *base,
1767 ArrayRef<std::pair<llvm::Constant*,
1768 llvm::GlobalVariable*>> addresses)
1769 : CGM(CGM), Base(base),
1770 PlaceholderAddresses(addresses.begin(), addresses.end()) {
1771 }
1772
1773 void replaceInInitializer(llvm::Constant *init) {
1774 // Remember the type of the top-most initializer.
1775 BaseValueTy = init->getType();
1776
1777 // Initialize the stack.
1778 Indices.push_back(Elt: 0);
1779 IndexValues.push_back(Elt: nullptr);
1780
1781 // Recurse into the initializer.
1782 findLocations(init);
1783
1784 // Check invariants.
1785 assert(IndexValues.size() == Indices.size() && "mismatch");
1786 assert(Indices.size() == 1 && "didn't pop all indices");
1787
1788 // Do the replacement; this basically invalidates 'init'.
1789 assert(Locations.size() == PlaceholderAddresses.size() &&
1790 "missed a placeholder?");
1791
1792 // We're iterating over a hashtable, so this would be a source of
1793 // non-determinism in compiler output *except* that we're just
1794 // messing around with llvm::Constant structures, which never itself
1795 // does anything that should be visible in compiler output.
1796 for (auto &entry : Locations) {
1797 assert(entry.first->getName() == "" && "not a placeholder!");
1798 entry.first->replaceAllUsesWith(V: entry.second);
1799 entry.first->eraseFromParent();
1800 }
1801 }
1802
1803 private:
1804 void findLocations(llvm::Constant *init) {
1805 // Recurse into aggregates.
1806 if (auto agg = dyn_cast<llvm::ConstantAggregate>(Val: init)) {
1807 for (unsigned i = 0, e = agg->getNumOperands(); i != e; ++i) {
1808 Indices.push_back(Elt: i);
1809 IndexValues.push_back(Elt: nullptr);
1810
1811 findLocations(init: agg->getOperand(i_nocapture: i));
1812
1813 IndexValues.pop_back();
1814 Indices.pop_back();
1815 }
1816 return;
1817 }
1818
1819 // Otherwise, check for registered constants.
1820 while (true) {
1821 auto it = PlaceholderAddresses.find(Val: init);
1822 if (it != PlaceholderAddresses.end()) {
1823 setLocation(it->second);
1824 break;
1825 }
1826
1827 // Look through bitcasts or other expressions.
1828 if (auto expr = dyn_cast<llvm::ConstantExpr>(Val: init)) {
1829 init = expr->getOperand(i_nocapture: 0);
1830 } else {
1831 break;
1832 }
1833 }
1834 }
1835
1836 void setLocation(llvm::GlobalVariable *placeholder) {
1837 assert(!Locations.contains(placeholder) &&
1838 "already found location for placeholder!");
1839
1840 // Lazily fill in IndexValues with the values from Indices.
1841 // We do this in reverse because we should always have a strict
1842 // prefix of indices from the start.
1843 assert(Indices.size() == IndexValues.size());
1844 for (size_t i = Indices.size() - 1; i != size_t(-1); --i) {
1845 if (IndexValues[i]) {
1846#ifndef NDEBUG
1847 for (size_t j = 0; j != i + 1; ++j) {
1848 assert(IndexValues[j] &&
1849 isa<llvm::ConstantInt>(IndexValues[j]) &&
1850 cast<llvm::ConstantInt>(IndexValues[j])->getZExtValue()
1851 == Indices[j]);
1852 }
1853#endif
1854 break;
1855 }
1856
1857 IndexValues[i] = llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: Indices[i]);
1858 }
1859
1860 llvm::Constant *location = llvm::ConstantExpr::getInBoundsGetElementPtr(
1861 Ty: BaseValueTy, C: Base, IdxList: IndexValues);
1862
1863 Locations.insert(KV: {placeholder, location});
1864 }
1865 };
1866}
1867
1868void ConstantEmitter::finalize(llvm::GlobalVariable *global) {
1869 assert(InitializedNonAbstract &&
1870 "finalizing emitter that was used for abstract emission?");
1871 assert(!Finalized && "finalizing emitter multiple times");
1872 assert(global->getInitializer());
1873
1874 // Note that we might also be Failed.
1875 Finalized = true;
1876
1877 if (!PlaceholderAddresses.empty()) {
1878 ReplacePlaceholders(CGM, global, PlaceholderAddresses)
1879 .replaceInInitializer(init: global->getInitializer());
1880 PlaceholderAddresses.clear(); // satisfy
1881 }
1882}
1883
1884ConstantEmitter::~ConstantEmitter() {
1885 assert((!InitializedNonAbstract || Finalized || Failed) &&
1886 "not finalized after being initialized for non-abstract emission");
1887 assert(PlaceholderAddresses.empty() && "unhandled placeholders");
1888}
1889
1890static QualType getNonMemoryType(CodeGenModule &CGM, QualType type) {
1891 if (auto AT = type->getAs<AtomicType>()) {
1892 return CGM.getContext().getQualifiedType(T: AT->getValueType(),
1893 Qs: type.getQualifiers());
1894 }
1895 return type;
1896}
1897
1898llvm::Constant *ConstantEmitter::tryEmitPrivateForVarInit(const VarDecl &D) {
1899 // Make a quick check if variable can be default NULL initialized
1900 // and avoid going through rest of code which may do, for c++11,
1901 // initialization of memory to all NULLs.
1902 if (!D.hasLocalStorage()) {
1903 QualType Ty = CGM.getContext().getBaseElementType(QT: D.getType());
1904 if (Ty->isRecordType())
1905 if (const CXXConstructExpr *E =
1906 dyn_cast_or_null<CXXConstructExpr>(Val: D.getInit())) {
1907 const CXXConstructorDecl *CD = E->getConstructor();
1908 if (CD->isTrivial() && CD->isDefaultConstructor())
1909 return CGM.EmitNullConstant(T: D.getType());
1910 }
1911 }
1912 InConstantContext = D.hasConstantInitialization();
1913
1914 QualType destType = D.getType();
1915 const Expr *E = D.getInit();
1916 assert(E && "No initializer to emit");
1917
1918 if (!destType->isReferenceType()) {
1919 QualType nonMemoryDestType = getNonMemoryType(CGM, type: destType);
1920 if (llvm::Constant *C = ConstExprEmitter(*this).Visit(S: E, P: nonMemoryDestType))
1921 return emitForMemory(C, T: destType);
1922 }
1923
1924 // Try to emit the initializer. Note that this can allow some things that
1925 // are not allowed by tryEmitPrivateForMemory alone.
1926 if (const APValue *value = D.evaluateValue()) {
1927 assert(!value->allowConstexprUnknown() &&
1928 "Constexpr unknown values are not allowed in CodeGen");
1929 return tryEmitPrivateForMemory(value: *value, T: destType);
1930 }
1931
1932 return nullptr;
1933}
1934
1935llvm::Constant *
1936ConstantEmitter::tryEmitAbstractForMemory(const Expr *E, QualType destType) {
1937 auto nonMemoryDestType = getNonMemoryType(CGM, type: destType);
1938 auto C = tryEmitAbstract(E, destType: nonMemoryDestType);
1939 return (C ? emitForMemory(C, T: destType) : nullptr);
1940}
1941
1942llvm::Constant *
1943ConstantEmitter::tryEmitAbstractForMemory(const APValue &value,
1944 QualType destType) {
1945 auto nonMemoryDestType = getNonMemoryType(CGM, type: destType);
1946 auto C = tryEmitAbstract(value, destType: nonMemoryDestType);
1947 return (C ? emitForMemory(C, T: destType) : nullptr);
1948}
1949
1950llvm::Constant *ConstantEmitter::tryEmitPrivateForMemory(const Expr *E,
1951 QualType destType) {
1952 auto nonMemoryDestType = getNonMemoryType(CGM, type: destType);
1953 llvm::Constant *C = tryEmitPrivate(E, T: nonMemoryDestType);
1954 return (C ? emitForMemory(C, T: destType) : nullptr);
1955}
1956
1957llvm::Constant *ConstantEmitter::tryEmitPrivateForMemory(const APValue &value,
1958 QualType destType) {
1959 auto nonMemoryDestType = getNonMemoryType(CGM, type: destType);
1960 auto C = tryEmitPrivate(value, T: nonMemoryDestType);
1961 return (C ? emitForMemory(C, T: destType) : nullptr);
1962}
1963
1964/// Try to emit a constant signed pointer, given a raw pointer and the
1965/// destination ptrauth qualifier.
1966///
1967/// This can fail if the qualifier needs address discrimination and the
1968/// emitter is in an abstract mode.
1969llvm::Constant *
1970ConstantEmitter::tryEmitConstantSignedPointer(llvm::Constant *UnsignedPointer,
1971 PointerAuthQualifier Schema) {
1972 assert(Schema && "applying trivial ptrauth schema");
1973
1974 if (Schema.hasKeyNone())
1975 return UnsignedPointer;
1976
1977 unsigned Key = Schema.getKey();
1978
1979 // Create an address placeholder if we're using address discrimination.
1980 llvm::GlobalValue *StorageAddress = nullptr;
1981 if (Schema.isAddressDiscriminated()) {
1982 // We can't do this if the emitter is in an abstract state.
1983 if (isAbstract())
1984 return nullptr;
1985
1986 StorageAddress = getCurrentAddrPrivate();
1987 }
1988
1989 llvm::ConstantInt *Discriminator =
1990 llvm::ConstantInt::get(Ty: CGM.IntPtrTy, V: Schema.getExtraDiscriminator());
1991
1992 llvm::Constant *SignedPointer = CGM.getConstantSignedPointer(
1993 Pointer: UnsignedPointer, Key, StorageAddress, OtherDiscriminator: Discriminator);
1994
1995 if (Schema.isAddressDiscriminated())
1996 registerCurrentAddrPrivate(signal: SignedPointer, placeholder: StorageAddress);
1997
1998 return SignedPointer;
1999}
2000
2001llvm::Constant *ConstantEmitter::emitForMemory(CodeGenModule &CGM,
2002 llvm::Constant *C,
2003 QualType destType) {
2004 // For an _Atomic-qualified constant, we may need to add tail padding.
2005 if (auto AT = destType->getAs<AtomicType>()) {
2006 QualType destValueType = AT->getValueType();
2007 C = emitForMemory(CGM, C, destType: destValueType);
2008
2009 uint64_t innerSize = CGM.getContext().getTypeSize(T: destValueType);
2010 uint64_t outerSize = CGM.getContext().getTypeSize(T: destType);
2011 if (innerSize == outerSize)
2012 return C;
2013
2014 assert(innerSize < outerSize && "emitted over-large constant for atomic");
2015 llvm::Constant *elts[] = {
2016 C,
2017 llvm::ConstantAggregateZero::get(
2018 Ty: llvm::ArrayType::get(ElementType: CGM.Int8Ty, NumElements: (outerSize - innerSize) / 8))
2019 };
2020 return llvm::ConstantStruct::getAnon(V: elts);
2021 }
2022
2023 // Zero-extend bool.
2024 // In HLSL bool vectors are stored in memory as a vector of i32
2025 if ((C->getType()->isIntegerTy(BitWidth: 1) && !destType->isBitIntType()) ||
2026 (destType->isExtVectorBoolType() &&
2027 !destType->isPackedVectorBoolType(ctx: CGM.getContext()))) {
2028 llvm::Type *boolTy = CGM.getTypes().ConvertTypeForMem(T: destType);
2029 llvm::Constant *Res = llvm::ConstantFoldCastOperand(
2030 Opcode: llvm::Instruction::ZExt, C, DestTy: boolTy, DL: CGM.getDataLayout());
2031 assert(Res && "Constant folding must succeed");
2032 return Res;
2033 }
2034
2035 if (destType->isBitIntType()) {
2036 llvm::Type *MemTy = CGM.getTypes().ConvertTypeForMem(T: destType);
2037 if (C->getType() != MemTy) {
2038 ConstantAggregateBuilder Builder(CGM);
2039 llvm::Type *LoadStoreTy =
2040 CGM.getTypes().convertTypeForLoadStore(T: destType);
2041 // ptrtoint/inttoptr should not involve _BitInt in constant expressions,
2042 // so casting to ConstantInt is safe here.
2043 auto *CI = cast<llvm::ConstantInt>(Val: C);
2044 llvm::Constant *Res = llvm::ConstantFoldCastOperand(
2045 Opcode: destType->isSignedIntegerOrEnumerationType()
2046 ? llvm::Instruction::SExt
2047 : llvm::Instruction::ZExt,
2048 C: CI, DestTy: LoadStoreTy, DL: CGM.getDataLayout());
2049 if (CGM.getTypes().typeRequiresSplitIntoByteArray(ASTTy: destType,
2050 LLVMTy: C->getType())) {
2051 // Long _BitInt has array of bytes as in-memory type.
2052 // So, split constant into individual bytes.
2053 llvm::APInt Value = cast<llvm::ConstantInt>(Val: Res)->getValue();
2054 Builder.addBits(Bits: Value, /*OffsetInBits=*/0, /*AllowOverwrite=*/false);
2055 return Builder.build(DesiredTy: MemTy, /*AllowOversized*/ false);
2056 }
2057 return Res;
2058 }
2059 }
2060
2061 return C;
2062}
2063
2064llvm::Constant *ConstantEmitter::tryEmitPrivate(const Expr *E,
2065 QualType destType) {
2066 assert(!destType->isVoidType() && "can't emit a void constant");
2067
2068 if (!destType->isReferenceType())
2069 if (llvm::Constant *C = ConstExprEmitter(*this).Visit(S: E, P: destType))
2070 return C;
2071
2072 Expr::EvalResult Result;
2073
2074 bool Success = false;
2075
2076 if (destType->isReferenceType())
2077 Success = E->EvaluateAsLValue(Result, Ctx: CGM.getContext());
2078 else
2079 Success = E->EvaluateAsRValue(Result, Ctx: CGM.getContext(), InConstantContext);
2080
2081 if (Success && !Result.HasSideEffects)
2082 return tryEmitPrivate(value: Result.Val, T: destType);
2083
2084 return nullptr;
2085}
2086
2087llvm::Constant *CodeGenModule::getNullPointer(llvm::PointerType *T, QualType QT) {
2088 return getTargetCodeGenInfo().getNullPointer(CGM: *this, T, QT);
2089}
2090
2091namespace {
2092/// A struct which can be used to peephole certain kinds of finalization
2093/// that normally happen during l-value emission.
2094struct ConstantLValue {
2095 llvm::Constant *Value;
2096 bool HasOffsetApplied;
2097 bool HasDestPointerAuth;
2098
2099 /*implicit*/ ConstantLValue(llvm::Constant *value,
2100 bool hasOffsetApplied = false,
2101 bool hasDestPointerAuth = false)
2102 : Value(value), HasOffsetApplied(hasOffsetApplied),
2103 HasDestPointerAuth(hasDestPointerAuth) {}
2104
2105 /*implicit*/ ConstantLValue(ConstantAddress address)
2106 : ConstantLValue(address.getPointer()) {}
2107};
2108
2109/// A helper class for emitting constant l-values.
2110class ConstantLValueEmitter : public ConstStmtVisitor<ConstantLValueEmitter,
2111 ConstantLValue> {
2112 CodeGenModule &CGM;
2113 ConstantEmitter &Emitter;
2114 const APValue &Value;
2115 QualType DestType;
2116 bool EnablePtrAuthFunctionTypeDiscrimination;
2117
2118 // Befriend StmtVisitorBase so that we don't have to expose Visit*.
2119 friend StmtVisitorBase;
2120
2121public:
2122 ConstantLValueEmitter(ConstantEmitter &emitter, const APValue &value,
2123 QualType destType,
2124 bool EnablePtrAuthFunctionTypeDiscrimination = true)
2125 : CGM(emitter.CGM), Emitter(emitter), Value(value), DestType(destType),
2126 EnablePtrAuthFunctionTypeDiscrimination(
2127 EnablePtrAuthFunctionTypeDiscrimination) {}
2128
2129 llvm::Constant *tryEmit();
2130
2131private:
2132 llvm::Constant *tryEmitAbsolute(llvm::Type *destTy);
2133 ConstantLValue tryEmitBase(const APValue::LValueBase &base);
2134
2135 ConstantLValue VisitStmt(const Stmt *S) { return nullptr; }
2136 ConstantLValue VisitConstantExpr(const ConstantExpr *E);
2137 ConstantLValue VisitCompoundLiteralExpr(const CompoundLiteralExpr *E);
2138 ConstantLValue VisitStringLiteral(const StringLiteral *E);
2139 ConstantLValue VisitObjCBoxedExpr(const ObjCBoxedExpr *E);
2140 ConstantLValue VisitObjCEncodeExpr(const ObjCEncodeExpr *E);
2141 ConstantLValue VisitObjCStringLiteral(const ObjCStringLiteral *E);
2142 llvm::Constant *VisitObjCCollectionElement(const Expr *E);
2143 ConstantLValue VisitObjCArrayLiteral(const ObjCArrayLiteral *E);
2144 ConstantLValue VisitObjCDictionaryLiteral(const ObjCDictionaryLiteral *E);
2145 ConstantLValue VisitPredefinedExpr(const PredefinedExpr *E);
2146 ConstantLValue VisitAddrLabelExpr(const AddrLabelExpr *E);
2147 ConstantLValue VisitCallExpr(const CallExpr *E);
2148 ConstantLValue VisitBlockExpr(const BlockExpr *E);
2149 ConstantLValue VisitCXXTypeidExpr(const CXXTypeidExpr *E);
2150 ConstantLValue VisitMaterializeTemporaryExpr(
2151 const MaterializeTemporaryExpr *E);
2152
2153 ConstantLValue emitPointerAuthSignConstant(const CallExpr *E);
2154 llvm::Constant *emitPointerAuthPointer(const Expr *E);
2155 unsigned emitPointerAuthKey(const Expr *E);
2156 std::pair<llvm::Constant *, llvm::ConstantInt *>
2157 emitPointerAuthDiscriminator(const Expr *E);
2158
2159 bool hasNonZeroOffset() const {
2160 return !Value.getLValueOffset().isZero();
2161 }
2162
2163 /// Return the value offset.
2164 llvm::Constant *getOffset() {
2165 return llvm::ConstantInt::get(Ty: CGM.Int64Ty,
2166 V: Value.getLValueOffset().getQuantity());
2167 }
2168
2169 /// Apply the value offset to the given constant.
2170 llvm::Constant *applyOffset(llvm::Constant *C) {
2171 if (!hasNonZeroOffset())
2172 return C;
2173
2174 return llvm::ConstantExpr::getPtrAdd(Ptr: C, Offset: getOffset());
2175 }
2176};
2177
2178}
2179
2180llvm::Constant *ConstantLValueEmitter::tryEmit() {
2181 const APValue::LValueBase &base = Value.getLValueBase();
2182
2183 // The destination type should be a pointer or reference
2184 // type, but it might also be a cast thereof.
2185 //
2186 // FIXME: the chain of casts required should be reflected in the APValue.
2187 // We need this in order to correctly handle things like a ptrtoint of a
2188 // non-zero null pointer and addrspace casts that aren't trivially
2189 // represented in LLVM IR.
2190 auto destTy = CGM.getTypes().ConvertTypeForMem(T: DestType);
2191 assert(isa<llvm::IntegerType>(destTy) || isa<llvm::PointerType>(destTy));
2192
2193 // If there's no base at all, this is a null or absolute pointer,
2194 // possibly cast back to an integer type.
2195 if (!base) {
2196 return tryEmitAbsolute(destTy);
2197 }
2198
2199 // Otherwise, try to emit the base.
2200 ConstantLValue result = tryEmitBase(base);
2201
2202 // If that failed, we're done.
2203 llvm::Constant *value = result.Value;
2204 if (!value) return nullptr;
2205
2206 // Apply the offset if necessary and not already done.
2207 if (!result.HasOffsetApplied) {
2208 value = applyOffset(C: value);
2209 }
2210
2211 // Apply pointer-auth signing from the destination type.
2212 if (PointerAuthQualifier PointerAuth = DestType.getPointerAuth();
2213 PointerAuth && !result.HasDestPointerAuth) {
2214 value = Emitter.tryEmitConstantSignedPointer(UnsignedPointer: value, Schema: PointerAuth);
2215 if (!value)
2216 return nullptr;
2217 }
2218
2219 // Convert to the appropriate type; this could be an lvalue for
2220 // an integer. FIXME: performAddrSpaceCast
2221 if (isa<llvm::PointerType>(Val: destTy))
2222 return llvm::ConstantExpr::getPointerCast(C: value, Ty: destTy);
2223
2224 return llvm::ConstantExpr::getPtrToInt(C: value, Ty: destTy);
2225}
2226
2227/// Try to emit an absolute l-value, such as a null pointer or an integer
2228/// bitcast to pointer type.
2229llvm::Constant *
2230ConstantLValueEmitter::tryEmitAbsolute(llvm::Type *destTy) {
2231 // If we're producing a pointer, this is easy.
2232 auto destPtrTy = cast<llvm::PointerType>(Val: destTy);
2233 if (Value.isNullPointer()) {
2234 // FIXME: integer offsets from non-zero null pointers.
2235 return CGM.getNullPointer(T: destPtrTy, QT: DestType);
2236 }
2237
2238 // Convert the integer to a pointer-sized integer before converting it
2239 // to a pointer.
2240 // FIXME: signedness depends on the original integer type.
2241 auto intptrTy = CGM.getDataLayout().getIntPtrType(destPtrTy);
2242 llvm::Constant *C;
2243 C = llvm::ConstantFoldIntegerCast(C: getOffset(), DestTy: intptrTy, /*isSigned*/ IsSigned: false,
2244 DL: CGM.getDataLayout());
2245 assert(C && "Must have folded, as Offset is a ConstantInt");
2246 C = llvm::ConstantExpr::getIntToPtr(C, Ty: destPtrTy);
2247 return C;
2248}
2249
2250ConstantLValue
2251ConstantLValueEmitter::tryEmitBase(const APValue::LValueBase &base) {
2252 // Handle values.
2253 if (const ValueDecl *D = base.dyn_cast<const ValueDecl*>()) {
2254 // The constant always points to the canonical declaration. We want to look
2255 // at properties of the most recent declaration at the point of emission.
2256 D = cast<ValueDecl>(Val: D->getMostRecentDecl());
2257
2258 if (D->hasAttr<WeakRefAttr>())
2259 return CGM.GetWeakRefReference(VD: D).getPointer();
2260
2261 auto PtrAuthSign = [&](llvm::Constant *C) {
2262 if (PointerAuthQualifier PointerAuth = DestType.getPointerAuth()) {
2263 C = applyOffset(C);
2264 C = Emitter.tryEmitConstantSignedPointer(UnsignedPointer: C, Schema: PointerAuth);
2265 return ConstantLValue(C, /*applied offset*/ true, /*signed*/ true);
2266 }
2267
2268 CGPointerAuthInfo AuthInfo;
2269
2270 if (EnablePtrAuthFunctionTypeDiscrimination)
2271 AuthInfo = CGM.getFunctionPointerAuthInfo(T: DestType);
2272
2273 if (AuthInfo) {
2274 if (hasNonZeroOffset())
2275 return ConstantLValue(nullptr);
2276
2277 C = applyOffset(C);
2278 C = CGM.getConstantSignedPointer(
2279 Pointer: C, Key: AuthInfo.getKey(), StorageAddress: nullptr,
2280 OtherDiscriminator: cast_or_null<llvm::ConstantInt>(Val: AuthInfo.getDiscriminator()));
2281 return ConstantLValue(C, /*applied offset*/ true, /*signed*/ true);
2282 }
2283
2284 return ConstantLValue(C);
2285 };
2286
2287 if (const auto *FD = dyn_cast<FunctionDecl>(Val: D)) {
2288 llvm::Constant *C = CGM.getRawFunctionPointer(GD: FD);
2289 if (FD->getType()->isCFIUncheckedCalleeFunctionType())
2290 C = llvm::NoCFIValue::get(GV: cast<llvm::GlobalValue>(Val: C));
2291 return PtrAuthSign(C);
2292 }
2293
2294 if (const auto *VD = dyn_cast<VarDecl>(Val: D)) {
2295 // We can never refer to a variable with local storage.
2296 if (!VD->hasLocalStorage()) {
2297 if (VD->isFileVarDecl() || VD->hasExternalStorage())
2298 return CGM.GetAddrOfGlobalVar(D: VD);
2299
2300 if (VD->isLocalVarDecl()) {
2301 return CGM.getOrCreateStaticVarDecl(
2302 D: *VD, Linkage: CGM.getLLVMLinkageVarDefinition(VD));
2303 }
2304 }
2305 }
2306
2307 if (const auto *GD = dyn_cast<MSGuidDecl>(Val: D))
2308 return CGM.GetAddrOfMSGuidDecl(GD);
2309
2310 if (const auto *GCD = dyn_cast<UnnamedGlobalConstantDecl>(Val: D))
2311 return CGM.GetAddrOfUnnamedGlobalConstantDecl(GCD);
2312
2313 if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(Val: D))
2314 return CGM.GetAddrOfTemplateParamObject(TPO);
2315
2316 return nullptr;
2317 }
2318
2319 // Handle typeid(T).
2320 if (TypeInfoLValue TI = base.dyn_cast<TypeInfoLValue>())
2321 return CGM.GetAddrOfRTTIDescriptor(Ty: QualType(TI.getType(), 0));
2322
2323 // Otherwise, it must be an expression.
2324 return Visit(S: base.get<const Expr*>());
2325}
2326
2327ConstantLValue
2328ConstantLValueEmitter::VisitConstantExpr(const ConstantExpr *E) {
2329 if (llvm::Constant *Result = Emitter.tryEmitConstantExpr(CE: E))
2330 return Result;
2331 return Visit(S: E->getSubExpr());
2332}
2333
2334ConstantLValue
2335ConstantLValueEmitter::VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) {
2336 ConstantEmitter CompoundLiteralEmitter(CGM, Emitter.CGF);
2337 CompoundLiteralEmitter.setInConstantContext(Emitter.isInConstantContext());
2338 return tryEmitGlobalCompoundLiteral(emitter&: CompoundLiteralEmitter, E);
2339}
2340
2341ConstantLValue
2342ConstantLValueEmitter::VisitStringLiteral(const StringLiteral *E) {
2343 return CGM.GetAddrOfConstantStringFromLiteral(S: E);
2344}
2345
2346ConstantLValue
2347ConstantLValueEmitter::VisitObjCEncodeExpr(const ObjCEncodeExpr *E) {
2348 return CGM.GetAddrOfConstantStringFromObjCEncode(E);
2349}
2350
2351static ConstantLValue emitConstantObjCStringLiteral(const StringLiteral *S,
2352 QualType T,
2353 CodeGenModule &CGM) {
2354 auto C = CGM.getObjCRuntime().GenerateConstantString(S);
2355 return C.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T));
2356}
2357
2358ConstantLValue
2359ConstantLValueEmitter::VisitObjCStringLiteral(const ObjCStringLiteral *E) {
2360 return emitConstantObjCStringLiteral(S: E->getString(), T: E->getType(), CGM);
2361}
2362
2363ConstantLValue
2364ConstantLValueEmitter::VisitObjCBoxedExpr(const ObjCBoxedExpr *E) {
2365 ASTContext &Context = CGM.getContext();
2366 CGObjCRuntime &Runtime = CGM.getObjCRuntime();
2367 const Expr *SubExpr = E->getSubExpr();
2368 const QualType &Ty = SubExpr->IgnoreParens()->getType();
2369
2370 assert(SubExpr->isEvaluatable(Context) &&
2371 "Non const NSNumber is being emitted as a constant");
2372
2373 if (const auto *SL = dyn_cast<StringLiteral>(Val: SubExpr->IgnoreParenCasts()))
2374 return emitConstantObjCStringLiteral(S: SL, T: E->getType(), CGM);
2375
2376 // Note `@YES` `@NO` need to be handled explicitly
2377 // to meet existing plist encoding / decoding expectations
2378 const bool IsBoolType =
2379 (Ty->isBooleanType() || NSAPI(Context).isObjCBOOLType(T: Ty));
2380 bool BoolValue = false;
2381 if (IsBoolType && SubExpr->EvaluateAsBooleanCondition(Result&: BoolValue, Ctx: Context)) {
2382 ConstantAddress C = Runtime.GenerateConstantNumber(Value: BoolValue, Ty);
2383 return C.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: E->getType()));
2384 }
2385
2386 Expr::EvalResult IntResult{};
2387 if (SubExpr->EvaluateAsInt(Result&: IntResult, Ctx: Context)) {
2388 ConstantAddress C =
2389 Runtime.GenerateConstantNumber(Value: IntResult.Val.getInt(), Ty);
2390 return C.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: E->getType()));
2391 }
2392
2393 llvm::APFloat FloatValue(0.0);
2394 if (SubExpr->EvaluateAsFloat(Result&: FloatValue, Ctx: Context)) {
2395 ConstantAddress C = Runtime.GenerateConstantNumber(Value: FloatValue, Ty);
2396 return C.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: E->getType()));
2397 }
2398
2399 llvm_unreachable("SubExpr is expected to be evaluated as a numeric type");
2400}
2401
2402llvm::Constant *
2403ConstantLValueEmitter::VisitObjCCollectionElement(const Expr *E) {
2404 auto CE = cast<CastExpr>(Val: E);
2405 const Expr *Elm = CE->getSubExpr();
2406 QualType DestTy = CE->getType();
2407
2408 assert(CE->getCastKind() == CK_BitCast &&
2409 "Expected a CK_BitCast type for valid items in constant objc "
2410 "collection literals");
2411
2412 llvm::Type *DstTy = CGM.getTypes().ConvertType(T: DestTy);
2413 ConstantLValue LV = Visit(S: Elm);
2414 llvm::Constant *ConstVal = cast<llvm::Constant>(Val: LV.Value);
2415 llvm::Constant *Val = llvm::ConstantExpr::getBitCast(C: ConstVal, Ty: DstTy);
2416 return Val;
2417}
2418
2419ConstantLValue
2420ConstantLValueEmitter::VisitObjCArrayLiteral(const ObjCArrayLiteral *E) {
2421 SmallVector<llvm::Constant *, 16> ObjectExpressions;
2422 uint64_t NumElements = E->getNumElements();
2423 ObjectExpressions.reserve(N: NumElements);
2424
2425 for (uint64_t i = 0; i < NumElements; i++) {
2426 llvm::Constant *Val = VisitObjCCollectionElement(E: E->getElement(Index: i));
2427 ObjectExpressions.push_back(Elt: Val);
2428 }
2429 ConstantAddress C =
2430 CGM.getObjCRuntime().GenerateConstantArray(Objects: ObjectExpressions);
2431 return C.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: E->getType()));
2432}
2433
2434ConstantLValue ConstantLValueEmitter::VisitObjCDictionaryLiteral(
2435 const ObjCDictionaryLiteral *E) {
2436 SmallVector<std::pair<llvm::Constant *, llvm::Constant *>, 16> KeysAndObjects;
2437 uint64_t NumElements = E->getNumElements();
2438 KeysAndObjects.reserve(N: NumElements);
2439
2440 for (uint64_t i = 0; i < NumElements; i++) {
2441 llvm::Constant *Key =
2442 VisitObjCCollectionElement(E: E->getKeyValueElement(Index: i).Key);
2443 llvm::Constant *Val =
2444 VisitObjCCollectionElement(E: E->getKeyValueElement(Index: i).Value);
2445 KeysAndObjects.push_back(Elt: {Key, Val});
2446 }
2447 ConstantAddress C =
2448 CGM.getObjCRuntime().GenerateConstantDictionary(E, KeysAndObjects);
2449 return C.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: E->getType()));
2450}
2451
2452ConstantLValue
2453ConstantLValueEmitter::VisitPredefinedExpr(const PredefinedExpr *E) {
2454 return CGM.GetAddrOfConstantStringFromLiteral(S: E->getFunctionName());
2455}
2456
2457ConstantLValue
2458ConstantLValueEmitter::VisitAddrLabelExpr(const AddrLabelExpr *E) {
2459 assert(Emitter.CGF && "Invalid address of label expression outside function");
2460 llvm::Constant *Ptr = Emitter.CGF->GetAddrOfLabel(L: E->getLabel());
2461 return Ptr;
2462}
2463
2464ConstantLValue
2465ConstantLValueEmitter::VisitCallExpr(const CallExpr *E) {
2466 unsigned builtin = E->getBuiltinCallee();
2467 if (builtin == Builtin::BI__builtin_function_start)
2468 return CGM.GetFunctionStart(
2469 Decl: E->getArg(Arg: 0)->getAsBuiltinConstantDeclRef(Context: CGM.getContext()));
2470
2471 if (builtin == Builtin::BI__builtin_ptrauth_sign_constant)
2472 return emitPointerAuthSignConstant(E);
2473
2474 if (builtin != Builtin::BI__builtin___CFStringMakeConstantString &&
2475 builtin != Builtin::BI__builtin___NSStringMakeConstantString)
2476 return nullptr;
2477
2478 const auto *Literal = cast<StringLiteral>(Val: E->getArg(Arg: 0)->IgnoreParenCasts());
2479 if (builtin == Builtin::BI__builtin___NSStringMakeConstantString) {
2480 return CGM.getObjCRuntime().GenerateConstantString(Literal);
2481 } else {
2482 // FIXME: need to deal with UCN conversion issues.
2483 return CGM.GetAddrOfConstantCFString(Literal);
2484 }
2485}
2486
2487ConstantLValue
2488ConstantLValueEmitter::emitPointerAuthSignConstant(const CallExpr *E) {
2489 llvm::Constant *UnsignedPointer = emitPointerAuthPointer(E: E->getArg(Arg: 0));
2490 unsigned Key = emitPointerAuthKey(E: E->getArg(Arg: 1));
2491 auto [StorageAddress, OtherDiscriminator] =
2492 emitPointerAuthDiscriminator(E: E->getArg(Arg: 2));
2493
2494 llvm::Constant *SignedPointer = CGM.getConstantSignedPointer(
2495 Pointer: UnsignedPointer, Key, StorageAddress, OtherDiscriminator);
2496 return SignedPointer;
2497}
2498
2499llvm::Constant *ConstantLValueEmitter::emitPointerAuthPointer(const Expr *E) {
2500 Expr::EvalResult Result;
2501 bool Succeeded = E->EvaluateAsRValue(Result, Ctx: CGM.getContext());
2502 assert(Succeeded);
2503 (void)Succeeded;
2504
2505 // The assertions here are all checked by Sema.
2506 assert(Result.Val.isLValue());
2507 if (isa<FunctionDecl>(Val: Result.Val.getLValueBase().get<const ValueDecl *>()))
2508 assert(Result.Val.getLValueOffset().isZero());
2509 return ConstantEmitter(CGM, Emitter.CGF)
2510 .emitAbstract(loc: E->getExprLoc(), value: Result.Val, destType: E->getType(), EnablePtrAuthFunctionTypeDiscrimination: false);
2511}
2512
2513unsigned ConstantLValueEmitter::emitPointerAuthKey(const Expr *E) {
2514 return E->EvaluateKnownConstInt(Ctx: CGM.getContext()).getZExtValue();
2515}
2516
2517std::pair<llvm::Constant *, llvm::ConstantInt *>
2518ConstantLValueEmitter::emitPointerAuthDiscriminator(const Expr *E) {
2519 E = E->IgnoreParens();
2520
2521 if (const auto *Call = dyn_cast<CallExpr>(Val: E)) {
2522 if (Call->getBuiltinCallee() ==
2523 Builtin::BI__builtin_ptrauth_blend_discriminator) {
2524 llvm::Constant *Pointer = ConstantEmitter(CGM).emitAbstract(
2525 E: Call->getArg(Arg: 0), destType: Call->getArg(Arg: 0)->getType());
2526 auto *Extra = cast<llvm::ConstantInt>(Val: ConstantEmitter(CGM).emitAbstract(
2527 E: Call->getArg(Arg: 1), destType: Call->getArg(Arg: 1)->getType()));
2528 return {Pointer, Extra};
2529 }
2530 }
2531
2532 llvm::Constant *Result = ConstantEmitter(CGM).emitAbstract(E, destType: E->getType());
2533 if (Result->getType()->isPointerTy())
2534 return {Result, nullptr};
2535 return {nullptr, cast<llvm::ConstantInt>(Val: Result)};
2536}
2537
2538ConstantLValue
2539ConstantLValueEmitter::VisitBlockExpr(const BlockExpr *E) {
2540 StringRef functionName;
2541 if (auto CGF = Emitter.CGF)
2542 functionName = CGF->CurFn->getName();
2543 else
2544 functionName = "global";
2545
2546 return CGM.GetAddrOfGlobalBlock(BE: E, Name: functionName);
2547}
2548
2549ConstantLValue
2550ConstantLValueEmitter::VisitCXXTypeidExpr(const CXXTypeidExpr *E) {
2551 QualType T;
2552 if (E->isTypeOperand())
2553 T = E->getTypeOperand(Context: CGM.getContext());
2554 else
2555 T = E->getExprOperand()->getType();
2556 return CGM.GetAddrOfRTTIDescriptor(Ty: T);
2557}
2558
2559ConstantLValue
2560ConstantLValueEmitter::VisitMaterializeTemporaryExpr(
2561 const MaterializeTemporaryExpr *E) {
2562 assert(E->getStorageDuration() == SD_Static);
2563 const Expr *Inner = E->getSubExpr()->skipRValueSubobjectAdjustments();
2564 return CGM.GetAddrOfGlobalTemporary(E, Inner);
2565}
2566
2567llvm::Constant *
2568ConstantEmitter::tryEmitPrivate(const APValue &Value, QualType DestType,
2569 bool EnablePtrAuthFunctionTypeDiscrimination) {
2570 switch (Value.getKind()) {
2571 case APValue::None:
2572 case APValue::Indeterminate:
2573 // Out-of-lifetime and indeterminate values can be modeled as 'undef'.
2574 return llvm::UndefValue::get(T: CGM.getTypes().ConvertType(T: DestType));
2575 case APValue::LValue:
2576 return ConstantLValueEmitter(*this, Value, DestType,
2577 EnablePtrAuthFunctionTypeDiscrimination)
2578 .tryEmit();
2579 case APValue::Int:
2580 if (PointerAuthQualifier PointerAuth = DestType.getPointerAuth();
2581 PointerAuth &&
2582 (PointerAuth.authenticatesNullValues() || Value.getInt() != 0))
2583 return nullptr;
2584 return llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: Value.getInt());
2585 case APValue::FixedPoint:
2586 return llvm::ConstantInt::get(Context&: CGM.getLLVMContext(),
2587 V: Value.getFixedPoint().getValue());
2588 case APValue::ComplexInt: {
2589 llvm::Constant *Complex[2];
2590
2591 Complex[0] = llvm::ConstantInt::get(Context&: CGM.getLLVMContext(),
2592 V: Value.getComplexIntReal());
2593 Complex[1] = llvm::ConstantInt::get(Context&: CGM.getLLVMContext(),
2594 V: Value.getComplexIntImag());
2595
2596 // FIXME: the target may want to specify that this is packed.
2597 llvm::StructType *STy =
2598 llvm::StructType::get(elt1: Complex[0]->getType(), elts: Complex[1]->getType());
2599 return llvm::ConstantStruct::get(T: STy, V: Complex);
2600 }
2601 case APValue::Float: {
2602 const llvm::APFloat &Init = Value.getFloat();
2603 if (&Init.getSemantics() == &llvm::APFloat::IEEEhalf() &&
2604 !CGM.getContext().getLangOpts().NativeHalfType &&
2605 CGM.getContext().getTargetInfo().useFP16ConversionIntrinsics())
2606 return llvm::ConstantInt::get(Context&: CGM.getLLVMContext(),
2607 V: Init.bitcastToAPInt());
2608 else
2609 return llvm::ConstantFP::get(Context&: CGM.getLLVMContext(), V: Init);
2610 }
2611 case APValue::ComplexFloat: {
2612 llvm::Constant *Complex[2];
2613
2614 Complex[0] = llvm::ConstantFP::get(Context&: CGM.getLLVMContext(),
2615 V: Value.getComplexFloatReal());
2616 Complex[1] = llvm::ConstantFP::get(Context&: CGM.getLLVMContext(),
2617 V: Value.getComplexFloatImag());
2618
2619 // FIXME: the target may want to specify that this is packed.
2620 llvm::StructType *STy =
2621 llvm::StructType::get(elt1: Complex[0]->getType(), elts: Complex[1]->getType());
2622 return llvm::ConstantStruct::get(T: STy, V: Complex);
2623 }
2624 case APValue::Vector: {
2625 unsigned NumElts = Value.getVectorLength();
2626 SmallVector<llvm::Constant *, 4> Inits(NumElts);
2627
2628 for (unsigned I = 0; I != NumElts; ++I) {
2629 const APValue &Elt = Value.getVectorElt(I);
2630 if (Elt.isInt())
2631 Inits[I] = llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: Elt.getInt());
2632 else if (Elt.isFloat())
2633 Inits[I] = llvm::ConstantFP::get(Context&: CGM.getLLVMContext(), V: Elt.getFloat());
2634 else if (Elt.isIndeterminate())
2635 Inits[I] = llvm::UndefValue::get(T: CGM.getTypes().ConvertType(
2636 T: DestType->castAs<VectorType>()->getElementType()));
2637 else
2638 llvm_unreachable("unsupported vector element type");
2639 }
2640 return llvm::ConstantVector::get(V: Inits);
2641 }
2642 case APValue::Matrix: {
2643 const auto *MT = DestType->castAs<ConstantMatrixType>();
2644 unsigned NumRows = Value.getMatrixNumRows();
2645 unsigned NumCols = Value.getMatrixNumColumns();
2646 unsigned NumElts = NumRows * NumCols;
2647 SmallVector<llvm::Constant *, 16> Inits(NumElts);
2648
2649 bool IsRowMajor = isMatrixRowMajor(LangOpts: CGM.getLangOpts(), T: DestType);
2650
2651 for (unsigned Row = 0; Row != NumRows; ++Row) {
2652 for (unsigned Col = 0; Col != NumCols; ++Col) {
2653 const APValue &Elt = Value.getMatrixElt(Row, Col);
2654 unsigned Idx = MT->getFlattenedIndex(Row, Column: Col, IsRowMajor);
2655 if (Elt.isInt())
2656 Inits[Idx] =
2657 llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: Elt.getInt());
2658 else if (Elt.isFloat())
2659 Inits[Idx] =
2660 llvm::ConstantFP::get(Context&: CGM.getLLVMContext(), V: Elt.getFloat());
2661 else if (Elt.isIndeterminate())
2662 Inits[Idx] = llvm::PoisonValue::get(
2663 T: CGM.getTypes().ConvertType(T: MT->getElementType()));
2664 else
2665 llvm_unreachable("unsupported matrix element type");
2666 }
2667 }
2668 return llvm::ConstantVector::get(V: Inits);
2669 }
2670 case APValue::AddrLabelDiff: {
2671 const AddrLabelExpr *LHSExpr = Value.getAddrLabelDiffLHS();
2672 const AddrLabelExpr *RHSExpr = Value.getAddrLabelDiffRHS();
2673 llvm::Constant *LHS = tryEmitPrivate(E: LHSExpr, destType: LHSExpr->getType());
2674 llvm::Constant *RHS = tryEmitPrivate(E: RHSExpr, destType: RHSExpr->getType());
2675 if (!LHS || !RHS) return nullptr;
2676
2677 // Compute difference
2678 llvm::Type *ResultType = CGM.getTypes().ConvertType(T: DestType);
2679 LHS = llvm::ConstantExpr::getPtrToInt(C: LHS, Ty: CGM.IntPtrTy);
2680 RHS = llvm::ConstantExpr::getPtrToInt(C: RHS, Ty: CGM.IntPtrTy);
2681 llvm::Constant *AddrLabelDiff = llvm::ConstantExpr::getSub(C1: LHS, C2: RHS);
2682
2683 // LLVM is a bit sensitive about the exact format of the
2684 // address-of-label difference; make sure to truncate after
2685 // the subtraction.
2686 return llvm::ConstantExpr::getTruncOrBitCast(C: AddrLabelDiff, Ty: ResultType);
2687 }
2688 case APValue::Struct:
2689 case APValue::Union:
2690 return ConstStructBuilder::BuildStruct(Emitter&: *this, Val: Value, ValTy: DestType);
2691 case APValue::Array: {
2692 const ArrayType *ArrayTy = CGM.getContext().getAsArrayType(T: DestType);
2693 unsigned NumElements = Value.getArraySize();
2694 unsigned NumInitElts = Value.getArrayInitializedElts();
2695
2696 // Emit array filler, if there is one.
2697 llvm::Constant *Filler = nullptr;
2698 if (Value.hasArrayFiller()) {
2699 Filler = tryEmitAbstractForMemory(value: Value.getArrayFiller(),
2700 destType: ArrayTy->getElementType());
2701 if (!Filler)
2702 return nullptr;
2703 }
2704
2705 // Emit initializer elements.
2706 SmallVector<llvm::Constant*, 16> Elts;
2707 if (Filler && Filler->isNullValue())
2708 Elts.reserve(N: NumInitElts + 1);
2709 else
2710 Elts.reserve(N: NumElements);
2711
2712 llvm::Type *CommonElementType = nullptr;
2713 for (unsigned I = 0; I < NumInitElts; ++I) {
2714 llvm::Constant *C = tryEmitPrivateForMemory(
2715 value: Value.getArrayInitializedElt(I), destType: ArrayTy->getElementType());
2716 if (!C) return nullptr;
2717
2718 if (I == 0)
2719 CommonElementType = C->getType();
2720 else if (C->getType() != CommonElementType)
2721 CommonElementType = nullptr;
2722 Elts.push_back(Elt: C);
2723 }
2724
2725 llvm::ArrayType *Desired =
2726 cast<llvm::ArrayType>(Val: CGM.getTypes().ConvertType(T: DestType));
2727
2728 // Fix the type of incomplete arrays if the initializer isn't empty.
2729 if (DestType->isIncompleteArrayType() && !Elts.empty())
2730 Desired = llvm::ArrayType::get(ElementType: Desired->getElementType(), NumElements: Elts.size());
2731
2732 return EmitArrayConstant(CGM, DesiredType: Desired, CommonElementType, ArrayBound: NumElements, Elements&: Elts,
2733 Filler);
2734 }
2735 case APValue::MemberPointer:
2736 return CGM.getCXXABI().EmitMemberPointer(MP: Value, MPT: DestType);
2737 }
2738 llvm_unreachable("Unknown APValue kind");
2739}
2740
2741llvm::GlobalVariable *CodeGenModule::getAddrOfConstantCompoundLiteralIfEmitted(
2742 const CompoundLiteralExpr *E) {
2743 return EmittedCompoundLiterals.lookup(Val: E);
2744}
2745
2746void CodeGenModule::setAddrOfConstantCompoundLiteral(
2747 const CompoundLiteralExpr *CLE, llvm::GlobalVariable *GV) {
2748 bool Ok = EmittedCompoundLiterals.insert(KV: std::make_pair(x&: CLE, y&: GV)).second;
2749 (void)Ok;
2750 assert(Ok && "CLE has already been emitted!");
2751}
2752
2753ConstantAddress
2754CodeGenModule::GetAddrOfConstantCompoundLiteral(const CompoundLiteralExpr *E) {
2755 assert(E->isFileScope() && "not a file-scope compound literal expr");
2756 ConstantEmitter emitter(*this);
2757 return tryEmitGlobalCompoundLiteral(emitter, E);
2758}
2759
2760llvm::Constant *
2761CodeGenModule::getMemberPointerConstant(const UnaryOperator *uo) {
2762 // Member pointer constants always have a very particular form.
2763 const MemberPointerType *type = cast<MemberPointerType>(Val: uo->getType());
2764 const ValueDecl *decl = cast<DeclRefExpr>(Val: uo->getSubExpr())->getDecl();
2765
2766 // A member function pointer.
2767 if (const CXXMethodDecl *method = dyn_cast<CXXMethodDecl>(Val: decl))
2768 return getCXXABI().EmitMemberFunctionPointer(MD: method);
2769
2770 // Otherwise, a member data pointer.
2771 getContext().recordMemberDataPointerEvaluation(VD: decl);
2772 uint64_t fieldOffset = getContext().getFieldOffset(FD: decl);
2773 CharUnits chars = getContext().toCharUnitsFromBits(BitSize: (int64_t) fieldOffset);
2774 return getCXXABI().EmitMemberDataPointer(MPT: type, offset: chars);
2775}
2776
2777static llvm::Constant *EmitNullConstantForBase(CodeGenModule &CGM,
2778 llvm::Type *baseType,
2779 const CXXRecordDecl *base);
2780
2781static llvm::Constant *EmitNullConstant(CodeGenModule &CGM,
2782 const RecordDecl *record,
2783 bool asCompleteObject) {
2784 const CGRecordLayout &layout = CGM.getTypes().getCGRecordLayout(record);
2785 llvm::StructType *structure =
2786 (asCompleteObject ? layout.getLLVMType()
2787 : layout.getBaseSubobjectLLVMType());
2788
2789 unsigned numElements = structure->getNumElements();
2790 std::vector<llvm::Constant *> elements(numElements);
2791
2792 auto CXXR = dyn_cast<CXXRecordDecl>(Val: record);
2793 // Fill in all the bases.
2794 if (CXXR) {
2795 for (const auto &I : CXXR->bases()) {
2796 if (I.isVirtual()) {
2797 // Ignore virtual bases; if we're laying out for a complete
2798 // object, we'll lay these out later.
2799 continue;
2800 }
2801
2802 const auto *base = I.getType()->castAsCXXRecordDecl();
2803 // Ignore empty bases.
2804 if (isEmptyRecordForLayout(Context: CGM.getContext(), T: I.getType()) ||
2805 CGM.getContext()
2806 .getASTRecordLayout(D: base)
2807 .getNonVirtualSize()
2808 .isZero())
2809 continue;
2810
2811 unsigned fieldIndex = layout.getNonVirtualBaseLLVMFieldNo(RD: base);
2812 llvm::Type *baseType = structure->getElementType(N: fieldIndex);
2813 elements[fieldIndex] = EmitNullConstantForBase(CGM, baseType, base);
2814 }
2815 }
2816
2817 // Fill in all the fields.
2818 for (const auto *Field : record->fields()) {
2819 // Fill in non-bitfields. (Bitfields always use a zero pattern, which we
2820 // will fill in later.)
2821 if (!Field->isBitField() &&
2822 !isEmptyFieldForLayout(Context: CGM.getContext(), FD: Field)) {
2823 unsigned fieldIndex = layout.getLLVMFieldNo(FD: Field);
2824 elements[fieldIndex] = CGM.EmitNullConstant(T: Field->getType());
2825 }
2826
2827 // For unions, stop after the first named field.
2828 if (record->isUnion()) {
2829 if (Field->getIdentifier())
2830 break;
2831 if (const auto *FieldRD = Field->getType()->getAsRecordDecl())
2832 if (FieldRD->findFirstNamedDataMember())
2833 break;
2834 }
2835 }
2836
2837 // Fill in the virtual bases, if we're working with the complete object.
2838 if (CXXR && asCompleteObject) {
2839 for (const auto &I : CXXR->vbases()) {
2840 const auto *base = I.getType()->castAsCXXRecordDecl();
2841 // Ignore empty bases.
2842 if (isEmptyRecordForLayout(Context: CGM.getContext(), T: I.getType()))
2843 continue;
2844
2845 unsigned fieldIndex = layout.getVirtualBaseIndex(base);
2846
2847 // We might have already laid this field out.
2848 if (elements[fieldIndex]) continue;
2849
2850 llvm::Type *baseType = structure->getElementType(N: fieldIndex);
2851 elements[fieldIndex] = EmitNullConstantForBase(CGM, baseType, base);
2852 }
2853 }
2854
2855 // Now go through all other fields and zero them out.
2856 for (unsigned i = 0; i != numElements; ++i) {
2857 if (!elements[i])
2858 elements[i] = llvm::Constant::getNullValue(Ty: structure->getElementType(N: i));
2859 }
2860
2861 return llvm::ConstantStruct::get(T: structure, V: elements);
2862}
2863
2864/// Emit the null constant for a base subobject.
2865static llvm::Constant *EmitNullConstantForBase(CodeGenModule &CGM,
2866 llvm::Type *baseType,
2867 const CXXRecordDecl *base) {
2868 const CGRecordLayout &baseLayout = CGM.getTypes().getCGRecordLayout(base);
2869
2870 // Just zero out bases that don't have any pointer to data members.
2871 if (baseLayout.isZeroInitializableAsBase())
2872 return llvm::Constant::getNullValue(Ty: baseType);
2873
2874 // Otherwise, we can just use its null constant.
2875 return EmitNullConstant(CGM, record: base, /*asCompleteObject=*/false);
2876}
2877
2878llvm::Constant *ConstantEmitter::emitNullForMemory(CodeGenModule &CGM,
2879 QualType T) {
2880 return emitForMemory(CGM, C: CGM.EmitNullConstant(T), destType: T);
2881}
2882
2883llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) {
2884 if (T->getAs<PointerType>()) {
2885 llvm::Type *LT = getTypes().ConvertTypeForMem(T);
2886 if (auto *PT = dyn_cast<llvm::PointerType>(Val: LT))
2887 return getNullPointer(T: PT, QT: T);
2888 // Some pointer types do not lower to an LLVM pointer (e.g. a WebAssembly
2889 // funcref, which is an opaque reference type). Use the type's zero value.
2890 return llvm::Constant::getNullValue(Ty: LT);
2891 }
2892
2893 if (getTypes().isZeroInitializable(T))
2894 return llvm::Constant::getNullValue(Ty: getTypes().ConvertTypeForMem(T));
2895
2896 if (const ConstantArrayType *CAT = Context.getAsConstantArrayType(T)) {
2897 llvm::ArrayType *ATy =
2898 cast<llvm::ArrayType>(Val: getTypes().ConvertTypeForMem(T));
2899
2900 QualType ElementTy = CAT->getElementType();
2901
2902 llvm::Constant *Element =
2903 ConstantEmitter::emitNullForMemory(CGM&: *this, T: ElementTy);
2904 unsigned NumElements = CAT->getZExtSize();
2905 SmallVector<llvm::Constant *, 8> Array(NumElements, Element);
2906 return llvm::ConstantArray::get(T: ATy, V: Array);
2907 }
2908
2909 if (const auto *RD = T->getAsRecordDecl())
2910 return ::EmitNullConstant(CGM&: *this, record: RD,
2911 /*asCompleteObject=*/true);
2912
2913 assert(T->isMemberDataPointerType() &&
2914 "Should only see pointers to data members here!");
2915
2916 return getCXXABI().EmitNullMemberPointer(MPT: T->castAs<MemberPointerType>());
2917}
2918
2919llvm::Constant *
2920CodeGenModule::EmitNullConstantForBase(const CXXRecordDecl *Record) {
2921 return ::EmitNullConstant(CGM&: *this, record: Record, asCompleteObject: false);
2922}
2923