1//===--- Descriptor.cpp - Types for the constexpr VM ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "Descriptor.h"
10#include "Boolean.h"
11#include "Floating.h"
12#include "FunctionPointer.h"
13#include "IntegralAP.h"
14#include "MemberPointer.h"
15#include "Pointer.h"
16#include "PrimType.h"
17#include "Record.h"
18
19using namespace clang;
20using namespace clang::interp;
21
22template <typename T>
23static void ctorTy(Block *, std::byte *Ptr, bool, bool, bool,
24 const Descriptor *) {
25 new (Ptr) T();
26}
27
28template <typename T>
29static void dtorTy(Block *, std::byte *Ptr, const Descriptor *) {
30 reinterpret_cast<T *>(Ptr)->~T();
31}
32
33template <typename T>
34static void moveTy(Block *, const std::byte *Src, std::byte *Dst,
35 const Descriptor *) {
36 const auto *SrcPtr = reinterpret_cast<const T *>(Src);
37 auto *DstPtr = reinterpret_cast<T *>(Dst);
38 new (DstPtr) T(std::move(*SrcPtr));
39}
40
41template <typename T>
42static void ctorArrayTy(Block *, std::byte *Ptr, bool, bool, bool,
43 const Descriptor *D) {
44 new (Ptr) InitMapPtr(std::nullopt);
45
46 Ptr += sizeof(InitMapPtr);
47 for (unsigned I = 0, NE = D->getNumElems(); I < NE; ++I) {
48 new (&reinterpret_cast<T *>(Ptr)[I]) T();
49 }
50}
51
52template <typename T>
53static void dtorArrayTy(Block *, std::byte *Ptr, const Descriptor *D) {
54 InitMapPtr &IMP = *reinterpret_cast<InitMapPtr *>(Ptr);
55
56 if (IMP)
57 IMP = std::nullopt;
58 Ptr += sizeof(InitMapPtr);
59 for (unsigned I = 0, NE = D->getNumElems(); I < NE; ++I) {
60 reinterpret_cast<T *>(Ptr)[I].~T();
61 }
62}
63
64template <typename T>
65static void moveArrayTy(Block *, const std::byte *Src, std::byte *Dst,
66 const Descriptor *D) {
67 // FIXME: Get rid of the const_cast.
68 InitMapPtr &SrcIMP =
69 *reinterpret_cast<InitMapPtr *>(const_cast<std::byte *>(Src));
70 if (SrcIMP) {
71 // We only ever invoke the moveFunc when moving block contents to a
72 // DeadBlock. DeadBlocks don't need InitMaps, so we destroy them here.
73 SrcIMP = std::nullopt;
74 }
75 Src += sizeof(InitMapPtr);
76 Dst += sizeof(InitMapPtr);
77 for (unsigned I = 0, NE = D->getNumElems(); I < NE; ++I) {
78 const auto *SrcPtr = &reinterpret_cast<const T *>(Src)[I];
79 auto *DstPtr = &reinterpret_cast<T *>(Dst)[I];
80 new (DstPtr) T(std::move(*SrcPtr));
81 }
82}
83
84static void ctorArrayDesc(Block *B, std::byte *Ptr, bool IsConst,
85 bool IsMutable, bool IsActive, const Descriptor *D) {
86 const unsigned NumElems = D->getNumElems();
87 const unsigned ElemSize =
88 D->ElemDesc->getAllocSize() + sizeof(InlineDescriptor);
89
90 unsigned ElemOffset = 0;
91 for (unsigned I = 0; I < NumElems; ++I, ElemOffset += ElemSize) {
92 auto *ElemPtr = Ptr + ElemOffset;
93 auto *Desc = reinterpret_cast<InlineDescriptor *>(ElemPtr);
94 auto *ElemLoc = reinterpret_cast<std::byte *>(Desc + 1);
95 auto *SD = D->ElemDesc;
96
97 Desc->Offset = ElemOffset + sizeof(InlineDescriptor);
98 Desc->Desc = SD;
99 Desc->IsInitialized = true;
100 Desc->IsBase = false;
101 Desc->IsActive = IsActive;
102 Desc->IsConst = IsConst || D->IsConst;
103 Desc->IsFieldMutable = IsMutable || D->IsMutable;
104 if (auto Fn = D->ElemDesc->CtorFn)
105 Fn(B, ElemLoc, Desc->IsConst, Desc->IsFieldMutable, IsActive,
106 D->ElemDesc);
107 }
108}
109
110static void dtorArrayDesc(Block *B, std::byte *Ptr, const Descriptor *D) {
111 const unsigned NumElems = D->getNumElems();
112 const unsigned ElemSize =
113 D->ElemDesc->getAllocSize() + sizeof(InlineDescriptor);
114
115 unsigned ElemOffset = 0;
116 for (unsigned I = 0; I < NumElems; ++I, ElemOffset += ElemSize) {
117 auto *ElemPtr = Ptr + ElemOffset;
118 auto *Desc = reinterpret_cast<InlineDescriptor *>(ElemPtr);
119 auto *ElemLoc = reinterpret_cast<std::byte *>(Desc + 1);
120 if (auto Fn = D->ElemDesc->DtorFn)
121 Fn(B, ElemLoc, D->ElemDesc);
122 }
123}
124
125static void moveArrayDesc(Block *B, const std::byte *Src, std::byte *Dst,
126 const Descriptor *D) {
127 const unsigned NumElems = D->getNumElems();
128 const unsigned ElemSize =
129 D->ElemDesc->getAllocSize() + sizeof(InlineDescriptor);
130
131 unsigned ElemOffset = 0;
132 for (unsigned I = 0; I < NumElems; ++I, ElemOffset += ElemSize) {
133 const auto *SrcPtr = Src + ElemOffset;
134 auto *DstPtr = Dst + ElemOffset;
135
136 const auto *SrcDesc = reinterpret_cast<const InlineDescriptor *>(SrcPtr);
137 const auto *SrcElemLoc = reinterpret_cast<const std::byte *>(SrcDesc + 1);
138 auto *DstDesc = reinterpret_cast<InlineDescriptor *>(DstPtr);
139 auto *DstElemLoc = reinterpret_cast<std::byte *>(DstDesc + 1);
140
141 *DstDesc = *SrcDesc;
142 if (auto Fn = D->ElemDesc->MoveFn)
143 Fn(B, SrcElemLoc, DstElemLoc, D->ElemDesc);
144 }
145}
146
147static void initField(Block *B, std::byte *Ptr, bool IsConst, bool IsMutable,
148 bool IsActive, bool IsUnion, const Descriptor *D,
149 unsigned FieldOffset) {
150 auto *Desc = reinterpret_cast<InlineDescriptor *>(Ptr + FieldOffset) - 1;
151 Desc->Offset = FieldOffset;
152 Desc->Desc = D;
153 Desc->IsInitialized = D->IsArray;
154 Desc->IsBase = false;
155 Desc->IsActive = IsActive && !IsUnion;
156 Desc->IsConst = IsConst || D->IsConst;
157 Desc->IsFieldMutable = IsMutable || D->IsMutable;
158
159 if (auto Fn = D->CtorFn)
160 Fn(B, Ptr + FieldOffset, Desc->IsConst, Desc->IsFieldMutable,
161 Desc->IsActive, D);
162}
163
164static void initBase(Block *B, std::byte *Ptr, bool IsConst, bool IsMutable,
165 bool IsActive, const Descriptor *D, unsigned FieldOffset,
166 bool IsVirtualBase) {
167 assert(D);
168 assert(D->ElemRecord);
169
170 bool IsUnion = D->ElemRecord->isUnion();
171 auto *Desc = reinterpret_cast<InlineDescriptor *>(Ptr + FieldOffset) - 1;
172 Desc->Offset = FieldOffset;
173 Desc->Desc = D;
174 Desc->IsInitialized = D->IsArray;
175 Desc->IsBase = true;
176 Desc->IsVirtualBase = IsVirtualBase;
177 Desc->IsActive = IsActive && !IsUnion;
178 Desc->IsConst = IsConst || D->IsConst;
179 Desc->IsFieldMutable = IsMutable || D->IsMutable;
180
181 for (const auto &V : D->ElemRecord->bases())
182 initBase(B, Ptr: Ptr + FieldOffset, IsConst, IsMutable, IsActive, D: V.Desc,
183 FieldOffset: V.Offset, IsVirtualBase: false);
184 for (const auto &F : D->ElemRecord->fields())
185 initField(B, Ptr: Ptr + FieldOffset, IsConst, IsMutable, IsActive, IsUnion,
186 D: F.Desc, FieldOffset: F.Offset);
187}
188
189static void ctorRecord(Block *B, std::byte *Ptr, bool IsConst, bool IsMutable,
190 bool IsActive, const Descriptor *D) {
191 for (const auto &V : D->ElemRecord->bases())
192 initBase(B, Ptr, IsConst, IsMutable, IsActive, D: V.Desc, FieldOffset: V.Offset, IsVirtualBase: false);
193 for (const auto &F : D->ElemRecord->fields())
194 initField(B, Ptr, IsConst, IsMutable, IsActive, IsUnion: D->ElemRecord->isUnion(), D: F.Desc, FieldOffset: F.Offset);
195 for (const auto &V : D->ElemRecord->virtual_bases())
196 initBase(B, Ptr, IsConst, IsMutable, IsActive, D: V.Desc, FieldOffset: V.Offset, IsVirtualBase: true);
197}
198
199static void destroyField(Block *B, std::byte *Ptr, const Descriptor *D,
200 unsigned FieldOffset) {
201 if (auto Fn = D->DtorFn)
202 Fn(B, Ptr + FieldOffset, D);
203}
204
205static void destroyBase(Block *B, std::byte *Ptr, const Descriptor *D,
206 unsigned FieldOffset) {
207 assert(D);
208 assert(D->ElemRecord);
209
210 for (const auto &V : D->ElemRecord->bases())
211 destroyBase(B, Ptr: Ptr + FieldOffset, D: V.Desc, FieldOffset: V.Offset);
212 for (const auto &F : D->ElemRecord->fields())
213 destroyField(B, Ptr: Ptr + FieldOffset, D: F.Desc, FieldOffset: F.Offset);
214}
215
216static void dtorRecord(Block *B, std::byte *Ptr, const Descriptor *D) {
217 for (const auto &F : D->ElemRecord->bases())
218 destroyBase(B, Ptr, D: F.Desc, FieldOffset: F.Offset);
219 for (const auto &F : D->ElemRecord->fields())
220 destroyField(B, Ptr, D: F.Desc, FieldOffset: F.Offset);
221 for (const auto &F : D->ElemRecord->virtual_bases())
222 destroyBase(B, Ptr, D: F.Desc, FieldOffset: F.Offset);
223}
224
225static void moveRecord(Block *B, const std::byte *Src, std::byte *Dst,
226 const Descriptor *D) {
227 for (const auto &F : D->ElemRecord->fields()) {
228 auto FieldOff = F.Offset;
229 auto *FieldDesc = F.Desc;
230
231 if (auto Fn = FieldDesc->MoveFn)
232 Fn(B, Src + FieldOff, Dst + FieldOff, FieldDesc);
233 }
234}
235
236static BlockCtorFn getCtorPrim(PrimType Type) {
237 // Floating types are special. They are primitives, but need their
238 // constructor called.
239 if (Type == PT_Float)
240 return ctorTy<PrimConv<PT_Float>::T>;
241 if (Type == PT_IntAP)
242 return ctorTy<PrimConv<PT_IntAP>::T>;
243 if (Type == PT_IntAPS)
244 return ctorTy<PrimConv<PT_IntAPS>::T>;
245 if (Type == PT_MemberPtr)
246 return ctorTy<PrimConv<PT_MemberPtr>::T>;
247
248 COMPOSITE_TYPE_SWITCH(Type, return ctorTy<T>, return nullptr);
249}
250
251static BlockDtorFn getDtorPrim(PrimType Type) {
252 // Floating types are special. They are primitives, but need their
253 // destructor called, since they might allocate memory.
254 if (Type == PT_Float)
255 return dtorTy<PrimConv<PT_Float>::T>;
256 if (Type == PT_IntAP)
257 return dtorTy<PrimConv<PT_IntAP>::T>;
258 if (Type == PT_IntAPS)
259 return dtorTy<PrimConv<PT_IntAPS>::T>;
260 if (Type == PT_MemberPtr)
261 return dtorTy<PrimConv<PT_MemberPtr>::T>;
262
263 COMPOSITE_TYPE_SWITCH(Type, return dtorTy<T>, return nullptr);
264}
265
266static BlockMoveFn getMovePrim(PrimType Type) {
267 COMPOSITE_TYPE_SWITCH(Type, return moveTy<T>, return nullptr);
268}
269
270static BlockCtorFn getCtorArrayPrim(PrimType Type) {
271 TYPE_SWITCH(Type, return ctorArrayTy<T>);
272 llvm_unreachable("unknown Expr");
273}
274
275static BlockDtorFn getDtorArrayPrim(PrimType Type) {
276 TYPE_SWITCH(Type, return dtorArrayTy<T>);
277 llvm_unreachable("unknown Expr");
278}
279
280static BlockMoveFn getMoveArrayPrim(PrimType Type) {
281 TYPE_SWITCH(Type, return moveArrayTy<T>);
282 llvm_unreachable("unknown Expr");
283}
284
285/// Primitives.
286Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD,
287 bool IsConst, bool IsTemporary, bool IsMutable)
288 : Source(D), ElemSize(primSize(Type)), Size(ElemSize),
289 MDSize(MD.value_or(u: 0)), AllocSize(align(Size: Size + MDSize)), PrimT(Type),
290 IsConst(IsConst), IsMutable(IsMutable), IsTemporary(IsTemporary),
291 CtorFn(getCtorPrim(Type)), DtorFn(getDtorPrim(Type)),
292 MoveFn(getMovePrim(Type)) {
293 assert(AllocSize >= Size);
294 assert(Source && "Missing source");
295}
296
297/// Primitive arrays.
298Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD,
299 size_t NumElems, bool IsConst, bool IsTemporary,
300 bool IsMutable)
301 : Source(D), ElemSize(primSize(Type)), Size(ElemSize * NumElems),
302 MDSize(MD.value_or(u: 0)),
303 AllocSize(align(Size: MDSize) + align(Size) + sizeof(InitMapPtr)), PrimT(Type),
304 IsConst(IsConst), IsMutable(IsMutable), IsTemporary(IsTemporary),
305 IsArray(true), CtorFn(getCtorArrayPrim(Type)),
306 DtorFn(getDtorArrayPrim(Type)), MoveFn(getMoveArrayPrim(Type)) {
307 assert(Source && "Missing source");
308 assert(NumElems <= (MaxArrayElemBytes / ElemSize));
309}
310
311/// Primitive unknown-size arrays.
312Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD,
313 bool IsTemporary, UnknownSize)
314 : Source(D), ElemSize(primSize(Type)), Size(UnknownSizeMark),
315 MDSize(MD.value_or(u: 0)),
316 AllocSize(MDSize + sizeof(InitMapPtr) + alignof(void *)), IsConst(true),
317 IsMutable(false), IsTemporary(IsTemporary), IsArray(true),
318 CtorFn(getCtorArrayPrim(Type)), DtorFn(getDtorArrayPrim(Type)),
319 MoveFn(getMoveArrayPrim(Type)) {
320 assert(Source && "Missing source");
321}
322
323/// Arrays of composite elements.
324Descriptor::Descriptor(const DeclTy &D, const Descriptor *Elem, MetadataSize MD,
325 unsigned NumElems, bool IsConst, bool IsTemporary,
326 bool IsMutable)
327 : Source(D), ElemSize(Elem->getAllocSize() + sizeof(InlineDescriptor)),
328 Size(ElemSize * NumElems), MDSize(MD.value_or(u: 0)),
329 AllocSize(std::max<size_t>(a: alignof(void *), b: Size) + MDSize),
330 ElemDesc(Elem), IsConst(IsConst), IsMutable(IsMutable),
331 IsTemporary(IsTemporary), IsArray(true), CtorFn(ctorArrayDesc),
332 DtorFn(dtorArrayDesc), MoveFn(moveArrayDesc) {
333 assert(Source && "Missing source");
334}
335
336/// Unknown-size arrays of composite elements.
337Descriptor::Descriptor(const DeclTy &D, const Descriptor *Elem, MetadataSize MD,
338 bool IsTemporary, UnknownSize)
339 : Source(D), ElemSize(Elem->getAllocSize() + sizeof(InlineDescriptor)),
340 Size(UnknownSizeMark), MDSize(MD.value_or(u: 0)),
341 AllocSize(MDSize + alignof(void *)), ElemDesc(Elem), IsConst(true),
342 IsMutable(false), IsTemporary(IsTemporary), IsArray(true),
343 CtorFn(ctorArrayDesc), DtorFn(dtorArrayDesc), MoveFn(moveArrayDesc) {
344 assert(Source && "Missing source");
345}
346
347/// Composite records.
348Descriptor::Descriptor(const DeclTy &D, const Record *R, MetadataSize MD,
349 bool IsConst, bool IsTemporary, bool IsMutable)
350 : Source(D), ElemSize(std::max<size_t>(a: alignof(void *), b: R->getFullSize())),
351 Size(ElemSize), MDSize(MD.value_or(u: 0)), AllocSize(Size + MDSize),
352 ElemRecord(R), IsConst(IsConst), IsMutable(IsMutable),
353 IsTemporary(IsTemporary), CtorFn(ctorRecord), DtorFn(dtorRecord),
354 MoveFn(moveRecord) {
355 assert(Source && "Missing source");
356}
357
358/// Dummy.
359Descriptor::Descriptor(const DeclTy &D)
360 : Source(D), ElemSize(1), Size(1), MDSize(0), AllocSize(MDSize),
361 ElemRecord(nullptr), IsConst(true), IsMutable(false), IsTemporary(false),
362 IsDummy(true) {
363 assert(Source && "Missing source");
364}
365
366QualType Descriptor::getType() const {
367 if (const auto *E = asExpr())
368 return E->getType();
369 if (const auto *D = asValueDecl())
370 return D->getType();
371 if (const auto *T = dyn_cast<TypeDecl>(Val: asDecl()))
372 return QualType(T->getTypeForDecl(), 0);
373 llvm_unreachable("Invalid descriptor type");
374}
375
376QualType Descriptor::getElemQualType() const {
377 assert(isArray());
378 QualType T = getType();
379 if (const auto *AT = T->getAsArrayTypeUnsafe())
380 return AT->getElementType();
381 if (const auto *CT = T->getAs<ComplexType>())
382 return CT->getElementType();
383 if (const auto *CT = T->getAs<VectorType>())
384 return CT->getElementType();
385 llvm_unreachable("Array that's not an array/complex/vector type?");
386}
387
388SourceLocation Descriptor::getLocation() const {
389 if (auto *D = Source.dyn_cast<const Decl *>())
390 return D->getLocation();
391 if (auto *E = Source.dyn_cast<const Expr *>())
392 return E->getExprLoc();
393 llvm_unreachable("Invalid descriptor type");
394}
395
396InitMap::InitMap(unsigned N)
397 : UninitFields(N), Data(std::make_unique<T[]>(num: numFields(N))) {
398 std::fill_n(data(), numFields(N), 0);
399}
400
401bool InitMap::initializeElement(unsigned I) {
402 unsigned Bucket = I / PER_FIELD;
403 T Mask = T(1) << (I % PER_FIELD);
404 if (!(data()[Bucket] & Mask)) {
405 data()[Bucket] |= Mask;
406 UninitFields -= 1;
407 }
408 return UninitFields == 0;
409}
410
411bool InitMap::isElementInitialized(unsigned I) const {
412 unsigned Bucket = I / PER_FIELD;
413 return data()[Bucket] & (T(1) << (I % PER_FIELD));
414}
415