1//===--- CGExpr.cpp - Emit LLVM Code from Expressions ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Expr nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ABIInfoImpl.h"
14#include "CGCUDARuntime.h"
15#include "CGCXXABI.h"
16#include "CGCall.h"
17#include "CGCleanup.h"
18#include "CGDebugInfo.h"
19#include "CGHLSLRuntime.h"
20#include "CGObjCRuntime.h"
21#include "CGOpenMPRuntime.h"
22#include "CGRecordLayout.h"
23#include "CodeGenFunction.h"
24#include "CodeGenModule.h"
25#include "CodeGenPGO.h"
26#include "ConstantEmitter.h"
27#include "TargetInfo.h"
28#include "clang/AST/ASTContext.h"
29#include "clang/AST/ASTLambda.h"
30#include "clang/AST/Attr.h"
31#include "clang/AST/DeclObjC.h"
32#include "clang/AST/Expr.h"
33#include "clang/AST/InferAlloc.h"
34#include "clang/AST/MatrixUtils.h"
35#include "clang/AST/NSAPI.h"
36#include "clang/AST/ParentMapContext.h"
37#include "clang/AST/StmtVisitor.h"
38#include "clang/Basic/Builtins.h"
39#include "clang/Basic/CodeGenOptions.h"
40#include "clang/Basic/Module.h"
41#include "clang/Basic/SourceManager.h"
42#include "llvm/ADT/STLExtras.h"
43#include "llvm/ADT/ScopeExit.h"
44#include "llvm/ADT/StringExtras.h"
45#include "llvm/IR/Constants.h"
46#include "llvm/IR/DataLayout.h"
47#include "llvm/IR/Intrinsics.h"
48#include "llvm/IR/IntrinsicsWebAssembly.h"
49#include "llvm/IR/LLVMContext.h"
50#include "llvm/IR/MDBuilder.h"
51#include "llvm/IR/MatrixBuilder.h"
52#include "llvm/Support/ConvertUTF.h"
53#include "llvm/Support/Endian.h"
54#include "llvm/Support/MathExtras.h"
55#include "llvm/Support/Path.h"
56#include "llvm/Support/xxhash.h"
57#include "llvm/Transforms/Utils/SanitizerStats.h"
58
59#include <numeric>
60#include <optional>
61#include <string>
62
63using namespace clang;
64using namespace CodeGen;
65
66namespace clang {
67// TODO: consider deprecating ClSanitizeGuardChecks; functionality is subsumed
68// by -fsanitize-skip-hot-cutoff
69llvm::cl::opt<bool> ClSanitizeGuardChecks(
70 "ubsan-guard-checks", llvm::cl::Optional,
71 llvm::cl::desc("Guard UBSAN checks with `llvm.allow.ubsan.check()`."));
72
73} // namespace clang
74
75//===--------------------------------------------------------------------===//
76// Defines for metadata
77//===--------------------------------------------------------------------===//
78
79// Those values are crucial to be the SAME as in ubsan runtime library.
80enum VariableTypeDescriptorKind : uint16_t {
81 /// An integer type.
82 TK_Integer = 0x0000,
83 /// A floating-point type.
84 TK_Float = 0x0001,
85 /// An _BitInt(N) type.
86 TK_BitInt = 0x0002,
87 /// Any other type. The value representation is unspecified.
88 TK_Unknown = 0xffff
89};
90
91//===--------------------------------------------------------------------===//
92// Miscellaneous Helper Methods
93//===--------------------------------------------------------------------===//
94
95static llvm::StringRef GetUBSanTrapForHandler(SanitizerHandler ID) {
96 switch (ID) {
97#define SANITIZER_CHECK(Enum, Name, Version, Msg) \
98 case SanitizerHandler::Enum: \
99 return Msg;
100 LIST_SANITIZER_CHECKS
101#undef SANITIZER_CHECK
102 }
103 llvm_unreachable("unhandled switch case");
104}
105
106/// CreateTempAlloca - This creates a alloca and inserts it into the entry
107/// block.
108RawAddress
109CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits Align,
110 const Twine &Name,
111 llvm::Value *ArraySize) {
112 if (getLangOpts().EmitLogicalPointer) {
113 auto Alloca = Builder.CreateStructuredAlloca(BaseType: Ty, Name);
114 return RawAddress(Alloca, Ty, Align, KnownNonNull);
115 }
116
117 auto *Alloca = CreateTempAlloca(Ty, Name, ArraySize);
118 Alloca->setAlignment(Align.getAsAlign());
119 return RawAddress(Alloca, Ty, Align, KnownNonNull);
120}
121
122RawAddress CodeGenFunction::MaybeCastStackAddressSpace(RawAddress Alloca,
123 LangAS DestLangAS,
124 llvm::Value *ArraySize) {
125
126 llvm::Value *V = Alloca.getPointer();
127 // Alloca always returns a pointer in alloca address space, which may
128 // be different from the type defined by the language. For example,
129 // in C++ the auto variables are in the default address space. Therefore
130 // cast alloca to the default address space when necessary.
131
132 unsigned DestAddrSpace = getContext().getTargetAddressSpace(AS: DestLangAS);
133 if (DestAddrSpace != Alloca.getAddressSpace()) {
134 llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
135 // When ArraySize is nullptr, alloca is inserted at AllocaInsertPt,
136 // otherwise alloca is inserted at the current insertion point of the
137 // builder.
138 if (!ArraySize)
139 Builder.SetInsertPoint(getPostAllocaInsertPoint());
140 V = performAddrSpaceCast(Src: V, DestTy: Builder.getPtrTy(AddrSpace: DestAddrSpace));
141 }
142
143 return RawAddress(V, Alloca.getElementType(), Alloca.getAlignment(),
144 KnownNonNull);
145}
146
147RawAddress CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, LangAS DestLangAS,
148 CharUnits Align, const Twine &Name,
149 llvm::Value *ArraySize,
150 RawAddress *AllocaAddr) {
151 RawAddress Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize);
152 if (AllocaAddr)
153 *AllocaAddr = Alloca;
154 return MaybeCastStackAddressSpace(Alloca, DestLangAS, ArraySize);
155}
156
157/// CreateTempAlloca - This creates an alloca and inserts it into the entry
158/// block if \p ArraySize is nullptr, otherwise inserts it at the current
159/// insertion point of the builder.
160llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
161 const Twine &Name,
162 llvm::Value *ArraySize) {
163 llvm::AllocaInst *Alloca;
164 if (ArraySize)
165 Alloca = Builder.CreateAlloca(Ty, ArraySize, Name);
166 else
167 Alloca =
168 new llvm::AllocaInst(Ty, CGM.getDataLayout().getAllocaAddrSpace(),
169 ArraySize, Name, AllocaInsertPt->getIterator());
170 if (SanOpts.Mask & SanitizerKind::Address) {
171 Alloca->addAnnotationMetadata(Annotations: {"alloca_name_altered", Name.str()});
172 }
173 if (Allocas) {
174 Allocas->Add(I: Alloca);
175 }
176 return Alloca;
177}
178
179/// CreateDefaultAlignTempAlloca - This creates an alloca with the
180/// default alignment of the corresponding LLVM type, which is *not*
181/// guaranteed to be related in any way to the expected alignment of
182/// an AST type that might have been lowered to Ty.
183RawAddress CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty,
184 const Twine &Name) {
185 CharUnits Align =
186 CharUnits::fromQuantity(Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty));
187 return CreateTempAlloca(Ty, DestLangAS: LangAS::Default, Align, Name);
188}
189
190RawAddress CodeGenFunction::CreateIRTempWithoutCast(QualType Ty,
191 const Twine &Name) {
192 CharUnits Align = getContext().getTypeAlignInChars(T: Ty);
193 return CreateTempAllocaWithoutCast(Ty: ConvertType(T: Ty), Align, Name, ArraySize: nullptr);
194}
195
196RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name,
197 RawAddress *Alloca) {
198 // FIXME: Should we prefer the preferred type alignment here?
199 return CreateMemTemp(T: Ty, Align: getContext().getTypeAlignInChars(T: Ty), Name, Alloca);
200}
201
202RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align,
203 const Twine &Name,
204 RawAddress *Alloca) {
205 RawAddress Result =
206 CreateTempAlloca(Ty: ConvertTypeForMem(T: Ty), DestLangAS: Ty.getAddressSpace(), Align, Name,
207 /*ArraySize=*/nullptr, AllocaAddr: Alloca);
208
209 if (Ty->isConstantMatrixType()) {
210 auto *ArrayTy = cast<llvm::ArrayType>(Val: Result.getElementType());
211 auto *ArrayElementTy = ArrayTy->getElementType();
212 auto ArrayElements = ArrayTy->getNumElements();
213 if (getContext().getLangOpts().HLSL) {
214 auto *VectorTy = cast<llvm::FixedVectorType>(Val: ArrayElementTy);
215 ArrayElementTy = VectorTy->getElementType();
216 ArrayElements *= VectorTy->getNumElements();
217 }
218 auto *VectorTy = llvm::FixedVectorType::get(ElementType: ArrayElementTy, NumElts: ArrayElements);
219
220 Result = Address(Result.getPointer(), VectorTy, Result.getAlignment(),
221 KnownNonNull);
222 }
223 return Result;
224}
225
226RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
227 CharUnits Align,
228 const Twine &Name) {
229 return CreateTempAllocaWithoutCast(Ty: ConvertTypeForMem(T: Ty), Align, Name);
230}
231
232RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
233 const Twine &Name) {
234 return CreateMemTempWithoutCast(Ty, Align: getContext().getTypeAlignInChars(T: Ty),
235 Name);
236}
237
238/// EvaluateExprAsBool - Perform the usual unary conversions on the specified
239/// expression and compare the result against zero, returning an Int1Ty value.
240llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
241 PGO->setCurrentStmt(E);
242 if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {
243 llvm::Value *MemPtr = EmitScalarExpr(E);
244 return CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF&: *this, MemPtr, MPT);
245 }
246
247 QualType BoolTy = getContext().BoolTy;
248 SourceLocation Loc = E->getExprLoc();
249 CGFPOptionsRAII FPOptsRAII(*this, E);
250 if (!E->getType()->isAnyComplexType())
251 return EmitScalarConversion(Src: EmitScalarExpr(E), SrcTy: E->getType(), DstTy: BoolTy, Loc);
252
253 return EmitComplexToScalarConversion(Src: EmitComplexExpr(E), SrcTy: E->getType(), DstTy: BoolTy,
254 Loc);
255}
256
257/// EmitIgnoredExpr - Emit code to compute the specified expression,
258/// ignoring the result.
259void CodeGenFunction::EmitIgnoredExpr(const Expr *E) {
260 if (E->isPRValue())
261 return (void)EmitAnyExpr(E, aggSlot: AggValueSlot::ignored(), ignoreResult: true);
262
263 // if this is a bitfield-resulting conditional operator, we can special case
264 // emit this. The normal 'EmitLValue' version of this is particularly
265 // difficult to codegen for, since creating a single "LValue" for two
266 // different sized arguments here is not particularly doable.
267 if (const auto *CondOp = dyn_cast<AbstractConditionalOperator>(
268 Val: E->IgnoreParenNoopCasts(Ctx: getContext()))) {
269 if (CondOp->getObjectKind() == OK_BitField)
270 return EmitIgnoredConditionalOperator(E: CondOp);
271 }
272
273 // Just emit it as an l-value and drop the result.
274 EmitLValue(E);
275}
276
277/// EmitAnyExpr - Emit code to compute the specified expression which
278/// can have any type. The result is returned as an RValue struct.
279/// If this is an aggregate expression, AggSlot indicates where the
280/// result should be returned.
281RValue CodeGenFunction::EmitAnyExpr(const Expr *E,
282 AggValueSlot aggSlot,
283 bool ignoreResult) {
284 switch (getEvaluationKind(T: E->getType())) {
285 case TEK_Scalar:
286 return RValue::get(V: EmitScalarExpr(E, IgnoreResultAssign: ignoreResult));
287 case TEK_Complex:
288 return RValue::getComplex(C: EmitComplexExpr(E, IgnoreReal: ignoreResult, IgnoreImag: ignoreResult));
289 case TEK_Aggregate:
290 if (!ignoreResult && aggSlot.isIgnored())
291 aggSlot = CreateAggTemp(T: E->getType().getUnqualifiedType(), Name: "agg-temp");
292 EmitAggExpr(E, AS: aggSlot);
293 return aggSlot.asRValue();
294 }
295 llvm_unreachable("bad evaluation kind");
296}
297
298/// EmitAnyExprToTemp - Similar to EmitAnyExpr(), however, the result will
299/// always be accessible even if no aggregate location is provided.
300RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E) {
301 AggValueSlot AggSlot = AggValueSlot::ignored();
302
303 if (hasAggregateEvaluationKind(T: E->getType()))
304 AggSlot = CreateAggTemp(T: E->getType(), Name: "agg.tmp");
305 return EmitAnyExpr(E, aggSlot: AggSlot);
306}
307
308/// EmitAnyExprToMem - Evaluate an expression into a given memory
309/// location.
310void CodeGenFunction::EmitAnyExprToMem(const Expr *E,
311 Address Location,
312 Qualifiers Quals,
313 bool IsInit) {
314 // FIXME: This function should take an LValue as an argument.
315 switch (getEvaluationKind(T: E->getType())) {
316 case TEK_Complex:
317 EmitComplexExprIntoLValue(E, dest: MakeAddrLValue(Addr: Location, T: E->getType()),
318 /*isInit*/ false);
319 return;
320
321 case TEK_Aggregate: {
322 EmitAggExpr(E, AS: AggValueSlot::forAddr(addr: Location, quals: Quals,
323 isDestructed: AggValueSlot::IsDestructed_t(IsInit),
324 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
325 isAliased: AggValueSlot::IsAliased_t(!IsInit),
326 mayOverlap: AggValueSlot::MayOverlap));
327 return;
328 }
329
330 case TEK_Scalar: {
331 RValue RV = RValue::get(V: EmitScalarExpr(E, /*Ignore*/ IgnoreResultAssign: false));
332 LValue LV = MakeAddrLValue(Addr: Location, T: E->getType());
333 EmitStoreThroughLValue(Src: RV, Dst: LV);
334 return;
335 }
336 }
337 llvm_unreachable("bad evaluation kind");
338}
339
340void CodeGenFunction::EmitInitializationToLValue(
341 const Expr *E, LValue LV, AggValueSlot::IsZeroed_t IsZeroed) {
342 QualType Type = LV.getType();
343 switch (getEvaluationKind(T: Type)) {
344 case TEK_Complex:
345 EmitComplexExprIntoLValue(E, dest: LV, /*isInit*/ true);
346 return;
347 case TEK_Aggregate:
348 EmitAggExpr(E, AS: AggValueSlot::forLValue(LV, isDestructed: AggValueSlot::IsDestructed,
349 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
350 isAliased: AggValueSlot::IsNotAliased,
351 mayOverlap: AggValueSlot::MayOverlap, isZeroed: IsZeroed));
352 return;
353 case TEK_Scalar:
354 if (LV.isSimple())
355 EmitScalarInit(init: E, /*D=*/nullptr, lvalue: LV, /*Captured=*/capturedByInit: false);
356 else
357 EmitStoreThroughLValue(Src: RValue::get(V: EmitScalarExpr(E)), Dst: LV);
358 return;
359 }
360 llvm_unreachable("bad evaluation kind");
361}
362
363static void
364pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M,
365 const Expr *E, Address ReferenceTemporary) {
366 // Objective-C++ ARC:
367 // If we are binding a reference to a temporary that has ownership, we
368 // need to perform retain/release operations on the temporary.
369 //
370 // FIXME: This should be looking at E, not M.
371 if (auto Lifetime = M->getType().getObjCLifetime()) {
372 switch (Lifetime) {
373 case Qualifiers::OCL_None:
374 case Qualifiers::OCL_ExplicitNone:
375 // Carry on to normal cleanup handling.
376 break;
377
378 case Qualifiers::OCL_Autoreleasing:
379 // Nothing to do; cleaned up by an autorelease pool.
380 return;
381
382 case Qualifiers::OCL_Strong:
383 case Qualifiers::OCL_Weak:
384 switch (StorageDuration Duration = M->getStorageDuration()) {
385 case SD_Static:
386 // Note: we intentionally do not register a cleanup to release
387 // the object on program termination.
388 return;
389
390 case SD_Thread:
391 // FIXME: We should probably register a cleanup in this case.
392 return;
393
394 case SD_Automatic:
395 case SD_FullExpression:
396 CodeGenFunction::Destroyer *Destroy;
397 CleanupKind CleanupKind;
398 if (Lifetime == Qualifiers::OCL_Strong) {
399 const ValueDecl *VD = M->getExtendingDecl();
400 bool Precise = isa_and_nonnull<VarDecl>(Val: VD) &&
401 VD->hasAttr<ObjCPreciseLifetimeAttr>();
402 CleanupKind = CGF.getARCCleanupKind();
403 Destroy = Precise ? &CodeGenFunction::destroyARCStrongPrecise
404 : &CodeGenFunction::destroyARCStrongImprecise;
405 } else {
406 // __weak objects always get EH cleanups; otherwise, exceptions
407 // could cause really nasty crashes instead of mere leaks.
408 CleanupKind = NormalAndEHCleanup;
409 Destroy = &CodeGenFunction::destroyARCWeak;
410 }
411 if (Duration == SD_FullExpression)
412 CGF.pushDestroy(kind: CleanupKind, addr: ReferenceTemporary,
413 type: M->getType(), destroyer: *Destroy,
414 useEHCleanupForArray: CleanupKind & EHCleanup);
415 else
416 CGF.pushLifetimeExtendedDestroy(kind: CleanupKind, addr: ReferenceTemporary,
417 type: M->getType(),
418 destroyer: *Destroy, useEHCleanupForArray: CleanupKind & EHCleanup);
419 return;
420
421 case SD_Dynamic:
422 llvm_unreachable("temporary cannot have dynamic storage duration");
423 }
424 llvm_unreachable("unknown storage duration");
425 }
426 }
427
428 QualType::DestructionKind DK = E->getType().isDestructedType();
429 if (DK != QualType::DK_none) {
430 switch (M->getStorageDuration()) {
431 case SD_Static:
432 case SD_Thread: {
433 CXXDestructorDecl *ReferenceTemporaryDtor = nullptr;
434 if (const auto *ClassDecl =
435 E->getType()->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
436 ClassDecl && !ClassDecl->hasTrivialDestructor())
437 // Get the destructor for the reference temporary.
438 ReferenceTemporaryDtor = ClassDecl->getDestructor();
439
440 if (!ReferenceTemporaryDtor)
441 return;
442
443 llvm::FunctionCallee CleanupFn;
444 llvm::Constant *CleanupArg;
445 if (E->getType()->isArrayType()) {
446 CleanupFn = CodeGenFunction(CGF.CGM).generateDestroyHelper(
447 addr: ReferenceTemporary, type: E->getType(), destroyer: CodeGenFunction::destroyCXXObject,
448 useEHCleanupForArray: CGF.getLangOpts().Exceptions,
449 VD: dyn_cast_or_null<VarDecl>(Val: M->getExtendingDecl()));
450 CleanupArg = llvm::Constant::getNullValue(Ty: CGF.Int8PtrTy);
451 } else {
452 CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor(
453 GD: GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete));
454 CleanupArg =
455 cast<llvm::Constant>(Val: ReferenceTemporary.emitRawPointer(CGF));
456 }
457 CGF.CGM.getCXXABI().registerGlobalDtor(
458 CGF, D: *cast<VarDecl>(Val: M->getExtendingDecl()), Dtor: CleanupFn, Addr: CleanupArg);
459 } break;
460 case SD_FullExpression:
461 CGF.pushDestroy(dtorKind: DK, addr: ReferenceTemporary, type: E->getType());
462 break;
463 case SD_Automatic:
464 CGF.pushLifetimeExtendedDestroy(dtorKind: DK, addr: ReferenceTemporary, type: E->getType());
465 break;
466 case SD_Dynamic:
467 llvm_unreachable("temporary cannot have dynamic storage duration");
468 }
469 }
470}
471
472static RawAddress createReferenceTemporary(CodeGenFunction &CGF,
473 const MaterializeTemporaryExpr *M,
474 const Expr *Inner,
475 RawAddress *Alloca = nullptr) {
476 switch (M->getStorageDuration()) {
477 case SD_FullExpression:
478 case SD_Automatic: {
479 // If we have a constant temporary array or record try to promote it into a
480 // constant global under the same rules a normal constant would've been
481 // promoted. This is easier on the optimizer and generally emits fewer
482 // instructions.
483 QualType Ty = Inner->getType();
484 if (CGF.CGM.getCodeGenOpts().MergeAllConstants &&
485 (Ty->isArrayType() || Ty->isRecordType()) &&
486 Ty.isConstantStorage(Ctx: CGF.getContext(), ExcludeCtor: true, ExcludeDtor: false))
487 if (auto Init = ConstantEmitter(CGF).tryEmitAbstract(E: Inner, T: Ty)) {
488 auto AS = CGF.CGM.GetGlobalConstantAddressSpace();
489 auto *GV = new llvm::GlobalVariable(
490 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
491 llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp", nullptr,
492 llvm::GlobalValue::NotThreadLocal,
493 CGF.getContext().getTargetAddressSpace(AS));
494 CharUnits alignment = CGF.getContext().getTypeAlignInChars(T: Ty);
495 GV->setAlignment(alignment.getAsAlign());
496 llvm::Constant *C = GV;
497 if (AS != Ty.getAddressSpace())
498 C = CGF.CGM.performAddrSpaceCast(
499 Src: GV, DestTy: llvm::PointerType::get(C&: CGF.getLLVMContext(),
500 AddressSpace: CGF.getContext().getTargetAddressSpace(
501 AS: Ty.getAddressSpace())));
502 // FIXME: Should we put the new global into a COMDAT?
503 return RawAddress(C, GV->getValueType(), alignment);
504 }
505 return CGF.CreateMemTemp(Ty, Name: "ref.tmp", Alloca);
506 }
507 case SD_Thread:
508 case SD_Static:
509 return CGF.CGM.GetAddrOfGlobalTemporary(E: M, Inner);
510
511 case SD_Dynamic:
512 llvm_unreachable("temporary can't have dynamic storage duration");
513 }
514 llvm_unreachable("unknown storage duration");
515}
516
517/// Helper method to check if the underlying ABI is AAPCS
518static bool isAAPCS(const TargetInfo &TargetInfo) {
519 return TargetInfo.getABI().starts_with(Prefix: "aapcs");
520}
521
522LValue CodeGenFunction::
523EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
524 const Expr *E = M->getSubExpr();
525
526 assert((!M->getExtendingDecl() || !isa<VarDecl>(M->getExtendingDecl()) ||
527 !cast<VarDecl>(M->getExtendingDecl())->isARCPseudoStrong()) &&
528 "Reference should never be pseudo-strong!");
529
530 // FIXME: ideally this would use EmitAnyExprToMem, however, we cannot do so
531 // as that will cause the lifetime adjustment to be lost for ARC
532 auto ownership = M->getType().getObjCLifetime();
533 if (ownership != Qualifiers::OCL_None &&
534 ownership != Qualifiers::OCL_ExplicitNone) {
535 RawAddress Object = createReferenceTemporary(CGF&: *this, M, Inner: E);
536 if (auto *Var = dyn_cast<llvm::GlobalVariable>(Val: Object.getPointer())) {
537 llvm::Type *Ty = ConvertTypeForMem(T: E->getType());
538 Object = Object.withElementType(ElemTy: Ty);
539
540 // createReferenceTemporary will promote the temporary to a global with a
541 // constant initializer if it can. It can only do this to a value of
542 // ARC-manageable type if the value is global and therefore "immune" to
543 // ref-counting operations. Therefore we have no need to emit either a
544 // dynamic initialization or a cleanup and we can just return the address
545 // of the temporary.
546 if (Var->hasInitializer())
547 return MakeAddrLValue(Addr: Object, T: M->getType(), Source: AlignmentSource::Decl);
548
549 Var->setInitializer(CGM.EmitNullConstant(T: E->getType()));
550 }
551 LValue RefTempDst = MakeAddrLValue(Addr: Object, T: M->getType(),
552 Source: AlignmentSource::Decl);
553
554 switch (getEvaluationKind(T: E->getType())) {
555 default: llvm_unreachable("expected scalar or aggregate expression");
556 case TEK_Scalar:
557 EmitScalarInit(init: E, D: M->getExtendingDecl(), lvalue: RefTempDst, capturedByInit: false);
558 break;
559 case TEK_Aggregate: {
560 EmitAggExpr(E, AS: AggValueSlot::forAddr(addr: Object,
561 quals: E->getType().getQualifiers(),
562 isDestructed: AggValueSlot::IsDestructed,
563 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
564 isAliased: AggValueSlot::IsNotAliased,
565 mayOverlap: AggValueSlot::DoesNotOverlap));
566 break;
567 }
568 }
569
570 pushTemporaryCleanup(CGF&: *this, M, E, ReferenceTemporary: Object);
571 return RefTempDst;
572 }
573
574 SmallVector<const Expr *, 2> CommaLHSs;
575 SmallVector<SubobjectAdjustment, 2> Adjustments;
576 E = E->skipRValueSubobjectAdjustments(CommaLHS&: CommaLHSs, Adjustments);
577
578 for (const auto &Ignored : CommaLHSs)
579 EmitIgnoredExpr(E: Ignored);
580
581 if (const auto *opaque = dyn_cast<OpaqueValueExpr>(Val: E)) {
582 if (opaque->getType()->isRecordType()) {
583 assert(Adjustments.empty());
584 return EmitOpaqueValueLValue(e: opaque);
585 }
586 }
587
588 // Create and initialize the reference temporary.
589 RawAddress Alloca = Address::invalid();
590 RawAddress Object = createReferenceTemporary(CGF&: *this, M, Inner: E, Alloca: &Alloca);
591 if (auto *Var = dyn_cast<llvm::GlobalVariable>(
592 Val: Object.getPointer()->stripPointerCasts())) {
593 llvm::Type *TemporaryType = ConvertTypeForMem(T: E->getType());
594 Object = Object.withElementType(ElemTy: TemporaryType);
595 // If the temporary is a global and has a constant initializer or is a
596 // constant temporary that we promoted to a global, we may have already
597 // initialized it.
598 if (!Var->hasInitializer()) {
599 Var->setInitializer(CGM.EmitNullConstant(T: E->getType()));
600 QualType RefType = M->getType().withoutLocalFastQualifiers();
601 if (RefType.getPointerAuth()) {
602 // Use the qualifier of the reference temporary to sign the pointer.
603 LValue LV = MakeRawAddrLValue(V: Object.getPointer(), T: RefType,
604 Alignment: Object.getAlignment());
605 EmitScalarInit(init: E, D: M->getExtendingDecl(), lvalue: LV, capturedByInit: false);
606 } else {
607 EmitAnyExprToMem(E, Location: Object, Quals: Qualifiers(), /*IsInit*/ true);
608 }
609 }
610 } else {
611 switch (M->getStorageDuration()) {
612 case SD_Automatic:
613 if (EmitLifetimeStart(Addr: Alloca.getPointer())) {
614 pushCleanupAfterFullExpr<CallLifetimeEnd>(Kind: NormalEHLifetimeMarker,
615 A: Alloca);
616 }
617 break;
618
619 case SD_FullExpression: {
620 if (!ShouldEmitLifetimeMarkers)
621 break;
622
623 // Avoid creating a conditional cleanup just to hold an llvm.lifetime.end
624 // marker. Instead, start the lifetime of a conditional temporary earlier
625 // so that it's unconditional. Don't do this with sanitizers which need
626 // more precise lifetime marks. However when inside an "await.suspend"
627 // block, we should always avoid conditional cleanup because it creates
628 // boolean marker that lives across await_suspend, which can destroy coro
629 // frame.
630 ConditionalEvaluation *OldConditional = nullptr;
631 CGBuilderTy::InsertPoint OldIP;
632 if (isInConditionalBranch() && !E->getType().isDestructedType() &&
633 ((!SanOpts.has(K: SanitizerKind::HWAddress) &&
634 !SanOpts.has(K: SanitizerKind::Memory) &&
635 !SanOpts.has(K: SanitizerKind::MemtagStack) &&
636 !CGM.getCodeGenOpts().SanitizeAddressUseAfterScope) ||
637 inSuspendBlock())) {
638 OldConditional = OutermostConditional;
639 OutermostConditional = nullptr;
640
641 OldIP = Builder.saveIP();
642 llvm::BasicBlock *Block = OldConditional->getStartingBlock();
643 Builder.restoreIP(IP: CGBuilderTy::InsertPoint(
644 Block, llvm::BasicBlock::iterator(Block->back())));
645 }
646
647 if (EmitLifetimeStart(Addr: Alloca.getPointer())) {
648 pushFullExprCleanup<CallLifetimeEnd>(kind: NormalEHLifetimeMarker, A: Alloca);
649 }
650
651 if (OldConditional) {
652 OutermostConditional = OldConditional;
653 Builder.restoreIP(IP: OldIP);
654 }
655 break;
656 }
657
658 default:
659 break;
660 }
661 EmitAnyExprToMem(E, Location: Object, Quals: Qualifiers(), /*IsInit*/true);
662 }
663 pushTemporaryCleanup(CGF&: *this, M, E, ReferenceTemporary: Object);
664
665 // Perform derived-to-base casts and/or field accesses, to get from the
666 // temporary object we created (and, potentially, for which we extended
667 // the lifetime) to the subobject we're binding the reference to.
668 for (SubobjectAdjustment &Adjustment : llvm::reverse(C&: Adjustments)) {
669 switch (Adjustment.Kind) {
670 case SubobjectAdjustment::DerivedToBaseAdjustment:
671 Object =
672 GetAddressOfBaseClass(Value: Object, Derived: Adjustment.DerivedToBase.DerivedClass,
673 PathBegin: Adjustment.DerivedToBase.BasePath->path_begin(),
674 PathEnd: Adjustment.DerivedToBase.BasePath->path_end(),
675 /*NullCheckValue=*/ false, Loc: E->getExprLoc());
676 break;
677
678 case SubobjectAdjustment::FieldAdjustment: {
679 LValue LV = MakeAddrLValue(Addr: Object, T: E->getType(), Source: AlignmentSource::Decl);
680 LV = EmitLValueForField(Base: LV, Field: Adjustment.Field);
681 assert(LV.isSimple() &&
682 "materialized temporary field is not a simple lvalue");
683 Object = LV.getAddress();
684 break;
685 }
686
687 case SubobjectAdjustment::MemberPointerAdjustment: {
688 llvm::Value *Ptr = EmitScalarExpr(E: Adjustment.Ptr.RHS);
689 Object = EmitCXXMemberDataPointerAddress(
690 E, base: Object, memberPtr: Ptr, memberPtrType: Adjustment.Ptr.MPT, /*IsInBounds=*/true);
691 break;
692 }
693 }
694 }
695
696 return MakeAddrLValue(Addr: Object, T: M->getType(), Source: AlignmentSource::Decl);
697}
698
699RValue
700CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E) {
701 // Emit the expression as an lvalue.
702 LValue LV = EmitLValue(E);
703 assert(LV.isSimple());
704 llvm::Value *Value = LV.getPointer(CGF&: *this);
705
706 if (sanitizePerformTypeCheck() && !E->getType()->isFunctionType()) {
707 // C++11 [dcl.ref]p5 (as amended by core issue 453):
708 // If a glvalue to which a reference is directly bound designates neither
709 // an existing object or function of an appropriate type nor a region of
710 // storage of suitable size and alignment to contain an object of the
711 // reference's type, the behavior is undefined.
712 QualType Ty = E->getType();
713 EmitTypeCheck(TCK: TCK_ReferenceBinding, Loc: E->getExprLoc(), V: Value, Type: Ty);
714 }
715
716 return RValue::get(V: Value);
717}
718
719
720/// getAccessedFieldNo - Given an encoded value and a result number, return the
721/// input field number being accessed.
722unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx,
723 const llvm::Constant *Elts) {
724 return cast<llvm::ConstantInt>(Val: Elts->getAggregateElement(Elt: Idx))
725 ->getZExtValue();
726}
727
728static llvm::Value *emitHashMix(CGBuilderTy &Builder, llvm::Value *Acc,
729 llvm::Value *Ptr) {
730 llvm::Value *A0 =
731 Builder.CreateMul(LHS: Ptr, RHS: Builder.getInt64(C: 0xbf58476d1ce4e5b9u));
732 llvm::Value *A1 =
733 Builder.CreateXor(LHS: A0, RHS: Builder.CreateLShr(LHS: A0, RHS: Builder.getInt64(C: 31)));
734 return Builder.CreateXor(LHS: Acc, RHS: A1);
735}
736
737bool CodeGenFunction::isNullPointerAllowed(TypeCheckKind TCK) {
738 return TCK == TCK_DowncastPointer || TCK == TCK_Upcast ||
739 TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation;
740}
741
742bool CodeGenFunction::isVptrCheckRequired(TypeCheckKind TCK, QualType Ty) {
743 CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
744 return (RD && RD->hasDefinition() && RD->isDynamicClass()) &&
745 (TCK == TCK_MemberAccess || TCK == TCK_MemberCall ||
746 TCK == TCK_DowncastPointer || TCK == TCK_DowncastReference ||
747 TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation);
748}
749
750bool CodeGenFunction::sanitizePerformTypeCheck() const {
751 return SanOpts.has(K: SanitizerKind::Null) ||
752 SanOpts.has(K: SanitizerKind::Alignment) ||
753 SanOpts.has(K: SanitizerKind::ObjectSize) ||
754 SanOpts.has(K: SanitizerKind::Vptr);
755}
756
757void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
758 llvm::Value *Ptr, QualType Ty,
759 CharUnits Alignment,
760 SanitizerSet SkippedChecks,
761 llvm::Value *ArraySize) {
762 if (!sanitizePerformTypeCheck())
763 return;
764
765 // Don't check pointers outside the default address space. The null check
766 // isn't correct, the object-size check isn't supported by LLVM, and we can't
767 // communicate the addresses to the runtime handler for the vptr check.
768 if (Ptr->getType()->getPointerAddressSpace())
769 return;
770
771 // Don't check pointers to volatile data. The behavior here is implementation-
772 // defined.
773 if (Ty.isVolatileQualified())
774 return;
775
776 // Quickly determine whether we have a pointer to an alloca. It's possible
777 // to skip null checks, and some alignment checks, for these pointers. This
778 // can reduce compile-time significantly.
779 auto PtrToAlloca = dyn_cast<llvm::AllocaInst>(Val: Ptr->stripPointerCasts());
780
781 llvm::Value *IsNonNull = nullptr;
782 bool IsGuaranteedNonNull =
783 SkippedChecks.has(K: SanitizerKind::Null) || PtrToAlloca;
784
785 llvm::BasicBlock *Done = nullptr;
786 bool DoneViaNullSanitize = false;
787
788 {
789 auto CheckHandler = SanitizerHandler::TypeMismatch;
790 SanitizerDebugLocation SanScope(this,
791 {SanitizerKind::SO_Null,
792 SanitizerKind::SO_ObjectSize,
793 SanitizerKind::SO_Alignment},
794 CheckHandler);
795
796 SmallVector<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>, 3>
797 Checks;
798
799 llvm::Value *True = llvm::ConstantInt::getTrue(Context&: getLLVMContext());
800 bool AllowNullPointers = isNullPointerAllowed(TCK);
801 if ((SanOpts.has(K: SanitizerKind::Null) || AllowNullPointers) &&
802 !IsGuaranteedNonNull) {
803 // The glvalue must not be an empty glvalue.
804 IsNonNull = Builder.CreateIsNotNull(Arg: Ptr);
805
806 // The IR builder can constant-fold the null check if the pointer points
807 // to a constant.
808 IsGuaranteedNonNull = IsNonNull == True;
809
810 // Skip the null check if the pointer is known to be non-null.
811 if (!IsGuaranteedNonNull) {
812 if (AllowNullPointers) {
813 // When performing pointer casts, it's OK if the value is null.
814 // Skip the remaining checks in that case.
815 Done = createBasicBlock(name: "null");
816 DoneViaNullSanitize = true;
817 llvm::BasicBlock *Rest = createBasicBlock(name: "not.null");
818 Builder.CreateCondBr(Cond: IsNonNull, True: Rest, False: Done);
819 EmitBlock(BB: Rest);
820 } else {
821 Checks.push_back(Elt: std::make_pair(x&: IsNonNull, y: SanitizerKind::SO_Null));
822 }
823 }
824 }
825
826 if (SanOpts.has(K: SanitizerKind::ObjectSize) &&
827 !SkippedChecks.has(K: SanitizerKind::ObjectSize) &&
828 !Ty->isIncompleteType()) {
829 uint64_t TySize = CGM.getMinimumObjectSize(Ty).getQuantity();
830 llvm::Value *Size = llvm::ConstantInt::get(Ty: IntPtrTy, V: TySize);
831 if (ArraySize)
832 Size = Builder.CreateMul(LHS: Size, RHS: ArraySize);
833
834 // Degenerate case: new X[0] does not need an objectsize check.
835 llvm::Constant *ConstantSize = dyn_cast<llvm::Constant>(Val: Size);
836 if (!ConstantSize || !ConstantSize->isNullValue()) {
837 // The glvalue must refer to a large enough storage region.
838 // FIXME: If Address Sanitizer is enabled, insert dynamic
839 // instrumentation
840 // to check this.
841 // FIXME: Get object address space
842 llvm::Type *Tys[2] = {IntPtrTy, Int8PtrTy};
843 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::objectsize, Tys);
844 llvm::Value *Min = Builder.getFalse();
845 llvm::Value *NullIsUnknown = Builder.getFalse();
846 llvm::Value *Dynamic = Builder.getFalse();
847 llvm::Value *LargeEnough = Builder.CreateICmpUGE(
848 LHS: Builder.CreateCall(Callee: F, Args: {Ptr, Min, NullIsUnknown, Dynamic}), RHS: Size);
849 Checks.push_back(
850 Elt: std::make_pair(x&: LargeEnough, y: SanitizerKind::SO_ObjectSize));
851 }
852 }
853
854 llvm::MaybeAlign AlignVal;
855 llvm::Value *PtrAsInt = nullptr;
856
857 if (SanOpts.has(K: SanitizerKind::Alignment) &&
858 !SkippedChecks.has(K: SanitizerKind::Alignment)) {
859 AlignVal = Alignment.getAsMaybeAlign();
860 if (!Ty->isIncompleteType() && !AlignVal)
861 AlignVal = CGM.getNaturalTypeAlignment(T: Ty, BaseInfo: nullptr, TBAAInfo: nullptr,
862 /*ForPointeeType=*/forPointeeType: true)
863 .getAsMaybeAlign();
864
865 // The glvalue must be suitably aligned.
866 if (AlignVal && *AlignVal > llvm::Align(1) &&
867 (!PtrToAlloca || PtrToAlloca->getAlign() < *AlignVal)) {
868 PtrAsInt = Builder.CreatePtrToInt(V: Ptr, DestTy: IntPtrTy);
869 llvm::Value *Align = Builder.CreateAnd(
870 LHS: PtrAsInt, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, V: AlignVal->value() - 1));
871 llvm::Value *Aligned =
872 Builder.CreateICmpEQ(LHS: Align, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, V: 0));
873 if (Aligned != True)
874 Checks.push_back(
875 Elt: std::make_pair(x&: Aligned, y: SanitizerKind::SO_Alignment));
876 }
877 }
878
879 if (Checks.size() > 0) {
880 llvm::Constant *StaticData[] = {
881 EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(T: Ty),
882 llvm::ConstantInt::get(Ty: Int8Ty, V: AlignVal ? llvm::Log2(A: *AlignVal) : 1),
883 llvm::ConstantInt::get(Ty: Int8Ty, V: TCK)};
884 EmitCheck(Checked: Checks, Check: CheckHandler, StaticArgs: StaticData, DynamicArgs: PtrAsInt ? PtrAsInt : Ptr);
885 }
886 }
887
888 // If possible, check that the vptr indicates that there is a subobject of
889 // type Ty at offset zero within this object.
890 //
891 // C++11 [basic.life]p5,6:
892 // [For storage which does not refer to an object within its lifetime]
893 // The program has undefined behavior if:
894 // -- the [pointer or glvalue] is used to access a non-static data member
895 // or call a non-static member function
896 if (SanOpts.has(K: SanitizerKind::Vptr) &&
897 !SkippedChecks.has(K: SanitizerKind::Vptr) && isVptrCheckRequired(TCK, Ty)) {
898 SanitizerDebugLocation SanScope(this, {SanitizerKind::SO_Vptr},
899 SanitizerHandler::DynamicTypeCacheMiss);
900
901 // Ensure that the pointer is non-null before loading it. If there is no
902 // compile-time guarantee, reuse the run-time null check or emit a new one.
903 if (!IsGuaranteedNonNull) {
904 if (!IsNonNull)
905 IsNonNull = Builder.CreateIsNotNull(Arg: Ptr);
906 if (!Done)
907 Done = createBasicBlock(name: "vptr.null");
908 llvm::BasicBlock *VptrNotNull = createBasicBlock(name: "vptr.not.null");
909 Builder.CreateCondBr(Cond: IsNonNull, True: VptrNotNull, False: Done);
910 EmitBlock(BB: VptrNotNull);
911 }
912
913 // Compute a deterministic hash of the mangled name of the type.
914 SmallString<64> MangledName;
915 llvm::raw_svector_ostream Out(MangledName);
916 CGM.getCXXABI().getMangleContext().mangleCXXRTTI(T: Ty.getUnqualifiedType(),
917 Out);
918
919 // Contained in NoSanitizeList based on the mangled type.
920 if (!CGM.getContext().getNoSanitizeList().containsType(Mask: SanitizerKind::Vptr,
921 MangledTypeName: Out.str())) {
922 // Load the vptr, and mix it with TypeHash.
923 llvm::Value *TypeHash =
924 llvm::ConstantInt::get(Ty: Int64Ty, V: xxh3_64bits(data: Out.str()));
925
926 llvm::Type *VPtrTy = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: 0);
927 Address VPtrAddr(Ptr, IntPtrTy, getPointerAlign());
928 llvm::Value *VPtrVal = GetVTablePtr(This: VPtrAddr, VTableTy: VPtrTy,
929 VTableClass: Ty->getAsCXXRecordDecl(),
930 AuthMode: VTableAuthMode::UnsafeUbsanStrip);
931 VPtrVal = Builder.CreateBitOrPointerCast(V: VPtrVal, DestTy: IntPtrTy);
932
933 llvm::Value *Hash =
934 emitHashMix(Builder, Acc: TypeHash, Ptr: Builder.CreateZExt(V: VPtrVal, DestTy: Int64Ty));
935 Hash = Builder.CreateTrunc(V: Hash, DestTy: IntPtrTy);
936
937 // Look the hash up in our cache.
938 const int CacheSize = 128;
939 llvm::Type *HashTable = llvm::ArrayType::get(ElementType: IntPtrTy, NumElements: CacheSize);
940 llvm::Value *Cache = CGM.CreateRuntimeVariable(Ty: HashTable,
941 Name: "__ubsan_vptr_type_cache");
942 llvm::Value *Slot = Builder.CreateAnd(LHS: Hash,
943 RHS: llvm::ConstantInt::get(Ty: IntPtrTy,
944 V: CacheSize-1));
945 llvm::Value *Indices[] = { Builder.getInt32(C: 0), Slot };
946 llvm::Value *CacheVal = Builder.CreateAlignedLoad(
947 Ty: IntPtrTy, Addr: Builder.CreateInBoundsGEP(Ty: HashTable, Ptr: Cache, IdxList: Indices),
948 Align: getPointerAlign());
949
950 // If the hash isn't in the cache, call a runtime handler to perform the
951 // hard work of checking whether the vptr is for an object of the right
952 // type. This will either fill in the cache and return, or produce a
953 // diagnostic.
954 llvm::Value *EqualHash = Builder.CreateICmpEQ(LHS: CacheVal, RHS: Hash);
955 llvm::Constant *StaticData[] = {
956 EmitCheckSourceLocation(Loc),
957 EmitCheckTypeDescriptor(T: Ty),
958 CGM.GetAddrOfRTTIDescriptor(Ty: Ty.getUnqualifiedType()),
959 llvm::ConstantInt::get(Ty: Int8Ty, V: TCK)
960 };
961 llvm::Value *DynamicData[] = { Ptr, Hash };
962 EmitCheck(Checked: std::make_pair(x&: EqualHash, y: SanitizerKind::SO_Vptr),
963 Check: SanitizerHandler::DynamicTypeCacheMiss, StaticArgs: StaticData,
964 DynamicArgs: DynamicData);
965 }
966 }
967
968 if (Done) {
969 SanitizerDebugLocation SanScope(
970 this,
971 {DoneViaNullSanitize ? SanitizerKind::SO_Null : SanitizerKind::SO_Vptr},
972 DoneViaNullSanitize ? SanitizerHandler::TypeMismatch
973 : SanitizerHandler::DynamicTypeCacheMiss);
974 Builder.CreateBr(Dest: Done);
975 EmitBlock(BB: Done);
976 }
977}
978
979llvm::Value *CodeGenFunction::LoadPassedObjectSize(const Expr *E,
980 QualType EltTy) {
981 ASTContext &C = getContext();
982 uint64_t EltSize = C.getTypeSizeInChars(T: EltTy).getQuantity();
983 if (!EltSize)
984 return nullptr;
985
986 auto *ArrayDeclRef = dyn_cast<DeclRefExpr>(Val: E->IgnoreParenImpCasts());
987 if (!ArrayDeclRef)
988 return nullptr;
989
990 auto *ParamDecl = dyn_cast<ParmVarDecl>(Val: ArrayDeclRef->getDecl());
991 if (!ParamDecl)
992 return nullptr;
993
994 auto *POSAttr = ParamDecl->getAttr<PassObjectSizeAttr>();
995 if (!POSAttr)
996 return nullptr;
997
998 // Don't load the size if it's a lower bound.
999 int POSType = POSAttr->getType();
1000 if (POSType != 0 && POSType != 1)
1001 return nullptr;
1002
1003 // Find the implicit size parameter.
1004 auto PassedSizeIt = SizeArguments.find(Val: ParamDecl);
1005 if (PassedSizeIt == SizeArguments.end())
1006 return nullptr;
1007
1008 const ImplicitParamDecl *PassedSizeDecl = PassedSizeIt->second;
1009 assert(LocalDeclMap.count(PassedSizeDecl) && "Passed size not loadable");
1010 Address AddrOfSize = LocalDeclMap.find(Val: PassedSizeDecl)->second;
1011 llvm::Value *SizeInBytes = EmitLoadOfScalar(Addr: AddrOfSize, /*Volatile=*/false,
1012 Ty: C.getSizeType(), Loc: E->getExprLoc());
1013 llvm::Value *SizeOfElement =
1014 llvm::ConstantInt::get(Ty: SizeInBytes->getType(), V: EltSize);
1015 return Builder.CreateUDiv(LHS: SizeInBytes, RHS: SizeOfElement);
1016}
1017
1018/// If Base is known to point to the start of an array, return the length of
1019/// that array. Return 0 if the length cannot be determined.
1020static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF,
1021 const Expr *Base,
1022 QualType &IndexedType,
1023 LangOptions::StrictFlexArraysLevelKind
1024 StrictFlexArraysLevel) {
1025 // For the vector indexing extension, the bound is the number of elements.
1026 if (const VectorType *VT = Base->getType()->getAs<VectorType>()) {
1027 IndexedType = Base->getType();
1028 return CGF.Builder.getInt32(C: VT->getNumElements());
1029 }
1030
1031 Base = Base->IgnoreParens();
1032
1033 if (const auto *CE = dyn_cast<CastExpr>(Val: Base)) {
1034 if (CE->getCastKind() == CK_ArrayToPointerDecay &&
1035 !CE->getSubExpr()->isFlexibleArrayMemberLike(Context: CGF.getContext(),
1036 StrictFlexArraysLevel)) {
1037 CodeGenFunction::SanitizerScope SanScope(&CGF);
1038
1039 IndexedType = CE->getSubExpr()->getType();
1040 const ArrayType *AT = IndexedType->castAsArrayTypeUnsafe();
1041 if (const auto *CAT = dyn_cast<ConstantArrayType>(Val: AT))
1042 return CGF.Builder.getInt(AI: CAT->getSize());
1043
1044 if (const auto *VAT = dyn_cast<VariableArrayType>(Val: AT))
1045 return CGF.getVLASize(vla: VAT).NumElts;
1046 // Ignore pass_object_size here. It's not applicable on decayed pointers.
1047 }
1048 }
1049
1050 CodeGenFunction::SanitizerScope SanScope(&CGF);
1051
1052 QualType EltTy{Base->getType()->getPointeeOrArrayElementType(), 0};
1053 if (llvm::Value *POS = CGF.LoadPassedObjectSize(E: Base, EltTy)) {
1054 IndexedType = Base->getType();
1055 return POS;
1056 }
1057
1058 return nullptr;
1059}
1060
1061/// Returns true if \p Field is reachable from \p RD either as a direct field or
1062/// through a chain of nested record fields (including anonymous
1063/// structs/unions). This mirrors the GEP path that getGEPIndicesToField builds,
1064/// and is used to identify the right anchor expression in Base.
1065static bool RecordContainsField(const RecordDecl *RD, const FieldDecl *Field) {
1066 for (const FieldDecl *FD : RD->fields()) {
1067 if (FD == Field)
1068 return true;
1069 QualType Ty = FD->getType();
1070 if (Ty->isRecordType())
1071 if (RecordContainsField(RD: Ty->getAsRecordDecl(), Field))
1072 return true;
1073 }
1074 return false;
1075}
1076
1077namespace {
1078
1079/// \p StructAccessBase returns the base \p Expr of a field access. It returns
1080/// either a \p DeclRefExpr, representing the base pointer to the struct, i.e.:
1081///
1082/// p in p-> a.b.c
1083///
1084/// or a \p MemberExpr, if the \p MemberExpr has the \p RecordDecl we're
1085/// looking for:
1086///
1087/// struct s {
1088/// struct s *ptr;
1089/// int count;
1090/// char array[] __attribute__((counted_by(count)));
1091/// };
1092///
1093/// If we have an expression like \p p->ptr->array[index], we want the
1094/// \p MemberExpr for \p p->ptr instead of \p p.
1095class StructAccessBase
1096 : public ConstStmtVisitor<StructAccessBase, const Expr *> {
1097 /// The count field we're navigating to. We stop at the innermost expression
1098 /// whose struct type transitively contains this field, so that
1099 /// getGEPIndicesToField can navigate from that struct down to it.
1100 const FieldDecl *CountDecl;
1101
1102 /// Returns true if E's record type (or pointee record type) transitively
1103 /// contains CountDecl. Handles both direct containment and nested structs,
1104 /// so we don't need a pre-computed RD from the caller.
1105 bool IsExpectedRecordDecl(const Expr *E) const {
1106 QualType Ty = E->getType();
1107 if (Ty->isPointerType())
1108 Ty = Ty->getPointeeType();
1109 const RecordDecl *RD = Ty->getAsRecordDecl();
1110 return RD && RecordContainsField(RD, Field: CountDecl);
1111 }
1112
1113public:
1114 StructAccessBase(const FieldDecl *CountDecl) : CountDecl(CountDecl) {}
1115
1116 //===--------------------------------------------------------------------===//
1117 // Visitor Methods
1118 //===--------------------------------------------------------------------===//
1119
1120 // NOTE: If we build C++ support for counted_by, then we'll have to handle
1121 // horrors like this:
1122 //
1123 // struct S {
1124 // int x, y;
1125 // int blah[] __attribute__((counted_by(x)));
1126 // } s;
1127 //
1128 // int foo(int index, int val) {
1129 // int (S::*IHatePMDs)[] = &S::blah;
1130 // (s.*IHatePMDs)[index] = val;
1131 // }
1132
1133 const Expr *Visit(const Expr *E) {
1134 return ConstStmtVisitor<StructAccessBase, const Expr *>::Visit(S: E);
1135 }
1136
1137 const Expr *VisitStmt(const Stmt *S) { return nullptr; }
1138
1139 // These are the types we expect to return (in order of most to least
1140 // likely):
1141 //
1142 // 1. DeclRefExpr - This is the expression for the base of the structure.
1143 // It's exactly what we want to build an access to the \p counted_by
1144 // field.
1145 // 2. MemberExpr - This is the expression that has the same \p RecordDecl
1146 // as the flexble array member's lexical enclosing \p RecordDecl. This
1147 // allows us to catch things like: "p->p->array"
1148 // 3. CompoundLiteralExpr - This is for people who create something
1149 // heretical like (struct foo has a flexible array member):
1150 //
1151 // (struct foo){ 1, 2 }.blah[idx];
1152 const Expr *VisitDeclRefExpr(const DeclRefExpr *E) {
1153 return IsExpectedRecordDecl(E) ? E : nullptr;
1154 }
1155 const Expr *VisitMemberExpr(const MemberExpr *E) {
1156 if (IsExpectedRecordDecl(E) && E->isArrow())
1157 return E;
1158 const Expr *Res = Visit(E: E->getBase());
1159 return !Res && IsExpectedRecordDecl(E) ? E : Res;
1160 }
1161 const Expr *VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) {
1162 return IsExpectedRecordDecl(E) ? E : nullptr;
1163 }
1164 const Expr *VisitCallExpr(const CallExpr *E) {
1165 return IsExpectedRecordDecl(E) ? E : nullptr;
1166 }
1167
1168 const Expr *VisitArraySubscriptExpr(const ArraySubscriptExpr *E) {
1169 if (IsExpectedRecordDecl(E))
1170 return E;
1171 return Visit(E: E->getBase());
1172 }
1173 const Expr *VisitCastExpr(const CastExpr *E) {
1174 if (E->getCastKind() == CK_LValueToRValue)
1175 return IsExpectedRecordDecl(E) ? E : nullptr;
1176 return Visit(E: E->getSubExpr());
1177 }
1178 const Expr *VisitParenExpr(const ParenExpr *E) {
1179 return Visit(E: E->getSubExpr());
1180 }
1181 const Expr *VisitUnaryAddrOf(const UnaryOperator *E) {
1182 return Visit(E: E->getSubExpr());
1183 }
1184 const Expr *VisitUnaryDeref(const UnaryOperator *E) {
1185 return Visit(E: E->getSubExpr());
1186 }
1187};
1188
1189} // end anonymous namespace
1190
1191using RecIndicesTy = SmallVector<llvm::Value *, 8>;
1192
1193static bool getGEPIndicesToField(CodeGenFunction &CGF, const RecordDecl *RD,
1194 const FieldDecl *Field,
1195 RecIndicesTy &Indices) {
1196 const CGRecordLayout &Layout = CGF.CGM.getTypes().getCGRecordLayout(RD);
1197 int64_t FieldNo = -1;
1198 for (const FieldDecl *FD : RD->fields()) {
1199 if (!Layout.containsFieldDecl(FD))
1200 // This could happen if the field has a struct type that's empty. I don't
1201 // know why either.
1202 continue;
1203
1204 FieldNo = Layout.getLLVMFieldNo(FD);
1205 if (FD == Field) {
1206 Indices.emplace_back(Args: CGF.Builder.getInt32(C: FieldNo));
1207 return true;
1208 }
1209
1210 QualType Ty = FD->getType();
1211 if (Ty->isRecordType()) {
1212 if (getGEPIndicesToField(CGF, RD: Ty->getAsRecordDecl(), Field, Indices)) {
1213 if (RD->isUnion())
1214 FieldNo = 0;
1215 Indices.emplace_back(Args: CGF.Builder.getInt32(C: FieldNo));
1216 return true;
1217 }
1218 }
1219 }
1220
1221 return false;
1222}
1223
1224llvm::Value *CodeGenFunction::GetCountedByFieldExprGEP(
1225 const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl) {
1226 // Walk Base to find the deepest sub-expression whose struct type transitively
1227 // contains CountDecl. This is our GEP anchor — getGEPIndicesToField then
1228 // builds the field indices from that struct down to CountDecl, handling any
1229 // intermediate nesting without requiring us to pre-compute a RecordDecl from
1230 // Base's type or from CountDecl's parent chain.
1231 const Expr *StructBase = StructAccessBase(CountDecl).Visit(E: Base);
1232 if (!StructBase || StructBase->HasSideEffects(Ctx: getContext()))
1233 return nullptr;
1234
1235 // Derive the record type from the anchor expression itself.
1236 QualType StructTy = StructBase->getType();
1237 if (StructTy->isPointerType())
1238 StructTy = StructTy->getPointeeType();
1239 const RecordDecl *RD = StructTy->getAsRecordDecl();
1240 if (!RD)
1241 return nullptr;
1242
1243 llvm::Value *Res = nullptr;
1244 if (StructBase->getType()->isPointerType()) {
1245 LValueBaseInfo BaseInfo;
1246 TBAAAccessInfo TBAAInfo;
1247 Address Addr = EmitPointerWithAlignment(Addr: StructBase, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
1248 Res = Addr.emitRawPointer(CGF&: *this);
1249 } else if (StructBase->isLValue()) {
1250 LValue LV = EmitLValue(E: StructBase);
1251 Address Addr = LV.getAddress();
1252 Res = Addr.emitRawPointer(CGF&: *this);
1253 } else {
1254 return nullptr;
1255 }
1256
1257 RecIndicesTy Indices;
1258 getGEPIndicesToField(CGF&: *this, RD, Field: CountDecl, Indices);
1259 if (Indices.empty())
1260 return nullptr;
1261
1262 Indices.push_back(Elt: Builder.getInt32(C: 0));
1263 CanQualType T = CGM.getContext().getCanonicalTagType(TD: RD);
1264 return Builder.CreateInBoundsGEP(Ty: ConvertType(T), Ptr: Res,
1265 IdxList: RecIndicesTy(llvm::reverse(C&: Indices)),
1266 Name: "counted_by.gep");
1267}
1268
1269/// This method is typically called in contexts where we can't generate
1270/// side-effects, like in __builtin_dynamic_object_size. When finding
1271/// expressions, only choose those that have either already been emitted or can
1272/// be loaded without side-effects.
1273///
1274/// - \p FAMDecl: the \p Decl for the flexible array member. It may not be
1275/// within the top-level struct.
1276/// - \p CountDecl: must be within the same non-anonymous struct as \p FAMDecl.
1277llvm::Value *CodeGenFunction::EmitLoadOfCountedByField(
1278 const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl) {
1279 if (llvm::Value *GEP = GetCountedByFieldExprGEP(Base, FAMDecl, CountDecl))
1280 return Builder.CreateAlignedLoad(Ty: ConvertType(T: CountDecl->getType()), Addr: GEP,
1281 Align: getIntAlign(), Name: "counted_by.load");
1282 return nullptr;
1283}
1284
1285void CodeGenFunction::EmitBoundsCheck(const Expr *ArrayExpr,
1286 const Expr *ArrayExprBase,
1287 llvm::Value *IndexVal, QualType IndexType,
1288 bool Accessed) {
1289 assert(SanOpts.has(SanitizerKind::ArrayBounds) &&
1290 "should not be called unless adding bounds checks");
1291 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
1292 getLangOpts().getStrictFlexArraysLevel();
1293 QualType ArrayExprBaseType;
1294 llvm::Value *BoundsVal = getArrayIndexingBound(
1295 CGF&: *this, Base: ArrayExprBase, IndexedType&: ArrayExprBaseType, StrictFlexArraysLevel);
1296
1297 EmitBoundsCheckImpl(ArrayExpr, ArrayBaseType: ArrayExprBaseType, IndexVal, IndexType,
1298 BoundsVal, BoundsType: getContext().getSizeType(), Accessed);
1299}
1300
1301void CodeGenFunction::EmitBoundsCheckImpl(const Expr *ArrayExpr,
1302 QualType ArrayBaseType,
1303 llvm::Value *IndexVal,
1304 QualType IndexType,
1305 llvm::Value *BoundsVal,
1306 QualType BoundsType, bool Accessed) {
1307 if (!BoundsVal)
1308 return;
1309
1310 auto CheckKind = SanitizerKind::SO_ArrayBounds;
1311 auto CheckHandler = SanitizerHandler::OutOfBounds;
1312 SanitizerDebugLocation SanScope(this, {CheckKind}, CheckHandler);
1313
1314 // All hail the C implicit type conversion rules!!!
1315 bool IndexSigned = IndexType->isSignedIntegerOrEnumerationType();
1316 bool BoundsSigned = BoundsType->isSignedIntegerOrEnumerationType();
1317
1318 const ASTContext &Ctx = getContext();
1319 llvm::Type *Ty = ConvertType(
1320 T: Ctx.getTypeSize(T: IndexType) >= Ctx.getTypeSize(T: BoundsType) ? IndexType
1321 : BoundsType);
1322
1323 llvm::Value *IndexInst = Builder.CreateIntCast(V: IndexVal, DestTy: Ty, isSigned: IndexSigned);
1324 llvm::Value *BoundsInst = Builder.CreateIntCast(V: BoundsVal, DestTy: Ty, isSigned: false);
1325
1326 llvm::Constant *StaticData[] = {
1327 EmitCheckSourceLocation(Loc: ArrayExpr->getExprLoc()),
1328 EmitCheckTypeDescriptor(T: ArrayBaseType),
1329 EmitCheckTypeDescriptor(T: IndexType),
1330 };
1331
1332 llvm::Value *Check = Accessed ? Builder.CreateICmpULT(LHS: IndexInst, RHS: BoundsInst)
1333 : Builder.CreateICmpULE(LHS: IndexInst, RHS: BoundsInst);
1334
1335 if (BoundsSigned) {
1336 // Don't allow a negative bounds.
1337 llvm::Value *Cmp = Builder.CreateICmpSGT(
1338 LHS: BoundsVal, RHS: llvm::ConstantInt::get(Ty: BoundsVal->getType(), V: 0));
1339 Check = Builder.CreateAnd(LHS: Cmp, RHS: Check);
1340 }
1341
1342 EmitCheck(Checked: std::make_pair(x&: Check, y&: CheckKind), Check: CheckHandler, StaticArgs: StaticData,
1343 DynamicArgs: IndexInst);
1344}
1345
1346llvm::MDNode *CodeGenFunction::buildAllocToken(QualType AllocType) {
1347 auto ATMD = infer_alloc::getAllocTokenMetadata(T: AllocType, Ctx: getContext());
1348 if (!ATMD)
1349 return nullptr;
1350
1351 llvm::MDBuilder MDB(getLLVMContext());
1352 auto *TypeNameMD = MDB.createString(Str: ATMD->TypeName);
1353 auto *ContainsPtrC = Builder.getInt1(V: ATMD->ContainsPointer);
1354 auto *ContainsPtrMD = MDB.createConstant(C: ContainsPtrC);
1355
1356 // Format: !{<type-name>, <contains-pointer>}
1357 return llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: {TypeNameMD, ContainsPtrMD});
1358}
1359
1360void CodeGenFunction::EmitAllocToken(llvm::CallBase *CB, QualType AllocType) {
1361 assert(SanOpts.has(SanitizerKind::AllocToken) &&
1362 "Only needed with -fsanitize=alloc-token");
1363 CB->setMetadata(KindID: llvm::LLVMContext::MD_alloc_token,
1364 Node: buildAllocToken(AllocType));
1365}
1366
1367llvm::MDNode *CodeGenFunction::buildAllocToken(const CallExpr *E) {
1368 QualType AllocType = infer_alloc::inferPossibleType(E, Ctx: getContext(), CastE: CurCast);
1369 if (!AllocType.isNull())
1370 return buildAllocToken(AllocType);
1371 return nullptr;
1372}
1373
1374void CodeGenFunction::EmitAllocToken(llvm::CallBase *CB, const CallExpr *E) {
1375 assert(SanOpts.has(SanitizerKind::AllocToken) &&
1376 "Only needed with -fsanitize=alloc-token");
1377 if (llvm::MDNode *MDN = buildAllocToken(E))
1378 CB->setMetadata(KindID: llvm::LLVMContext::MD_alloc_token, Node: MDN);
1379}
1380
1381CodeGenFunction::ComplexPairTy CodeGenFunction::
1382EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
1383 bool isInc, bool isPre) {
1384 ComplexPairTy InVal = EmitLoadOfComplex(src: LV, loc: E->getExprLoc());
1385
1386 llvm::Value *NextVal;
1387 if (isa<llvm::IntegerType>(Val: InVal.first->getType())) {
1388 uint64_t AmountVal = isInc ? 1 : -1;
1389 NextVal = llvm::ConstantInt::get(Ty: InVal.first->getType(), V: AmountVal, IsSigned: true);
1390
1391 // Add the inc/dec to the real part.
1392 NextVal = Builder.CreateAdd(LHS: InVal.first, RHS: NextVal, Name: isInc ? "inc" : "dec");
1393 } else {
1394 QualType ElemTy = E->getType()->castAs<ComplexType>()->getElementType();
1395 llvm::APFloat FVal(getContext().getFloatTypeSemantics(T: ElemTy), 1);
1396 if (!isInc)
1397 FVal.changeSign();
1398 NextVal = llvm::ConstantFP::get(Context&: getLLVMContext(), V: FVal);
1399
1400 // Add the inc/dec to the real part.
1401 NextVal = Builder.CreateFAdd(L: InVal.first, R: NextVal, Name: isInc ? "inc" : "dec");
1402 }
1403
1404 ComplexPairTy IncVal(NextVal, InVal.second);
1405
1406 // Store the updated result through the lvalue.
1407 EmitStoreOfComplex(V: IncVal, dest: LV, /*init*/ isInit: false);
1408 if (getLangOpts().OpenMP)
1409 CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF&: *this,
1410 LHS: E->getSubExpr());
1411
1412 // If this is a postinc, return the value read from memory, otherwise use the
1413 // updated value.
1414 return isPre ? IncVal : InVal;
1415}
1416
1417void CodeGenModule::EmitExplicitCastExprType(const ExplicitCastExpr *E,
1418 CodeGenFunction *CGF) {
1419 // Bind VLAs in the cast type.
1420 if (CGF && E->getType()->isVariablyModifiedType())
1421 CGF->EmitVariablyModifiedType(Ty: E->getType());
1422
1423 if (CGDebugInfo *DI = getModuleDebugInfo())
1424 DI->EmitExplicitCastType(Ty: E->getType());
1425}
1426
1427//===----------------------------------------------------------------------===//
1428// LValue Expression Emission
1429//===----------------------------------------------------------------------===//
1430
1431static CharUnits getArrayElementAlign(CharUnits arrayAlign, llvm::Value *idx,
1432 CharUnits eltSize) {
1433 // If we have a constant index, we can use the exact offset of the
1434 // element we're accessing.
1435 if (auto *constantIdx = dyn_cast<llvm::ConstantInt>(Val: idx)) {
1436 CharUnits offset = constantIdx->getZExtValue() * eltSize;
1437 return arrayAlign.alignmentAtOffset(offset);
1438 }
1439
1440 // Otherwise, use the worst-case alignment for any element.
1441 return arrayAlign.alignmentOfArrayElement(elementSize: eltSize);
1442}
1443
1444/// Emit pointer + index arithmetic.
1445static Address emitPointerArithmetic(CodeGenFunction &CGF,
1446 const BinaryOperator *BO,
1447 LValueBaseInfo *BaseInfo,
1448 TBAAAccessInfo *TBAAInfo,
1449 KnownNonNull_t IsKnownNonNull) {
1450 assert(BO->isAdditiveOp() && "Expect an addition or subtraction.");
1451 Expr *pointerOperand = BO->getLHS();
1452 Expr *indexOperand = BO->getRHS();
1453 bool isSubtraction = BO->getOpcode() == BO_Sub;
1454
1455 Address BaseAddr = Address::invalid();
1456 llvm::Value *index = nullptr;
1457 // In a subtraction, the LHS is always the pointer.
1458 // Note: do not change the evaluation order.
1459 if (!isSubtraction && !pointerOperand->getType()->isAnyPointerType()) {
1460 std::swap(a&: pointerOperand, b&: indexOperand);
1461 index = CGF.EmitScalarExpr(E: indexOperand);
1462 BaseAddr = CGF.EmitPointerWithAlignment(Addr: pointerOperand, BaseInfo, TBAAInfo,
1463 IsKnownNonNull: NotKnownNonNull);
1464 } else {
1465 BaseAddr = CGF.EmitPointerWithAlignment(Addr: pointerOperand, BaseInfo, TBAAInfo,
1466 IsKnownNonNull: NotKnownNonNull);
1467 index = CGF.EmitScalarExpr(E: indexOperand);
1468 }
1469
1470 llvm::Value *pointer = BaseAddr.getBasePointer();
1471 llvm::Value *Res = CGF.EmitPointerArithmetic(
1472 BO, pointerOperand, pointer, indexOperand, index, isSubtraction);
1473 QualType PointeeTy = BO->getType()->getPointeeType();
1474 CharUnits Align =
1475 getArrayElementAlign(arrayAlign: BaseAddr.getAlignment(), idx: index,
1476 eltSize: CGF.getContext().getTypeSizeInChars(T: PointeeTy));
1477 return Address(Res, CGF.ConvertTypeForMem(T: PointeeTy), Align,
1478 CGF.CGM.getPointerAuthInfoForPointeeType(type: PointeeTy),
1479 /*Offset=*/nullptr, IsKnownNonNull);
1480}
1481
1482static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo,
1483 TBAAAccessInfo *TBAAInfo,
1484 KnownNonNull_t IsKnownNonNull,
1485 CodeGenFunction &CGF) {
1486 // We allow this with ObjC object pointers because of fragile ABIs.
1487 assert(E->getType()->isPointerType() ||
1488 E->getType()->isObjCObjectPointerType());
1489 E = E->IgnoreParens();
1490
1491 // Casts:
1492 if (const CastExpr *CE = dyn_cast<CastExpr>(Val: E)) {
1493 if (const auto *ECE = dyn_cast<ExplicitCastExpr>(Val: CE))
1494 CGF.CGM.EmitExplicitCastExprType(E: ECE, CGF: &CGF);
1495
1496 switch (CE->getCastKind()) {
1497 // Non-converting casts (but not C's implicit conversion from void*).
1498 case CK_BitCast:
1499 case CK_NoOp:
1500 case CK_AddressSpaceConversion:
1501 if (auto PtrTy = CE->getSubExpr()->getType()->getAs<PointerType>()) {
1502 if (PtrTy->getPointeeType()->isVoidType())
1503 break;
1504
1505 LValueBaseInfo InnerBaseInfo;
1506 TBAAAccessInfo InnerTBAAInfo;
1507 Address Addr = CGF.EmitPointerWithAlignment(
1508 Addr: CE->getSubExpr(), BaseInfo: &InnerBaseInfo, TBAAInfo: &InnerTBAAInfo, IsKnownNonNull);
1509 if (BaseInfo) *BaseInfo = InnerBaseInfo;
1510 if (TBAAInfo) *TBAAInfo = InnerTBAAInfo;
1511
1512 if (isa<ExplicitCastExpr>(Val: CE)) {
1513 LValueBaseInfo TargetTypeBaseInfo;
1514 TBAAAccessInfo TargetTypeTBAAInfo;
1515 CharUnits Align = CGF.CGM.getNaturalPointeeTypeAlignment(
1516 T: E->getType(), BaseInfo: &TargetTypeBaseInfo, TBAAInfo: &TargetTypeTBAAInfo);
1517 if (TBAAInfo)
1518 *TBAAInfo =
1519 CGF.CGM.mergeTBAAInfoForCast(SourceInfo: *TBAAInfo, TargetInfo: TargetTypeTBAAInfo);
1520 // If the source l-value is opaque, honor the alignment of the
1521 // casted-to type.
1522 if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) {
1523 if (BaseInfo)
1524 BaseInfo->mergeForCast(Info: TargetTypeBaseInfo);
1525 Addr.setAlignment(Align);
1526 }
1527 }
1528
1529 if (CGF.SanOpts.has(K: SanitizerKind::CFIUnrelatedCast) &&
1530 CE->getCastKind() == CK_BitCast) {
1531 if (auto PT = E->getType()->getAs<PointerType>())
1532 CGF.EmitVTablePtrCheckForCast(T: PT->getPointeeType(), Derived: Addr,
1533 /*MayBeNull=*/true,
1534 TCK: CodeGenFunction::CFITCK_UnrelatedCast,
1535 Loc: CE->getBeginLoc());
1536 }
1537
1538 llvm::Type *ElemTy =
1539 CGF.ConvertTypeForMem(T: E->getType()->getPointeeType());
1540 Addr = Addr.withElementType(ElemTy);
1541 if (CE->getCastKind() == CK_AddressSpaceConversion)
1542 Addr = CGF.Builder.CreateAddrSpaceCast(
1543 Addr, Ty: CGF.ConvertType(T: E->getType()), ElementTy: ElemTy);
1544
1545 return CGF.authPointerToPointerCast(Ptr: Addr, SourceType: CE->getSubExpr()->getType(),
1546 DestType: CE->getType());
1547 }
1548 break;
1549
1550 // Array-to-pointer decay.
1551 case CK_ArrayToPointerDecay:
1552 return CGF.EmitArrayToPointerDecay(Array: CE->getSubExpr(), BaseInfo, TBAAInfo);
1553
1554 // Derived-to-base conversions.
1555 case CK_UncheckedDerivedToBase:
1556 case CK_DerivedToBase: {
1557 // TODO: Support accesses to members of base classes in TBAA. For now, we
1558 // conservatively pretend that the complete object is of the base class
1559 // type.
1560 if (TBAAInfo)
1561 *TBAAInfo = CGF.CGM.getTBAAAccessInfo(AccessType: E->getType());
1562 Address Addr = CGF.EmitPointerWithAlignment(
1563 Addr: CE->getSubExpr(), BaseInfo, TBAAInfo: nullptr,
1564 IsKnownNonNull: (KnownNonNull_t)(IsKnownNonNull ||
1565 CE->getCastKind() == CK_UncheckedDerivedToBase));
1566 auto Derived = CE->getSubExpr()->getType()->getPointeeCXXRecordDecl();
1567 return CGF.GetAddressOfBaseClass(
1568 Value: Addr, Derived, PathBegin: CE->path_begin(), PathEnd: CE->path_end(),
1569 NullCheckValue: CGF.ShouldNullCheckClassCastValue(Cast: CE), Loc: CE->getExprLoc());
1570 }
1571
1572 // TODO: Is there any reason to treat base-to-derived conversions
1573 // specially?
1574 default:
1575 break;
1576 }
1577 }
1578
1579 // Unary &.
1580 if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(Val: E)) {
1581 if (UO->getOpcode() == UO_AddrOf) {
1582 LValue LV = CGF.EmitLValue(E: UO->getSubExpr(), IsKnownNonNull);
1583 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1584 if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1585 return LV.getAddress();
1586 }
1587 }
1588
1589 // std::addressof and variants.
1590 if (auto *Call = dyn_cast<CallExpr>(Val: E)) {
1591 switch (Call->getBuiltinCallee()) {
1592 default:
1593 break;
1594 case Builtin::BIaddressof:
1595 case Builtin::BI__addressof:
1596 case Builtin::BI__builtin_addressof: {
1597 LValue LV = CGF.EmitLValue(E: Call->getArg(Arg: 0), IsKnownNonNull);
1598 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1599 if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1600 return LV.getAddress();
1601 }
1602 }
1603 }
1604
1605 // Pointer arithmetic: pointer +/- index.
1606 if (auto *BO = dyn_cast<BinaryOperator>(Val: E)) {
1607 if (BO->isAdditiveOp())
1608 return emitPointerArithmetic(CGF, BO, BaseInfo, TBAAInfo, IsKnownNonNull);
1609 }
1610
1611 // TODO: conditional operators, comma.
1612
1613 // Otherwise, use the alignment of the type.
1614 return CGF.makeNaturalAddressForPointer(
1615 Ptr: CGF.EmitScalarExpr(E), T: E->getType()->getPointeeType(), Alignment: CharUnits(),
1616 /*ForPointeeType=*/true, BaseInfo, TBAAInfo, IsKnownNonNull);
1617}
1618
1619/// EmitPointerWithAlignment - Given an expression of pointer type, try to
1620/// derive a more accurate bound on the alignment of the pointer.
1621Address CodeGenFunction::EmitPointerWithAlignment(
1622 const Expr *E, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo,
1623 KnownNonNull_t IsKnownNonNull) {
1624 Address Addr =
1625 ::EmitPointerWithAlignment(E, BaseInfo, TBAAInfo, IsKnownNonNull, CGF&: *this);
1626 if (IsKnownNonNull && !Addr.isKnownNonNull())
1627 Addr.setKnownNonNull();
1628 return Addr;
1629}
1630
1631llvm::Value *CodeGenFunction::EmitNonNullRValueCheck(RValue RV, QualType T) {
1632 llvm::Value *V = RV.getScalarVal();
1633 if (auto MPT = T->getAs<MemberPointerType>())
1634 return CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF&: *this, MemPtr: V, MPT);
1635 return Builder.CreateICmpNE(LHS: V, RHS: llvm::Constant::getNullValue(Ty: V->getType()));
1636}
1637
1638RValue CodeGenFunction::GetUndefRValue(QualType Ty) {
1639 if (Ty->isVoidType())
1640 return RValue::get(V: nullptr);
1641
1642 switch (getEvaluationKind(T: Ty)) {
1643 case TEK_Complex: {
1644 llvm::Type *EltTy =
1645 ConvertType(T: Ty->castAs<ComplexType>()->getElementType());
1646 llvm::Value *U = llvm::UndefValue::get(T: EltTy);
1647 return RValue::getComplex(C: std::make_pair(x&: U, y&: U));
1648 }
1649
1650 // If this is a use of an undefined aggregate type, the aggregate must have an
1651 // identifiable address. Just because the contents of the value are undefined
1652 // doesn't mean that the address can't be taken and compared.
1653 case TEK_Aggregate: {
1654 Address DestPtr = CreateMemTemp(Ty, Name: "undef.agg.tmp");
1655 return RValue::getAggregate(addr: DestPtr);
1656 }
1657
1658 case TEK_Scalar:
1659 return RValue::get(V: llvm::UndefValue::get(T: ConvertType(T: Ty)));
1660 }
1661 llvm_unreachable("bad evaluation kind");
1662}
1663
1664RValue CodeGenFunction::EmitUnsupportedRValue(const Expr *E,
1665 const char *Name) {
1666 ErrorUnsupported(S: E, Type: Name);
1667 return GetUndefRValue(Ty: E->getType());
1668}
1669
1670LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E,
1671 const char *Name) {
1672 ErrorUnsupported(S: E, Type: Name);
1673 llvm::Type *ElTy = ConvertType(T: E->getType());
1674 llvm::Type *Ty = DefaultPtrTy;
1675 return MakeAddrLValue(
1676 Addr: Address(llvm::UndefValue::get(T: Ty), ElTy, CharUnits::One()), T: E->getType());
1677}
1678
1679bool CodeGenFunction::IsWrappedCXXThis(const Expr *Obj) {
1680 const Expr *Base = Obj;
1681 while (!isa<CXXThisExpr>(Val: Base)) {
1682 // The result of a dynamic_cast can be null.
1683 if (isa<CXXDynamicCastExpr>(Val: Base))
1684 return false;
1685
1686 if (const auto *CE = dyn_cast<CastExpr>(Val: Base)) {
1687 Base = CE->getSubExpr();
1688 } else if (const auto *PE = dyn_cast<ParenExpr>(Val: Base)) {
1689 Base = PE->getSubExpr();
1690 } else if (const auto *UO = dyn_cast<UnaryOperator>(Val: Base)) {
1691 if (UO->getOpcode() == UO_Extension)
1692 Base = UO->getSubExpr();
1693 else
1694 return false;
1695 } else {
1696 return false;
1697 }
1698 }
1699 return true;
1700}
1701
1702LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {
1703 LValue LV;
1704 if (SanOpts.has(K: SanitizerKind::ArrayBounds) && isa<ArraySubscriptExpr>(Val: E))
1705 LV = EmitArraySubscriptExpr(E: cast<ArraySubscriptExpr>(Val: E), /*Accessed*/true);
1706 else
1707 LV = EmitLValue(E);
1708 if (!isa<DeclRefExpr>(Val: E) && !LV.isBitField() && LV.isSimple()) {
1709 SanitizerSet SkippedChecks;
1710 if (const auto *ME = dyn_cast<MemberExpr>(Val: E)) {
1711 bool IsBaseCXXThis = IsWrappedCXXThis(Obj: ME->getBase());
1712 if (IsBaseCXXThis)
1713 SkippedChecks.set(K: SanitizerKind::Alignment, Value: true);
1714 if (IsBaseCXXThis || isa<DeclRefExpr>(Val: ME->getBase()))
1715 SkippedChecks.set(K: SanitizerKind::Null, Value: true);
1716 }
1717 EmitTypeCheck(TCK, Loc: E->getExprLoc(), LV, Type: E->getType(), SkippedChecks);
1718 }
1719 return LV;
1720}
1721
1722/// EmitLValue - Emit code to compute a designator that specifies the location
1723/// of the expression.
1724///
1725/// This can return one of two things: a simple address or a bitfield reference.
1726/// In either case, the LLVM Value* in the LValue structure is guaranteed to be
1727/// an LLVM pointer type.
1728///
1729/// If this returns a bitfield reference, nothing about the pointee type of the
1730/// LLVM value is known: For example, it may not be a pointer to an integer.
1731///
1732/// If this returns a normal address, and if the lvalue's C type is fixed size,
1733/// this method guarantees that the returned pointer type will point to an LLVM
1734/// type of the same size of the lvalue's type. If the lvalue has a variable
1735/// length type, this is not possible.
1736///
1737LValue CodeGenFunction::EmitLValue(const Expr *E,
1738 KnownNonNull_t IsKnownNonNull) {
1739 // Running with sufficient stack space to avoid deeply nested expressions
1740 // cause a stack overflow.
1741 LValue LV;
1742 CGM.runWithSufficientStackSpace(
1743 Loc: E->getExprLoc(), Fn: [&] { LV = EmitLValueHelper(E, IsKnownNonNull); });
1744
1745 if (IsKnownNonNull && !LV.isKnownNonNull())
1746 LV.setKnownNonNull();
1747 return LV;
1748}
1749
1750LValue CodeGenFunction::EmitLValueHelper(const Expr *E,
1751 KnownNonNull_t IsKnownNonNull) {
1752 ApplyDebugLocation DL(*this, E);
1753 switch (E->getStmtClass()) {
1754 default: return EmitUnsupportedLValue(E, Name: "l-value expression");
1755
1756 case Expr::ObjCPropertyRefExprClass:
1757 llvm_unreachable("cannot emit a property reference directly");
1758
1759 case Expr::ObjCSelectorExprClass:
1760 return EmitObjCSelectorLValue(E: cast<ObjCSelectorExpr>(Val: E));
1761 case Expr::ObjCIsaExprClass:
1762 return EmitObjCIsaExpr(E: cast<ObjCIsaExpr>(Val: E));
1763 case Expr::BinaryOperatorClass:
1764 return EmitBinaryOperatorLValue(E: cast<BinaryOperator>(Val: E));
1765 case Expr::CompoundAssignOperatorClass: {
1766 QualType Ty = E->getType();
1767 if (const AtomicType *AT = Ty->getAs<AtomicType>())
1768 Ty = AT->getValueType();
1769 if (!Ty->isAnyComplexType())
1770 return EmitCompoundAssignmentLValue(E: cast<CompoundAssignOperator>(Val: E));
1771 return EmitComplexCompoundAssignmentLValue(E: cast<CompoundAssignOperator>(Val: E));
1772 }
1773 case Expr::CallExprClass:
1774 case Expr::CXXMemberCallExprClass:
1775 case Expr::CXXOperatorCallExprClass:
1776 case Expr::UserDefinedLiteralClass:
1777 return EmitCallExprLValue(E: cast<CallExpr>(Val: E));
1778 case Expr::CXXRewrittenBinaryOperatorClass:
1779 return EmitLValue(E: cast<CXXRewrittenBinaryOperator>(Val: E)->getSemanticForm(),
1780 IsKnownNonNull);
1781 case Expr::VAArgExprClass:
1782 return EmitVAArgExprLValue(E: cast<VAArgExpr>(Val: E));
1783 case Expr::DeclRefExprClass:
1784 return EmitDeclRefLValue(E: cast<DeclRefExpr>(Val: E));
1785 case Expr::ConstantExprClass: {
1786 const ConstantExpr *CE = cast<ConstantExpr>(Val: E);
1787 if (llvm::Value *Result = ConstantEmitter(*this).tryEmitConstantExpr(CE))
1788 return MakeNaturalAlignPointeeAddrLValue(V: Result, T: CE->getType());
1789 return EmitLValue(E: cast<ConstantExpr>(Val: E)->getSubExpr(), IsKnownNonNull);
1790 }
1791 case Expr::ParenExprClass:
1792 return EmitLValue(E: cast<ParenExpr>(Val: E)->getSubExpr(), IsKnownNonNull);
1793 case Expr::GenericSelectionExprClass:
1794 return EmitLValue(E: cast<GenericSelectionExpr>(Val: E)->getResultExpr(),
1795 IsKnownNonNull);
1796 case Expr::PredefinedExprClass:
1797 return EmitPredefinedLValue(E: cast<PredefinedExpr>(Val: E));
1798 case Expr::StringLiteralClass:
1799 return EmitStringLiteralLValue(E: cast<StringLiteral>(Val: E));
1800 case Expr::ObjCEncodeExprClass:
1801 return EmitObjCEncodeExprLValue(E: cast<ObjCEncodeExpr>(Val: E));
1802 case Expr::PseudoObjectExprClass:
1803 return EmitPseudoObjectLValue(e: cast<PseudoObjectExpr>(Val: E));
1804 case Expr::InitListExprClass:
1805 return EmitInitListLValue(E: cast<InitListExpr>(Val: E));
1806 case Expr::CXXTemporaryObjectExprClass:
1807 case Expr::CXXConstructExprClass:
1808 return EmitCXXConstructLValue(E: cast<CXXConstructExpr>(Val: E));
1809 case Expr::CXXBindTemporaryExprClass:
1810 return EmitCXXBindTemporaryLValue(E: cast<CXXBindTemporaryExpr>(Val: E));
1811 case Expr::CXXUuidofExprClass:
1812 return EmitCXXUuidofLValue(E: cast<CXXUuidofExpr>(Val: E));
1813 case Expr::LambdaExprClass:
1814 return EmitAggExprToLValue(E);
1815
1816 case Expr::ExprWithCleanupsClass: {
1817 const auto *cleanups = cast<ExprWithCleanups>(Val: E);
1818 RunCleanupsScope Scope(*this);
1819 LValue LV = EmitLValue(E: cleanups->getSubExpr(), IsKnownNonNull);
1820 if (LV.isSimple()) {
1821 // Defend against branches out of gnu statement expressions surrounded by
1822 // cleanups.
1823 Address Addr = LV.getAddress();
1824 llvm::Value *V = Addr.getBasePointer();
1825 Scope.ForceCleanup(ValuesToReload: {&V});
1826 Addr.replaceBasePointer(P: V);
1827 return LValue::MakeAddr(Addr, type: LV.getType(), Context&: getContext(),
1828 BaseInfo: LV.getBaseInfo(), TBAAInfo: LV.getTBAAInfo());
1829 }
1830 // FIXME: Is it possible to create an ExprWithCleanups that produces a
1831 // bitfield lvalue or some other non-simple lvalue?
1832 return LV;
1833 }
1834
1835 case Expr::CXXDefaultArgExprClass: {
1836 auto *DAE = cast<CXXDefaultArgExpr>(Val: E);
1837 CXXDefaultArgExprScope Scope(*this, DAE);
1838 return EmitLValue(E: DAE->getExpr(), IsKnownNonNull);
1839 }
1840 case Expr::CXXDefaultInitExprClass: {
1841 auto *DIE = cast<CXXDefaultInitExpr>(Val: E);
1842 CXXDefaultInitExprScope Scope(*this, DIE);
1843 return EmitLValue(E: DIE->getExpr(), IsKnownNonNull);
1844 }
1845 case Expr::CXXTypeidExprClass:
1846 return EmitCXXTypeidLValue(E: cast<CXXTypeidExpr>(Val: E));
1847
1848 case Expr::ObjCMessageExprClass:
1849 return EmitObjCMessageExprLValue(E: cast<ObjCMessageExpr>(Val: E));
1850 case Expr::ObjCIvarRefExprClass:
1851 return EmitObjCIvarRefLValue(E: cast<ObjCIvarRefExpr>(Val: E));
1852 case Expr::StmtExprClass:
1853 return EmitStmtExprLValue(E: cast<StmtExpr>(Val: E));
1854 case Expr::UnaryOperatorClass:
1855 return EmitUnaryOpLValue(E: cast<UnaryOperator>(Val: E));
1856 case Expr::ArraySubscriptExprClass:
1857 return EmitArraySubscriptExpr(E: cast<ArraySubscriptExpr>(Val: E));
1858 case Expr::MatrixSingleSubscriptExprClass:
1859 return EmitMatrixSingleSubscriptExpr(E: cast<MatrixSingleSubscriptExpr>(Val: E));
1860 case Expr::MatrixSubscriptExprClass:
1861 return EmitMatrixSubscriptExpr(E: cast<MatrixSubscriptExpr>(Val: E));
1862 case Expr::ArraySectionExprClass:
1863 return EmitArraySectionExpr(E: cast<ArraySectionExpr>(Val: E));
1864 case Expr::ExtVectorElementExprClass:
1865 return EmitExtVectorElementExpr(E: cast<ExtVectorElementExpr>(Val: E));
1866 case Expr::MatrixElementExprClass:
1867 return EmitMatrixElementExpr(E: cast<MatrixElementExpr>(Val: E));
1868 case Expr::CXXThisExprClass:
1869 return MakeAddrLValue(Addr: LoadCXXThisAddress(), T: E->getType());
1870 case Expr::MemberExprClass:
1871 return EmitMemberExpr(E: cast<MemberExpr>(Val: E));
1872 case Expr::CompoundLiteralExprClass:
1873 return EmitCompoundLiteralLValue(E: cast<CompoundLiteralExpr>(Val: E));
1874 case Expr::ConditionalOperatorClass:
1875 return EmitConditionalOperatorLValue(E: cast<ConditionalOperator>(Val: E));
1876 case Expr::BinaryConditionalOperatorClass:
1877 return EmitConditionalOperatorLValue(E: cast<BinaryConditionalOperator>(Val: E));
1878 case Expr::ChooseExprClass:
1879 return EmitLValue(E: cast<ChooseExpr>(Val: E)->getChosenSubExpr(), IsKnownNonNull);
1880 case Expr::OpaqueValueExprClass:
1881 return EmitOpaqueValueLValue(e: cast<OpaqueValueExpr>(Val: E));
1882 case Expr::SubstNonTypeTemplateParmExprClass:
1883 return EmitLValue(E: cast<SubstNonTypeTemplateParmExpr>(Val: E)->getReplacement(),
1884 IsKnownNonNull);
1885 case Expr::ImplicitCastExprClass:
1886 case Expr::CStyleCastExprClass:
1887 case Expr::CXXFunctionalCastExprClass:
1888 case Expr::CXXStaticCastExprClass:
1889 case Expr::CXXDynamicCastExprClass:
1890 case Expr::CXXReinterpretCastExprClass:
1891 case Expr::CXXConstCastExprClass:
1892 case Expr::CXXAddrspaceCastExprClass:
1893 case Expr::ObjCBridgedCastExprClass:
1894 return EmitCastLValue(E: cast<CastExpr>(Val: E));
1895
1896 case Expr::MaterializeTemporaryExprClass:
1897 return EmitMaterializeTemporaryExpr(M: cast<MaterializeTemporaryExpr>(Val: E));
1898
1899 case Expr::CoawaitExprClass:
1900 return EmitCoawaitLValue(E: cast<CoawaitExpr>(Val: E));
1901 case Expr::CoyieldExprClass:
1902 return EmitCoyieldLValue(E: cast<CoyieldExpr>(Val: E));
1903 case Expr::PackIndexingExprClass:
1904 return EmitLValue(E: cast<PackIndexingExpr>(Val: E)->getSelectedExpr());
1905 case Expr::HLSLOutArgExprClass:
1906 llvm_unreachable("cannot emit a HLSL out argument directly");
1907 }
1908}
1909
1910/// Given an object of the given canonical type, can we safely copy a
1911/// value out of it based on its initializer?
1912static bool isConstantEmittableObjectType(QualType type) {
1913 assert(type.isCanonical());
1914 assert(!type->isReferenceType());
1915
1916 // Must be const-qualified but non-volatile.
1917 Qualifiers qs = type.getLocalQualifiers();
1918 if (!qs.hasConst() || qs.hasVolatile()) return false;
1919
1920 // Otherwise, all object types satisfy this except C++ classes with
1921 // mutable subobjects or non-trivial copy/destroy behavior.
1922 if (const auto *RT = dyn_cast<RecordType>(Val&: type))
1923 if (const auto *RD = dyn_cast<CXXRecordDecl>(Val: RT->getDecl())) {
1924 RD = RD->getDefinitionOrSelf();
1925 if (RD->hasMutableFields() || !RD->isTrivial())
1926 return false;
1927 }
1928
1929 return true;
1930}
1931
1932/// Can we constant-emit a load of a reference to a variable of the
1933/// given type? This is different from predicates like
1934/// Decl::mightBeUsableInConstantExpressions because we do want it to apply
1935/// in situations that don't necessarily satisfy the language's rules
1936/// for this (e.g. C++'s ODR-use rules). For example, we want to able
1937/// to do this with const float variables even if those variables
1938/// aren't marked 'constexpr'.
1939enum ConstantEmissionKind {
1940 CEK_None,
1941 CEK_AsReferenceOnly,
1942 CEK_AsValueOrReference,
1943 CEK_AsValueOnly
1944};
1945static ConstantEmissionKind checkVarTypeForConstantEmission(QualType type) {
1946 type = type.getCanonicalType();
1947 if (const auto *ref = dyn_cast<ReferenceType>(Val&: type)) {
1948 if (isConstantEmittableObjectType(type: ref->getPointeeType()))
1949 return CEK_AsValueOrReference;
1950 return CEK_AsReferenceOnly;
1951 }
1952 if (isConstantEmittableObjectType(type))
1953 return CEK_AsValueOnly;
1954 return CEK_None;
1955}
1956
1957/// Try to emit a reference to the given value without producing it as
1958/// an l-value. This is just an optimization, but it avoids us needing
1959/// to emit global copies of variables if they're named without triggering
1960/// a formal use in a context where we can't emit a direct reference to them,
1961/// for instance if a block or lambda or a member of a local class uses a
1962/// const int variable or constexpr variable from an enclosing function.
1963CodeGenFunction::ConstantEmission
1964CodeGenFunction::tryEmitAsConstant(const DeclRefExpr *RefExpr) {
1965 const ValueDecl *Value = RefExpr->getDecl();
1966
1967 // The value needs to be an enum constant or a constant variable.
1968 ConstantEmissionKind CEK;
1969 if (isa<ParmVarDecl>(Val: Value)) {
1970 CEK = CEK_None;
1971 } else if (const auto *var = dyn_cast<VarDecl>(Val: Value)) {
1972 CEK = checkVarTypeForConstantEmission(type: var->getType());
1973 } else if (isa<EnumConstantDecl>(Val: Value)) {
1974 CEK = CEK_AsValueOnly;
1975 } else {
1976 CEK = CEK_None;
1977 }
1978 if (CEK == CEK_None) return ConstantEmission();
1979
1980 Expr::EvalResult result;
1981 bool resultIsReference;
1982 QualType resultType;
1983
1984 // It's best to evaluate all the way as an r-value if that's permitted.
1985 if (CEK != CEK_AsReferenceOnly &&
1986 RefExpr->EvaluateAsRValue(Result&: result, Ctx: getContext())) {
1987 resultIsReference = false;
1988 resultType = RefExpr->getType().getUnqualifiedType();
1989
1990 // Otherwise, try to evaluate as an l-value.
1991 } else if (CEK != CEK_AsValueOnly &&
1992 RefExpr->EvaluateAsLValue(Result&: result, Ctx: getContext())) {
1993 resultIsReference = true;
1994 resultType = Value->getType();
1995
1996 // Failure.
1997 } else {
1998 return ConstantEmission();
1999 }
2000
2001 // In any case, if the initializer has side-effects, abandon ship.
2002 if (result.HasSideEffects)
2003 return ConstantEmission();
2004
2005 // In CUDA/HIP device compilation, a lambda may capture a reference variable
2006 // referencing a global host variable by copy. In this case the lambda should
2007 // make a copy of the value of the global host variable. The DRE of the
2008 // captured reference variable cannot be emitted as load from the host
2009 // global variable as compile time constant, since the host variable is not
2010 // accessible on device. The DRE of the captured reference variable has to be
2011 // loaded from captures.
2012 if (CGM.getLangOpts().CUDAIsDevice && result.Val.isLValue() &&
2013 RefExpr->refersToEnclosingVariableOrCapture()) {
2014 auto *MD = dyn_cast_or_null<CXXMethodDecl>(Val: CurCodeDecl);
2015 if (isLambdaMethod(DC: MD) && MD->getOverloadedOperator() == OO_Call) {
2016 const APValue::LValueBase &base = result.Val.getLValueBase();
2017 if (const ValueDecl *D = base.dyn_cast<const ValueDecl *>()) {
2018 if (const VarDecl *VD = dyn_cast<const VarDecl>(Val: D)) {
2019 if (!VD->hasAttr<CUDADeviceAttr>()) {
2020 return ConstantEmission();
2021 }
2022 }
2023 }
2024 }
2025 }
2026
2027 // Emit as a constant.
2028 llvm::Constant *C = ConstantEmitter(*this).emitAbstract(
2029 loc: RefExpr->getLocation(), value: result.Val, T: resultType);
2030
2031 // Make sure we emit a debug reference to the global variable.
2032 // This should probably fire even for
2033 if (isa<VarDecl>(Val: Value)) {
2034 if (!getContext().DeclMustBeEmitted(D: cast<VarDecl>(Val: Value)))
2035 EmitDeclRefExprDbgValue(E: RefExpr, Init: result.Val);
2036 } else {
2037 assert(isa<EnumConstantDecl>(Value));
2038 EmitDeclRefExprDbgValue(E: RefExpr, Init: result.Val);
2039 }
2040
2041 // If we emitted a reference constant, we need to dereference that.
2042 if (resultIsReference)
2043 return ConstantEmission::forReference(C);
2044
2045 return ConstantEmission::forValue(C);
2046}
2047
2048static DeclRefExpr *tryToConvertMemberExprToDeclRefExpr(CodeGenFunction &CGF,
2049 const MemberExpr *ME) {
2050 if (auto *VD = dyn_cast<VarDecl>(Val: ME->getMemberDecl())) {
2051 // Try to emit static variable member expressions as DREs.
2052 return DeclRefExpr::Create(
2053 Context: CGF.getContext(), QualifierLoc: NestedNameSpecifierLoc(), TemplateKWLoc: SourceLocation(), D: VD,
2054 /*RefersToEnclosingVariableOrCapture=*/false, NameLoc: ME->getExprLoc(),
2055 T: ME->getType(), VK: ME->getValueKind(), FoundD: nullptr, TemplateArgs: nullptr, NOUR: ME->isNonOdrUse());
2056 }
2057 return nullptr;
2058}
2059
2060CodeGenFunction::ConstantEmission
2061CodeGenFunction::tryEmitAsConstant(const MemberExpr *ME) {
2062 if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(CGF&: *this, ME))
2063 return tryEmitAsConstant(RefExpr: DRE);
2064 return ConstantEmission();
2065}
2066
2067llvm::Value *CodeGenFunction::emitScalarConstant(
2068 const CodeGenFunction::ConstantEmission &Constant, Expr *E) {
2069 assert(Constant && "not a constant");
2070 if (Constant.isReference())
2071 return EmitLoadOfLValue(V: Constant.getReferenceLValue(CGF&: *this, RefExpr: E),
2072 Loc: E->getExprLoc())
2073 .getScalarVal();
2074 return Constant.getValue();
2075}
2076
2077llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue,
2078 SourceLocation Loc) {
2079 return EmitLoadOfScalar(Addr: lvalue.getAddress(), Volatile: lvalue.isVolatile(),
2080 Ty: lvalue.getType(), Loc, BaseInfo: lvalue.getBaseInfo(),
2081 TBAAInfo: lvalue.getTBAAInfo(), isNontemporal: lvalue.isNontemporal());
2082}
2083
2084// This method SHOULD NOT be extended to support additional types, like BitInt
2085// types, without an opt-in bool controlled by a CodeGenOptions setting (like
2086// -fstrict-bool) and a new UBSan check (like SanitizerKind::Bool) as breaking
2087// that assumption would lead to memory corruption. See link for examples of how
2088// having a bool that has a value different from 0 or 1 in memory can lead to
2089// memory corruption.
2090// https://discourse.llvm.org/t/defining-what-happens-when-a-bool-isn-t-0-or-1/86778
2091static bool getRangeForType(CodeGenFunction &CGF, QualType Ty, llvm::APInt &Min,
2092 llvm::APInt &End, bool StrictEnums, bool StrictBool,
2093 bool IsBool) {
2094 const auto *ED = Ty->getAsEnumDecl();
2095 bool IsRegularCPlusPlusEnum =
2096 CGF.getLangOpts().CPlusPlus && StrictEnums && ED && !ED->isFixed();
2097 if (!IsBool && !IsRegularCPlusPlusEnum)
2098 return false;
2099
2100 if (IsBool) {
2101 if (!StrictBool)
2102 return false;
2103 Min = llvm::APInt(CGF.getContext().getTypeSize(T: Ty), 0);
2104 End = llvm::APInt(CGF.getContext().getTypeSize(T: Ty), 2);
2105 } else {
2106 ED->getValueRange(Max&: End, Min);
2107 }
2108 return true;
2109}
2110
2111llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) {
2112 llvm::APInt Min, End;
2113 bool IsBool = Ty->hasBooleanRepresentation() && !Ty->isVectorType();
2114 bool StrictBoolEnabled = CGM.getCodeGenOpts().getLoadBoolFromMem() ==
2115 CodeGenOptions::BoolFromMem::Strict;
2116 if (!getRangeForType(CGF&: *this, Ty, Min, End,
2117 /*StrictEnums=*/CGM.getCodeGenOpts().StrictEnums,
2118 /*StrictBool=*/StrictBoolEnabled, /*IsBool=*/IsBool))
2119 return nullptr;
2120
2121 llvm::MDBuilder MDHelper(getLLVMContext());
2122 return MDHelper.createRange(Lo: Min, Hi: End);
2123}
2124
2125void CodeGenFunction::maybeAttachRangeForLoad(llvm::LoadInst *Load, QualType Ty,
2126 SourceLocation Loc) {
2127 if (EmitScalarRangeCheck(Value: Load, Ty, Loc)) {
2128 // In order to prevent the optimizer from throwing away the check, don't
2129 // attach range metadata to the load.
2130 } else if (CGM.getCodeGenOpts().isOptimizedBuild()) {
2131 if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty)) {
2132 Load->setMetadata(KindID: llvm::LLVMContext::MD_range, Node: RangeInfo);
2133 Load->setMetadata(KindID: llvm::LLVMContext::MD_noundef,
2134 Node: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: {}));
2135 }
2136 }
2137}
2138
2139bool CodeGenFunction::EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
2140 SourceLocation Loc) {
2141 bool HasBoolCheck = SanOpts.has(K: SanitizerKind::Bool);
2142 bool HasEnumCheck = SanOpts.has(K: SanitizerKind::Enum);
2143 if (!HasBoolCheck && !HasEnumCheck)
2144 return false;
2145
2146 bool IsBool = (Ty->hasBooleanRepresentation() && !Ty->isVectorType()) ||
2147 NSAPI(CGM.getContext()).isObjCBOOLType(T: Ty);
2148 bool NeedsBoolCheck = HasBoolCheck && IsBool;
2149 bool NeedsEnumCheck = HasEnumCheck && Ty->isEnumeralType();
2150 if (!NeedsBoolCheck && !NeedsEnumCheck)
2151 return false;
2152
2153 // Single-bit booleans don't need to be checked. Special-case this to avoid
2154 // a bit width mismatch when handling bitfield values. This is handled by
2155 // EmitFromMemory for the non-bitfield case.
2156 if (IsBool &&
2157 cast<llvm::IntegerType>(Val: Value->getType())->getBitWidth() == 1)
2158 return false;
2159
2160 if (NeedsEnumCheck &&
2161 getContext().isTypeIgnoredBySanitizer(Mask: SanitizerKind::Enum, Ty))
2162 return false;
2163
2164 llvm::APInt Min, End;
2165 if (!getRangeForType(CGF&: *this, Ty, Min, End, /*StrictEnums=*/true,
2166 /*StrictBool=*/true, IsBool))
2167 return true;
2168
2169 SanitizerKind::SanitizerOrdinal Kind =
2170 NeedsEnumCheck ? SanitizerKind::SO_Enum : SanitizerKind::SO_Bool;
2171
2172 auto &Ctx = getLLVMContext();
2173 auto CheckHandler = SanitizerHandler::LoadInvalidValue;
2174 SanitizerDebugLocation SanScope(this, {Kind}, CheckHandler);
2175 llvm::Value *Check;
2176 --End;
2177 if (!Min) {
2178 Check = Builder.CreateICmpULE(LHS: Value, RHS: llvm::ConstantInt::get(Context&: Ctx, V: End));
2179 } else {
2180 llvm::Value *Upper =
2181 Builder.CreateICmpSLE(LHS: Value, RHS: llvm::ConstantInt::get(Context&: Ctx, V: End));
2182 llvm::Value *Lower =
2183 Builder.CreateICmpSGE(LHS: Value, RHS: llvm::ConstantInt::get(Context&: Ctx, V: Min));
2184 Check = Builder.CreateAnd(LHS: Upper, RHS: Lower);
2185 }
2186 llvm::Constant *StaticArgs[] = {EmitCheckSourceLocation(Loc),
2187 EmitCheckTypeDescriptor(T: Ty)};
2188 EmitCheck(Checked: std::make_pair(x&: Check, y&: Kind), Check: CheckHandler, StaticArgs, DynamicArgs: Value);
2189 return true;
2190}
2191
2192llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
2193 QualType Ty,
2194 SourceLocation Loc,
2195 LValueBaseInfo BaseInfo,
2196 TBAAAccessInfo TBAAInfo,
2197 bool isNontemporal) {
2198 if (auto *GV = dyn_cast<llvm::GlobalValue>(Val: Addr.getBasePointer()))
2199 if (GV->isThreadLocal())
2200 Addr = Addr.withPointer(NewPointer: Builder.CreateThreadLocalAddress(Ptr: GV),
2201 IsKnownNonNull: NotKnownNonNull);
2202
2203 if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
2204 // Boolean vectors use `iN` as storage type.
2205 if (ClangVecTy->isPackedVectorBoolType(ctx: getContext())) {
2206 llvm::Type *ValTy = ConvertType(T: Ty);
2207 unsigned ValNumElems =
2208 cast<llvm::FixedVectorType>(Val: ValTy)->getNumElements();
2209 // Load the `iP` storage object (P is the padded vector size).
2210 auto *RawIntV = Builder.CreateLoad(Addr, IsVolatile: Volatile, Name: "load_bits");
2211 const auto *RawIntTy = RawIntV->getType();
2212 assert(RawIntTy->isIntegerTy() && "compressed iN storage for bitvectors");
2213 // Bitcast iP --> <P x i1>.
2214 auto *PaddedVecTy = llvm::FixedVectorType::get(
2215 ElementType: Builder.getInt1Ty(), NumElts: RawIntTy->getPrimitiveSizeInBits());
2216 llvm::Value *V = Builder.CreateBitCast(V: RawIntV, DestTy: PaddedVecTy);
2217 // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
2218 V = emitBoolVecConversion(SrcVec: V, NumElementsDst: ValNumElems, Name: "extractvec");
2219
2220 return EmitFromMemory(Value: V, Ty);
2221 }
2222
2223 // Handles vectors of sizes that are likely to be expanded to a larger size
2224 // to optimize performance.
2225 auto *VTy = cast<llvm::FixedVectorType>(Val: Addr.getElementType());
2226 auto *NewVecTy =
2227 CGM.getABIInfo().getOptimalVectorMemoryType(T: VTy, Opt: getLangOpts());
2228
2229 if (VTy != NewVecTy) {
2230 Address Cast = Addr.withElementType(ElemTy: NewVecTy);
2231 llvm::Value *V = Builder.CreateLoad(Addr: Cast, IsVolatile: Volatile, Name: "loadVecN");
2232 unsigned OldNumElements = VTy->getNumElements();
2233 SmallVector<int, 16> Mask(OldNumElements);
2234 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
2235 V = Builder.CreateShuffleVector(V, Mask, Name: "extractVec");
2236 return EmitFromMemory(Value: V, Ty);
2237 }
2238 }
2239
2240 // Atomic operations have to be done on integral types.
2241 LValue AtomicLValue =
2242 LValue::MakeAddr(Addr, type: Ty, Context&: getContext(), BaseInfo, TBAAInfo);
2243 if (Ty->isAtomicType() || LValueIsSuitableForInlineAtomic(Src: AtomicLValue)) {
2244 return EmitAtomicLoad(LV: AtomicLValue, SL: Loc).getScalarVal();
2245 }
2246
2247 Addr =
2248 Addr.withElementType(ElemTy: convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Addr.getElementType()));
2249
2250 llvm::LoadInst *Load = Builder.CreateLoad(Addr, IsVolatile: Volatile);
2251 if (isNontemporal) {
2252 llvm::MDNode *Node = llvm::MDNode::get(
2253 Context&: Load->getContext(), MDs: llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: 1)));
2254 Load->setMetadata(KindID: llvm::LLVMContext::MD_nontemporal, Node);
2255 }
2256
2257 CGM.DecorateInstructionWithTBAA(Inst: Load, TBAAInfo);
2258
2259 maybeAttachRangeForLoad(Load, Ty, Loc);
2260
2261 return EmitFromMemory(Value: Load, Ty);
2262}
2263
2264/// Converts a scalar value from its primary IR type (as returned
2265/// by ConvertType) to its load/store type (as returned by
2266/// convertTypeForLoadStore).
2267llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) {
2268 if (auto *AtomicTy = Ty->getAs<AtomicType>())
2269 Ty = AtomicTy->getValueType();
2270
2271 if (Ty->isExtVectorBoolType() || Ty->isConstantMatrixBoolType()) {
2272 llvm::Type *StoreTy = convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Value->getType());
2273
2274 if (Value->getType() == StoreTy)
2275 return Value;
2276
2277 if (StoreTy->isVectorTy() && StoreTy->getScalarSizeInBits() >
2278 Value->getType()->getScalarSizeInBits())
2279 return Builder.CreateZExt(V: Value, DestTy: StoreTy);
2280
2281 // Expand to the memory bit width.
2282 unsigned MemNumElems = StoreTy->getPrimitiveSizeInBits();
2283 // <N x i1> --> <P x i1>.
2284 Value = emitBoolVecConversion(SrcVec: Value, NumElementsDst: MemNumElems, Name: "insertvec");
2285 // <P x i1> --> iP.
2286 Value = Builder.CreateBitCast(V: Value, DestTy: StoreTy);
2287 }
2288
2289 if (Ty->hasBooleanRepresentation() || Ty->isBitIntType()) {
2290 llvm::Type *StoreTy = convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Value->getType());
2291 bool Signed = Ty->isSignedIntegerOrEnumerationType();
2292 return Builder.CreateIntCast(V: Value, DestTy: StoreTy, isSigned: Signed, Name: "storedv");
2293 }
2294
2295 return Value;
2296}
2297
2298/// Converts a scalar value from its load/store type (as returned
2299/// by convertTypeForLoadStore) to its primary IR type (as returned
2300/// by ConvertType).
2301llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
2302 if (auto *AtomicTy = Ty->getAs<AtomicType>())
2303 Ty = AtomicTy->getValueType();
2304
2305 if (Ty->isPackedVectorBoolType(ctx: getContext())) {
2306 const auto *RawIntTy = Value->getType();
2307
2308 // Bitcast iP --> <P x i1>.
2309 auto *PaddedVecTy = llvm::FixedVectorType::get(
2310 ElementType: Builder.getInt1Ty(), NumElts: RawIntTy->getPrimitiveSizeInBits());
2311 auto *V = Builder.CreateBitCast(V: Value, DestTy: PaddedVecTy);
2312 // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
2313 llvm::Type *ValTy = ConvertType(T: Ty);
2314 unsigned ValNumElems = cast<llvm::FixedVectorType>(Val: ValTy)->getNumElements();
2315 return emitBoolVecConversion(SrcVec: V, NumElementsDst: ValNumElems, Name: "extractvec");
2316 }
2317
2318 llvm::Type *ResTy = ConvertType(T: Ty);
2319 bool HasBoolRep = Ty->hasBooleanRepresentation() || Ty->isExtVectorBoolType();
2320 if (HasBoolRep && CGM.getCodeGenOpts().isConvertingBoolWithCmp0()) {
2321 return Builder.CreateICmpNE(
2322 LHS: Value, RHS: llvm::Constant::getNullValue(Ty: Value->getType()), Name: "loadedv");
2323 }
2324 if (HasBoolRep || Ty->isBitIntType())
2325 return Builder.CreateTrunc(V: Value, DestTy: ResTy, Name: "loadedv");
2326
2327 return Value;
2328}
2329
2330// Convert the pointer of \p Addr to a pointer to a vector (the value type of
2331// MatrixType), if it points to a array (the memory type of MatrixType).
2332static RawAddress MaybeConvertMatrixAddress(RawAddress Addr,
2333 CodeGenFunction &CGF,
2334 bool IsVector = true) {
2335 auto *ArrayTy = dyn_cast<llvm::ArrayType>(Val: Addr.getElementType());
2336 if (ArrayTy && IsVector) {
2337 auto ArrayElements = ArrayTy->getNumElements();
2338 auto *ArrayElementTy = ArrayTy->getElementType();
2339 if (CGF.getContext().getLangOpts().HLSL) {
2340 auto *VectorTy = cast<llvm::FixedVectorType>(Val: ArrayElementTy);
2341 ArrayElementTy = VectorTy->getElementType();
2342 ArrayElements *= VectorTy->getNumElements();
2343 }
2344 auto *VectorTy = llvm::FixedVectorType::get(ElementType: ArrayElementTy, NumElts: ArrayElements);
2345
2346 return Addr.withElementType(ElemTy: VectorTy);
2347 }
2348 auto *VectorTy = dyn_cast<llvm::VectorType>(Val: Addr.getElementType());
2349 if (VectorTy && !IsVector) {
2350 auto *ArrayTy = llvm::ArrayType::get(
2351 ElementType: VectorTy->getElementType(),
2352 NumElements: cast<llvm::FixedVectorType>(Val: VectorTy)->getNumElements());
2353
2354 return Addr.withElementType(ElemTy: ArrayTy);
2355 }
2356
2357 return Addr;
2358}
2359
2360LValue CodeGenFunction::EmitMatrixElementExpr(const MatrixElementExpr *E) {
2361 LValue Base;
2362 if (E->getBase()->isGLValue())
2363 Base = EmitLValue(E: E->getBase());
2364 else {
2365 assert(E->getBase()->getType()->isConstantMatrixType() &&
2366 "Result must be a Constant Matrix");
2367 llvm::Value *Mat = EmitScalarExpr(E: E->getBase());
2368 Address MatMem = CreateMemTemp(Ty: E->getBase()->getType());
2369 QualType Ty = E->getBase()->getType();
2370 llvm::Type *LTy = convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Mat->getType());
2371 if (LTy->getScalarSizeInBits() > Mat->getType()->getScalarSizeInBits())
2372 Mat = Builder.CreateZExt(V: Mat, DestTy: LTy);
2373 Builder.CreateStore(Val: Mat, Addr: MatMem);
2374 Base = MakeAddrLValue(Addr: MatMem, T: Ty, Source: AlignmentSource::Decl);
2375 }
2376 QualType ResultType =
2377 E->getType().withCVRQualifiers(CVR: Base.getQuals().getCVRQualifiers());
2378
2379 // Encode the element access list into a vector of unsigned indices.
2380 // getEncodedElementAccess returns row-major linearized indices.
2381 SmallVector<uint32_t, 4> Indices;
2382 E->getEncodedElementAccess(Elts&: Indices);
2383
2384 // getEncodedElementAccess returns row-major linearized indices
2385 // If the matrix memory layout is column-major, convert indices
2386 // to column-major indices.
2387 bool IsRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: E->getBase()->getType());
2388 if (!IsRowMajor) {
2389 const auto *MT = E->getBase()->getType()->castAs<ConstantMatrixType>();
2390 unsigned NumCols = MT->getNumColumns();
2391 for (uint32_t &Idx : Indices) {
2392 // Decompose row-major index: Row = Idx / NumCols, Col = Idx % NumCols
2393 unsigned Row = Idx / NumCols;
2394 unsigned Col = Idx % NumCols;
2395 // Re-linearize as column-major
2396 Idx = MT->getColumnMajorFlattenedIndex(Row, Column: Col);
2397 }
2398 }
2399
2400 if (Base.isSimple()) {
2401 RawAddress MatAddr = Base.getAddress();
2402 if (getLangOpts().HLSL &&
2403 E->getBase()->getType().getAddressSpace() == LangAS::hlsl_constant)
2404 MatAddr = CGM.getHLSLRuntime().createBufferMatrixTempAddress(LV: Base, CGF&: *this);
2405
2406 llvm::Constant *CV =
2407 llvm::ConstantDataVector::get(Context&: getLLVMContext(), Elts: Indices);
2408 return LValue::MakeExtVectorElt(Addr: MaybeConvertMatrixAddress(Addr: MatAddr, CGF&: *this),
2409 Elts: CV, type: ResultType, BaseInfo: Base.getBaseInfo(),
2410 TBAAInfo: TBAAAccessInfo());
2411 }
2412 assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
2413
2414 llvm::Constant *BaseElts = Base.getExtVectorElts();
2415 SmallVector<llvm::Constant *, 4> CElts;
2416
2417 for (unsigned Index : Indices)
2418 CElts.push_back(Elt: BaseElts->getAggregateElement(Elt: Index));
2419 llvm::Constant *CV = llvm::ConstantVector::get(V: CElts);
2420
2421 return LValue::MakeExtVectorElt(
2422 Addr: MaybeConvertMatrixAddress(Addr: Base.getExtVectorAddress(), CGF&: *this), Elts: CV,
2423 type: ResultType, BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
2424}
2425
2426// Emit a store of a matrix LValue. This may require casting the original
2427// pointer to memory address (ArrayType) to a pointer to the value type
2428// (VectorType).
2429static void EmitStoreOfMatrixScalar(llvm::Value *value, LValue lvalue,
2430 bool isInit, CodeGenFunction &CGF) {
2431 Address Addr = MaybeConvertMatrixAddress(Addr: lvalue.getAddress(), CGF,
2432 IsVector: value->getType()->isVectorTy());
2433 CGF.EmitStoreOfScalar(Value: value, Addr, Volatile: lvalue.isVolatile(), Ty: lvalue.getType(),
2434 BaseInfo: lvalue.getBaseInfo(), TBAAInfo: lvalue.getTBAAInfo(), isInit,
2435 isNontemporal: lvalue.isNontemporal());
2436}
2437
2438void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr,
2439 bool Volatile, QualType Ty,
2440 LValueBaseInfo BaseInfo,
2441 TBAAAccessInfo TBAAInfo,
2442 bool isInit, bool isNontemporal) {
2443 if (auto *GV = dyn_cast<llvm::GlobalValue>(Val: Addr.getBasePointer()))
2444 if (GV->isThreadLocal())
2445 Addr = Addr.withPointer(NewPointer: Builder.CreateThreadLocalAddress(Ptr: GV),
2446 IsKnownNonNull: NotKnownNonNull);
2447
2448 // Handles vectors of sizes that are likely to be expanded to a larger size
2449 // to optimize performance.
2450 llvm::Type *SrcTy = Value->getType();
2451 if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
2452 if (auto *VecTy = dyn_cast<llvm::FixedVectorType>(Val: SrcTy)) {
2453 auto *NewVecTy =
2454 CGM.getABIInfo().getOptimalVectorMemoryType(T: VecTy, Opt: getLangOpts());
2455 if (!ClangVecTy->isPackedVectorBoolType(ctx: getContext()) &&
2456 VecTy != NewVecTy) {
2457 SmallVector<int, 16> Mask(NewVecTy->getNumElements(),
2458 VecTy->getNumElements());
2459 std::iota(first: Mask.begin(), last: Mask.begin() + VecTy->getNumElements(), value: 0);
2460 // Use undef instead of poison for the padding lanes, to make sure no
2461 // padding bits are poisoned, which may break coercion.
2462 Value = Builder.CreateShuffleVector(V1: Value, V2: llvm::UndefValue::get(T: VecTy),
2463 Mask, Name: "extractVec");
2464 SrcTy = NewVecTy;
2465 }
2466 if (Addr.getElementType() != SrcTy)
2467 Addr = Addr.withElementType(ElemTy: SrcTy);
2468 }
2469 }
2470
2471 Value = EmitToMemory(Value, Ty);
2472
2473 LValue AtomicLValue =
2474 LValue::MakeAddr(Addr, type: Ty, Context&: getContext(), BaseInfo, TBAAInfo);
2475 if (Ty->isAtomicType() ||
2476 (!isInit && LValueIsSuitableForInlineAtomic(Src: AtomicLValue))) {
2477 EmitAtomicStore(rvalue: RValue::get(V: Value), lvalue: AtomicLValue, isInit);
2478 return;
2479 }
2480
2481 llvm::StoreInst *Store = Builder.CreateStore(Val: Value, Addr, IsVolatile: Volatile);
2482 addInstToCurrentSourceAtom(KeyInstruction: Store, Backup: Value);
2483
2484 if (isNontemporal) {
2485 llvm::MDNode *Node =
2486 llvm::MDNode::get(Context&: Store->getContext(),
2487 MDs: llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: 1)));
2488 Store->setMetadata(KindID: llvm::LLVMContext::MD_nontemporal, Node);
2489 }
2490
2491 CGM.DecorateInstructionWithTBAA(Inst: Store, TBAAInfo);
2492}
2493
2494void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
2495 bool isInit) {
2496 if (lvalue.getType()->isConstantMatrixType()) {
2497 EmitStoreOfMatrixScalar(value, lvalue, isInit, CGF&: *this);
2498 return;
2499 }
2500
2501 EmitStoreOfScalar(Value: value, Addr: lvalue.getAddress(), Volatile: lvalue.isVolatile(),
2502 Ty: lvalue.getType(), BaseInfo: lvalue.getBaseInfo(),
2503 TBAAInfo: lvalue.getTBAAInfo(), isInit, isNontemporal: lvalue.isNontemporal());
2504}
2505
2506// Emit a load of a LValue of matrix type. This may require casting the pointer
2507// to memory address (ArrayType) to a pointer to the value type (VectorType).
2508static RValue EmitLoadOfMatrixLValue(LValue LV, SourceLocation Loc,
2509 CodeGenFunction &CGF) {
2510 assert(LV.getType()->isConstantMatrixType());
2511 RawAddress DestAddr = LV.getAddress();
2512
2513 // HLSL constant buffers may pad matrix layouts, so copy elements into a
2514 // non-padded local alloca before loading.
2515 if (CGF.getLangOpts().HLSL &&
2516 LV.getType().getAddressSpace() == LangAS::hlsl_constant)
2517 DestAddr = CGF.CGM.getHLSLRuntime().createBufferMatrixTempAddress(LV, CGF);
2518
2519 Address Addr = MaybeConvertMatrixAddress(Addr: DestAddr, CGF);
2520 LV.setAddress(Addr);
2521 return RValue::get(V: CGF.EmitLoadOfScalar(lvalue: LV, Loc));
2522}
2523
2524RValue CodeGenFunction::EmitLoadOfAnyValue(LValue LV, AggValueSlot Slot,
2525 SourceLocation Loc) {
2526 QualType Ty = LV.getType();
2527 switch (getEvaluationKind(T: Ty)) {
2528 case TEK_Scalar:
2529 return EmitLoadOfLValue(V: LV, Loc);
2530 case TEK_Complex:
2531 return RValue::getComplex(C: EmitLoadOfComplex(src: LV, loc: Loc));
2532 case TEK_Aggregate:
2533 EmitAggFinalDestCopy(Type: Ty, Dest: Slot, Src: LV, SrcKind: EVK_NonRValue);
2534 return Slot.asRValue();
2535 }
2536 llvm_unreachable("bad evaluation kind");
2537}
2538
2539/// EmitLoadOfLValue - Given an expression that represents a value lvalue, this
2540/// method emits the address of the lvalue, then loads the result as an rvalue,
2541/// returning the rvalue.
2542RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) {
2543 // Load from __ptrauth.
2544 if (PointerAuthQualifier PtrAuth = LV.getQuals().getPointerAuth()) {
2545 LV.getQuals().removePointerAuth();
2546 llvm::Value *Value = EmitLoadOfLValue(LV, Loc).getScalarVal();
2547 return RValue::get(V: EmitPointerAuthUnqualify(Qualifier: PtrAuth, Pointer: Value, PointerType: LV.getType(),
2548 StorageAddress: LV.getAddress(),
2549 /*known nonnull*/ IsKnownNonNull: false));
2550 }
2551
2552 if (LV.isObjCWeak()) {
2553 // load of a __weak object.
2554 Address AddrWeakObj = LV.getAddress();
2555 return RValue::get(V: CGM.getObjCRuntime().EmitObjCWeakRead(CGF&: *this,
2556 AddrWeakObj));
2557 }
2558 if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) {
2559 // In MRC mode, we do a load+autorelease.
2560 if (!getLangOpts().ObjCAutoRefCount) {
2561 return RValue::get(V: EmitARCLoadWeak(addr: LV.getAddress()));
2562 }
2563
2564 // In ARC mode, we load retained and then consume the value.
2565 llvm::Value *Object = EmitARCLoadWeakRetained(addr: LV.getAddress());
2566 Object = EmitObjCConsumeObject(T: LV.getType(), Ptr: Object);
2567 return RValue::get(V: Object);
2568 }
2569
2570 if (LV.isSimple()) {
2571 assert(!LV.getType()->isFunctionType());
2572
2573 if (LV.getType()->isConstantMatrixType())
2574 return EmitLoadOfMatrixLValue(LV, Loc, CGF&: *this);
2575
2576 // Everything needs a load.
2577 return RValue::get(V: EmitLoadOfScalar(lvalue: LV, Loc));
2578 }
2579
2580 if (LV.isVectorElt()) {
2581 llvm::LoadInst *Load = Builder.CreateLoad(Addr: LV.getVectorAddress(),
2582 IsVolatile: LV.isVolatileQualified());
2583 llvm::Value *Elt =
2584 Builder.CreateExtractElement(Vec: Load, Idx: LV.getVectorIdx(), Name: "vecext");
2585 return RValue::get(V: EmitFromMemory(Value: Elt, Ty: LV.getType()));
2586 }
2587
2588 // If this is a reference to a subset of the elements of a vector, either
2589 // shuffle the input or extract/insert them as appropriate.
2590 if (LV.isExtVectorElt()) {
2591 return EmitLoadOfExtVectorElementLValue(V: LV);
2592 }
2593
2594 // Global Register variables always invoke intrinsics
2595 if (LV.isGlobalReg())
2596 return EmitLoadOfGlobalRegLValue(LV);
2597
2598 if (LV.isMatrixElt()) {
2599 llvm::Value *Idx = LV.getMatrixIdx();
2600 QualType EltTy = LV.getType();
2601 if (const auto *MatTy = EltTy->getAs<ConstantMatrixType>()) {
2602 EltTy = MatTy->getElementType();
2603 if (CGM.getCodeGenOpts().isOptimizedBuild()) {
2604 llvm::MatrixBuilder MB(Builder);
2605 MB.CreateIndexAssumption(Idx, NumElements: MatTy->getNumElementsFlattened());
2606 }
2607 }
2608 llvm::LoadInst *Load =
2609 Builder.CreateLoad(Addr: LV.getMatrixAddress(), IsVolatile: LV.isVolatileQualified());
2610 llvm::Value *Elt = Builder.CreateExtractElement(Vec: Load, Idx, Name: "matrixext");
2611 return RValue::get(V: EmitFromMemory(Value: Elt, Ty: EltTy));
2612 }
2613 if (LV.isMatrixRow()) {
2614 QualType MatTy = LV.getType();
2615 const ConstantMatrixType *MT = MatTy->castAs<ConstantMatrixType>();
2616
2617 unsigned NumRows = MT->getNumRows();
2618 unsigned NumCols = MT->getNumColumns();
2619 unsigned NumLanes = NumCols;
2620 llvm::Value *MatrixVec = EmitLoadOfScalar(lvalue: LV, Loc);
2621 llvm::Value *Row = LV.getMatrixRowIdx();
2622 llvm::Type *ElemTy = ConvertType(T: MT->getElementType());
2623 llvm::Constant *ColConstsIndices = nullptr;
2624 llvm::MatrixBuilder MB(Builder);
2625
2626 if (LV.isMatrixRowSwizzle()) {
2627 ColConstsIndices = LV.getMatrixRowElts();
2628 NumLanes = llvm::cast<llvm::FixedVectorType>(Val: ColConstsIndices->getType())
2629 ->getNumElements();
2630 }
2631
2632 llvm::Type *RowTy = llvm::FixedVectorType::get(ElementType: ElemTy, NumElts: NumLanes);
2633 llvm::Value *Result = llvm::PoisonValue::get(T: RowTy); // <NumLanes x T>
2634
2635 for (unsigned Col = 0; Col < NumLanes; ++Col) {
2636 llvm::Value *ColIdx;
2637 if (ColConstsIndices)
2638 ColIdx = ColConstsIndices->getAggregateElement(Elt: Col);
2639 else
2640 ColIdx = llvm::ConstantInt::get(Ty: Row->getType(), V: Col);
2641 bool IsMatrixRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: MatTy);
2642 llvm::Value *EltIndex =
2643 MB.CreateIndex(RowIdx: Row, ColumnIdx: ColIdx, NumRows, NumCols, IsMatrixRowMajor);
2644 llvm::Value *Elt = Builder.CreateExtractElement(Vec: MatrixVec, Idx: EltIndex);
2645 llvm::Value *Lane = llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: Col);
2646 Result = Builder.CreateInsertElement(Vec: Result, NewElt: Elt, Idx: Lane);
2647 }
2648
2649 return RValue::get(V: Result);
2650 }
2651
2652 assert(LV.isBitField() && "Unknown LValue type!");
2653 return EmitLoadOfBitfieldLValue(LV, Loc);
2654}
2655
2656RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV,
2657 SourceLocation Loc) {
2658 const CGBitFieldInfo &Info = LV.getBitFieldInfo();
2659
2660 // Get the output type.
2661 llvm::Type *ResLTy = ConvertType(T: LV.getType());
2662
2663 Address Ptr = LV.getBitFieldAddress();
2664 llvm::Value *Val =
2665 Builder.CreateLoad(Addr: Ptr, IsVolatile: LV.isVolatileQualified(), Name: "bf.load");
2666
2667 bool UseVolatile = LV.isVolatileQualified() &&
2668 Info.VolatileStorageSize != 0 && isAAPCS(TargetInfo: CGM.getTarget());
2669 const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
2670 const unsigned StorageSize =
2671 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
2672 if (Info.IsSigned) {
2673 assert(static_cast<unsigned>(Offset + Info.Size) <= StorageSize);
2674 unsigned HighBits = StorageSize - Offset - Info.Size;
2675 if (HighBits)
2676 Val = Builder.CreateShl(LHS: Val, RHS: HighBits, Name: "bf.shl");
2677 if (Offset + HighBits)
2678 Val = Builder.CreateAShr(LHS: Val, RHS: Offset + HighBits, Name: "bf.ashr");
2679 } else {
2680 if (Offset)
2681 Val = Builder.CreateLShr(LHS: Val, RHS: Offset, Name: "bf.lshr");
2682 if (static_cast<unsigned>(Offset) + Info.Size < StorageSize)
2683 Val = Builder.CreateAnd(
2684 LHS: Val, RHS: llvm::APInt::getLowBitsSet(numBits: StorageSize, loBitsSet: Info.Size), Name: "bf.clear");
2685 }
2686 Val = Builder.CreateIntCast(V: Val, DestTy: ResLTy, isSigned: Info.IsSigned, Name: "bf.cast");
2687 EmitScalarRangeCheck(Value: Val, Ty: LV.getType(), Loc);
2688 return RValue::get(V: Val);
2689}
2690
2691// If this is a reference to a subset of the elements of a vector, create an
2692// appropriate shufflevector.
2693RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {
2694 llvm::Value *Vec = Builder.CreateLoad(Addr: LV.getExtVectorAddress(),
2695 IsVolatile: LV.isVolatileQualified());
2696
2697 // HLSL allows treating scalars as one-element vectors. Converting the scalar
2698 // IR value to a vector here allows the rest of codegen to behave as normal.
2699 if (getLangOpts().HLSL && !Vec->getType()->isVectorTy()) {
2700 llvm::Type *DstTy = llvm::FixedVectorType::get(ElementType: Vec->getType(), NumElts: 1);
2701 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: CGM.Int64Ty);
2702 Vec = Builder.CreateInsertElement(VecTy: DstTy, NewElt: Vec, Idx: Zero, Name: "cast.splat");
2703 }
2704
2705 const llvm::Constant *Elts = LV.getExtVectorElts();
2706
2707 // If the result of the expression is a non-vector type, we must be extracting
2708 // a single element. Just codegen as an extractelement.
2709 const VectorType *ExprVT = LV.getType()->getAs<VectorType>();
2710 if (!ExprVT) {
2711 unsigned InIdx = getAccessedFieldNo(Idx: 0, Elts);
2712 llvm::Value *Elt = llvm::ConstantInt::get(Ty: SizeTy, V: InIdx);
2713
2714 llvm::Value *Element = Builder.CreateExtractElement(Vec, Idx: Elt);
2715
2716 llvm::Type *LVTy = ConvertType(T: LV.getType());
2717 if (Element->getType()->getPrimitiveSizeInBits() >
2718 LVTy->getPrimitiveSizeInBits()) {
2719 if (LV.getType()->hasBooleanRepresentation() &&
2720 CGM.getCodeGenOpts().isConvertingBoolWithCmp0())
2721 Element = Builder.CreateICmpNE(
2722 LHS: Element, RHS: llvm::Constant::getNullValue(Ty: Element->getType()));
2723 else
2724 Element = Builder.CreateTrunc(V: Element, DestTy: LVTy);
2725 }
2726
2727 return RValue::get(V: Element);
2728 }
2729
2730 // Always use shuffle vector to try to retain the original program structure
2731 unsigned NumResultElts = ExprVT->getNumElements();
2732
2733 SmallVector<int, 4> Mask;
2734 for (unsigned i = 0; i != NumResultElts; ++i)
2735 Mask.push_back(Elt: getAccessedFieldNo(Idx: i, Elts));
2736
2737 Vec = Builder.CreateShuffleVector(V: Vec, Mask);
2738
2739 if (LV.getType()->isExtVectorBoolType()) {
2740 if (CGM.getCodeGenOpts().isConvertingBoolWithCmp0())
2741 Vec = Builder.CreateICmpNE(LHS: Vec,
2742 RHS: llvm::Constant::getNullValue(Ty: Vec->getType()));
2743 else
2744 Vec = Builder.CreateTrunc(V: Vec, DestTy: ConvertType(T: LV.getType()), Name: "truncv");
2745 }
2746
2747 return RValue::get(V: Vec);
2748}
2749
2750/// Generates lvalue for partial ext_vector access.
2751Address CodeGenFunction::EmitExtVectorElementLValue(LValue LV) {
2752 Address VectorAddress = LV.getExtVectorAddress();
2753 QualType EQT = LV.getType()->castAs<VectorType>()->getElementType();
2754 llvm::Type *VectorElementTy = CGM.getTypes().ConvertType(T: EQT);
2755
2756 Address CastToPointerElement = VectorAddress.withElementType(ElemTy: VectorElementTy);
2757
2758 const llvm::Constant *Elts = LV.getExtVectorElts();
2759 unsigned ix = getAccessedFieldNo(Idx: 0, Elts);
2760
2761 Address VectorBasePtrPlusIx =
2762 Builder.CreateConstInBoundsGEP(Addr: CastToPointerElement, Index: ix,
2763 Name: "vector.elt");
2764
2765 return VectorBasePtrPlusIx;
2766}
2767
2768/// Load of global named registers are always calls to intrinsics.
2769RValue CodeGenFunction::EmitLoadOfGlobalRegLValue(LValue LV) {
2770 assert((LV.getType()->isIntegerType() || LV.getType()->isPointerType()) &&
2771 "Bad type for register variable");
2772 llvm::MDNode *RegName = cast<llvm::MDNode>(
2773 Val: cast<llvm::MetadataAsValue>(Val: LV.getGlobalReg())->getMetadata());
2774
2775 // We accept integer and pointer types only
2776 llvm::Type *OrigTy = CGM.getTypes().ConvertType(T: LV.getType());
2777 llvm::Type *Ty = OrigTy;
2778 if (OrigTy->isPointerTy())
2779 Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
2780 llvm::Type *Types[] = { Ty };
2781
2782 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::read_register, Tys: Types);
2783 llvm::Value *Call = Builder.CreateCall(
2784 Callee: F, Args: llvm::MetadataAsValue::get(Context&: Ty->getContext(), MD: RegName));
2785 if (OrigTy->isPointerTy())
2786 Call = Builder.CreateIntToPtr(V: Call, DestTy: OrigTy);
2787 return RValue::get(V: Call);
2788}
2789
2790/// EmitStoreThroughLValue - Store the specified rvalue into the specified
2791/// lvalue, where both are guaranteed to the have the same type, and that type
2792/// is 'Ty'.
2793void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
2794 bool isInit) {
2795 if (!Dst.isSimple()) {
2796 if (Dst.isVectorElt()) {
2797 if (getLangOpts().HLSL) {
2798 // HLSL allows direct access to vector elements, so storing to
2799 // individual elements of a vector through VectorElt is handled as
2800 // separate store instructions.
2801 Address DstAddr = Dst.getVectorAddress();
2802 llvm::Type *DestAddrTy = DstAddr.getElementType();
2803 llvm::Type *ElemTy = DestAddrTy->getScalarType();
2804 CharUnits ElemAlign = CharUnits::fromQuantity(
2805 Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty: ElemTy));
2806
2807 assert(ElemTy->getScalarSizeInBits() >= 8 &&
2808 "vector element type must be at least byte-sized");
2809
2810 llvm::Value *Val = Src.getScalarVal();
2811 if (Val->getType()->getPrimitiveSizeInBits() <
2812 ElemTy->getScalarSizeInBits())
2813 Val = Builder.CreateZExt(V: Val, DestTy: ElemTy->getScalarType());
2814
2815 llvm::Value *Idx = Dst.getVectorIdx();
2816 llvm::Value *Zero = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
2817 Address DstElemAddr =
2818 Builder.CreateGEP(Addr: DstAddr, IdxList: {Zero, Idx}, ElementType: DestAddrTy, Align: ElemAlign);
2819 Builder.CreateStore(Val, Addr: DstElemAddr, IsVolatile: Dst.isVolatileQualified());
2820 return;
2821 }
2822
2823 // Read/modify/write the vector, inserting the new element.
2824 llvm::Value *Vec = Builder.CreateLoad(Addr: Dst.getVectorAddress(),
2825 IsVolatile: Dst.isVolatileQualified());
2826 llvm::Type *VecTy = Vec->getType();
2827 llvm::Value *SrcVal = Src.getScalarVal();
2828
2829 if (VecTy->isVectorTy() && SrcVal->getType()->getPrimitiveSizeInBits() <
2830 VecTy->getScalarSizeInBits())
2831 SrcVal = Builder.CreateZExt(V: SrcVal, DestTy: VecTy->getScalarType());
2832
2833 auto *IRStoreTy = dyn_cast<llvm::IntegerType>(Val: Vec->getType());
2834 if (IRStoreTy) {
2835 auto *IRVecTy = llvm::FixedVectorType::get(
2836 ElementType: Builder.getInt1Ty(), NumElts: IRStoreTy->getPrimitiveSizeInBits());
2837 Vec = Builder.CreateBitCast(V: Vec, DestTy: IRVecTy);
2838 // iN --> <N x i1>.
2839 }
2840
2841 // Allow inserting `<1 x T>` into an `<N x T>`. It can happen with scalar
2842 // types which are mapped to vector LLVM IR types (e.g. for implementing
2843 // an ABI).
2844 if (auto *EltTy = dyn_cast<llvm::FixedVectorType>(Val: SrcVal->getType());
2845 EltTy && EltTy->getNumElements() == 1)
2846 SrcVal = Builder.CreateBitCast(V: SrcVal, DestTy: EltTy->getElementType());
2847
2848 Vec = Builder.CreateInsertElement(Vec, NewElt: SrcVal, Idx: Dst.getVectorIdx(),
2849 Name: "vecins");
2850 if (IRStoreTy) {
2851 // <N x i1> --> <iN>.
2852 Vec = Builder.CreateBitCast(V: Vec, DestTy: IRStoreTy);
2853 }
2854
2855 auto *I = Builder.CreateStore(Val: Vec, Addr: Dst.getVectorAddress(),
2856 IsVolatile: Dst.isVolatileQualified());
2857 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Vec);
2858 return;
2859 }
2860
2861 // If this is an update of extended vector elements, insert them as
2862 // appropriate.
2863 if (Dst.isExtVectorElt())
2864 return EmitStoreThroughExtVectorComponentLValue(Src, Dst);
2865
2866 if (Dst.isGlobalReg())
2867 return EmitStoreThroughGlobalRegLValue(Src, Dst);
2868
2869 if (Dst.isMatrixElt()) {
2870 if (getLangOpts().HLSL) {
2871 // HLSL allows direct access to matrix elements, so storing to
2872 // individual elements of a matrix through MatrixElt is handled as
2873 // separate store instructions.
2874 Address DstAddr = Dst.getMatrixAddress();
2875 llvm::Type *DestAddrTy = DstAddr.getElementType();
2876 llvm::Type *ElemTy = DestAddrTy->getScalarType();
2877 CharUnits ElemAlign = CharUnits::fromQuantity(
2878 Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty: ElemTy));
2879
2880 assert(ElemTy->getScalarSizeInBits() >= 8 &&
2881 "matrix element type must be at least byte-sized");
2882
2883 llvm::Value *Val = Src.getScalarVal();
2884 if (Val->getType()->getPrimitiveSizeInBits() <
2885 ElemTy->getScalarSizeInBits())
2886 Val = Builder.CreateZExt(V: Val, DestTy: ElemTy->getScalarType());
2887
2888 llvm::Value *Idx = Dst.getMatrixIdx();
2889 llvm::Value *Zero = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
2890 Address DstElemAddr =
2891 Builder.CreateGEP(Addr: DstAddr, IdxList: {Zero, Idx}, ElementType: DestAddrTy, Align: ElemAlign);
2892 Builder.CreateStore(Val, Addr: DstElemAddr, IsVolatile: Dst.isVolatileQualified());
2893 return;
2894 }
2895
2896 llvm::Value *Idx = Dst.getMatrixIdx();
2897 if (CGM.getCodeGenOpts().isOptimizedBuild()) {
2898 const auto *const MatTy = Dst.getType()->castAs<ConstantMatrixType>();
2899 llvm::MatrixBuilder MB(Builder);
2900 MB.CreateIndexAssumption(Idx, NumElements: MatTy->getNumElementsFlattened());
2901 }
2902 llvm::Instruction *Load = Builder.CreateLoad(Addr: Dst.getMatrixAddress());
2903 llvm::Value *InsertVal = Src.getScalarVal();
2904 llvm::Value *Vec =
2905 Builder.CreateInsertElement(Vec: Load, NewElt: InsertVal, Idx, Name: "matins");
2906 auto *I = Builder.CreateStore(Val: Vec, Addr: Dst.getMatrixAddress(),
2907 IsVolatile: Dst.isVolatileQualified());
2908 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Vec);
2909 return;
2910 }
2911 if (Dst.isMatrixRow()) {
2912 // NOTE: Since there are no other languages that implement matrix single
2913 // subscripting, the logic here is specific to HLSL which allows
2914 // per-element stores to rows of matrices.
2915 assert(getLangOpts().HLSL &&
2916 "Store through matrix row LValues is only implemented for HLSL!");
2917 QualType MatTy = Dst.getType();
2918 const ConstantMatrixType *MT = MatTy->castAs<ConstantMatrixType>();
2919
2920 unsigned NumRows = MT->getNumRows();
2921 unsigned NumCols = MT->getNumColumns();
2922 unsigned NumLanes = NumCols;
2923
2924 Address DstAddr = Dst.getMatrixAddress();
2925 llvm::Type *DestAddrTy = DstAddr.getElementType();
2926 llvm::Type *ElemTy = DestAddrTy->getScalarType();
2927 CharUnits ElemAlign =
2928 CharUnits::fromQuantity(Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty: ElemTy));
2929
2930 assert(ElemTy->getScalarSizeInBits() >= 8 &&
2931 "matrix element type must be at least byte-sized");
2932
2933 llvm::Value *RowVal = Src.getScalarVal();
2934 if (RowVal->getType()->getScalarType()->getPrimitiveSizeInBits() <
2935 ElemTy->getScalarSizeInBits()) {
2936 auto *RowValVecTy = cast<llvm::FixedVectorType>(Val: RowVal->getType());
2937 llvm::Type *StorageElmTy = llvm::FixedVectorType::get(
2938 ElementType: ElemTy->getScalarType(), NumElts: RowValVecTy->getNumElements());
2939 RowVal = Builder.CreateZExt(V: RowVal, DestTy: StorageElmTy);
2940 }
2941
2942 llvm::MatrixBuilder MB(Builder);
2943
2944 llvm::Constant *ColConstsIndices = nullptr;
2945 if (Dst.isMatrixRowSwizzle()) {
2946 ColConstsIndices = Dst.getMatrixRowElts();
2947 NumLanes =
2948 llvm::cast<llvm::FixedVectorType>(Val: ColConstsIndices->getType())
2949 ->getNumElements();
2950 }
2951
2952 llvm::Value *Row = Dst.getMatrixRowIdx();
2953 for (unsigned Col = 0; Col < NumLanes; ++Col) {
2954 llvm::Value *ColIdx;
2955 if (ColConstsIndices)
2956 ColIdx = ColConstsIndices->getAggregateElement(Elt: Col);
2957 else
2958 ColIdx = llvm::ConstantInt::get(Ty: Row->getType(), V: Col);
2959 bool IsMatrixRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: Dst.getType());
2960 llvm::Value *EltIndex =
2961 MB.CreateIndex(RowIdx: Row, ColumnIdx: ColIdx, NumRows, NumCols, IsMatrixRowMajor);
2962 llvm::Value *Lane = llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: Col);
2963 llvm::Value *Zero = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
2964 llvm::Value *NewElt = Builder.CreateExtractElement(Vec: RowVal, Idx: Lane);
2965 Address DstElemAddr =
2966 Builder.CreateGEP(Addr: DstAddr, IdxList: {Zero, EltIndex}, ElementType: DestAddrTy, Align: ElemAlign);
2967 Builder.CreateStore(Val: NewElt, Addr: DstElemAddr, IsVolatile: Dst.isVolatileQualified());
2968 }
2969
2970 return;
2971 }
2972
2973 assert(Dst.isBitField() && "Unknown LValue type");
2974 return EmitStoreThroughBitfieldLValue(Src, Dst);
2975 }
2976
2977 // Handle __ptrauth qualification by re-signing the value.
2978 if (PointerAuthQualifier PointerAuth = Dst.getQuals().getPointerAuth()) {
2979 Src = RValue::get(V: EmitPointerAuthQualify(Qualifier: PointerAuth, Pointer: Src.getScalarVal(),
2980 ValueType: Dst.getType(), StorageAddress: Dst.getAddress(),
2981 /*known nonnull*/ IsKnownNonNull: false));
2982 }
2983
2984 // There's special magic for assigning into an ARC-qualified l-value.
2985 if (Qualifiers::ObjCLifetime Lifetime = Dst.getQuals().getObjCLifetime()) {
2986 switch (Lifetime) {
2987 case Qualifiers::OCL_None:
2988 llvm_unreachable("present but none");
2989
2990 case Qualifiers::OCL_ExplicitNone:
2991 // nothing special
2992 break;
2993
2994 case Qualifiers::OCL_Strong:
2995 if (isInit) {
2996 Src = RValue::get(V: EmitARCRetain(type: Dst.getType(), value: Src.getScalarVal()));
2997 break;
2998 }
2999 EmitARCStoreStrong(lvalue: Dst, value: Src.getScalarVal(), /*ignore*/ resultIgnored: true);
3000 return;
3001
3002 case Qualifiers::OCL_Weak:
3003 if (isInit)
3004 // Initialize and then skip the primitive store.
3005 EmitARCInitWeak(addr: Dst.getAddress(), value: Src.getScalarVal());
3006 else
3007 EmitARCStoreWeak(addr: Dst.getAddress(), value: Src.getScalarVal(),
3008 /*ignore*/ ignored: true);
3009 return;
3010
3011 case Qualifiers::OCL_Autoreleasing:
3012 Src = RValue::get(V: EmitObjCExtendObjectLifetime(T: Dst.getType(),
3013 Ptr: Src.getScalarVal()));
3014 // fall into the normal path
3015 break;
3016 }
3017 }
3018
3019 if (Dst.isObjCWeak() && !Dst.isNonGC()) {
3020 // load of a __weak object.
3021 Address LvalueDst = Dst.getAddress();
3022 llvm::Value *src = Src.getScalarVal();
3023 CGM.getObjCRuntime().EmitObjCWeakAssign(CGF&: *this, src, dest: LvalueDst);
3024 return;
3025 }
3026
3027 if (Dst.isObjCStrong() && !Dst.isNonGC()) {
3028 // load of a __strong object.
3029 Address LvalueDst = Dst.getAddress();
3030 llvm::Value *src = Src.getScalarVal();
3031 if (Dst.isObjCIvar()) {
3032 assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");
3033 llvm::Type *ResultType = IntPtrTy;
3034 Address dst = EmitPointerWithAlignment(E: Dst.getBaseIvarExp());
3035 llvm::Value *RHS = dst.emitRawPointer(CGF&: *this);
3036 RHS = Builder.CreatePtrToInt(V: RHS, DestTy: ResultType, Name: "sub.ptr.rhs.cast");
3037 llvm::Value *LHS = Builder.CreatePtrToInt(V: LvalueDst.emitRawPointer(CGF&: *this),
3038 DestTy: ResultType, Name: "sub.ptr.lhs.cast");
3039 llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, Name: "ivar.offset");
3040 CGM.getObjCRuntime().EmitObjCIvarAssign(CGF&: *this, src, dest: dst, ivarOffset: BytesBetween);
3041 } else if (Dst.isGlobalObjCRef()) {
3042 CGM.getObjCRuntime().EmitObjCGlobalAssign(CGF&: *this, src, dest: LvalueDst,
3043 threadlocal: Dst.isThreadLocalRef());
3044 }
3045 else
3046 CGM.getObjCRuntime().EmitObjCStrongCastAssign(CGF&: *this, src, dest: LvalueDst);
3047 return;
3048 }
3049
3050 assert(Src.isScalar() && "Can't emit an agg store with this method");
3051 EmitStoreOfScalar(value: Src.getScalarVal(), lvalue: Dst, isInit);
3052}
3053
3054void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
3055 llvm::Value **Result) {
3056 const CGBitFieldInfo &Info = Dst.getBitFieldInfo();
3057 llvm::Type *ResLTy = convertTypeForLoadStore(ASTTy: Dst.getType());
3058 Address Ptr = Dst.getBitFieldAddress();
3059
3060 // Get the source value, truncated to the width of the bit-field.
3061 llvm::Value *SrcVal = Src.getScalarVal();
3062
3063 // Cast the source to the storage type and shift it into place.
3064 SrcVal = Builder.CreateIntCast(V: SrcVal, DestTy: Ptr.getElementType(),
3065 /*isSigned=*/false);
3066 llvm::Value *MaskedVal = SrcVal;
3067
3068 const bool UseVolatile =
3069 CGM.getCodeGenOpts().AAPCSBitfieldWidth && Dst.isVolatileQualified() &&
3070 Info.VolatileStorageSize != 0 && isAAPCS(TargetInfo: CGM.getTarget());
3071 const unsigned StorageSize =
3072 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
3073 const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
3074 // See if there are other bits in the bitfield's storage we'll need to load
3075 // and mask together with source before storing.
3076 if (StorageSize != Info.Size) {
3077 assert(StorageSize > Info.Size && "Invalid bitfield size.");
3078 llvm::Value *Val =
3079 Builder.CreateLoad(Addr: Ptr, IsVolatile: Dst.isVolatileQualified(), Name: "bf.load");
3080
3081 // Mask the source value as needed.
3082 if (!Dst.getType()->hasBooleanRepresentation())
3083 SrcVal = Builder.CreateAnd(
3084 LHS: SrcVal, RHS: llvm::APInt::getLowBitsSet(numBits: StorageSize, loBitsSet: Info.Size),
3085 Name: "bf.value");
3086 MaskedVal = SrcVal;
3087 if (Offset)
3088 SrcVal = Builder.CreateShl(LHS: SrcVal, RHS: Offset, Name: "bf.shl");
3089
3090 // Mask out the original value.
3091 Val = Builder.CreateAnd(
3092 LHS: Val, RHS: ~llvm::APInt::getBitsSet(numBits: StorageSize, loBit: Offset, hiBit: Offset + Info.Size),
3093 Name: "bf.clear");
3094
3095 // Or together the unchanged values and the source value.
3096 SrcVal = Builder.CreateOr(LHS: Val, RHS: SrcVal, Name: "bf.set");
3097 } else {
3098 assert(Offset == 0);
3099 // According to the AACPS:
3100 // When a volatile bit-field is written, and its container does not overlap
3101 // with any non-bit-field member, its container must be read exactly once
3102 // and written exactly once using the access width appropriate to the type
3103 // of the container. The two accesses are not atomic.
3104 if (Dst.isVolatileQualified() && isAAPCS(TargetInfo: CGM.getTarget()) &&
3105 CGM.getCodeGenOpts().ForceAAPCSBitfieldLoad)
3106 Builder.CreateLoad(Addr: Ptr, IsVolatile: true, Name: "bf.load");
3107 }
3108
3109 // Write the new value back out.
3110 auto *I = Builder.CreateStore(Val: SrcVal, Addr: Ptr, IsVolatile: Dst.isVolatileQualified());
3111 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: SrcVal);
3112
3113 // Return the new value of the bit-field, if requested.
3114 if (Result) {
3115 llvm::Value *ResultVal = MaskedVal;
3116
3117 // Sign extend the value if needed.
3118 if (Info.IsSigned) {
3119 assert(Info.Size <= StorageSize);
3120 unsigned HighBits = StorageSize - Info.Size;
3121 if (HighBits) {
3122 ResultVal = Builder.CreateShl(LHS: ResultVal, RHS: HighBits, Name: "bf.result.shl");
3123 ResultVal = Builder.CreateAShr(LHS: ResultVal, RHS: HighBits, Name: "bf.result.ashr");
3124 }
3125 }
3126
3127 ResultVal = Builder.CreateIntCast(V: ResultVal, DestTy: ResLTy, isSigned: Info.IsSigned,
3128 Name: "bf.result.cast");
3129 *Result = EmitFromMemory(Value: ResultVal, Ty: Dst.getType());
3130 }
3131}
3132
3133void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
3134 LValue Dst) {
3135 llvm::Value *SrcVal = Src.getScalarVal();
3136 Address DstAddr = Dst.getExtVectorAddress();
3137 const llvm::Constant *Elts = Dst.getExtVectorElts();
3138 if (DstAddr.getElementType()->getScalarSizeInBits() >
3139 SrcVal->getType()->getScalarSizeInBits())
3140 SrcVal = Builder.CreateZExt(
3141 V: SrcVal, DestTy: convertTypeForLoadStore(ASTTy: Dst.getType(), LLVMTy: SrcVal->getType()));
3142
3143 if (getLangOpts().HLSL) {
3144 llvm::Type *DestAddrTy = DstAddr.getElementType();
3145 // HLSL allows storing to scalar values through ExtVector component LValues.
3146 // To support this we need to handle the case where the destination address
3147 // is a scalar.
3148 if (!DestAddrTy->isVectorTy()) {
3149 assert(!Dst.getType()->isVectorType() &&
3150 "this should only occur for non-vector l-values");
3151 Builder.CreateStore(Val: SrcVal, Addr: DstAddr, IsVolatile: Dst.isVolatileQualified());
3152 return;
3153 }
3154
3155 // HLSL allows direct access to vector elements, so storing to individual
3156 // elements of a vector through ExtVector is handled as separate store
3157 // instructions.
3158 // If we are updating multiple elements, Dst and Src are vectors; for
3159 // a single element update they are scalars.
3160 const VectorType *VTy = Dst.getType()->getAs<VectorType>();
3161 unsigned NumSrcElts = VTy ? VTy->getNumElements() : 1;
3162 CharUnits ElemAlign = CharUnits::fromQuantity(
3163 Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty: DestAddrTy->getScalarType()));
3164 llvm::Value *Zero = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
3165
3166 for (unsigned I = 0; I != NumSrcElts; ++I) {
3167 llvm::Value *Val = VTy ? Builder.CreateExtractElement(
3168 Vec: SrcVal, Idx: llvm::ConstantInt::get(Ty: Int32Ty, V: I))
3169 : SrcVal;
3170 unsigned FieldNo = getAccessedFieldNo(Idx: I, Elts);
3171 Address DstElemAddr = Address::invalid();
3172 if (FieldNo == 0)
3173 DstElemAddr = DstAddr.withAlignment(NewAlignment: ElemAlign);
3174 else
3175 DstElemAddr = Builder.CreateGEP(
3176 Addr: DstAddr, IdxList: {Zero, llvm::ConstantInt::get(Ty: Int32Ty, V: FieldNo)},
3177 ElementType: DestAddrTy, Align: ElemAlign);
3178 Builder.CreateStore(Val, Addr: DstElemAddr, IsVolatile: Dst.isVolatileQualified());
3179 }
3180 return;
3181 }
3182
3183 // This access turns into a read/modify/write of the vector. Load the input
3184 // value now.
3185 llvm::Value *Vec = Builder.CreateLoad(Addr: DstAddr, IsVolatile: Dst.isVolatileQualified());
3186 llvm::Type *VecTy = Vec->getType();
3187
3188 if (const VectorType *VTy = Dst.getType()->getAs<VectorType>()) {
3189 unsigned NumSrcElts = VTy->getNumElements();
3190 unsigned NumDstElts = cast<llvm::FixedVectorType>(Val: VecTy)->getNumElements();
3191 if (NumDstElts == NumSrcElts) {
3192 // Use shuffle vector is the src and destination are the same number of
3193 // elements and restore the vector mask since it is on the side it will be
3194 // stored.
3195 SmallVector<int, 4> Mask(NumDstElts);
3196 for (unsigned i = 0; i != NumSrcElts; ++i)
3197 Mask[getAccessedFieldNo(Idx: i, Elts)] = i;
3198
3199 Vec = Builder.CreateShuffleVector(V: SrcVal, Mask);
3200 } else if (NumDstElts > NumSrcElts) {
3201 // Extended the source vector to the same length and then shuffle it
3202 // into the destination.
3203 // FIXME: since we're shuffling with undef, can we just use the indices
3204 // into that? This could be simpler.
3205 SmallVector<int, 4> ExtMask;
3206 for (unsigned i = 0; i != NumSrcElts; ++i)
3207 ExtMask.push_back(Elt: i);
3208 ExtMask.resize(N: NumDstElts, NV: -1);
3209 llvm::Value *ExtSrcVal = Builder.CreateShuffleVector(V: SrcVal, Mask: ExtMask);
3210 // build identity
3211 SmallVector<int, 4> Mask;
3212 for (unsigned i = 0; i != NumDstElts; ++i)
3213 Mask.push_back(Elt: i);
3214
3215 // When the vector size is odd and .odd or .hi is used, the last element
3216 // of the Elts constant array will be one past the size of the vector.
3217 // Ignore the last element here, if it is greater than the mask size.
3218 if (getAccessedFieldNo(Idx: NumSrcElts - 1, Elts) == Mask.size())
3219 NumSrcElts--;
3220
3221 // modify when what gets shuffled in
3222 for (unsigned i = 0; i != NumSrcElts; ++i)
3223 Mask[getAccessedFieldNo(Idx: i, Elts)] = i + NumDstElts;
3224 Vec = Builder.CreateShuffleVector(V1: Vec, V2: ExtSrcVal, Mask);
3225 } else {
3226 // We should never shorten the vector
3227 llvm_unreachable("unexpected shorten vector length");
3228 }
3229 } else {
3230 // If the Src is a scalar (not a vector), and the target is a vector it must
3231 // be updating one element.
3232 unsigned InIdx = getAccessedFieldNo(Idx: 0, Elts);
3233 llvm::Value *Elt = llvm::ConstantInt::get(Ty: SizeTy, V: InIdx);
3234
3235 Vec = Builder.CreateInsertElement(Vec, NewElt: SrcVal, Idx: Elt);
3236 }
3237
3238 Builder.CreateStore(Val: Vec, Addr: Dst.getExtVectorAddress(),
3239 IsVolatile: Dst.isVolatileQualified());
3240}
3241
3242/// Store of global named registers are always calls to intrinsics.
3243void CodeGenFunction::EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst) {
3244 assert((Dst.getType()->isIntegerType() || Dst.getType()->isPointerType()) &&
3245 "Bad type for register variable");
3246 llvm::MDNode *RegName = cast<llvm::MDNode>(
3247 Val: cast<llvm::MetadataAsValue>(Val: Dst.getGlobalReg())->getMetadata());
3248 assert(RegName && "Register LValue is not metadata");
3249
3250 // We accept integer and pointer types only
3251 llvm::Type *OrigTy = CGM.getTypes().ConvertType(T: Dst.getType());
3252 llvm::Type *Ty = OrigTy;
3253 if (OrigTy->isPointerTy())
3254 Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
3255 llvm::Type *Types[] = { Ty };
3256
3257 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::write_register, Tys: Types);
3258 llvm::Value *Value = Src.getScalarVal();
3259 if (OrigTy->isPointerTy())
3260 Value = Builder.CreatePtrToInt(V: Value, DestTy: Ty);
3261 Builder.CreateCall(
3262 Callee: F, Args: {llvm::MetadataAsValue::get(Context&: Ty->getContext(), MD: RegName), Value});
3263}
3264
3265// setObjCGCLValueClass - sets class of the lvalue for the purpose of
3266// generating write-barries API. It is currently a global, ivar,
3267// or neither.
3268static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
3269 LValue &LV,
3270 bool IsMemberAccess=false) {
3271 if (Ctx.getLangOpts().getGC() == LangOptions::NonGC)
3272 return;
3273
3274 if (isa<ObjCIvarRefExpr>(Val: E)) {
3275 QualType ExpTy = E->getType();
3276 if (IsMemberAccess && ExpTy->isPointerType()) {
3277 // If ivar is a structure pointer, assigning to field of
3278 // this struct follows gcc's behavior and makes it a non-ivar
3279 // writer-barrier conservatively.
3280 ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
3281 if (ExpTy->isRecordType()) {
3282 LV.setObjCIvar(false);
3283 return;
3284 }
3285 }
3286 LV.setObjCIvar(true);
3287 auto *Exp = cast<ObjCIvarRefExpr>(Val: const_cast<Expr *>(E));
3288 LV.setBaseIvarExp(Exp->getBase());
3289 LV.setObjCArray(E->getType()->isArrayType());
3290 return;
3291 }
3292
3293 if (const auto *Exp = dyn_cast<DeclRefExpr>(Val: E)) {
3294 if (const auto *VD = dyn_cast<VarDecl>(Val: Exp->getDecl())) {
3295 if (VD->hasGlobalStorage()) {
3296 LV.setGlobalObjCRef(true);
3297 LV.setThreadLocalRef(VD->getTLSKind() != VarDecl::TLS_None);
3298 }
3299 }
3300 LV.setObjCArray(E->getType()->isArrayType());
3301 return;
3302 }
3303
3304 if (const auto *Exp = dyn_cast<UnaryOperator>(Val: E)) {
3305 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3306 return;
3307 }
3308
3309 if (const auto *Exp = dyn_cast<ParenExpr>(Val: E)) {
3310 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3311 if (LV.isObjCIvar()) {
3312 // If cast is to a structure pointer, follow gcc's behavior and make it
3313 // a non-ivar write-barrier.
3314 QualType ExpTy = E->getType();
3315 if (ExpTy->isPointerType())
3316 ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
3317 if (ExpTy->isRecordType())
3318 LV.setObjCIvar(false);
3319 }
3320 return;
3321 }
3322
3323 if (const auto *Exp = dyn_cast<GenericSelectionExpr>(Val: E)) {
3324 setObjCGCLValueClass(Ctx, E: Exp->getResultExpr(), LV);
3325 return;
3326 }
3327
3328 if (const auto *Exp = dyn_cast<ImplicitCastExpr>(Val: E)) {
3329 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3330 return;
3331 }
3332
3333 if (const auto *Exp = dyn_cast<CStyleCastExpr>(Val: E)) {
3334 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3335 return;
3336 }
3337
3338 if (const auto *Exp = dyn_cast<ObjCBridgedCastExpr>(Val: E)) {
3339 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3340 return;
3341 }
3342
3343 if (const auto *Exp = dyn_cast<ArraySubscriptExpr>(Val: E)) {
3344 setObjCGCLValueClass(Ctx, E: Exp->getBase(), LV);
3345 if (LV.isObjCIvar() && !LV.isObjCArray())
3346 // Using array syntax to assigning to what an ivar points to is not
3347 // same as assigning to the ivar itself. {id *Names;} Names[i] = 0;
3348 LV.setObjCIvar(false);
3349 else if (LV.isGlobalObjCRef() && !LV.isObjCArray())
3350 // Using array syntax to assigning to what global points to is not
3351 // same as assigning to the global itself. {id *G;} G[i] = 0;
3352 LV.setGlobalObjCRef(false);
3353 return;
3354 }
3355
3356 if (const auto *Exp = dyn_cast<MemberExpr>(Val: E)) {
3357 setObjCGCLValueClass(Ctx, E: Exp->getBase(), LV, IsMemberAccess: true);
3358 // We don't know if member is an 'ivar', but this flag is looked at
3359 // only in the context of LV.isObjCIvar().
3360 LV.setObjCArray(E->getType()->isArrayType());
3361 return;
3362 }
3363}
3364
3365static LValue EmitThreadPrivateVarDeclLValue(
3366 CodeGenFunction &CGF, const VarDecl *VD, QualType T, Address Addr,
3367 llvm::Type *RealVarTy, SourceLocation Loc) {
3368 if (CGF.CGM.getLangOpts().OpenMPIRBuilder)
3369 Addr = CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(
3370 CGF, VD, VDAddr: Addr, Loc);
3371 else
3372 Addr =
3373 CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, VDAddr: Addr, Loc);
3374
3375 Addr = Addr.withElementType(ElemTy: RealVarTy);
3376 return CGF.MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3377}
3378
3379static Address emitDeclTargetVarDeclLValue(CodeGenFunction &CGF,
3380 const VarDecl *VD, QualType T) {
3381 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
3382 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
3383 // Always return an invalid address for MT_Local, and also for
3384 // MT_To/MT_Enter when unified memory is not enabled. These use direct
3385 // access (global exists in device image). Otherwise, return a valid
3386 // address.
3387 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Local ||
3388 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
3389 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
3390 !CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory()))
3391 return Address::invalid();
3392 assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
3393 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
3394 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
3395 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) &&
3396 "Expected link clause OR to clause with unified memory enabled.");
3397 QualType PtrTy = CGF.getContext().getPointerType(T: VD->getType());
3398 Address Addr = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
3399 return CGF.EmitLoadOfPointer(Ptr: Addr, PtrTy: PtrTy->castAs<PointerType>());
3400}
3401
3402Address
3403CodeGenFunction::EmitLoadOfReference(LValue RefLVal,
3404 LValueBaseInfo *PointeeBaseInfo,
3405 TBAAAccessInfo *PointeeTBAAInfo) {
3406 llvm::LoadInst *Load =
3407 Builder.CreateLoad(Addr: RefLVal.getAddress(), IsVolatile: RefLVal.isVolatile());
3408 CGM.DecorateInstructionWithTBAA(Inst: Load, TBAAInfo: RefLVal.getTBAAInfo());
3409 QualType PTy = RefLVal.getType()->getPointeeType();
3410 CharUnits Align = CGM.getNaturalTypeAlignment(
3411 T: PTy, BaseInfo: PointeeBaseInfo, TBAAInfo: PointeeTBAAInfo, /*ForPointeeType=*/forPointeeType: true);
3412 if (!PTy->isIncompleteType()) {
3413 llvm::LLVMContext &Ctx = getLLVMContext();
3414 llvm::MDBuilder MDB(Ctx);
3415 // Emit !nonnull metadata
3416 if (CGM.getTypes().getTargetAddressSpace(T: PTy) == 0 &&
3417 !CGM.getCodeGenOpts().NullPointerIsValid)
3418 Load->setMetadata(KindID: llvm::LLVMContext::MD_nonnull,
3419 Node: llvm::MDNode::get(Context&: Ctx, MDs: {}));
3420 // Emit !align metadata
3421 if (PTy->isObjectType()) {
3422 auto AlignVal = Align.getQuantity();
3423 if (AlignVal > 1) {
3424 Load->setMetadata(
3425 KindID: llvm::LLVMContext::MD_align,
3426 Node: llvm::MDNode::get(Context&: Ctx, MDs: MDB.createConstant(C: llvm::ConstantInt::get(
3427 Ty: Builder.getInt64Ty(), V: AlignVal))));
3428 }
3429 }
3430 }
3431 return makeNaturalAddressForPointer(Ptr: Load, T: PTy, Alignment: Align,
3432 /*ForPointeeType=*/true, BaseInfo: PointeeBaseInfo,
3433 TBAAInfo: PointeeTBAAInfo);
3434}
3435
3436LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) {
3437 LValueBaseInfo PointeeBaseInfo;
3438 TBAAAccessInfo PointeeTBAAInfo;
3439 Address PointeeAddr = EmitLoadOfReference(RefLVal, PointeeBaseInfo: &PointeeBaseInfo,
3440 PointeeTBAAInfo: &PointeeTBAAInfo);
3441 return MakeAddrLValue(Addr: PointeeAddr, T: RefLVal.getType()->getPointeeType(),
3442 BaseInfo: PointeeBaseInfo, TBAAInfo: PointeeTBAAInfo);
3443}
3444
3445Address CodeGenFunction::EmitLoadOfPointer(Address Ptr,
3446 const PointerType *PtrTy,
3447 LValueBaseInfo *BaseInfo,
3448 TBAAAccessInfo *TBAAInfo) {
3449 llvm::Value *Addr = Builder.CreateLoad(Addr: Ptr);
3450 return makeNaturalAddressForPointer(Ptr: Addr, T: PtrTy->getPointeeType(),
3451 Alignment: CharUnits(), /*ForPointeeType=*/true,
3452 BaseInfo, TBAAInfo);
3453}
3454
3455LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr,
3456 const PointerType *PtrTy) {
3457 LValueBaseInfo BaseInfo;
3458 TBAAAccessInfo TBAAInfo;
3459 Address Addr = EmitLoadOfPointer(Ptr: PtrAddr, PtrTy, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
3460 return MakeAddrLValue(Addr, T: PtrTy->getPointeeType(), BaseInfo, TBAAInfo);
3461}
3462
3463static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
3464 const Expr *E, const VarDecl *VD) {
3465 QualType T = E->getType();
3466
3467 // If it's thread_local, emit a call to its wrapper function instead.
3468 if (VD->getTLSKind() == VarDecl::TLS_Dynamic &&
3469 CGF.CGM.getCXXABI().usesThreadWrapperFunction(VD))
3470 return CGF.CGM.getCXXABI().EmitThreadLocalVarDeclLValue(CGF, VD, LValType: T);
3471 // Check if the variable is marked as declare target with link clause in
3472 // device codegen.
3473 if (CGF.getLangOpts().OpenMPIsTargetDevice) {
3474 Address Addr = emitDeclTargetVarDeclLValue(CGF, VD, T);
3475 if (Addr.isValid())
3476 return CGF.MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3477 }
3478
3479 // Global HLSL resource arrays initialized on access; create a temporary with
3480 // the initialized global resource array.
3481 if (CGF.getLangOpts().HLSL && VD->getType()->isHLSLResourceRecordArray()) {
3482 std::optional<LValue> LV =
3483 CGF.CGM.getHLSLRuntime().emitGlobalResourceArrayAsLValue(CGF, ArrayDecl: VD);
3484 if (LV.has_value())
3485 return LV.value();
3486 }
3487
3488 llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(D: VD);
3489
3490 if (VD->getTLSKind() != VarDecl::TLS_None)
3491 V = CGF.Builder.CreateThreadLocalAddress(Ptr: V);
3492
3493 llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(T: VD->getType());
3494 CharUnits Alignment = CGF.getContext().getDeclAlign(D: VD);
3495 Address Addr(V, RealVarTy, Alignment);
3496 // Emit reference to the private copy of the variable if it is an OpenMP
3497 // threadprivate variable.
3498 if (CGF.getLangOpts().OpenMP && !CGF.getLangOpts().OpenMPSimd &&
3499 VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
3500 return EmitThreadPrivateVarDeclLValue(CGF, VD, T, Addr, RealVarTy,
3501 Loc: E->getExprLoc());
3502 }
3503 LValue LV = VD->getType()->isReferenceType() ?
3504 CGF.EmitLoadOfReferenceLValue(RefAddr: Addr, RefTy: VD->getType(),
3505 Source: AlignmentSource::Decl) :
3506 CGF.MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3507 setObjCGCLValueClass(Ctx: CGF.getContext(), E, LV);
3508 return LV;
3509}
3510
3511llvm::Constant *CodeGenModule::getRawFunctionPointer(GlobalDecl GD,
3512 llvm::Type *Ty) {
3513 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
3514 if (FD->hasAttr<WeakRefAttr>()) {
3515 ConstantAddress aliasee = GetWeakRefReference(VD: FD);
3516 return aliasee.getPointer();
3517 }
3518
3519 llvm::Constant *V = GetAddrOfFunction(GD, Ty);
3520 return V;
3521}
3522
3523static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF, const Expr *E,
3524 GlobalDecl GD) {
3525 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
3526 llvm::Constant *V = CGF.CGM.getFunctionPointer(GD);
3527 QualType ETy = E->getType();
3528 if (ETy->isCFIUncheckedCalleeFunctionType()) {
3529 if (auto *GV = dyn_cast<llvm::GlobalValue>(Val: V))
3530 V = llvm::NoCFIValue::get(GV);
3531 }
3532 CharUnits Alignment = CGF.getContext().getDeclAlign(D: FD);
3533 return CGF.MakeAddrLValue(V, T: ETy, Alignment, Source: AlignmentSource::Decl);
3534}
3535
3536static LValue EmitCapturedFieldLValue(CodeGenFunction &CGF, const FieldDecl *FD,
3537 llvm::Value *ThisValue) {
3538
3539 return CGF.EmitLValueForLambdaField(Field: FD, ThisValue);
3540}
3541
3542/// Named Registers are named metadata pointing to the register name
3543/// which will be read from/written to as an argument to the intrinsic
3544/// @llvm.read/write_register.
3545/// So far, only the name is being passed down, but other options such as
3546/// register type, allocation type or even optimization options could be
3547/// passed down via the metadata node.
3548static LValue EmitGlobalNamedRegister(const VarDecl *VD, CodeGenModule &CGM) {
3549 SmallString<64> Name("llvm.named.register.");
3550 AsmLabelAttr *Asm = VD->getAttr<AsmLabelAttr>();
3551 assert(Asm->getLabel().size() < 64-Name.size() &&
3552 "Register name too big");
3553 Name.append(RHS: Asm->getLabel());
3554 llvm::NamedMDNode *M =
3555 CGM.getModule().getOrInsertNamedMetadata(Name);
3556 if (M->getNumOperands() == 0) {
3557 llvm::MDString *Str = llvm::MDString::get(Context&: CGM.getLLVMContext(),
3558 Str: Asm->getLabel());
3559 llvm::Metadata *Ops[] = {Str};
3560 M->addOperand(M: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Ops));
3561 }
3562
3563 CharUnits Alignment = CGM.getContext().getDeclAlign(D: VD);
3564
3565 llvm::Value *Ptr =
3566 llvm::MetadataAsValue::get(Context&: CGM.getLLVMContext(), MD: M->getOperand(i: 0));
3567 return LValue::MakeGlobalReg(V: Ptr, alignment: Alignment, type: VD->getType());
3568}
3569
3570/// Determine whether we can emit a reference to \p VD from the current
3571/// context, despite not necessarily having seen an odr-use of the variable in
3572/// this context.
3573static bool canEmitSpuriousReferenceToVariable(CodeGenFunction &CGF,
3574 const DeclRefExpr *E,
3575 const VarDecl *VD) {
3576 // For a variable declared in an enclosing scope, do not emit a spurious
3577 // reference even if we have a capture, as that will emit an unwarranted
3578 // reference to our capture state, and will likely generate worse code than
3579 // emitting a local copy.
3580 if (E->refersToEnclosingVariableOrCapture())
3581 return false;
3582
3583 // For a local declaration declared in this function, we can always reference
3584 // it even if we don't have an odr-use.
3585 if (VD->hasLocalStorage()) {
3586 return VD->getDeclContext() ==
3587 dyn_cast_or_null<DeclContext>(Val: CGF.CurCodeDecl);
3588 }
3589
3590 // For a global declaration, we can emit a reference to it if we know
3591 // for sure that we are able to emit a definition of it.
3592 VD = VD->getDefinition(C&: CGF.getContext());
3593 if (!VD)
3594 return false;
3595
3596 // Don't emit a spurious reference if it might be to a variable that only
3597 // exists on a different device / target.
3598 // FIXME: This is unnecessarily broad. Check whether this would actually be a
3599 // cross-target reference.
3600 if (CGF.getLangOpts().OpenMP || CGF.getLangOpts().CUDA ||
3601 CGF.getLangOpts().OpenCL) {
3602 return false;
3603 }
3604
3605 // We can emit a spurious reference only if the linkage implies that we'll
3606 // be emitting a non-interposable symbol that will be retained until link
3607 // time.
3608 switch (CGF.CGM.getLLVMLinkageVarDefinition(VD)) {
3609 case llvm::GlobalValue::ExternalLinkage:
3610 case llvm::GlobalValue::LinkOnceODRLinkage:
3611 case llvm::GlobalValue::WeakODRLinkage:
3612 case llvm::GlobalValue::InternalLinkage:
3613 case llvm::GlobalValue::PrivateLinkage:
3614 return true;
3615 default:
3616 return false;
3617 }
3618}
3619
3620LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
3621 const NamedDecl *ND = E->getDecl();
3622 QualType T = E->getType();
3623
3624 assert(E->isNonOdrUse() != NOUR_Unevaluated &&
3625 "should not emit an unevaluated operand");
3626
3627 if (const auto *VD = dyn_cast<VarDecl>(Val: ND)) {
3628 // Global Named registers access via intrinsics only
3629 if (VD->getStorageClass() == SC_Register &&
3630 VD->hasAttr<AsmLabelAttr>() && !VD->isLocalVarDecl())
3631 return EmitGlobalNamedRegister(VD, CGM);
3632
3633 // If this DeclRefExpr does not constitute an odr-use of the variable,
3634 // we're not permitted to emit a reference to it in general, and it might
3635 // not be captured if capture would be necessary for a use. Emit the
3636 // constant value directly instead.
3637 if (E->isNonOdrUse() == NOUR_Constant &&
3638 (VD->getType()->isReferenceType() ||
3639 !canEmitSpuriousReferenceToVariable(CGF&: *this, E, VD))) {
3640 VD->getAnyInitializer(D&: VD);
3641 llvm::Constant *Val = ConstantEmitter(*this).emitAbstract(
3642 loc: E->getLocation(), value: *VD->evaluateValue(), T: VD->getType());
3643 assert(Val && "failed to emit constant expression");
3644
3645 Address Addr = Address::invalid();
3646 if (!VD->getType()->isReferenceType()) {
3647 // Spill the constant value to a global.
3648 Addr = CGM.createUnnamedGlobalFrom(D: *VD, Constant: Val,
3649 Align: getContext().getDeclAlign(D: VD));
3650 llvm::Type *VarTy = getTypes().ConvertTypeForMem(T: VD->getType());
3651 auto *PTy = llvm::PointerType::get(
3652 C&: getLLVMContext(), AddressSpace: getTypes().getTargetAddressSpace(T: VD->getType()));
3653 Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty: PTy, ElementTy: VarTy);
3654 } else {
3655 // Should we be using the alignment of the constant pointer we emitted?
3656 CharUnits Alignment =
3657 CGM.getNaturalTypeAlignment(T: E->getType(),
3658 /* BaseInfo= */ nullptr,
3659 /* TBAAInfo= */ nullptr,
3660 /* forPointeeType= */ true);
3661 Addr = makeNaturalAddressForPointer(Ptr: Val, T, Alignment);
3662 }
3663 return MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3664 }
3665
3666 // FIXME: Handle other kinds of non-odr-use DeclRefExprs.
3667
3668 // Check for captured variables.
3669 if (E->refersToEnclosingVariableOrCapture()) {
3670 VD = VD->getCanonicalDecl();
3671 if (auto *FD = LambdaCaptureFields.lookup(Val: VD))
3672 return EmitCapturedFieldLValue(CGF&: *this, FD, ThisValue: CXXABIThisValue);
3673 if (CapturedStmtInfo) {
3674 auto I = LocalDeclMap.find(Val: VD);
3675 if (I != LocalDeclMap.end()) {
3676 LValue CapLVal;
3677 if (VD->getType()->isReferenceType())
3678 CapLVal = EmitLoadOfReferenceLValue(RefAddr: I->second, RefTy: VD->getType(),
3679 Source: AlignmentSource::Decl);
3680 else
3681 CapLVal = MakeAddrLValue(Addr: I->second, T);
3682 // Mark lvalue as nontemporal if the variable is marked as nontemporal
3683 // in simd context.
3684 if (getLangOpts().OpenMP &&
3685 CGM.getOpenMPRuntime().isNontemporalDecl(VD))
3686 CapLVal.setNontemporal(/*Value=*/true);
3687 return CapLVal;
3688 }
3689 LValue CapLVal =
3690 EmitCapturedFieldLValue(CGF&: *this, FD: CapturedStmtInfo->lookup(VD),
3691 ThisValue: CapturedStmtInfo->getContextValue());
3692 Address LValueAddress = CapLVal.getAddress();
3693 CapLVal = MakeAddrLValue(Addr: Address(LValueAddress.emitRawPointer(CGF&: *this),
3694 LValueAddress.getElementType(),
3695 getContext().getDeclAlign(D: VD)),
3696 T: CapLVal.getType(),
3697 BaseInfo: LValueBaseInfo(AlignmentSource::Decl),
3698 TBAAInfo: CapLVal.getTBAAInfo());
3699 // Mark lvalue as nontemporal if the variable is marked as nontemporal
3700 // in simd context.
3701 if (getLangOpts().OpenMP &&
3702 CGM.getOpenMPRuntime().isNontemporalDecl(VD))
3703 CapLVal.setNontemporal(/*Value=*/true);
3704 return CapLVal;
3705 }
3706
3707 assert(isa<BlockDecl>(CurCodeDecl));
3708 Address addr = GetAddrOfBlockDecl(var: VD);
3709 return MakeAddrLValue(Addr: addr, T, Source: AlignmentSource::Decl);
3710 }
3711 }
3712
3713 // FIXME: We should be able to assert this for FunctionDecls as well!
3714 // FIXME: We should be able to assert this for all DeclRefExprs, not just
3715 // those with a valid source location.
3716 assert((ND->isUsed(false) || !isa<VarDecl>(ND) || E->isNonOdrUse() ||
3717 !E->getLocation().isValid()) &&
3718 "Should not use decl without marking it used!");
3719
3720 if (ND->hasAttr<WeakRefAttr>()) {
3721 const auto *VD = cast<ValueDecl>(Val: ND);
3722 ConstantAddress Aliasee = CGM.GetWeakRefReference(VD);
3723 return MakeAddrLValue(Addr: Aliasee, T, Source: AlignmentSource::Decl);
3724 }
3725
3726 if (const auto *VD = dyn_cast<VarDecl>(Val: ND)) {
3727 // Check if this is a global variable.
3728 if (VD->hasLinkage() || VD->isStaticDataMember())
3729 return EmitGlobalVarDeclLValue(CGF&: *this, E, VD);
3730
3731 Address addr = Address::invalid();
3732
3733 // The variable should generally be present in the local decl map.
3734 auto iter = LocalDeclMap.find(Val: VD);
3735 if (iter != LocalDeclMap.end()) {
3736 addr = iter->second;
3737
3738 // Otherwise, it might be static local we haven't emitted yet for
3739 // some reason; most likely, because it's in an outer function.
3740 } else if (VD->isStaticLocal()) {
3741 llvm::Constant *var = CGM.getOrCreateStaticVarDecl(
3742 D: *VD, Linkage: CGM.getLLVMLinkageVarDefinition(VD));
3743 addr = Address(
3744 var, ConvertTypeForMem(T: VD->getType()), getContext().getDeclAlign(D: VD));
3745
3746 // No other cases for now.
3747 } else {
3748 llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?");
3749 }
3750
3751 // Handle threadlocal function locals.
3752 if (VD->getTLSKind() != VarDecl::TLS_None)
3753 addr = addr.withPointer(
3754 NewPointer: Builder.CreateThreadLocalAddress(Ptr: addr.getBasePointer()),
3755 IsKnownNonNull: NotKnownNonNull);
3756
3757 // Check for OpenMP threadprivate variables.
3758 if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd &&
3759 VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
3760 return EmitThreadPrivateVarDeclLValue(
3761 CGF&: *this, VD, T, Addr: addr, RealVarTy: getTypes().ConvertTypeForMem(T: VD->getType()),
3762 Loc: E->getExprLoc());
3763 }
3764
3765 // Drill into block byref variables.
3766 bool isBlockByref = VD->isEscapingByref();
3767 if (isBlockByref) {
3768 addr = emitBlockByrefAddress(baseAddr: addr, V: VD);
3769 }
3770
3771 // Drill into reference types.
3772 LValue LV = VD->getType()->isReferenceType() ?
3773 EmitLoadOfReferenceLValue(RefAddr: addr, RefTy: VD->getType(), Source: AlignmentSource::Decl) :
3774 MakeAddrLValue(Addr: addr, T, Source: AlignmentSource::Decl);
3775
3776 bool isLocalStorage = VD->hasLocalStorage();
3777
3778 bool NonGCable = isLocalStorage &&
3779 !VD->getType()->isReferenceType() &&
3780 !isBlockByref;
3781 if (NonGCable) {
3782 LV.getQuals().removeObjCGCAttr();
3783 LV.setNonGC(true);
3784 }
3785
3786 bool isImpreciseLifetime =
3787 (isLocalStorage && !VD->hasAttr<ObjCPreciseLifetimeAttr>());
3788 if (isImpreciseLifetime)
3789 LV.setARCPreciseLifetime(ARCImpreciseLifetime);
3790 setObjCGCLValueClass(Ctx: getContext(), E, LV);
3791 return LV;
3792 }
3793
3794 if (const auto *FD = dyn_cast<FunctionDecl>(Val: ND))
3795 return EmitFunctionDeclLValue(CGF&: *this, E, GD: FD);
3796
3797 // FIXME: While we're emitting a binding from an enclosing scope, all other
3798 // DeclRefExprs we see should be implicitly treated as if they also refer to
3799 // an enclosing scope.
3800 if (const auto *BD = dyn_cast<BindingDecl>(Val: ND)) {
3801 if (E->refersToEnclosingVariableOrCapture()) {
3802 auto *FD = LambdaCaptureFields.lookup(Val: BD);
3803 return EmitCapturedFieldLValue(CGF&: *this, FD, ThisValue: CXXABIThisValue);
3804 }
3805 // Suppress debug location updates when visiting the binding, since the
3806 // binding may emit instructions that would otherwise be associated with the
3807 // binding itself, rather than the expression referencing the binding. (this
3808 // leads to jumpy debug stepping behavior where the location/debugger jump
3809 // back to the binding declaration, then back to the expression referencing
3810 // the binding)
3811 DisableDebugLocationUpdates D(*this);
3812 return EmitLValue(E: BD->getBinding(), IsKnownNonNull: NotKnownNonNull);
3813 }
3814
3815 // We can form DeclRefExprs naming GUID declarations when reconstituting
3816 // non-type template parameters into expressions.
3817 if (const auto *GD = dyn_cast<MSGuidDecl>(Val: ND))
3818 return MakeAddrLValue(Addr: CGM.GetAddrOfMSGuidDecl(GD), T,
3819 Source: AlignmentSource::Decl);
3820
3821 if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(Val: ND)) {
3822 ConstantAddress ATPO = CGM.GetAddrOfTemplateParamObject(TPO);
3823 auto AS = getLangASFromTargetAS(TargetAS: ATPO.getAddressSpace());
3824
3825 if (AS != T.getAddressSpace()) {
3826 auto TargetAS = getContext().getTargetAddressSpace(AS: T.getAddressSpace());
3827 llvm::Type *PtrTy =
3828 llvm::PointerType::get(C&: CGM.getLLVMContext(), AddressSpace: TargetAS);
3829 llvm::Constant *ASC = CGM.performAddrSpaceCast(Src: ATPO.getPointer(), DestTy: PtrTy);
3830 ATPO = ConstantAddress(ASC, ATPO.getElementType(), ATPO.getAlignment());
3831 }
3832
3833 return MakeAddrLValue(Addr: ATPO, T, Source: AlignmentSource::Decl);
3834 }
3835
3836 llvm_unreachable("Unhandled DeclRefExpr");
3837}
3838
3839LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
3840 // __extension__ doesn't affect lvalue-ness.
3841 if (E->getOpcode() == UO_Extension)
3842 return EmitLValue(E: E->getSubExpr());
3843
3844 QualType ExprTy = getContext().getCanonicalType(T: E->getSubExpr()->getType());
3845 switch (E->getOpcode()) {
3846 default: llvm_unreachable("Unknown unary operator lvalue!");
3847 case UO_Deref: {
3848 QualType T = E->getSubExpr()->getType()->getPointeeType();
3849 assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
3850
3851 LValueBaseInfo BaseInfo;
3852 TBAAAccessInfo TBAAInfo;
3853 Address Addr = EmitPointerWithAlignment(E: E->getSubExpr(), BaseInfo: &BaseInfo,
3854 TBAAInfo: &TBAAInfo);
3855 LValue LV = MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
3856 LV.getQuals().setAddressSpace(ExprTy.getAddressSpace());
3857
3858 // We should not generate __weak write barrier on indirect reference
3859 // of a pointer to object; as in void foo (__weak id *param); *param = 0;
3860 // But, we continue to generate __strong write barrier on indirect write
3861 // into a pointer to object.
3862 if (getLangOpts().ObjC &&
3863 getLangOpts().getGC() != LangOptions::NonGC &&
3864 LV.isObjCWeak())
3865 LV.setNonGC(!E->isOBJCGCCandidate(Ctx&: getContext()));
3866 return LV;
3867 }
3868 case UO_Real:
3869 case UO_Imag: {
3870 LValue LV = EmitLValue(E: E->getSubExpr());
3871 assert(LV.isSimple() && "real/imag on non-ordinary l-value");
3872
3873 // __real is valid on scalars. This is a faster way of testing that.
3874 // __imag can only produce an rvalue on scalars.
3875 if (E->getOpcode() == UO_Real &&
3876 !LV.getAddress().getElementType()->isStructTy()) {
3877 assert(E->getSubExpr()->getType()->isArithmeticType());
3878 return LV;
3879 }
3880
3881 QualType T = ExprTy->castAs<ComplexType>()->getElementType();
3882
3883 Address Component =
3884 (E->getOpcode() == UO_Real
3885 ? emitAddrOfRealComponent(complex: LV.getAddress(), complexType: LV.getType())
3886 : emitAddrOfImagComponent(complex: LV.getAddress(), complexType: LV.getType()));
3887 LValue ElemLV = MakeAddrLValue(Addr: Component, T, BaseInfo: LV.getBaseInfo(),
3888 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: T));
3889 ElemLV.getQuals().addQualifiers(Q: LV.getQuals());
3890 return ElemLV;
3891 }
3892 case UO_PreInc:
3893 case UO_PreDec: {
3894 LValue LV = EmitLValue(E: E->getSubExpr());
3895 bool isInc = E->getOpcode() == UO_PreInc;
3896
3897 if (E->getType()->isAnyComplexType())
3898 EmitComplexPrePostIncDec(E, LV, isInc, isPre: true/*isPre*/);
3899 else
3900 EmitScalarPrePostIncDec(E, LV, isInc, isPre: true/*isPre*/);
3901 return LV;
3902 }
3903 }
3904}
3905
3906LValue CodeGenFunction::EmitStringLiteralLValue(const StringLiteral *E) {
3907 return MakeAddrLValue(Addr: CGM.GetAddrOfConstantStringFromLiteral(S: E),
3908 T: E->getType(), Source: AlignmentSource::Decl);
3909}
3910
3911LValue CodeGenFunction::EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E) {
3912 return MakeAddrLValue(Addr: CGM.GetAddrOfConstantStringFromObjCEncode(E),
3913 T: E->getType(), Source: AlignmentSource::Decl);
3914}
3915
3916LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
3917 auto SL = E->getFunctionName();
3918 assert(SL != nullptr && "No StringLiteral name in PredefinedExpr");
3919 StringRef FnName = CurFn->getName();
3920 FnName.consume_front(Prefix: "\01");
3921 StringRef NameItems[] = {
3922 PredefinedExpr::getIdentKindName(IK: E->getIdentKind()), FnName};
3923 std::string GVName = llvm::join(Begin: NameItems, End: NameItems + 2, Separator: ".");
3924 if (auto *BD = dyn_cast_or_null<BlockDecl>(Val: CurCodeDecl)) {
3925 std::string Name = std::string(SL->getString());
3926 if (!Name.empty()) {
3927 unsigned Discriminator =
3928 CGM.getCXXABI().getMangleContext().getBlockId(BD, Local: true);
3929 if (Discriminator)
3930 Name += "_" + Twine(Discriminator + 1).str();
3931 auto C = CGM.GetAddrOfConstantCString(Str: Name, GlobalName: GVName);
3932 return MakeAddrLValue(Addr: C, T: E->getType(), Source: AlignmentSource::Decl);
3933 } else {
3934 auto C = CGM.GetAddrOfConstantCString(Str: std::string(FnName), GlobalName: GVName);
3935 return MakeAddrLValue(Addr: C, T: E->getType(), Source: AlignmentSource::Decl);
3936 }
3937 }
3938 auto C = CGM.GetAddrOfConstantStringFromLiteral(S: SL, Name: GVName);
3939 return MakeAddrLValue(Addr: C, T: E->getType(), Source: AlignmentSource::Decl);
3940}
3941
3942/// Emit a type description suitable for use by a runtime sanitizer library. The
3943/// format of a type descriptor is
3944///
3945/// \code
3946/// { i16 TypeKind, i16 TypeInfo }
3947/// \endcode
3948///
3949/// followed by an array of i8 containing the type name with extra information
3950/// for BitInt. TypeKind is TK_Integer(0) for an integer, TK_Float(1) for a
3951/// floating point value, TK_BitInt(2) for BitInt and TK_Unknown(0xFFFF) for
3952/// anything else.
3953llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) {
3954 // Only emit each type's descriptor once.
3955 if (llvm::Constant *C = CGM.getTypeDescriptorFromMap(Ty: T))
3956 return C;
3957
3958 uint16_t TypeKind = TK_Unknown;
3959 uint16_t TypeInfo = 0;
3960 bool IsBitInt = false;
3961
3962 if (T->isIntegerType()) {
3963 TypeKind = TK_Integer;
3964 TypeInfo = (llvm::Log2_32(Value: getContext().getTypeSize(T)) << 1) |
3965 (T->isSignedIntegerType() ? 1 : 0);
3966 // Follow suggestion from discussion of issue 64100.
3967 // So we can write the exact amount of bits in TypeName after '\0'
3968 // making it <diagnostic-like type name>.'\0'.<32-bit width>.
3969 if (T->isSignedIntegerType() && T->getAs<BitIntType>()) {
3970 // Do a sanity checks as we are using 32-bit type to store bit length.
3971 assert(getContext().getTypeSize(T) > 0 &&
3972 " non positive amount of bits in __BitInt type");
3973 assert(getContext().getTypeSize(T) <= 0xFFFFFFFF &&
3974 " too many bits in __BitInt type");
3975
3976 // Redefine TypeKind with the actual __BitInt type if we have signed
3977 // BitInt.
3978 TypeKind = TK_BitInt;
3979 IsBitInt = true;
3980 }
3981 } else if (T->isFloatingType()) {
3982 TypeKind = TK_Float;
3983 TypeInfo = getContext().getTypeSize(T);
3984 }
3985
3986 // Format the type name as if for a diagnostic, including quotes and
3987 // optionally an 'aka'.
3988 SmallString<32> Buffer;
3989 CGM.getDiags().ConvertArgToString(Kind: DiagnosticsEngine::ak_qualtype,
3990 Val: (intptr_t)T.getAsOpaquePtr(), Modifier: StringRef(),
3991 Argument: StringRef(), PrevArgs: {}, Output&: Buffer, QualTypeVals: {});
3992
3993 if (IsBitInt) {
3994 // The Structure is: 0 to end the string, 32 bit unsigned integer in target
3995 // endianness, zero.
3996 char S[6] = {'\0', '\0', '\0', '\0', '\0', '\0'};
3997 const auto *EIT = T->castAs<BitIntType>();
3998 uint32_t Bits = EIT->getNumBits();
3999 llvm::support::endian::write32(P: S + 1, V: Bits,
4000 E: getTarget().isBigEndian()
4001 ? llvm::endianness::big
4002 : llvm::endianness::little);
4003 StringRef Str = StringRef(S, sizeof(S) / sizeof(decltype(S[0])));
4004 Buffer.append(RHS: Str);
4005 }
4006
4007 llvm::Constant *Components[] = {
4008 Builder.getInt16(C: TypeKind), Builder.getInt16(C: TypeInfo),
4009 llvm::ConstantDataArray::getString(Context&: getLLVMContext(), Initializer: Buffer)
4010 };
4011 llvm::Constant *Descriptor = llvm::ConstantStruct::getAnon(V: Components);
4012
4013 auto *GV = new llvm::GlobalVariable(
4014 CGM.getModule(), Descriptor->getType(),
4015 /*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage, Descriptor);
4016 GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4017 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV);
4018
4019 // Remember the descriptor for this type.
4020 CGM.setTypeDescriptorInMap(Ty: T, C: GV);
4021
4022 return GV;
4023}
4024
4025llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) {
4026 llvm::Type *TargetTy = IntPtrTy;
4027
4028 if (V->getType() == TargetTy)
4029 return V;
4030
4031 // Floating-point types which fit into intptr_t are bitcast to integers
4032 // and then passed directly (after zero-extension, if necessary).
4033 if (V->getType()->isFloatingPointTy()) {
4034 unsigned Bits = V->getType()->getPrimitiveSizeInBits().getFixedValue();
4035 if (Bits <= TargetTy->getIntegerBitWidth())
4036 V = Builder.CreateBitCast(V, DestTy: llvm::Type::getIntNTy(C&: getLLVMContext(),
4037 N: Bits));
4038 }
4039
4040 // Integers which fit in intptr_t are zero-extended and passed directly.
4041 if (V->getType()->isIntegerTy() &&
4042 V->getType()->getIntegerBitWidth() <= TargetTy->getIntegerBitWidth())
4043 return Builder.CreateZExt(V, DestTy: TargetTy);
4044
4045 // Pointers are passed directly, everything else is passed by address.
4046 if (!V->getType()->isPointerTy()) {
4047 RawAddress Ptr = CreateDefaultAlignTempAlloca(Ty: V->getType());
4048 Builder.CreateStore(Val: V, Addr: Ptr);
4049 V = Ptr.getPointer();
4050 }
4051 return Builder.CreatePtrToInt(V, DestTy: TargetTy);
4052}
4053
4054/// Emit a representation of a SourceLocation for passing to a handler
4055/// in a sanitizer runtime library. The format for this data is:
4056/// \code
4057/// struct SourceLocation {
4058/// const char *Filename;
4059/// int32_t Line, Column;
4060/// };
4061/// \endcode
4062/// For an invalid SourceLocation, the Filename pointer is null.
4063llvm::Constant *CodeGenFunction::EmitCheckSourceLocation(SourceLocation Loc) {
4064 llvm::Constant *Filename;
4065 int Line, Column;
4066
4067 PresumedLoc PLoc = getContext().getSourceManager().getPresumedLoc(Loc);
4068 if (PLoc.isValid()) {
4069 StringRef FilenameString = PLoc.getFilename();
4070
4071 int PathComponentsToStrip =
4072 CGM.getCodeGenOpts().EmitCheckPathComponentsToStrip;
4073 if (PathComponentsToStrip < 0) {
4074 assert(PathComponentsToStrip != INT_MIN);
4075 int PathComponentsToKeep = -PathComponentsToStrip;
4076 auto I = llvm::sys::path::rbegin(path: FilenameString);
4077 auto E = llvm::sys::path::rend(path: FilenameString);
4078 while (I != E && --PathComponentsToKeep)
4079 ++I;
4080
4081 FilenameString = FilenameString.substr(Start: I - E);
4082 } else if (PathComponentsToStrip > 0) {
4083 auto I = llvm::sys::path::begin(path: FilenameString);
4084 auto E = llvm::sys::path::end(path: FilenameString);
4085 while (I != E && PathComponentsToStrip--)
4086 ++I;
4087
4088 if (I != E)
4089 FilenameString =
4090 FilenameString.substr(Start: I - llvm::sys::path::begin(path: FilenameString));
4091 else
4092 FilenameString = llvm::sys::path::filename(path: FilenameString);
4093 }
4094
4095 auto FilenameGV =
4096 CGM.GetAddrOfConstantCString(Str: std::string(FilenameString), GlobalName: ".src");
4097 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(
4098 GV: cast<llvm::GlobalVariable>(
4099 Val: FilenameGV.getPointer()->stripPointerCasts()));
4100 Filename = FilenameGV.getPointer();
4101 Line = PLoc.getLine();
4102 Column = PLoc.getColumn();
4103 } else {
4104 Filename = llvm::Constant::getNullValue(Ty: Int8PtrTy);
4105 Line = Column = 0;
4106 }
4107
4108 llvm::Constant *Data[] = {Filename, Builder.getInt32(C: Line),
4109 Builder.getInt32(C: Column)};
4110
4111 return llvm::ConstantStruct::getAnon(V: Data);
4112}
4113
4114namespace {
4115/// Specify under what conditions this check can be recovered
4116enum class CheckRecoverableKind {
4117 /// Always terminate program execution if this check fails.
4118 Unrecoverable,
4119 /// Check supports recovering, runtime has both fatal (noreturn) and
4120 /// non-fatal handlers for this check.
4121 Recoverable,
4122 /// Runtime conditionally aborts, always need to support recovery.
4123 AlwaysRecoverable
4124};
4125}
4126
4127static CheckRecoverableKind
4128getRecoverableKind(SanitizerKind::SanitizerOrdinal Ordinal) {
4129 if (Ordinal == SanitizerKind::SO_Vptr)
4130 return CheckRecoverableKind::AlwaysRecoverable;
4131 else if (Ordinal == SanitizerKind::SO_Return ||
4132 Ordinal == SanitizerKind::SO_Unreachable)
4133 return CheckRecoverableKind::Unrecoverable;
4134 else
4135 return CheckRecoverableKind::Recoverable;
4136}
4137
4138namespace {
4139struct SanitizerHandlerInfo {
4140 char const *const Name;
4141 unsigned Version;
4142};
4143}
4144
4145const SanitizerHandlerInfo SanitizerHandlers[] = {
4146#define SANITIZER_CHECK(Enum, Name, Version, Msg) {#Name, Version},
4147 LIST_SANITIZER_CHECKS
4148#undef SANITIZER_CHECK
4149};
4150
4151static void emitCheckHandlerCall(CodeGenFunction &CGF,
4152 llvm::FunctionType *FnType,
4153 ArrayRef<llvm::Value *> FnArgs,
4154 SanitizerHandler CheckHandler,
4155 CheckRecoverableKind RecoverKind, bool IsFatal,
4156 llvm::BasicBlock *ContBB, bool NoMerge) {
4157 assert(IsFatal || RecoverKind != CheckRecoverableKind::Unrecoverable);
4158 std::optional<ApplyDebugLocation> DL;
4159 if (!CGF.Builder.getCurrentDebugLocation()) {
4160 // Ensure that the call has at least an artificial debug location.
4161 DL.emplace(args&: CGF, args: SourceLocation());
4162 }
4163 bool NeedsAbortSuffix =
4164 IsFatal && RecoverKind != CheckRecoverableKind::Unrecoverable;
4165 bool MinimalRuntime = CGF.CGM.getCodeGenOpts().SanitizeMinimalRuntime;
4166 bool HandlerPreserveAllRegs =
4167 CGF.CGM.getCodeGenOpts().SanitizeHandlerPreserveAllRegs;
4168 const SanitizerHandlerInfo &CheckInfo = SanitizerHandlers[CheckHandler];
4169 const StringRef CheckName = CheckInfo.Name;
4170 std::string FnName = "__ubsan_handle_" + CheckName.str();
4171 if (CheckInfo.Version && !MinimalRuntime)
4172 FnName += "_v" + llvm::utostr(X: CheckInfo.Version);
4173 if (MinimalRuntime)
4174 FnName += "_minimal";
4175 if (NeedsAbortSuffix)
4176 FnName += "_abort";
4177 if (HandlerPreserveAllRegs && !NeedsAbortSuffix)
4178 FnName += "_preserve";
4179 bool MayReturn =
4180 !IsFatal || RecoverKind == CheckRecoverableKind::AlwaysRecoverable;
4181
4182 llvm::AttrBuilder B(CGF.getLLVMContext());
4183 if (!MayReturn) {
4184 B.addAttribute(Val: llvm::Attribute::NoReturn)
4185 .addAttribute(Val: llvm::Attribute::NoUnwind);
4186 }
4187 B.addUWTableAttr(Kind: llvm::UWTableKind::Default);
4188
4189 llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(
4190 Ty: FnType, Name: FnName,
4191 ExtraAttrs: llvm::AttributeList::get(C&: CGF.getLLVMContext(),
4192 Index: llvm::AttributeList::FunctionIndex, B),
4193 /*Local=*/true);
4194 llvm::CallInst *HandlerCall = CGF.EmitNounwindRuntimeCall(callee: Fn, args: FnArgs);
4195 NoMerge = NoMerge || !CGF.CGM.getCodeGenOpts().isOptimizedBuild() ||
4196 (CGF.CurCodeDecl && CGF.CurCodeDecl->hasAttr<OptimizeNoneAttr>());
4197 if (NoMerge)
4198 HandlerCall->addFnAttr(Kind: llvm::Attribute::NoMerge);
4199 if (HandlerPreserveAllRegs && !NeedsAbortSuffix) {
4200 // N.B. there is also a clang::CallingConv which is not what we want here.
4201 HandlerCall->setCallingConv(llvm::CallingConv::PreserveAll);
4202 }
4203 if (!MayReturn) {
4204 HandlerCall->setDoesNotReturn();
4205 CGF.Builder.CreateUnreachable();
4206 } else {
4207 CGF.Builder.CreateBr(Dest: ContBB);
4208 }
4209}
4210
4211void CodeGenFunction::EmitCheck(
4212 ArrayRef<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>> Checked,
4213 SanitizerHandler CheckHandler, ArrayRef<llvm::Constant *> StaticArgs,
4214 ArrayRef<llvm::Value *> DynamicArgs, const TrapReason *TR) {
4215 assert(IsSanitizerScope);
4216 assert(Checked.size() > 0);
4217 assert(CheckHandler >= 0 &&
4218 size_t(CheckHandler) < std::size(SanitizerHandlers));
4219 const StringRef CheckName = SanitizerHandlers[CheckHandler].Name;
4220
4221 llvm::Value *FatalCond = nullptr;
4222 llvm::Value *RecoverableCond = nullptr;
4223 llvm::Value *TrapCond = nullptr;
4224 bool NoMerge = false;
4225 // Expand checks into:
4226 // (Check1 || !allow_ubsan_check) && (Check2 || !allow_ubsan_check) ...
4227 // We need separate allow_ubsan_check intrinsics because they have separately
4228 // specified cutoffs.
4229 // This expression looks expensive but will be simplified after
4230 // LowerAllowCheckPass.
4231 for (auto &[Check, Ord] : Checked) {
4232 llvm::Value *GuardedCheck = Check;
4233 if (ClSanitizeGuardChecks ||
4234 (CGM.getCodeGenOpts().SanitizeSkipHotCutoffs[Ord] > 0)) {
4235 llvm::Value *Allow = Builder.CreateCall(
4236 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::allow_ubsan_check),
4237 Args: llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: Ord));
4238 GuardedCheck = Builder.CreateOr(LHS: Check, RHS: Builder.CreateNot(V: Allow));
4239 }
4240
4241 // -fsanitize-trap= overrides -fsanitize-recover=.
4242 llvm::Value *&Cond = CGM.getCodeGenOpts().SanitizeTrap.has(O: Ord) ? TrapCond
4243 : CGM.getCodeGenOpts().SanitizeRecover.has(O: Ord)
4244 ? RecoverableCond
4245 : FatalCond;
4246 Cond = Cond ? Builder.CreateAnd(LHS: Cond, RHS: GuardedCheck) : GuardedCheck;
4247
4248 if (!CGM.getCodeGenOpts().SanitizeMergeHandlers.has(O: Ord))
4249 NoMerge = true;
4250 }
4251
4252 if (TrapCond)
4253 EmitTrapCheck(Checked: TrapCond, CheckHandlerID: CheckHandler, NoMerge, TR);
4254 if (!FatalCond && !RecoverableCond)
4255 return;
4256
4257 llvm::Value *JointCond;
4258 if (FatalCond && RecoverableCond)
4259 JointCond = Builder.CreateAnd(LHS: FatalCond, RHS: RecoverableCond);
4260 else
4261 JointCond = FatalCond ? FatalCond : RecoverableCond;
4262 assert(JointCond);
4263
4264 CheckRecoverableKind RecoverKind = getRecoverableKind(Ordinal: Checked[0].second);
4265 assert(SanOpts.has(Checked[0].second));
4266#ifndef NDEBUG
4267 for (int i = 1, n = Checked.size(); i < n; ++i) {
4268 assert(RecoverKind == getRecoverableKind(Checked[i].second) &&
4269 "All recoverable kinds in a single check must be same!");
4270 assert(SanOpts.has(Checked[i].second));
4271 }
4272#endif
4273
4274 llvm::BasicBlock *Cont = createBasicBlock(name: "cont");
4275 llvm::BasicBlock *Handlers = createBasicBlock(name: "handler." + CheckName);
4276 llvm::Instruction *Branch = Builder.CreateCondBr(Cond: JointCond, True: Cont, False: Handlers);
4277 // Give hint that we very much don't expect to execute the handler
4278 llvm::MDBuilder MDHelper(getLLVMContext());
4279 llvm::MDNode *Node = MDHelper.createLikelyBranchWeights();
4280 Branch->setMetadata(KindID: llvm::LLVMContext::MD_prof, Node);
4281 EmitBlock(BB: Handlers);
4282
4283 // Clear arguments for the MinimalRuntime handler.
4284 if (CGM.getCodeGenOpts().SanitizeMinimalRuntime) {
4285 StaticArgs = {};
4286 DynamicArgs = {};
4287 }
4288
4289 // Handler functions take an i8* pointing to the (handler-specific) static
4290 // information block, followed by a sequence of intptr_t arguments
4291 // representing operand values.
4292 SmallVector<llvm::Value *, 4> Args;
4293 SmallVector<llvm::Type *, 4> ArgTypes;
4294
4295 Args.reserve(N: DynamicArgs.size() + 1);
4296 ArgTypes.reserve(N: DynamicArgs.size() + 1);
4297
4298 // Emit handler arguments and create handler function type.
4299 if (!StaticArgs.empty()) {
4300 llvm::Constant *Info = llvm::ConstantStruct::getAnon(V: StaticArgs);
4301 auto *InfoPtr = new llvm::GlobalVariable(
4302 CGM.getModule(), Info->getType(),
4303 // Non-constant global is used in a handler to deduplicate reports.
4304 // TODO: change deduplication logic and make it constant.
4305 /*isConstant=*/false, llvm::GlobalVariable::PrivateLinkage, Info, "",
4306 nullptr, llvm::GlobalVariable::NotThreadLocal,
4307 CGM.getDataLayout().getDefaultGlobalsAddressSpace());
4308 InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4309 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV: InfoPtr);
4310 Args.push_back(Elt: InfoPtr);
4311 ArgTypes.push_back(Elt: Args.back()->getType());
4312 }
4313
4314 for (llvm::Value *DynamicArg : DynamicArgs) {
4315 Args.push_back(Elt: EmitCheckValue(V: DynamicArg));
4316 ArgTypes.push_back(Elt: IntPtrTy);
4317 }
4318
4319 llvm::FunctionType *FnType =
4320 llvm::FunctionType::get(Result: CGM.VoidTy, Params: ArgTypes, isVarArg: false);
4321
4322 if (!FatalCond || !RecoverableCond) {
4323 // Simple case: we need to generate a single handler call, either
4324 // fatal, or non-fatal.
4325 emitCheckHandlerCall(CGF&: *this, FnType, FnArgs: Args, CheckHandler, RecoverKind,
4326 IsFatal: (FatalCond != nullptr), ContBB: Cont, NoMerge);
4327 } else {
4328 // Emit two handler calls: first one for set of unrecoverable checks,
4329 // another one for recoverable.
4330 llvm::BasicBlock *NonFatalHandlerBB =
4331 createBasicBlock(name: "non_fatal." + CheckName);
4332 llvm::BasicBlock *FatalHandlerBB = createBasicBlock(name: "fatal." + CheckName);
4333 Builder.CreateCondBr(Cond: FatalCond, True: NonFatalHandlerBB, False: FatalHandlerBB);
4334 EmitBlock(BB: FatalHandlerBB);
4335 emitCheckHandlerCall(CGF&: *this, FnType, FnArgs: Args, CheckHandler, RecoverKind, IsFatal: true,
4336 ContBB: NonFatalHandlerBB, NoMerge);
4337 EmitBlock(BB: NonFatalHandlerBB);
4338 emitCheckHandlerCall(CGF&: *this, FnType, FnArgs: Args, CheckHandler, RecoverKind, IsFatal: false,
4339 ContBB: Cont, NoMerge);
4340 }
4341
4342 EmitBlock(BB: Cont);
4343}
4344
4345void CodeGenFunction::EmitCfiSlowPathCheck(
4346 SanitizerKind::SanitizerOrdinal Ordinal, llvm::Value *Cond,
4347 llvm::ConstantInt *TypeId, llvm::Value *Ptr,
4348 ArrayRef<llvm::Constant *> StaticArgs) {
4349 llvm::BasicBlock *Cont = createBasicBlock(name: "cfi.cont");
4350
4351 llvm::BasicBlock *CheckBB = createBasicBlock(name: "cfi.slowpath");
4352 llvm::CondBrInst *BI = Builder.CreateCondBr(Cond, True: Cont, False: CheckBB);
4353
4354 llvm::MDBuilder MDHelper(getLLVMContext());
4355 llvm::MDNode *Node = MDHelper.createLikelyBranchWeights();
4356 BI->setMetadata(KindID: llvm::LLVMContext::MD_prof, Node);
4357
4358 EmitBlock(BB: CheckBB);
4359
4360 bool WithDiag = !CGM.getCodeGenOpts().SanitizeTrap.has(O: Ordinal);
4361
4362 llvm::CallInst *CheckCall;
4363 llvm::FunctionCallee SlowPathFn;
4364 if (WithDiag) {
4365 llvm::Constant *Info = llvm::ConstantStruct::getAnon(V: StaticArgs);
4366 auto *InfoPtr =
4367 new llvm::GlobalVariable(CGM.getModule(), Info->getType(), false,
4368 llvm::GlobalVariable::PrivateLinkage, Info);
4369 InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4370 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV: InfoPtr);
4371
4372 SlowPathFn = CGM.getModule().getOrInsertFunction(
4373 Name: "__cfi_slowpath_diag",
4374 T: llvm::FunctionType::get(Result: VoidTy, Params: {Int64Ty, Int8PtrTy, Int8PtrTy},
4375 isVarArg: false));
4376 CheckCall = Builder.CreateCall(Callee: SlowPathFn, Args: {TypeId, Ptr, InfoPtr});
4377 } else {
4378 SlowPathFn = CGM.getModule().getOrInsertFunction(
4379 Name: "__cfi_slowpath",
4380 T: llvm::FunctionType::get(Result: VoidTy, Params: {Int64Ty, Int8PtrTy}, isVarArg: false));
4381 CheckCall = Builder.CreateCall(Callee: SlowPathFn, Args: {TypeId, Ptr});
4382 }
4383
4384 CGM.setDSOLocal(
4385 cast<llvm::GlobalValue>(Val: SlowPathFn.getCallee()->stripPointerCasts()));
4386 CheckCall->setDoesNotThrow();
4387
4388 EmitBlock(BB: Cont);
4389}
4390
4391// Emit a stub for __cfi_check function so that the linker knows about this
4392// symbol in LTO mode.
4393void CodeGenFunction::EmitCfiCheckStub() {
4394 llvm::Module *M = &CGM.getModule();
4395 ASTContext &C = getContext();
4396 QualType QInt64Ty = C.getIntTypeForBitwidth(DestWidth: 64, Signed: false);
4397
4398 auto *ArgCallsiteTypeId =
4399 ImplicitParamDecl::Create(C, T: QInt64Ty, ParamKind: ImplicitParamKind::Other);
4400 auto *ArgAddr =
4401 ImplicitParamDecl::Create(C, T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
4402 auto *ArgCFICheckFailData =
4403 ImplicitParamDecl::Create(C, T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
4404 FunctionArgList FnArgs{ArgCallsiteTypeId, ArgAddr, ArgCFICheckFailData};
4405 const CGFunctionInfo &FI =
4406 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: FnArgs);
4407
4408 llvm::Function *F = llvm::Function::Create(
4409 Ty: llvm::FunctionType::get(Result: VoidTy, Params: {Int64Ty, VoidPtrTy, VoidPtrTy}, isVarArg: false),
4410 Linkage: llvm::GlobalValue::WeakAnyLinkage, N: "__cfi_check", M);
4411 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F, /*IsThunk=*/false);
4412 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F);
4413 F->setAlignment(llvm::Align(4096));
4414 CGM.setDSOLocal(F);
4415
4416 llvm::LLVMContext &Ctx = M->getContext();
4417 llvm::BasicBlock *BB = llvm::BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: F);
4418 // CrossDSOCFI pass is not executed if there is no executable code.
4419 SmallVector<llvm::Value*> Args{F->getArg(i: 2), F->getArg(i: 1)};
4420 llvm::CallInst::Create(Func: M->getFunction(Name: "__cfi_check_fail"), Args, NameStr: "", InsertBefore: BB);
4421 llvm::ReturnInst::Create(C&: Ctx, retVal: nullptr, InsertBefore: BB);
4422}
4423
4424// This function is basically a switch over the CFI failure kind, which is
4425// extracted from CFICheckFailData (1st function argument). Each case is either
4426// llvm.trap or a call to one of the two runtime handlers, based on
4427// -fsanitize-trap and -fsanitize-recover settings. Default case (invalid
4428// failure kind) traps, but this should really never happen. CFICheckFailData
4429// can be nullptr if the calling module has -fsanitize-trap behavior for this
4430// check kind; in this case __cfi_check_fail traps as well.
4431void CodeGenFunction::EmitCfiCheckFail() {
4432 auto CheckHandler = SanitizerHandler::CFICheckFail;
4433 // TODO: the SanitizerKind is not yet determined for this check (and might
4434 // not even be available, if Data == nullptr). However, we still want to
4435 // annotate the instrumentation. We approximate this by using all the CFI
4436 // kinds.
4437 SanitizerDebugLocation SanScope(
4438 this,
4439 {SanitizerKind::SO_CFIVCall, SanitizerKind::SO_CFINVCall,
4440 SanitizerKind::SO_CFIDerivedCast, SanitizerKind::SO_CFIUnrelatedCast,
4441 SanitizerKind::SO_CFIICall},
4442 CheckHandler);
4443 auto *ArgData = ImplicitParamDecl::Create(
4444 C&: getContext(), T: getContext().VoidPtrTy, ParamKind: ImplicitParamKind::Other);
4445 auto *ArgAddr = ImplicitParamDecl::Create(
4446 C&: getContext(), T: getContext().VoidPtrTy, ParamKind: ImplicitParamKind::Other);
4447
4448 FunctionArgList Args{ArgData, ArgAddr};
4449 const CGFunctionInfo &FI =
4450 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: getContext().VoidTy, args: Args);
4451
4452 llvm::Function *F = llvm::Function::Create(
4453 Ty: llvm::FunctionType::get(Result: VoidTy, Params: {VoidPtrTy, VoidPtrTy}, isVarArg: false),
4454 Linkage: llvm::GlobalValue::WeakODRLinkage, N: "__cfi_check_fail", M: &CGM.getModule());
4455
4456 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F, /*IsThunk=*/false);
4457 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F);
4458 F->setVisibility(llvm::GlobalValue::HiddenVisibility);
4459
4460 StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidTy, Fn: F, FnInfo: FI, Args,
4461 Loc: SourceLocation());
4462
4463 ApplyDebugLocation ADL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
4464
4465 // This function is not affected by NoSanitizeList. This function does
4466 // not have a source location, but "src:*" would still apply. Revert any
4467 // changes to SanOpts made in StartFunction.
4468 SanOpts = CGM.getLangOpts().Sanitize;
4469
4470 llvm::Value *Data =
4471 EmitLoadOfScalar(Addr: GetAddrOfLocalVar(VD: ArgData), /*Volatile=*/false,
4472 Ty: CGM.getContext().VoidPtrTy, Loc: ArgData->getLocation());
4473 llvm::Value *Addr =
4474 EmitLoadOfScalar(Addr: GetAddrOfLocalVar(VD: ArgAddr), /*Volatile=*/false,
4475 Ty: CGM.getContext().VoidPtrTy, Loc: ArgAddr->getLocation());
4476
4477 // Data == nullptr means the calling module has trap behaviour for this check.
4478 llvm::Value *DataIsNotNullPtr =
4479 Builder.CreateICmpNE(LHS: Data, RHS: llvm::ConstantPointerNull::get(T: Int8PtrTy));
4480 // TODO: since there is no data, we don't know the CheckKind, and therefore
4481 // cannot inspect CGM.getCodeGenOpts().SanitizeMergeHandlers. We default to
4482 // NoMerge = false. Users can disable merging by disabling optimization.
4483 EmitTrapCheck(Checked: DataIsNotNullPtr, CheckHandlerID: SanitizerHandler::CFICheckFail,
4484 /*NoMerge=*/false);
4485
4486 llvm::StructType *SourceLocationTy =
4487 llvm::StructType::get(elt1: VoidPtrTy, elts: Int32Ty, elts: Int32Ty);
4488 llvm::StructType *CfiCheckFailDataTy =
4489 llvm::StructType::get(elt1: Int8Ty, elts: SourceLocationTy, elts: VoidPtrTy);
4490
4491 llvm::Value *V = Builder.CreateConstGEP2_32(
4492 Ty: CfiCheckFailDataTy, Ptr: Builder.CreatePointerCast(V: Data, DestTy: DefaultPtrTy), Idx0: 0, Idx1: 0);
4493
4494 Address CheckKindAddr(V, Int8Ty, getIntAlign());
4495 llvm::Value *CheckKind = Builder.CreateLoad(Addr: CheckKindAddr);
4496
4497 llvm::Value *AllVtables = llvm::MetadataAsValue::get(
4498 Context&: CGM.getLLVMContext(),
4499 MD: llvm::MDString::get(Context&: CGM.getLLVMContext(), Str: "all-vtables"));
4500 llvm::Value *ValidVtable = Builder.CreateZExt(
4501 V: Builder.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::type_test),
4502 Args: {Addr, AllVtables}),
4503 DestTy: IntPtrTy);
4504
4505 const std::pair<int, SanitizerKind::SanitizerOrdinal> CheckKinds[] = {
4506 {CFITCK_VCall, SanitizerKind::SO_CFIVCall},
4507 {CFITCK_NVCall, SanitizerKind::SO_CFINVCall},
4508 {CFITCK_DerivedCast, SanitizerKind::SO_CFIDerivedCast},
4509 {CFITCK_UnrelatedCast, SanitizerKind::SO_CFIUnrelatedCast},
4510 {CFITCK_ICall, SanitizerKind::SO_CFIICall}};
4511
4512 for (auto CheckKindOrdinalPair : CheckKinds) {
4513 int Kind = CheckKindOrdinalPair.first;
4514 SanitizerKind::SanitizerOrdinal Ordinal = CheckKindOrdinalPair.second;
4515
4516 // TODO: we could apply SanitizerAnnotateDebugInfo(Ordinal) instead of
4517 // relying on the SanitizerScope with all CFI ordinals
4518
4519 llvm::Value *Cond =
4520 Builder.CreateICmpNE(LHS: CheckKind, RHS: llvm::ConstantInt::get(Ty: Int8Ty, V: Kind));
4521 if (CGM.getLangOpts().Sanitize.has(O: Ordinal))
4522 EmitCheck(Checked: std::make_pair(x&: Cond, y&: Ordinal), CheckHandler: SanitizerHandler::CFICheckFail,
4523 StaticArgs: {}, DynamicArgs: {Data, Addr, ValidVtable});
4524 else
4525 // TODO: we can't rely on CGM.getCodeGenOpts().SanitizeMergeHandlers.
4526 // Although the compiler allows SanitizeMergeHandlers to be set
4527 // independently of CGM.getLangOpts().Sanitize, Driver/SanitizerArgs.cpp
4528 // requires that SanitizeMergeHandlers is a subset of Sanitize.
4529 EmitTrapCheck(Checked: Cond, CheckHandlerID: CheckHandler, /*NoMerge=*/false);
4530 }
4531
4532 FinishFunction();
4533 // The only reference to this function will be created during LTO link.
4534 // Make sure it survives until then.
4535 CGM.addUsedGlobal(GV: F);
4536}
4537
4538void CodeGenFunction::EmitUnreachable(SourceLocation Loc) {
4539 if (SanOpts.has(K: SanitizerKind::Unreachable)) {
4540 auto CheckOrdinal = SanitizerKind::SO_Unreachable;
4541 auto CheckHandler = SanitizerHandler::BuiltinUnreachable;
4542 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
4543 EmitCheck(Checked: std::make_pair(x: static_cast<llvm::Value *>(Builder.getFalse()),
4544 y&: CheckOrdinal),
4545 CheckHandler, StaticArgs: EmitCheckSourceLocation(Loc), DynamicArgs: {});
4546 }
4547 Builder.CreateUnreachable();
4548}
4549
4550void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked,
4551 SanitizerHandler CheckHandlerID,
4552 bool NoMerge, const TrapReason *TR) {
4553 llvm::BasicBlock *Cont = createBasicBlock(name: "cont");
4554
4555 // If we're optimizing, collapse all calls to trap down to just one per
4556 // check-type per function to save on code size.
4557 if ((int)TrapBBs.size() <= CheckHandlerID)
4558 TrapBBs.resize(N: CheckHandlerID + 1);
4559
4560 llvm::BasicBlock *&TrapBB = TrapBBs[CheckHandlerID];
4561
4562 llvm::DILocation *TrapLocation = Builder.getCurrentDebugLocation();
4563 llvm::StringRef TrapMessage;
4564 llvm::StringRef TrapCategory;
4565 auto DebugTrapReasonKind = CGM.getCodeGenOpts().getSanitizeDebugTrapReasons();
4566 if (TR && !TR->isEmpty() &&
4567 DebugTrapReasonKind ==
4568 CodeGenOptions::SanitizeDebugTrapReasonKind::Detailed) {
4569 TrapMessage = TR->getMessage();
4570 TrapCategory = TR->getCategory();
4571 } else {
4572 TrapMessage = GetUBSanTrapForHandler(ID: CheckHandlerID);
4573 TrapCategory = "Undefined Behavior Sanitizer";
4574 }
4575
4576 if (getDebugInfo() && !TrapMessage.empty() &&
4577 DebugTrapReasonKind !=
4578 CodeGenOptions::SanitizeDebugTrapReasonKind::None &&
4579 TrapLocation) {
4580 TrapLocation = getDebugInfo()->CreateTrapFailureMessageFor(
4581 TrapLocation, Category: TrapCategory, FailureMsg: TrapMessage);
4582 }
4583
4584 NoMerge = NoMerge || !CGM.getCodeGenOpts().isOptimizedBuild() ||
4585 (CurCodeDecl && CurCodeDecl->hasAttr<OptimizeNoneAttr>());
4586
4587 llvm::MDBuilder MDHelper(getLLVMContext());
4588 if (TrapBB && !NoMerge) {
4589 auto Call = TrapBB->begin();
4590 assert(isa<llvm::CallInst>(Call) && "Expected call in trap BB");
4591
4592 Call->applyMergedLocation(LocA: Call->getDebugLoc(), LocB: TrapLocation);
4593
4594 Builder.CreateCondBr(Cond: Checked, True: Cont, False: TrapBB,
4595 BranchWeights: MDHelper.createLikelyBranchWeights());
4596 } else {
4597 TrapBB = createBasicBlock(name: "trap");
4598 Builder.CreateCondBr(Cond: Checked, True: Cont, False: TrapBB,
4599 BranchWeights: MDHelper.createLikelyBranchWeights());
4600 EmitBlock(BB: TrapBB);
4601
4602 ApplyDebugLocation applyTrapDI(*this, TrapLocation);
4603
4604 llvm::CallInst *TrapCall;
4605 if (CGM.getCodeGenOpts().SanitizeTrapLoop)
4606 TrapCall =
4607 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::looptrap));
4608 else
4609 TrapCall = Builder.CreateCall(
4610 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::ubsantrap),
4611 Args: llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: CheckHandlerID));
4612
4613 if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
4614 auto A = llvm::Attribute::get(Context&: getLLVMContext(), Kind: "trap-func-name",
4615 Val: CGM.getCodeGenOpts().TrapFuncName);
4616 TrapCall->addFnAttr(Attr: A);
4617 }
4618 if (NoMerge)
4619 TrapCall->addFnAttr(Kind: llvm::Attribute::NoMerge);
4620 TrapCall->setDoesNotReturn();
4621 TrapCall->setDoesNotThrow();
4622 Builder.CreateUnreachable();
4623 }
4624
4625 EmitBlock(BB: Cont);
4626}
4627
4628llvm::CallInst *CodeGenFunction::EmitTrapCall(llvm::Intrinsic::ID IntrID) {
4629 llvm::Function *TrapIntrinsic = CGM.getIntrinsic(IID: IntrID);
4630 llvm::CallInst *TrapCall = Builder.CreateCall(Callee: TrapIntrinsic);
4631
4632 if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
4633 auto A = llvm::Attribute::get(Context&: getLLVMContext(), Kind: "trap-func-name",
4634 Val: CGM.getCodeGenOpts().TrapFuncName);
4635 TrapCall->addFnAttr(Attr: A);
4636 }
4637
4638 if (InNoMergeAttributedStmt)
4639 TrapCall->addFnAttr(Kind: llvm::Attribute::NoMerge);
4640 if (TrapIntrinsic->doesNotThrow())
4641 TrapCall->setDoesNotThrow();
4642 if (TrapIntrinsic->doesNotReturn()) {
4643 TrapCall->setDoesNotReturn();
4644 Builder.CreateUnreachable();
4645 EmitBlock(BB: createBasicBlock());
4646 }
4647 return TrapCall;
4648}
4649
4650Address CodeGenFunction::EmitArrayToPointerDecay(const Expr *E,
4651 LValueBaseInfo *BaseInfo,
4652 TBAAAccessInfo *TBAAInfo) {
4653 assert(E->getType()->isArrayType() &&
4654 "Array to pointer decay must have array source type!");
4655
4656 // Expressions of array type can't be bitfields or vector elements.
4657 LValue LV = EmitLValue(E);
4658 Address Addr = LV.getAddress();
4659
4660 // If the array type was an incomplete type, we need to make sure
4661 // the decay ends up being the right type.
4662 llvm::Type *NewTy = ConvertType(T: E->getType());
4663 Addr = Addr.withElementType(ElemTy: NewTy);
4664
4665 // Note that VLA pointers are always decayed, so we don't need to do
4666 // anything here.
4667 if (!E->getType()->isVariableArrayType()) {
4668 assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
4669 "Expected pointer to array");
4670
4671 if (getLangOpts().EmitLogicalPointer) {
4672 // Array-to-pointer decay for an SGEP is a no-op as we don't do any
4673 // logical indexing. See #179951 for some additional context.
4674 auto *SGEP =
4675 Builder.CreateStructuredGEP(BaseType: NewTy, PtrBase: Addr.emitRawPointer(CGF&: *this), Indices: {});
4676 Addr = Address(SGEP, NewTy, Addr.getAlignment(), Addr.isKnownNonNull());
4677 } else {
4678 Addr = Builder.CreateConstArrayGEP(Addr, Index: 0, Name: "arraydecay");
4679 }
4680 }
4681
4682 // The result of this decay conversion points to an array element within the
4683 // base lvalue. However, since TBAA currently does not support representing
4684 // accesses to elements of member arrays, we conservatively represent accesses
4685 // to the pointee object as if it had no any base lvalue specified.
4686 // TODO: Support TBAA for member arrays.
4687 QualType EltType = E->getType()->castAsArrayTypeUnsafe()->getElementType();
4688 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
4689 if (TBAAInfo) *TBAAInfo = CGM.getTBAAAccessInfo(AccessType: EltType);
4690
4691 return Addr.withElementType(ElemTy: ConvertTypeForMem(T: EltType));
4692}
4693
4694/// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an
4695/// array to pointer, return the array subexpression.
4696static const Expr *isSimpleArrayDecayOperand(const Expr *E) {
4697 // If this isn't just an array->pointer decay, bail out.
4698 const auto *CE = dyn_cast<CastExpr>(Val: E);
4699 if (!CE || CE->getCastKind() != CK_ArrayToPointerDecay)
4700 return nullptr;
4701
4702 // If this is a decay from variable width array, bail out.
4703 const Expr *SubExpr = CE->getSubExpr();
4704 if (SubExpr->getType()->isVariableArrayType())
4705 return nullptr;
4706
4707 return SubExpr;
4708}
4709
4710static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF,
4711 llvm::Type *elemType,
4712 llvm::Value *ptr,
4713 ArrayRef<llvm::Value*> indices,
4714 bool inbounds,
4715 bool signedIndices,
4716 SourceLocation loc,
4717 const llvm::Twine &name = "arrayidx") {
4718 if (inbounds && CGF.getLangOpts().EmitLogicalPointer)
4719 return CGF.Builder.CreateStructuredGEP(BaseType: elemType, PtrBase: ptr, Indices: indices);
4720
4721 if (inbounds) {
4722 return CGF.EmitCheckedInBoundsGEP(ElemTy: elemType, Ptr: ptr, IdxList: indices, SignedIndices: signedIndices,
4723 IsSubtraction: CodeGenFunction::NotSubtraction, Loc: loc,
4724 Name: name);
4725 } else {
4726 return CGF.Builder.CreateGEP(Ty: elemType, Ptr: ptr, IdxList: indices, Name: name);
4727 }
4728}
4729
4730static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
4731 ArrayRef<llvm::Value *> indices,
4732 llvm::Type *arrayType,
4733 llvm::Type *elementType, bool inbounds,
4734 bool signedIndices, SourceLocation loc,
4735 CharUnits align,
4736 const llvm::Twine &name = "arrayidx") {
4737 if (inbounds && CGF.getLangOpts().EmitLogicalPointer)
4738 return RawAddress(CGF.Builder.CreateStructuredGEP(BaseType: arrayType,
4739 PtrBase: addr.emitRawPointer(CGF),
4740 Indices: indices.drop_front()),
4741 elementType, align);
4742
4743 if (inbounds) {
4744 return CGF.EmitCheckedInBoundsGEP(Addr: addr, IdxList: indices, elementType, SignedIndices: signedIndices,
4745 IsSubtraction: CodeGenFunction::NotSubtraction, Loc: loc,
4746 Align: align, Name: name);
4747 } else {
4748 return CGF.Builder.CreateGEP(Addr: addr, IdxList: indices, ElementType: elementType, Align: align, Name: name);
4749 }
4750}
4751
4752static QualType getFixedSizeElementType(const ASTContext &ctx,
4753 const VariableArrayType *vla) {
4754 QualType eltType;
4755 do {
4756 eltType = vla->getElementType();
4757 } while ((vla = ctx.getAsVariableArrayType(T: eltType)));
4758 return eltType;
4759}
4760
4761static bool hasBPFPreserveStaticOffset(const RecordDecl *D) {
4762 return D && D->hasAttr<BPFPreserveStaticOffsetAttr>();
4763}
4764
4765static bool hasBPFPreserveStaticOffset(const Expr *E) {
4766 if (!E)
4767 return false;
4768 QualType PointeeType = E->getType()->getPointeeType();
4769 if (PointeeType.isNull())
4770 return false;
4771 if (const auto *BaseDecl = PointeeType->getAsRecordDecl())
4772 return hasBPFPreserveStaticOffset(D: BaseDecl);
4773 return false;
4774}
4775
4776// Wraps Addr with a call to llvm.preserve.static.offset intrinsic.
4777static Address wrapWithBPFPreserveStaticOffset(CodeGenFunction &CGF,
4778 Address &Addr) {
4779 if (!CGF.getTarget().getTriple().isBPF())
4780 return Addr;
4781
4782 llvm::Function *Fn =
4783 CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::preserve_static_offset);
4784 llvm::CallInst *Call = CGF.Builder.CreateCall(Callee: Fn, Args: {Addr.emitRawPointer(CGF)});
4785 return Address(Call, Addr.getElementType(), Addr.getAlignment());
4786}
4787
4788/// Given an array base, check whether its member access belongs to a record
4789/// with preserve_access_index attribute or not.
4790static bool IsPreserveAIArrayBase(CodeGenFunction &CGF, const Expr *ArrayBase) {
4791 if (!ArrayBase || !CGF.getDebugInfo())
4792 return false;
4793
4794 // Only support base as either a MemberExpr or DeclRefExpr.
4795 // DeclRefExpr to cover cases like:
4796 // struct s { int a; int b[10]; };
4797 // struct s *p;
4798 // p[1].a
4799 // p[1] will generate a DeclRefExpr and p[1].a is a MemberExpr.
4800 // p->b[5] is a MemberExpr example.
4801 const Expr *E = ArrayBase->IgnoreImpCasts();
4802 if (const auto *ME = dyn_cast<MemberExpr>(Val: E))
4803 return ME->getMemberDecl()->hasAttr<BPFPreserveAccessIndexAttr>();
4804
4805 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
4806 const auto *VarDef = dyn_cast<VarDecl>(Val: DRE->getDecl());
4807 if (!VarDef)
4808 return false;
4809
4810 const auto *PtrT = VarDef->getType()->getAs<PointerType>();
4811 if (!PtrT)
4812 return false;
4813
4814 const auto *PointeeT = PtrT->getPointeeType()
4815 ->getUnqualifiedDesugaredType();
4816 if (const auto *RecT = dyn_cast<RecordType>(Val: PointeeT))
4817 return RecT->getDecl()
4818 ->getMostRecentDecl()
4819 ->hasAttr<BPFPreserveAccessIndexAttr>();
4820 return false;
4821 }
4822
4823 return false;
4824}
4825
4826static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
4827 ArrayRef<llvm::Value *> indices,
4828 QualType eltType, bool inbounds,
4829 bool signedIndices, SourceLocation loc,
4830 QualType *arrayType = nullptr,
4831 const Expr *Base = nullptr,
4832 const llvm::Twine &name = "arrayidx") {
4833 // All the indices except that last must be zero.
4834#ifndef NDEBUG
4835 for (auto *idx : indices.drop_back())
4836 assert(isa<llvm::ConstantInt>(idx) &&
4837 cast<llvm::ConstantInt>(idx)->isZero());
4838#endif
4839
4840 // Determine the element size of the statically-sized base. This is
4841 // the thing that the indices are expressed in terms of.
4842 if (auto vla = CGF.getContext().getAsVariableArrayType(T: eltType)) {
4843 eltType = getFixedSizeElementType(ctx: CGF.getContext(), vla);
4844 }
4845
4846 // We can use that to compute the best alignment of the element.
4847 CharUnits eltSize = CGF.getContext().getTypeSizeInChars(T: eltType);
4848 CharUnits eltAlign =
4849 getArrayElementAlign(arrayAlign: addr.getAlignment(), idx: indices.back(), eltSize);
4850
4851 if (hasBPFPreserveStaticOffset(E: Base))
4852 addr = wrapWithBPFPreserveStaticOffset(CGF, Addr&: addr);
4853
4854 llvm::Value *eltPtr;
4855 auto LastIndex = dyn_cast<llvm::ConstantInt>(Val: indices.back());
4856 if (!LastIndex ||
4857 (!CGF.IsInPreservedAIRegion && !IsPreserveAIArrayBase(CGF, ArrayBase: Base))) {
4858 addr = emitArraySubscriptGEP(CGF, addr, indices,
4859 arrayType: arrayType ? CGF.ConvertTypeForMem(T: *arrayType)
4860 : nullptr,
4861 elementType: CGF.ConvertTypeForMem(T: eltType), inbounds,
4862 signedIndices, loc, align: eltAlign, name);
4863 return addr;
4864 } else {
4865 // Remember the original array subscript for bpf target
4866 unsigned idx = LastIndex->getZExtValue();
4867 llvm::DIType *DbgInfo = nullptr;
4868 if (arrayType)
4869 DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(Ty: *arrayType, Loc: loc);
4870 eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(
4871 ElTy: addr.getElementType(), Base: addr.emitRawPointer(CGF), Dimension: indices.size() - 1,
4872 LastIndex: idx, DbgInfo);
4873 }
4874
4875 return Address(eltPtr, CGF.ConvertTypeForMem(T: eltType), eltAlign);
4876}
4877
4878namespace {
4879
4880/// StructFieldAccess is a simple visitor class to grab the first l-value to
4881/// r-value cast Expr.
4882struct StructFieldAccess
4883 : public ConstStmtVisitor<StructFieldAccess, const Expr *> {
4884 const Expr *VisitCastExpr(const CastExpr *E) {
4885 if (E->getCastKind() == CK_LValueToRValue)
4886 return E;
4887 return Visit(S: E->getSubExpr());
4888 }
4889 const Expr *VisitParenExpr(const ParenExpr *E) {
4890 return Visit(S: E->getSubExpr());
4891 }
4892};
4893
4894} // end anonymous namespace
4895
4896/// The offset of a field from the beginning of the record.
4897static bool getFieldOffsetInBits(CodeGenFunction &CGF, const RecordDecl *RD,
4898 const FieldDecl *Field, int64_t &Offset) {
4899 ASTContext &Ctx = CGF.getContext();
4900 const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(D: RD);
4901 unsigned FieldNo = 0;
4902
4903 for (const FieldDecl *FD : RD->fields()) {
4904 if (FD == Field) {
4905 Offset += Layout.getFieldOffset(FieldNo);
4906 return true;
4907 }
4908
4909 QualType Ty = FD->getType();
4910 if (Ty->isRecordType())
4911 if (getFieldOffsetInBits(CGF, RD: Ty->getAsRecordDecl(), Field, Offset)) {
4912 Offset += Layout.getFieldOffset(FieldNo);
4913 return true;
4914 }
4915
4916 if (!RD->isUnion())
4917 ++FieldNo;
4918 }
4919
4920 return false;
4921}
4922
4923/// Returns the relative offset difference between \p FD1 and \p FD2.
4924/// \code
4925/// offsetof(struct foo, FD1) - offsetof(struct foo, FD2)
4926/// \endcode
4927/// Both fields must be within the same struct.
4928static std::optional<int64_t> getOffsetDifferenceInBits(CodeGenFunction &CGF,
4929 const FieldDecl *FD1,
4930 const FieldDecl *FD2) {
4931 const RecordDecl *FD1OuterRec =
4932 FD1->getParent()->getOuterLexicalRecordContext();
4933 const RecordDecl *FD2OuterRec =
4934 FD2->getParent()->getOuterLexicalRecordContext();
4935
4936 if (FD1OuterRec != FD2OuterRec)
4937 // Fields must be within the same RecordDecl.
4938 return std::optional<int64_t>();
4939
4940 int64_t FD1Offset = 0;
4941 if (!getFieldOffsetInBits(CGF, RD: FD1OuterRec, Field: FD1, Offset&: FD1Offset))
4942 return std::optional<int64_t>();
4943
4944 int64_t FD2Offset = 0;
4945 if (!getFieldOffsetInBits(CGF, RD: FD2OuterRec, Field: FD2, Offset&: FD2Offset))
4946 return std::optional<int64_t>();
4947
4948 return std::make_optional<int64_t>(t: FD1Offset - FD2Offset);
4949}
4950
4951/// EmitCountedByBoundsChecking - If the array being accessed has a "counted_by"
4952/// attribute, generate bounds checking code. The "count" field is at the top
4953/// level of the struct or in an anonymous struct, that's also at the top level.
4954/// Future expansions may allow the "count" to reside at any place in the
4955/// struct, but the value of "counted_by" will be a "simple" path to the count,
4956/// i.e. "a.b.count", so we shouldn't need the full force of EmitLValue or
4957/// similar to emit the correct GEP.
4958void CodeGenFunction::EmitCountedByBoundsChecking(
4959 const Expr *ArrayExpr, QualType ArrayType, Address ArrayInst,
4960 QualType IndexType, llvm::Value *IndexVal, bool Accessed,
4961 bool FlexibleArray) {
4962 const auto *ME = dyn_cast<MemberExpr>(Val: ArrayExpr->IgnoreImpCasts());
4963 if (!ME || !ME->getMemberDecl()->getType()->isCountAttributedType())
4964 return;
4965
4966 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
4967 getLangOpts().getStrictFlexArraysLevel();
4968 if (FlexibleArray &&
4969 !ME->isFlexibleArrayMemberLike(Context: getContext(), StrictFlexArraysLevel))
4970 return;
4971
4972 const FieldDecl *FD = cast<FieldDecl>(Val: ME->getMemberDecl());
4973 const FieldDecl *CountFD = FD->findCountedByField();
4974 if (!CountFD)
4975 return;
4976
4977 if (std::optional<int64_t> Diff =
4978 getOffsetDifferenceInBits(CGF&: *this, FD1: CountFD, FD2: FD)) {
4979 if (!ArrayInst.isValid()) {
4980 // An invalid Address indicates we're checking a pointer array access.
4981 // Emit the checked L-Value here.
4982 LValue LV = EmitCheckedLValue(E: ArrayExpr, TCK: TCK_MemberAccess);
4983 ArrayInst = LV.getAddress();
4984 }
4985
4986 // FIXME: The 'static_cast' is necessary, otherwise the result turns into a
4987 // uint64_t, which messes things up if we have a negative offset difference.
4988 Diff = *Diff / static_cast<int64_t>(CGM.getContext().getCharWidth());
4989
4990 // Create a GEP with the byte offset between the counted object and the
4991 // count and use that to load the count value.
4992 ArrayInst = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr: ArrayInst,
4993 Ty: Int8PtrTy, ElementTy: Int8Ty);
4994
4995 llvm::Type *BoundsType = ConvertType(T: CountFD->getType());
4996 llvm::Value *BoundsVal =
4997 Builder.CreateInBoundsGEP(Ty: Int8Ty, Ptr: ArrayInst.emitRawPointer(CGF&: *this),
4998 IdxList: Builder.getInt32(C: *Diff), Name: ".counted_by.gep");
4999 BoundsVal = Builder.CreateAlignedLoad(Ty: BoundsType, Addr: BoundsVal, Align: getIntAlign(),
5000 Name: ".counted_by.load");
5001
5002 // Now emit the bounds checking.
5003 EmitBoundsCheckImpl(ArrayExpr, ArrayBaseType: ArrayType, IndexVal, IndexType, BoundsVal,
5004 BoundsType: CountFD->getType(), Accessed);
5005 }
5006}
5007
5008LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
5009 bool Accessed) {
5010 // The index must always be an integer, which is not an aggregate. Emit it
5011 // in lexical order (this complexity is, sadly, required by C++17).
5012 llvm::Value *IdxPre =
5013 (E->getLHS() == E->getIdx()) ? EmitScalarExpr(E: E->getIdx()) : nullptr;
5014 bool SignedIndices = false;
5015 auto EmitIdxAfterBase = [&, IdxPre](bool Promote) -> llvm::Value * {
5016 auto *Idx = IdxPre;
5017 if (E->getLHS() != E->getIdx()) {
5018 assert(E->getRHS() == E->getIdx() && "index was neither LHS nor RHS");
5019 Idx = EmitScalarExpr(E: E->getIdx());
5020 }
5021
5022 QualType IdxTy = E->getIdx()->getType();
5023 bool IdxSigned = IdxTy->isSignedIntegerOrEnumerationType();
5024 SignedIndices |= IdxSigned;
5025
5026 if (SanOpts.has(K: SanitizerKind::ArrayBounds))
5027 EmitBoundsCheck(ArrayExpr: E, ArrayExprBase: E->getBase(), IndexVal: Idx, IndexType: IdxTy, Accessed);
5028
5029 // Extend or truncate the index type to 32 or 64-bits.
5030 if (Promote && Idx->getType() != IntPtrTy)
5031 Idx = Builder.CreateIntCast(V: Idx, DestTy: IntPtrTy, isSigned: IdxSigned, Name: "idxprom");
5032
5033 return Idx;
5034 };
5035 IdxPre = nullptr;
5036
5037 // If the base is a vector type, then we are forming a vector element lvalue
5038 // with this subscript.
5039 if (E->getBase()->getType()->isSubscriptableVectorType() &&
5040 !isa<ExtVectorElementExpr>(Val: E->getBase())) {
5041 // Emit the vector as an lvalue to get its address.
5042 LValue LHS = EmitLValue(E: E->getBase());
5043 auto *Idx = EmitIdxAfterBase(/*Promote*/false);
5044 assert(LHS.isSimple() && "Can only subscript lvalue vectors here!");
5045 return LValue::MakeVectorElt(vecAddress: LHS.getAddress(), Idx, type: E->getBase()->getType(),
5046 BaseInfo: LHS.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5047 }
5048
5049 // The HLSL runtime handles subscript expressions on global resource arrays
5050 // and objects with HLSL buffer layouts.
5051 if (getLangOpts().HLSL) {
5052 std::optional<LValue> LV;
5053 if (E->getType()->isHLSLResourceRecord() ||
5054 E->getType()->isHLSLResourceRecordArray()) {
5055 LV = CGM.getHLSLRuntime().emitResourceArraySubscriptExpr(E, CGF&: *this);
5056 } else if (E->getType().getAddressSpace() == LangAS::hlsl_constant) {
5057 LV = CGM.getHLSLRuntime().emitBufferArraySubscriptExpr(E, CGF&: *this,
5058 EmitIdxAfterBase);
5059 }
5060 if (LV.has_value())
5061 return *LV;
5062 }
5063
5064 // All the other cases basically behave like simple offsetting.
5065
5066 // Handle the extvector case we ignored above.
5067 if (isa<ExtVectorElementExpr>(Val: E->getBase())) {
5068 LValue LV = EmitLValue(E: E->getBase());
5069 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5070 Address Addr = EmitExtVectorElementLValue(LV);
5071
5072 QualType EltType = LV.getType()->castAs<VectorType>()->getElementType();
5073 Addr = emitArraySubscriptGEP(CGF&: *this, addr: Addr, indices: Idx, eltType: EltType, /*inbounds*/ true,
5074 signedIndices: SignedIndices, loc: E->getExprLoc());
5075 return MakeAddrLValue(Addr, T: EltType, BaseInfo: LV.getBaseInfo(),
5076 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: EltType));
5077 }
5078
5079 LValueBaseInfo EltBaseInfo;
5080 TBAAAccessInfo EltTBAAInfo;
5081 Address Addr = Address::invalid();
5082 if (const VariableArrayType *vla =
5083 getContext().getAsVariableArrayType(T: E->getType())) {
5084 // The base must be a pointer, which is not an aggregate. Emit
5085 // it. It needs to be emitted first in case it's what captures
5086 // the VLA bounds.
5087 Addr = EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
5088 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5089
5090 // The element count here is the total number of non-VLA elements.
5091 llvm::Value *numElements = getVLASize(vla).NumElts;
5092
5093 // Effectively, the multiply by the VLA size is part of the GEP.
5094 // GEP indexes are signed, and scaling an index isn't permitted to
5095 // signed-overflow, so we use the same semantics for our explicit
5096 // multiply. We suppress this if overflow is not undefined behavior.
5097 if (getLangOpts().PointerOverflowDefined) {
5098 Idx = Builder.CreateMul(LHS: Idx, RHS: numElements);
5099 } else {
5100 Idx = Builder.CreateNSWMul(LHS: Idx, RHS: numElements);
5101 }
5102
5103 Addr = emitArraySubscriptGEP(CGF&: *this, addr: Addr, indices: Idx, eltType: vla->getElementType(),
5104 inbounds: !getLangOpts().PointerOverflowDefined,
5105 signedIndices: SignedIndices, loc: E->getExprLoc());
5106
5107 } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){
5108 // Indexing over an interface, as in "NSString *P; P[4];"
5109
5110 // Emit the base pointer.
5111 Addr = EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
5112 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5113
5114 CharUnits InterfaceSize = getContext().getTypeSizeInChars(T: OIT);
5115 llvm::Value *InterfaceSizeVal =
5116 llvm::ConstantInt::get(Ty: Idx->getType(), V: InterfaceSize.getQuantity());
5117
5118 llvm::Value *ScaledIdx = Builder.CreateMul(LHS: Idx, RHS: InterfaceSizeVal);
5119
5120 // We don't necessarily build correct LLVM struct types for ObjC
5121 // interfaces, so we can't rely on GEP to do this scaling
5122 // correctly, so we need to cast to i8*. FIXME: is this actually
5123 // true? A lot of other things in the fragile ABI would break...
5124 llvm::Type *OrigBaseElemTy = Addr.getElementType();
5125
5126 // Do the GEP.
5127 CharUnits EltAlign =
5128 getArrayElementAlign(arrayAlign: Addr.getAlignment(), idx: Idx, eltSize: InterfaceSize);
5129 llvm::Value *EltPtr =
5130 emitArraySubscriptGEP(CGF&: *this, elemType: Int8Ty, ptr: Addr.emitRawPointer(CGF&: *this),
5131 indices: ScaledIdx, inbounds: false, signedIndices: SignedIndices, loc: E->getExprLoc());
5132 Addr = Address(EltPtr, OrigBaseElemTy, EltAlign);
5133 } else if (const Expr *Array = isSimpleArrayDecayOperand(E: E->getBase())) {
5134 // If this is A[i] where A is an array, the frontend will have decayed the
5135 // base to be a ArrayToPointerDecay implicit cast. While correct, it is
5136 // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
5137 // "gep x, i" here. Emit one "gep A, 0, i".
5138 assert(Array->getType()->isArrayType() &&
5139 "Array to pointer decay must have array source type!");
5140 LValue ArrayLV;
5141 // For simple multidimensional array indexing, set the 'accessed' flag for
5142 // better bounds-checking of the base expression.
5143 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Array))
5144 ArrayLV = EmitArraySubscriptExpr(E: ASE, /*Accessed*/ true);
5145 else
5146 ArrayLV = EmitLValue(E: Array);
5147 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5148
5149 if (SanOpts.has(K: SanitizerKind::ArrayBounds))
5150 EmitCountedByBoundsChecking(ArrayExpr: Array, ArrayType: Array->getType(), ArrayInst: ArrayLV.getAddress(),
5151 IndexType: E->getIdx()->getType(), IndexVal: Idx, Accessed,
5152 /*FlexibleArray=*/true);
5153
5154 // Propagate the alignment from the array itself to the result.
5155 QualType arrayType = Array->getType();
5156 Addr = emitArraySubscriptGEP(
5157 CGF&: *this, addr: ArrayLV.getAddress(), indices: {CGM.getSize(numChars: CharUnits::Zero()), Idx},
5158 eltType: E->getType(), inbounds: !getLangOpts().PointerOverflowDefined, signedIndices: SignedIndices,
5159 loc: E->getExprLoc(), arrayType: &arrayType, Base: E->getBase());
5160 EltBaseInfo = ArrayLV.getBaseInfo();
5161 if (!CGM.getCodeGenOpts().NewStructPathTBAA) {
5162 // Since CodeGenTBAA::getTypeInfoHelper only handles array types for
5163 // new struct path TBAA, we must a use a plain access.
5164 EltTBAAInfo = CGM.getTBAAInfoForSubobject(Base: ArrayLV, AccessType: E->getType());
5165 } else if (ArrayLV.getTBAAInfo().isMayAlias()) {
5166 EltTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
5167 } else if (ArrayLV.getTBAAInfo().isIncomplete()) {
5168 // The array element is complete, even if the array is not.
5169 EltTBAAInfo = CGM.getTBAAAccessInfo(AccessType: E->getType());
5170 } else {
5171 // The TBAA access info from the array (base) lvalue is ordinary. We will
5172 // adapt it to create access info for the element.
5173 EltTBAAInfo = ArrayLV.getTBAAInfo();
5174
5175 // We retain the TBAA struct path (BaseType and Offset members) from the
5176 // array. In the TBAA representation, we map any array access to the
5177 // element at index 0, as the index is generally a runtime value. This
5178 // element has the same offset in the base type as the array itself.
5179 // If the array lvalue had no base type, there is no point trying to
5180 // generate one, since an array itself is not a valid base type.
5181
5182 // We also retain the access type from the base lvalue, but the access
5183 // size must be updated to the size of an individual element.
5184 EltTBAAInfo.Size =
5185 getContext().getTypeSizeInChars(T: E->getType()).getQuantity();
5186 }
5187 } else {
5188 // The base must be a pointer; emit it with an estimate of its alignment.
5189 Address BaseAddr =
5190 EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
5191 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5192 QualType ptrType = E->getBase()->getType();
5193 Addr = emitArraySubscriptGEP(CGF&: *this, addr: BaseAddr, indices: Idx, eltType: E->getType(),
5194 inbounds: !getLangOpts().PointerOverflowDefined,
5195 signedIndices: SignedIndices, loc: E->getExprLoc(), arrayType: &ptrType,
5196 Base: E->getBase());
5197
5198 if (SanOpts.has(K: SanitizerKind::ArrayBounds)) {
5199 StructFieldAccess Visitor;
5200 const Expr *Base = Visitor.Visit(S: E->getBase());
5201
5202 if (const auto *CE = dyn_cast_if_present<CastExpr>(Val: Base);
5203 CE && CE->getCastKind() == CK_LValueToRValue)
5204 EmitCountedByBoundsChecking(ArrayExpr: CE, ArrayType: ptrType, ArrayInst: Address::invalid(),
5205 IndexType: E->getIdx()->getType(), IndexVal: Idx, Accessed,
5206 /*FlexibleArray=*/false);
5207 }
5208 }
5209
5210 LValue LV = MakeAddrLValue(Addr, T: E->getType(), BaseInfo: EltBaseInfo, TBAAInfo: EltTBAAInfo);
5211
5212 if (getLangOpts().ObjC &&
5213 getLangOpts().getGC() != LangOptions::NonGC) {
5214 LV.setNonGC(!E->isOBJCGCCandidate(Ctx&: getContext()));
5215 setObjCGCLValueClass(Ctx: getContext(), E, LV);
5216 }
5217 return LV;
5218}
5219
5220llvm::Value *CodeGenFunction::EmitMatrixIndexExpr(const Expr *E) {
5221 llvm::Value *Idx = EmitScalarExpr(E);
5222 if (Idx->getType() == IntPtrTy)
5223 return Idx;
5224 bool IsSigned = E->getType()->isSignedIntegerOrEnumerationType();
5225 return Builder.CreateIntCast(V: Idx, DestTy: IntPtrTy, isSigned: IsSigned);
5226}
5227
5228LValue CodeGenFunction::EmitMatrixSingleSubscriptExpr(
5229 const MatrixSingleSubscriptExpr *E) {
5230 LValue Base = EmitLValue(E: E->getBase());
5231 llvm::Value *RowIdx = EmitMatrixIndexExpr(E: E->getRowIdx());
5232
5233 RawAddress MatAddr = Base.getAddress();
5234 if (getLangOpts().HLSL &&
5235 E->getBase()->getType().getAddressSpace() == LangAS::hlsl_constant)
5236 MatAddr = CGM.getHLSLRuntime().createBufferMatrixTempAddress(LV: Base, CGF&: *this);
5237
5238 return LValue::MakeMatrixRow(Addr: MaybeConvertMatrixAddress(Addr: MatAddr, CGF&: *this),
5239 RowIdx, MatrixTy: E->getBase()->getType(),
5240 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5241}
5242
5243LValue CodeGenFunction::EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E) {
5244 assert(
5245 !E->isIncomplete() &&
5246 "incomplete matrix subscript expressions should be rejected during Sema");
5247 LValue Base = EmitLValue(E: E->getBase());
5248
5249 // Extend or truncate the index type to 32 or 64-bits if needed.
5250 llvm::Value *RowIdx = EmitMatrixIndexExpr(E: E->getRowIdx());
5251 llvm::Value *ColIdx = EmitMatrixIndexExpr(E: E->getColumnIdx());
5252 llvm::MatrixBuilder MB(Builder);
5253 const auto *MatrixTy = E->getBase()->getType()->castAs<ConstantMatrixType>();
5254 unsigned NumCols = MatrixTy->getNumColumns();
5255 unsigned NumRows = MatrixTy->getNumRows();
5256 bool IsMatrixRowMajor =
5257 isMatrixRowMajor(LangOpts: getLangOpts(), T: E->getBase()->getType());
5258 llvm::Value *FinalIdx =
5259 MB.CreateIndex(RowIdx, ColumnIdx: ColIdx, NumRows, NumCols, IsMatrixRowMajor);
5260
5261 return LValue::MakeMatrixElt(
5262 matAddress: MaybeConvertMatrixAddress(Addr: Base.getAddress(), CGF&: *this), Idx: FinalIdx,
5263 type: E->getBase()->getType(), BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5264}
5265
5266static Address emitOMPArraySectionBase(CodeGenFunction &CGF, const Expr *Base,
5267 LValueBaseInfo &BaseInfo,
5268 TBAAAccessInfo &TBAAInfo,
5269 QualType BaseTy, QualType ElTy,
5270 bool IsLowerBound) {
5271 LValue BaseLVal;
5272 if (auto *ASE = dyn_cast<ArraySectionExpr>(Val: Base->IgnoreParenImpCasts())) {
5273 BaseLVal = CGF.EmitArraySectionExpr(E: ASE, IsLowerBound);
5274 if (BaseTy->isArrayType()) {
5275 Address Addr = BaseLVal.getAddress();
5276 BaseInfo = BaseLVal.getBaseInfo();
5277
5278 // If the array type was an incomplete type, we need to make sure
5279 // the decay ends up being the right type.
5280 llvm::Type *NewTy = CGF.ConvertType(T: BaseTy);
5281 Addr = Addr.withElementType(ElemTy: NewTy);
5282
5283 // Note that VLA pointers are always decayed, so we don't need to do
5284 // anything here.
5285 if (!BaseTy->isVariableArrayType()) {
5286 assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
5287 "Expected pointer to array");
5288 Addr = CGF.Builder.CreateConstArrayGEP(Addr, Index: 0, Name: "arraydecay");
5289 }
5290
5291 return Addr.withElementType(ElemTy: CGF.ConvertTypeForMem(T: ElTy));
5292 }
5293 LValueBaseInfo TypeBaseInfo;
5294 TBAAAccessInfo TypeTBAAInfo;
5295 CharUnits Align =
5296 CGF.CGM.getNaturalTypeAlignment(T: ElTy, BaseInfo: &TypeBaseInfo, TBAAInfo: &TypeTBAAInfo);
5297 BaseInfo.mergeForCast(Info: TypeBaseInfo);
5298 TBAAInfo = CGF.CGM.mergeTBAAInfoForCast(SourceInfo: TBAAInfo, TargetInfo: TypeTBAAInfo);
5299 return Address(CGF.Builder.CreateLoad(Addr: BaseLVal.getAddress()),
5300 CGF.ConvertTypeForMem(T: ElTy), Align);
5301 }
5302 return CGF.EmitPointerWithAlignment(E: Base, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
5303}
5304
5305LValue CodeGenFunction::EmitArraySectionExpr(const ArraySectionExpr *E,
5306 bool IsLowerBound) {
5307
5308 assert(!E->isOpenACCArraySection() &&
5309 "OpenACC Array section codegen not implemented");
5310
5311 QualType BaseTy = ArraySectionExpr::getBaseOriginalType(Base: E->getBase());
5312 QualType ResultExprTy;
5313 if (auto *AT = getContext().getAsArrayType(T: BaseTy))
5314 ResultExprTy = AT->getElementType();
5315 else
5316 ResultExprTy = BaseTy->getPointeeType();
5317 llvm::Value *Idx = nullptr;
5318 if (IsLowerBound || E->getColonLocFirst().isInvalid()) {
5319 // Requesting lower bound or upper bound, but without provided length and
5320 // without ':' symbol for the default length -> length = 1.
5321 // Idx = LowerBound ?: 0;
5322 if (auto *LowerBound = E->getLowerBound()) {
5323 Idx = Builder.CreateIntCast(
5324 V: EmitScalarExpr(E: LowerBound), DestTy: IntPtrTy,
5325 isSigned: LowerBound->getType()->hasSignedIntegerRepresentation());
5326 } else
5327 Idx = llvm::ConstantInt::getNullValue(Ty: IntPtrTy);
5328 } else {
5329 // Try to emit length or lower bound as constant. If this is possible, 1
5330 // is subtracted from constant length or lower bound. Otherwise, emit LLVM
5331 // IR (LB + Len) - 1.
5332 auto &C = CGM.getContext();
5333 auto *Length = E->getLength();
5334 llvm::APSInt ConstLength;
5335 if (Length) {
5336 // Idx = LowerBound + Length - 1;
5337 if (std::optional<llvm::APSInt> CL = Length->getIntegerConstantExpr(Ctx: C)) {
5338 ConstLength = CL->zextOrTrunc(width: PointerWidthInBits);
5339 Length = nullptr;
5340 }
5341 auto *LowerBound = E->getLowerBound();
5342 llvm::APSInt ConstLowerBound(PointerWidthInBits, /*isUnsigned=*/false);
5343 if (LowerBound) {
5344 if (std::optional<llvm::APSInt> LB =
5345 LowerBound->getIntegerConstantExpr(Ctx: C)) {
5346 ConstLowerBound = LB->zextOrTrunc(width: PointerWidthInBits);
5347 LowerBound = nullptr;
5348 }
5349 }
5350 if (!Length)
5351 --ConstLength;
5352 else if (!LowerBound)
5353 --ConstLowerBound;
5354
5355 if (Length || LowerBound) {
5356 auto *LowerBoundVal =
5357 LowerBound
5358 ? Builder.CreateIntCast(
5359 V: EmitScalarExpr(E: LowerBound), DestTy: IntPtrTy,
5360 isSigned: LowerBound->getType()->hasSignedIntegerRepresentation())
5361 : llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLowerBound);
5362 auto *LengthVal =
5363 Length
5364 ? Builder.CreateIntCast(
5365 V: EmitScalarExpr(E: Length), DestTy: IntPtrTy,
5366 isSigned: Length->getType()->hasSignedIntegerRepresentation())
5367 : llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLength);
5368 Idx = Builder.CreateAdd(LHS: LowerBoundVal, RHS: LengthVal, Name: "lb_add_len",
5369 /*HasNUW=*/false,
5370 HasNSW: !getLangOpts().PointerOverflowDefined);
5371 if (Length && LowerBound) {
5372 Idx = Builder.CreateSub(
5373 LHS: Idx, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, /*V=*/1), Name: "idx_sub_1",
5374 /*HasNUW=*/false, HasNSW: !getLangOpts().PointerOverflowDefined);
5375 }
5376 } else
5377 Idx = llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLength + ConstLowerBound);
5378 } else {
5379 // Idx = ArraySize - 1;
5380 QualType ArrayTy = BaseTy->isPointerType()
5381 ? E->getBase()->IgnoreParenImpCasts()->getType()
5382 : BaseTy;
5383 if (auto *VAT = C.getAsVariableArrayType(T: ArrayTy)) {
5384 Length = VAT->getSizeExpr();
5385 if (std::optional<llvm::APSInt> L = Length->getIntegerConstantExpr(Ctx: C)) {
5386 ConstLength = *L;
5387 Length = nullptr;
5388 }
5389 } else {
5390 auto *CAT = C.getAsConstantArrayType(T: ArrayTy);
5391 assert(CAT && "unexpected type for array initializer");
5392 ConstLength = CAT->getSize();
5393 }
5394 if (Length) {
5395 auto *LengthVal = Builder.CreateIntCast(
5396 V: EmitScalarExpr(E: Length), DestTy: IntPtrTy,
5397 isSigned: Length->getType()->hasSignedIntegerRepresentation());
5398 Idx = Builder.CreateSub(
5399 LHS: LengthVal, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, /*V=*/1), Name: "len_sub_1",
5400 /*HasNUW=*/false, HasNSW: !getLangOpts().PointerOverflowDefined);
5401 } else {
5402 ConstLength = ConstLength.zextOrTrunc(width: PointerWidthInBits);
5403 --ConstLength;
5404 Idx = llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLength);
5405 }
5406 }
5407 }
5408 assert(Idx);
5409
5410 Address EltPtr = Address::invalid();
5411 LValueBaseInfo BaseInfo;
5412 TBAAAccessInfo TBAAInfo;
5413 if (auto *VLA = getContext().getAsVariableArrayType(T: ResultExprTy)) {
5414 // The base must be a pointer, which is not an aggregate. Emit
5415 // it. It needs to be emitted first in case it's what captures
5416 // the VLA bounds.
5417 Address Base =
5418 emitOMPArraySectionBase(CGF&: *this, Base: E->getBase(), BaseInfo, TBAAInfo,
5419 BaseTy, ElTy: VLA->getElementType(), IsLowerBound);
5420 // The element count here is the total number of non-VLA elements.
5421 llvm::Value *NumElements = getVLASize(vla: VLA).NumElts;
5422
5423 // Effectively, the multiply by the VLA size is part of the GEP.
5424 // GEP indexes are signed, and scaling an index isn't permitted to
5425 // signed-overflow, so we use the same semantics for our explicit
5426 // multiply. We suppress this if overflow is not undefined behavior.
5427 if (getLangOpts().PointerOverflowDefined)
5428 Idx = Builder.CreateMul(LHS: Idx, RHS: NumElements);
5429 else
5430 Idx = Builder.CreateNSWMul(LHS: Idx, RHS: NumElements);
5431 EltPtr = emitArraySubscriptGEP(CGF&: *this, addr: Base, indices: Idx, eltType: VLA->getElementType(),
5432 inbounds: !getLangOpts().PointerOverflowDefined,
5433 /*signedIndices=*/false, loc: E->getExprLoc());
5434 } else if (const Expr *Array = isSimpleArrayDecayOperand(E: E->getBase())) {
5435 // If this is A[i] where A is an array, the frontend will have decayed the
5436 // base to be a ArrayToPointerDecay implicit cast. While correct, it is
5437 // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
5438 // "gep x, i" here. Emit one "gep A, 0, i".
5439 assert(Array->getType()->isArrayType() &&
5440 "Array to pointer decay must have array source type!");
5441 LValue ArrayLV;
5442 // For simple multidimensional array indexing, set the 'accessed' flag for
5443 // better bounds-checking of the base expression.
5444 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Array))
5445 ArrayLV = EmitArraySubscriptExpr(E: ASE, /*Accessed*/ true);
5446 else
5447 ArrayLV = EmitLValue(E: Array);
5448
5449 // Propagate the alignment from the array itself to the result.
5450 EltPtr = emitArraySubscriptGEP(
5451 CGF&: *this, addr: ArrayLV.getAddress(), indices: {CGM.getSize(numChars: CharUnits::Zero()), Idx},
5452 eltType: ResultExprTy, inbounds: !getLangOpts().PointerOverflowDefined,
5453 /*signedIndices=*/false, loc: E->getExprLoc());
5454 BaseInfo = ArrayLV.getBaseInfo();
5455 TBAAInfo = CGM.getTBAAInfoForSubobject(Base: ArrayLV, AccessType: ResultExprTy);
5456 } else {
5457 Address Base =
5458 emitOMPArraySectionBase(CGF&: *this, Base: E->getBase(), BaseInfo, TBAAInfo, BaseTy,
5459 ElTy: ResultExprTy, IsLowerBound);
5460 EltPtr = emitArraySubscriptGEP(CGF&: *this, addr: Base, indices: Idx, eltType: ResultExprTy,
5461 inbounds: !getLangOpts().PointerOverflowDefined,
5462 /*signedIndices=*/false, loc: E->getExprLoc());
5463 }
5464
5465 return MakeAddrLValue(Addr: EltPtr, T: ResultExprTy, BaseInfo, TBAAInfo);
5466}
5467
5468LValue CodeGenFunction::
5469EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
5470 // Emit the base vector as an l-value.
5471 LValue Base;
5472
5473 // ExtVectorElementExpr's base can either be a vector or pointer to vector.
5474 if (E->isArrow()) {
5475 // If it is a pointer to a vector, emit the address and form an lvalue with
5476 // it.
5477 LValueBaseInfo BaseInfo;
5478 TBAAAccessInfo TBAAInfo;
5479 Address Ptr = EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
5480 const auto *PT = E->getBase()->getType()->castAs<PointerType>();
5481 Base = MakeAddrLValue(Addr: Ptr, T: PT->getPointeeType(), BaseInfo, TBAAInfo);
5482 Base.getQuals().removeObjCGCAttr();
5483 } else if (E->getBase()->isGLValue()) {
5484 // Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
5485 // emit the base as an lvalue.
5486 assert(E->getBase()->getType()->isVectorType());
5487 Base = EmitLValue(E: E->getBase());
5488 } else {
5489 // Otherwise, the base is a normal rvalue (as in (V+V).x), emit it as such.
5490 assert(E->getBase()->getType()->isVectorType() &&
5491 "Result must be a vector");
5492 llvm::Value *Vec = EmitScalarExpr(E: E->getBase());
5493
5494 // Store the vector to memory (because LValue wants an address).
5495 Address VecMem = CreateMemTemp(Ty: E->getBase()->getType());
5496 // need to zero extend an hlsl boolean vector to store it back to memory
5497 QualType Ty = E->getBase()->getType();
5498 llvm::Type *LTy = convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Vec->getType());
5499 if (LTy->getScalarSizeInBits() > Vec->getType()->getScalarSizeInBits())
5500 Vec = Builder.CreateZExt(V: Vec, DestTy: LTy);
5501 Builder.CreateStore(Val: Vec, Addr: VecMem);
5502 Base = MakeAddrLValue(Addr: VecMem, T: Ty, Source: AlignmentSource::Decl);
5503 }
5504
5505 QualType type =
5506 E->getType().withCVRQualifiers(CVR: Base.getQuals().getCVRQualifiers());
5507
5508 // Encode the element access list into a vector of unsigned indices.
5509 SmallVector<uint32_t, 4> Indices;
5510 E->getEncodedElementAccess(Elts&: Indices);
5511
5512 if (Base.isSimple()) {
5513 llvm::Constant *CV =
5514 llvm::ConstantDataVector::get(Context&: getLLVMContext(), Elts: Indices);
5515 return LValue::MakeExtVectorElt(Addr: Base.getAddress(), Elts: CV, type,
5516 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5517 }
5518
5519 if (Base.isMatrixRow()) {
5520 if (auto *RowIdx =
5521 llvm::dyn_cast<llvm::ConstantInt>(Val: Base.getMatrixRowIdx())) {
5522 llvm::SmallVector<llvm::Constant *> MatIndices;
5523 QualType MatTy = Base.getType();
5524 const ConstantMatrixType *MT = MatTy->castAs<ConstantMatrixType>();
5525 unsigned NumCols = Indices.size();
5526 unsigned NumRows = MT->getNumRows();
5527 unsigned Row = RowIdx->getZExtValue();
5528 QualType VecQT = E->getBase()->getType();
5529 if (NumCols != MT->getNumColumns()) {
5530 const auto *EVT = VecQT->getAs<ExtVectorType>();
5531 QualType ElemQT = EVT->getElementType();
5532 VecQT = getContext().getExtVectorType(VectorType: ElemQT, NumElts: NumCols);
5533 }
5534 for (unsigned C = 0; C < NumCols; ++C) {
5535 unsigned Col = Indices[C];
5536 unsigned Linear = Col * NumRows + Row;
5537 MatIndices.push_back(Elt: llvm::ConstantInt::get(Ty: Int32Ty, V: Linear));
5538 }
5539
5540 llvm::Constant *ConstIdxs = llvm::ConstantVector::get(V: MatIndices);
5541 return LValue::MakeExtVectorElt(Addr: Base.getMatrixAddress(), Elts: ConstIdxs, type: VecQT,
5542 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5543 }
5544 llvm::Constant *Cols =
5545 llvm::ConstantDataVector::get(Context&: getLLVMContext(), Elts: Indices);
5546 // Note: intentionally not using E.getType() so we can reuse isMatrixRow()
5547 // implementations in EmitLoadOfLValue & EmitStoreThroughLValue and don't
5548 // need the LValue to have its own number of rows and columns when the
5549 // type is a vector.
5550 return LValue::MakeMatrixRowSwizzle(
5551 MatAddr: Base.getMatrixAddress(), RowIdx: Base.getMatrixRowIdx(), Cols, MatrixTy: Base.getType(),
5552 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5553 }
5554
5555 assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
5556
5557 llvm::Constant *BaseElts = Base.getExtVectorElts();
5558 SmallVector<llvm::Constant *, 4> CElts;
5559
5560 for (unsigned Index : Indices)
5561 CElts.push_back(Elt: BaseElts->getAggregateElement(Elt: Index));
5562 llvm::Constant *CV = llvm::ConstantVector::get(V: CElts);
5563 return LValue::MakeExtVectorElt(Addr: Base.getExtVectorAddress(), Elts: CV, type,
5564 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5565}
5566
5567bool CodeGenFunction::isUnderlyingBasePointerConstantNull(const Expr *E) {
5568 const Expr *UnderlyingBaseExpr = E->IgnoreParens();
5569 while (auto *BaseMemberExpr = dyn_cast<MemberExpr>(Val: UnderlyingBaseExpr))
5570 UnderlyingBaseExpr = BaseMemberExpr->getBase()->IgnoreParens();
5571 return getContext().isSentinelNullExpr(E: UnderlyingBaseExpr);
5572}
5573
5574LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
5575 if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(CGF&: *this, ME: E)) {
5576 EmitIgnoredExpr(E: E->getBase());
5577 return EmitDeclRefLValue(E: DRE);
5578 }
5579
5580 if (getLangOpts().HLSL) {
5581 QualType QT = E->getType();
5582 if (QT.getAddressSpace() == LangAS::hlsl_constant)
5583 return CGM.getHLSLRuntime().emitBufferMemberExpr(CGF&: *this, E);
5584
5585 if (QT->isHLSLResourceRecord() || QT->isHLSLResourceRecordArray()) {
5586 std::optional<LValue> LV;
5587 LV = CGM.getHLSLRuntime().emitResourceMemberExpr(CGF&: *this, E);
5588 if (LV.has_value())
5589 return *LV;
5590 }
5591 }
5592
5593 Expr *BaseExpr = E->getBase();
5594 // Check whether the underlying base pointer is a constant null.
5595 // If so, we do not set inbounds flag for GEP to avoid breaking some
5596 // old-style offsetof idioms.
5597 bool IsInBounds = !getLangOpts().PointerOverflowDefined &&
5598 !isUnderlyingBasePointerConstantNull(E: BaseExpr);
5599 // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar.
5600 LValue BaseLV;
5601 if (E->isArrow()) {
5602 LValueBaseInfo BaseInfo;
5603 TBAAAccessInfo TBAAInfo;
5604 Address Addr = EmitPointerWithAlignment(E: BaseExpr, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
5605 QualType PtrTy = BaseExpr->getType()->getPointeeType();
5606 SanitizerSet SkippedChecks;
5607 bool IsBaseCXXThis = IsWrappedCXXThis(Obj: BaseExpr);
5608 if (IsBaseCXXThis)
5609 SkippedChecks.set(K: SanitizerKind::Alignment, Value: true);
5610 if (IsBaseCXXThis || isa<DeclRefExpr>(Val: BaseExpr))
5611 SkippedChecks.set(K: SanitizerKind::Null, Value: true);
5612 EmitTypeCheck(TCK: TCK_MemberAccess, Loc: E->getExprLoc(), Addr, Type: PtrTy,
5613 /*Alignment=*/CharUnits::Zero(), SkippedChecks);
5614 BaseLV = MakeAddrLValue(Addr, T: PtrTy, BaseInfo, TBAAInfo);
5615 } else
5616 BaseLV = EmitCheckedLValue(E: BaseExpr, TCK: TCK_MemberAccess);
5617
5618 NamedDecl *ND = E->getMemberDecl();
5619 if (auto *Field = dyn_cast<FieldDecl>(Val: ND)) {
5620 LValue LV = EmitLValueForField(Base: BaseLV, Field, IsInBounds);
5621 setObjCGCLValueClass(Ctx: getContext(), E, LV);
5622 if (getLangOpts().OpenMP) {
5623 // If the member was explicitly marked as nontemporal, mark it as
5624 // nontemporal. If the base lvalue is marked as nontemporal, mark access
5625 // to children as nontemporal too.
5626 if ((IsWrappedCXXThis(Obj: BaseExpr) &&
5627 CGM.getOpenMPRuntime().isNontemporalDecl(VD: Field)) ||
5628 BaseLV.isNontemporal())
5629 LV.setNontemporal(/*Value=*/true);
5630 }
5631 return LV;
5632 }
5633
5634 if (const auto *FD = dyn_cast<FunctionDecl>(Val: ND))
5635 return EmitFunctionDeclLValue(CGF&: *this, E, GD: FD);
5636
5637 llvm_unreachable("Unhandled member declaration!");
5638}
5639
5640/// Given that we are currently emitting a lambda, emit an l-value for
5641/// one of its members.
5642///
5643LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field,
5644 llvm::Value *ThisValue) {
5645 bool HasExplicitObjectParameter = false;
5646 const auto *MD = dyn_cast_if_present<CXXMethodDecl>(Val: CurCodeDecl);
5647 if (MD) {
5648 HasExplicitObjectParameter = MD->isExplicitObjectMemberFunction();
5649 assert(MD->getParent()->isLambda());
5650 assert(MD->getParent() == Field->getParent());
5651 }
5652 LValue LambdaLV;
5653 if (HasExplicitObjectParameter) {
5654 const VarDecl *D = cast<CXXMethodDecl>(Val: CurCodeDecl)->getParamDecl(i: 0);
5655 auto It = LocalDeclMap.find(Val: D);
5656 assert(It != LocalDeclMap.end() && "explicit parameter not loaded?");
5657 Address AddrOfExplicitObject = It->getSecond();
5658 if (D->getType()->isReferenceType())
5659 LambdaLV = EmitLoadOfReferenceLValue(RefAddr: AddrOfExplicitObject, RefTy: D->getType(),
5660 Source: AlignmentSource::Decl);
5661 else
5662 LambdaLV = MakeAddrLValue(Addr: AddrOfExplicitObject,
5663 T: D->getType().getNonReferenceType());
5664
5665 // Make sure we have an lvalue to the lambda itself and not a derived class.
5666 auto *ThisTy = D->getType().getNonReferenceType()->getAsCXXRecordDecl();
5667 auto *LambdaTy = cast<CXXRecordDecl>(Val: Field->getParent());
5668 if (ThisTy != LambdaTy) {
5669 const CXXCastPath &BasePathArray = getContext().LambdaCastPaths.at(Val: MD);
5670 Address Base = GetAddressOfBaseClass(
5671 Value: LambdaLV.getAddress(), Derived: ThisTy, PathBegin: BasePathArray.begin(),
5672 PathEnd: BasePathArray.end(), /*NullCheckValue=*/false, Loc: SourceLocation());
5673 CanQualType T = getContext().getCanonicalTagType(TD: LambdaTy);
5674 LambdaLV = MakeAddrLValue(Addr: Base, T);
5675 }
5676 } else {
5677 CanQualType LambdaTagType =
5678 getContext().getCanonicalTagType(TD: Field->getParent());
5679 LambdaLV = MakeNaturalAlignAddrLValue(V: ThisValue, T: LambdaTagType);
5680 }
5681 return EmitLValueForField(Base: LambdaLV, Field);
5682}
5683
5684LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field) {
5685 return EmitLValueForLambdaField(Field, ThisValue: CXXABIThisValue);
5686}
5687
5688/// Get the field index in the debug info. The debug info structure/union
5689/// will ignore the unnamed bitfields.
5690unsigned CodeGenFunction::getDebugInfoFIndex(const RecordDecl *Rec,
5691 unsigned FieldIndex) {
5692 unsigned I = 0, Skipped = 0;
5693
5694 for (auto *F : Rec->getDefinition()->fields()) {
5695 if (I == FieldIndex)
5696 break;
5697 if (F->isUnnamedBitField())
5698 Skipped++;
5699 I++;
5700 }
5701
5702 return FieldIndex - Skipped;
5703}
5704
5705/// Get the address of a zero-sized field within a record. The resulting
5706/// address doesn't necessarily have the right type.
5707static Address emitAddrOfZeroSizeField(CodeGenFunction &CGF, Address Base,
5708 const FieldDecl *Field,
5709 bool IsInBounds) {
5710 CharUnits Offset = CGF.getContext().toCharUnitsFromBits(
5711 BitSize: CGF.getContext().getFieldOffset(FD: Field));
5712 if (Offset.isZero())
5713 return Base;
5714 Base = Base.withElementType(ElemTy: CGF.Int8Ty);
5715 if (!IsInBounds)
5716 return CGF.Builder.CreateConstByteGEP(Addr: Base, Offset);
5717 return CGF.Builder.CreateConstInBoundsByteGEP(Addr: Base, Offset);
5718}
5719
5720/// Drill down to the storage of a field without walking into reference types,
5721/// and without respect for pointer field protection.
5722///
5723/// The resulting address doesn't necessarily have the right type.
5724static Address emitRawAddrOfFieldStorage(CodeGenFunction &CGF, Address base,
5725 const FieldDecl *field,
5726 bool IsInBounds) {
5727 if (isEmptyFieldForLayout(Context: CGF.getContext(), FD: field))
5728 return emitAddrOfZeroSizeField(CGF, Base: base, Field: field, IsInBounds);
5729
5730 const RecordDecl *rec = field->getParent();
5731
5732 unsigned idx =
5733 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(FD: field);
5734 llvm::Type *StructType =
5735 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMType();
5736
5737 if (CGF.getLangOpts().EmitLogicalPointer)
5738 return RawAddress(
5739 CGF.Builder.CreateStructuredGEP(BaseType: StructType, PtrBase: base.emitRawPointer(CGF),
5740 Indices: {CGF.Builder.getSize(N: idx)}),
5741 base.getElementType(), base.getAlignment());
5742
5743 if (!IsInBounds)
5744 return CGF.Builder.CreateConstGEP2_32(Addr: base, Idx0: 0, Idx1: idx, Name: field->getName());
5745
5746 return CGF.Builder.CreateStructGEP(Addr: base, Index: idx, Name: field->getName());
5747}
5748
5749/// Drill down to the storage of a field without walking into reference types,
5750/// wrapping the address in an llvm.protected.field.ptr intrinsic for the
5751/// pointer field protection feature if necessary.
5752///
5753/// The resulting address doesn't necessarily have the right type.
5754static Address emitAddrOfFieldStorage(CodeGenFunction &CGF, Address base,
5755 const FieldDecl *field, bool IsInBounds) {
5756 Address Addr = emitRawAddrOfFieldStorage(CGF, base, field, IsInBounds);
5757
5758 if (!CGF.getContext().isPFPField(Field: field))
5759 return Addr;
5760
5761 return CGF.EmitAddressOfPFPField(RecordPtr: base, FieldPtr: Addr, Field: field);
5762}
5763
5764static Address emitPreserveStructAccess(CodeGenFunction &CGF, LValue base,
5765 Address addr, const FieldDecl *field) {
5766 const RecordDecl *rec = field->getParent();
5767 llvm::DIType *DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(
5768 Ty: base.getType(), Loc: rec->getLocation());
5769
5770 unsigned idx =
5771 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(FD: field);
5772
5773 return CGF.Builder.CreatePreserveStructAccessIndex(
5774 Addr: addr, Index: idx, FieldIndex: CGF.getDebugInfoFIndex(Rec: rec, FieldIndex: field->getFieldIndex()), DbgInfo);
5775}
5776
5777static bool hasAnyVptr(const QualType Type, const ASTContext &Context) {
5778 const auto *RD = Type.getTypePtr()->getAsCXXRecordDecl();
5779 if (!RD)
5780 return false;
5781
5782 if (RD->isDynamicClass())
5783 return true;
5784
5785 for (const auto &Base : RD->bases())
5786 if (hasAnyVptr(Type: Base.getType(), Context))
5787 return true;
5788
5789 for (const FieldDecl *Field : RD->fields())
5790 if (hasAnyVptr(Type: Field->getType(), Context))
5791 return true;
5792
5793 return false;
5794}
5795
5796LValue CodeGenFunction::EmitLValueForField(LValue base, const FieldDecl *field,
5797 bool IsInBounds) {
5798 LValueBaseInfo BaseInfo = base.getBaseInfo();
5799
5800 if (field->isBitField()) {
5801 const CGRecordLayout &RL =
5802 CGM.getTypes().getCGRecordLayout(field->getParent());
5803 const CGBitFieldInfo &Info = RL.getBitFieldInfo(FD: field);
5804 const bool UseVolatile = isAAPCS(TargetInfo: CGM.getTarget()) &&
5805 CGM.getCodeGenOpts().AAPCSBitfieldWidth &&
5806 Info.VolatileStorageSize != 0 &&
5807 field->getType()
5808 .withCVRQualifiers(CVR: base.getVRQualifiers())
5809 .isVolatileQualified();
5810 Address Addr = base.getAddress();
5811 unsigned Idx = RL.getLLVMFieldNo(FD: field);
5812 const RecordDecl *rec = field->getParent();
5813 if (hasBPFPreserveStaticOffset(D: rec))
5814 Addr = wrapWithBPFPreserveStaticOffset(CGF&: *this, Addr);
5815 if (!UseVolatile) {
5816 if (!IsInPreservedAIRegion &&
5817 (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
5818 if (Idx != 0) {
5819 // For structs, we GEP to the field that the record layout suggests.
5820 if (!IsInBounds)
5821 Addr = Builder.CreateConstGEP2_32(Addr, Idx0: 0, Idx1: Idx, Name: field->getName());
5822 else
5823 Addr = Builder.CreateStructGEP(Addr, Index: Idx, Name: field->getName());
5824 }
5825 } else {
5826 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType(
5827 Ty: getContext().getCanonicalTagType(TD: rec), L: rec->getLocation());
5828 Addr = Builder.CreatePreserveStructAccessIndex(
5829 Addr, Index: Idx, FieldIndex: getDebugInfoFIndex(Rec: rec, FieldIndex: field->getFieldIndex()),
5830 DbgInfo);
5831 }
5832 }
5833 const unsigned SS =
5834 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
5835 // Get the access type.
5836 llvm::Type *FieldIntTy = llvm::Type::getIntNTy(C&: getLLVMContext(), N: SS);
5837 Addr = Addr.withElementType(ElemTy: FieldIntTy);
5838 if (UseVolatile) {
5839 const unsigned VolatileOffset = Info.VolatileStorageOffset.getQuantity();
5840 if (VolatileOffset)
5841 Addr = Builder.CreateConstInBoundsGEP(Addr, Index: VolatileOffset);
5842 }
5843
5844 QualType fieldType =
5845 field->getType().withCVRQualifiers(CVR: base.getVRQualifiers());
5846 // TODO: Support TBAA for bit fields.
5847 LValueBaseInfo FieldBaseInfo(BaseInfo.getAlignmentSource());
5848 return LValue::MakeBitfield(Addr, Info, type: fieldType, BaseInfo: FieldBaseInfo,
5849 TBAAInfo: TBAAAccessInfo());
5850 }
5851
5852 // Fields of may-alias structures are may-alias themselves.
5853 // FIXME: this should get propagated down through anonymous structs
5854 // and unions.
5855 QualType FieldType = field->getType();
5856 const RecordDecl *rec = field->getParent();
5857 AlignmentSource BaseAlignSource = BaseInfo.getAlignmentSource();
5858 LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(Source: BaseAlignSource));
5859 TBAAAccessInfo FieldTBAAInfo;
5860 if (base.getTBAAInfo().isMayAlias() ||
5861 rec->hasAttr<MayAliasAttr>() || FieldType->isVectorType()) {
5862 FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
5863 } else if (rec->isUnion()) {
5864 // TODO: Support TBAA for unions.
5865 FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
5866 } else {
5867 // If no base type been assigned for the base access, then try to generate
5868 // one for this base lvalue.
5869 FieldTBAAInfo = base.getTBAAInfo();
5870 if (!FieldTBAAInfo.BaseType) {
5871 FieldTBAAInfo.BaseType = CGM.getTBAABaseTypeInfo(QTy: base.getType());
5872 assert(!FieldTBAAInfo.Offset &&
5873 "Nonzero offset for an access with no base type!");
5874 }
5875
5876 // Adjust offset to be relative to the base type.
5877 const ASTRecordLayout &Layout =
5878 getContext().getASTRecordLayout(D: field->getParent());
5879 unsigned CharWidth = getContext().getCharWidth();
5880 if (FieldTBAAInfo.BaseType)
5881 FieldTBAAInfo.Offset +=
5882 Layout.getFieldOffset(FieldNo: field->getFieldIndex()) / CharWidth;
5883
5884 // Update the final access type and size.
5885 FieldTBAAInfo.AccessType = CGM.getTBAATypeInfo(QTy: FieldType);
5886 FieldTBAAInfo.Size =
5887 getContext().getTypeSizeInChars(T: FieldType).getQuantity();
5888 }
5889
5890 Address addr = base.getAddress();
5891 if (hasBPFPreserveStaticOffset(D: rec))
5892 addr = wrapWithBPFPreserveStaticOffset(CGF&: *this, Addr&: addr);
5893 if (auto *ClassDef = dyn_cast<CXXRecordDecl>(Val: rec)) {
5894 if (CGM.getCodeGenOpts().StrictVTablePointers &&
5895 ClassDef->isDynamicClass()) {
5896 // Getting to any field of dynamic object requires stripping dynamic
5897 // information provided by invariant.group. This is because accessing
5898 // fields may leak the real address of dynamic object, which could result
5899 // in miscompilation when leaked pointer would be compared.
5900 auto *stripped =
5901 Builder.CreateStripInvariantGroup(Ptr: addr.emitRawPointer(CGF&: *this));
5902 addr = Address(stripped, addr.getElementType(), addr.getAlignment());
5903 }
5904 }
5905
5906 unsigned RecordCVR = base.getVRQualifiers();
5907 if (rec->isUnion()) {
5908 // For unions, there is no pointer adjustment.
5909 if (CGM.getCodeGenOpts().StrictVTablePointers &&
5910 hasAnyVptr(Type: FieldType, Context: getContext()))
5911 // Because unions can easily skip invariant.barriers, we need to add
5912 // a barrier every time CXXRecord field with vptr is referenced.
5913 addr = Builder.CreateLaunderInvariantGroup(Addr: addr);
5914
5915 if (IsInPreservedAIRegion ||
5916 (getDebugInfo() && rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
5917 // Remember the original union field index
5918 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(Ty: base.getType(),
5919 Loc: rec->getLocation());
5920 addr =
5921 Address(Builder.CreatePreserveUnionAccessIndex(
5922 Base: addr.emitRawPointer(CGF&: *this),
5923 FieldIndex: getDebugInfoFIndex(Rec: rec, FieldIndex: field->getFieldIndex()), DbgInfo),
5924 addr.getElementType(), addr.getAlignment());
5925 }
5926
5927 if (FieldType->isReferenceType())
5928 addr = addr.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: FieldType));
5929 } else {
5930 if (!IsInPreservedAIRegion &&
5931 (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>()))
5932 // For structs, we GEP to the field that the record layout suggests.
5933 addr = emitAddrOfFieldStorage(CGF&: *this, base: addr, field, IsInBounds);
5934 else
5935 // Remember the original struct field index
5936 addr = emitPreserveStructAccess(CGF&: *this, base, addr, field);
5937 }
5938
5939 // If this is a reference field, load the reference right now.
5940 if (FieldType->isReferenceType()) {
5941 LValue RefLVal =
5942 MakeAddrLValue(Addr: addr, T: FieldType, BaseInfo: FieldBaseInfo, TBAAInfo: FieldTBAAInfo);
5943 if (RecordCVR & Qualifiers::Volatile)
5944 RefLVal.getQuals().addVolatile();
5945 addr = EmitLoadOfReference(RefLVal, PointeeBaseInfo: &FieldBaseInfo, PointeeTBAAInfo: &FieldTBAAInfo);
5946
5947 // Qualifiers on the struct don't apply to the referencee.
5948 RecordCVR = 0;
5949 FieldType = FieldType->getPointeeType();
5950 }
5951
5952 // Make sure that the address is pointing to the right type. This is critical
5953 // for both unions and structs.
5954 addr = addr.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: FieldType));
5955
5956 if (field->hasAttr<AnnotateAttr>())
5957 addr = EmitFieldAnnotations(D: field, V: addr);
5958
5959 LValue LV = MakeAddrLValue(Addr: addr, T: FieldType, BaseInfo: FieldBaseInfo, TBAAInfo: FieldTBAAInfo);
5960 LV.getQuals().addCVRQualifiers(mask: RecordCVR);
5961
5962 // __weak attribute on a field is ignored.
5963 if (LV.getQuals().getObjCGCAttr() == Qualifiers::Weak)
5964 LV.getQuals().removeObjCGCAttr();
5965
5966 return LV;
5967}
5968
5969LValue
5970CodeGenFunction::EmitLValueForFieldInitialization(LValue Base,
5971 const FieldDecl *Field) {
5972 QualType FieldType = Field->getType();
5973
5974 if (!FieldType->isReferenceType())
5975 return EmitLValueForField(base: Base, field: Field);
5976
5977 Address V = emitAddrOfFieldStorage(
5978 CGF&: *this, base: Base.getAddress(), field: Field,
5979 /*IsInBounds=*/!getLangOpts().PointerOverflowDefined);
5980
5981 // Make sure that the address is pointing to the right type.
5982 llvm::Type *llvmType = ConvertTypeForMem(T: FieldType);
5983 V = V.withElementType(ElemTy: llvmType);
5984
5985 // TODO: Generate TBAA information that describes this access as a structure
5986 // member access and not just an access to an object of the field's type. This
5987 // should be similar to what we do in EmitLValueForField().
5988 LValueBaseInfo BaseInfo = Base.getBaseInfo();
5989 AlignmentSource FieldAlignSource = BaseInfo.getAlignmentSource();
5990 LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(Source: FieldAlignSource));
5991 return MakeAddrLValue(Addr: V, T: FieldType, BaseInfo: FieldBaseInfo,
5992 TBAAInfo: CGM.getTBAAInfoForSubobject(Base, AccessType: FieldType));
5993}
5994
5995LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr *E){
5996 if (E->isFileScope()) {
5997 ConstantAddress GlobalPtr = CGM.GetAddrOfConstantCompoundLiteral(E);
5998 return MakeAddrLValue(Addr: GlobalPtr, T: E->getType(), Source: AlignmentSource::Decl);
5999 }
6000 if (E->getType()->isVariablyModifiedType())
6001 // make sure to emit the VLA size.
6002 EmitVariablyModifiedType(Ty: E->getType());
6003
6004 Address DeclPtr = CreateMemTempWithoutCast(Ty: E->getType(), Name: ".compoundliteral");
6005 const Expr *InitExpr = E->getInitializer();
6006 LValue Result = MakeAddrLValue(Addr: DeclPtr, T: E->getType(), Source: AlignmentSource::Decl);
6007
6008 EmitAnyExprToMem(E: InitExpr, Location: DeclPtr, Quals: E->getType().getQualifiers(),
6009 /*Init*/ IsInit: true);
6010
6011 // Block-scope compound literals are destroyed at the end of the enclosing
6012 // scope in C.
6013 if (!getLangOpts().CPlusPlus)
6014 if (QualType::DestructionKind DtorKind = E->getType().isDestructedType())
6015 pushLifetimeExtendedDestroy(kind: getCleanupKind(kind: DtorKind), addr: DeclPtr,
6016 type: E->getType(), destroyer: getDestroyer(destructionKind: DtorKind),
6017 useEHCleanupForArray: DtorKind & EHCleanup);
6018
6019 return Result;
6020}
6021
6022LValue CodeGenFunction::EmitInitListLValue(const InitListExpr *E) {
6023 if (!E->isGLValue())
6024 // Initializing an aggregate temporary in C++11: T{...}.
6025 return EmitAggExprToLValue(E);
6026
6027 // An lvalue initializer list must be initializing a reference.
6028 assert(E->isTransparent() && "non-transparent glvalue init list");
6029 return EmitLValue(E: E->getInit(Init: 0));
6030}
6031
6032/// Emit the operand of a glvalue conditional operator. This is either a glvalue
6033/// or a (possibly-parenthesized) throw-expression. If this is a throw, no
6034/// LValue is returned and the current block has been terminated.
6035static std::optional<LValue> EmitLValueOrThrowExpression(CodeGenFunction &CGF,
6036 const Expr *Operand) {
6037 if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Val: Operand->IgnoreParens())) {
6038 CGF.EmitCXXThrowExpr(E: ThrowExpr, /*KeepInsertionPoint*/false);
6039 return std::nullopt;
6040 }
6041
6042 return CGF.EmitLValue(E: Operand);
6043}
6044
6045namespace {
6046// Handle the case where the condition is a constant evaluatable simple integer,
6047// which means we don't have to separately handle the true/false blocks.
6048std::optional<LValue> HandleConditionalOperatorLValueSimpleCase(
6049 CodeGenFunction &CGF, const AbstractConditionalOperator *E) {
6050 const Expr *condExpr = E->getCond();
6051 bool CondExprBool;
6052 if (CGF.ConstantFoldsToSimpleInteger(Cond: condExpr, Result&: CondExprBool)) {
6053 const Expr *Live = E->getTrueExpr(), *Dead = E->getFalseExpr();
6054 if (!CondExprBool)
6055 std::swap(a&: Live, b&: Dead);
6056
6057 if (!CGF.ContainsLabel(S: Dead)) {
6058 // If the true case is live, we need to track its region.
6059 CGF.incrementProfileCounter(ExecSkip: CondExprBool ? CGF.UseExecPath
6060 : CGF.UseSkipPath,
6061 S: E, /*UseBoth=*/true);
6062 CGF.markStmtMaybeUsed(S: Dead);
6063 // If a throw expression we emit it and return an undefined lvalue
6064 // because it can't be used.
6065 if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Val: Live->IgnoreParens())) {
6066 CGF.EmitCXXThrowExpr(E: ThrowExpr);
6067 llvm::Type *ElemTy = CGF.ConvertType(T: Dead->getType());
6068 llvm::Type *Ty = CGF.DefaultPtrTy;
6069 return CGF.MakeAddrLValue(
6070 Addr: Address(llvm::UndefValue::get(T: Ty), ElemTy, CharUnits::One()),
6071 T: Dead->getType());
6072 }
6073 return CGF.EmitLValue(E: Live);
6074 }
6075 }
6076 return std::nullopt;
6077}
6078struct ConditionalInfo {
6079 llvm::BasicBlock *lhsBlock, *rhsBlock;
6080 std::optional<LValue> LHS, RHS;
6081};
6082
6083// Create and generate the 3 blocks for a conditional operator.
6084// Leaves the 'current block' in the continuation basic block.
6085template<typename FuncTy>
6086ConditionalInfo EmitConditionalBlocks(CodeGenFunction &CGF,
6087 const AbstractConditionalOperator *E,
6088 const FuncTy &BranchGenFunc) {
6089 ConditionalInfo Info{.lhsBlock: CGF.createBasicBlock(name: "cond.true"),
6090 .rhsBlock: CGF.createBasicBlock(name: "cond.false"), .LHS: std::nullopt,
6091 .RHS: std::nullopt};
6092 llvm::BasicBlock *endBlock = CGF.createBasicBlock(name: "cond.end");
6093
6094 CodeGenFunction::ConditionalEvaluation eval(CGF);
6095 CGF.EmitBranchOnBoolExpr(Cond: E->getCond(), TrueBlock: Info.lhsBlock, FalseBlock: Info.rhsBlock,
6096 TrueCount: CGF.getProfileCount(S: E));
6097
6098 // Any temporaries created here are conditional.
6099 CGF.EmitBlock(BB: Info.lhsBlock);
6100 CGF.incrementProfileCounter(ExecSkip: CGF.UseExecPath, S: E);
6101 eval.begin(CGF);
6102 Info.LHS = BranchGenFunc(CGF, E->getTrueExpr());
6103 eval.end(CGF);
6104 Info.lhsBlock = CGF.Builder.GetInsertBlock();
6105
6106 if (Info.LHS)
6107 CGF.Builder.CreateBr(Dest: endBlock);
6108
6109 // Any temporaries created here are conditional.
6110 CGF.EmitBlock(BB: Info.rhsBlock);
6111 CGF.incrementProfileCounter(ExecSkip: CGF.UseSkipPath, S: E);
6112 eval.begin(CGF);
6113 Info.RHS = BranchGenFunc(CGF, E->getFalseExpr());
6114 eval.end(CGF);
6115 Info.rhsBlock = CGF.Builder.GetInsertBlock();
6116 CGF.EmitBlock(BB: endBlock);
6117
6118 return Info;
6119}
6120} // namespace
6121
6122void CodeGenFunction::EmitIgnoredConditionalOperator(
6123 const AbstractConditionalOperator *E) {
6124 if (!E->isGLValue()) {
6125 // ?: here should be an aggregate.
6126 assert(hasAggregateEvaluationKind(E->getType()) &&
6127 "Unexpected conditional operator!");
6128 return (void)EmitAggExprToLValue(E);
6129 }
6130
6131 OpaqueValueMapping binding(*this, E);
6132 if (HandleConditionalOperatorLValueSimpleCase(CGF&: *this, E))
6133 return;
6134
6135 EmitConditionalBlocks(CGF&: *this, E, BranchGenFunc: [](CodeGenFunction &CGF, const Expr *E) {
6136 CGF.EmitIgnoredExpr(E);
6137 return LValue{};
6138 });
6139}
6140LValue CodeGenFunction::EmitConditionalOperatorLValue(
6141 const AbstractConditionalOperator *expr) {
6142 if (!expr->isGLValue()) {
6143 // ?: here should be an aggregate.
6144 assert(hasAggregateEvaluationKind(expr->getType()) &&
6145 "Unexpected conditional operator!");
6146 return EmitAggExprToLValue(E: expr);
6147 }
6148
6149 OpaqueValueMapping binding(*this, expr);
6150 if (std::optional<LValue> Res =
6151 HandleConditionalOperatorLValueSimpleCase(CGF&: *this, E: expr))
6152 return *Res;
6153
6154 ConditionalInfo Info = EmitConditionalBlocks(
6155 CGF&: *this, E: expr, BranchGenFunc: [](CodeGenFunction &CGF, const Expr *E) {
6156 return EmitLValueOrThrowExpression(CGF, Operand: E);
6157 });
6158
6159 if ((Info.LHS && !Info.LHS->isSimple()) ||
6160 (Info.RHS && !Info.RHS->isSimple()))
6161 return EmitUnsupportedLValue(E: expr, Name: "conditional operator");
6162
6163 if (Info.LHS && Info.RHS) {
6164 Address lhsAddr = Info.LHS->getAddress();
6165 Address rhsAddr = Info.RHS->getAddress();
6166 Address result = mergeAddressesInConditionalExpr(
6167 LHS: lhsAddr, RHS: rhsAddr, LHSBlock: Info.lhsBlock, RHSBlock: Info.rhsBlock,
6168 MergeBlock: Builder.GetInsertBlock(), MergedType: expr->getType());
6169 AlignmentSource alignSource =
6170 std::max(a: Info.LHS->getBaseInfo().getAlignmentSource(),
6171 b: Info.RHS->getBaseInfo().getAlignmentSource());
6172 TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForConditionalOperator(
6173 InfoA: Info.LHS->getTBAAInfo(), InfoB: Info.RHS->getTBAAInfo());
6174 return MakeAddrLValue(Addr: result, T: expr->getType(), BaseInfo: LValueBaseInfo(alignSource),
6175 TBAAInfo);
6176 } else {
6177 assert((Info.LHS || Info.RHS) &&
6178 "both operands of glvalue conditional are throw-expressions?");
6179 return Info.LHS ? *Info.LHS : *Info.RHS;
6180 }
6181}
6182
6183/// EmitCastLValue - Casts are never lvalues unless that cast is to a reference
6184/// type. If the cast is to a reference, we can have the usual lvalue result,
6185/// otherwise if a cast is needed by the code generator in an lvalue context,
6186/// then it must mean that we need the address of an aggregate in order to
6187/// access one of its members. This can happen for all the reasons that casts
6188/// are permitted with aggregate result, including noop aggregate casts, and
6189/// cast from scalar to union.
6190LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
6191 llvm::scope_exit RestoreCurCast([this, Prev = CurCast] { CurCast = Prev; });
6192 CurCast = E;
6193 switch (E->getCastKind()) {
6194 case CK_ToVoid:
6195 case CK_BitCast:
6196 case CK_LValueToRValueBitCast:
6197 case CK_ArrayToPointerDecay:
6198 case CK_FunctionToPointerDecay:
6199 case CK_NullToMemberPointer:
6200 case CK_NullToPointer:
6201 case CK_IntegralToPointer:
6202 case CK_PointerToIntegral:
6203 case CK_PointerToBoolean:
6204 case CK_IntegralCast:
6205 case CK_BooleanToSignedIntegral:
6206 case CK_IntegralToBoolean:
6207 case CK_IntegralToFloating:
6208 case CK_FloatingToIntegral:
6209 case CK_FloatingToBoolean:
6210 case CK_FloatingCast:
6211 case CK_FloatingRealToComplex:
6212 case CK_FloatingComplexToReal:
6213 case CK_FloatingComplexToBoolean:
6214 case CK_FloatingComplexCast:
6215 case CK_FloatingComplexToIntegralComplex:
6216 case CK_IntegralRealToComplex:
6217 case CK_IntegralComplexToReal:
6218 case CK_IntegralComplexToBoolean:
6219 case CK_IntegralComplexCast:
6220 case CK_IntegralComplexToFloatingComplex:
6221 case CK_DerivedToBaseMemberPointer:
6222 case CK_BaseToDerivedMemberPointer:
6223 case CK_MemberPointerToBoolean:
6224 case CK_ReinterpretMemberPointer:
6225 case CK_AnyPointerToBlockPointerCast:
6226 case CK_ARCProduceObject:
6227 case CK_ARCConsumeObject:
6228 case CK_ARCReclaimReturnedObject:
6229 case CK_ARCExtendBlockObject:
6230 case CK_CopyAndAutoreleaseBlockObject:
6231 case CK_IntToOCLSampler:
6232 case CK_FloatingToFixedPoint:
6233 case CK_FixedPointToFloating:
6234 case CK_FixedPointCast:
6235 case CK_FixedPointToBoolean:
6236 case CK_FixedPointToIntegral:
6237 case CK_IntegralToFixedPoint:
6238 case CK_MatrixCast:
6239 case CK_HLSLVectorTruncation:
6240 case CK_HLSLMatrixTruncation:
6241 case CK_HLSLArrayRValue:
6242 case CK_HLSLElementwiseCast:
6243 case CK_HLSLAggregateSplatCast:
6244 return EmitUnsupportedLValue(E, Name: "unexpected cast lvalue");
6245
6246 case CK_Dependent:
6247 llvm_unreachable("dependent cast kind in IR gen!");
6248
6249 case CK_BuiltinFnToFnPtr:
6250 llvm_unreachable("builtin functions are handled elsewhere");
6251
6252 // These are never l-values; just use the aggregate emission code.
6253 case CK_NonAtomicToAtomic:
6254 case CK_AtomicToNonAtomic:
6255 return EmitAggExprToLValue(E);
6256
6257 case CK_Dynamic: {
6258 LValue LV = EmitLValue(E: E->getSubExpr());
6259 Address V = LV.getAddress();
6260 const auto *DCE = cast<CXXDynamicCastExpr>(Val: E);
6261 return MakeNaturalAlignRawAddrLValue(V: EmitDynamicCast(V, DCE), T: E->getType());
6262 }
6263
6264 case CK_ConstructorConversion:
6265 case CK_UserDefinedConversion:
6266 case CK_CPointerToObjCPointerCast:
6267 case CK_BlockPointerToObjCPointerCast:
6268 case CK_LValueToRValue:
6269 return EmitLValue(E: E->getSubExpr());
6270
6271 case CK_NoOp: {
6272 // CK_NoOp can model a qualification conversion, which can remove an array
6273 // bound and change the IR type.
6274 // FIXME: Once pointee types are removed from IR, remove this.
6275 LValue LV = EmitLValue(E: E->getSubExpr());
6276 // Propagate the volatile qualifer to LValue, if exist in E.
6277 if (E->changesVolatileQualification())
6278 LV.getQuals() = E->getType().getQualifiers();
6279 if (LV.isSimple()) {
6280 Address V = LV.getAddress();
6281 if (V.isValid()) {
6282 llvm::Type *T = ConvertTypeForMem(T: E->getType());
6283 if (V.getElementType() != T)
6284 LV.setAddress(V.withElementType(ElemTy: T));
6285 }
6286 }
6287 return LV;
6288 }
6289
6290 case CK_UncheckedDerivedToBase:
6291 case CK_DerivedToBase: {
6292 auto *DerivedClassDecl = E->getSubExpr()->getType()->castAsCXXRecordDecl();
6293 LValue LV = EmitLValue(E: E->getSubExpr());
6294 Address This = LV.getAddress();
6295
6296 // Perform the derived-to-base conversion
6297 Address Base = GetAddressOfBaseClass(
6298 Value: This, Derived: DerivedClassDecl, PathBegin: E->path_begin(), PathEnd: E->path_end(),
6299 /*NullCheckValue=*/false, Loc: E->getExprLoc());
6300
6301 // TODO: Support accesses to members of base classes in TBAA. For now, we
6302 // conservatively pretend that the complete object is of the base class
6303 // type.
6304 return MakeAddrLValue(Addr: Base, T: E->getType(), BaseInfo: LV.getBaseInfo(),
6305 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
6306 }
6307 case CK_ToUnion:
6308 return EmitAggExprToLValue(E);
6309 case CK_BaseToDerived: {
6310 auto *DerivedClassDecl = E->getType()->castAsCXXRecordDecl();
6311 LValue LV = EmitLValue(E: E->getSubExpr());
6312
6313 // Perform the base-to-derived conversion
6314 Address Derived = GetAddressOfDerivedClass(
6315 Value: LV.getAddress(), Derived: DerivedClassDecl, PathBegin: E->path_begin(), PathEnd: E->path_end(),
6316 /*NullCheckValue=*/false);
6317
6318 // C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is
6319 // performed and the object is not of the derived type.
6320 if (sanitizePerformTypeCheck())
6321 EmitTypeCheck(TCK: TCK_DowncastReference, Loc: E->getExprLoc(), Addr: Derived,
6322 Type: E->getType());
6323
6324 if (SanOpts.has(K: SanitizerKind::CFIDerivedCast))
6325 EmitVTablePtrCheckForCast(T: E->getType(), Derived,
6326 /*MayBeNull=*/false, TCK: CFITCK_DerivedCast,
6327 Loc: E->getBeginLoc());
6328
6329 return MakeAddrLValue(Addr: Derived, T: E->getType(), BaseInfo: LV.getBaseInfo(),
6330 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
6331 }
6332 case CK_LValueBitCast: {
6333 // This must be a reinterpret_cast (or c-style equivalent).
6334 const auto *CE = cast<ExplicitCastExpr>(Val: E);
6335
6336 CGM.EmitExplicitCastExprType(E: CE, CGF: this);
6337 LValue LV = EmitLValue(E: E->getSubExpr());
6338 Address V = LV.getAddress().withElementType(
6339 ElemTy: ConvertTypeForMem(T: CE->getTypeAsWritten()->getPointeeType()));
6340
6341 if (SanOpts.has(K: SanitizerKind::CFIUnrelatedCast))
6342 EmitVTablePtrCheckForCast(T: E->getType(), Derived: V,
6343 /*MayBeNull=*/false, TCK: CFITCK_UnrelatedCast,
6344 Loc: E->getBeginLoc());
6345
6346 return MakeAddrLValue(Addr: V, T: E->getType(), BaseInfo: LV.getBaseInfo(),
6347 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
6348 }
6349 case CK_AddressSpaceConversion: {
6350 LValue LV = EmitLValue(E: E->getSubExpr());
6351 QualType DestTy = getContext().getPointerType(T: E->getType());
6352 llvm::Value *V =
6353 performAddrSpaceCast(Src: LV.getPointer(CGF&: *this), DestTy: ConvertType(T: DestTy));
6354 return MakeAddrLValue(Addr: Address(V, ConvertTypeForMem(T: E->getType()),
6355 LV.getAddress().getAlignment()),
6356 T: E->getType(), BaseInfo: LV.getBaseInfo(), TBAAInfo: LV.getTBAAInfo());
6357 }
6358 case CK_ObjCObjectLValueCast: {
6359 LValue LV = EmitLValue(E: E->getSubExpr());
6360 Address V = LV.getAddress().withElementType(ElemTy: ConvertType(T: E->getType()));
6361 return MakeAddrLValue(Addr: V, T: E->getType(), BaseInfo: LV.getBaseInfo(),
6362 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
6363 }
6364 case CK_ZeroToOCLOpaqueType:
6365 llvm_unreachable("NULL to OpenCL opaque type lvalue cast is not valid");
6366
6367 case CK_VectorSplat: {
6368 // LValue results of vector splats are only supported in HLSL.
6369 if (!getLangOpts().HLSL)
6370 return EmitUnsupportedLValue(E, Name: "unexpected cast lvalue");
6371 return EmitLValue(E: E->getSubExpr());
6372 }
6373 }
6374
6375 llvm_unreachable("Unhandled lvalue cast kind?");
6376}
6377
6378LValue CodeGenFunction::EmitOpaqueValueLValue(const OpaqueValueExpr *e) {
6379 assert(OpaqueValueMappingData::shouldBindAsLValue(e));
6380 return getOrCreateOpaqueLValueMapping(e);
6381}
6382
6383std::pair<LValue, LValue>
6384CodeGenFunction::EmitHLSLOutArgLValues(const HLSLOutArgExpr *E, QualType Ty) {
6385 // Emitting the casted temporary through an opaque value.
6386 LValue BaseLV = EmitLValue(E: E->getArgLValue());
6387 OpaqueValueMappingData::bind(CGF&: *this, ov: E->getOpaqueArgLValue(), lv: BaseLV);
6388
6389 QualType ExprTy = E->getType();
6390 Address OutTemp = CreateIRTempWithoutCast(Ty: ExprTy);
6391 LValue TempLV = MakeAddrLValue(Addr: OutTemp, T: ExprTy);
6392
6393 // Start the lifetime before the copy-in so that the temporary is live when
6394 // the initial value is written. This ensures the store is within the
6395 // lifetime and is not killed by a store undef inserted at lifetime.start.
6396 EmitLifetimeStart(Addr: OutTemp.getBasePointer());
6397
6398 if (E->isInOut())
6399 EmitInitializationToLValue(E: E->getCastedTemporary()->getSourceExpr(),
6400 LV: TempLV);
6401
6402 OpaqueValueMappingData::bind(CGF&: *this, ov: E->getCastedTemporary(), lv: TempLV);
6403 return std::make_pair(x&: BaseLV, y&: TempLV);
6404}
6405
6406LValue CodeGenFunction::EmitHLSLOutArgExpr(const HLSLOutArgExpr *E,
6407 CallArgList &Args, QualType Ty) {
6408
6409 auto [BaseLV, TempLV] = EmitHLSLOutArgLValues(E, Ty);
6410
6411 llvm::Value *Addr = TempLV.getAddress().getBasePointer();
6412 llvm::Type *ElTy = ConvertTypeForMem(T: TempLV.getType());
6413
6414 Address TmpAddr(Addr, ElTy, TempLV.getAlignment());
6415 Args.addWriteback(srcLV: BaseLV, temporary: TmpAddr, toUse: nullptr, writebackExpr: E->getWritebackCast());
6416 Args.add(rvalue: RValue::get(Addr: TmpAddr, CGF&: *this), type: Ty);
6417 return TempLV;
6418}
6419
6420LValue
6421CodeGenFunction::getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e) {
6422 assert(OpaqueValueMapping::shouldBindAsLValue(e));
6423
6424 llvm::DenseMap<const OpaqueValueExpr*,LValue>::iterator
6425 it = OpaqueLValues.find(Val: e);
6426
6427 if (it != OpaqueLValues.end())
6428 return it->second;
6429
6430 assert(e->isUnique() && "LValue for a nonunique OVE hasn't been emitted");
6431 return EmitLValue(E: e->getSourceExpr());
6432}
6433
6434RValue
6435CodeGenFunction::getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e) {
6436 assert(!OpaqueValueMapping::shouldBindAsLValue(e));
6437
6438 llvm::DenseMap<const OpaqueValueExpr*,RValue>::iterator
6439 it = OpaqueRValues.find(Val: e);
6440
6441 if (it != OpaqueRValues.end())
6442 return it->second;
6443
6444 assert(e->isUnique() && "RValue for a nonunique OVE hasn't been emitted");
6445 return EmitAnyExpr(E: e->getSourceExpr());
6446}
6447
6448bool CodeGenFunction::isOpaqueValueEmitted(const OpaqueValueExpr *E) {
6449 if (OpaqueValueMapping::shouldBindAsLValue(expr: E))
6450 return OpaqueLValues.contains(Val: E);
6451 return OpaqueRValues.contains(Val: E);
6452}
6453
6454RValue CodeGenFunction::EmitRValueForField(LValue LV,
6455 const FieldDecl *FD,
6456 SourceLocation Loc) {
6457 QualType FT = FD->getType();
6458 LValue FieldLV = EmitLValueForField(base: LV, field: FD);
6459 switch (getEvaluationKind(T: FT)) {
6460 case TEK_Complex:
6461 return RValue::getComplex(C: EmitLoadOfComplex(src: FieldLV, loc: Loc));
6462 case TEK_Aggregate:
6463 return FieldLV.asAggregateRValue();
6464 case TEK_Scalar:
6465 // This routine is used to load fields one-by-one to perform a copy, so
6466 // don't load reference fields.
6467 if (FD->getType()->isReferenceType())
6468 return RValue::get(V: FieldLV.getPointer(CGF&: *this));
6469 // Call EmitLoadOfScalar except when the lvalue is a bitfield to emit a
6470 // primitive load.
6471 if (FieldLV.isBitField())
6472 return EmitLoadOfLValue(LV: FieldLV, Loc);
6473 return RValue::get(V: EmitLoadOfScalar(lvalue: FieldLV, Loc));
6474 }
6475 llvm_unreachable("bad evaluation kind");
6476}
6477
6478//===--------------------------------------------------------------------===//
6479// Expression Emission
6480//===--------------------------------------------------------------------===//
6481
6482RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
6483 ReturnValueSlot ReturnValue,
6484 llvm::CallBase **CallOrInvoke) {
6485 llvm::CallBase *CallOrInvokeStorage;
6486 if (!CallOrInvoke) {
6487 CallOrInvoke = &CallOrInvokeStorage;
6488 }
6489
6490 llvm::scope_exit AddCoroElideSafeOnExit([&] {
6491 if (E->isCoroElideSafe()) {
6492 auto *I = *CallOrInvoke;
6493 if (I)
6494 I->addFnAttr(Kind: llvm::Attribute::CoroElideSafe);
6495 }
6496 });
6497
6498 // Builtins never have block type.
6499 if (E->getCallee()->getType()->isBlockPointerType())
6500 return EmitBlockCallExpr(E, ReturnValue, CallOrInvoke);
6501
6502 if (const auto *CE = dyn_cast<CXXMemberCallExpr>(Val: E))
6503 return EmitCXXMemberCallExpr(E: CE, ReturnValue, CallOrInvoke);
6504
6505 if (const auto *CE = dyn_cast<CUDAKernelCallExpr>(Val: E))
6506 return EmitCUDAKernelCallExpr(E: CE, ReturnValue, CallOrInvoke);
6507
6508 // A CXXOperatorCallExpr is created even for explicit object methods, but
6509 // these should be treated like static function call.
6510 if (const auto *CE = dyn_cast<CXXOperatorCallExpr>(Val: E))
6511 if (const auto *MD =
6512 dyn_cast_if_present<CXXMethodDecl>(Val: CE->getCalleeDecl());
6513 MD && MD->isImplicitObjectMemberFunction())
6514 return EmitCXXOperatorMemberCallExpr(E: CE, MD, ReturnValue, CallOrInvoke);
6515
6516 CGCallee callee = EmitCallee(E: E->getCallee());
6517
6518 if (callee.isBuiltin()) {
6519 return EmitBuiltinExpr(GD: callee.getBuiltinDecl(), BuiltinID: callee.getBuiltinID(),
6520 E, ReturnValue);
6521 }
6522
6523 if (callee.isPseudoDestructor()) {
6524 return EmitCXXPseudoDestructorExpr(E: callee.getPseudoDestructorExpr());
6525 }
6526
6527 return EmitCall(FnType: E->getCallee()->getType(), Callee: callee, E, ReturnValue,
6528 /*Chain=*/nullptr, CallOrInvoke);
6529}
6530
6531/// Emit a CallExpr without considering whether it might be a subclass.
6532RValue CodeGenFunction::EmitSimpleCallExpr(const CallExpr *E,
6533 ReturnValueSlot ReturnValue,
6534 llvm::CallBase **CallOrInvoke) {
6535 CGCallee Callee = EmitCallee(E: E->getCallee());
6536 return EmitCall(FnType: E->getCallee()->getType(), Callee, E, ReturnValue,
6537 /*Chain=*/nullptr, CallOrInvoke);
6538}
6539
6540// Detect the unusual situation where an inline version is shadowed by a
6541// non-inline version. In that case we should pick the external one
6542// everywhere. That's GCC behavior too.
6543static bool OnlyHasInlineBuiltinDeclaration(const FunctionDecl *FD) {
6544 for (const FunctionDecl *PD = FD; PD; PD = PD->getPreviousDecl())
6545 if (!PD->isInlineBuiltinDeclaration())
6546 return false;
6547 return true;
6548}
6549
6550static CGCallee EmitDirectCallee(CodeGenFunction &CGF, GlobalDecl GD) {
6551 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
6552
6553 if (auto builtinID = FD->getBuiltinID()) {
6554 std::string NoBuiltinFD = ("no-builtin-" + FD->getName()).str();
6555 std::string NoBuiltins = "no-builtins";
6556
6557 StringRef Ident = CGF.CGM.getMangledName(GD);
6558 std::string FDInlineName = (Ident + ".inline").str();
6559
6560 bool IsPredefinedLibFunction =
6561 CGF.getContext().BuiltinInfo.isPredefinedLibFunction(ID: builtinID);
6562 bool HasAttributeNoBuiltin =
6563 CGF.CurFn->getAttributes().hasFnAttr(Kind: NoBuiltinFD) ||
6564 CGF.CurFn->getAttributes().hasFnAttr(Kind: NoBuiltins);
6565
6566 // When directing calling an inline builtin, call it through it's mangled
6567 // name to make it clear it's not the actual builtin.
6568 if (CGF.CurFn->getName() != FDInlineName &&
6569 OnlyHasInlineBuiltinDeclaration(FD)) {
6570 llvm::Constant *CalleePtr = CGF.CGM.getRawFunctionPointer(GD);
6571 llvm::Function *Fn = llvm::cast<llvm::Function>(Val: CalleePtr);
6572 llvm::Module *M = Fn->getParent();
6573 llvm::Function *Clone = M->getFunction(Name: FDInlineName);
6574 if (!Clone) {
6575 Clone = llvm::Function::Create(Ty: Fn->getFunctionType(),
6576 Linkage: llvm::GlobalValue::InternalLinkage,
6577 AddrSpace: Fn->getAddressSpace(), N: FDInlineName, M);
6578 Clone->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
6579 }
6580 return CGCallee::forDirect(functionPtr: Clone, abstractInfo: GD);
6581 }
6582
6583 // Replaceable builtins provide their own implementation of a builtin. If we
6584 // are in an inline builtin implementation, avoid trivial infinite
6585 // recursion. Honor __attribute__((no_builtin("foo"))) or
6586 // __attribute__((no_builtin)) on the current function unless foo is
6587 // not a predefined library function which means we must generate the
6588 // builtin no matter what.
6589 else if (!IsPredefinedLibFunction || !HasAttributeNoBuiltin)
6590 return CGCallee::forBuiltin(builtinID, builtinDecl: FD);
6591 }
6592
6593 llvm::Constant *CalleePtr = CGF.CGM.getRawFunctionPointer(GD);
6594 if (CGF.CGM.getLangOpts().CUDA && !CGF.CGM.getLangOpts().CUDAIsDevice &&
6595 FD->hasAttr<CUDAGlobalAttr>())
6596 CalleePtr = CGF.CGM.getCUDARuntime().getKernelStub(
6597 Handle: cast<llvm::GlobalValue>(Val: CalleePtr->stripPointerCasts()));
6598
6599 return CGCallee::forDirect(functionPtr: CalleePtr, abstractInfo: GD);
6600}
6601
6602static GlobalDecl getGlobalDeclForDirectCall(const FunctionDecl *FD) {
6603 if (DeviceKernelAttr::isOpenCLSpelling(A: FD->getAttr<DeviceKernelAttr>()))
6604 return GlobalDecl(FD, KernelReferenceKind::Stub);
6605 return GlobalDecl(FD);
6606}
6607
6608CGCallee CodeGenFunction::EmitCallee(const Expr *E) {
6609 E = E->IgnoreParens();
6610
6611 // A WebAssembly funcref is an opaque reference type and llvm only accepts
6612 // function pointers as the call target. To make an indirect call through a
6613 // reference type, first use the llvm.wasm.funcref.to_ptr intrinsic to make a
6614 // fake function pointer to it. The backend lowers the resulting indirect call
6615 // to a table.set into a single element dummy table + call_indirect 0.
6616 auto ConvertFuncrefToPtr = [&](llvm::Value *CalleePtr) -> llvm::Value * {
6617 if (auto *TET = dyn_cast<llvm::TargetExtType>(Val: CalleePtr->getType());
6618 TET && TET->getName() == "wasm.funcref") {
6619 llvm::Function *ToPtr =
6620 CGM.getIntrinsic(IID: llvm::Intrinsic::wasm_funcref_to_ptr);
6621 return Builder.CreateCall(Callee: ToPtr, Args: {CalleePtr});
6622 }
6623 return CalleePtr;
6624 };
6625
6626 // Look through function-to-pointer decay.
6627 if (auto ICE = dyn_cast<ImplicitCastExpr>(Val: E)) {
6628 if (ICE->getCastKind() == CK_FunctionToPointerDecay ||
6629 ICE->getCastKind() == CK_BuiltinFnToFnPtr) {
6630 return EmitCallee(E: ICE->getSubExpr());
6631 }
6632
6633 // Try to remember the original __ptrauth qualifier for loads of
6634 // function pointers.
6635 if (ICE->getCastKind() == CK_LValueToRValue) {
6636 const Expr *SubExpr = ICE->getSubExpr();
6637 if (const auto *PtrType = SubExpr->getType()->getAs<PointerType>()) {
6638 std::pair<llvm::Value *, CGPointerAuthInfo> Result =
6639 EmitOrigPointerRValue(E);
6640
6641 QualType FunctionType = PtrType->getPointeeType();
6642 assert(FunctionType->isFunctionType());
6643
6644 GlobalDecl GD;
6645 if (const auto *VD =
6646 dyn_cast_or_null<VarDecl>(Val: E->getReferencedDeclOfCallee())) {
6647 GD = GlobalDecl(VD);
6648 }
6649 CGCalleeInfo CalleeInfo(FunctionType->getAs<FunctionProtoType>(), GD);
6650 CGCallee Callee(CalleeInfo, ConvertFuncrefToPtr(Result.first),
6651 Result.second);
6652 return Callee;
6653 }
6654 }
6655
6656 // Resolve direct calls.
6657 } else if (auto DRE = dyn_cast<DeclRefExpr>(Val: E)) {
6658 if (auto FD = dyn_cast<FunctionDecl>(Val: DRE->getDecl())) {
6659 return EmitDirectCallee(CGF&: *this, GD: getGlobalDeclForDirectCall(FD));
6660 }
6661 } else if (auto ME = dyn_cast<MemberExpr>(Val: E)) {
6662 if (auto FD = dyn_cast<FunctionDecl>(Val: ME->getMemberDecl())) {
6663 EmitIgnoredExpr(E: ME->getBase());
6664 return EmitDirectCallee(CGF&: *this, GD: FD);
6665 }
6666
6667 // Look through template substitutions.
6668 } else if (auto NTTP = dyn_cast<SubstNonTypeTemplateParmExpr>(Val: E)) {
6669 return EmitCallee(E: NTTP->getReplacement());
6670
6671 // Treat pseudo-destructor calls differently.
6672 } else if (auto PDE = dyn_cast<CXXPseudoDestructorExpr>(Val: E)) {
6673 return CGCallee::forPseudoDestructor(E: PDE);
6674 }
6675
6676 // Otherwise, we have an indirect reference.
6677 llvm::Value *calleePtr;
6678 QualType functionType;
6679 if (auto ptrType = E->getType()->getAs<PointerType>()) {
6680 calleePtr = EmitScalarExpr(E);
6681 functionType = ptrType->getPointeeType();
6682 } else {
6683 functionType = E->getType();
6684 calleePtr = EmitLValue(E, IsKnownNonNull: KnownNonNull).getPointer(CGF&: *this);
6685 }
6686 assert(functionType->isFunctionType());
6687
6688 GlobalDecl GD;
6689 if (const auto *VD =
6690 dyn_cast_or_null<VarDecl>(Val: E->getReferencedDeclOfCallee()))
6691 GD = GlobalDecl(VD);
6692
6693 CGCalleeInfo calleeInfo(functionType->getAs<FunctionProtoType>(), GD);
6694 CGPointerAuthInfo pointerAuth = CGM.getFunctionPointerAuthInfo(T: functionType);
6695 CGCallee callee(calleeInfo, ConvertFuncrefToPtr(calleePtr), pointerAuth);
6696 return callee;
6697}
6698
6699LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
6700 // Comma expressions just emit their LHS then their RHS as an l-value.
6701 if (E->getOpcode() == BO_Comma) {
6702 EmitIgnoredExpr(E: E->getLHS());
6703 EnsureInsertPoint();
6704 return EmitLValue(E: E->getRHS());
6705 }
6706
6707 if (E->getOpcode() == BO_PtrMemD ||
6708 E->getOpcode() == BO_PtrMemI)
6709 return EmitPointerToDataMemberBinaryExpr(E);
6710
6711 assert(E->getOpcode() == BO_Assign && "unexpected binary l-value");
6712
6713 // Create a Key Instructions source location atom group that covers both
6714 // LHS and RHS expressions. Nested RHS expressions may get subsequently
6715 // separately grouped (1 below):
6716 //
6717 // 1. `a = b = c` -> Two atoms.
6718 // 2. `x = new(1)` -> One atom (for both addr store and value store).
6719 // 3. Complex and agg assignment -> One atom.
6720 ApplyAtomGroup Grp(getDebugInfo());
6721
6722 // Note that in all of these cases, __block variables need the RHS
6723 // evaluated first just in case the variable gets moved by the RHS.
6724
6725 switch (getEvaluationKind(T: E->getType())) {
6726 case TEK_Scalar: {
6727 if (PointerAuthQualifier PtrAuth =
6728 E->getLHS()->getType().getPointerAuth()) {
6729 LValue LV = EmitCheckedLValue(E: E->getLHS(), TCK: TCK_Store);
6730 LValue CopiedLV = LV;
6731 CopiedLV.getQuals().removePointerAuth();
6732 llvm::Value *RV =
6733 EmitPointerAuthQualify(Qualifier: PtrAuth, PointerExpr: E->getRHS(), StorageAddress: CopiedLV.getAddress());
6734 EmitNullabilityCheck(LHS: CopiedLV, RHS: RV, Loc: E->getExprLoc());
6735 EmitStoreThroughLValue(Src: RValue::get(V: RV), Dst: CopiedLV);
6736 return LV;
6737 }
6738
6739 switch (E->getLHS()->getType().getObjCLifetime()) {
6740 case Qualifiers::OCL_Strong:
6741 return EmitARCStoreStrong(e: E, /*ignored*/ false).first;
6742
6743 case Qualifiers::OCL_Autoreleasing:
6744 return EmitARCStoreAutoreleasing(e: E).first;
6745
6746 // No reason to do any of these differently.
6747 case Qualifiers::OCL_None:
6748 case Qualifiers::OCL_ExplicitNone:
6749 case Qualifiers::OCL_Weak:
6750 break;
6751 }
6752
6753 // TODO: Can we de-duplicate this code with the corresponding code in
6754 // CGExprScalar, similar to the way EmitCompoundAssignmentLValue works?
6755 RValue RV;
6756 llvm::Value *Previous = nullptr;
6757 QualType SrcType = E->getRHS()->getType();
6758 // Check if LHS is a bitfield, if RHS contains an implicit cast expression
6759 // we want to extract that value and potentially (if the bitfield sanitizer
6760 // is enabled) use it to check for an implicit conversion.
6761 if (E->getLHS()->refersToBitField()) {
6762 llvm::Value *RHS =
6763 EmitWithOriginalRHSBitfieldAssignment(E, Previous: &Previous, SrcType: &SrcType);
6764 RV = RValue::get(V: RHS);
6765 } else
6766 RV = EmitAnyExpr(E: E->getRHS());
6767
6768 LValue LV = EmitCheckedLValue(E: E->getLHS(), TCK: TCK_Store);
6769
6770 if (RV.isScalar())
6771 EmitNullabilityCheck(LHS: LV, RHS: RV.getScalarVal(), Loc: E->getExprLoc());
6772
6773 if (LV.isBitField()) {
6774 llvm::Value *Result = nullptr;
6775 // If bitfield sanitizers are enabled we want to use the result
6776 // to check whether a truncation or sign change has occurred.
6777 if (SanOpts.has(K: SanitizerKind::ImplicitBitfieldConversion))
6778 EmitStoreThroughBitfieldLValue(Src: RV, Dst: LV, Result: &Result);
6779 else
6780 EmitStoreThroughBitfieldLValue(Src: RV, Dst: LV);
6781
6782 // If the expression contained an implicit conversion, make sure
6783 // to use the value before the scalar conversion.
6784 llvm::Value *Src = Previous ? Previous : RV.getScalarVal();
6785 QualType DstType = E->getLHS()->getType();
6786 EmitBitfieldConversionCheck(Src, SrcType, Dst: Result, DstType,
6787 Info: LV.getBitFieldInfo(), Loc: E->getExprLoc());
6788 } else
6789 EmitStoreThroughLValue(Src: RV, Dst: LV);
6790
6791 if (getLangOpts().OpenMP)
6792 CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF&: *this,
6793 LHS: E->getLHS());
6794 return LV;
6795 }
6796
6797 case TEK_Complex:
6798 return EmitComplexAssignmentLValue(E);
6799
6800 case TEK_Aggregate:
6801 // If the lang opt is HLSL and the LHS is a constant array
6802 // then we are performing a copy assignment and call a special
6803 // function because EmitAggExprToLValue emits to a temporary LValue
6804 if (getLangOpts().HLSL && E->getLHS()->getType()->isConstantArrayType())
6805 return EmitHLSLArrayAssignLValue(E);
6806
6807 return EmitAggExprToLValue(E);
6808 }
6809 llvm_unreachable("bad evaluation kind");
6810}
6811
6812// This function implements trivial copy assignment for HLSL's
6813// assignable constant arrays.
6814LValue CodeGenFunction::EmitHLSLArrayAssignLValue(const BinaryOperator *E) {
6815 // Don't emit an LValue for the RHS because it might not be an LValue
6816 LValue LHS = EmitLValue(E: E->getLHS());
6817
6818 // If the RHS is a global resource array, copy all individual resources
6819 // into LHS.
6820 if (E->getRHS()->getType()->isHLSLResourceRecordArray()) {
6821 AggValueSlot Slot = AggValueSlot::forAddr(
6822 addr: LHS.getAddress(), quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed_t(true),
6823 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsAliased_t(false),
6824 mayOverlap: AggValueSlot::DoesNotOverlap);
6825 if (CGM.getHLSLRuntime().emitGlobalResourceArray(CGF&: *this, E: E->getRHS(), DestSlot&: Slot))
6826 return LHS;
6827 }
6828
6829 // In C the RHS of an assignment operator is an RValue.
6830 // EmitAggregateAssign takes an LValue for the RHS. Instead we can call
6831 // EmitInitializationToLValue to emit an RValue into an LValue.
6832 EmitInitializationToLValue(E: E->getRHS(), LV: LHS);
6833 return LHS;
6834}
6835
6836LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E,
6837 llvm::CallBase **CallOrInvoke) {
6838 RValue RV = EmitCallExpr(E, ReturnValue: ReturnValueSlot(), CallOrInvoke);
6839
6840 if (!RV.isScalar())
6841 return MakeAddrLValue(Addr: RV.getAggregateAddress(), T: E->getType(),
6842 Source: AlignmentSource::Decl);
6843
6844 assert(E->getCallReturnType(getContext())->isReferenceType() &&
6845 "Can't have a scalar return unless the return type is a "
6846 "reference type!");
6847
6848 return MakeNaturalAlignPointeeAddrLValue(V: RV.getScalarVal(), T: E->getType());
6849}
6850
6851LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {
6852 // FIXME: This shouldn't require another copy.
6853 return EmitAggExprToLValue(E);
6854}
6855
6856LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
6857 assert(E->getType()->getAsCXXRecordDecl()->hasTrivialDestructor()
6858 && "binding l-value to type which needs a temporary");
6859 AggValueSlot Slot = CreateAggTemp(T: E->getType());
6860 EmitCXXConstructExpr(E, Dest: Slot);
6861 return MakeAddrLValue(Addr: Slot.getAddress(), T: E->getType(), Source: AlignmentSource::Decl);
6862}
6863
6864LValue
6865CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
6866 return MakeNaturalAlignRawAddrLValue(V: EmitCXXTypeidExpr(E), T: E->getType());
6867}
6868
6869Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) {
6870 return CGM.GetAddrOfMSGuidDecl(GD: E->getGuidDecl())
6871 .withElementType(ElemTy: ConvertType(T: E->getType()));
6872}
6873
6874LValue CodeGenFunction::EmitCXXUuidofLValue(const CXXUuidofExpr *E) {
6875 return MakeAddrLValue(Addr: EmitCXXUuidofExpr(E), T: E->getType(),
6876 Source: AlignmentSource::Decl);
6877}
6878
6879LValue
6880CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) {
6881 AggValueSlot Slot = CreateAggTemp(T: E->getType(), Name: "temp.lvalue");
6882 Slot.setExternallyDestructed();
6883 EmitAggExpr(E: E->getSubExpr(), AS: Slot);
6884 EmitCXXTemporary(Temporary: E->getTemporary(), TempType: E->getType(), Ptr: Slot.getAddress());
6885 return MakeAddrLValue(Addr: Slot.getAddress(), T: E->getType(), Source: AlignmentSource::Decl);
6886}
6887
6888LValue CodeGenFunction::EmitObjCMessageExprLValue(const ObjCMessageExpr *E) {
6889 RValue RV = EmitObjCMessageExpr(E);
6890
6891 if (!RV.isScalar())
6892 return MakeAddrLValue(Addr: RV.getAggregateAddress(), T: E->getType(),
6893 Source: AlignmentSource::Decl);
6894
6895 assert(E->getMethodDecl()->getReturnType()->isReferenceType() &&
6896 "Can't have a scalar return unless the return type is a "
6897 "reference type!");
6898
6899 return MakeNaturalAlignPointeeAddrLValue(V: RV.getScalarVal(), T: E->getType());
6900}
6901
6902LValue CodeGenFunction::EmitObjCSelectorLValue(const ObjCSelectorExpr *E) {
6903 Address V =
6904 CGM.getObjCRuntime().GetAddrOfSelector(CGF&: *this, Sel: E->getSelector());
6905 return MakeAddrLValue(Addr: V, T: E->getType(), Source: AlignmentSource::Decl);
6906}
6907
6908llvm::Value *CodeGenFunction::EmitIvarOffset(const ObjCInterfaceDecl *Interface,
6909 const ObjCIvarDecl *Ivar) {
6910 return CGM.getObjCRuntime().EmitIvarOffset(CGF&: *this, Interface, Ivar);
6911}
6912
6913llvm::Value *
6914CodeGenFunction::EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,
6915 const ObjCIvarDecl *Ivar) {
6916 llvm::Value *OffsetValue = EmitIvarOffset(Interface, Ivar);
6917 QualType PointerDiffType = getContext().getPointerDiffType();
6918 return Builder.CreateZExtOrTrunc(V: OffsetValue,
6919 DestTy: getTypes().ConvertType(T: PointerDiffType));
6920}
6921
6922LValue CodeGenFunction::EmitLValueForIvar(QualType ObjectTy,
6923 llvm::Value *BaseValue,
6924 const ObjCIvarDecl *Ivar,
6925 unsigned CVRQualifiers) {
6926 return CGM.getObjCRuntime().EmitObjCValueForIvar(CGF&: *this, ObjectTy, BaseValue,
6927 Ivar, CVRQualifiers);
6928}
6929
6930LValue CodeGenFunction::EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E) {
6931 // FIXME: A lot of the code below could be shared with EmitMemberExpr.
6932 llvm::Value *BaseValue = nullptr;
6933 const Expr *BaseExpr = E->getBase();
6934 Qualifiers BaseQuals;
6935 QualType ObjectTy;
6936 if (E->isArrow()) {
6937 BaseValue = EmitScalarExpr(E: BaseExpr);
6938 ObjectTy = BaseExpr->getType()->getPointeeType();
6939 BaseQuals = ObjectTy.getQualifiers();
6940 } else {
6941 LValue BaseLV = EmitLValue(E: BaseExpr);
6942 BaseValue = BaseLV.getPointer(CGF&: *this);
6943 ObjectTy = BaseExpr->getType();
6944 BaseQuals = ObjectTy.getQualifiers();
6945 }
6946
6947 LValue LV =
6948 EmitLValueForIvar(ObjectTy, BaseValue, Ivar: E->getDecl(),
6949 CVRQualifiers: BaseQuals.getCVRQualifiers());
6950 setObjCGCLValueClass(Ctx: getContext(), E, LV);
6951 return LV;
6952}
6953
6954LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {
6955 // Can only get l-value for message expression returning aggregate type
6956 RValue RV = EmitAnyExprToTemp(E);
6957 return MakeAddrLValue(Addr: RV.getAggregateAddress(), T: E->getType(),
6958 Source: AlignmentSource::Decl);
6959}
6960
6961RValue CodeGenFunction::EmitCall(QualType CalleeType,
6962 const CGCallee &OrigCallee, const CallExpr *E,
6963 ReturnValueSlot ReturnValue,
6964 llvm::Value *Chain,
6965 llvm::CallBase **CallOrInvoke,
6966 CGFunctionInfo const **ResolvedFnInfo) {
6967 // Get the actual function type. The callee type will always be a pointer to
6968 // function type or a block pointer type.
6969 assert(CalleeType->isFunctionPointerType() &&
6970 "Call must have function pointer type!");
6971
6972 const Decl *TargetDecl =
6973 OrigCallee.getAbstractInfo().getCalleeDecl().getDecl();
6974
6975 assert((!isa_and_present<FunctionDecl>(TargetDecl) ||
6976 !cast<FunctionDecl>(TargetDecl)->isImmediateFunction()) &&
6977 "trying to emit a call to an immediate function");
6978
6979 CalleeType = getContext().getCanonicalType(T: CalleeType);
6980
6981 auto PointeeType = cast<PointerType>(Val&: CalleeType)->getPointeeType();
6982
6983 CGCallee Callee = OrigCallee;
6984
6985 bool CFIUnchecked = CalleeType->hasPointeeToCFIUncheckedCalleeFunctionType();
6986
6987 if (SanOpts.has(K: SanitizerKind::Function) &&
6988 (!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl)) &&
6989 !isa<FunctionNoProtoType>(Val: PointeeType) && !CFIUnchecked) {
6990 if (llvm::Constant *PrefixSig =
6991 CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM)) {
6992 auto CheckOrdinal = SanitizerKind::SO_Function;
6993 auto CheckHandler = SanitizerHandler::FunctionTypeMismatch;
6994 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
6995 auto *TypeHash = getUBSanFunctionTypeHash(T: PointeeType);
6996
6997 llvm::Type *PrefixSigType = PrefixSig->getType();
6998 llvm::StructType *PrefixStructTy = llvm::StructType::get(
6999 Context&: CGM.getLLVMContext(), Elements: {PrefixSigType, Int32Ty}, /*isPacked=*/true);
7000
7001 llvm::Value *CalleePtr = Callee.getFunctionPointer();
7002 if (CGM.getCodeGenOpts().PointerAuth.FunctionPointers) {
7003 // Use raw pointer since we are using the callee pointer as data here.
7004 Address Addr =
7005 Address(CalleePtr, CalleePtr->getType(),
7006 CharUnits::fromQuantity(
7007 Quantity: CalleePtr->getPointerAlignment(DL: CGM.getDataLayout())),
7008 Callee.getPointerAuthInfo(), nullptr);
7009 CalleePtr = Addr.emitRawPointer(CGF&: *this);
7010 }
7011
7012 // On 32-bit Arm, the low bit of a function pointer indicates whether
7013 // it's using the Arm or Thumb instruction set. The actual first
7014 // instruction lives at the same address either way, so we must clear
7015 // that low bit before using the function address to find the prefix
7016 // structure.
7017 //
7018 // This applies to both Arm and Thumb target triples, because
7019 // either one could be used in an interworking context where it
7020 // might be passed function pointers of both types.
7021 llvm::Value *AlignedCalleePtr;
7022 if (CGM.getTriple().isARM() || CGM.getTriple().isThumb()) {
7023 AlignedCalleePtr = Builder.CreateIntrinsic(
7024 RetTy: CalleePtr->getType(), ID: llvm::Intrinsic::ptrmask,
7025 Args: {CalleePtr, llvm::ConstantInt::getSigned(Ty: IntPtrTy, V: ~1)});
7026 } else {
7027 AlignedCalleePtr = CalleePtr;
7028 }
7029
7030 llvm::Value *CalleePrefixStruct = AlignedCalleePtr;
7031 llvm::Value *CalleeSigPtr =
7032 Builder.CreateConstGEP2_32(Ty: PrefixStructTy, Ptr: CalleePrefixStruct, Idx0: -1, Idx1: 0);
7033 llvm::Value *CalleeSig =
7034 Builder.CreateAlignedLoad(Ty: PrefixSigType, Addr: CalleeSigPtr, Align: getIntAlign());
7035 llvm::Value *CalleeSigMatch = Builder.CreateICmpEQ(LHS: CalleeSig, RHS: PrefixSig);
7036
7037 llvm::BasicBlock *Cont = createBasicBlock(name: "cont");
7038 llvm::BasicBlock *TypeCheck = createBasicBlock(name: "typecheck");
7039 Builder.CreateCondBr(Cond: CalleeSigMatch, True: TypeCheck, False: Cont);
7040
7041 EmitBlock(BB: TypeCheck);
7042 llvm::Value *CalleeTypeHash = Builder.CreateAlignedLoad(
7043 Ty: Int32Ty,
7044 Addr: Builder.CreateConstGEP2_32(Ty: PrefixStructTy, Ptr: CalleePrefixStruct, Idx0: -1, Idx1: 1),
7045 Align: getPointerAlign());
7046 llvm::Value *CalleeTypeHashMatch =
7047 Builder.CreateICmpEQ(LHS: CalleeTypeHash, RHS: TypeHash);
7048 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc: E->getBeginLoc()),
7049 EmitCheckTypeDescriptor(T: CalleeType)};
7050 EmitCheck(Checked: std::make_pair(x&: CalleeTypeHashMatch, y&: CheckOrdinal), CheckHandler,
7051 StaticArgs: StaticData, DynamicArgs: {CalleePtr});
7052
7053 Builder.CreateBr(Dest: Cont);
7054 EmitBlock(BB: Cont);
7055 }
7056 }
7057
7058 const auto *FnType = cast<FunctionType>(Val&: PointeeType);
7059
7060 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
7061 FD && DeviceKernelAttr::isOpenCLSpelling(A: FD->getAttr<DeviceKernelAttr>()))
7062 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FnType);
7063
7064 // If we are checking indirect calls and this call is indirect, check that the
7065 // function pointer is a member of the bit set for the function type.
7066 if (SanOpts.has(K: SanitizerKind::CFIICall) &&
7067 (!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl)) && !CFIUnchecked) {
7068 auto CheckOrdinal = SanitizerKind::SO_CFIICall;
7069 auto CheckHandler = SanitizerHandler::CFICheckFail;
7070 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
7071 EmitSanitizerStatReport(SSK: llvm::SanStat_CFI_ICall);
7072
7073 llvm::Metadata *MD =
7074 CGM.CreateMetadataIdentifierForFnType(T: QualType(FnType, 0));
7075
7076 llvm::Value *TypeId = llvm::MetadataAsValue::get(Context&: getLLVMContext(), MD);
7077
7078 llvm::Value *CalleePtr = Callee.getFunctionPointer();
7079 llvm::Value *TypeTest = Builder.CreateCall(
7080 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::type_test), Args: {CalleePtr, TypeId});
7081
7082 auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
7083 llvm::Constant *StaticData[] = {
7084 llvm::ConstantInt::get(Ty: Int8Ty, V: CFITCK_ICall),
7085 EmitCheckSourceLocation(Loc: E->getBeginLoc()),
7086 EmitCheckTypeDescriptor(T: QualType(FnType, 0)),
7087 };
7088 if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) {
7089 EmitCfiSlowPathCheck(Ordinal: CheckOrdinal, Cond: TypeTest, TypeId: CrossDsoTypeId, Ptr: CalleePtr,
7090 StaticArgs: StaticData);
7091 } else {
7092 EmitCheck(Checked: std::make_pair(x&: TypeTest, y&: CheckOrdinal), CheckHandler,
7093 StaticArgs: StaticData, DynamicArgs: {CalleePtr, llvm::UndefValue::get(T: IntPtrTy)});
7094 }
7095 }
7096
7097 CallArgList Args;
7098 if (Chain)
7099 Args.add(rvalue: RValue::get(V: Chain), type: CGM.getContext().VoidPtrTy);
7100
7101 // C++17 requires that we evaluate arguments to a call using assignment syntax
7102 // right-to-left, and that we evaluate arguments to certain other operators
7103 // left-to-right. Note that we allow this to override the order dictated by
7104 // the calling convention on the MS ABI, which means that parameter
7105 // destruction order is not necessarily reverse construction order.
7106 // FIXME: Revisit this based on C++ committee response to unimplementability.
7107 EvaluationOrder Order = EvaluationOrder::Default;
7108 bool StaticOperator = false;
7109 if (auto *OCE = dyn_cast<CXXOperatorCallExpr>(Val: E)) {
7110 if (OCE->isAssignmentOp())
7111 Order = EvaluationOrder::ForceRightToLeft;
7112 else {
7113 switch (OCE->getOperator()) {
7114 case OO_LessLess:
7115 case OO_GreaterGreater:
7116 case OO_AmpAmp:
7117 case OO_PipePipe:
7118 case OO_Comma:
7119 case OO_ArrowStar:
7120 Order = EvaluationOrder::ForceLeftToRight;
7121 break;
7122 default:
7123 break;
7124 }
7125 }
7126
7127 if (const auto *MD =
7128 dyn_cast_if_present<CXXMethodDecl>(Val: OCE->getCalleeDecl());
7129 MD && MD->isStatic())
7130 StaticOperator = true;
7131 }
7132
7133 auto Arguments = E->arguments();
7134 if (StaticOperator) {
7135 // If we're calling a static operator, we need to emit the object argument
7136 // and ignore it.
7137 EmitIgnoredExpr(E: E->getArg(Arg: 0));
7138 Arguments = drop_begin(RangeOrContainer&: Arguments, N: 1);
7139 }
7140 EmitCallArgs(Args, Prototype: dyn_cast<FunctionProtoType>(Val: FnType), ArgRange: Arguments,
7141 AC: E->getDirectCallee(), /*ParamsToSkip=*/0, Order);
7142
7143 const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(
7144 Args, Ty: FnType, /*ChainCall=*/Chain, ABIInfoFD: getCurrentFunctionDecl());
7145
7146 if (ResolvedFnInfo)
7147 *ResolvedFnInfo = &FnInfo;
7148
7149 // HIP function pointer contains kernel handle when it is used in triple
7150 // chevron. The kernel stub needs to be loaded from kernel handle and used
7151 // as callee.
7152 if (CGM.getLangOpts().HIP && !CGM.getLangOpts().CUDAIsDevice &&
7153 isa<CUDAKernelCallExpr>(Val: E) &&
7154 (!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl))) {
7155 llvm::Value *Handle = Callee.getFunctionPointer();
7156 auto *Stub = Builder.CreateLoad(
7157 Addr: Address(Handle, Handle->getType(), CGM.getPointerAlign()));
7158 Callee.setFunctionPointer(Stub);
7159 }
7160
7161 // Insert function pointer lookup if this is a target call
7162 //
7163 // This is used for the indirect function case, virtual function case is
7164 // handled in ItaniumCXXABI.cpp
7165 if (getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU() &&
7166 (!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl))) {
7167 const Expr *CalleeExpr = E->getCallee()->IgnoreParenImpCasts();
7168 const DeclRefExpr *DRE = nullptr;
7169 while (CalleeExpr) {
7170 if ((DRE = dyn_cast<DeclRefExpr>(Val: CalleeExpr)))
7171 break;
7172 if (const auto *ME = dyn_cast<MemberExpr>(Val: CalleeExpr))
7173 CalleeExpr = ME->getBase()->IgnoreParenImpCasts();
7174 else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: CalleeExpr))
7175 CalleeExpr = ASE->getBase()->IgnoreParenImpCasts();
7176 else
7177 break;
7178 }
7179
7180 const auto *VD = DRE ? dyn_cast<VarDecl>(Val: DRE->getDecl()) : nullptr;
7181 if (VD && VD->hasAttr<OMPTargetIndirectCallAttr>()) {
7182 auto *FuncPtrTy = llvm::PointerType::get(
7183 C&: CGM.getLLVMContext(), AddressSpace: CGM.getDataLayout().getProgramAddressSpace());
7184 llvm::Type *RtlFnArgs[] = {FuncPtrTy};
7185 llvm::FunctionCallee DeviceRtlFn = CGM.CreateRuntimeFunction(
7186 Ty: llvm::FunctionType::get(Result: FuncPtrTy, Params: RtlFnArgs, isVarArg: false),
7187 Name: "__llvm_omp_indirect_call_lookup");
7188 llvm::Value *Func = Callee.getFunctionPointer();
7189 llvm::Type *BackupTy = Func->getType();
7190 Func = Builder.CreatePointerBitCastOrAddrSpaceCast(V: Func, DestTy: FuncPtrTy);
7191 Func = EmitRuntimeCall(callee: DeviceRtlFn, args: {Func});
7192 Func = Builder.CreatePointerBitCastOrAddrSpaceCast(V: Func, DestTy: BackupTy);
7193 Callee.setFunctionPointer(Func);
7194 }
7195 }
7196
7197 llvm::CallBase *LocalCallOrInvoke = nullptr;
7198 RValue Call = EmitCall(CallInfo: FnInfo, Callee, ReturnValue, Args, CallOrInvoke: &LocalCallOrInvoke,
7199 IsMustTail: E == MustTailCall, Loc: E->getExprLoc());
7200
7201 if (auto *CalleeDecl = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
7202 if (CalleeDecl->hasAttr<RestrictAttr>() ||
7203 CalleeDecl->hasAttr<MallocSpanAttr>() ||
7204 CalleeDecl->hasAttr<AllocSizeAttr>()) {
7205 // Function has 'malloc' (aka. 'restrict') or 'alloc_size' attribute.
7206 if (SanOpts.has(K: SanitizerKind::AllocToken)) {
7207 // Set !alloc_token metadata.
7208 EmitAllocToken(CB: LocalCallOrInvoke, E);
7209 }
7210 }
7211 }
7212 if (CallOrInvoke)
7213 *CallOrInvoke = LocalCallOrInvoke;
7214
7215 return Call;
7216}
7217
7218LValue CodeGenFunction::
7219EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) {
7220 Address BaseAddr = Address::invalid();
7221 if (E->getOpcode() == BO_PtrMemI) {
7222 BaseAddr = EmitPointerWithAlignment(E: E->getLHS());
7223 } else {
7224 BaseAddr = EmitLValue(E: E->getLHS()).getAddress();
7225 }
7226
7227 llvm::Value *OffsetV = EmitScalarExpr(E: E->getRHS());
7228 const auto *MPT = E->getRHS()->getType()->castAs<MemberPointerType>();
7229
7230 LValueBaseInfo BaseInfo;
7231 TBAAAccessInfo TBAAInfo;
7232 bool IsInBounds = !getLangOpts().PointerOverflowDefined &&
7233 !isUnderlyingBasePointerConstantNull(E: E->getLHS());
7234 Address MemberAddr = EmitCXXMemberDataPointerAddress(
7235 E, base: BaseAddr, memberPtr: OffsetV, memberPtrType: MPT, IsInBounds, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
7236
7237 return MakeAddrLValue(Addr: MemberAddr, T: MPT->getPointeeType(), BaseInfo, TBAAInfo);
7238}
7239
7240/// Given the address of a temporary variable, produce an r-value of
7241/// its type.
7242RValue CodeGenFunction::convertTempToRValue(Address addr,
7243 QualType type,
7244 SourceLocation loc) {
7245 LValue lvalue = MakeAddrLValue(Addr: addr, T: type, Source: AlignmentSource::Decl);
7246 switch (getEvaluationKind(T: type)) {
7247 case TEK_Complex:
7248 return RValue::getComplex(C: EmitLoadOfComplex(src: lvalue, loc));
7249 case TEK_Aggregate:
7250 return lvalue.asAggregateRValue();
7251 case TEK_Scalar:
7252 return RValue::get(V: EmitLoadOfScalar(lvalue, Loc: loc));
7253 }
7254 llvm_unreachable("bad evaluation kind");
7255}
7256
7257void CodeGenFunction::SetFPAccuracy(llvm::Value *Val, float Accuracy) {
7258 assert(Val->getType()->isFPOrFPVectorTy());
7259 if (Accuracy == 0.0 || !isa<llvm::Instruction>(Val))
7260 return;
7261
7262 llvm::MDBuilder MDHelper(getLLVMContext());
7263 llvm::MDNode *Node = MDHelper.createFPMath(Accuracy);
7264
7265 cast<llvm::Instruction>(Val)->setMetadata(KindID: llvm::LLVMContext::MD_fpmath, Node);
7266}
7267
7268void CodeGenFunction::SetSqrtFPAccuracy(llvm::Value *Val) {
7269 llvm::Type *EltTy = Val->getType()->getScalarType();
7270 if (!EltTy->isFloatTy() && !EltTy->isHalfTy())
7271 return;
7272
7273 if ((getLangOpts().OpenCL &&
7274 !CGM.getCodeGenOpts().OpenCLCorrectlyRoundedDivSqrt) ||
7275 (getLangOpts().HIP && getLangOpts().CUDAIsDevice &&
7276 !CGM.getCodeGenOpts().HIPCorrectlyRoundedDivSqrt)) {
7277 // OpenCL v1.1 s7.4: minimum accuracy of single precision sqrt is 3 ulp.
7278 // OpenCL v3.0 s7.4: minimum accuracy of half precision sqrt is 1.5 ulp.
7279 //
7280 // OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt
7281 // build option allows an application to specify that single precision
7282 // floating-point divide (x/y and 1/x) and sqrt used in the program
7283 // source are correctly rounded.
7284 //
7285 // TODO: CUDA has a prec-sqrt flag
7286 SetFPAccuracy(Val, Accuracy: EltTy->isFloatTy() ? 3.0f : 1.5f);
7287 }
7288}
7289
7290void CodeGenFunction::SetDivFPAccuracy(llvm::Value *Val) {
7291 llvm::Type *EltTy = Val->getType()->getScalarType();
7292 if (!EltTy->isFloatTy() && !EltTy->isHalfTy())
7293 return;
7294
7295 if ((getLangOpts().OpenCL &&
7296 !CGM.getCodeGenOpts().OpenCLCorrectlyRoundedDivSqrt) ||
7297 (getLangOpts().HIP && getLangOpts().CUDAIsDevice &&
7298 !CGM.getCodeGenOpts().HIPCorrectlyRoundedDivSqrt)) {
7299 // OpenCL v1.1 s7.4: minimum accuracy of single precision / is 2.5 ulp.
7300 // OpenCL v3.0 s7.4: minimum accuracy of half precision / is 1 ulp.
7301 //
7302 // OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt
7303 // build option allows an application to specify that single precision
7304 // floating-point divide (x/y and 1/x) and sqrt used in the program
7305 // source are correctly rounded.
7306 //
7307 // TODO: CUDA has a prec-div flag
7308 SetFPAccuracy(Val, Accuracy: EltTy->isFloatTy() ? 2.5f : 1.f);
7309 }
7310}
7311
7312namespace {
7313 struct LValueOrRValue {
7314 LValue LV;
7315 RValue RV;
7316 };
7317}
7318
7319static LValueOrRValue emitPseudoObjectExpr(CodeGenFunction &CGF,
7320 const PseudoObjectExpr *E,
7321 bool forLValue,
7322 AggValueSlot slot) {
7323 SmallVector<CodeGenFunction::OpaqueValueMappingData, 4> opaques;
7324
7325 // Find the result expression, if any.
7326 const Expr *resultExpr = E->getResultExpr();
7327 LValueOrRValue result;
7328
7329 for (PseudoObjectExpr::const_semantics_iterator
7330 i = E->semantics_begin(), e = E->semantics_end(); i != e; ++i) {
7331 const Expr *semantic = *i;
7332
7333 // If this semantic expression is an opaque value, bind it
7334 // to the result of its source expression.
7335 if (const auto *ov = dyn_cast<OpaqueValueExpr>(Val: semantic)) {
7336 // Skip unique OVEs.
7337 if (ov->isUnique()) {
7338 assert(ov != resultExpr &&
7339 "A unique OVE cannot be used as the result expression");
7340 continue;
7341 }
7342
7343 // If this is the result expression, we may need to evaluate
7344 // directly into the slot.
7345 typedef CodeGenFunction::OpaqueValueMappingData OVMA;
7346 OVMA opaqueData;
7347 if (ov == resultExpr && ov->isPRValue() && !forLValue &&
7348 CodeGenFunction::hasAggregateEvaluationKind(T: ov->getType())) {
7349 CGF.EmitAggExpr(E: ov->getSourceExpr(), AS: slot);
7350 LValue LV = CGF.MakeAddrLValue(Addr: slot.getAddress(), T: ov->getType(),
7351 Source: AlignmentSource::Decl);
7352 opaqueData = OVMA::bind(CGF, ov, lv: LV);
7353 result.RV = slot.asRValue();
7354
7355 // Otherwise, emit as normal.
7356 } else {
7357 opaqueData = OVMA::bind(CGF, ov, e: ov->getSourceExpr());
7358
7359 // If this is the result, also evaluate the result now.
7360 if (ov == resultExpr) {
7361 if (forLValue)
7362 result.LV = CGF.EmitLValue(E: ov);
7363 else
7364 result.RV = CGF.EmitAnyExpr(E: ov, aggSlot: slot);
7365 }
7366 }
7367
7368 opaques.push_back(Elt: opaqueData);
7369
7370 // Otherwise, if the expression is the result, evaluate it
7371 // and remember the result.
7372 } else if (semantic == resultExpr) {
7373 if (forLValue)
7374 result.LV = CGF.EmitLValue(E: semantic);
7375 else
7376 result.RV = CGF.EmitAnyExpr(E: semantic, aggSlot: slot);
7377
7378 // Otherwise, evaluate the expression in an ignored context.
7379 } else {
7380 CGF.EmitIgnoredExpr(E: semantic);
7381 }
7382 }
7383
7384 // Unbind all the opaques now.
7385 for (CodeGenFunction::OpaqueValueMappingData &opaque : opaques)
7386 opaque.unbind(CGF);
7387
7388 return result;
7389}
7390
7391RValue CodeGenFunction::EmitPseudoObjectRValue(const PseudoObjectExpr *E,
7392 AggValueSlot slot) {
7393 return emitPseudoObjectExpr(CGF&: *this, E, forLValue: false, slot).RV;
7394}
7395
7396LValue CodeGenFunction::EmitPseudoObjectLValue(const PseudoObjectExpr *E) {
7397 return emitPseudoObjectExpr(CGF&: *this, E, forLValue: true, slot: AggValueSlot::ignored()).LV;
7398}
7399
7400void CodeGenFunction::FlattenAccessAndTypeLValue(
7401 LValue Val, SmallVectorImpl<LValue> &AccessList) {
7402
7403 llvm::SmallVector<
7404 std::tuple<LValue, QualType, llvm::SmallVector<llvm::Value *, 4>>, 16>
7405 WorkList;
7406 llvm::IntegerType *IdxTy = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: 32);
7407 WorkList.push_back(Elt: {Val, Val.getType(), {llvm::ConstantInt::get(Ty: IdxTy, V: 0)}});
7408
7409 while (!WorkList.empty()) {
7410 auto [LVal, T, IdxList] = WorkList.pop_back_val();
7411 T = T.getCanonicalType().getUnqualifiedType();
7412 if (const auto *CAT = dyn_cast<ConstantArrayType>(Val&: T)) {
7413 uint64_t Size = CAT->getZExtSize();
7414 for (int64_t I = Size - 1; I > -1; I--) {
7415 llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList;
7416 IdxListCopy.push_back(Elt: llvm::ConstantInt::get(Ty: IdxTy, V: I));
7417 WorkList.emplace_back(Args&: LVal, Args: CAT->getElementType(), Args&: IdxListCopy);
7418 }
7419 } else if (const auto *RT = dyn_cast<RecordType>(Val&: T)) {
7420 const RecordDecl *Record = RT->getDecl()->getDefinitionOrSelf();
7421 assert(!Record->isUnion() && "Union types not supported in flat cast.");
7422
7423 const CXXRecordDecl *CXXD = dyn_cast<CXXRecordDecl>(Val: Record);
7424
7425 llvm::SmallVector<
7426 std::tuple<LValue, QualType, llvm::SmallVector<llvm::Value *, 4>>, 16>
7427 ReverseList;
7428 if (CXXD && CXXD->isStandardLayout())
7429 Record = CXXD->getStandardLayoutBaseWithFields();
7430
7431 // deal with potential base classes
7432 if (CXXD && !CXXD->isStandardLayout()) {
7433 if (CXXD->getNumBases() > 0) {
7434 assert(CXXD->getNumBases() == 1 &&
7435 "HLSL doesn't support multiple inheritance.");
7436 auto Base = CXXD->bases_begin();
7437 llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList;
7438 IdxListCopy.push_back(Elt: llvm::ConstantInt::get(
7439 Ty: IdxTy, V: 0)); // base struct should be at index zero
7440 ReverseList.emplace_back(Args&: LVal, Args: Base->getType(), Args&: IdxListCopy);
7441 }
7442 }
7443
7444 const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(Record);
7445
7446 llvm::Type *LLVMT = ConvertTypeForMem(T);
7447 CharUnits Align = getContext().getTypeAlignInChars(T);
7448 LValue RLValue;
7449 bool createdGEP = false;
7450 for (auto *FD : Record->fields()) {
7451 if (FD->isBitField()) {
7452 if (FD->isUnnamedBitField())
7453 continue;
7454 if (!createdGEP) {
7455 createdGEP = true;
7456 Address GEP = Builder.CreateInBoundsGEP(Addr: LVal.getAddress(), IdxList,
7457 ElementType: LLVMT, Align, Name: "gep");
7458 RLValue = MakeAddrLValue(Addr: GEP, T);
7459 }
7460 LValue FieldLVal = EmitLValueForField(base: RLValue, field: FD, IsInBounds: true);
7461 ReverseList.push_back(Elt: {FieldLVal, FD->getType(), {}});
7462 } else {
7463 llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList;
7464 IdxListCopy.push_back(
7465 Elt: llvm::ConstantInt::get(Ty: IdxTy, V: Layout.getLLVMFieldNo(FD)));
7466 ReverseList.emplace_back(Args&: LVal, Args: FD->getType(), Args&: IdxListCopy);
7467 }
7468 }
7469
7470 std::reverse(first: ReverseList.begin(), last: ReverseList.end());
7471 llvm::append_range(C&: WorkList, R&: ReverseList);
7472 } else if (const auto *VT = dyn_cast<VectorType>(Val&: T)) {
7473 llvm::Type *LLVMT = ConvertTypeForMem(T);
7474 CharUnits Align = getContext().getTypeAlignInChars(T);
7475 Address GEP = Builder.CreateInBoundsGEP(Addr: LVal.getAddress(), IdxList, ElementType: LLVMT,
7476 Align, Name: "vector.gep");
7477 LValue Base = MakeAddrLValue(Addr: GEP, T);
7478 for (unsigned I = 0, E = VT->getNumElements(); I < E; I++) {
7479 llvm::Constant *Idx = llvm::ConstantInt::get(Ty: IdxTy, V: I);
7480 LValue LV =
7481 LValue::MakeVectorElt(vecAddress: Base.getAddress(), Idx, type: VT->getElementType(),
7482 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
7483 AccessList.emplace_back(Args&: LV);
7484 }
7485 } else if (const auto *MT = dyn_cast<ConstantMatrixType>(Val&: T)) {
7486 // Matrices are represented as flat arrays in memory, but has a vector
7487 // value type. So we use ConvertMatrixAddress to convert the address from
7488 // array to vector, and extract elements similar to the vector case above.
7489 // The matrix elements are iterated over in row-major order regardless of
7490 // the memory layout of the matrix.
7491 llvm::Type *LLVMT = ConvertTypeForMem(T);
7492 CharUnits Align = getContext().getTypeAlignInChars(T);
7493 Address GEP = Builder.CreateInBoundsGEP(Addr: LVal.getAddress(), IdxList, ElementType: LLVMT,
7494 Align, Name: "matrix.gep");
7495 LValue Base = MakeAddrLValue(Addr: GEP, T);
7496 Address MatAddr = MaybeConvertMatrixAddress(Addr: Base.getAddress(), CGF&: *this);
7497 unsigned NumRows = MT->getNumRows();
7498 unsigned NumCols = MT->getNumColumns();
7499 bool IsMatrixRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T);
7500 llvm::MatrixBuilder MB(Builder);
7501 for (unsigned Row = 0; Row < MT->getNumRows(); Row++) {
7502 for (unsigned Col = 0; Col < MT->getNumColumns(); Col++) {
7503 llvm::Value *RowIdx = llvm::ConstantInt::get(Ty: IdxTy, V: Row);
7504 llvm::Value *ColIdx = llvm::ConstantInt::get(Ty: IdxTy, V: Col);
7505 llvm::Value *Idx = MB.CreateIndex(RowIdx, ColumnIdx: ColIdx, NumRows, NumCols,
7506 IsMatrixRowMajor);
7507 LValue LV =
7508 LValue::MakeMatrixElt(matAddress: MatAddr, Idx, type: MT->getElementType(),
7509 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
7510 AccessList.emplace_back(Args&: LV);
7511 }
7512 }
7513 } else { // a scalar/builtin type
7514 if (!IdxList.empty()) {
7515 llvm::Type *LLVMT = ConvertTypeForMem(T);
7516 CharUnits Align = getContext().getTypeAlignInChars(T);
7517 Address GEP = Builder.CreateInBoundsGEP(Addr: LVal.getAddress(), IdxList,
7518 ElementType: LLVMT, Align, Name: "gep");
7519 AccessList.emplace_back(Args: MakeAddrLValue(Addr: GEP, T));
7520 } else // must be a bitfield we already created an lvalue for
7521 AccessList.emplace_back(Args&: LVal);
7522 }
7523 }
7524}
7525