1//===--- CGExpr.cpp - Emit LLVM Code from Expressions ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Expr nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ABIInfoImpl.h"
14#include "CGCUDARuntime.h"
15#include "CGCXXABI.h"
16#include "CGCall.h"
17#include "CGCleanup.h"
18#include "CGDebugInfo.h"
19#include "CGHLSLRuntime.h"
20#include "CGObjCRuntime.h"
21#include "CGOpenMPRuntime.h"
22#include "CGRecordLayout.h"
23#include "CodeGenFunction.h"
24#include "CodeGenModule.h"
25#include "CodeGenPGO.h"
26#include "ConstantEmitter.h"
27#include "TargetInfo.h"
28#include "clang/AST/ASTContext.h"
29#include "clang/AST/ASTLambda.h"
30#include "clang/AST/Attr.h"
31#include "clang/AST/DeclObjC.h"
32#include "clang/AST/Expr.h"
33#include "clang/AST/InferAlloc.h"
34#include "clang/AST/NSAPI.h"
35#include "clang/AST/ParentMapContext.h"
36#include "clang/AST/StmtVisitor.h"
37#include "clang/Basic/Builtins.h"
38#include "clang/Basic/CodeGenOptions.h"
39#include "clang/Basic/Module.h"
40#include "clang/Basic/SourceManager.h"
41#include "llvm/ADT/STLExtras.h"
42#include "llvm/ADT/ScopeExit.h"
43#include "llvm/ADT/StringExtras.h"
44#include "llvm/IR/Constants.h"
45#include "llvm/IR/DataLayout.h"
46#include "llvm/IR/Intrinsics.h"
47#include "llvm/IR/LLVMContext.h"
48#include "llvm/IR/MDBuilder.h"
49#include "llvm/IR/MatrixBuilder.h"
50#include "llvm/Support/ConvertUTF.h"
51#include "llvm/Support/Endian.h"
52#include "llvm/Support/MathExtras.h"
53#include "llvm/Support/Path.h"
54#include "llvm/Support/xxhash.h"
55#include "llvm/Transforms/Utils/SanitizerStats.h"
56
57#include <numeric>
58#include <optional>
59#include <string>
60
61using namespace clang;
62using namespace CodeGen;
63
64namespace clang {
65// TODO: consider deprecating ClSanitizeGuardChecks; functionality is subsumed
66// by -fsanitize-skip-hot-cutoff
67llvm::cl::opt<bool> ClSanitizeGuardChecks(
68 "ubsan-guard-checks", llvm::cl::Optional,
69 llvm::cl::desc("Guard UBSAN checks with `llvm.allow.ubsan.check()`."));
70
71} // namespace clang
72
73//===--------------------------------------------------------------------===//
74// Defines for metadata
75//===--------------------------------------------------------------------===//
76
77// Those values are crucial to be the SAME as in ubsan runtime library.
78enum VariableTypeDescriptorKind : uint16_t {
79 /// An integer type.
80 TK_Integer = 0x0000,
81 /// A floating-point type.
82 TK_Float = 0x0001,
83 /// An _BitInt(N) type.
84 TK_BitInt = 0x0002,
85 /// Any other type. The value representation is unspecified.
86 TK_Unknown = 0xffff
87};
88
89//===--------------------------------------------------------------------===//
90// Miscellaneous Helper Methods
91//===--------------------------------------------------------------------===//
92
93static llvm::StringRef GetUBSanTrapForHandler(SanitizerHandler ID) {
94 switch (ID) {
95#define SANITIZER_CHECK(Enum, Name, Version, Msg) \
96 case SanitizerHandler::Enum: \
97 return Msg;
98 LIST_SANITIZER_CHECKS
99#undef SANITIZER_CHECK
100 }
101 llvm_unreachable("unhandled switch case");
102}
103
104/// CreateTempAlloca - This creates a alloca and inserts it into the entry
105/// block.
106RawAddress
107CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits Align,
108 const Twine &Name,
109 llvm::Value *ArraySize) {
110 auto Alloca = CreateTempAlloca(Ty, Name, ArraySize);
111 Alloca->setAlignment(Align.getAsAlign());
112 return RawAddress(Alloca, Ty, Align, KnownNonNull);
113}
114
115RawAddress CodeGenFunction::MaybeCastStackAddressSpace(RawAddress Alloca,
116 LangAS DestLangAS,
117 llvm::Value *ArraySize) {
118
119 llvm::Value *V = Alloca.getPointer();
120 // Alloca always returns a pointer in alloca address space, which may
121 // be different from the type defined by the language. For example,
122 // in C++ the auto variables are in the default address space. Therefore
123 // cast alloca to the default address space when necessary.
124
125 unsigned DestAddrSpace = getContext().getTargetAddressSpace(AS: DestLangAS);
126 if (DestAddrSpace != Alloca.getAddressSpace()) {
127 llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
128 // When ArraySize is nullptr, alloca is inserted at AllocaInsertPt,
129 // otherwise alloca is inserted at the current insertion point of the
130 // builder.
131 if (!ArraySize)
132 Builder.SetInsertPoint(getPostAllocaInsertPoint());
133 V = performAddrSpaceCast(Src: V, DestTy: Builder.getPtrTy(AddrSpace: DestAddrSpace));
134 }
135
136 return RawAddress(V, Alloca.getElementType(), Alloca.getAlignment(),
137 KnownNonNull);
138}
139
140RawAddress CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, LangAS DestLangAS,
141 CharUnits Align, const Twine &Name,
142 llvm::Value *ArraySize,
143 RawAddress *AllocaAddr) {
144 RawAddress Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize);
145 if (AllocaAddr)
146 *AllocaAddr = Alloca;
147 return MaybeCastStackAddressSpace(Alloca, DestLangAS, ArraySize);
148}
149
150/// CreateTempAlloca - This creates an alloca and inserts it into the entry
151/// block if \p ArraySize is nullptr, otherwise inserts it at the current
152/// insertion point of the builder.
153llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
154 const Twine &Name,
155 llvm::Value *ArraySize) {
156 llvm::AllocaInst *Alloca;
157 if (ArraySize)
158 Alloca = Builder.CreateAlloca(Ty, ArraySize, Name);
159 else
160 Alloca =
161 new llvm::AllocaInst(Ty, CGM.getDataLayout().getAllocaAddrSpace(),
162 ArraySize, Name, AllocaInsertPt->getIterator());
163 if (SanOpts.Mask & SanitizerKind::Address) {
164 Alloca->addAnnotationMetadata(Annotations: {"alloca_name_altered", Name.str()});
165 }
166 if (Allocas) {
167 Allocas->Add(I: Alloca);
168 }
169 return Alloca;
170}
171
172/// CreateDefaultAlignTempAlloca - This creates an alloca with the
173/// default alignment of the corresponding LLVM type, which is *not*
174/// guaranteed to be related in any way to the expected alignment of
175/// an AST type that might have been lowered to Ty.
176RawAddress CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty,
177 const Twine &Name) {
178 CharUnits Align =
179 CharUnits::fromQuantity(Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty));
180 return CreateTempAlloca(Ty, align: Align, Name);
181}
182
183RawAddress CodeGenFunction::CreateIRTempWithoutCast(QualType Ty,
184 const Twine &Name) {
185 CharUnits Align = getContext().getTypeAlignInChars(T: Ty);
186 return CreateTempAllocaWithoutCast(Ty: ConvertType(T: Ty), Align, Name, ArraySize: nullptr);
187}
188
189RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name,
190 RawAddress *Alloca) {
191 // FIXME: Should we prefer the preferred type alignment here?
192 return CreateMemTemp(T: Ty, Align: getContext().getTypeAlignInChars(T: Ty), Name, Alloca);
193}
194
195RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align,
196 const Twine &Name,
197 RawAddress *Alloca) {
198 RawAddress Result = CreateTempAlloca(Ty: ConvertTypeForMem(T: Ty), align: Align, Name,
199 /*ArraySize=*/nullptr, Alloca);
200
201 if (Ty->isConstantMatrixType()) {
202 auto *ArrayTy = cast<llvm::ArrayType>(Val: Result.getElementType());
203 auto *ArrayElementTy = ArrayTy->getElementType();
204 auto ArrayElements = ArrayTy->getNumElements();
205 if (getContext().getLangOpts().HLSL) {
206 auto *VectorTy = cast<llvm::FixedVectorType>(Val: ArrayElementTy);
207 ArrayElementTy = VectorTy->getElementType();
208 ArrayElements *= VectorTy->getNumElements();
209 }
210 auto *VectorTy = llvm::FixedVectorType::get(ElementType: ArrayElementTy, NumElts: ArrayElements);
211
212 Result = Address(Result.getPointer(), VectorTy, Result.getAlignment(),
213 KnownNonNull);
214 }
215 return Result;
216}
217
218RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
219 CharUnits Align,
220 const Twine &Name) {
221 return CreateTempAllocaWithoutCast(Ty: ConvertTypeForMem(T: Ty), Align, Name);
222}
223
224RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
225 const Twine &Name) {
226 return CreateMemTempWithoutCast(Ty, Align: getContext().getTypeAlignInChars(T: Ty),
227 Name);
228}
229
230/// EvaluateExprAsBool - Perform the usual unary conversions on the specified
231/// expression and compare the result against zero, returning an Int1Ty value.
232llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
233 PGO->setCurrentStmt(E);
234 if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {
235 llvm::Value *MemPtr = EmitScalarExpr(E);
236 return CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF&: *this, MemPtr, MPT);
237 }
238
239 QualType BoolTy = getContext().BoolTy;
240 SourceLocation Loc = E->getExprLoc();
241 CGFPOptionsRAII FPOptsRAII(*this, E);
242 if (!E->getType()->isAnyComplexType())
243 return EmitScalarConversion(Src: EmitScalarExpr(E), SrcTy: E->getType(), DstTy: BoolTy, Loc);
244
245 return EmitComplexToScalarConversion(Src: EmitComplexExpr(E), SrcTy: E->getType(), DstTy: BoolTy,
246 Loc);
247}
248
249/// EmitIgnoredExpr - Emit code to compute the specified expression,
250/// ignoring the result.
251void CodeGenFunction::EmitIgnoredExpr(const Expr *E) {
252 if (E->isPRValue())
253 return (void)EmitAnyExpr(E, aggSlot: AggValueSlot::ignored(), ignoreResult: true);
254
255 // if this is a bitfield-resulting conditional operator, we can special case
256 // emit this. The normal 'EmitLValue' version of this is particularly
257 // difficult to codegen for, since creating a single "LValue" for two
258 // different sized arguments here is not particularly doable.
259 if (const auto *CondOp = dyn_cast<AbstractConditionalOperator>(
260 Val: E->IgnoreParenNoopCasts(Ctx: getContext()))) {
261 if (CondOp->getObjectKind() == OK_BitField)
262 return EmitIgnoredConditionalOperator(E: CondOp);
263 }
264
265 // Just emit it as an l-value and drop the result.
266 EmitLValue(E);
267}
268
269/// EmitAnyExpr - Emit code to compute the specified expression which
270/// can have any type. The result is returned as an RValue struct.
271/// If this is an aggregate expression, AggSlot indicates where the
272/// result should be returned.
273RValue CodeGenFunction::EmitAnyExpr(const Expr *E,
274 AggValueSlot aggSlot,
275 bool ignoreResult) {
276 switch (getEvaluationKind(T: E->getType())) {
277 case TEK_Scalar:
278 return RValue::get(V: EmitScalarExpr(E, IgnoreResultAssign: ignoreResult));
279 case TEK_Complex:
280 return RValue::getComplex(C: EmitComplexExpr(E, IgnoreReal: ignoreResult, IgnoreImag: ignoreResult));
281 case TEK_Aggregate:
282 if (!ignoreResult && aggSlot.isIgnored())
283 aggSlot = CreateAggTemp(T: E->getType(), Name: "agg-temp");
284 EmitAggExpr(E, AS: aggSlot);
285 return aggSlot.asRValue();
286 }
287 llvm_unreachable("bad evaluation kind");
288}
289
290/// EmitAnyExprToTemp - Similar to EmitAnyExpr(), however, the result will
291/// always be accessible even if no aggregate location is provided.
292RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E) {
293 AggValueSlot AggSlot = AggValueSlot::ignored();
294
295 if (hasAggregateEvaluationKind(T: E->getType()))
296 AggSlot = CreateAggTemp(T: E->getType(), Name: "agg.tmp");
297 return EmitAnyExpr(E, aggSlot: AggSlot);
298}
299
300/// EmitAnyExprToMem - Evaluate an expression into a given memory
301/// location.
302void CodeGenFunction::EmitAnyExprToMem(const Expr *E,
303 Address Location,
304 Qualifiers Quals,
305 bool IsInit) {
306 // FIXME: This function should take an LValue as an argument.
307 switch (getEvaluationKind(T: E->getType())) {
308 case TEK_Complex:
309 EmitComplexExprIntoLValue(E, dest: MakeAddrLValue(Addr: Location, T: E->getType()),
310 /*isInit*/ false);
311 return;
312
313 case TEK_Aggregate: {
314 EmitAggExpr(E, AS: AggValueSlot::forAddr(addr: Location, quals: Quals,
315 isDestructed: AggValueSlot::IsDestructed_t(IsInit),
316 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
317 isAliased: AggValueSlot::IsAliased_t(!IsInit),
318 mayOverlap: AggValueSlot::MayOverlap));
319 return;
320 }
321
322 case TEK_Scalar: {
323 RValue RV = RValue::get(V: EmitScalarExpr(E, /*Ignore*/ IgnoreResultAssign: false));
324 LValue LV = MakeAddrLValue(Addr: Location, T: E->getType());
325 EmitStoreThroughLValue(Src: RV, Dst: LV);
326 return;
327 }
328 }
329 llvm_unreachable("bad evaluation kind");
330}
331
332void CodeGenFunction::EmitInitializationToLValue(
333 const Expr *E, LValue LV, AggValueSlot::IsZeroed_t IsZeroed) {
334 QualType Type = LV.getType();
335 switch (getEvaluationKind(T: Type)) {
336 case TEK_Complex:
337 EmitComplexExprIntoLValue(E, dest: LV, /*isInit*/ true);
338 return;
339 case TEK_Aggregate:
340 EmitAggExpr(E, AS: AggValueSlot::forLValue(LV, isDestructed: AggValueSlot::IsDestructed,
341 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
342 isAliased: AggValueSlot::IsNotAliased,
343 mayOverlap: AggValueSlot::MayOverlap, isZeroed: IsZeroed));
344 return;
345 case TEK_Scalar:
346 if (LV.isSimple())
347 EmitScalarInit(init: E, /*D=*/nullptr, lvalue: LV, /*Captured=*/capturedByInit: false);
348 else
349 EmitStoreThroughLValue(Src: RValue::get(V: EmitScalarExpr(E)), Dst: LV);
350 return;
351 }
352 llvm_unreachable("bad evaluation kind");
353}
354
355static void
356pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M,
357 const Expr *E, Address ReferenceTemporary) {
358 // Objective-C++ ARC:
359 // If we are binding a reference to a temporary that has ownership, we
360 // need to perform retain/release operations on the temporary.
361 //
362 // FIXME: This should be looking at E, not M.
363 if (auto Lifetime = M->getType().getObjCLifetime()) {
364 switch (Lifetime) {
365 case Qualifiers::OCL_None:
366 case Qualifiers::OCL_ExplicitNone:
367 // Carry on to normal cleanup handling.
368 break;
369
370 case Qualifiers::OCL_Autoreleasing:
371 // Nothing to do; cleaned up by an autorelease pool.
372 return;
373
374 case Qualifiers::OCL_Strong:
375 case Qualifiers::OCL_Weak:
376 switch (StorageDuration Duration = M->getStorageDuration()) {
377 case SD_Static:
378 // Note: we intentionally do not register a cleanup to release
379 // the object on program termination.
380 return;
381
382 case SD_Thread:
383 // FIXME: We should probably register a cleanup in this case.
384 return;
385
386 case SD_Automatic:
387 case SD_FullExpression:
388 CodeGenFunction::Destroyer *Destroy;
389 CleanupKind CleanupKind;
390 if (Lifetime == Qualifiers::OCL_Strong) {
391 const ValueDecl *VD = M->getExtendingDecl();
392 bool Precise = isa_and_nonnull<VarDecl>(Val: VD) &&
393 VD->hasAttr<ObjCPreciseLifetimeAttr>();
394 CleanupKind = CGF.getARCCleanupKind();
395 Destroy = Precise ? &CodeGenFunction::destroyARCStrongPrecise
396 : &CodeGenFunction::destroyARCStrongImprecise;
397 } else {
398 // __weak objects always get EH cleanups; otherwise, exceptions
399 // could cause really nasty crashes instead of mere leaks.
400 CleanupKind = NormalAndEHCleanup;
401 Destroy = &CodeGenFunction::destroyARCWeak;
402 }
403 if (Duration == SD_FullExpression)
404 CGF.pushDestroy(kind: CleanupKind, addr: ReferenceTemporary,
405 type: M->getType(), destroyer: *Destroy,
406 useEHCleanupForArray: CleanupKind & EHCleanup);
407 else
408 CGF.pushLifetimeExtendedDestroy(kind: CleanupKind, addr: ReferenceTemporary,
409 type: M->getType(),
410 destroyer: *Destroy, useEHCleanupForArray: CleanupKind & EHCleanup);
411 return;
412
413 case SD_Dynamic:
414 llvm_unreachable("temporary cannot have dynamic storage duration");
415 }
416 llvm_unreachable("unknown storage duration");
417 }
418 }
419
420 QualType::DestructionKind DK = E->getType().isDestructedType();
421 if (DK != QualType::DK_none) {
422 switch (M->getStorageDuration()) {
423 case SD_Static:
424 case SD_Thread: {
425 CXXDestructorDecl *ReferenceTemporaryDtor = nullptr;
426 if (const auto *ClassDecl =
427 E->getType()->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
428 ClassDecl && !ClassDecl->hasTrivialDestructor())
429 // Get the destructor for the reference temporary.
430 ReferenceTemporaryDtor = ClassDecl->getDestructor();
431
432 if (!ReferenceTemporaryDtor)
433 return;
434
435 llvm::FunctionCallee CleanupFn;
436 llvm::Constant *CleanupArg;
437 if (E->getType()->isArrayType()) {
438 CleanupFn = CodeGenFunction(CGF.CGM).generateDestroyHelper(
439 addr: ReferenceTemporary, type: E->getType(), destroyer: CodeGenFunction::destroyCXXObject,
440 useEHCleanupForArray: CGF.getLangOpts().Exceptions,
441 VD: dyn_cast_or_null<VarDecl>(Val: M->getExtendingDecl()));
442 CleanupArg = llvm::Constant::getNullValue(Ty: CGF.Int8PtrTy);
443 } else {
444 CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor(
445 GD: GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete));
446 CleanupArg =
447 cast<llvm::Constant>(Val: ReferenceTemporary.emitRawPointer(CGF));
448 }
449 CGF.CGM.getCXXABI().registerGlobalDtor(
450 CGF, D: *cast<VarDecl>(Val: M->getExtendingDecl()), Dtor: CleanupFn, Addr: CleanupArg);
451 } break;
452 case SD_FullExpression:
453 CGF.pushDestroy(dtorKind: DK, addr: ReferenceTemporary, type: E->getType());
454 break;
455 case SD_Automatic:
456 CGF.pushLifetimeExtendedDestroy(dtorKind: DK, addr: ReferenceTemporary, type: E->getType());
457 break;
458 case SD_Dynamic:
459 llvm_unreachable("temporary cannot have dynamic storage duration");
460 }
461 }
462}
463
464static RawAddress createReferenceTemporary(CodeGenFunction &CGF,
465 const MaterializeTemporaryExpr *M,
466 const Expr *Inner,
467 RawAddress *Alloca = nullptr) {
468 switch (M->getStorageDuration()) {
469 case SD_FullExpression:
470 case SD_Automatic: {
471 // If we have a constant temporary array or record try to promote it into a
472 // constant global under the same rules a normal constant would've been
473 // promoted. This is easier on the optimizer and generally emits fewer
474 // instructions.
475 QualType Ty = Inner->getType();
476 if (CGF.CGM.getCodeGenOpts().MergeAllConstants &&
477 (Ty->isArrayType() || Ty->isRecordType()) &&
478 Ty.isConstantStorage(Ctx: CGF.getContext(), ExcludeCtor: true, ExcludeDtor: false))
479 if (auto Init = ConstantEmitter(CGF).tryEmitAbstract(E: Inner, T: Ty)) {
480 auto AS = CGF.CGM.GetGlobalConstantAddressSpace();
481 auto *GV = new llvm::GlobalVariable(
482 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
483 llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp", nullptr,
484 llvm::GlobalValue::NotThreadLocal,
485 CGF.getContext().getTargetAddressSpace(AS));
486 CharUnits alignment = CGF.getContext().getTypeAlignInChars(T: Ty);
487 GV->setAlignment(alignment.getAsAlign());
488 llvm::Constant *C = GV;
489 if (AS != LangAS::Default)
490 C = CGF.CGM.performAddrSpaceCast(
491 Src: GV, DestTy: llvm::PointerType::get(
492 C&: CGF.getLLVMContext(),
493 AddressSpace: CGF.getContext().getTargetAddressSpace(AS: LangAS::Default)));
494 // FIXME: Should we put the new global into a COMDAT?
495 return RawAddress(C, GV->getValueType(), alignment);
496 }
497 return CGF.CreateMemTemp(Ty, Name: "ref.tmp", Alloca);
498 }
499 case SD_Thread:
500 case SD_Static:
501 return CGF.CGM.GetAddrOfGlobalTemporary(E: M, Inner);
502
503 case SD_Dynamic:
504 llvm_unreachable("temporary can't have dynamic storage duration");
505 }
506 llvm_unreachable("unknown storage duration");
507}
508
509/// Helper method to check if the underlying ABI is AAPCS
510static bool isAAPCS(const TargetInfo &TargetInfo) {
511 return TargetInfo.getABI().starts_with(Prefix: "aapcs");
512}
513
514LValue CodeGenFunction::
515EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
516 const Expr *E = M->getSubExpr();
517
518 assert((!M->getExtendingDecl() || !isa<VarDecl>(M->getExtendingDecl()) ||
519 !cast<VarDecl>(M->getExtendingDecl())->isARCPseudoStrong()) &&
520 "Reference should never be pseudo-strong!");
521
522 // FIXME: ideally this would use EmitAnyExprToMem, however, we cannot do so
523 // as that will cause the lifetime adjustment to be lost for ARC
524 auto ownership = M->getType().getObjCLifetime();
525 if (ownership != Qualifiers::OCL_None &&
526 ownership != Qualifiers::OCL_ExplicitNone) {
527 RawAddress Object = createReferenceTemporary(CGF&: *this, M, Inner: E);
528 if (auto *Var = dyn_cast<llvm::GlobalVariable>(Val: Object.getPointer())) {
529 llvm::Type *Ty = ConvertTypeForMem(T: E->getType());
530 Object = Object.withElementType(ElemTy: Ty);
531
532 // createReferenceTemporary will promote the temporary to a global with a
533 // constant initializer if it can. It can only do this to a value of
534 // ARC-manageable type if the value is global and therefore "immune" to
535 // ref-counting operations. Therefore we have no need to emit either a
536 // dynamic initialization or a cleanup and we can just return the address
537 // of the temporary.
538 if (Var->hasInitializer())
539 return MakeAddrLValue(Addr: Object, T: M->getType(), Source: AlignmentSource::Decl);
540
541 Var->setInitializer(CGM.EmitNullConstant(T: E->getType()));
542 }
543 LValue RefTempDst = MakeAddrLValue(Addr: Object, T: M->getType(),
544 Source: AlignmentSource::Decl);
545
546 switch (getEvaluationKind(T: E->getType())) {
547 default: llvm_unreachable("expected scalar or aggregate expression");
548 case TEK_Scalar:
549 EmitScalarInit(init: E, D: M->getExtendingDecl(), lvalue: RefTempDst, capturedByInit: false);
550 break;
551 case TEK_Aggregate: {
552 EmitAggExpr(E, AS: AggValueSlot::forAddr(addr: Object,
553 quals: E->getType().getQualifiers(),
554 isDestructed: AggValueSlot::IsDestructed,
555 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
556 isAliased: AggValueSlot::IsNotAliased,
557 mayOverlap: AggValueSlot::DoesNotOverlap));
558 break;
559 }
560 }
561
562 pushTemporaryCleanup(CGF&: *this, M, E, ReferenceTemporary: Object);
563 return RefTempDst;
564 }
565
566 SmallVector<const Expr *, 2> CommaLHSs;
567 SmallVector<SubobjectAdjustment, 2> Adjustments;
568 E = E->skipRValueSubobjectAdjustments(CommaLHS&: CommaLHSs, Adjustments);
569
570 for (const auto &Ignored : CommaLHSs)
571 EmitIgnoredExpr(E: Ignored);
572
573 if (const auto *opaque = dyn_cast<OpaqueValueExpr>(Val: E)) {
574 if (opaque->getType()->isRecordType()) {
575 assert(Adjustments.empty());
576 return EmitOpaqueValueLValue(e: opaque);
577 }
578 }
579
580 // Create and initialize the reference temporary.
581 RawAddress Alloca = Address::invalid();
582 RawAddress Object = createReferenceTemporary(CGF&: *this, M, Inner: E, Alloca: &Alloca);
583 if (auto *Var = dyn_cast<llvm::GlobalVariable>(
584 Val: Object.getPointer()->stripPointerCasts())) {
585 llvm::Type *TemporaryType = ConvertTypeForMem(T: E->getType());
586 Object = Object.withElementType(ElemTy: TemporaryType);
587 // If the temporary is a global and has a constant initializer or is a
588 // constant temporary that we promoted to a global, we may have already
589 // initialized it.
590 if (!Var->hasInitializer()) {
591 Var->setInitializer(CGM.EmitNullConstant(T: E->getType()));
592 QualType RefType = M->getType().withoutLocalFastQualifiers();
593 if (RefType.getPointerAuth()) {
594 // Use the qualifier of the reference temporary to sign the pointer.
595 LValue LV = MakeRawAddrLValue(V: Object.getPointer(), T: RefType,
596 Alignment: Object.getAlignment());
597 EmitScalarInit(init: E, D: M->getExtendingDecl(), lvalue: LV, capturedByInit: false);
598 } else {
599 EmitAnyExprToMem(E, Location: Object, Quals: Qualifiers(), /*IsInit*/ true);
600 }
601 }
602 } else {
603 switch (M->getStorageDuration()) {
604 case SD_Automatic:
605 if (EmitLifetimeStart(Addr: Alloca.getPointer())) {
606 pushCleanupAfterFullExpr<CallLifetimeEnd>(Kind: NormalEHLifetimeMarker,
607 A: Alloca);
608 }
609 break;
610
611 case SD_FullExpression: {
612 if (!ShouldEmitLifetimeMarkers)
613 break;
614
615 // Avoid creating a conditional cleanup just to hold an llvm.lifetime.end
616 // marker. Instead, start the lifetime of a conditional temporary earlier
617 // so that it's unconditional. Don't do this with sanitizers which need
618 // more precise lifetime marks. However when inside an "await.suspend"
619 // block, we should always avoid conditional cleanup because it creates
620 // boolean marker that lives across await_suspend, which can destroy coro
621 // frame.
622 ConditionalEvaluation *OldConditional = nullptr;
623 CGBuilderTy::InsertPoint OldIP;
624 if (isInConditionalBranch() && !E->getType().isDestructedType() &&
625 ((!SanOpts.has(K: SanitizerKind::HWAddress) &&
626 !SanOpts.has(K: SanitizerKind::Memory) &&
627 !SanOpts.has(K: SanitizerKind::MemtagStack) &&
628 !CGM.getCodeGenOpts().SanitizeAddressUseAfterScope) ||
629 inSuspendBlock())) {
630 OldConditional = OutermostConditional;
631 OutermostConditional = nullptr;
632
633 OldIP = Builder.saveIP();
634 llvm::BasicBlock *Block = OldConditional->getStartingBlock();
635 Builder.restoreIP(IP: CGBuilderTy::InsertPoint(
636 Block, llvm::BasicBlock::iterator(Block->back())));
637 }
638
639 if (EmitLifetimeStart(Addr: Alloca.getPointer())) {
640 pushFullExprCleanup<CallLifetimeEnd>(kind: NormalEHLifetimeMarker, A: Alloca);
641 }
642
643 if (OldConditional) {
644 OutermostConditional = OldConditional;
645 Builder.restoreIP(IP: OldIP);
646 }
647 break;
648 }
649
650 default:
651 break;
652 }
653 EmitAnyExprToMem(E, Location: Object, Quals: Qualifiers(), /*IsInit*/true);
654 }
655 pushTemporaryCleanup(CGF&: *this, M, E, ReferenceTemporary: Object);
656
657 // Perform derived-to-base casts and/or field accesses, to get from the
658 // temporary object we created (and, potentially, for which we extended
659 // the lifetime) to the subobject we're binding the reference to.
660 for (SubobjectAdjustment &Adjustment : llvm::reverse(C&: Adjustments)) {
661 switch (Adjustment.Kind) {
662 case SubobjectAdjustment::DerivedToBaseAdjustment:
663 Object =
664 GetAddressOfBaseClass(Value: Object, Derived: Adjustment.DerivedToBase.DerivedClass,
665 PathBegin: Adjustment.DerivedToBase.BasePath->path_begin(),
666 PathEnd: Adjustment.DerivedToBase.BasePath->path_end(),
667 /*NullCheckValue=*/ false, Loc: E->getExprLoc());
668 break;
669
670 case SubobjectAdjustment::FieldAdjustment: {
671 LValue LV = MakeAddrLValue(Addr: Object, T: E->getType(), Source: AlignmentSource::Decl);
672 LV = EmitLValueForField(Base: LV, Field: Adjustment.Field);
673 assert(LV.isSimple() &&
674 "materialized temporary field is not a simple lvalue");
675 Object = LV.getAddress();
676 break;
677 }
678
679 case SubobjectAdjustment::MemberPointerAdjustment: {
680 llvm::Value *Ptr = EmitScalarExpr(E: Adjustment.Ptr.RHS);
681 Object = EmitCXXMemberDataPointerAddress(
682 E, base: Object, memberPtr: Ptr, memberPtrType: Adjustment.Ptr.MPT, /*IsInBounds=*/true);
683 break;
684 }
685 }
686 }
687
688 return MakeAddrLValue(Addr: Object, T: M->getType(), Source: AlignmentSource::Decl);
689}
690
691RValue
692CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E) {
693 // Emit the expression as an lvalue.
694 LValue LV = EmitLValue(E);
695 assert(LV.isSimple());
696 llvm::Value *Value = LV.getPointer(CGF&: *this);
697
698 if (sanitizePerformTypeCheck() && !E->getType()->isFunctionType()) {
699 // C++11 [dcl.ref]p5 (as amended by core issue 453):
700 // If a glvalue to which a reference is directly bound designates neither
701 // an existing object or function of an appropriate type nor a region of
702 // storage of suitable size and alignment to contain an object of the
703 // reference's type, the behavior is undefined.
704 QualType Ty = E->getType();
705 EmitTypeCheck(TCK: TCK_ReferenceBinding, Loc: E->getExprLoc(), V: Value, Type: Ty);
706 }
707
708 return RValue::get(V: Value);
709}
710
711
712/// getAccessedFieldNo - Given an encoded value and a result number, return the
713/// input field number being accessed.
714unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx,
715 const llvm::Constant *Elts) {
716 return cast<llvm::ConstantInt>(Val: Elts->getAggregateElement(Elt: Idx))
717 ->getZExtValue();
718}
719
720static llvm::Value *emitHashMix(CGBuilderTy &Builder, llvm::Value *Acc,
721 llvm::Value *Ptr) {
722 llvm::Value *A0 =
723 Builder.CreateMul(LHS: Ptr, RHS: Builder.getInt64(C: 0xbf58476d1ce4e5b9u));
724 llvm::Value *A1 =
725 Builder.CreateXor(LHS: A0, RHS: Builder.CreateLShr(LHS: A0, RHS: Builder.getInt64(C: 31)));
726 return Builder.CreateXor(LHS: Acc, RHS: A1);
727}
728
729bool CodeGenFunction::isNullPointerAllowed(TypeCheckKind TCK) {
730 return TCK == TCK_DowncastPointer || TCK == TCK_Upcast ||
731 TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation;
732}
733
734bool CodeGenFunction::isVptrCheckRequired(TypeCheckKind TCK, QualType Ty) {
735 CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
736 return (RD && RD->hasDefinition() && RD->isDynamicClass()) &&
737 (TCK == TCK_MemberAccess || TCK == TCK_MemberCall ||
738 TCK == TCK_DowncastPointer || TCK == TCK_DowncastReference ||
739 TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation);
740}
741
742bool CodeGenFunction::sanitizePerformTypeCheck() const {
743 return SanOpts.has(K: SanitizerKind::Null) ||
744 SanOpts.has(K: SanitizerKind::Alignment) ||
745 SanOpts.has(K: SanitizerKind::ObjectSize) ||
746 SanOpts.has(K: SanitizerKind::Vptr);
747}
748
749void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
750 llvm::Value *Ptr, QualType Ty,
751 CharUnits Alignment,
752 SanitizerSet SkippedChecks,
753 llvm::Value *ArraySize) {
754 if (!sanitizePerformTypeCheck())
755 return;
756
757 // Don't check pointers outside the default address space. The null check
758 // isn't correct, the object-size check isn't supported by LLVM, and we can't
759 // communicate the addresses to the runtime handler for the vptr check.
760 if (Ptr->getType()->getPointerAddressSpace())
761 return;
762
763 // Don't check pointers to volatile data. The behavior here is implementation-
764 // defined.
765 if (Ty.isVolatileQualified())
766 return;
767
768 // Quickly determine whether we have a pointer to an alloca. It's possible
769 // to skip null checks, and some alignment checks, for these pointers. This
770 // can reduce compile-time significantly.
771 auto PtrToAlloca = dyn_cast<llvm::AllocaInst>(Val: Ptr->stripPointerCasts());
772
773 llvm::Value *IsNonNull = nullptr;
774 bool IsGuaranteedNonNull =
775 SkippedChecks.has(K: SanitizerKind::Null) || PtrToAlloca;
776
777 llvm::BasicBlock *Done = nullptr;
778 bool DoneViaNullSanitize = false;
779
780 {
781 auto CheckHandler = SanitizerHandler::TypeMismatch;
782 SanitizerDebugLocation SanScope(this,
783 {SanitizerKind::SO_Null,
784 SanitizerKind::SO_ObjectSize,
785 SanitizerKind::SO_Alignment},
786 CheckHandler);
787
788 SmallVector<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>, 3>
789 Checks;
790
791 llvm::Value *True = llvm::ConstantInt::getTrue(Context&: getLLVMContext());
792 bool AllowNullPointers = isNullPointerAllowed(TCK);
793 if ((SanOpts.has(K: SanitizerKind::Null) || AllowNullPointers) &&
794 !IsGuaranteedNonNull) {
795 // The glvalue must not be an empty glvalue.
796 IsNonNull = Builder.CreateIsNotNull(Arg: Ptr);
797
798 // The IR builder can constant-fold the null check if the pointer points
799 // to a constant.
800 IsGuaranteedNonNull = IsNonNull == True;
801
802 // Skip the null check if the pointer is known to be non-null.
803 if (!IsGuaranteedNonNull) {
804 if (AllowNullPointers) {
805 // When performing pointer casts, it's OK if the value is null.
806 // Skip the remaining checks in that case.
807 Done = createBasicBlock(name: "null");
808 DoneViaNullSanitize = true;
809 llvm::BasicBlock *Rest = createBasicBlock(name: "not.null");
810 Builder.CreateCondBr(Cond: IsNonNull, True: Rest, False: Done);
811 EmitBlock(BB: Rest);
812 } else {
813 Checks.push_back(Elt: std::make_pair(x&: IsNonNull, y: SanitizerKind::SO_Null));
814 }
815 }
816 }
817
818 if (SanOpts.has(K: SanitizerKind::ObjectSize) &&
819 !SkippedChecks.has(K: SanitizerKind::ObjectSize) &&
820 !Ty->isIncompleteType()) {
821 uint64_t TySize = CGM.getMinimumObjectSize(Ty).getQuantity();
822 llvm::Value *Size = llvm::ConstantInt::get(Ty: IntPtrTy, V: TySize);
823 if (ArraySize)
824 Size = Builder.CreateMul(LHS: Size, RHS: ArraySize);
825
826 // Degenerate case: new X[0] does not need an objectsize check.
827 llvm::Constant *ConstantSize = dyn_cast<llvm::Constant>(Val: Size);
828 if (!ConstantSize || !ConstantSize->isNullValue()) {
829 // The glvalue must refer to a large enough storage region.
830 // FIXME: If Address Sanitizer is enabled, insert dynamic
831 // instrumentation
832 // to check this.
833 // FIXME: Get object address space
834 llvm::Type *Tys[2] = {IntPtrTy, Int8PtrTy};
835 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::objectsize, Tys);
836 llvm::Value *Min = Builder.getFalse();
837 llvm::Value *NullIsUnknown = Builder.getFalse();
838 llvm::Value *Dynamic = Builder.getFalse();
839 llvm::Value *LargeEnough = Builder.CreateICmpUGE(
840 LHS: Builder.CreateCall(Callee: F, Args: {Ptr, Min, NullIsUnknown, Dynamic}), RHS: Size);
841 Checks.push_back(
842 Elt: std::make_pair(x&: LargeEnough, y: SanitizerKind::SO_ObjectSize));
843 }
844 }
845
846 llvm::MaybeAlign AlignVal;
847 llvm::Value *PtrAsInt = nullptr;
848
849 if (SanOpts.has(K: SanitizerKind::Alignment) &&
850 !SkippedChecks.has(K: SanitizerKind::Alignment)) {
851 AlignVal = Alignment.getAsMaybeAlign();
852 if (!Ty->isIncompleteType() && !AlignVal)
853 AlignVal = CGM.getNaturalTypeAlignment(T: Ty, BaseInfo: nullptr, TBAAInfo: nullptr,
854 /*ForPointeeType=*/forPointeeType: true)
855 .getAsMaybeAlign();
856
857 // The glvalue must be suitably aligned.
858 if (AlignVal && *AlignVal > llvm::Align(1) &&
859 (!PtrToAlloca || PtrToAlloca->getAlign() < *AlignVal)) {
860 PtrAsInt = Builder.CreatePtrToInt(V: Ptr, DestTy: IntPtrTy);
861 llvm::Value *Align = Builder.CreateAnd(
862 LHS: PtrAsInt, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, V: AlignVal->value() - 1));
863 llvm::Value *Aligned =
864 Builder.CreateICmpEQ(LHS: Align, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, V: 0));
865 if (Aligned != True)
866 Checks.push_back(
867 Elt: std::make_pair(x&: Aligned, y: SanitizerKind::SO_Alignment));
868 }
869 }
870
871 if (Checks.size() > 0) {
872 llvm::Constant *StaticData[] = {
873 EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(T: Ty),
874 llvm::ConstantInt::get(Ty: Int8Ty, V: AlignVal ? llvm::Log2(A: *AlignVal) : 1),
875 llvm::ConstantInt::get(Ty: Int8Ty, V: TCK)};
876 EmitCheck(Checked: Checks, Check: CheckHandler, StaticArgs: StaticData, DynamicArgs: PtrAsInt ? PtrAsInt : Ptr);
877 }
878 }
879
880 // If possible, check that the vptr indicates that there is a subobject of
881 // type Ty at offset zero within this object.
882 //
883 // C++11 [basic.life]p5,6:
884 // [For storage which does not refer to an object within its lifetime]
885 // The program has undefined behavior if:
886 // -- the [pointer or glvalue] is used to access a non-static data member
887 // or call a non-static member function
888 if (SanOpts.has(K: SanitizerKind::Vptr) &&
889 !SkippedChecks.has(K: SanitizerKind::Vptr) && isVptrCheckRequired(TCK, Ty)) {
890 SanitizerDebugLocation SanScope(this, {SanitizerKind::SO_Vptr},
891 SanitizerHandler::DynamicTypeCacheMiss);
892
893 // Ensure that the pointer is non-null before loading it. If there is no
894 // compile-time guarantee, reuse the run-time null check or emit a new one.
895 if (!IsGuaranteedNonNull) {
896 if (!IsNonNull)
897 IsNonNull = Builder.CreateIsNotNull(Arg: Ptr);
898 if (!Done)
899 Done = createBasicBlock(name: "vptr.null");
900 llvm::BasicBlock *VptrNotNull = createBasicBlock(name: "vptr.not.null");
901 Builder.CreateCondBr(Cond: IsNonNull, True: VptrNotNull, False: Done);
902 EmitBlock(BB: VptrNotNull);
903 }
904
905 // Compute a deterministic hash of the mangled name of the type.
906 SmallString<64> MangledName;
907 llvm::raw_svector_ostream Out(MangledName);
908 CGM.getCXXABI().getMangleContext().mangleCXXRTTI(T: Ty.getUnqualifiedType(),
909 Out);
910
911 // Contained in NoSanitizeList based on the mangled type.
912 if (!CGM.getContext().getNoSanitizeList().containsType(Mask: SanitizerKind::Vptr,
913 MangledTypeName: Out.str())) {
914 // Load the vptr, and mix it with TypeHash.
915 llvm::Value *TypeHash =
916 llvm::ConstantInt::get(Ty: Int64Ty, V: xxh3_64bits(data: Out.str()));
917
918 llvm::Type *VPtrTy = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: 0);
919 Address VPtrAddr(Ptr, IntPtrTy, getPointerAlign());
920 llvm::Value *VPtrVal = GetVTablePtr(This: VPtrAddr, VTableTy: VPtrTy,
921 VTableClass: Ty->getAsCXXRecordDecl(),
922 AuthMode: VTableAuthMode::UnsafeUbsanStrip);
923 VPtrVal = Builder.CreateBitOrPointerCast(V: VPtrVal, DestTy: IntPtrTy);
924
925 llvm::Value *Hash =
926 emitHashMix(Builder, Acc: TypeHash, Ptr: Builder.CreateZExt(V: VPtrVal, DestTy: Int64Ty));
927 Hash = Builder.CreateTrunc(V: Hash, DestTy: IntPtrTy);
928
929 // Look the hash up in our cache.
930 const int CacheSize = 128;
931 llvm::Type *HashTable = llvm::ArrayType::get(ElementType: IntPtrTy, NumElements: CacheSize);
932 llvm::Value *Cache = CGM.CreateRuntimeVariable(Ty: HashTable,
933 Name: "__ubsan_vptr_type_cache");
934 llvm::Value *Slot = Builder.CreateAnd(LHS: Hash,
935 RHS: llvm::ConstantInt::get(Ty: IntPtrTy,
936 V: CacheSize-1));
937 llvm::Value *Indices[] = { Builder.getInt32(C: 0), Slot };
938 llvm::Value *CacheVal = Builder.CreateAlignedLoad(
939 Ty: IntPtrTy, Addr: Builder.CreateInBoundsGEP(Ty: HashTable, Ptr: Cache, IdxList: Indices),
940 Align: getPointerAlign());
941
942 // If the hash isn't in the cache, call a runtime handler to perform the
943 // hard work of checking whether the vptr is for an object of the right
944 // type. This will either fill in the cache and return, or produce a
945 // diagnostic.
946 llvm::Value *EqualHash = Builder.CreateICmpEQ(LHS: CacheVal, RHS: Hash);
947 llvm::Constant *StaticData[] = {
948 EmitCheckSourceLocation(Loc),
949 EmitCheckTypeDescriptor(T: Ty),
950 CGM.GetAddrOfRTTIDescriptor(Ty: Ty.getUnqualifiedType()),
951 llvm::ConstantInt::get(Ty: Int8Ty, V: TCK)
952 };
953 llvm::Value *DynamicData[] = { Ptr, Hash };
954 EmitCheck(Checked: std::make_pair(x&: EqualHash, y: SanitizerKind::SO_Vptr),
955 Check: SanitizerHandler::DynamicTypeCacheMiss, StaticArgs: StaticData,
956 DynamicArgs: DynamicData);
957 }
958 }
959
960 if (Done) {
961 SanitizerDebugLocation SanScope(
962 this,
963 {DoneViaNullSanitize ? SanitizerKind::SO_Null : SanitizerKind::SO_Vptr},
964 DoneViaNullSanitize ? SanitizerHandler::TypeMismatch
965 : SanitizerHandler::DynamicTypeCacheMiss);
966 Builder.CreateBr(Dest: Done);
967 EmitBlock(BB: Done);
968 }
969}
970
971llvm::Value *CodeGenFunction::LoadPassedObjectSize(const Expr *E,
972 QualType EltTy) {
973 ASTContext &C = getContext();
974 uint64_t EltSize = C.getTypeSizeInChars(T: EltTy).getQuantity();
975 if (!EltSize)
976 return nullptr;
977
978 auto *ArrayDeclRef = dyn_cast<DeclRefExpr>(Val: E->IgnoreParenImpCasts());
979 if (!ArrayDeclRef)
980 return nullptr;
981
982 auto *ParamDecl = dyn_cast<ParmVarDecl>(Val: ArrayDeclRef->getDecl());
983 if (!ParamDecl)
984 return nullptr;
985
986 auto *POSAttr = ParamDecl->getAttr<PassObjectSizeAttr>();
987 if (!POSAttr)
988 return nullptr;
989
990 // Don't load the size if it's a lower bound.
991 int POSType = POSAttr->getType();
992 if (POSType != 0 && POSType != 1)
993 return nullptr;
994
995 // Find the implicit size parameter.
996 auto PassedSizeIt = SizeArguments.find(Val: ParamDecl);
997 if (PassedSizeIt == SizeArguments.end())
998 return nullptr;
999
1000 const ImplicitParamDecl *PassedSizeDecl = PassedSizeIt->second;
1001 assert(LocalDeclMap.count(PassedSizeDecl) && "Passed size not loadable");
1002 Address AddrOfSize = LocalDeclMap.find(Val: PassedSizeDecl)->second;
1003 llvm::Value *SizeInBytes = EmitLoadOfScalar(Addr: AddrOfSize, /*Volatile=*/false,
1004 Ty: C.getSizeType(), Loc: E->getExprLoc());
1005 llvm::Value *SizeOfElement =
1006 llvm::ConstantInt::get(Ty: SizeInBytes->getType(), V: EltSize);
1007 return Builder.CreateUDiv(LHS: SizeInBytes, RHS: SizeOfElement);
1008}
1009
1010/// If Base is known to point to the start of an array, return the length of
1011/// that array. Return 0 if the length cannot be determined.
1012static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF,
1013 const Expr *Base,
1014 QualType &IndexedType,
1015 LangOptions::StrictFlexArraysLevelKind
1016 StrictFlexArraysLevel) {
1017 // For the vector indexing extension, the bound is the number of elements.
1018 if (const VectorType *VT = Base->getType()->getAs<VectorType>()) {
1019 IndexedType = Base->getType();
1020 return CGF.Builder.getInt32(C: VT->getNumElements());
1021 }
1022
1023 Base = Base->IgnoreParens();
1024
1025 if (const auto *CE = dyn_cast<CastExpr>(Val: Base)) {
1026 if (CE->getCastKind() == CK_ArrayToPointerDecay &&
1027 !CE->getSubExpr()->isFlexibleArrayMemberLike(Context: CGF.getContext(),
1028 StrictFlexArraysLevel)) {
1029 CodeGenFunction::SanitizerScope SanScope(&CGF);
1030
1031 IndexedType = CE->getSubExpr()->getType();
1032 const ArrayType *AT = IndexedType->castAsArrayTypeUnsafe();
1033 if (const auto *CAT = dyn_cast<ConstantArrayType>(Val: AT))
1034 return CGF.Builder.getInt(AI: CAT->getSize());
1035
1036 if (const auto *VAT = dyn_cast<VariableArrayType>(Val: AT))
1037 return CGF.getVLASize(vla: VAT).NumElts;
1038 // Ignore pass_object_size here. It's not applicable on decayed pointers.
1039 }
1040 }
1041
1042 CodeGenFunction::SanitizerScope SanScope(&CGF);
1043
1044 QualType EltTy{Base->getType()->getPointeeOrArrayElementType(), 0};
1045 if (llvm::Value *POS = CGF.LoadPassedObjectSize(E: Base, EltTy)) {
1046 IndexedType = Base->getType();
1047 return POS;
1048 }
1049
1050 return nullptr;
1051}
1052
1053namespace {
1054
1055/// \p StructAccessBase returns the base \p Expr of a field access. It returns
1056/// either a \p DeclRefExpr, representing the base pointer to the struct, i.e.:
1057///
1058/// p in p-> a.b.c
1059///
1060/// or a \p MemberExpr, if the \p MemberExpr has the \p RecordDecl we're
1061/// looking for:
1062///
1063/// struct s {
1064/// struct s *ptr;
1065/// int count;
1066/// char array[] __attribute__((counted_by(count)));
1067/// };
1068///
1069/// If we have an expression like \p p->ptr->array[index], we want the
1070/// \p MemberExpr for \p p->ptr instead of \p p.
1071class StructAccessBase
1072 : public ConstStmtVisitor<StructAccessBase, const Expr *> {
1073 const RecordDecl *ExpectedRD;
1074
1075 bool IsExpectedRecordDecl(const Expr *E) const {
1076 QualType Ty = E->getType();
1077 if (Ty->isPointerType())
1078 Ty = Ty->getPointeeType();
1079 return ExpectedRD == Ty->getAsRecordDecl();
1080 }
1081
1082public:
1083 StructAccessBase(const RecordDecl *ExpectedRD) : ExpectedRD(ExpectedRD) {}
1084
1085 //===--------------------------------------------------------------------===//
1086 // Visitor Methods
1087 //===--------------------------------------------------------------------===//
1088
1089 // NOTE: If we build C++ support for counted_by, then we'll have to handle
1090 // horrors like this:
1091 //
1092 // struct S {
1093 // int x, y;
1094 // int blah[] __attribute__((counted_by(x)));
1095 // } s;
1096 //
1097 // int foo(int index, int val) {
1098 // int (S::*IHatePMDs)[] = &S::blah;
1099 // (s.*IHatePMDs)[index] = val;
1100 // }
1101
1102 const Expr *Visit(const Expr *E) {
1103 return ConstStmtVisitor<StructAccessBase, const Expr *>::Visit(S: E);
1104 }
1105
1106 const Expr *VisitStmt(const Stmt *S) { return nullptr; }
1107
1108 // These are the types we expect to return (in order of most to least
1109 // likely):
1110 //
1111 // 1. DeclRefExpr - This is the expression for the base of the structure.
1112 // It's exactly what we want to build an access to the \p counted_by
1113 // field.
1114 // 2. MemberExpr - This is the expression that has the same \p RecordDecl
1115 // as the flexble array member's lexical enclosing \p RecordDecl. This
1116 // allows us to catch things like: "p->p->array"
1117 // 3. CompoundLiteralExpr - This is for people who create something
1118 // heretical like (struct foo has a flexible array member):
1119 //
1120 // (struct foo){ 1, 2 }.blah[idx];
1121 const Expr *VisitDeclRefExpr(const DeclRefExpr *E) {
1122 return IsExpectedRecordDecl(E) ? E : nullptr;
1123 }
1124 const Expr *VisitMemberExpr(const MemberExpr *E) {
1125 if (IsExpectedRecordDecl(E) && E->isArrow())
1126 return E;
1127 const Expr *Res = Visit(E: E->getBase());
1128 return !Res && IsExpectedRecordDecl(E) ? E : Res;
1129 }
1130 const Expr *VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) {
1131 return IsExpectedRecordDecl(E) ? E : nullptr;
1132 }
1133 const Expr *VisitCallExpr(const CallExpr *E) {
1134 return IsExpectedRecordDecl(E) ? E : nullptr;
1135 }
1136
1137 const Expr *VisitArraySubscriptExpr(const ArraySubscriptExpr *E) {
1138 if (IsExpectedRecordDecl(E))
1139 return E;
1140 return Visit(E: E->getBase());
1141 }
1142 const Expr *VisitCastExpr(const CastExpr *E) {
1143 if (E->getCastKind() == CK_LValueToRValue)
1144 return IsExpectedRecordDecl(E) ? E : nullptr;
1145 return Visit(E: E->getSubExpr());
1146 }
1147 const Expr *VisitParenExpr(const ParenExpr *E) {
1148 return Visit(E: E->getSubExpr());
1149 }
1150 const Expr *VisitUnaryAddrOf(const UnaryOperator *E) {
1151 return Visit(E: E->getSubExpr());
1152 }
1153 const Expr *VisitUnaryDeref(const UnaryOperator *E) {
1154 return Visit(E: E->getSubExpr());
1155 }
1156};
1157
1158} // end anonymous namespace
1159
1160using RecIndicesTy = SmallVector<llvm::Value *, 8>;
1161
1162static bool getGEPIndicesToField(CodeGenFunction &CGF, const RecordDecl *RD,
1163 const FieldDecl *Field,
1164 RecIndicesTy &Indices) {
1165 const CGRecordLayout &Layout = CGF.CGM.getTypes().getCGRecordLayout(RD);
1166 int64_t FieldNo = -1;
1167 for (const FieldDecl *FD : RD->fields()) {
1168 if (!Layout.containsFieldDecl(FD))
1169 // This could happen if the field has a struct type that's empty. I don't
1170 // know why either.
1171 continue;
1172
1173 FieldNo = Layout.getLLVMFieldNo(FD);
1174 if (FD == Field) {
1175 Indices.emplace_back(Args: CGF.Builder.getInt32(C: FieldNo));
1176 return true;
1177 }
1178
1179 QualType Ty = FD->getType();
1180 if (Ty->isRecordType()) {
1181 if (getGEPIndicesToField(CGF, RD: Ty->getAsRecordDecl(), Field, Indices)) {
1182 if (RD->isUnion())
1183 FieldNo = 0;
1184 Indices.emplace_back(Args: CGF.Builder.getInt32(C: FieldNo));
1185 return true;
1186 }
1187 }
1188 }
1189
1190 return false;
1191}
1192
1193llvm::Value *CodeGenFunction::GetCountedByFieldExprGEP(
1194 const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl) {
1195 // Find the record containing the count field. Walk up through anonymous
1196 // structs/unions (which are transparent in C) but stop at named records.
1197 // Using getOuterLexicalRecordContext() here would be wrong because it walks
1198 // past named nested structs to the outermost record, causing a crash when a
1199 // struct with a counted_by FAM is defined nested inside another struct.
1200 const RecordDecl *RD = CountDecl->getParent();
1201 while (RD->isAnonymousStructOrUnion()) {
1202 const auto *Parent = dyn_cast<RecordDecl>(Val: RD->getLexicalParent());
1203 if (!Parent)
1204 break;
1205 RD = Parent;
1206 }
1207
1208 // Find the base struct expr (i.e. p in p->a.b.c.d).
1209 const Expr *StructBase = StructAccessBase(RD).Visit(E: Base);
1210 if (!StructBase || StructBase->HasSideEffects(Ctx: getContext()))
1211 return nullptr;
1212
1213 llvm::Value *Res = nullptr;
1214 if (StructBase->getType()->isPointerType()) {
1215 LValueBaseInfo BaseInfo;
1216 TBAAAccessInfo TBAAInfo;
1217 Address Addr = EmitPointerWithAlignment(Addr: StructBase, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
1218 Res = Addr.emitRawPointer(CGF&: *this);
1219 } else if (StructBase->isLValue()) {
1220 LValue LV = EmitLValue(E: StructBase);
1221 Address Addr = LV.getAddress();
1222 Res = Addr.emitRawPointer(CGF&: *this);
1223 } else {
1224 return nullptr;
1225 }
1226
1227 RecIndicesTy Indices;
1228 getGEPIndicesToField(CGF&: *this, RD, Field: CountDecl, Indices);
1229 if (Indices.empty())
1230 return nullptr;
1231
1232 Indices.push_back(Elt: Builder.getInt32(C: 0));
1233 CanQualType T = CGM.getContext().getCanonicalTagType(TD: RD);
1234 return Builder.CreateInBoundsGEP(Ty: ConvertType(T), Ptr: Res,
1235 IdxList: RecIndicesTy(llvm::reverse(C&: Indices)),
1236 Name: "counted_by.gep");
1237}
1238
1239/// This method is typically called in contexts where we can't generate
1240/// side-effects, like in __builtin_dynamic_object_size. When finding
1241/// expressions, only choose those that have either already been emitted or can
1242/// be loaded without side-effects.
1243///
1244/// - \p FAMDecl: the \p Decl for the flexible array member. It may not be
1245/// within the top-level struct.
1246/// - \p CountDecl: must be within the same non-anonymous struct as \p FAMDecl.
1247llvm::Value *CodeGenFunction::EmitLoadOfCountedByField(
1248 const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl) {
1249 if (llvm::Value *GEP = GetCountedByFieldExprGEP(Base, FAMDecl, CountDecl))
1250 return Builder.CreateAlignedLoad(Ty: ConvertType(T: CountDecl->getType()), Addr: GEP,
1251 Align: getIntAlign(), Name: "counted_by.load");
1252 return nullptr;
1253}
1254
1255void CodeGenFunction::EmitBoundsCheck(const Expr *ArrayExpr,
1256 const Expr *ArrayExprBase,
1257 llvm::Value *IndexVal, QualType IndexType,
1258 bool Accessed) {
1259 assert(SanOpts.has(SanitizerKind::ArrayBounds) &&
1260 "should not be called unless adding bounds checks");
1261 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
1262 getLangOpts().getStrictFlexArraysLevel();
1263 QualType ArrayExprBaseType;
1264 llvm::Value *BoundsVal = getArrayIndexingBound(
1265 CGF&: *this, Base: ArrayExprBase, IndexedType&: ArrayExprBaseType, StrictFlexArraysLevel);
1266
1267 EmitBoundsCheckImpl(ArrayExpr, ArrayBaseType: ArrayExprBaseType, IndexVal, IndexType,
1268 BoundsVal, BoundsType: getContext().getSizeType(), Accessed);
1269}
1270
1271void CodeGenFunction::EmitBoundsCheckImpl(const Expr *ArrayExpr,
1272 QualType ArrayBaseType,
1273 llvm::Value *IndexVal,
1274 QualType IndexType,
1275 llvm::Value *BoundsVal,
1276 QualType BoundsType, bool Accessed) {
1277 if (!BoundsVal)
1278 return;
1279
1280 auto CheckKind = SanitizerKind::SO_ArrayBounds;
1281 auto CheckHandler = SanitizerHandler::OutOfBounds;
1282 SanitizerDebugLocation SanScope(this, {CheckKind}, CheckHandler);
1283
1284 // All hail the C implicit type conversion rules!!!
1285 bool IndexSigned = IndexType->isSignedIntegerOrEnumerationType();
1286 bool BoundsSigned = BoundsType->isSignedIntegerOrEnumerationType();
1287
1288 const ASTContext &Ctx = getContext();
1289 llvm::Type *Ty = ConvertType(
1290 T: Ctx.getTypeSize(T: IndexType) >= Ctx.getTypeSize(T: BoundsType) ? IndexType
1291 : BoundsType);
1292
1293 llvm::Value *IndexInst = Builder.CreateIntCast(V: IndexVal, DestTy: Ty, isSigned: IndexSigned);
1294 llvm::Value *BoundsInst = Builder.CreateIntCast(V: BoundsVal, DestTy: Ty, isSigned: false);
1295
1296 llvm::Constant *StaticData[] = {
1297 EmitCheckSourceLocation(Loc: ArrayExpr->getExprLoc()),
1298 EmitCheckTypeDescriptor(T: ArrayBaseType),
1299 EmitCheckTypeDescriptor(T: IndexType),
1300 };
1301
1302 llvm::Value *Check = Accessed ? Builder.CreateICmpULT(LHS: IndexInst, RHS: BoundsInst)
1303 : Builder.CreateICmpULE(LHS: IndexInst, RHS: BoundsInst);
1304
1305 if (BoundsSigned) {
1306 // Don't allow a negative bounds.
1307 llvm::Value *Cmp = Builder.CreateICmpSGT(
1308 LHS: BoundsVal, RHS: llvm::ConstantInt::get(Ty: BoundsVal->getType(), V: 0));
1309 Check = Builder.CreateAnd(LHS: Cmp, RHS: Check);
1310 }
1311
1312 EmitCheck(Checked: std::make_pair(x&: Check, y&: CheckKind), Check: CheckHandler, StaticArgs: StaticData,
1313 DynamicArgs: IndexInst);
1314}
1315
1316llvm::MDNode *CodeGenFunction::buildAllocToken(QualType AllocType) {
1317 auto ATMD = infer_alloc::getAllocTokenMetadata(T: AllocType, Ctx: getContext());
1318 if (!ATMD)
1319 return nullptr;
1320
1321 llvm::MDBuilder MDB(getLLVMContext());
1322 auto *TypeNameMD = MDB.createString(Str: ATMD->TypeName);
1323 auto *ContainsPtrC = Builder.getInt1(V: ATMD->ContainsPointer);
1324 auto *ContainsPtrMD = MDB.createConstant(C: ContainsPtrC);
1325
1326 // Format: !{<type-name>, <contains-pointer>}
1327 return llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: {TypeNameMD, ContainsPtrMD});
1328}
1329
1330void CodeGenFunction::EmitAllocToken(llvm::CallBase *CB, QualType AllocType) {
1331 assert(SanOpts.has(SanitizerKind::AllocToken) &&
1332 "Only needed with -fsanitize=alloc-token");
1333 CB->setMetadata(KindID: llvm::LLVMContext::MD_alloc_token,
1334 Node: buildAllocToken(AllocType));
1335}
1336
1337llvm::MDNode *CodeGenFunction::buildAllocToken(const CallExpr *E) {
1338 QualType AllocType = infer_alloc::inferPossibleType(E, Ctx: getContext(), CastE: CurCast);
1339 if (!AllocType.isNull())
1340 return buildAllocToken(AllocType);
1341 return nullptr;
1342}
1343
1344void CodeGenFunction::EmitAllocToken(llvm::CallBase *CB, const CallExpr *E) {
1345 assert(SanOpts.has(SanitizerKind::AllocToken) &&
1346 "Only needed with -fsanitize=alloc-token");
1347 if (llvm::MDNode *MDN = buildAllocToken(E))
1348 CB->setMetadata(KindID: llvm::LLVMContext::MD_alloc_token, Node: MDN);
1349}
1350
1351CodeGenFunction::ComplexPairTy CodeGenFunction::
1352EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
1353 bool isInc, bool isPre) {
1354 ComplexPairTy InVal = EmitLoadOfComplex(src: LV, loc: E->getExprLoc());
1355
1356 llvm::Value *NextVal;
1357 if (isa<llvm::IntegerType>(Val: InVal.first->getType())) {
1358 uint64_t AmountVal = isInc ? 1 : -1;
1359 NextVal = llvm::ConstantInt::get(Ty: InVal.first->getType(), V: AmountVal, IsSigned: true);
1360
1361 // Add the inc/dec to the real part.
1362 NextVal = Builder.CreateAdd(LHS: InVal.first, RHS: NextVal, Name: isInc ? "inc" : "dec");
1363 } else {
1364 QualType ElemTy = E->getType()->castAs<ComplexType>()->getElementType();
1365 llvm::APFloat FVal(getContext().getFloatTypeSemantics(T: ElemTy), 1);
1366 if (!isInc)
1367 FVal.changeSign();
1368 NextVal = llvm::ConstantFP::get(Context&: getLLVMContext(), V: FVal);
1369
1370 // Add the inc/dec to the real part.
1371 NextVal = Builder.CreateFAdd(L: InVal.first, R: NextVal, Name: isInc ? "inc" : "dec");
1372 }
1373
1374 ComplexPairTy IncVal(NextVal, InVal.second);
1375
1376 // Store the updated result through the lvalue.
1377 EmitStoreOfComplex(V: IncVal, dest: LV, /*init*/ isInit: false);
1378 if (getLangOpts().OpenMP)
1379 CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF&: *this,
1380 LHS: E->getSubExpr());
1381
1382 // If this is a postinc, return the value read from memory, otherwise use the
1383 // updated value.
1384 return isPre ? IncVal : InVal;
1385}
1386
1387void CodeGenModule::EmitExplicitCastExprType(const ExplicitCastExpr *E,
1388 CodeGenFunction *CGF) {
1389 // Bind VLAs in the cast type.
1390 if (CGF && E->getType()->isVariablyModifiedType())
1391 CGF->EmitVariablyModifiedType(Ty: E->getType());
1392
1393 if (CGDebugInfo *DI = getModuleDebugInfo())
1394 DI->EmitExplicitCastType(Ty: E->getType());
1395}
1396
1397//===----------------------------------------------------------------------===//
1398// LValue Expression Emission
1399//===----------------------------------------------------------------------===//
1400
1401static CharUnits getArrayElementAlign(CharUnits arrayAlign, llvm::Value *idx,
1402 CharUnits eltSize) {
1403 // If we have a constant index, we can use the exact offset of the
1404 // element we're accessing.
1405 if (auto *constantIdx = dyn_cast<llvm::ConstantInt>(Val: idx)) {
1406 CharUnits offset = constantIdx->getZExtValue() * eltSize;
1407 return arrayAlign.alignmentAtOffset(offset);
1408 }
1409
1410 // Otherwise, use the worst-case alignment for any element.
1411 return arrayAlign.alignmentOfArrayElement(elementSize: eltSize);
1412}
1413
1414/// Emit pointer + index arithmetic.
1415static Address emitPointerArithmetic(CodeGenFunction &CGF,
1416 const BinaryOperator *BO,
1417 LValueBaseInfo *BaseInfo,
1418 TBAAAccessInfo *TBAAInfo,
1419 KnownNonNull_t IsKnownNonNull) {
1420 assert(BO->isAdditiveOp() && "Expect an addition or subtraction.");
1421 Expr *pointerOperand = BO->getLHS();
1422 Expr *indexOperand = BO->getRHS();
1423 bool isSubtraction = BO->getOpcode() == BO_Sub;
1424
1425 Address BaseAddr = Address::invalid();
1426 llvm::Value *index = nullptr;
1427 // In a subtraction, the LHS is always the pointer.
1428 // Note: do not change the evaluation order.
1429 if (!isSubtraction && !pointerOperand->getType()->isAnyPointerType()) {
1430 std::swap(a&: pointerOperand, b&: indexOperand);
1431 index = CGF.EmitScalarExpr(E: indexOperand);
1432 BaseAddr = CGF.EmitPointerWithAlignment(Addr: pointerOperand, BaseInfo, TBAAInfo,
1433 IsKnownNonNull: NotKnownNonNull);
1434 } else {
1435 BaseAddr = CGF.EmitPointerWithAlignment(Addr: pointerOperand, BaseInfo, TBAAInfo,
1436 IsKnownNonNull: NotKnownNonNull);
1437 index = CGF.EmitScalarExpr(E: indexOperand);
1438 }
1439
1440 llvm::Value *pointer = BaseAddr.getBasePointer();
1441 llvm::Value *Res = CGF.EmitPointerArithmetic(
1442 BO, pointerOperand, pointer, indexOperand, index, isSubtraction);
1443 QualType PointeeTy = BO->getType()->getPointeeType();
1444 CharUnits Align =
1445 getArrayElementAlign(arrayAlign: BaseAddr.getAlignment(), idx: index,
1446 eltSize: CGF.getContext().getTypeSizeInChars(T: PointeeTy));
1447 return Address(Res, CGF.ConvertTypeForMem(T: PointeeTy), Align,
1448 CGF.CGM.getPointerAuthInfoForPointeeType(type: PointeeTy),
1449 /*Offset=*/nullptr, IsKnownNonNull);
1450}
1451
1452static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo,
1453 TBAAAccessInfo *TBAAInfo,
1454 KnownNonNull_t IsKnownNonNull,
1455 CodeGenFunction &CGF) {
1456 // We allow this with ObjC object pointers because of fragile ABIs.
1457 assert(E->getType()->isPointerType() ||
1458 E->getType()->isObjCObjectPointerType());
1459 E = E->IgnoreParens();
1460
1461 // Casts:
1462 if (const CastExpr *CE = dyn_cast<CastExpr>(Val: E)) {
1463 if (const auto *ECE = dyn_cast<ExplicitCastExpr>(Val: CE))
1464 CGF.CGM.EmitExplicitCastExprType(E: ECE, CGF: &CGF);
1465
1466 switch (CE->getCastKind()) {
1467 // Non-converting casts (but not C's implicit conversion from void*).
1468 case CK_BitCast:
1469 case CK_NoOp:
1470 case CK_AddressSpaceConversion:
1471 if (auto PtrTy = CE->getSubExpr()->getType()->getAs<PointerType>()) {
1472 if (PtrTy->getPointeeType()->isVoidType())
1473 break;
1474
1475 LValueBaseInfo InnerBaseInfo;
1476 TBAAAccessInfo InnerTBAAInfo;
1477 Address Addr = CGF.EmitPointerWithAlignment(
1478 Addr: CE->getSubExpr(), BaseInfo: &InnerBaseInfo, TBAAInfo: &InnerTBAAInfo, IsKnownNonNull);
1479 if (BaseInfo) *BaseInfo = InnerBaseInfo;
1480 if (TBAAInfo) *TBAAInfo = InnerTBAAInfo;
1481
1482 if (isa<ExplicitCastExpr>(Val: CE)) {
1483 LValueBaseInfo TargetTypeBaseInfo;
1484 TBAAAccessInfo TargetTypeTBAAInfo;
1485 CharUnits Align = CGF.CGM.getNaturalPointeeTypeAlignment(
1486 T: E->getType(), BaseInfo: &TargetTypeBaseInfo, TBAAInfo: &TargetTypeTBAAInfo);
1487 if (TBAAInfo)
1488 *TBAAInfo =
1489 CGF.CGM.mergeTBAAInfoForCast(SourceInfo: *TBAAInfo, TargetInfo: TargetTypeTBAAInfo);
1490 // If the source l-value is opaque, honor the alignment of the
1491 // casted-to type.
1492 if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) {
1493 if (BaseInfo)
1494 BaseInfo->mergeForCast(Info: TargetTypeBaseInfo);
1495 Addr.setAlignment(Align);
1496 }
1497 }
1498
1499 if (CGF.SanOpts.has(K: SanitizerKind::CFIUnrelatedCast) &&
1500 CE->getCastKind() == CK_BitCast) {
1501 if (auto PT = E->getType()->getAs<PointerType>())
1502 CGF.EmitVTablePtrCheckForCast(T: PT->getPointeeType(), Derived: Addr,
1503 /*MayBeNull=*/true,
1504 TCK: CodeGenFunction::CFITCK_UnrelatedCast,
1505 Loc: CE->getBeginLoc());
1506 }
1507
1508 llvm::Type *ElemTy =
1509 CGF.ConvertTypeForMem(T: E->getType()->getPointeeType());
1510 Addr = Addr.withElementType(ElemTy);
1511 if (CE->getCastKind() == CK_AddressSpaceConversion)
1512 Addr = CGF.Builder.CreateAddrSpaceCast(
1513 Addr, Ty: CGF.ConvertType(T: E->getType()), ElementTy: ElemTy);
1514
1515 return CGF.authPointerToPointerCast(Ptr: Addr, SourceType: CE->getSubExpr()->getType(),
1516 DestType: CE->getType());
1517 }
1518 break;
1519
1520 // Array-to-pointer decay.
1521 case CK_ArrayToPointerDecay:
1522 return CGF.EmitArrayToPointerDecay(Array: CE->getSubExpr(), BaseInfo, TBAAInfo);
1523
1524 // Derived-to-base conversions.
1525 case CK_UncheckedDerivedToBase:
1526 case CK_DerivedToBase: {
1527 // TODO: Support accesses to members of base classes in TBAA. For now, we
1528 // conservatively pretend that the complete object is of the base class
1529 // type.
1530 if (TBAAInfo)
1531 *TBAAInfo = CGF.CGM.getTBAAAccessInfo(AccessType: E->getType());
1532 Address Addr = CGF.EmitPointerWithAlignment(
1533 Addr: CE->getSubExpr(), BaseInfo, TBAAInfo: nullptr,
1534 IsKnownNonNull: (KnownNonNull_t)(IsKnownNonNull ||
1535 CE->getCastKind() == CK_UncheckedDerivedToBase));
1536 auto Derived = CE->getSubExpr()->getType()->getPointeeCXXRecordDecl();
1537 return CGF.GetAddressOfBaseClass(
1538 Value: Addr, Derived, PathBegin: CE->path_begin(), PathEnd: CE->path_end(),
1539 NullCheckValue: CGF.ShouldNullCheckClassCastValue(Cast: CE), Loc: CE->getExprLoc());
1540 }
1541
1542 // TODO: Is there any reason to treat base-to-derived conversions
1543 // specially?
1544 default:
1545 break;
1546 }
1547 }
1548
1549 // Unary &.
1550 if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(Val: E)) {
1551 if (UO->getOpcode() == UO_AddrOf) {
1552 LValue LV = CGF.EmitLValue(E: UO->getSubExpr(), IsKnownNonNull);
1553 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1554 if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1555 return LV.getAddress();
1556 }
1557 }
1558
1559 // std::addressof and variants.
1560 if (auto *Call = dyn_cast<CallExpr>(Val: E)) {
1561 switch (Call->getBuiltinCallee()) {
1562 default:
1563 break;
1564 case Builtin::BIaddressof:
1565 case Builtin::BI__addressof:
1566 case Builtin::BI__builtin_addressof: {
1567 LValue LV = CGF.EmitLValue(E: Call->getArg(Arg: 0), IsKnownNonNull);
1568 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1569 if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1570 return LV.getAddress();
1571 }
1572 }
1573 }
1574
1575 // Pointer arithmetic: pointer +/- index.
1576 if (auto *BO = dyn_cast<BinaryOperator>(Val: E)) {
1577 if (BO->isAdditiveOp())
1578 return emitPointerArithmetic(CGF, BO, BaseInfo, TBAAInfo, IsKnownNonNull);
1579 }
1580
1581 // TODO: conditional operators, comma.
1582
1583 // Otherwise, use the alignment of the type.
1584 return CGF.makeNaturalAddressForPointer(
1585 Ptr: CGF.EmitScalarExpr(E), T: E->getType()->getPointeeType(), Alignment: CharUnits(),
1586 /*ForPointeeType=*/true, BaseInfo, TBAAInfo, IsKnownNonNull);
1587}
1588
1589/// EmitPointerWithAlignment - Given an expression of pointer type, try to
1590/// derive a more accurate bound on the alignment of the pointer.
1591Address CodeGenFunction::EmitPointerWithAlignment(
1592 const Expr *E, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo,
1593 KnownNonNull_t IsKnownNonNull) {
1594 Address Addr =
1595 ::EmitPointerWithAlignment(E, BaseInfo, TBAAInfo, IsKnownNonNull, CGF&: *this);
1596 if (IsKnownNonNull && !Addr.isKnownNonNull())
1597 Addr.setKnownNonNull();
1598 return Addr;
1599}
1600
1601llvm::Value *CodeGenFunction::EmitNonNullRValueCheck(RValue RV, QualType T) {
1602 llvm::Value *V = RV.getScalarVal();
1603 if (auto MPT = T->getAs<MemberPointerType>())
1604 return CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF&: *this, MemPtr: V, MPT);
1605 return Builder.CreateICmpNE(LHS: V, RHS: llvm::Constant::getNullValue(Ty: V->getType()));
1606}
1607
1608RValue CodeGenFunction::GetUndefRValue(QualType Ty) {
1609 if (Ty->isVoidType())
1610 return RValue::get(V: nullptr);
1611
1612 switch (getEvaluationKind(T: Ty)) {
1613 case TEK_Complex: {
1614 llvm::Type *EltTy =
1615 ConvertType(T: Ty->castAs<ComplexType>()->getElementType());
1616 llvm::Value *U = llvm::UndefValue::get(T: EltTy);
1617 return RValue::getComplex(C: std::make_pair(x&: U, y&: U));
1618 }
1619
1620 // If this is a use of an undefined aggregate type, the aggregate must have an
1621 // identifiable address. Just because the contents of the value are undefined
1622 // doesn't mean that the address can't be taken and compared.
1623 case TEK_Aggregate: {
1624 Address DestPtr = CreateMemTemp(Ty, Name: "undef.agg.tmp");
1625 return RValue::getAggregate(addr: DestPtr);
1626 }
1627
1628 case TEK_Scalar:
1629 return RValue::get(V: llvm::UndefValue::get(T: ConvertType(T: Ty)));
1630 }
1631 llvm_unreachable("bad evaluation kind");
1632}
1633
1634RValue CodeGenFunction::EmitUnsupportedRValue(const Expr *E,
1635 const char *Name) {
1636 ErrorUnsupported(S: E, Type: Name);
1637 return GetUndefRValue(Ty: E->getType());
1638}
1639
1640LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E,
1641 const char *Name) {
1642 ErrorUnsupported(S: E, Type: Name);
1643 llvm::Type *ElTy = ConvertType(T: E->getType());
1644 llvm::Type *Ty = DefaultPtrTy;
1645 return MakeAddrLValue(
1646 Addr: Address(llvm::UndefValue::get(T: Ty), ElTy, CharUnits::One()), T: E->getType());
1647}
1648
1649bool CodeGenFunction::IsWrappedCXXThis(const Expr *Obj) {
1650 const Expr *Base = Obj;
1651 while (!isa<CXXThisExpr>(Val: Base)) {
1652 // The result of a dynamic_cast can be null.
1653 if (isa<CXXDynamicCastExpr>(Val: Base))
1654 return false;
1655
1656 if (const auto *CE = dyn_cast<CastExpr>(Val: Base)) {
1657 Base = CE->getSubExpr();
1658 } else if (const auto *PE = dyn_cast<ParenExpr>(Val: Base)) {
1659 Base = PE->getSubExpr();
1660 } else if (const auto *UO = dyn_cast<UnaryOperator>(Val: Base)) {
1661 if (UO->getOpcode() == UO_Extension)
1662 Base = UO->getSubExpr();
1663 else
1664 return false;
1665 } else {
1666 return false;
1667 }
1668 }
1669 return true;
1670}
1671
1672LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {
1673 LValue LV;
1674 if (SanOpts.has(K: SanitizerKind::ArrayBounds) && isa<ArraySubscriptExpr>(Val: E))
1675 LV = EmitArraySubscriptExpr(E: cast<ArraySubscriptExpr>(Val: E), /*Accessed*/true);
1676 else
1677 LV = EmitLValue(E);
1678 if (!isa<DeclRefExpr>(Val: E) && !LV.isBitField() && LV.isSimple()) {
1679 SanitizerSet SkippedChecks;
1680 if (const auto *ME = dyn_cast<MemberExpr>(Val: E)) {
1681 bool IsBaseCXXThis = IsWrappedCXXThis(Obj: ME->getBase());
1682 if (IsBaseCXXThis)
1683 SkippedChecks.set(K: SanitizerKind::Alignment, Value: true);
1684 if (IsBaseCXXThis || isa<DeclRefExpr>(Val: ME->getBase()))
1685 SkippedChecks.set(K: SanitizerKind::Null, Value: true);
1686 }
1687 EmitTypeCheck(TCK, Loc: E->getExprLoc(), LV, Type: E->getType(), SkippedChecks);
1688 }
1689 return LV;
1690}
1691
1692/// EmitLValue - Emit code to compute a designator that specifies the location
1693/// of the expression.
1694///
1695/// This can return one of two things: a simple address or a bitfield reference.
1696/// In either case, the LLVM Value* in the LValue structure is guaranteed to be
1697/// an LLVM pointer type.
1698///
1699/// If this returns a bitfield reference, nothing about the pointee type of the
1700/// LLVM value is known: For example, it may not be a pointer to an integer.
1701///
1702/// If this returns a normal address, and if the lvalue's C type is fixed size,
1703/// this method guarantees that the returned pointer type will point to an LLVM
1704/// type of the same size of the lvalue's type. If the lvalue has a variable
1705/// length type, this is not possible.
1706///
1707LValue CodeGenFunction::EmitLValue(const Expr *E,
1708 KnownNonNull_t IsKnownNonNull) {
1709 // Running with sufficient stack space to avoid deeply nested expressions
1710 // cause a stack overflow.
1711 LValue LV;
1712 CGM.runWithSufficientStackSpace(
1713 Loc: E->getExprLoc(), Fn: [&] { LV = EmitLValueHelper(E, IsKnownNonNull); });
1714
1715 if (IsKnownNonNull && !LV.isKnownNonNull())
1716 LV.setKnownNonNull();
1717 return LV;
1718}
1719
1720LValue CodeGenFunction::EmitLValueHelper(const Expr *E,
1721 KnownNonNull_t IsKnownNonNull) {
1722 ApplyDebugLocation DL(*this, E);
1723 switch (E->getStmtClass()) {
1724 default: return EmitUnsupportedLValue(E, Name: "l-value expression");
1725
1726 case Expr::ObjCPropertyRefExprClass:
1727 llvm_unreachable("cannot emit a property reference directly");
1728
1729 case Expr::ObjCSelectorExprClass:
1730 return EmitObjCSelectorLValue(E: cast<ObjCSelectorExpr>(Val: E));
1731 case Expr::ObjCIsaExprClass:
1732 return EmitObjCIsaExpr(E: cast<ObjCIsaExpr>(Val: E));
1733 case Expr::BinaryOperatorClass:
1734 return EmitBinaryOperatorLValue(E: cast<BinaryOperator>(Val: E));
1735 case Expr::CompoundAssignOperatorClass: {
1736 QualType Ty = E->getType();
1737 if (const AtomicType *AT = Ty->getAs<AtomicType>())
1738 Ty = AT->getValueType();
1739 if (!Ty->isAnyComplexType())
1740 return EmitCompoundAssignmentLValue(E: cast<CompoundAssignOperator>(Val: E));
1741 return EmitComplexCompoundAssignmentLValue(E: cast<CompoundAssignOperator>(Val: E));
1742 }
1743 case Expr::CallExprClass:
1744 case Expr::CXXMemberCallExprClass:
1745 case Expr::CXXOperatorCallExprClass:
1746 case Expr::UserDefinedLiteralClass:
1747 return EmitCallExprLValue(E: cast<CallExpr>(Val: E));
1748 case Expr::CXXRewrittenBinaryOperatorClass:
1749 return EmitLValue(E: cast<CXXRewrittenBinaryOperator>(Val: E)->getSemanticForm(),
1750 IsKnownNonNull);
1751 case Expr::VAArgExprClass:
1752 return EmitVAArgExprLValue(E: cast<VAArgExpr>(Val: E));
1753 case Expr::DeclRefExprClass:
1754 return EmitDeclRefLValue(E: cast<DeclRefExpr>(Val: E));
1755 case Expr::ConstantExprClass: {
1756 const ConstantExpr *CE = cast<ConstantExpr>(Val: E);
1757 if (llvm::Value *Result = ConstantEmitter(*this).tryEmitConstantExpr(CE))
1758 return MakeNaturalAlignPointeeAddrLValue(V: Result, T: CE->getType());
1759 return EmitLValue(E: cast<ConstantExpr>(Val: E)->getSubExpr(), IsKnownNonNull);
1760 }
1761 case Expr::ParenExprClass:
1762 return EmitLValue(E: cast<ParenExpr>(Val: E)->getSubExpr(), IsKnownNonNull);
1763 case Expr::GenericSelectionExprClass:
1764 return EmitLValue(E: cast<GenericSelectionExpr>(Val: E)->getResultExpr(),
1765 IsKnownNonNull);
1766 case Expr::PredefinedExprClass:
1767 return EmitPredefinedLValue(E: cast<PredefinedExpr>(Val: E));
1768 case Expr::StringLiteralClass:
1769 return EmitStringLiteralLValue(E: cast<StringLiteral>(Val: E));
1770 case Expr::ObjCEncodeExprClass:
1771 return EmitObjCEncodeExprLValue(E: cast<ObjCEncodeExpr>(Val: E));
1772 case Expr::PseudoObjectExprClass:
1773 return EmitPseudoObjectLValue(e: cast<PseudoObjectExpr>(Val: E));
1774 case Expr::InitListExprClass:
1775 return EmitInitListLValue(E: cast<InitListExpr>(Val: E));
1776 case Expr::CXXTemporaryObjectExprClass:
1777 case Expr::CXXConstructExprClass:
1778 return EmitCXXConstructLValue(E: cast<CXXConstructExpr>(Val: E));
1779 case Expr::CXXBindTemporaryExprClass:
1780 return EmitCXXBindTemporaryLValue(E: cast<CXXBindTemporaryExpr>(Val: E));
1781 case Expr::CXXUuidofExprClass:
1782 return EmitCXXUuidofLValue(E: cast<CXXUuidofExpr>(Val: E));
1783 case Expr::LambdaExprClass:
1784 return EmitAggExprToLValue(E);
1785
1786 case Expr::ExprWithCleanupsClass: {
1787 const auto *cleanups = cast<ExprWithCleanups>(Val: E);
1788 RunCleanupsScope Scope(*this);
1789 LValue LV = EmitLValue(E: cleanups->getSubExpr(), IsKnownNonNull);
1790 if (LV.isSimple()) {
1791 // Defend against branches out of gnu statement expressions surrounded by
1792 // cleanups.
1793 Address Addr = LV.getAddress();
1794 llvm::Value *V = Addr.getBasePointer();
1795 Scope.ForceCleanup(ValuesToReload: {&V});
1796 Addr.replaceBasePointer(P: V);
1797 return LValue::MakeAddr(Addr, type: LV.getType(), Context&: getContext(),
1798 BaseInfo: LV.getBaseInfo(), TBAAInfo: LV.getTBAAInfo());
1799 }
1800 // FIXME: Is it possible to create an ExprWithCleanups that produces a
1801 // bitfield lvalue or some other non-simple lvalue?
1802 return LV;
1803 }
1804
1805 case Expr::CXXDefaultArgExprClass: {
1806 auto *DAE = cast<CXXDefaultArgExpr>(Val: E);
1807 CXXDefaultArgExprScope Scope(*this, DAE);
1808 return EmitLValue(E: DAE->getExpr(), IsKnownNonNull);
1809 }
1810 case Expr::CXXDefaultInitExprClass: {
1811 auto *DIE = cast<CXXDefaultInitExpr>(Val: E);
1812 CXXDefaultInitExprScope Scope(*this, DIE);
1813 return EmitLValue(E: DIE->getExpr(), IsKnownNonNull);
1814 }
1815 case Expr::CXXTypeidExprClass:
1816 return EmitCXXTypeidLValue(E: cast<CXXTypeidExpr>(Val: E));
1817
1818 case Expr::ObjCMessageExprClass:
1819 return EmitObjCMessageExprLValue(E: cast<ObjCMessageExpr>(Val: E));
1820 case Expr::ObjCIvarRefExprClass:
1821 return EmitObjCIvarRefLValue(E: cast<ObjCIvarRefExpr>(Val: E));
1822 case Expr::StmtExprClass:
1823 return EmitStmtExprLValue(E: cast<StmtExpr>(Val: E));
1824 case Expr::UnaryOperatorClass:
1825 return EmitUnaryOpLValue(E: cast<UnaryOperator>(Val: E));
1826 case Expr::ArraySubscriptExprClass:
1827 return EmitArraySubscriptExpr(E: cast<ArraySubscriptExpr>(Val: E));
1828 case Expr::MatrixSingleSubscriptExprClass:
1829 return EmitMatrixSingleSubscriptExpr(E: cast<MatrixSingleSubscriptExpr>(Val: E));
1830 case Expr::MatrixSubscriptExprClass:
1831 return EmitMatrixSubscriptExpr(E: cast<MatrixSubscriptExpr>(Val: E));
1832 case Expr::ArraySectionExprClass:
1833 return EmitArraySectionExpr(E: cast<ArraySectionExpr>(Val: E));
1834 case Expr::ExtVectorElementExprClass:
1835 return EmitExtVectorElementExpr(E: cast<ExtVectorElementExpr>(Val: E));
1836 case Expr::MatrixElementExprClass:
1837 return EmitMatrixElementExpr(E: cast<MatrixElementExpr>(Val: E));
1838 case Expr::CXXThisExprClass:
1839 return MakeAddrLValue(Addr: LoadCXXThisAddress(), T: E->getType());
1840 case Expr::MemberExprClass:
1841 return EmitMemberExpr(E: cast<MemberExpr>(Val: E));
1842 case Expr::CompoundLiteralExprClass:
1843 return EmitCompoundLiteralLValue(E: cast<CompoundLiteralExpr>(Val: E));
1844 case Expr::ConditionalOperatorClass:
1845 return EmitConditionalOperatorLValue(E: cast<ConditionalOperator>(Val: E));
1846 case Expr::BinaryConditionalOperatorClass:
1847 return EmitConditionalOperatorLValue(E: cast<BinaryConditionalOperator>(Val: E));
1848 case Expr::ChooseExprClass:
1849 return EmitLValue(E: cast<ChooseExpr>(Val: E)->getChosenSubExpr(), IsKnownNonNull);
1850 case Expr::OpaqueValueExprClass:
1851 return EmitOpaqueValueLValue(e: cast<OpaqueValueExpr>(Val: E));
1852 case Expr::SubstNonTypeTemplateParmExprClass:
1853 return EmitLValue(E: cast<SubstNonTypeTemplateParmExpr>(Val: E)->getReplacement(),
1854 IsKnownNonNull);
1855 case Expr::ImplicitCastExprClass:
1856 case Expr::CStyleCastExprClass:
1857 case Expr::CXXFunctionalCastExprClass:
1858 case Expr::CXXStaticCastExprClass:
1859 case Expr::CXXDynamicCastExprClass:
1860 case Expr::CXXReinterpretCastExprClass:
1861 case Expr::CXXConstCastExprClass:
1862 case Expr::CXXAddrspaceCastExprClass:
1863 case Expr::ObjCBridgedCastExprClass:
1864 return EmitCastLValue(E: cast<CastExpr>(Val: E));
1865
1866 case Expr::MaterializeTemporaryExprClass:
1867 return EmitMaterializeTemporaryExpr(M: cast<MaterializeTemporaryExpr>(Val: E));
1868
1869 case Expr::CoawaitExprClass:
1870 return EmitCoawaitLValue(E: cast<CoawaitExpr>(Val: E));
1871 case Expr::CoyieldExprClass:
1872 return EmitCoyieldLValue(E: cast<CoyieldExpr>(Val: E));
1873 case Expr::PackIndexingExprClass:
1874 return EmitLValue(E: cast<PackIndexingExpr>(Val: E)->getSelectedExpr());
1875 case Expr::HLSLOutArgExprClass:
1876 llvm_unreachable("cannot emit a HLSL out argument directly");
1877 }
1878}
1879
1880/// Given an object of the given canonical type, can we safely copy a
1881/// value out of it based on its initializer?
1882static bool isConstantEmittableObjectType(QualType type) {
1883 assert(type.isCanonical());
1884 assert(!type->isReferenceType());
1885
1886 // Must be const-qualified but non-volatile.
1887 Qualifiers qs = type.getLocalQualifiers();
1888 if (!qs.hasConst() || qs.hasVolatile()) return false;
1889
1890 // Otherwise, all object types satisfy this except C++ classes with
1891 // mutable subobjects or non-trivial copy/destroy behavior.
1892 if (const auto *RT = dyn_cast<RecordType>(Val&: type))
1893 if (const auto *RD = dyn_cast<CXXRecordDecl>(Val: RT->getDecl())) {
1894 RD = RD->getDefinitionOrSelf();
1895 if (RD->hasMutableFields() || !RD->isTrivial())
1896 return false;
1897 }
1898
1899 return true;
1900}
1901
1902/// Can we constant-emit a load of a reference to a variable of the
1903/// given type? This is different from predicates like
1904/// Decl::mightBeUsableInConstantExpressions because we do want it to apply
1905/// in situations that don't necessarily satisfy the language's rules
1906/// for this (e.g. C++'s ODR-use rules). For example, we want to able
1907/// to do this with const float variables even if those variables
1908/// aren't marked 'constexpr'.
1909enum ConstantEmissionKind {
1910 CEK_None,
1911 CEK_AsReferenceOnly,
1912 CEK_AsValueOrReference,
1913 CEK_AsValueOnly
1914};
1915static ConstantEmissionKind checkVarTypeForConstantEmission(QualType type) {
1916 type = type.getCanonicalType();
1917 if (const auto *ref = dyn_cast<ReferenceType>(Val&: type)) {
1918 if (isConstantEmittableObjectType(type: ref->getPointeeType()))
1919 return CEK_AsValueOrReference;
1920 return CEK_AsReferenceOnly;
1921 }
1922 if (isConstantEmittableObjectType(type))
1923 return CEK_AsValueOnly;
1924 return CEK_None;
1925}
1926
1927/// Try to emit a reference to the given value without producing it as
1928/// an l-value. This is just an optimization, but it avoids us needing
1929/// to emit global copies of variables if they're named without triggering
1930/// a formal use in a context where we can't emit a direct reference to them,
1931/// for instance if a block or lambda or a member of a local class uses a
1932/// const int variable or constexpr variable from an enclosing function.
1933CodeGenFunction::ConstantEmission
1934CodeGenFunction::tryEmitAsConstant(const DeclRefExpr *RefExpr) {
1935 const ValueDecl *Value = RefExpr->getDecl();
1936
1937 // The value needs to be an enum constant or a constant variable.
1938 ConstantEmissionKind CEK;
1939 if (isa<ParmVarDecl>(Val: Value)) {
1940 CEK = CEK_None;
1941 } else if (const auto *var = dyn_cast<VarDecl>(Val: Value)) {
1942 CEK = checkVarTypeForConstantEmission(type: var->getType());
1943 } else if (isa<EnumConstantDecl>(Val: Value)) {
1944 CEK = CEK_AsValueOnly;
1945 } else {
1946 CEK = CEK_None;
1947 }
1948 if (CEK == CEK_None) return ConstantEmission();
1949
1950 Expr::EvalResult result;
1951 bool resultIsReference;
1952 QualType resultType;
1953
1954 // It's best to evaluate all the way as an r-value if that's permitted.
1955 if (CEK != CEK_AsReferenceOnly &&
1956 RefExpr->EvaluateAsRValue(Result&: result, Ctx: getContext())) {
1957 resultIsReference = false;
1958 resultType = RefExpr->getType().getUnqualifiedType();
1959
1960 // Otherwise, try to evaluate as an l-value.
1961 } else if (CEK != CEK_AsValueOnly &&
1962 RefExpr->EvaluateAsLValue(Result&: result, Ctx: getContext())) {
1963 resultIsReference = true;
1964 resultType = Value->getType();
1965
1966 // Failure.
1967 } else {
1968 return ConstantEmission();
1969 }
1970
1971 // In any case, if the initializer has side-effects, abandon ship.
1972 if (result.HasSideEffects)
1973 return ConstantEmission();
1974
1975 // In CUDA/HIP device compilation, a lambda may capture a reference variable
1976 // referencing a global host variable by copy. In this case the lambda should
1977 // make a copy of the value of the global host variable. The DRE of the
1978 // captured reference variable cannot be emitted as load from the host
1979 // global variable as compile time constant, since the host variable is not
1980 // accessible on device. The DRE of the captured reference variable has to be
1981 // loaded from captures.
1982 if (CGM.getLangOpts().CUDAIsDevice && result.Val.isLValue() &&
1983 RefExpr->refersToEnclosingVariableOrCapture()) {
1984 auto *MD = dyn_cast_or_null<CXXMethodDecl>(Val: CurCodeDecl);
1985 if (isLambdaMethod(DC: MD) && MD->getOverloadedOperator() == OO_Call) {
1986 const APValue::LValueBase &base = result.Val.getLValueBase();
1987 if (const ValueDecl *D = base.dyn_cast<const ValueDecl *>()) {
1988 if (const VarDecl *VD = dyn_cast<const VarDecl>(Val: D)) {
1989 if (!VD->hasAttr<CUDADeviceAttr>()) {
1990 return ConstantEmission();
1991 }
1992 }
1993 }
1994 }
1995 }
1996
1997 // Emit as a constant.
1998 llvm::Constant *C = ConstantEmitter(*this).emitAbstract(
1999 loc: RefExpr->getLocation(), value: result.Val, T: resultType);
2000
2001 // Make sure we emit a debug reference to the global variable.
2002 // This should probably fire even for
2003 if (isa<VarDecl>(Val: Value)) {
2004 if (!getContext().DeclMustBeEmitted(D: cast<VarDecl>(Val: Value)))
2005 EmitDeclRefExprDbgValue(E: RefExpr, Init: result.Val);
2006 } else {
2007 assert(isa<EnumConstantDecl>(Value));
2008 EmitDeclRefExprDbgValue(E: RefExpr, Init: result.Val);
2009 }
2010
2011 // If we emitted a reference constant, we need to dereference that.
2012 if (resultIsReference)
2013 return ConstantEmission::forReference(C);
2014
2015 return ConstantEmission::forValue(C);
2016}
2017
2018static DeclRefExpr *tryToConvertMemberExprToDeclRefExpr(CodeGenFunction &CGF,
2019 const MemberExpr *ME) {
2020 if (auto *VD = dyn_cast<VarDecl>(Val: ME->getMemberDecl())) {
2021 // Try to emit static variable member expressions as DREs.
2022 return DeclRefExpr::Create(
2023 Context: CGF.getContext(), QualifierLoc: NestedNameSpecifierLoc(), TemplateKWLoc: SourceLocation(), D: VD,
2024 /*RefersToEnclosingVariableOrCapture=*/false, NameLoc: ME->getExprLoc(),
2025 T: ME->getType(), VK: ME->getValueKind(), FoundD: nullptr, TemplateArgs: nullptr, NOUR: ME->isNonOdrUse());
2026 }
2027 return nullptr;
2028}
2029
2030CodeGenFunction::ConstantEmission
2031CodeGenFunction::tryEmitAsConstant(const MemberExpr *ME) {
2032 if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(CGF&: *this, ME))
2033 return tryEmitAsConstant(RefExpr: DRE);
2034 return ConstantEmission();
2035}
2036
2037llvm::Value *CodeGenFunction::emitScalarConstant(
2038 const CodeGenFunction::ConstantEmission &Constant, Expr *E) {
2039 assert(Constant && "not a constant");
2040 if (Constant.isReference())
2041 return EmitLoadOfLValue(V: Constant.getReferenceLValue(CGF&: *this, RefExpr: E),
2042 Loc: E->getExprLoc())
2043 .getScalarVal();
2044 return Constant.getValue();
2045}
2046
2047llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue,
2048 SourceLocation Loc) {
2049 return EmitLoadOfScalar(Addr: lvalue.getAddress(), Volatile: lvalue.isVolatile(),
2050 Ty: lvalue.getType(), Loc, BaseInfo: lvalue.getBaseInfo(),
2051 TBAAInfo: lvalue.getTBAAInfo(), isNontemporal: lvalue.isNontemporal());
2052}
2053
2054static bool getRangeForType(CodeGenFunction &CGF, QualType Ty,
2055 llvm::APInt &Min, llvm::APInt &End,
2056 bool StrictEnums, bool IsBool) {
2057 const auto *ED = Ty->getAsEnumDecl();
2058 bool IsRegularCPlusPlusEnum =
2059 CGF.getLangOpts().CPlusPlus && StrictEnums && ED && !ED->isFixed();
2060 if (!IsBool && !IsRegularCPlusPlusEnum)
2061 return false;
2062
2063 if (IsBool) {
2064 Min = llvm::APInt(CGF.getContext().getTypeSize(T: Ty), 0);
2065 End = llvm::APInt(CGF.getContext().getTypeSize(T: Ty), 2);
2066 } else {
2067 ED->getValueRange(Max&: End, Min);
2068 }
2069 return true;
2070}
2071
2072llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) {
2073 llvm::APInt Min, End;
2074 if (!getRangeForType(CGF&: *this, Ty, Min, End, StrictEnums: CGM.getCodeGenOpts().StrictEnums,
2075 IsBool: Ty->hasBooleanRepresentation() && !Ty->isVectorType()))
2076 return nullptr;
2077
2078 llvm::MDBuilder MDHelper(getLLVMContext());
2079 return MDHelper.createRange(Lo: Min, Hi: End);
2080}
2081
2082void CodeGenFunction::maybeAttachRangeForLoad(llvm::LoadInst *Load, QualType Ty,
2083 SourceLocation Loc) {
2084 if (EmitScalarRangeCheck(Value: Load, Ty, Loc)) {
2085 // In order to prevent the optimizer from throwing away the check, don't
2086 // attach range metadata to the load.
2087 } else if (CGM.getCodeGenOpts().OptimizationLevel > 0) {
2088 if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty)) {
2089 Load->setMetadata(KindID: llvm::LLVMContext::MD_range, Node: RangeInfo);
2090 Load->setMetadata(KindID: llvm::LLVMContext::MD_noundef,
2091 Node: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: {}));
2092 }
2093 }
2094}
2095
2096bool CodeGenFunction::EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
2097 SourceLocation Loc) {
2098 bool HasBoolCheck = SanOpts.has(K: SanitizerKind::Bool);
2099 bool HasEnumCheck = SanOpts.has(K: SanitizerKind::Enum);
2100 if (!HasBoolCheck && !HasEnumCheck)
2101 return false;
2102
2103 bool IsBool = (Ty->hasBooleanRepresentation() && !Ty->isVectorType()) ||
2104 NSAPI(CGM.getContext()).isObjCBOOLType(T: Ty);
2105 bool NeedsBoolCheck = HasBoolCheck && IsBool;
2106 bool NeedsEnumCheck = HasEnumCheck && Ty->isEnumeralType();
2107 if (!NeedsBoolCheck && !NeedsEnumCheck)
2108 return false;
2109
2110 // Single-bit booleans don't need to be checked. Special-case this to avoid
2111 // a bit width mismatch when handling bitfield values. This is handled by
2112 // EmitFromMemory for the non-bitfield case.
2113 if (IsBool &&
2114 cast<llvm::IntegerType>(Val: Value->getType())->getBitWidth() == 1)
2115 return false;
2116
2117 if (NeedsEnumCheck &&
2118 getContext().isTypeIgnoredBySanitizer(Mask: SanitizerKind::Enum, Ty))
2119 return false;
2120
2121 llvm::APInt Min, End;
2122 if (!getRangeForType(CGF&: *this, Ty, Min, End, /*StrictEnums=*/true, IsBool))
2123 return true;
2124
2125 SanitizerKind::SanitizerOrdinal Kind =
2126 NeedsEnumCheck ? SanitizerKind::SO_Enum : SanitizerKind::SO_Bool;
2127
2128 auto &Ctx = getLLVMContext();
2129 auto CheckHandler = SanitizerHandler::LoadInvalidValue;
2130 SanitizerDebugLocation SanScope(this, {Kind}, CheckHandler);
2131 llvm::Value *Check;
2132 --End;
2133 if (!Min) {
2134 Check = Builder.CreateICmpULE(LHS: Value, RHS: llvm::ConstantInt::get(Context&: Ctx, V: End));
2135 } else {
2136 llvm::Value *Upper =
2137 Builder.CreateICmpSLE(LHS: Value, RHS: llvm::ConstantInt::get(Context&: Ctx, V: End));
2138 llvm::Value *Lower =
2139 Builder.CreateICmpSGE(LHS: Value, RHS: llvm::ConstantInt::get(Context&: Ctx, V: Min));
2140 Check = Builder.CreateAnd(LHS: Upper, RHS: Lower);
2141 }
2142 llvm::Constant *StaticArgs[] = {EmitCheckSourceLocation(Loc),
2143 EmitCheckTypeDescriptor(T: Ty)};
2144 EmitCheck(Checked: std::make_pair(x&: Check, y&: Kind), Check: CheckHandler, StaticArgs, DynamicArgs: Value);
2145 return true;
2146}
2147
2148llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
2149 QualType Ty,
2150 SourceLocation Loc,
2151 LValueBaseInfo BaseInfo,
2152 TBAAAccessInfo TBAAInfo,
2153 bool isNontemporal) {
2154 if (auto *GV = dyn_cast<llvm::GlobalValue>(Val: Addr.getBasePointer()))
2155 if (GV->isThreadLocal())
2156 Addr = Addr.withPointer(NewPointer: Builder.CreateThreadLocalAddress(Ptr: GV),
2157 IsKnownNonNull: NotKnownNonNull);
2158
2159 if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
2160 // Boolean vectors use `iN` as storage type.
2161 if (ClangVecTy->isPackedVectorBoolType(ctx: getContext())) {
2162 llvm::Type *ValTy = ConvertType(T: Ty);
2163 unsigned ValNumElems =
2164 cast<llvm::FixedVectorType>(Val: ValTy)->getNumElements();
2165 // Load the `iP` storage object (P is the padded vector size).
2166 auto *RawIntV = Builder.CreateLoad(Addr, IsVolatile: Volatile, Name: "load_bits");
2167 const auto *RawIntTy = RawIntV->getType();
2168 assert(RawIntTy->isIntegerTy() && "compressed iN storage for bitvectors");
2169 // Bitcast iP --> <P x i1>.
2170 auto *PaddedVecTy = llvm::FixedVectorType::get(
2171 ElementType: Builder.getInt1Ty(), NumElts: RawIntTy->getPrimitiveSizeInBits());
2172 llvm::Value *V = Builder.CreateBitCast(V: RawIntV, DestTy: PaddedVecTy);
2173 // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
2174 V = emitBoolVecConversion(SrcVec: V, NumElementsDst: ValNumElems, Name: "extractvec");
2175
2176 return EmitFromMemory(Value: V, Ty);
2177 }
2178
2179 // Handles vectors of sizes that are likely to be expanded to a larger size
2180 // to optimize performance.
2181 auto *VTy = cast<llvm::FixedVectorType>(Val: Addr.getElementType());
2182 auto *NewVecTy =
2183 CGM.getABIInfo().getOptimalVectorMemoryType(T: VTy, Opt: getLangOpts());
2184
2185 if (VTy != NewVecTy) {
2186 Address Cast = Addr.withElementType(ElemTy: NewVecTy);
2187 llvm::Value *V = Builder.CreateLoad(Addr: Cast, IsVolatile: Volatile, Name: "loadVecN");
2188 unsigned OldNumElements = VTy->getNumElements();
2189 SmallVector<int, 16> Mask(OldNumElements);
2190 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
2191 V = Builder.CreateShuffleVector(V, Mask, Name: "extractVec");
2192 return EmitFromMemory(Value: V, Ty);
2193 }
2194 }
2195
2196 // Atomic operations have to be done on integral types.
2197 LValue AtomicLValue =
2198 LValue::MakeAddr(Addr, type: Ty, Context&: getContext(), BaseInfo, TBAAInfo);
2199 if (Ty->isAtomicType() || LValueIsSuitableForInlineAtomic(Src: AtomicLValue)) {
2200 return EmitAtomicLoad(LV: AtomicLValue, SL: Loc).getScalarVal();
2201 }
2202
2203 Addr =
2204 Addr.withElementType(ElemTy: convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Addr.getElementType()));
2205
2206 llvm::LoadInst *Load = Builder.CreateLoad(Addr, IsVolatile: Volatile);
2207 if (isNontemporal) {
2208 llvm::MDNode *Node = llvm::MDNode::get(
2209 Context&: Load->getContext(), MDs: llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: 1)));
2210 Load->setMetadata(KindID: llvm::LLVMContext::MD_nontemporal, Node);
2211 }
2212
2213 CGM.DecorateInstructionWithTBAA(Inst: Load, TBAAInfo);
2214
2215 maybeAttachRangeForLoad(Load, Ty, Loc);
2216
2217 return EmitFromMemory(Value: Load, Ty);
2218}
2219
2220/// Converts a scalar value from its primary IR type (as returned
2221/// by ConvertType) to its load/store type (as returned by
2222/// convertTypeForLoadStore).
2223llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) {
2224 if (auto *AtomicTy = Ty->getAs<AtomicType>())
2225 Ty = AtomicTy->getValueType();
2226
2227 if (Ty->isExtVectorBoolType() || Ty->isConstantMatrixBoolType()) {
2228 llvm::Type *StoreTy = convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Value->getType());
2229
2230 if (Value->getType() == StoreTy)
2231 return Value;
2232
2233 if (StoreTy->isVectorTy() && StoreTy->getScalarSizeInBits() >
2234 Value->getType()->getScalarSizeInBits())
2235 return Builder.CreateZExt(V: Value, DestTy: StoreTy);
2236
2237 // Expand to the memory bit width.
2238 unsigned MemNumElems = StoreTy->getPrimitiveSizeInBits();
2239 // <N x i1> --> <P x i1>.
2240 Value = emitBoolVecConversion(SrcVec: Value, NumElementsDst: MemNumElems, Name: "insertvec");
2241 // <P x i1> --> iP.
2242 Value = Builder.CreateBitCast(V: Value, DestTy: StoreTy);
2243 }
2244
2245 if (Ty->hasBooleanRepresentation() || Ty->isBitIntType()) {
2246 llvm::Type *StoreTy = convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Value->getType());
2247 bool Signed = Ty->isSignedIntegerOrEnumerationType();
2248 return Builder.CreateIntCast(V: Value, DestTy: StoreTy, isSigned: Signed, Name: "storedv");
2249 }
2250
2251 return Value;
2252}
2253
2254/// Converts a scalar value from its load/store type (as returned
2255/// by convertTypeForLoadStore) to its primary IR type (as returned
2256/// by ConvertType).
2257llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
2258 if (auto *AtomicTy = Ty->getAs<AtomicType>())
2259 Ty = AtomicTy->getValueType();
2260
2261 if (Ty->isPackedVectorBoolType(ctx: getContext())) {
2262 const auto *RawIntTy = Value->getType();
2263
2264 // Bitcast iP --> <P x i1>.
2265 auto *PaddedVecTy = llvm::FixedVectorType::get(
2266 ElementType: Builder.getInt1Ty(), NumElts: RawIntTy->getPrimitiveSizeInBits());
2267 auto *V = Builder.CreateBitCast(V: Value, DestTy: PaddedVecTy);
2268 // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
2269 llvm::Type *ValTy = ConvertType(T: Ty);
2270 unsigned ValNumElems = cast<llvm::FixedVectorType>(Val: ValTy)->getNumElements();
2271 return emitBoolVecConversion(SrcVec: V, NumElementsDst: ValNumElems, Name: "extractvec");
2272 }
2273
2274 llvm::Type *ResTy = ConvertType(T: Ty);
2275 if (Ty->hasBooleanRepresentation() || Ty->isBitIntType() ||
2276 Ty->isExtVectorBoolType())
2277 return Builder.CreateTrunc(V: Value, DestTy: ResTy, Name: "loadedv");
2278
2279 return Value;
2280}
2281
2282// Convert the pointer of \p Addr to a pointer to a vector (the value type of
2283// MatrixType), if it points to a array (the memory type of MatrixType).
2284static RawAddress MaybeConvertMatrixAddress(RawAddress Addr,
2285 CodeGenFunction &CGF,
2286 bool IsVector = true) {
2287 auto *ArrayTy = dyn_cast<llvm::ArrayType>(Val: Addr.getElementType());
2288 if (ArrayTy && IsVector) {
2289 auto ArrayElements = ArrayTy->getNumElements();
2290 auto *ArrayElementTy = ArrayTy->getElementType();
2291 if (CGF.getContext().getLangOpts().HLSL) {
2292 auto *VectorTy = cast<llvm::FixedVectorType>(Val: ArrayElementTy);
2293 ArrayElementTy = VectorTy->getElementType();
2294 ArrayElements *= VectorTy->getNumElements();
2295 }
2296 auto *VectorTy = llvm::FixedVectorType::get(ElementType: ArrayElementTy, NumElts: ArrayElements);
2297
2298 return Addr.withElementType(ElemTy: VectorTy);
2299 }
2300 auto *VectorTy = dyn_cast<llvm::VectorType>(Val: Addr.getElementType());
2301 if (VectorTy && !IsVector) {
2302 auto *ArrayTy = llvm::ArrayType::get(
2303 ElementType: VectorTy->getElementType(),
2304 NumElements: cast<llvm::FixedVectorType>(Val: VectorTy)->getNumElements());
2305
2306 return Addr.withElementType(ElemTy: ArrayTy);
2307 }
2308
2309 return Addr;
2310}
2311
2312LValue CodeGenFunction::EmitMatrixElementExpr(const MatrixElementExpr *E) {
2313 LValue Base;
2314 if (E->getBase()->isGLValue())
2315 Base = EmitLValue(E: E->getBase());
2316 else {
2317 assert(E->getBase()->getType()->isConstantMatrixType() &&
2318 "Result must be a Constant Matrix");
2319 llvm::Value *Mat = EmitScalarExpr(E: E->getBase());
2320 Address MatMem = CreateMemTemp(Ty: E->getBase()->getType());
2321 QualType Ty = E->getBase()->getType();
2322 llvm::Type *LTy = convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Mat->getType());
2323 if (LTy->getScalarSizeInBits() > Mat->getType()->getScalarSizeInBits())
2324 Mat = Builder.CreateZExt(V: Mat, DestTy: LTy);
2325 Builder.CreateStore(Val: Mat, Addr: MatMem);
2326 Base = MakeAddrLValue(Addr: MatMem, T: Ty, Source: AlignmentSource::Decl);
2327 }
2328 QualType ResultType =
2329 E->getType().withCVRQualifiers(CVR: Base.getQuals().getCVRQualifiers());
2330
2331 // Encode the element access list into a vector of unsigned indices.
2332 // getEncodedElementAccess returns row-major linearized indices.
2333 SmallVector<uint32_t, 4> Indices;
2334 E->getEncodedElementAccess(Elts&: Indices);
2335
2336 // getEncodedElementAccess returns row-major linearized indices
2337 // If the matrix memory layout is column-major, convert indices
2338 // to column-major indices.
2339 bool IsColMajor = getLangOpts().getDefaultMatrixMemoryLayout() ==
2340 LangOptions::MatrixMemoryLayout::MatrixColMajor;
2341 if (IsColMajor) {
2342 const auto *MT = E->getBase()->getType()->castAs<ConstantMatrixType>();
2343 unsigned NumCols = MT->getNumColumns();
2344 for (uint32_t &Idx : Indices) {
2345 // Decompose row-major index: Row = Idx / NumCols, Col = Idx % NumCols
2346 unsigned Row = Idx / NumCols;
2347 unsigned Col = Idx % NumCols;
2348 // Re-linearize as column-major
2349 Idx = MT->getColumnMajorFlattenedIndex(Row, Column: Col);
2350 }
2351 }
2352
2353 if (Base.isSimple()) {
2354 RawAddress MatAddr = Base.getAddress();
2355 if (getLangOpts().HLSL &&
2356 E->getBase()->getType().getAddressSpace() == LangAS::hlsl_constant)
2357 MatAddr = CGM.getHLSLRuntime().createBufferMatrixTempAddress(
2358 LV: Base, Loc: E->getExprLoc(), CGF&: *this);
2359
2360 llvm::Constant *CV =
2361 llvm::ConstantDataVector::get(Context&: getLLVMContext(), Elts: Indices);
2362 return LValue::MakeExtVectorElt(Addr: MaybeConvertMatrixAddress(Addr: MatAddr, CGF&: *this),
2363 Elts: CV, type: ResultType, BaseInfo: Base.getBaseInfo(),
2364 TBAAInfo: TBAAAccessInfo());
2365 }
2366 assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
2367
2368 llvm::Constant *BaseElts = Base.getExtVectorElts();
2369 SmallVector<llvm::Constant *, 4> CElts;
2370
2371 for (unsigned Index : Indices)
2372 CElts.push_back(Elt: BaseElts->getAggregateElement(Elt: Index));
2373 llvm::Constant *CV = llvm::ConstantVector::get(V: CElts);
2374
2375 return LValue::MakeExtVectorElt(
2376 Addr: MaybeConvertMatrixAddress(Addr: Base.getExtVectorAddress(), CGF&: *this), Elts: CV,
2377 type: ResultType, BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
2378}
2379
2380// Emit a store of a matrix LValue. This may require casting the original
2381// pointer to memory address (ArrayType) to a pointer to the value type
2382// (VectorType).
2383static void EmitStoreOfMatrixScalar(llvm::Value *value, LValue lvalue,
2384 bool isInit, CodeGenFunction &CGF) {
2385 Address Addr = MaybeConvertMatrixAddress(Addr: lvalue.getAddress(), CGF,
2386 IsVector: value->getType()->isVectorTy());
2387 CGF.EmitStoreOfScalar(Value: value, Addr, Volatile: lvalue.isVolatile(), Ty: lvalue.getType(),
2388 BaseInfo: lvalue.getBaseInfo(), TBAAInfo: lvalue.getTBAAInfo(), isInit,
2389 isNontemporal: lvalue.isNontemporal());
2390}
2391
2392void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr,
2393 bool Volatile, QualType Ty,
2394 LValueBaseInfo BaseInfo,
2395 TBAAAccessInfo TBAAInfo,
2396 bool isInit, bool isNontemporal) {
2397 if (auto *GV = dyn_cast<llvm::GlobalValue>(Val: Addr.getBasePointer()))
2398 if (GV->isThreadLocal())
2399 Addr = Addr.withPointer(NewPointer: Builder.CreateThreadLocalAddress(Ptr: GV),
2400 IsKnownNonNull: NotKnownNonNull);
2401
2402 // Handles vectors of sizes that are likely to be expanded to a larger size
2403 // to optimize performance.
2404 llvm::Type *SrcTy = Value->getType();
2405 if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
2406 if (auto *VecTy = dyn_cast<llvm::FixedVectorType>(Val: SrcTy)) {
2407 auto *NewVecTy =
2408 CGM.getABIInfo().getOptimalVectorMemoryType(T: VecTy, Opt: getLangOpts());
2409 if (!ClangVecTy->isPackedVectorBoolType(ctx: getContext()) &&
2410 VecTy != NewVecTy) {
2411 SmallVector<int, 16> Mask(NewVecTy->getNumElements(),
2412 VecTy->getNumElements());
2413 std::iota(first: Mask.begin(), last: Mask.begin() + VecTy->getNumElements(), value: 0);
2414 // Use undef instead of poison for the padding lanes, to make sure no
2415 // padding bits are poisoned, which may break coercion.
2416 Value = Builder.CreateShuffleVector(V1: Value, V2: llvm::UndefValue::get(T: VecTy),
2417 Mask, Name: "extractVec");
2418 SrcTy = NewVecTy;
2419 }
2420 if (Addr.getElementType() != SrcTy)
2421 Addr = Addr.withElementType(ElemTy: SrcTy);
2422 }
2423 }
2424
2425 Value = EmitToMemory(Value, Ty);
2426
2427 LValue AtomicLValue =
2428 LValue::MakeAddr(Addr, type: Ty, Context&: getContext(), BaseInfo, TBAAInfo);
2429 if (Ty->isAtomicType() ||
2430 (!isInit && LValueIsSuitableForInlineAtomic(Src: AtomicLValue))) {
2431 EmitAtomicStore(rvalue: RValue::get(V: Value), lvalue: AtomicLValue, isInit);
2432 return;
2433 }
2434
2435 llvm::StoreInst *Store = Builder.CreateStore(Val: Value, Addr, IsVolatile: Volatile);
2436 addInstToCurrentSourceAtom(KeyInstruction: Store, Backup: Value);
2437
2438 if (isNontemporal) {
2439 llvm::MDNode *Node =
2440 llvm::MDNode::get(Context&: Store->getContext(),
2441 MDs: llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: 1)));
2442 Store->setMetadata(KindID: llvm::LLVMContext::MD_nontemporal, Node);
2443 }
2444
2445 CGM.DecorateInstructionWithTBAA(Inst: Store, TBAAInfo);
2446}
2447
2448void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
2449 bool isInit) {
2450 if (lvalue.getType()->isConstantMatrixType()) {
2451 EmitStoreOfMatrixScalar(value, lvalue, isInit, CGF&: *this);
2452 return;
2453 }
2454
2455 EmitStoreOfScalar(Value: value, Addr: lvalue.getAddress(), Volatile: lvalue.isVolatile(),
2456 Ty: lvalue.getType(), BaseInfo: lvalue.getBaseInfo(),
2457 TBAAInfo: lvalue.getTBAAInfo(), isInit, isNontemporal: lvalue.isNontemporal());
2458}
2459
2460// Emit a load of a LValue of matrix type. This may require casting the pointer
2461// to memory address (ArrayType) to a pointer to the value type (VectorType).
2462static RValue EmitLoadOfMatrixLValue(LValue LV, SourceLocation Loc,
2463 CodeGenFunction &CGF) {
2464 assert(LV.getType()->isConstantMatrixType());
2465 RawAddress DestAddr = LV.getAddress();
2466
2467 // HLSL constant buffers may pad matrix layouts, so copy elements into a
2468 // non-padded local alloca before loading.
2469 if (CGF.getLangOpts().HLSL &&
2470 LV.getType().getAddressSpace() == LangAS::hlsl_constant)
2471 DestAddr =
2472 CGF.CGM.getHLSLRuntime().createBufferMatrixTempAddress(LV, Loc, CGF);
2473
2474 Address Addr = MaybeConvertMatrixAddress(Addr: DestAddr, CGF);
2475 LV.setAddress(Addr);
2476 return RValue::get(V: CGF.EmitLoadOfScalar(lvalue: LV, Loc));
2477}
2478
2479RValue CodeGenFunction::EmitLoadOfAnyValue(LValue LV, AggValueSlot Slot,
2480 SourceLocation Loc) {
2481 QualType Ty = LV.getType();
2482 switch (getEvaluationKind(T: Ty)) {
2483 case TEK_Scalar:
2484 return EmitLoadOfLValue(V: LV, Loc);
2485 case TEK_Complex:
2486 return RValue::getComplex(C: EmitLoadOfComplex(src: LV, loc: Loc));
2487 case TEK_Aggregate:
2488 EmitAggFinalDestCopy(Type: Ty, Dest: Slot, Src: LV, SrcKind: EVK_NonRValue);
2489 return Slot.asRValue();
2490 }
2491 llvm_unreachable("bad evaluation kind");
2492}
2493
2494/// EmitLoadOfLValue - Given an expression that represents a value lvalue, this
2495/// method emits the address of the lvalue, then loads the result as an rvalue,
2496/// returning the rvalue.
2497RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) {
2498 // Load from __ptrauth.
2499 if (PointerAuthQualifier PtrAuth = LV.getQuals().getPointerAuth()) {
2500 LV.getQuals().removePointerAuth();
2501 llvm::Value *Value = EmitLoadOfLValue(LV, Loc).getScalarVal();
2502 return RValue::get(V: EmitPointerAuthUnqualify(Qualifier: PtrAuth, Pointer: Value, PointerType: LV.getType(),
2503 StorageAddress: LV.getAddress(),
2504 /*known nonnull*/ IsKnownNonNull: false));
2505 }
2506
2507 if (LV.isObjCWeak()) {
2508 // load of a __weak object.
2509 Address AddrWeakObj = LV.getAddress();
2510 return RValue::get(V: CGM.getObjCRuntime().EmitObjCWeakRead(CGF&: *this,
2511 AddrWeakObj));
2512 }
2513 if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) {
2514 // In MRC mode, we do a load+autorelease.
2515 if (!getLangOpts().ObjCAutoRefCount) {
2516 return RValue::get(V: EmitARCLoadWeak(addr: LV.getAddress()));
2517 }
2518
2519 // In ARC mode, we load retained and then consume the value.
2520 llvm::Value *Object = EmitARCLoadWeakRetained(addr: LV.getAddress());
2521 Object = EmitObjCConsumeObject(T: LV.getType(), Ptr: Object);
2522 return RValue::get(V: Object);
2523 }
2524
2525 if (LV.isSimple()) {
2526 assert(!LV.getType()->isFunctionType());
2527
2528 if (LV.getType()->isConstantMatrixType())
2529 return EmitLoadOfMatrixLValue(LV, Loc, CGF&: *this);
2530
2531 // Everything needs a load.
2532 return RValue::get(V: EmitLoadOfScalar(lvalue: LV, Loc));
2533 }
2534
2535 if (LV.isVectorElt()) {
2536 llvm::LoadInst *Load = Builder.CreateLoad(Addr: LV.getVectorAddress(),
2537 IsVolatile: LV.isVolatileQualified());
2538 llvm::Value *Elt =
2539 Builder.CreateExtractElement(Vec: Load, Idx: LV.getVectorIdx(), Name: "vecext");
2540 return RValue::get(V: EmitFromMemory(Value: Elt, Ty: LV.getType()));
2541 }
2542
2543 // If this is a reference to a subset of the elements of a vector, either
2544 // shuffle the input or extract/insert them as appropriate.
2545 if (LV.isExtVectorElt()) {
2546 return EmitLoadOfExtVectorElementLValue(V: LV);
2547 }
2548
2549 // Global Register variables always invoke intrinsics
2550 if (LV.isGlobalReg())
2551 return EmitLoadOfGlobalRegLValue(LV);
2552
2553 if (LV.isMatrixElt()) {
2554 llvm::Value *Idx = LV.getMatrixIdx();
2555 QualType EltTy = LV.getType();
2556 if (const auto *MatTy = EltTy->getAs<ConstantMatrixType>()) {
2557 EltTy = MatTy->getElementType();
2558 if (CGM.getCodeGenOpts().OptimizationLevel > 0) {
2559 llvm::MatrixBuilder MB(Builder);
2560 MB.CreateIndexAssumption(Idx, NumElements: MatTy->getNumElementsFlattened());
2561 }
2562 }
2563 llvm::LoadInst *Load =
2564 Builder.CreateLoad(Addr: LV.getMatrixAddress(), IsVolatile: LV.isVolatileQualified());
2565 llvm::Value *Elt = Builder.CreateExtractElement(Vec: Load, Idx, Name: "matrixext");
2566 return RValue::get(V: EmitFromMemory(Value: Elt, Ty: EltTy));
2567 }
2568 if (LV.isMatrixRow()) {
2569 QualType MatTy = LV.getType();
2570 const ConstantMatrixType *MT = MatTy->castAs<ConstantMatrixType>();
2571
2572 unsigned NumRows = MT->getNumRows();
2573 unsigned NumCols = MT->getNumColumns();
2574 unsigned NumLanes = NumCols;
2575 llvm::Value *MatrixVec = EmitLoadOfScalar(lvalue: LV, Loc);
2576 llvm::Value *Row = LV.getMatrixRowIdx();
2577 llvm::Type *ElemTy = ConvertType(T: MT->getElementType());
2578 llvm::Constant *ColConstsIndices = nullptr;
2579 llvm::MatrixBuilder MB(Builder);
2580
2581 if (LV.isMatrixRowSwizzle()) {
2582 ColConstsIndices = LV.getMatrixRowElts();
2583 NumLanes = llvm::cast<llvm::FixedVectorType>(Val: ColConstsIndices->getType())
2584 ->getNumElements();
2585 }
2586
2587 llvm::Type *RowTy = llvm::FixedVectorType::get(ElementType: ElemTy, NumElts: NumLanes);
2588 llvm::Value *Result = llvm::PoisonValue::get(T: RowTy); // <NumLanes x T>
2589
2590 for (unsigned Col = 0; Col < NumLanes; ++Col) {
2591 llvm::Value *ColIdx;
2592 if (ColConstsIndices)
2593 ColIdx = ColConstsIndices->getAggregateElement(Elt: Col);
2594 else
2595 ColIdx = llvm::ConstantInt::get(Ty: Row->getType(), V: Col);
2596 bool IsMatrixRowMajor = getLangOpts().getDefaultMatrixMemoryLayout() ==
2597 LangOptions::MatrixMemoryLayout::MatrixRowMajor;
2598 llvm::Value *EltIndex =
2599 MB.CreateIndex(RowIdx: Row, ColumnIdx: ColIdx, NumRows, NumCols, IsMatrixRowMajor);
2600 llvm::Value *Elt = Builder.CreateExtractElement(Vec: MatrixVec, Idx: EltIndex);
2601 llvm::Value *Lane = llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: Col);
2602 Result = Builder.CreateInsertElement(Vec: Result, NewElt: Elt, Idx: Lane);
2603 }
2604
2605 return RValue::get(V: Result);
2606 }
2607
2608 assert(LV.isBitField() && "Unknown LValue type!");
2609 return EmitLoadOfBitfieldLValue(LV, Loc);
2610}
2611
2612RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV,
2613 SourceLocation Loc) {
2614 const CGBitFieldInfo &Info = LV.getBitFieldInfo();
2615
2616 // Get the output type.
2617 llvm::Type *ResLTy = ConvertType(T: LV.getType());
2618
2619 Address Ptr = LV.getBitFieldAddress();
2620 llvm::Value *Val =
2621 Builder.CreateLoad(Addr: Ptr, IsVolatile: LV.isVolatileQualified(), Name: "bf.load");
2622
2623 bool UseVolatile = LV.isVolatileQualified() &&
2624 Info.VolatileStorageSize != 0 && isAAPCS(TargetInfo: CGM.getTarget());
2625 const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
2626 const unsigned StorageSize =
2627 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
2628 if (Info.IsSigned) {
2629 assert(static_cast<unsigned>(Offset + Info.Size) <= StorageSize);
2630 unsigned HighBits = StorageSize - Offset - Info.Size;
2631 if (HighBits)
2632 Val = Builder.CreateShl(LHS: Val, RHS: HighBits, Name: "bf.shl");
2633 if (Offset + HighBits)
2634 Val = Builder.CreateAShr(LHS: Val, RHS: Offset + HighBits, Name: "bf.ashr");
2635 } else {
2636 if (Offset)
2637 Val = Builder.CreateLShr(LHS: Val, RHS: Offset, Name: "bf.lshr");
2638 if (static_cast<unsigned>(Offset) + Info.Size < StorageSize)
2639 Val = Builder.CreateAnd(
2640 LHS: Val, RHS: llvm::APInt::getLowBitsSet(numBits: StorageSize, loBitsSet: Info.Size), Name: "bf.clear");
2641 }
2642 Val = Builder.CreateIntCast(V: Val, DestTy: ResLTy, isSigned: Info.IsSigned, Name: "bf.cast");
2643 EmitScalarRangeCheck(Value: Val, Ty: LV.getType(), Loc);
2644 return RValue::get(V: Val);
2645}
2646
2647// If this is a reference to a subset of the elements of a vector, create an
2648// appropriate shufflevector.
2649RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {
2650 llvm::Value *Vec = Builder.CreateLoad(Addr: LV.getExtVectorAddress(),
2651 IsVolatile: LV.isVolatileQualified());
2652
2653 // HLSL allows treating scalars as one-element vectors. Converting the scalar
2654 // IR value to a vector here allows the rest of codegen to behave as normal.
2655 if (getLangOpts().HLSL && !Vec->getType()->isVectorTy()) {
2656 llvm::Type *DstTy = llvm::FixedVectorType::get(ElementType: Vec->getType(), NumElts: 1);
2657 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: CGM.Int64Ty);
2658 Vec = Builder.CreateInsertElement(VecTy: DstTy, NewElt: Vec, Idx: Zero, Name: "cast.splat");
2659 }
2660
2661 const llvm::Constant *Elts = LV.getExtVectorElts();
2662
2663 // If the result of the expression is a non-vector type, we must be extracting
2664 // a single element. Just codegen as an extractelement.
2665 const VectorType *ExprVT = LV.getType()->getAs<VectorType>();
2666 if (!ExprVT) {
2667 unsigned InIdx = getAccessedFieldNo(Idx: 0, Elts);
2668 llvm::Value *Elt = llvm::ConstantInt::get(Ty: SizeTy, V: InIdx);
2669
2670 llvm::Value *Element = Builder.CreateExtractElement(Vec, Idx: Elt);
2671
2672 llvm::Type *LVTy = ConvertType(T: LV.getType());
2673 if (Element->getType()->getPrimitiveSizeInBits() >
2674 LVTy->getPrimitiveSizeInBits())
2675 Element = Builder.CreateTrunc(V: Element, DestTy: LVTy);
2676
2677 return RValue::get(V: Element);
2678 }
2679
2680 // Always use shuffle vector to try to retain the original program structure
2681 unsigned NumResultElts = ExprVT->getNumElements();
2682
2683 SmallVector<int, 4> Mask;
2684 for (unsigned i = 0; i != NumResultElts; ++i)
2685 Mask.push_back(Elt: getAccessedFieldNo(Idx: i, Elts));
2686
2687 Vec = Builder.CreateShuffleVector(V: Vec, Mask);
2688
2689 if (LV.getType()->isExtVectorBoolType())
2690 Vec = Builder.CreateTrunc(V: Vec, DestTy: ConvertType(T: LV.getType()), Name: "truncv");
2691
2692 return RValue::get(V: Vec);
2693}
2694
2695/// Generates lvalue for partial ext_vector access.
2696Address CodeGenFunction::EmitExtVectorElementLValue(LValue LV) {
2697 Address VectorAddress = LV.getExtVectorAddress();
2698 QualType EQT = LV.getType()->castAs<VectorType>()->getElementType();
2699 llvm::Type *VectorElementTy = CGM.getTypes().ConvertType(T: EQT);
2700
2701 Address CastToPointerElement = VectorAddress.withElementType(ElemTy: VectorElementTy);
2702
2703 const llvm::Constant *Elts = LV.getExtVectorElts();
2704 unsigned ix = getAccessedFieldNo(Idx: 0, Elts);
2705
2706 Address VectorBasePtrPlusIx =
2707 Builder.CreateConstInBoundsGEP(Addr: CastToPointerElement, Index: ix,
2708 Name: "vector.elt");
2709
2710 return VectorBasePtrPlusIx;
2711}
2712
2713/// Load of global named registers are always calls to intrinsics.
2714RValue CodeGenFunction::EmitLoadOfGlobalRegLValue(LValue LV) {
2715 assert((LV.getType()->isIntegerType() || LV.getType()->isPointerType()) &&
2716 "Bad type for register variable");
2717 llvm::MDNode *RegName = cast<llvm::MDNode>(
2718 Val: cast<llvm::MetadataAsValue>(Val: LV.getGlobalReg())->getMetadata());
2719
2720 // We accept integer and pointer types only
2721 llvm::Type *OrigTy = CGM.getTypes().ConvertType(T: LV.getType());
2722 llvm::Type *Ty = OrigTy;
2723 if (OrigTy->isPointerTy())
2724 Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
2725 llvm::Type *Types[] = { Ty };
2726
2727 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::read_register, Tys: Types);
2728 llvm::Value *Call = Builder.CreateCall(
2729 Callee: F, Args: llvm::MetadataAsValue::get(Context&: Ty->getContext(), MD: RegName));
2730 if (OrigTy->isPointerTy())
2731 Call = Builder.CreateIntToPtr(V: Call, DestTy: OrigTy);
2732 return RValue::get(V: Call);
2733}
2734
2735/// EmitStoreThroughLValue - Store the specified rvalue into the specified
2736/// lvalue, where both are guaranteed to the have the same type, and that type
2737/// is 'Ty'.
2738void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
2739 bool isInit) {
2740 if (!Dst.isSimple()) {
2741 if (Dst.isVectorElt()) {
2742 if (getLangOpts().HLSL) {
2743 // HLSL allows direct access to vector elements, so storing to
2744 // individual elements of a vector through VectorElt is handled as
2745 // separate store instructions.
2746 Address DstAddr = Dst.getVectorAddress();
2747 llvm::Type *DestAddrTy = DstAddr.getElementType();
2748 llvm::Type *ElemTy = DestAddrTy->getScalarType();
2749 CharUnits ElemAlign = CharUnits::fromQuantity(
2750 Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty: ElemTy));
2751
2752 assert(ElemTy->getScalarSizeInBits() >= 8 &&
2753 "vector element type must be at least byte-sized");
2754
2755 llvm::Value *Val = Src.getScalarVal();
2756 if (Val->getType()->getPrimitiveSizeInBits() <
2757 ElemTy->getScalarSizeInBits())
2758 Val = Builder.CreateZExt(V: Val, DestTy: ElemTy->getScalarType());
2759
2760 llvm::Value *Idx = Dst.getVectorIdx();
2761 llvm::Value *Zero = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
2762 Address DstElemAddr =
2763 Builder.CreateGEP(Addr: DstAddr, IdxList: {Zero, Idx}, ElementType: DestAddrTy, Align: ElemAlign);
2764 Builder.CreateStore(Val, Addr: DstElemAddr, IsVolatile: Dst.isVolatileQualified());
2765 return;
2766 }
2767
2768 // Read/modify/write the vector, inserting the new element.
2769 llvm::Value *Vec = Builder.CreateLoad(Addr: Dst.getVectorAddress(),
2770 IsVolatile: Dst.isVolatileQualified());
2771 llvm::Type *VecTy = Vec->getType();
2772 llvm::Value *SrcVal = Src.getScalarVal();
2773
2774 if (SrcVal->getType()->getPrimitiveSizeInBits() <
2775 VecTy->getScalarSizeInBits())
2776 SrcVal = Builder.CreateZExt(V: SrcVal, DestTy: VecTy->getScalarType());
2777
2778 auto *IRStoreTy = dyn_cast<llvm::IntegerType>(Val: Vec->getType());
2779 if (IRStoreTy) {
2780 auto *IRVecTy = llvm::FixedVectorType::get(
2781 ElementType: Builder.getInt1Ty(), NumElts: IRStoreTy->getPrimitiveSizeInBits());
2782 Vec = Builder.CreateBitCast(V: Vec, DestTy: IRVecTy);
2783 // iN --> <N x i1>.
2784 }
2785
2786 // Allow inserting `<1 x T>` into an `<N x T>`. It can happen with scalar
2787 // types which are mapped to vector LLVM IR types (e.g. for implementing
2788 // an ABI).
2789 if (auto *EltTy = dyn_cast<llvm::FixedVectorType>(Val: SrcVal->getType());
2790 EltTy && EltTy->getNumElements() == 1)
2791 SrcVal = Builder.CreateBitCast(V: SrcVal, DestTy: EltTy->getElementType());
2792
2793 Vec = Builder.CreateInsertElement(Vec, NewElt: SrcVal, Idx: Dst.getVectorIdx(),
2794 Name: "vecins");
2795 if (IRStoreTy) {
2796 // <N x i1> --> <iN>.
2797 Vec = Builder.CreateBitCast(V: Vec, DestTy: IRStoreTy);
2798 }
2799
2800 auto *I = Builder.CreateStore(Val: Vec, Addr: Dst.getVectorAddress(),
2801 IsVolatile: Dst.isVolatileQualified());
2802 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Vec);
2803 return;
2804 }
2805
2806 // If this is an update of extended vector elements, insert them as
2807 // appropriate.
2808 if (Dst.isExtVectorElt())
2809 return EmitStoreThroughExtVectorComponentLValue(Src, Dst);
2810
2811 if (Dst.isGlobalReg())
2812 return EmitStoreThroughGlobalRegLValue(Src, Dst);
2813
2814 if (Dst.isMatrixElt()) {
2815 if (getLangOpts().HLSL) {
2816 // HLSL allows direct access to matrix elements, so storing to
2817 // individual elements of a matrix through MatrixElt is handled as
2818 // separate store instructions.
2819 Address DstAddr = Dst.getMatrixAddress();
2820 llvm::Type *DestAddrTy = DstAddr.getElementType();
2821 llvm::Type *ElemTy = DestAddrTy->getScalarType();
2822 CharUnits ElemAlign = CharUnits::fromQuantity(
2823 Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty: ElemTy));
2824
2825 assert(ElemTy->getScalarSizeInBits() >= 8 &&
2826 "matrix element type must be at least byte-sized");
2827
2828 llvm::Value *Val = Src.getScalarVal();
2829 if (Val->getType()->getPrimitiveSizeInBits() <
2830 ElemTy->getScalarSizeInBits())
2831 Val = Builder.CreateZExt(V: Val, DestTy: ElemTy->getScalarType());
2832
2833 llvm::Value *Idx = Dst.getMatrixIdx();
2834 llvm::Value *Zero = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
2835 Address DstElemAddr =
2836 Builder.CreateGEP(Addr: DstAddr, IdxList: {Zero, Idx}, ElementType: DestAddrTy, Align: ElemAlign);
2837 Builder.CreateStore(Val, Addr: DstElemAddr, IsVolatile: Dst.isVolatileQualified());
2838 return;
2839 }
2840
2841 llvm::Value *Idx = Dst.getMatrixIdx();
2842 if (CGM.getCodeGenOpts().OptimizationLevel > 0) {
2843 const auto *const MatTy = Dst.getType()->castAs<ConstantMatrixType>();
2844 llvm::MatrixBuilder MB(Builder);
2845 MB.CreateIndexAssumption(Idx, NumElements: MatTy->getNumElementsFlattened());
2846 }
2847 llvm::Instruction *Load = Builder.CreateLoad(Addr: Dst.getMatrixAddress());
2848 llvm::Value *InsertVal = Src.getScalarVal();
2849 llvm::Value *Vec =
2850 Builder.CreateInsertElement(Vec: Load, NewElt: InsertVal, Idx, Name: "matins");
2851 auto *I = Builder.CreateStore(Val: Vec, Addr: Dst.getMatrixAddress(),
2852 IsVolatile: Dst.isVolatileQualified());
2853 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Vec);
2854 return;
2855 }
2856 if (Dst.isMatrixRow()) {
2857 // NOTE: Since there are no other languages that implement matrix single
2858 // subscripting, the logic here is specific to HLSL which allows
2859 // per-element stores to rows of matrices.
2860 assert(getLangOpts().HLSL &&
2861 "Store through matrix row LValues is only implemented for HLSL!");
2862 QualType MatTy = Dst.getType();
2863 const ConstantMatrixType *MT = MatTy->castAs<ConstantMatrixType>();
2864
2865 unsigned NumRows = MT->getNumRows();
2866 unsigned NumCols = MT->getNumColumns();
2867 unsigned NumLanes = NumCols;
2868
2869 Address DstAddr = Dst.getMatrixAddress();
2870 llvm::Type *DestAddrTy = DstAddr.getElementType();
2871 llvm::Type *ElemTy = DestAddrTy->getScalarType();
2872 CharUnits ElemAlign =
2873 CharUnits::fromQuantity(Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty: ElemTy));
2874
2875 assert(ElemTy->getScalarSizeInBits() >= 8 &&
2876 "matrix element type must be at least byte-sized");
2877
2878 llvm::Value *RowVal = Src.getScalarVal();
2879 if (RowVal->getType()->getScalarType()->getPrimitiveSizeInBits() <
2880 ElemTy->getScalarSizeInBits()) {
2881 auto *RowValVecTy = cast<llvm::FixedVectorType>(Val: RowVal->getType());
2882 llvm::Type *StorageElmTy = llvm::FixedVectorType::get(
2883 ElementType: ElemTy->getScalarType(), NumElts: RowValVecTy->getNumElements());
2884 RowVal = Builder.CreateZExt(V: RowVal, DestTy: StorageElmTy);
2885 }
2886
2887 llvm::MatrixBuilder MB(Builder);
2888
2889 llvm::Constant *ColConstsIndices = nullptr;
2890 if (Dst.isMatrixRowSwizzle()) {
2891 ColConstsIndices = Dst.getMatrixRowElts();
2892 NumLanes =
2893 llvm::cast<llvm::FixedVectorType>(Val: ColConstsIndices->getType())
2894 ->getNumElements();
2895 }
2896
2897 llvm::Value *Row = Dst.getMatrixRowIdx();
2898 for (unsigned Col = 0; Col < NumLanes; ++Col) {
2899 llvm::Value *ColIdx;
2900 if (ColConstsIndices)
2901 ColIdx = ColConstsIndices->getAggregateElement(Elt: Col);
2902 else
2903 ColIdx = llvm::ConstantInt::get(Ty: Row->getType(), V: Col);
2904 bool IsMatrixRowMajor = getLangOpts().getDefaultMatrixMemoryLayout() ==
2905 LangOptions::MatrixMemoryLayout::MatrixRowMajor;
2906 llvm::Value *EltIndex =
2907 MB.CreateIndex(RowIdx: Row, ColumnIdx: ColIdx, NumRows, NumCols, IsMatrixRowMajor);
2908 llvm::Value *Lane = llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: Col);
2909 llvm::Value *Zero = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
2910 llvm::Value *NewElt = Builder.CreateExtractElement(Vec: RowVal, Idx: Lane);
2911 Address DstElemAddr =
2912 Builder.CreateGEP(Addr: DstAddr, IdxList: {Zero, EltIndex}, ElementType: DestAddrTy, Align: ElemAlign);
2913 Builder.CreateStore(Val: NewElt, Addr: DstElemAddr, IsVolatile: Dst.isVolatileQualified());
2914 }
2915
2916 return;
2917 }
2918
2919 assert(Dst.isBitField() && "Unknown LValue type");
2920 return EmitStoreThroughBitfieldLValue(Src, Dst);
2921 }
2922
2923 // Handle __ptrauth qualification by re-signing the value.
2924 if (PointerAuthQualifier PointerAuth = Dst.getQuals().getPointerAuth()) {
2925 Src = RValue::get(V: EmitPointerAuthQualify(Qualifier: PointerAuth, Pointer: Src.getScalarVal(),
2926 ValueType: Dst.getType(), StorageAddress: Dst.getAddress(),
2927 /*known nonnull*/ IsKnownNonNull: false));
2928 }
2929
2930 // There's special magic for assigning into an ARC-qualified l-value.
2931 if (Qualifiers::ObjCLifetime Lifetime = Dst.getQuals().getObjCLifetime()) {
2932 switch (Lifetime) {
2933 case Qualifiers::OCL_None:
2934 llvm_unreachable("present but none");
2935
2936 case Qualifiers::OCL_ExplicitNone:
2937 // nothing special
2938 break;
2939
2940 case Qualifiers::OCL_Strong:
2941 if (isInit) {
2942 Src = RValue::get(V: EmitARCRetain(type: Dst.getType(), value: Src.getScalarVal()));
2943 break;
2944 }
2945 EmitARCStoreStrong(lvalue: Dst, value: Src.getScalarVal(), /*ignore*/ resultIgnored: true);
2946 return;
2947
2948 case Qualifiers::OCL_Weak:
2949 if (isInit)
2950 // Initialize and then skip the primitive store.
2951 EmitARCInitWeak(addr: Dst.getAddress(), value: Src.getScalarVal());
2952 else
2953 EmitARCStoreWeak(addr: Dst.getAddress(), value: Src.getScalarVal(),
2954 /*ignore*/ ignored: true);
2955 return;
2956
2957 case Qualifiers::OCL_Autoreleasing:
2958 Src = RValue::get(V: EmitObjCExtendObjectLifetime(T: Dst.getType(),
2959 Ptr: Src.getScalarVal()));
2960 // fall into the normal path
2961 break;
2962 }
2963 }
2964
2965 if (Dst.isObjCWeak() && !Dst.isNonGC()) {
2966 // load of a __weak object.
2967 Address LvalueDst = Dst.getAddress();
2968 llvm::Value *src = Src.getScalarVal();
2969 CGM.getObjCRuntime().EmitObjCWeakAssign(CGF&: *this, src, dest: LvalueDst);
2970 return;
2971 }
2972
2973 if (Dst.isObjCStrong() && !Dst.isNonGC()) {
2974 // load of a __strong object.
2975 Address LvalueDst = Dst.getAddress();
2976 llvm::Value *src = Src.getScalarVal();
2977 if (Dst.isObjCIvar()) {
2978 assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");
2979 llvm::Type *ResultType = IntPtrTy;
2980 Address dst = EmitPointerWithAlignment(E: Dst.getBaseIvarExp());
2981 llvm::Value *RHS = dst.emitRawPointer(CGF&: *this);
2982 RHS = Builder.CreatePtrToInt(V: RHS, DestTy: ResultType, Name: "sub.ptr.rhs.cast");
2983 llvm::Value *LHS = Builder.CreatePtrToInt(V: LvalueDst.emitRawPointer(CGF&: *this),
2984 DestTy: ResultType, Name: "sub.ptr.lhs.cast");
2985 llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, Name: "ivar.offset");
2986 CGM.getObjCRuntime().EmitObjCIvarAssign(CGF&: *this, src, dest: dst, ivarOffset: BytesBetween);
2987 } else if (Dst.isGlobalObjCRef()) {
2988 CGM.getObjCRuntime().EmitObjCGlobalAssign(CGF&: *this, src, dest: LvalueDst,
2989 threadlocal: Dst.isThreadLocalRef());
2990 }
2991 else
2992 CGM.getObjCRuntime().EmitObjCStrongCastAssign(CGF&: *this, src, dest: LvalueDst);
2993 return;
2994 }
2995
2996 assert(Src.isScalar() && "Can't emit an agg store with this method");
2997 EmitStoreOfScalar(value: Src.getScalarVal(), lvalue: Dst, isInit);
2998}
2999
3000void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
3001 llvm::Value **Result) {
3002 const CGBitFieldInfo &Info = Dst.getBitFieldInfo();
3003 llvm::Type *ResLTy = convertTypeForLoadStore(ASTTy: Dst.getType());
3004 Address Ptr = Dst.getBitFieldAddress();
3005
3006 // Get the source value, truncated to the width of the bit-field.
3007 llvm::Value *SrcVal = Src.getScalarVal();
3008
3009 // Cast the source to the storage type and shift it into place.
3010 SrcVal = Builder.CreateIntCast(V: SrcVal, DestTy: Ptr.getElementType(),
3011 /*isSigned=*/false);
3012 llvm::Value *MaskedVal = SrcVal;
3013
3014 const bool UseVolatile =
3015 CGM.getCodeGenOpts().AAPCSBitfieldWidth && Dst.isVolatileQualified() &&
3016 Info.VolatileStorageSize != 0 && isAAPCS(TargetInfo: CGM.getTarget());
3017 const unsigned StorageSize =
3018 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
3019 const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
3020 // See if there are other bits in the bitfield's storage we'll need to load
3021 // and mask together with source before storing.
3022 if (StorageSize != Info.Size) {
3023 assert(StorageSize > Info.Size && "Invalid bitfield size.");
3024 llvm::Value *Val =
3025 Builder.CreateLoad(Addr: Ptr, IsVolatile: Dst.isVolatileQualified(), Name: "bf.load");
3026
3027 // Mask the source value as needed.
3028 if (!Dst.getType()->hasBooleanRepresentation())
3029 SrcVal = Builder.CreateAnd(
3030 LHS: SrcVal, RHS: llvm::APInt::getLowBitsSet(numBits: StorageSize, loBitsSet: Info.Size),
3031 Name: "bf.value");
3032 MaskedVal = SrcVal;
3033 if (Offset)
3034 SrcVal = Builder.CreateShl(LHS: SrcVal, RHS: Offset, Name: "bf.shl");
3035
3036 // Mask out the original value.
3037 Val = Builder.CreateAnd(
3038 LHS: Val, RHS: ~llvm::APInt::getBitsSet(numBits: StorageSize, loBit: Offset, hiBit: Offset + Info.Size),
3039 Name: "bf.clear");
3040
3041 // Or together the unchanged values and the source value.
3042 SrcVal = Builder.CreateOr(LHS: Val, RHS: SrcVal, Name: "bf.set");
3043 } else {
3044 assert(Offset == 0);
3045 // According to the AACPS:
3046 // When a volatile bit-field is written, and its container does not overlap
3047 // with any non-bit-field member, its container must be read exactly once
3048 // and written exactly once using the access width appropriate to the type
3049 // of the container. The two accesses are not atomic.
3050 if (Dst.isVolatileQualified() && isAAPCS(TargetInfo: CGM.getTarget()) &&
3051 CGM.getCodeGenOpts().ForceAAPCSBitfieldLoad)
3052 Builder.CreateLoad(Addr: Ptr, IsVolatile: true, Name: "bf.load");
3053 }
3054
3055 // Write the new value back out.
3056 auto *I = Builder.CreateStore(Val: SrcVal, Addr: Ptr, IsVolatile: Dst.isVolatileQualified());
3057 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: SrcVal);
3058
3059 // Return the new value of the bit-field, if requested.
3060 if (Result) {
3061 llvm::Value *ResultVal = MaskedVal;
3062
3063 // Sign extend the value if needed.
3064 if (Info.IsSigned) {
3065 assert(Info.Size <= StorageSize);
3066 unsigned HighBits = StorageSize - Info.Size;
3067 if (HighBits) {
3068 ResultVal = Builder.CreateShl(LHS: ResultVal, RHS: HighBits, Name: "bf.result.shl");
3069 ResultVal = Builder.CreateAShr(LHS: ResultVal, RHS: HighBits, Name: "bf.result.ashr");
3070 }
3071 }
3072
3073 ResultVal = Builder.CreateIntCast(V: ResultVal, DestTy: ResLTy, isSigned: Info.IsSigned,
3074 Name: "bf.result.cast");
3075 *Result = EmitFromMemory(Value: ResultVal, Ty: Dst.getType());
3076 }
3077}
3078
3079void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
3080 LValue Dst) {
3081 llvm::Value *SrcVal = Src.getScalarVal();
3082 Address DstAddr = Dst.getExtVectorAddress();
3083 const llvm::Constant *Elts = Dst.getExtVectorElts();
3084 if (DstAddr.getElementType()->getScalarSizeInBits() >
3085 SrcVal->getType()->getScalarSizeInBits())
3086 SrcVal = Builder.CreateZExt(
3087 V: SrcVal, DestTy: convertTypeForLoadStore(ASTTy: Dst.getType(), LLVMTy: SrcVal->getType()));
3088
3089 if (getLangOpts().HLSL) {
3090 llvm::Type *DestAddrTy = DstAddr.getElementType();
3091 // HLSL allows storing to scalar values through ExtVector component LValues.
3092 // To support this we need to handle the case where the destination address
3093 // is a scalar.
3094 if (!DestAddrTy->isVectorTy()) {
3095 assert(!Dst.getType()->isVectorType() &&
3096 "this should only occur for non-vector l-values");
3097 Builder.CreateStore(Val: SrcVal, Addr: DstAddr, IsVolatile: Dst.isVolatileQualified());
3098 return;
3099 }
3100
3101 // HLSL allows direct access to vector elements, so storing to individual
3102 // elements of a vector through ExtVector is handled as separate store
3103 // instructions.
3104 // If we are updating multiple elements, Dst and Src are vectors; for
3105 // a single element update they are scalars.
3106 const VectorType *VTy = Dst.getType()->getAs<VectorType>();
3107 unsigned NumSrcElts = VTy ? VTy->getNumElements() : 1;
3108 CharUnits ElemAlign = CharUnits::fromQuantity(
3109 Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty: DestAddrTy->getScalarType()));
3110 llvm::Value *Zero = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
3111
3112 for (unsigned I = 0; I != NumSrcElts; ++I) {
3113 llvm::Value *Val = VTy ? Builder.CreateExtractElement(
3114 Vec: SrcVal, Idx: llvm::ConstantInt::get(Ty: Int32Ty, V: I))
3115 : SrcVal;
3116 unsigned FieldNo = getAccessedFieldNo(Idx: I, Elts);
3117 Address DstElemAddr = Address::invalid();
3118 if (FieldNo == 0)
3119 DstElemAddr = DstAddr.withAlignment(NewAlignment: ElemAlign);
3120 else
3121 DstElemAddr = Builder.CreateGEP(
3122 Addr: DstAddr, IdxList: {Zero, llvm::ConstantInt::get(Ty: Int32Ty, V: FieldNo)},
3123 ElementType: DestAddrTy, Align: ElemAlign);
3124 Builder.CreateStore(Val, Addr: DstElemAddr, IsVolatile: Dst.isVolatileQualified());
3125 }
3126 return;
3127 }
3128
3129 // This access turns into a read/modify/write of the vector. Load the input
3130 // value now.
3131 llvm::Value *Vec = Builder.CreateLoad(Addr: DstAddr, IsVolatile: Dst.isVolatileQualified());
3132 llvm::Type *VecTy = Vec->getType();
3133
3134 if (const VectorType *VTy = Dst.getType()->getAs<VectorType>()) {
3135 unsigned NumSrcElts = VTy->getNumElements();
3136 unsigned NumDstElts = cast<llvm::FixedVectorType>(Val: VecTy)->getNumElements();
3137 if (NumDstElts == NumSrcElts) {
3138 // Use shuffle vector is the src and destination are the same number of
3139 // elements and restore the vector mask since it is on the side it will be
3140 // stored.
3141 SmallVector<int, 4> Mask(NumDstElts);
3142 for (unsigned i = 0; i != NumSrcElts; ++i)
3143 Mask[getAccessedFieldNo(Idx: i, Elts)] = i;
3144
3145 Vec = Builder.CreateShuffleVector(V: SrcVal, Mask);
3146 } else if (NumDstElts > NumSrcElts) {
3147 // Extended the source vector to the same length and then shuffle it
3148 // into the destination.
3149 // FIXME: since we're shuffling with undef, can we just use the indices
3150 // into that? This could be simpler.
3151 SmallVector<int, 4> ExtMask;
3152 for (unsigned i = 0; i != NumSrcElts; ++i)
3153 ExtMask.push_back(Elt: i);
3154 ExtMask.resize(N: NumDstElts, NV: -1);
3155 llvm::Value *ExtSrcVal = Builder.CreateShuffleVector(V: SrcVal, Mask: ExtMask);
3156 // build identity
3157 SmallVector<int, 4> Mask;
3158 for (unsigned i = 0; i != NumDstElts; ++i)
3159 Mask.push_back(Elt: i);
3160
3161 // When the vector size is odd and .odd or .hi is used, the last element
3162 // of the Elts constant array will be one past the size of the vector.
3163 // Ignore the last element here, if it is greater than the mask size.
3164 if (getAccessedFieldNo(Idx: NumSrcElts - 1, Elts) == Mask.size())
3165 NumSrcElts--;
3166
3167 // modify when what gets shuffled in
3168 for (unsigned i = 0; i != NumSrcElts; ++i)
3169 Mask[getAccessedFieldNo(Idx: i, Elts)] = i + NumDstElts;
3170 Vec = Builder.CreateShuffleVector(V1: Vec, V2: ExtSrcVal, Mask);
3171 } else {
3172 // We should never shorten the vector
3173 llvm_unreachable("unexpected shorten vector length");
3174 }
3175 } else {
3176 // If the Src is a scalar (not a vector), and the target is a vector it must
3177 // be updating one element.
3178 unsigned InIdx = getAccessedFieldNo(Idx: 0, Elts);
3179 llvm::Value *Elt = llvm::ConstantInt::get(Ty: SizeTy, V: InIdx);
3180
3181 Vec = Builder.CreateInsertElement(Vec, NewElt: SrcVal, Idx: Elt);
3182 }
3183
3184 Builder.CreateStore(Val: Vec, Addr: Dst.getExtVectorAddress(),
3185 IsVolatile: Dst.isVolatileQualified());
3186}
3187
3188/// Store of global named registers are always calls to intrinsics.
3189void CodeGenFunction::EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst) {
3190 assert((Dst.getType()->isIntegerType() || Dst.getType()->isPointerType()) &&
3191 "Bad type for register variable");
3192 llvm::MDNode *RegName = cast<llvm::MDNode>(
3193 Val: cast<llvm::MetadataAsValue>(Val: Dst.getGlobalReg())->getMetadata());
3194 assert(RegName && "Register LValue is not metadata");
3195
3196 // We accept integer and pointer types only
3197 llvm::Type *OrigTy = CGM.getTypes().ConvertType(T: Dst.getType());
3198 llvm::Type *Ty = OrigTy;
3199 if (OrigTy->isPointerTy())
3200 Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
3201 llvm::Type *Types[] = { Ty };
3202
3203 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::write_register, Tys: Types);
3204 llvm::Value *Value = Src.getScalarVal();
3205 if (OrigTy->isPointerTy())
3206 Value = Builder.CreatePtrToInt(V: Value, DestTy: Ty);
3207 Builder.CreateCall(
3208 Callee: F, Args: {llvm::MetadataAsValue::get(Context&: Ty->getContext(), MD: RegName), Value});
3209}
3210
3211// setObjCGCLValueClass - sets class of the lvalue for the purpose of
3212// generating write-barries API. It is currently a global, ivar,
3213// or neither.
3214static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
3215 LValue &LV,
3216 bool IsMemberAccess=false) {
3217 if (Ctx.getLangOpts().getGC() == LangOptions::NonGC)
3218 return;
3219
3220 if (isa<ObjCIvarRefExpr>(Val: E)) {
3221 QualType ExpTy = E->getType();
3222 if (IsMemberAccess && ExpTy->isPointerType()) {
3223 // If ivar is a structure pointer, assigning to field of
3224 // this struct follows gcc's behavior and makes it a non-ivar
3225 // writer-barrier conservatively.
3226 ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
3227 if (ExpTy->isRecordType()) {
3228 LV.setObjCIvar(false);
3229 return;
3230 }
3231 }
3232 LV.setObjCIvar(true);
3233 auto *Exp = cast<ObjCIvarRefExpr>(Val: const_cast<Expr *>(E));
3234 LV.setBaseIvarExp(Exp->getBase());
3235 LV.setObjCArray(E->getType()->isArrayType());
3236 return;
3237 }
3238
3239 if (const auto *Exp = dyn_cast<DeclRefExpr>(Val: E)) {
3240 if (const auto *VD = dyn_cast<VarDecl>(Val: Exp->getDecl())) {
3241 if (VD->hasGlobalStorage()) {
3242 LV.setGlobalObjCRef(true);
3243 LV.setThreadLocalRef(VD->getTLSKind() != VarDecl::TLS_None);
3244 }
3245 }
3246 LV.setObjCArray(E->getType()->isArrayType());
3247 return;
3248 }
3249
3250 if (const auto *Exp = dyn_cast<UnaryOperator>(Val: E)) {
3251 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3252 return;
3253 }
3254
3255 if (const auto *Exp = dyn_cast<ParenExpr>(Val: E)) {
3256 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3257 if (LV.isObjCIvar()) {
3258 // If cast is to a structure pointer, follow gcc's behavior and make it
3259 // a non-ivar write-barrier.
3260 QualType ExpTy = E->getType();
3261 if (ExpTy->isPointerType())
3262 ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
3263 if (ExpTy->isRecordType())
3264 LV.setObjCIvar(false);
3265 }
3266 return;
3267 }
3268
3269 if (const auto *Exp = dyn_cast<GenericSelectionExpr>(Val: E)) {
3270 setObjCGCLValueClass(Ctx, E: Exp->getResultExpr(), LV);
3271 return;
3272 }
3273
3274 if (const auto *Exp = dyn_cast<ImplicitCastExpr>(Val: E)) {
3275 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3276 return;
3277 }
3278
3279 if (const auto *Exp = dyn_cast<CStyleCastExpr>(Val: E)) {
3280 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3281 return;
3282 }
3283
3284 if (const auto *Exp = dyn_cast<ObjCBridgedCastExpr>(Val: E)) {
3285 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3286 return;
3287 }
3288
3289 if (const auto *Exp = dyn_cast<ArraySubscriptExpr>(Val: E)) {
3290 setObjCGCLValueClass(Ctx, E: Exp->getBase(), LV);
3291 if (LV.isObjCIvar() && !LV.isObjCArray())
3292 // Using array syntax to assigning to what an ivar points to is not
3293 // same as assigning to the ivar itself. {id *Names;} Names[i] = 0;
3294 LV.setObjCIvar(false);
3295 else if (LV.isGlobalObjCRef() && !LV.isObjCArray())
3296 // Using array syntax to assigning to what global points to is not
3297 // same as assigning to the global itself. {id *G;} G[i] = 0;
3298 LV.setGlobalObjCRef(false);
3299 return;
3300 }
3301
3302 if (const auto *Exp = dyn_cast<MemberExpr>(Val: E)) {
3303 setObjCGCLValueClass(Ctx, E: Exp->getBase(), LV, IsMemberAccess: true);
3304 // We don't know if member is an 'ivar', but this flag is looked at
3305 // only in the context of LV.isObjCIvar().
3306 LV.setObjCArray(E->getType()->isArrayType());
3307 return;
3308 }
3309}
3310
3311static LValue EmitThreadPrivateVarDeclLValue(
3312 CodeGenFunction &CGF, const VarDecl *VD, QualType T, Address Addr,
3313 llvm::Type *RealVarTy, SourceLocation Loc) {
3314 if (CGF.CGM.getLangOpts().OpenMPIRBuilder)
3315 Addr = CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(
3316 CGF, VD, VDAddr: Addr, Loc);
3317 else
3318 Addr =
3319 CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, VDAddr: Addr, Loc);
3320
3321 Addr = Addr.withElementType(ElemTy: RealVarTy);
3322 return CGF.MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3323}
3324
3325static Address emitDeclTargetVarDeclLValue(CodeGenFunction &CGF,
3326 const VarDecl *VD, QualType T) {
3327 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
3328 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
3329 // Return an invalid address if variable is MT_To (or MT_Enter starting with
3330 // OpenMP 5.2, or MT_Local in OpenMP 6.0) and unified memory is not enabled.
3331 // For all other cases: MT_Link and MT_To (or MT_Enter/MT_Local) with unified
3332 // memory, return a valid address.
3333 if (!Res || ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
3334 *Res == OMPDeclareTargetDeclAttr::MT_Enter ||
3335 *Res == OMPDeclareTargetDeclAttr::MT_Local) &&
3336 !CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory()))
3337 return Address::invalid();
3338 assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
3339 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
3340 *Res == OMPDeclareTargetDeclAttr::MT_Enter ||
3341 *Res == OMPDeclareTargetDeclAttr::MT_Local) &&
3342 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) &&
3343 "Expected link clause OR to clause with unified memory enabled.");
3344 QualType PtrTy = CGF.getContext().getPointerType(T: VD->getType());
3345 Address Addr = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
3346 return CGF.EmitLoadOfPointer(Ptr: Addr, PtrTy: PtrTy->castAs<PointerType>());
3347}
3348
3349Address
3350CodeGenFunction::EmitLoadOfReference(LValue RefLVal,
3351 LValueBaseInfo *PointeeBaseInfo,
3352 TBAAAccessInfo *PointeeTBAAInfo) {
3353 llvm::LoadInst *Load =
3354 Builder.CreateLoad(Addr: RefLVal.getAddress(), IsVolatile: RefLVal.isVolatile());
3355 CGM.DecorateInstructionWithTBAA(Inst: Load, TBAAInfo: RefLVal.getTBAAInfo());
3356 QualType PTy = RefLVal.getType()->getPointeeType();
3357 CharUnits Align = CGM.getNaturalTypeAlignment(
3358 T: PTy, BaseInfo: PointeeBaseInfo, TBAAInfo: PointeeTBAAInfo, /*ForPointeeType=*/forPointeeType: true);
3359 if (!PTy->isIncompleteType()) {
3360 llvm::LLVMContext &Ctx = getLLVMContext();
3361 llvm::MDBuilder MDB(Ctx);
3362 // Emit !nonnull metadata
3363 if (CGM.getTypes().getTargetAddressSpace(T: PTy) == 0 &&
3364 !CGM.getCodeGenOpts().NullPointerIsValid)
3365 Load->setMetadata(KindID: llvm::LLVMContext::MD_nonnull,
3366 Node: llvm::MDNode::get(Context&: Ctx, MDs: {}));
3367 // Emit !align metadata
3368 if (PTy->isObjectType()) {
3369 auto AlignVal = Align.getQuantity();
3370 if (AlignVal > 1) {
3371 Load->setMetadata(
3372 KindID: llvm::LLVMContext::MD_align,
3373 Node: llvm::MDNode::get(Context&: Ctx, MDs: MDB.createConstant(C: llvm::ConstantInt::get(
3374 Ty: Builder.getInt64Ty(), V: AlignVal))));
3375 }
3376 }
3377 }
3378 return makeNaturalAddressForPointer(Ptr: Load, T: PTy, Alignment: Align,
3379 /*ForPointeeType=*/true, BaseInfo: PointeeBaseInfo,
3380 TBAAInfo: PointeeTBAAInfo);
3381}
3382
3383LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) {
3384 LValueBaseInfo PointeeBaseInfo;
3385 TBAAAccessInfo PointeeTBAAInfo;
3386 Address PointeeAddr = EmitLoadOfReference(RefLVal, PointeeBaseInfo: &PointeeBaseInfo,
3387 PointeeTBAAInfo: &PointeeTBAAInfo);
3388 return MakeAddrLValue(Addr: PointeeAddr, T: RefLVal.getType()->getPointeeType(),
3389 BaseInfo: PointeeBaseInfo, TBAAInfo: PointeeTBAAInfo);
3390}
3391
3392Address CodeGenFunction::EmitLoadOfPointer(Address Ptr,
3393 const PointerType *PtrTy,
3394 LValueBaseInfo *BaseInfo,
3395 TBAAAccessInfo *TBAAInfo) {
3396 llvm::Value *Addr = Builder.CreateLoad(Addr: Ptr);
3397 return makeNaturalAddressForPointer(Ptr: Addr, T: PtrTy->getPointeeType(),
3398 Alignment: CharUnits(), /*ForPointeeType=*/true,
3399 BaseInfo, TBAAInfo);
3400}
3401
3402LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr,
3403 const PointerType *PtrTy) {
3404 LValueBaseInfo BaseInfo;
3405 TBAAAccessInfo TBAAInfo;
3406 Address Addr = EmitLoadOfPointer(Ptr: PtrAddr, PtrTy, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
3407 return MakeAddrLValue(Addr, T: PtrTy->getPointeeType(), BaseInfo, TBAAInfo);
3408}
3409
3410static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
3411 const Expr *E, const VarDecl *VD) {
3412 QualType T = E->getType();
3413
3414 // If it's thread_local, emit a call to its wrapper function instead.
3415 if (VD->getTLSKind() == VarDecl::TLS_Dynamic &&
3416 CGF.CGM.getCXXABI().usesThreadWrapperFunction(VD))
3417 return CGF.CGM.getCXXABI().EmitThreadLocalVarDeclLValue(CGF, VD, LValType: T);
3418 // Check if the variable is marked as declare target with link clause in
3419 // device codegen.
3420 if (CGF.getLangOpts().OpenMPIsTargetDevice) {
3421 Address Addr = emitDeclTargetVarDeclLValue(CGF, VD, T);
3422 if (Addr.isValid())
3423 return CGF.MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3424 }
3425
3426 llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(D: VD);
3427
3428 if (VD->getTLSKind() != VarDecl::TLS_None)
3429 V = CGF.Builder.CreateThreadLocalAddress(Ptr: V);
3430
3431 llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(T: VD->getType());
3432 CharUnits Alignment = CGF.getContext().getDeclAlign(D: VD);
3433 Address Addr(V, RealVarTy, Alignment);
3434 // Emit reference to the private copy of the variable if it is an OpenMP
3435 // threadprivate variable.
3436 if (CGF.getLangOpts().OpenMP && !CGF.getLangOpts().OpenMPSimd &&
3437 VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
3438 return EmitThreadPrivateVarDeclLValue(CGF, VD, T, Addr, RealVarTy,
3439 Loc: E->getExprLoc());
3440 }
3441 LValue LV = VD->getType()->isReferenceType() ?
3442 CGF.EmitLoadOfReferenceLValue(RefAddr: Addr, RefTy: VD->getType(),
3443 Source: AlignmentSource::Decl) :
3444 CGF.MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3445 setObjCGCLValueClass(Ctx: CGF.getContext(), E, LV);
3446 return LV;
3447}
3448
3449llvm::Constant *CodeGenModule::getRawFunctionPointer(GlobalDecl GD,
3450 llvm::Type *Ty) {
3451 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
3452 if (FD->hasAttr<WeakRefAttr>()) {
3453 ConstantAddress aliasee = GetWeakRefReference(VD: FD);
3454 return aliasee.getPointer();
3455 }
3456
3457 llvm::Constant *V = GetAddrOfFunction(GD, Ty);
3458 return V;
3459}
3460
3461static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF, const Expr *E,
3462 GlobalDecl GD) {
3463 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
3464 llvm::Constant *V = CGF.CGM.getFunctionPointer(GD);
3465 QualType ETy = E->getType();
3466 if (ETy->isCFIUncheckedCalleeFunctionType()) {
3467 if (auto *GV = dyn_cast<llvm::GlobalValue>(Val: V))
3468 V = llvm::NoCFIValue::get(GV);
3469 }
3470 CharUnits Alignment = CGF.getContext().getDeclAlign(D: FD);
3471 return CGF.MakeAddrLValue(V, T: ETy, Alignment, Source: AlignmentSource::Decl);
3472}
3473
3474static LValue EmitCapturedFieldLValue(CodeGenFunction &CGF, const FieldDecl *FD,
3475 llvm::Value *ThisValue) {
3476
3477 return CGF.EmitLValueForLambdaField(Field: FD, ThisValue);
3478}
3479
3480/// Named Registers are named metadata pointing to the register name
3481/// which will be read from/written to as an argument to the intrinsic
3482/// @llvm.read/write_register.
3483/// So far, only the name is being passed down, but other options such as
3484/// register type, allocation type or even optimization options could be
3485/// passed down via the metadata node.
3486static LValue EmitGlobalNamedRegister(const VarDecl *VD, CodeGenModule &CGM) {
3487 SmallString<64> Name("llvm.named.register.");
3488 AsmLabelAttr *Asm = VD->getAttr<AsmLabelAttr>();
3489 assert(Asm->getLabel().size() < 64-Name.size() &&
3490 "Register name too big");
3491 Name.append(RHS: Asm->getLabel());
3492 llvm::NamedMDNode *M =
3493 CGM.getModule().getOrInsertNamedMetadata(Name);
3494 if (M->getNumOperands() == 0) {
3495 llvm::MDString *Str = llvm::MDString::get(Context&: CGM.getLLVMContext(),
3496 Str: Asm->getLabel());
3497 llvm::Metadata *Ops[] = {Str};
3498 M->addOperand(M: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Ops));
3499 }
3500
3501 CharUnits Alignment = CGM.getContext().getDeclAlign(D: VD);
3502
3503 llvm::Value *Ptr =
3504 llvm::MetadataAsValue::get(Context&: CGM.getLLVMContext(), MD: M->getOperand(i: 0));
3505 return LValue::MakeGlobalReg(V: Ptr, alignment: Alignment, type: VD->getType());
3506}
3507
3508/// Determine whether we can emit a reference to \p VD from the current
3509/// context, despite not necessarily having seen an odr-use of the variable in
3510/// this context.
3511static bool canEmitSpuriousReferenceToVariable(CodeGenFunction &CGF,
3512 const DeclRefExpr *E,
3513 const VarDecl *VD) {
3514 // For a variable declared in an enclosing scope, do not emit a spurious
3515 // reference even if we have a capture, as that will emit an unwarranted
3516 // reference to our capture state, and will likely generate worse code than
3517 // emitting a local copy.
3518 if (E->refersToEnclosingVariableOrCapture())
3519 return false;
3520
3521 // For a local declaration declared in this function, we can always reference
3522 // it even if we don't have an odr-use.
3523 if (VD->hasLocalStorage()) {
3524 return VD->getDeclContext() ==
3525 dyn_cast_or_null<DeclContext>(Val: CGF.CurCodeDecl);
3526 }
3527
3528 // For a global declaration, we can emit a reference to it if we know
3529 // for sure that we are able to emit a definition of it.
3530 VD = VD->getDefinition(C&: CGF.getContext());
3531 if (!VD)
3532 return false;
3533
3534 // Don't emit a spurious reference if it might be to a variable that only
3535 // exists on a different device / target.
3536 // FIXME: This is unnecessarily broad. Check whether this would actually be a
3537 // cross-target reference.
3538 if (CGF.getLangOpts().OpenMP || CGF.getLangOpts().CUDA ||
3539 CGF.getLangOpts().OpenCL) {
3540 return false;
3541 }
3542
3543 // We can emit a spurious reference only if the linkage implies that we'll
3544 // be emitting a non-interposable symbol that will be retained until link
3545 // time.
3546 switch (CGF.CGM.getLLVMLinkageVarDefinition(VD)) {
3547 case llvm::GlobalValue::ExternalLinkage:
3548 case llvm::GlobalValue::LinkOnceODRLinkage:
3549 case llvm::GlobalValue::WeakODRLinkage:
3550 case llvm::GlobalValue::InternalLinkage:
3551 case llvm::GlobalValue::PrivateLinkage:
3552 return true;
3553 default:
3554 return false;
3555 }
3556}
3557
3558LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
3559 const NamedDecl *ND = E->getDecl();
3560 QualType T = E->getType();
3561
3562 assert(E->isNonOdrUse() != NOUR_Unevaluated &&
3563 "should not emit an unevaluated operand");
3564
3565 if (const auto *VD = dyn_cast<VarDecl>(Val: ND)) {
3566 // Global Named registers access via intrinsics only
3567 if (VD->getStorageClass() == SC_Register &&
3568 VD->hasAttr<AsmLabelAttr>() && !VD->isLocalVarDecl())
3569 return EmitGlobalNamedRegister(VD, CGM);
3570
3571 // If this DeclRefExpr does not constitute an odr-use of the variable,
3572 // we're not permitted to emit a reference to it in general, and it might
3573 // not be captured if capture would be necessary for a use. Emit the
3574 // constant value directly instead.
3575 if (E->isNonOdrUse() == NOUR_Constant &&
3576 (VD->getType()->isReferenceType() ||
3577 !canEmitSpuriousReferenceToVariable(CGF&: *this, E, VD))) {
3578 VD->getAnyInitializer(D&: VD);
3579 llvm::Constant *Val = ConstantEmitter(*this).emitAbstract(
3580 loc: E->getLocation(), value: *VD->evaluateValue(), T: VD->getType());
3581 assert(Val && "failed to emit constant expression");
3582
3583 Address Addr = Address::invalid();
3584 if (!VD->getType()->isReferenceType()) {
3585 // Spill the constant value to a global.
3586 Addr = CGM.createUnnamedGlobalFrom(D: *VD, Constant: Val,
3587 Align: getContext().getDeclAlign(D: VD));
3588 llvm::Type *VarTy = getTypes().ConvertTypeForMem(T: VD->getType());
3589 auto *PTy = llvm::PointerType::get(
3590 C&: getLLVMContext(), AddressSpace: getTypes().getTargetAddressSpace(T: VD->getType()));
3591 Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty: PTy, ElementTy: VarTy);
3592 } else {
3593 // Should we be using the alignment of the constant pointer we emitted?
3594 CharUnits Alignment =
3595 CGM.getNaturalTypeAlignment(T: E->getType(),
3596 /* BaseInfo= */ nullptr,
3597 /* TBAAInfo= */ nullptr,
3598 /* forPointeeType= */ true);
3599 Addr = makeNaturalAddressForPointer(Ptr: Val, T, Alignment);
3600 }
3601 return MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3602 }
3603
3604 // FIXME: Handle other kinds of non-odr-use DeclRefExprs.
3605
3606 // Check for captured variables.
3607 if (E->refersToEnclosingVariableOrCapture()) {
3608 VD = VD->getCanonicalDecl();
3609 if (auto *FD = LambdaCaptureFields.lookup(Val: VD))
3610 return EmitCapturedFieldLValue(CGF&: *this, FD, ThisValue: CXXABIThisValue);
3611 if (CapturedStmtInfo) {
3612 auto I = LocalDeclMap.find(Val: VD);
3613 if (I != LocalDeclMap.end()) {
3614 LValue CapLVal;
3615 if (VD->getType()->isReferenceType())
3616 CapLVal = EmitLoadOfReferenceLValue(RefAddr: I->second, RefTy: VD->getType(),
3617 Source: AlignmentSource::Decl);
3618 else
3619 CapLVal = MakeAddrLValue(Addr: I->second, T);
3620 // Mark lvalue as nontemporal if the variable is marked as nontemporal
3621 // in simd context.
3622 if (getLangOpts().OpenMP &&
3623 CGM.getOpenMPRuntime().isNontemporalDecl(VD))
3624 CapLVal.setNontemporal(/*Value=*/true);
3625 return CapLVal;
3626 }
3627 LValue CapLVal =
3628 EmitCapturedFieldLValue(CGF&: *this, FD: CapturedStmtInfo->lookup(VD),
3629 ThisValue: CapturedStmtInfo->getContextValue());
3630 Address LValueAddress = CapLVal.getAddress();
3631 CapLVal = MakeAddrLValue(Addr: Address(LValueAddress.emitRawPointer(CGF&: *this),
3632 LValueAddress.getElementType(),
3633 getContext().getDeclAlign(D: VD)),
3634 T: CapLVal.getType(),
3635 BaseInfo: LValueBaseInfo(AlignmentSource::Decl),
3636 TBAAInfo: CapLVal.getTBAAInfo());
3637 // Mark lvalue as nontemporal if the variable is marked as nontemporal
3638 // in simd context.
3639 if (getLangOpts().OpenMP &&
3640 CGM.getOpenMPRuntime().isNontemporalDecl(VD))
3641 CapLVal.setNontemporal(/*Value=*/true);
3642 return CapLVal;
3643 }
3644
3645 assert(isa<BlockDecl>(CurCodeDecl));
3646 Address addr = GetAddrOfBlockDecl(var: VD);
3647 return MakeAddrLValue(Addr: addr, T, Source: AlignmentSource::Decl);
3648 }
3649 }
3650
3651 // FIXME: We should be able to assert this for FunctionDecls as well!
3652 // FIXME: We should be able to assert this for all DeclRefExprs, not just
3653 // those with a valid source location.
3654 assert((ND->isUsed(false) || !isa<VarDecl>(ND) || E->isNonOdrUse() ||
3655 !E->getLocation().isValid()) &&
3656 "Should not use decl without marking it used!");
3657
3658 if (ND->hasAttr<WeakRefAttr>()) {
3659 const auto *VD = cast<ValueDecl>(Val: ND);
3660 ConstantAddress Aliasee = CGM.GetWeakRefReference(VD);
3661 return MakeAddrLValue(Addr: Aliasee, T, Source: AlignmentSource::Decl);
3662 }
3663
3664 if (const auto *VD = dyn_cast<VarDecl>(Val: ND)) {
3665 // Check if this is a global variable.
3666 if (VD->hasLinkage() || VD->isStaticDataMember())
3667 return EmitGlobalVarDeclLValue(CGF&: *this, E, VD);
3668
3669 Address addr = Address::invalid();
3670
3671 // The variable should generally be present in the local decl map.
3672 auto iter = LocalDeclMap.find(Val: VD);
3673 if (iter != LocalDeclMap.end()) {
3674 addr = iter->second;
3675
3676 // Otherwise, it might be static local we haven't emitted yet for
3677 // some reason; most likely, because it's in an outer function.
3678 } else if (VD->isStaticLocal()) {
3679 llvm::Constant *var = CGM.getOrCreateStaticVarDecl(
3680 D: *VD, Linkage: CGM.getLLVMLinkageVarDefinition(VD));
3681 addr = Address(
3682 var, ConvertTypeForMem(T: VD->getType()), getContext().getDeclAlign(D: VD));
3683
3684 // No other cases for now.
3685 } else {
3686 llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?");
3687 }
3688
3689 // Handle threadlocal function locals.
3690 if (VD->getTLSKind() != VarDecl::TLS_None)
3691 addr = addr.withPointer(
3692 NewPointer: Builder.CreateThreadLocalAddress(Ptr: addr.getBasePointer()),
3693 IsKnownNonNull: NotKnownNonNull);
3694
3695 // Check for OpenMP threadprivate variables.
3696 if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd &&
3697 VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
3698 return EmitThreadPrivateVarDeclLValue(
3699 CGF&: *this, VD, T, Addr: addr, RealVarTy: getTypes().ConvertTypeForMem(T: VD->getType()),
3700 Loc: E->getExprLoc());
3701 }
3702
3703 // Drill into block byref variables.
3704 bool isBlockByref = VD->isEscapingByref();
3705 if (isBlockByref) {
3706 addr = emitBlockByrefAddress(baseAddr: addr, V: VD);
3707 }
3708
3709 // Drill into reference types.
3710 LValue LV = VD->getType()->isReferenceType() ?
3711 EmitLoadOfReferenceLValue(RefAddr: addr, RefTy: VD->getType(), Source: AlignmentSource::Decl) :
3712 MakeAddrLValue(Addr: addr, T, Source: AlignmentSource::Decl);
3713
3714 bool isLocalStorage = VD->hasLocalStorage();
3715
3716 bool NonGCable = isLocalStorage &&
3717 !VD->getType()->isReferenceType() &&
3718 !isBlockByref;
3719 if (NonGCable) {
3720 LV.getQuals().removeObjCGCAttr();
3721 LV.setNonGC(true);
3722 }
3723
3724 bool isImpreciseLifetime =
3725 (isLocalStorage && !VD->hasAttr<ObjCPreciseLifetimeAttr>());
3726 if (isImpreciseLifetime)
3727 LV.setARCPreciseLifetime(ARCImpreciseLifetime);
3728 setObjCGCLValueClass(Ctx: getContext(), E, LV);
3729 return LV;
3730 }
3731
3732 if (const auto *FD = dyn_cast<FunctionDecl>(Val: ND))
3733 return EmitFunctionDeclLValue(CGF&: *this, E, GD: FD);
3734
3735 // FIXME: While we're emitting a binding from an enclosing scope, all other
3736 // DeclRefExprs we see should be implicitly treated as if they also refer to
3737 // an enclosing scope.
3738 if (const auto *BD = dyn_cast<BindingDecl>(Val: ND)) {
3739 if (E->refersToEnclosingVariableOrCapture()) {
3740 auto *FD = LambdaCaptureFields.lookup(Val: BD);
3741 return EmitCapturedFieldLValue(CGF&: *this, FD, ThisValue: CXXABIThisValue);
3742 }
3743 // Suppress debug location updates when visiting the binding, since the
3744 // binding may emit instructions that would otherwise be associated with the
3745 // binding itself, rather than the expression referencing the binding. (this
3746 // leads to jumpy debug stepping behavior where the location/debugger jump
3747 // back to the binding declaration, then back to the expression referencing
3748 // the binding)
3749 DisableDebugLocationUpdates D(*this);
3750 return EmitLValue(E: BD->getBinding(), IsKnownNonNull: NotKnownNonNull);
3751 }
3752
3753 // We can form DeclRefExprs naming GUID declarations when reconstituting
3754 // non-type template parameters into expressions.
3755 if (const auto *GD = dyn_cast<MSGuidDecl>(Val: ND))
3756 return MakeAddrLValue(Addr: CGM.GetAddrOfMSGuidDecl(GD), T,
3757 Source: AlignmentSource::Decl);
3758
3759 if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(Val: ND)) {
3760 ConstantAddress ATPO = CGM.GetAddrOfTemplateParamObject(TPO);
3761 auto AS = getLangASFromTargetAS(TargetAS: ATPO.getAddressSpace());
3762
3763 if (AS != T.getAddressSpace()) {
3764 auto TargetAS = getContext().getTargetAddressSpace(AS: T.getAddressSpace());
3765 llvm::Type *PtrTy =
3766 llvm::PointerType::get(C&: CGM.getLLVMContext(), AddressSpace: TargetAS);
3767 llvm::Constant *ASC = CGM.performAddrSpaceCast(Src: ATPO.getPointer(), DestTy: PtrTy);
3768 ATPO = ConstantAddress(ASC, ATPO.getElementType(), ATPO.getAlignment());
3769 }
3770
3771 return MakeAddrLValue(Addr: ATPO, T, Source: AlignmentSource::Decl);
3772 }
3773
3774 llvm_unreachable("Unhandled DeclRefExpr");
3775}
3776
3777LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
3778 // __extension__ doesn't affect lvalue-ness.
3779 if (E->getOpcode() == UO_Extension)
3780 return EmitLValue(E: E->getSubExpr());
3781
3782 QualType ExprTy = getContext().getCanonicalType(T: E->getSubExpr()->getType());
3783 switch (E->getOpcode()) {
3784 default: llvm_unreachable("Unknown unary operator lvalue!");
3785 case UO_Deref: {
3786 QualType T = E->getSubExpr()->getType()->getPointeeType();
3787 assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
3788
3789 LValueBaseInfo BaseInfo;
3790 TBAAAccessInfo TBAAInfo;
3791 Address Addr = EmitPointerWithAlignment(E: E->getSubExpr(), BaseInfo: &BaseInfo,
3792 TBAAInfo: &TBAAInfo);
3793 LValue LV = MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
3794 LV.getQuals().setAddressSpace(ExprTy.getAddressSpace());
3795
3796 // We should not generate __weak write barrier on indirect reference
3797 // of a pointer to object; as in void foo (__weak id *param); *param = 0;
3798 // But, we continue to generate __strong write barrier on indirect write
3799 // into a pointer to object.
3800 if (getLangOpts().ObjC &&
3801 getLangOpts().getGC() != LangOptions::NonGC &&
3802 LV.isObjCWeak())
3803 LV.setNonGC(!E->isOBJCGCCandidate(Ctx&: getContext()));
3804 return LV;
3805 }
3806 case UO_Real:
3807 case UO_Imag: {
3808 LValue LV = EmitLValue(E: E->getSubExpr());
3809 assert(LV.isSimple() && "real/imag on non-ordinary l-value");
3810
3811 // __real is valid on scalars. This is a faster way of testing that.
3812 // __imag can only produce an rvalue on scalars.
3813 if (E->getOpcode() == UO_Real &&
3814 !LV.getAddress().getElementType()->isStructTy()) {
3815 assert(E->getSubExpr()->getType()->isArithmeticType());
3816 return LV;
3817 }
3818
3819 QualType T = ExprTy->castAs<ComplexType>()->getElementType();
3820
3821 Address Component =
3822 (E->getOpcode() == UO_Real
3823 ? emitAddrOfRealComponent(complex: LV.getAddress(), complexType: LV.getType())
3824 : emitAddrOfImagComponent(complex: LV.getAddress(), complexType: LV.getType()));
3825 LValue ElemLV = MakeAddrLValue(Addr: Component, T, BaseInfo: LV.getBaseInfo(),
3826 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: T));
3827 ElemLV.getQuals().addQualifiers(Q: LV.getQuals());
3828 return ElemLV;
3829 }
3830 case UO_PreInc:
3831 case UO_PreDec: {
3832 LValue LV = EmitLValue(E: E->getSubExpr());
3833 bool isInc = E->getOpcode() == UO_PreInc;
3834
3835 if (E->getType()->isAnyComplexType())
3836 EmitComplexPrePostIncDec(E, LV, isInc, isPre: true/*isPre*/);
3837 else
3838 EmitScalarPrePostIncDec(E, LV, isInc, isPre: true/*isPre*/);
3839 return LV;
3840 }
3841 }
3842}
3843
3844LValue CodeGenFunction::EmitStringLiteralLValue(const StringLiteral *E) {
3845 return MakeAddrLValue(Addr: CGM.GetAddrOfConstantStringFromLiteral(S: E),
3846 T: E->getType(), Source: AlignmentSource::Decl);
3847}
3848
3849LValue CodeGenFunction::EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E) {
3850 return MakeAddrLValue(Addr: CGM.GetAddrOfConstantStringFromObjCEncode(E),
3851 T: E->getType(), Source: AlignmentSource::Decl);
3852}
3853
3854LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
3855 auto SL = E->getFunctionName();
3856 assert(SL != nullptr && "No StringLiteral name in PredefinedExpr");
3857 StringRef FnName = CurFn->getName();
3858 FnName.consume_front(Prefix: "\01");
3859 StringRef NameItems[] = {
3860 PredefinedExpr::getIdentKindName(IK: E->getIdentKind()), FnName};
3861 std::string GVName = llvm::join(Begin: NameItems, End: NameItems + 2, Separator: ".");
3862 if (auto *BD = dyn_cast_or_null<BlockDecl>(Val: CurCodeDecl)) {
3863 std::string Name = std::string(SL->getString());
3864 if (!Name.empty()) {
3865 unsigned Discriminator =
3866 CGM.getCXXABI().getMangleContext().getBlockId(BD, Local: true);
3867 if (Discriminator)
3868 Name += "_" + Twine(Discriminator + 1).str();
3869 auto C = CGM.GetAddrOfConstantCString(Str: Name, GlobalName: GVName);
3870 return MakeAddrLValue(Addr: C, T: E->getType(), Source: AlignmentSource::Decl);
3871 } else {
3872 auto C = CGM.GetAddrOfConstantCString(Str: std::string(FnName), GlobalName: GVName);
3873 return MakeAddrLValue(Addr: C, T: E->getType(), Source: AlignmentSource::Decl);
3874 }
3875 }
3876 auto C = CGM.GetAddrOfConstantStringFromLiteral(S: SL, Name: GVName);
3877 return MakeAddrLValue(Addr: C, T: E->getType(), Source: AlignmentSource::Decl);
3878}
3879
3880/// Emit a type description suitable for use by a runtime sanitizer library. The
3881/// format of a type descriptor is
3882///
3883/// \code
3884/// { i16 TypeKind, i16 TypeInfo }
3885/// \endcode
3886///
3887/// followed by an array of i8 containing the type name with extra information
3888/// for BitInt. TypeKind is TK_Integer(0) for an integer, TK_Float(1) for a
3889/// floating point value, TK_BitInt(2) for BitInt and TK_Unknown(0xFFFF) for
3890/// anything else.
3891llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) {
3892 // Only emit each type's descriptor once.
3893 if (llvm::Constant *C = CGM.getTypeDescriptorFromMap(Ty: T))
3894 return C;
3895
3896 uint16_t TypeKind = TK_Unknown;
3897 uint16_t TypeInfo = 0;
3898 bool IsBitInt = false;
3899
3900 if (T->isIntegerType()) {
3901 TypeKind = TK_Integer;
3902 TypeInfo = (llvm::Log2_32(Value: getContext().getTypeSize(T)) << 1) |
3903 (T->isSignedIntegerType() ? 1 : 0);
3904 // Follow suggestion from discussion of issue 64100.
3905 // So we can write the exact amount of bits in TypeName after '\0'
3906 // making it <diagnostic-like type name>.'\0'.<32-bit width>.
3907 if (T->isSignedIntegerType() && T->getAs<BitIntType>()) {
3908 // Do a sanity checks as we are using 32-bit type to store bit length.
3909 assert(getContext().getTypeSize(T) > 0 &&
3910 " non positive amount of bits in __BitInt type");
3911 assert(getContext().getTypeSize(T) <= 0xFFFFFFFF &&
3912 " too many bits in __BitInt type");
3913
3914 // Redefine TypeKind with the actual __BitInt type if we have signed
3915 // BitInt.
3916 TypeKind = TK_BitInt;
3917 IsBitInt = true;
3918 }
3919 } else if (T->isFloatingType()) {
3920 TypeKind = TK_Float;
3921 TypeInfo = getContext().getTypeSize(T);
3922 }
3923
3924 // Format the type name as if for a diagnostic, including quotes and
3925 // optionally an 'aka'.
3926 SmallString<32> Buffer;
3927 CGM.getDiags().ConvertArgToString(Kind: DiagnosticsEngine::ak_qualtype,
3928 Val: (intptr_t)T.getAsOpaquePtr(), Modifier: StringRef(),
3929 Argument: StringRef(), PrevArgs: {}, Output&: Buffer, QualTypeVals: {});
3930
3931 if (IsBitInt) {
3932 // The Structure is: 0 to end the string, 32 bit unsigned integer in target
3933 // endianness, zero.
3934 char S[6] = {'\0', '\0', '\0', '\0', '\0', '\0'};
3935 const auto *EIT = T->castAs<BitIntType>();
3936 uint32_t Bits = EIT->getNumBits();
3937 llvm::support::endian::write32(P: S + 1, V: Bits,
3938 E: getTarget().isBigEndian()
3939 ? llvm::endianness::big
3940 : llvm::endianness::little);
3941 StringRef Str = StringRef(S, sizeof(S) / sizeof(decltype(S[0])));
3942 Buffer.append(RHS: Str);
3943 }
3944
3945 llvm::Constant *Components[] = {
3946 Builder.getInt16(C: TypeKind), Builder.getInt16(C: TypeInfo),
3947 llvm::ConstantDataArray::getString(Context&: getLLVMContext(), Initializer: Buffer)
3948 };
3949 llvm::Constant *Descriptor = llvm::ConstantStruct::getAnon(V: Components);
3950
3951 auto *GV = new llvm::GlobalVariable(
3952 CGM.getModule(), Descriptor->getType(),
3953 /*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage, Descriptor);
3954 GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3955 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV);
3956
3957 // Remember the descriptor for this type.
3958 CGM.setTypeDescriptorInMap(Ty: T, C: GV);
3959
3960 return GV;
3961}
3962
3963llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) {
3964 llvm::Type *TargetTy = IntPtrTy;
3965
3966 if (V->getType() == TargetTy)
3967 return V;
3968
3969 // Floating-point types which fit into intptr_t are bitcast to integers
3970 // and then passed directly (after zero-extension, if necessary).
3971 if (V->getType()->isFloatingPointTy()) {
3972 unsigned Bits = V->getType()->getPrimitiveSizeInBits().getFixedValue();
3973 if (Bits <= TargetTy->getIntegerBitWidth())
3974 V = Builder.CreateBitCast(V, DestTy: llvm::Type::getIntNTy(C&: getLLVMContext(),
3975 N: Bits));
3976 }
3977
3978 // Integers which fit in intptr_t are zero-extended and passed directly.
3979 if (V->getType()->isIntegerTy() &&
3980 V->getType()->getIntegerBitWidth() <= TargetTy->getIntegerBitWidth())
3981 return Builder.CreateZExt(V, DestTy: TargetTy);
3982
3983 // Pointers are passed directly, everything else is passed by address.
3984 if (!V->getType()->isPointerTy()) {
3985 RawAddress Ptr = CreateDefaultAlignTempAlloca(Ty: V->getType());
3986 Builder.CreateStore(Val: V, Addr: Ptr);
3987 V = Ptr.getPointer();
3988 }
3989 return Builder.CreatePtrToInt(V, DestTy: TargetTy);
3990}
3991
3992/// Emit a representation of a SourceLocation for passing to a handler
3993/// in a sanitizer runtime library. The format for this data is:
3994/// \code
3995/// struct SourceLocation {
3996/// const char *Filename;
3997/// int32_t Line, Column;
3998/// };
3999/// \endcode
4000/// For an invalid SourceLocation, the Filename pointer is null.
4001llvm::Constant *CodeGenFunction::EmitCheckSourceLocation(SourceLocation Loc) {
4002 llvm::Constant *Filename;
4003 int Line, Column;
4004
4005 PresumedLoc PLoc = getContext().getSourceManager().getPresumedLoc(Loc);
4006 if (PLoc.isValid()) {
4007 StringRef FilenameString = PLoc.getFilename();
4008
4009 int PathComponentsToStrip =
4010 CGM.getCodeGenOpts().EmitCheckPathComponentsToStrip;
4011 if (PathComponentsToStrip < 0) {
4012 assert(PathComponentsToStrip != INT_MIN);
4013 int PathComponentsToKeep = -PathComponentsToStrip;
4014 auto I = llvm::sys::path::rbegin(path: FilenameString);
4015 auto E = llvm::sys::path::rend(path: FilenameString);
4016 while (I != E && --PathComponentsToKeep)
4017 ++I;
4018
4019 FilenameString = FilenameString.substr(Start: I - E);
4020 } else if (PathComponentsToStrip > 0) {
4021 auto I = llvm::sys::path::begin(path: FilenameString);
4022 auto E = llvm::sys::path::end(path: FilenameString);
4023 while (I != E && PathComponentsToStrip--)
4024 ++I;
4025
4026 if (I != E)
4027 FilenameString =
4028 FilenameString.substr(Start: I - llvm::sys::path::begin(path: FilenameString));
4029 else
4030 FilenameString = llvm::sys::path::filename(path: FilenameString);
4031 }
4032
4033 auto FilenameGV =
4034 CGM.GetAddrOfConstantCString(Str: std::string(FilenameString), GlobalName: ".src");
4035 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(
4036 GV: cast<llvm::GlobalVariable>(
4037 Val: FilenameGV.getPointer()->stripPointerCasts()));
4038 Filename = FilenameGV.getPointer();
4039 Line = PLoc.getLine();
4040 Column = PLoc.getColumn();
4041 } else {
4042 Filename = llvm::Constant::getNullValue(Ty: Int8PtrTy);
4043 Line = Column = 0;
4044 }
4045
4046 llvm::Constant *Data[] = {Filename, Builder.getInt32(C: Line),
4047 Builder.getInt32(C: Column)};
4048
4049 return llvm::ConstantStruct::getAnon(V: Data);
4050}
4051
4052namespace {
4053/// Specify under what conditions this check can be recovered
4054enum class CheckRecoverableKind {
4055 /// Always terminate program execution if this check fails.
4056 Unrecoverable,
4057 /// Check supports recovering, runtime has both fatal (noreturn) and
4058 /// non-fatal handlers for this check.
4059 Recoverable,
4060 /// Runtime conditionally aborts, always need to support recovery.
4061 AlwaysRecoverable
4062};
4063}
4064
4065static CheckRecoverableKind
4066getRecoverableKind(SanitizerKind::SanitizerOrdinal Ordinal) {
4067 if (Ordinal == SanitizerKind::SO_Vptr)
4068 return CheckRecoverableKind::AlwaysRecoverable;
4069 else if (Ordinal == SanitizerKind::SO_Return ||
4070 Ordinal == SanitizerKind::SO_Unreachable)
4071 return CheckRecoverableKind::Unrecoverable;
4072 else
4073 return CheckRecoverableKind::Recoverable;
4074}
4075
4076namespace {
4077struct SanitizerHandlerInfo {
4078 char const *const Name;
4079 unsigned Version;
4080};
4081}
4082
4083const SanitizerHandlerInfo SanitizerHandlers[] = {
4084#define SANITIZER_CHECK(Enum, Name, Version, Msg) {#Name, Version},
4085 LIST_SANITIZER_CHECKS
4086#undef SANITIZER_CHECK
4087};
4088
4089static void emitCheckHandlerCall(CodeGenFunction &CGF,
4090 llvm::FunctionType *FnType,
4091 ArrayRef<llvm::Value *> FnArgs,
4092 SanitizerHandler CheckHandler,
4093 CheckRecoverableKind RecoverKind, bool IsFatal,
4094 llvm::BasicBlock *ContBB, bool NoMerge) {
4095 assert(IsFatal || RecoverKind != CheckRecoverableKind::Unrecoverable);
4096 std::optional<ApplyDebugLocation> DL;
4097 if (!CGF.Builder.getCurrentDebugLocation()) {
4098 // Ensure that the call has at least an artificial debug location.
4099 DL.emplace(args&: CGF, args: SourceLocation());
4100 }
4101 bool NeedsAbortSuffix =
4102 IsFatal && RecoverKind != CheckRecoverableKind::Unrecoverable;
4103 bool MinimalRuntime = CGF.CGM.getCodeGenOpts().SanitizeMinimalRuntime;
4104 bool HandlerPreserveAllRegs =
4105 CGF.CGM.getCodeGenOpts().SanitizeHandlerPreserveAllRegs;
4106 const SanitizerHandlerInfo &CheckInfo = SanitizerHandlers[CheckHandler];
4107 const StringRef CheckName = CheckInfo.Name;
4108 std::string FnName = "__ubsan_handle_" + CheckName.str();
4109 if (CheckInfo.Version && !MinimalRuntime)
4110 FnName += "_v" + llvm::utostr(X: CheckInfo.Version);
4111 if (MinimalRuntime)
4112 FnName += "_minimal";
4113 if (NeedsAbortSuffix)
4114 FnName += "_abort";
4115 if (HandlerPreserveAllRegs && !NeedsAbortSuffix)
4116 FnName += "_preserve";
4117 bool MayReturn =
4118 !IsFatal || RecoverKind == CheckRecoverableKind::AlwaysRecoverable;
4119
4120 llvm::AttrBuilder B(CGF.getLLVMContext());
4121 if (!MayReturn) {
4122 B.addAttribute(Val: llvm::Attribute::NoReturn)
4123 .addAttribute(Val: llvm::Attribute::NoUnwind);
4124 }
4125 B.addUWTableAttr(Kind: llvm::UWTableKind::Default);
4126
4127 llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(
4128 Ty: FnType, Name: FnName,
4129 ExtraAttrs: llvm::AttributeList::get(C&: CGF.getLLVMContext(),
4130 Index: llvm::AttributeList::FunctionIndex, B),
4131 /*Local=*/true);
4132 llvm::CallInst *HandlerCall = CGF.EmitNounwindRuntimeCall(callee: Fn, args: FnArgs);
4133 NoMerge = NoMerge || !CGF.CGM.getCodeGenOpts().OptimizationLevel ||
4134 (CGF.CurCodeDecl && CGF.CurCodeDecl->hasAttr<OptimizeNoneAttr>());
4135 if (NoMerge)
4136 HandlerCall->addFnAttr(Kind: llvm::Attribute::NoMerge);
4137 if (HandlerPreserveAllRegs && !NeedsAbortSuffix) {
4138 // N.B. there is also a clang::CallingConv which is not what we want here.
4139 HandlerCall->setCallingConv(llvm::CallingConv::PreserveAll);
4140 }
4141 if (!MayReturn) {
4142 HandlerCall->setDoesNotReturn();
4143 CGF.Builder.CreateUnreachable();
4144 } else {
4145 CGF.Builder.CreateBr(Dest: ContBB);
4146 }
4147}
4148
4149void CodeGenFunction::EmitCheck(
4150 ArrayRef<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>> Checked,
4151 SanitizerHandler CheckHandler, ArrayRef<llvm::Constant *> StaticArgs,
4152 ArrayRef<llvm::Value *> DynamicArgs, const TrapReason *TR) {
4153 assert(IsSanitizerScope);
4154 assert(Checked.size() > 0);
4155 assert(CheckHandler >= 0 &&
4156 size_t(CheckHandler) < std::size(SanitizerHandlers));
4157 const StringRef CheckName = SanitizerHandlers[CheckHandler].Name;
4158
4159 llvm::Value *FatalCond = nullptr;
4160 llvm::Value *RecoverableCond = nullptr;
4161 llvm::Value *TrapCond = nullptr;
4162 bool NoMerge = false;
4163 // Expand checks into:
4164 // (Check1 || !allow_ubsan_check) && (Check2 || !allow_ubsan_check) ...
4165 // We need separate allow_ubsan_check intrinsics because they have separately
4166 // specified cutoffs.
4167 // This expression looks expensive but will be simplified after
4168 // LowerAllowCheckPass.
4169 for (auto &[Check, Ord] : Checked) {
4170 llvm::Value *GuardedCheck = Check;
4171 if (ClSanitizeGuardChecks ||
4172 (CGM.getCodeGenOpts().SanitizeSkipHotCutoffs[Ord] > 0)) {
4173 llvm::Value *Allow = Builder.CreateCall(
4174 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::allow_ubsan_check),
4175 Args: llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: Ord));
4176 GuardedCheck = Builder.CreateOr(LHS: Check, RHS: Builder.CreateNot(V: Allow));
4177 }
4178
4179 // -fsanitize-trap= overrides -fsanitize-recover=.
4180 llvm::Value *&Cond = CGM.getCodeGenOpts().SanitizeTrap.has(O: Ord) ? TrapCond
4181 : CGM.getCodeGenOpts().SanitizeRecover.has(O: Ord)
4182 ? RecoverableCond
4183 : FatalCond;
4184 Cond = Cond ? Builder.CreateAnd(LHS: Cond, RHS: GuardedCheck) : GuardedCheck;
4185
4186 if (!CGM.getCodeGenOpts().SanitizeMergeHandlers.has(O: Ord))
4187 NoMerge = true;
4188 }
4189
4190 if (TrapCond)
4191 EmitTrapCheck(Checked: TrapCond, CheckHandlerID: CheckHandler, NoMerge, TR);
4192 if (!FatalCond && !RecoverableCond)
4193 return;
4194
4195 llvm::Value *JointCond;
4196 if (FatalCond && RecoverableCond)
4197 JointCond = Builder.CreateAnd(LHS: FatalCond, RHS: RecoverableCond);
4198 else
4199 JointCond = FatalCond ? FatalCond : RecoverableCond;
4200 assert(JointCond);
4201
4202 CheckRecoverableKind RecoverKind = getRecoverableKind(Ordinal: Checked[0].second);
4203 assert(SanOpts.has(Checked[0].second));
4204#ifndef NDEBUG
4205 for (int i = 1, n = Checked.size(); i < n; ++i) {
4206 assert(RecoverKind == getRecoverableKind(Checked[i].second) &&
4207 "All recoverable kinds in a single check must be same!");
4208 assert(SanOpts.has(Checked[i].second));
4209 }
4210#endif
4211
4212 llvm::BasicBlock *Cont = createBasicBlock(name: "cont");
4213 llvm::BasicBlock *Handlers = createBasicBlock(name: "handler." + CheckName);
4214 llvm::Instruction *Branch = Builder.CreateCondBr(Cond: JointCond, True: Cont, False: Handlers);
4215 // Give hint that we very much don't expect to execute the handler
4216 llvm::MDBuilder MDHelper(getLLVMContext());
4217 llvm::MDNode *Node = MDHelper.createLikelyBranchWeights();
4218 Branch->setMetadata(KindID: llvm::LLVMContext::MD_prof, Node);
4219 EmitBlock(BB: Handlers);
4220
4221 // Clear arguments for the MinimalRuntime handler.
4222 if (CGM.getCodeGenOpts().SanitizeMinimalRuntime) {
4223 StaticArgs = {};
4224 DynamicArgs = {};
4225 }
4226
4227 // Handler functions take an i8* pointing to the (handler-specific) static
4228 // information block, followed by a sequence of intptr_t arguments
4229 // representing operand values.
4230 SmallVector<llvm::Value *, 4> Args;
4231 SmallVector<llvm::Type *, 4> ArgTypes;
4232
4233 Args.reserve(N: DynamicArgs.size() + 1);
4234 ArgTypes.reserve(N: DynamicArgs.size() + 1);
4235
4236 // Emit handler arguments and create handler function type.
4237 if (!StaticArgs.empty()) {
4238 llvm::Constant *Info = llvm::ConstantStruct::getAnon(V: StaticArgs);
4239 auto *InfoPtr = new llvm::GlobalVariable(
4240 CGM.getModule(), Info->getType(),
4241 // Non-constant global is used in a handler to deduplicate reports.
4242 // TODO: change deduplication logic and make it constant.
4243 /*isConstant=*/false, llvm::GlobalVariable::PrivateLinkage, Info, "",
4244 nullptr, llvm::GlobalVariable::NotThreadLocal,
4245 CGM.getDataLayout().getDefaultGlobalsAddressSpace());
4246 InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4247 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV: InfoPtr);
4248 Args.push_back(Elt: InfoPtr);
4249 ArgTypes.push_back(Elt: Args.back()->getType());
4250 }
4251
4252 for (llvm::Value *DynamicArg : DynamicArgs) {
4253 Args.push_back(Elt: EmitCheckValue(V: DynamicArg));
4254 ArgTypes.push_back(Elt: IntPtrTy);
4255 }
4256
4257 llvm::FunctionType *FnType =
4258 llvm::FunctionType::get(Result: CGM.VoidTy, Params: ArgTypes, isVarArg: false);
4259
4260 if (!FatalCond || !RecoverableCond) {
4261 // Simple case: we need to generate a single handler call, either
4262 // fatal, or non-fatal.
4263 emitCheckHandlerCall(CGF&: *this, FnType, FnArgs: Args, CheckHandler, RecoverKind,
4264 IsFatal: (FatalCond != nullptr), ContBB: Cont, NoMerge);
4265 } else {
4266 // Emit two handler calls: first one for set of unrecoverable checks,
4267 // another one for recoverable.
4268 llvm::BasicBlock *NonFatalHandlerBB =
4269 createBasicBlock(name: "non_fatal." + CheckName);
4270 llvm::BasicBlock *FatalHandlerBB = createBasicBlock(name: "fatal." + CheckName);
4271 Builder.CreateCondBr(Cond: FatalCond, True: NonFatalHandlerBB, False: FatalHandlerBB);
4272 EmitBlock(BB: FatalHandlerBB);
4273 emitCheckHandlerCall(CGF&: *this, FnType, FnArgs: Args, CheckHandler, RecoverKind, IsFatal: true,
4274 ContBB: NonFatalHandlerBB, NoMerge);
4275 EmitBlock(BB: NonFatalHandlerBB);
4276 emitCheckHandlerCall(CGF&: *this, FnType, FnArgs: Args, CheckHandler, RecoverKind, IsFatal: false,
4277 ContBB: Cont, NoMerge);
4278 }
4279
4280 EmitBlock(BB: Cont);
4281}
4282
4283void CodeGenFunction::EmitCfiSlowPathCheck(
4284 SanitizerKind::SanitizerOrdinal Ordinal, llvm::Value *Cond,
4285 llvm::ConstantInt *TypeId, llvm::Value *Ptr,
4286 ArrayRef<llvm::Constant *> StaticArgs) {
4287 llvm::BasicBlock *Cont = createBasicBlock(name: "cfi.cont");
4288
4289 llvm::BasicBlock *CheckBB = createBasicBlock(name: "cfi.slowpath");
4290 llvm::BranchInst *BI = Builder.CreateCondBr(Cond, True: Cont, False: CheckBB);
4291
4292 llvm::MDBuilder MDHelper(getLLVMContext());
4293 llvm::MDNode *Node = MDHelper.createLikelyBranchWeights();
4294 BI->setMetadata(KindID: llvm::LLVMContext::MD_prof, Node);
4295
4296 EmitBlock(BB: CheckBB);
4297
4298 bool WithDiag = !CGM.getCodeGenOpts().SanitizeTrap.has(O: Ordinal);
4299
4300 llvm::CallInst *CheckCall;
4301 llvm::FunctionCallee SlowPathFn;
4302 if (WithDiag) {
4303 llvm::Constant *Info = llvm::ConstantStruct::getAnon(V: StaticArgs);
4304 auto *InfoPtr =
4305 new llvm::GlobalVariable(CGM.getModule(), Info->getType(), false,
4306 llvm::GlobalVariable::PrivateLinkage, Info);
4307 InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4308 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV: InfoPtr);
4309
4310 SlowPathFn = CGM.getModule().getOrInsertFunction(
4311 Name: "__cfi_slowpath_diag",
4312 T: llvm::FunctionType::get(Result: VoidTy, Params: {Int64Ty, Int8PtrTy, Int8PtrTy},
4313 isVarArg: false));
4314 CheckCall = Builder.CreateCall(Callee: SlowPathFn, Args: {TypeId, Ptr, InfoPtr});
4315 } else {
4316 SlowPathFn = CGM.getModule().getOrInsertFunction(
4317 Name: "__cfi_slowpath",
4318 T: llvm::FunctionType::get(Result: VoidTy, Params: {Int64Ty, Int8PtrTy}, isVarArg: false));
4319 CheckCall = Builder.CreateCall(Callee: SlowPathFn, Args: {TypeId, Ptr});
4320 }
4321
4322 CGM.setDSOLocal(
4323 cast<llvm::GlobalValue>(Val: SlowPathFn.getCallee()->stripPointerCasts()));
4324 CheckCall->setDoesNotThrow();
4325
4326 EmitBlock(BB: Cont);
4327}
4328
4329// Emit a stub for __cfi_check function so that the linker knows about this
4330// symbol in LTO mode.
4331void CodeGenFunction::EmitCfiCheckStub() {
4332 llvm::Module *M = &CGM.getModule();
4333 ASTContext &C = getContext();
4334 QualType QInt64Ty = C.getIntTypeForBitwidth(DestWidth: 64, Signed: false);
4335
4336 FunctionArgList FnArgs;
4337 ImplicitParamDecl ArgCallsiteTypeId(C, QInt64Ty, ImplicitParamKind::Other);
4338 ImplicitParamDecl ArgAddr(C, C.VoidPtrTy, ImplicitParamKind::Other);
4339 ImplicitParamDecl ArgCFICheckFailData(C, C.VoidPtrTy,
4340 ImplicitParamKind::Other);
4341 FnArgs.push_back(Elt: &ArgCallsiteTypeId);
4342 FnArgs.push_back(Elt: &ArgAddr);
4343 FnArgs.push_back(Elt: &ArgCFICheckFailData);
4344 const CGFunctionInfo &FI =
4345 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: FnArgs);
4346
4347 llvm::Function *F = llvm::Function::Create(
4348 Ty: llvm::FunctionType::get(Result: VoidTy, Params: {Int64Ty, VoidPtrTy, VoidPtrTy}, isVarArg: false),
4349 Linkage: llvm::GlobalValue::WeakAnyLinkage, N: "__cfi_check", M);
4350 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F, /*IsThunk=*/false);
4351 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F);
4352 F->setAlignment(llvm::Align(4096));
4353 CGM.setDSOLocal(F);
4354
4355 llvm::LLVMContext &Ctx = M->getContext();
4356 llvm::BasicBlock *BB = llvm::BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: F);
4357 // CrossDSOCFI pass is not executed if there is no executable code.
4358 SmallVector<llvm::Value*> Args{F->getArg(i: 2), F->getArg(i: 1)};
4359 llvm::CallInst::Create(Func: M->getFunction(Name: "__cfi_check_fail"), Args, NameStr: "", InsertBefore: BB);
4360 llvm::ReturnInst::Create(C&: Ctx, retVal: nullptr, InsertBefore: BB);
4361}
4362
4363// This function is basically a switch over the CFI failure kind, which is
4364// extracted from CFICheckFailData (1st function argument). Each case is either
4365// llvm.trap or a call to one of the two runtime handlers, based on
4366// -fsanitize-trap and -fsanitize-recover settings. Default case (invalid
4367// failure kind) traps, but this should really never happen. CFICheckFailData
4368// can be nullptr if the calling module has -fsanitize-trap behavior for this
4369// check kind; in this case __cfi_check_fail traps as well.
4370void CodeGenFunction::EmitCfiCheckFail() {
4371 auto CheckHandler = SanitizerHandler::CFICheckFail;
4372 // TODO: the SanitizerKind is not yet determined for this check (and might
4373 // not even be available, if Data == nullptr). However, we still want to
4374 // annotate the instrumentation. We approximate this by using all the CFI
4375 // kinds.
4376 SanitizerDebugLocation SanScope(
4377 this,
4378 {SanitizerKind::SO_CFIVCall, SanitizerKind::SO_CFINVCall,
4379 SanitizerKind::SO_CFIDerivedCast, SanitizerKind::SO_CFIUnrelatedCast,
4380 SanitizerKind::SO_CFIICall},
4381 CheckHandler);
4382 FunctionArgList Args;
4383 ImplicitParamDecl ArgData(getContext(), getContext().VoidPtrTy,
4384 ImplicitParamKind::Other);
4385 ImplicitParamDecl ArgAddr(getContext(), getContext().VoidPtrTy,
4386 ImplicitParamKind::Other);
4387 Args.push_back(Elt: &ArgData);
4388 Args.push_back(Elt: &ArgAddr);
4389
4390 const CGFunctionInfo &FI =
4391 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: getContext().VoidTy, args: Args);
4392
4393 llvm::Function *F = llvm::Function::Create(
4394 Ty: llvm::FunctionType::get(Result: VoidTy, Params: {VoidPtrTy, VoidPtrTy}, isVarArg: false),
4395 Linkage: llvm::GlobalValue::WeakODRLinkage, N: "__cfi_check_fail", M: &CGM.getModule());
4396
4397 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F, /*IsThunk=*/false);
4398 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F);
4399 F->setVisibility(llvm::GlobalValue::HiddenVisibility);
4400
4401 StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidTy, Fn: F, FnInfo: FI, Args,
4402 Loc: SourceLocation());
4403
4404 ApplyDebugLocation ADL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
4405
4406 // This function is not affected by NoSanitizeList. This function does
4407 // not have a source location, but "src:*" would still apply. Revert any
4408 // changes to SanOpts made in StartFunction.
4409 SanOpts = CGM.getLangOpts().Sanitize;
4410
4411 llvm::Value *Data =
4412 EmitLoadOfScalar(Addr: GetAddrOfLocalVar(VD: &ArgData), /*Volatile=*/false,
4413 Ty: CGM.getContext().VoidPtrTy, Loc: ArgData.getLocation());
4414 llvm::Value *Addr =
4415 EmitLoadOfScalar(Addr: GetAddrOfLocalVar(VD: &ArgAddr), /*Volatile=*/false,
4416 Ty: CGM.getContext().VoidPtrTy, Loc: ArgAddr.getLocation());
4417
4418 // Data == nullptr means the calling module has trap behaviour for this check.
4419 llvm::Value *DataIsNotNullPtr =
4420 Builder.CreateICmpNE(LHS: Data, RHS: llvm::ConstantPointerNull::get(T: Int8PtrTy));
4421 // TODO: since there is no data, we don't know the CheckKind, and therefore
4422 // cannot inspect CGM.getCodeGenOpts().SanitizeMergeHandlers. We default to
4423 // NoMerge = false. Users can disable merging by disabling optimization.
4424 EmitTrapCheck(Checked: DataIsNotNullPtr, CheckHandlerID: SanitizerHandler::CFICheckFail,
4425 /*NoMerge=*/false);
4426
4427 llvm::StructType *SourceLocationTy =
4428 llvm::StructType::get(elt1: VoidPtrTy, elts: Int32Ty, elts: Int32Ty);
4429 llvm::StructType *CfiCheckFailDataTy =
4430 llvm::StructType::get(elt1: Int8Ty, elts: SourceLocationTy, elts: VoidPtrTy);
4431
4432 llvm::Value *V = Builder.CreateConstGEP2_32(
4433 Ty: CfiCheckFailDataTy, Ptr: Builder.CreatePointerCast(V: Data, DestTy: DefaultPtrTy), Idx0: 0, Idx1: 0);
4434
4435 Address CheckKindAddr(V, Int8Ty, getIntAlign());
4436 llvm::Value *CheckKind = Builder.CreateLoad(Addr: CheckKindAddr);
4437
4438 llvm::Value *AllVtables = llvm::MetadataAsValue::get(
4439 Context&: CGM.getLLVMContext(),
4440 MD: llvm::MDString::get(Context&: CGM.getLLVMContext(), Str: "all-vtables"));
4441 llvm::Value *ValidVtable = Builder.CreateZExt(
4442 V: Builder.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::type_test),
4443 Args: {Addr, AllVtables}),
4444 DestTy: IntPtrTy);
4445
4446 const std::pair<int, SanitizerKind::SanitizerOrdinal> CheckKinds[] = {
4447 {CFITCK_VCall, SanitizerKind::SO_CFIVCall},
4448 {CFITCK_NVCall, SanitizerKind::SO_CFINVCall},
4449 {CFITCK_DerivedCast, SanitizerKind::SO_CFIDerivedCast},
4450 {CFITCK_UnrelatedCast, SanitizerKind::SO_CFIUnrelatedCast},
4451 {CFITCK_ICall, SanitizerKind::SO_CFIICall}};
4452
4453 for (auto CheckKindOrdinalPair : CheckKinds) {
4454 int Kind = CheckKindOrdinalPair.first;
4455 SanitizerKind::SanitizerOrdinal Ordinal = CheckKindOrdinalPair.second;
4456
4457 // TODO: we could apply SanitizerAnnotateDebugInfo(Ordinal) instead of
4458 // relying on the SanitizerScope with all CFI ordinals
4459
4460 llvm::Value *Cond =
4461 Builder.CreateICmpNE(LHS: CheckKind, RHS: llvm::ConstantInt::get(Ty: Int8Ty, V: Kind));
4462 if (CGM.getLangOpts().Sanitize.has(O: Ordinal))
4463 EmitCheck(Checked: std::make_pair(x&: Cond, y&: Ordinal), CheckHandler: SanitizerHandler::CFICheckFail,
4464 StaticArgs: {}, DynamicArgs: {Data, Addr, ValidVtable});
4465 else
4466 // TODO: we can't rely on CGM.getCodeGenOpts().SanitizeMergeHandlers.
4467 // Although the compiler allows SanitizeMergeHandlers to be set
4468 // independently of CGM.getLangOpts().Sanitize, Driver/SanitizerArgs.cpp
4469 // requires that SanitizeMergeHandlers is a subset of Sanitize.
4470 EmitTrapCheck(Checked: Cond, CheckHandlerID: CheckHandler, /*NoMerge=*/false);
4471 }
4472
4473 FinishFunction();
4474 // The only reference to this function will be created during LTO link.
4475 // Make sure it survives until then.
4476 CGM.addUsedGlobal(GV: F);
4477}
4478
4479void CodeGenFunction::EmitUnreachable(SourceLocation Loc) {
4480 if (SanOpts.has(K: SanitizerKind::Unreachable)) {
4481 auto CheckOrdinal = SanitizerKind::SO_Unreachable;
4482 auto CheckHandler = SanitizerHandler::BuiltinUnreachable;
4483 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
4484 EmitCheck(Checked: std::make_pair(x: static_cast<llvm::Value *>(Builder.getFalse()),
4485 y&: CheckOrdinal),
4486 CheckHandler, StaticArgs: EmitCheckSourceLocation(Loc), DynamicArgs: {});
4487 }
4488 Builder.CreateUnreachable();
4489}
4490
4491void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked,
4492 SanitizerHandler CheckHandlerID,
4493 bool NoMerge, const TrapReason *TR) {
4494 llvm::BasicBlock *Cont = createBasicBlock(name: "cont");
4495
4496 // If we're optimizing, collapse all calls to trap down to just one per
4497 // check-type per function to save on code size.
4498 if ((int)TrapBBs.size() <= CheckHandlerID)
4499 TrapBBs.resize(N: CheckHandlerID + 1);
4500
4501 llvm::BasicBlock *&TrapBB = TrapBBs[CheckHandlerID];
4502
4503 llvm::DILocation *TrapLocation = Builder.getCurrentDebugLocation();
4504 llvm::StringRef TrapMessage;
4505 llvm::StringRef TrapCategory;
4506 auto DebugTrapReasonKind = CGM.getCodeGenOpts().getSanitizeDebugTrapReasons();
4507 if (TR && !TR->isEmpty() &&
4508 DebugTrapReasonKind ==
4509 CodeGenOptions::SanitizeDebugTrapReasonKind::Detailed) {
4510 TrapMessage = TR->getMessage();
4511 TrapCategory = TR->getCategory();
4512 } else {
4513 TrapMessage = GetUBSanTrapForHandler(ID: CheckHandlerID);
4514 TrapCategory = "Undefined Behavior Sanitizer";
4515 }
4516
4517 if (getDebugInfo() && !TrapMessage.empty() &&
4518 DebugTrapReasonKind !=
4519 CodeGenOptions::SanitizeDebugTrapReasonKind::None &&
4520 TrapLocation) {
4521 TrapLocation = getDebugInfo()->CreateTrapFailureMessageFor(
4522 TrapLocation, Category: TrapCategory, FailureMsg: TrapMessage);
4523 }
4524
4525 NoMerge = NoMerge || !CGM.getCodeGenOpts().OptimizationLevel ||
4526 (CurCodeDecl && CurCodeDecl->hasAttr<OptimizeNoneAttr>());
4527
4528 llvm::MDBuilder MDHelper(getLLVMContext());
4529 if (TrapBB && !NoMerge) {
4530 auto Call = TrapBB->begin();
4531 assert(isa<llvm::CallInst>(Call) && "Expected call in trap BB");
4532
4533 Call->applyMergedLocation(LocA: Call->getDebugLoc(), LocB: TrapLocation);
4534
4535 Builder.CreateCondBr(Cond: Checked, True: Cont, False: TrapBB,
4536 BranchWeights: MDHelper.createLikelyBranchWeights());
4537 } else {
4538 TrapBB = createBasicBlock(name: "trap");
4539 Builder.CreateCondBr(Cond: Checked, True: Cont, False: TrapBB,
4540 BranchWeights: MDHelper.createLikelyBranchWeights());
4541 EmitBlock(BB: TrapBB);
4542
4543 ApplyDebugLocation applyTrapDI(*this, TrapLocation);
4544
4545 llvm::CallInst *TrapCall;
4546 if (CGM.getCodeGenOpts().SanitizeTrapLoop)
4547 TrapCall =
4548 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::looptrap));
4549 else
4550 TrapCall = Builder.CreateCall(
4551 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::ubsantrap),
4552 Args: llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: CheckHandlerID));
4553
4554 if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
4555 auto A = llvm::Attribute::get(Context&: getLLVMContext(), Kind: "trap-func-name",
4556 Val: CGM.getCodeGenOpts().TrapFuncName);
4557 TrapCall->addFnAttr(Attr: A);
4558 }
4559 if (NoMerge)
4560 TrapCall->addFnAttr(Kind: llvm::Attribute::NoMerge);
4561 TrapCall->setDoesNotReturn();
4562 TrapCall->setDoesNotThrow();
4563 Builder.CreateUnreachable();
4564 }
4565
4566 EmitBlock(BB: Cont);
4567}
4568
4569llvm::CallInst *CodeGenFunction::EmitTrapCall(llvm::Intrinsic::ID IntrID) {
4570 llvm::CallInst *TrapCall =
4571 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: IntrID));
4572
4573 if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
4574 auto A = llvm::Attribute::get(Context&: getLLVMContext(), Kind: "trap-func-name",
4575 Val: CGM.getCodeGenOpts().TrapFuncName);
4576 TrapCall->addFnAttr(Attr: A);
4577 }
4578
4579 if (InNoMergeAttributedStmt)
4580 TrapCall->addFnAttr(Kind: llvm::Attribute::NoMerge);
4581 return TrapCall;
4582}
4583
4584Address CodeGenFunction::EmitArrayToPointerDecay(const Expr *E,
4585 LValueBaseInfo *BaseInfo,
4586 TBAAAccessInfo *TBAAInfo) {
4587 assert(E->getType()->isArrayType() &&
4588 "Array to pointer decay must have array source type!");
4589
4590 // Expressions of array type can't be bitfields or vector elements.
4591 LValue LV = EmitLValue(E);
4592 Address Addr = LV.getAddress();
4593
4594 // If the array type was an incomplete type, we need to make sure
4595 // the decay ends up being the right type.
4596 llvm::Type *NewTy = ConvertType(T: E->getType());
4597 Addr = Addr.withElementType(ElemTy: NewTy);
4598
4599 // Note that VLA pointers are always decayed, so we don't need to do
4600 // anything here.
4601 if (!E->getType()->isVariableArrayType()) {
4602 assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
4603 "Expected pointer to array");
4604
4605 if (getLangOpts().EmitStructuredGEP) {
4606 // Array-to-pointer decay for an SGEP is a no-op as we don't do any
4607 // logical indexing. See #179951 for some additional context.
4608 auto *SGEP =
4609 Builder.CreateStructuredGEP(BaseType: NewTy, PtrBase: Addr.emitRawPointer(CGF&: *this), Indices: {});
4610 Addr = Address(SGEP, NewTy, Addr.getAlignment(), Addr.isKnownNonNull());
4611 } else {
4612 Addr = Builder.CreateConstArrayGEP(Addr, Index: 0, Name: "arraydecay");
4613 }
4614 }
4615
4616 // The result of this decay conversion points to an array element within the
4617 // base lvalue. However, since TBAA currently does not support representing
4618 // accesses to elements of member arrays, we conservatively represent accesses
4619 // to the pointee object as if it had no any base lvalue specified.
4620 // TODO: Support TBAA for member arrays.
4621 QualType EltType = E->getType()->castAsArrayTypeUnsafe()->getElementType();
4622 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
4623 if (TBAAInfo) *TBAAInfo = CGM.getTBAAAccessInfo(AccessType: EltType);
4624
4625 return Addr.withElementType(ElemTy: ConvertTypeForMem(T: EltType));
4626}
4627
4628/// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an
4629/// array to pointer, return the array subexpression.
4630static const Expr *isSimpleArrayDecayOperand(const Expr *E) {
4631 // If this isn't just an array->pointer decay, bail out.
4632 const auto *CE = dyn_cast<CastExpr>(Val: E);
4633 if (!CE || CE->getCastKind() != CK_ArrayToPointerDecay)
4634 return nullptr;
4635
4636 // If this is a decay from variable width array, bail out.
4637 const Expr *SubExpr = CE->getSubExpr();
4638 if (SubExpr->getType()->isVariableArrayType())
4639 return nullptr;
4640
4641 return SubExpr;
4642}
4643
4644static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF,
4645 llvm::Type *elemType,
4646 llvm::Value *ptr,
4647 ArrayRef<llvm::Value*> indices,
4648 bool inbounds,
4649 bool signedIndices,
4650 SourceLocation loc,
4651 const llvm::Twine &name = "arrayidx") {
4652 if (inbounds && CGF.getLangOpts().EmitStructuredGEP)
4653 return CGF.Builder.CreateStructuredGEP(BaseType: elemType, PtrBase: ptr, Indices: indices);
4654
4655 if (inbounds) {
4656 return CGF.EmitCheckedInBoundsGEP(ElemTy: elemType, Ptr: ptr, IdxList: indices, SignedIndices: signedIndices,
4657 IsSubtraction: CodeGenFunction::NotSubtraction, Loc: loc,
4658 Name: name);
4659 } else {
4660 return CGF.Builder.CreateGEP(Ty: elemType, Ptr: ptr, IdxList: indices, Name: name);
4661 }
4662}
4663
4664static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
4665 ArrayRef<llvm::Value *> indices,
4666 llvm::Type *arrayType,
4667 llvm::Type *elementType, bool inbounds,
4668 bool signedIndices, SourceLocation loc,
4669 CharUnits align,
4670 const llvm::Twine &name = "arrayidx") {
4671 if (inbounds && CGF.getLangOpts().EmitStructuredGEP)
4672 return RawAddress(CGF.Builder.CreateStructuredGEP(BaseType: arrayType,
4673 PtrBase: addr.emitRawPointer(CGF),
4674 Indices: indices.drop_front()),
4675 elementType, align);
4676
4677 if (inbounds) {
4678 return CGF.EmitCheckedInBoundsGEP(Addr: addr, IdxList: indices, elementType, SignedIndices: signedIndices,
4679 IsSubtraction: CodeGenFunction::NotSubtraction, Loc: loc,
4680 Align: align, Name: name);
4681 } else {
4682 return CGF.Builder.CreateGEP(Addr: addr, IdxList: indices, ElementType: elementType, Align: align, Name: name);
4683 }
4684}
4685
4686static QualType getFixedSizeElementType(const ASTContext &ctx,
4687 const VariableArrayType *vla) {
4688 QualType eltType;
4689 do {
4690 eltType = vla->getElementType();
4691 } while ((vla = ctx.getAsVariableArrayType(T: eltType)));
4692 return eltType;
4693}
4694
4695static bool hasBPFPreserveStaticOffset(const RecordDecl *D) {
4696 return D && D->hasAttr<BPFPreserveStaticOffsetAttr>();
4697}
4698
4699static bool hasBPFPreserveStaticOffset(const Expr *E) {
4700 if (!E)
4701 return false;
4702 QualType PointeeType = E->getType()->getPointeeType();
4703 if (PointeeType.isNull())
4704 return false;
4705 if (const auto *BaseDecl = PointeeType->getAsRecordDecl())
4706 return hasBPFPreserveStaticOffset(D: BaseDecl);
4707 return false;
4708}
4709
4710// Wraps Addr with a call to llvm.preserve.static.offset intrinsic.
4711static Address wrapWithBPFPreserveStaticOffset(CodeGenFunction &CGF,
4712 Address &Addr) {
4713 if (!CGF.getTarget().getTriple().isBPF())
4714 return Addr;
4715
4716 llvm::Function *Fn =
4717 CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::preserve_static_offset);
4718 llvm::CallInst *Call = CGF.Builder.CreateCall(Callee: Fn, Args: {Addr.emitRawPointer(CGF)});
4719 return Address(Call, Addr.getElementType(), Addr.getAlignment());
4720}
4721
4722/// Given an array base, check whether its member access belongs to a record
4723/// with preserve_access_index attribute or not.
4724static bool IsPreserveAIArrayBase(CodeGenFunction &CGF, const Expr *ArrayBase) {
4725 if (!ArrayBase || !CGF.getDebugInfo())
4726 return false;
4727
4728 // Only support base as either a MemberExpr or DeclRefExpr.
4729 // DeclRefExpr to cover cases like:
4730 // struct s { int a; int b[10]; };
4731 // struct s *p;
4732 // p[1].a
4733 // p[1] will generate a DeclRefExpr and p[1].a is a MemberExpr.
4734 // p->b[5] is a MemberExpr example.
4735 const Expr *E = ArrayBase->IgnoreImpCasts();
4736 if (const auto *ME = dyn_cast<MemberExpr>(Val: E))
4737 return ME->getMemberDecl()->hasAttr<BPFPreserveAccessIndexAttr>();
4738
4739 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
4740 const auto *VarDef = dyn_cast<VarDecl>(Val: DRE->getDecl());
4741 if (!VarDef)
4742 return false;
4743
4744 const auto *PtrT = VarDef->getType()->getAs<PointerType>();
4745 if (!PtrT)
4746 return false;
4747
4748 const auto *PointeeT = PtrT->getPointeeType()
4749 ->getUnqualifiedDesugaredType();
4750 if (const auto *RecT = dyn_cast<RecordType>(Val: PointeeT))
4751 return RecT->getDecl()
4752 ->getMostRecentDecl()
4753 ->hasAttr<BPFPreserveAccessIndexAttr>();
4754 return false;
4755 }
4756
4757 return false;
4758}
4759
4760static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
4761 ArrayRef<llvm::Value *> indices,
4762 QualType eltType, bool inbounds,
4763 bool signedIndices, SourceLocation loc,
4764 QualType *arrayType = nullptr,
4765 const Expr *Base = nullptr,
4766 const llvm::Twine &name = "arrayidx") {
4767 // All the indices except that last must be zero.
4768#ifndef NDEBUG
4769 for (auto *idx : indices.drop_back())
4770 assert(isa<llvm::ConstantInt>(idx) &&
4771 cast<llvm::ConstantInt>(idx)->isZero());
4772#endif
4773
4774 // Determine the element size of the statically-sized base. This is
4775 // the thing that the indices are expressed in terms of.
4776 if (auto vla = CGF.getContext().getAsVariableArrayType(T: eltType)) {
4777 eltType = getFixedSizeElementType(ctx: CGF.getContext(), vla);
4778 }
4779
4780 // We can use that to compute the best alignment of the element.
4781 CharUnits eltSize = CGF.getContext().getTypeSizeInChars(T: eltType);
4782 CharUnits eltAlign =
4783 getArrayElementAlign(arrayAlign: addr.getAlignment(), idx: indices.back(), eltSize);
4784
4785 if (hasBPFPreserveStaticOffset(E: Base))
4786 addr = wrapWithBPFPreserveStaticOffset(CGF, Addr&: addr);
4787
4788 llvm::Value *eltPtr;
4789 auto LastIndex = dyn_cast<llvm::ConstantInt>(Val: indices.back());
4790 if (!LastIndex ||
4791 (!CGF.IsInPreservedAIRegion && !IsPreserveAIArrayBase(CGF, ArrayBase: Base))) {
4792 addr = emitArraySubscriptGEP(CGF, addr, indices,
4793 arrayType: arrayType ? CGF.ConvertTypeForMem(T: *arrayType)
4794 : nullptr,
4795 elementType: CGF.ConvertTypeForMem(T: eltType), inbounds,
4796 signedIndices, loc, align: eltAlign, name);
4797 return addr;
4798 } else {
4799 // Remember the original array subscript for bpf target
4800 unsigned idx = LastIndex->getZExtValue();
4801 llvm::DIType *DbgInfo = nullptr;
4802 if (arrayType)
4803 DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(Ty: *arrayType, Loc: loc);
4804 eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(
4805 ElTy: addr.getElementType(), Base: addr.emitRawPointer(CGF), Dimension: indices.size() - 1,
4806 LastIndex: idx, DbgInfo);
4807 }
4808
4809 return Address(eltPtr, CGF.ConvertTypeForMem(T: eltType), eltAlign);
4810}
4811
4812namespace {
4813
4814/// StructFieldAccess is a simple visitor class to grab the first l-value to
4815/// r-value cast Expr.
4816struct StructFieldAccess
4817 : public ConstStmtVisitor<StructFieldAccess, const Expr *> {
4818 const Expr *VisitCastExpr(const CastExpr *E) {
4819 if (E->getCastKind() == CK_LValueToRValue)
4820 return E;
4821 return Visit(S: E->getSubExpr());
4822 }
4823 const Expr *VisitParenExpr(const ParenExpr *E) {
4824 return Visit(S: E->getSubExpr());
4825 }
4826};
4827
4828} // end anonymous namespace
4829
4830/// The offset of a field from the beginning of the record.
4831static bool getFieldOffsetInBits(CodeGenFunction &CGF, const RecordDecl *RD,
4832 const FieldDecl *Field, int64_t &Offset) {
4833 ASTContext &Ctx = CGF.getContext();
4834 const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(D: RD);
4835 unsigned FieldNo = 0;
4836
4837 for (const FieldDecl *FD : RD->fields()) {
4838 if (FD == Field) {
4839 Offset += Layout.getFieldOffset(FieldNo);
4840 return true;
4841 }
4842
4843 QualType Ty = FD->getType();
4844 if (Ty->isRecordType())
4845 if (getFieldOffsetInBits(CGF, RD: Ty->getAsRecordDecl(), Field, Offset)) {
4846 Offset += Layout.getFieldOffset(FieldNo);
4847 return true;
4848 }
4849
4850 if (!RD->isUnion())
4851 ++FieldNo;
4852 }
4853
4854 return false;
4855}
4856
4857/// Returns the relative offset difference between \p FD1 and \p FD2.
4858/// \code
4859/// offsetof(struct foo, FD1) - offsetof(struct foo, FD2)
4860/// \endcode
4861/// Both fields must be within the same struct.
4862static std::optional<int64_t> getOffsetDifferenceInBits(CodeGenFunction &CGF,
4863 const FieldDecl *FD1,
4864 const FieldDecl *FD2) {
4865 const RecordDecl *FD1OuterRec =
4866 FD1->getParent()->getOuterLexicalRecordContext();
4867 const RecordDecl *FD2OuterRec =
4868 FD2->getParent()->getOuterLexicalRecordContext();
4869
4870 if (FD1OuterRec != FD2OuterRec)
4871 // Fields must be within the same RecordDecl.
4872 return std::optional<int64_t>();
4873
4874 int64_t FD1Offset = 0;
4875 if (!getFieldOffsetInBits(CGF, RD: FD1OuterRec, Field: FD1, Offset&: FD1Offset))
4876 return std::optional<int64_t>();
4877
4878 int64_t FD2Offset = 0;
4879 if (!getFieldOffsetInBits(CGF, RD: FD2OuterRec, Field: FD2, Offset&: FD2Offset))
4880 return std::optional<int64_t>();
4881
4882 return std::make_optional<int64_t>(t: FD1Offset - FD2Offset);
4883}
4884
4885/// EmitCountedByBoundsChecking - If the array being accessed has a "counted_by"
4886/// attribute, generate bounds checking code. The "count" field is at the top
4887/// level of the struct or in an anonymous struct, that's also at the top level.
4888/// Future expansions may allow the "count" to reside at any place in the
4889/// struct, but the value of "counted_by" will be a "simple" path to the count,
4890/// i.e. "a.b.count", so we shouldn't need the full force of EmitLValue or
4891/// similar to emit the correct GEP.
4892void CodeGenFunction::EmitCountedByBoundsChecking(
4893 const Expr *ArrayExpr, QualType ArrayType, Address ArrayInst,
4894 QualType IndexType, llvm::Value *IndexVal, bool Accessed,
4895 bool FlexibleArray) {
4896 const auto *ME = dyn_cast<MemberExpr>(Val: ArrayExpr->IgnoreImpCasts());
4897 if (!ME || !ME->getMemberDecl()->getType()->isCountAttributedType())
4898 return;
4899
4900 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
4901 getLangOpts().getStrictFlexArraysLevel();
4902 if (FlexibleArray &&
4903 !ME->isFlexibleArrayMemberLike(Context: getContext(), StrictFlexArraysLevel))
4904 return;
4905
4906 const FieldDecl *FD = cast<FieldDecl>(Val: ME->getMemberDecl());
4907 const FieldDecl *CountFD = FD->findCountedByField();
4908 if (!CountFD)
4909 return;
4910
4911 if (std::optional<int64_t> Diff =
4912 getOffsetDifferenceInBits(CGF&: *this, FD1: CountFD, FD2: FD)) {
4913 if (!ArrayInst.isValid()) {
4914 // An invalid Address indicates we're checking a pointer array access.
4915 // Emit the checked L-Value here.
4916 LValue LV = EmitCheckedLValue(E: ArrayExpr, TCK: TCK_MemberAccess);
4917 ArrayInst = LV.getAddress();
4918 }
4919
4920 // FIXME: The 'static_cast' is necessary, otherwise the result turns into a
4921 // uint64_t, which messes things up if we have a negative offset difference.
4922 Diff = *Diff / static_cast<int64_t>(CGM.getContext().getCharWidth());
4923
4924 // Create a GEP with the byte offset between the counted object and the
4925 // count and use that to load the count value.
4926 ArrayInst = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr: ArrayInst,
4927 Ty: Int8PtrTy, ElementTy: Int8Ty);
4928
4929 llvm::Type *BoundsType = ConvertType(T: CountFD->getType());
4930 llvm::Value *BoundsVal =
4931 Builder.CreateInBoundsGEP(Ty: Int8Ty, Ptr: ArrayInst.emitRawPointer(CGF&: *this),
4932 IdxList: Builder.getInt32(C: *Diff), Name: ".counted_by.gep");
4933 BoundsVal = Builder.CreateAlignedLoad(Ty: BoundsType, Addr: BoundsVal, Align: getIntAlign(),
4934 Name: ".counted_by.load");
4935
4936 // Now emit the bounds checking.
4937 EmitBoundsCheckImpl(ArrayExpr, ArrayBaseType: ArrayType, IndexVal, IndexType, BoundsVal,
4938 BoundsType: CountFD->getType(), Accessed);
4939 }
4940}
4941
4942LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
4943 bool Accessed) {
4944 // The index must always be an integer, which is not an aggregate. Emit it
4945 // in lexical order (this complexity is, sadly, required by C++17).
4946 llvm::Value *IdxPre =
4947 (E->getLHS() == E->getIdx()) ? EmitScalarExpr(E: E->getIdx()) : nullptr;
4948 bool SignedIndices = false;
4949 auto EmitIdxAfterBase = [&, IdxPre](bool Promote) -> llvm::Value * {
4950 auto *Idx = IdxPre;
4951 if (E->getLHS() != E->getIdx()) {
4952 assert(E->getRHS() == E->getIdx() && "index was neither LHS nor RHS");
4953 Idx = EmitScalarExpr(E: E->getIdx());
4954 }
4955
4956 QualType IdxTy = E->getIdx()->getType();
4957 bool IdxSigned = IdxTy->isSignedIntegerOrEnumerationType();
4958 SignedIndices |= IdxSigned;
4959
4960 if (SanOpts.has(K: SanitizerKind::ArrayBounds))
4961 EmitBoundsCheck(ArrayExpr: E, ArrayExprBase: E->getBase(), IndexVal: Idx, IndexType: IdxTy, Accessed);
4962
4963 // Extend or truncate the index type to 32 or 64-bits.
4964 if (Promote && Idx->getType() != IntPtrTy)
4965 Idx = Builder.CreateIntCast(V: Idx, DestTy: IntPtrTy, isSigned: IdxSigned, Name: "idxprom");
4966
4967 return Idx;
4968 };
4969 IdxPre = nullptr;
4970
4971 // If the base is a vector type, then we are forming a vector element lvalue
4972 // with this subscript.
4973 if (E->getBase()->getType()->isSubscriptableVectorType() &&
4974 !isa<ExtVectorElementExpr>(Val: E->getBase())) {
4975 // Emit the vector as an lvalue to get its address.
4976 LValue LHS = EmitLValue(E: E->getBase());
4977 auto *Idx = EmitIdxAfterBase(/*Promote*/false);
4978 assert(LHS.isSimple() && "Can only subscript lvalue vectors here!");
4979 return LValue::MakeVectorElt(vecAddress: LHS.getAddress(), Idx, type: E->getBase()->getType(),
4980 BaseInfo: LHS.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
4981 }
4982
4983 // The HLSL runtime handles subscript expressions on global resource arrays
4984 // and objects with HLSL buffer layouts.
4985 if (getLangOpts().HLSL) {
4986 std::optional<LValue> LV;
4987 if (E->getType()->isHLSLResourceRecord() ||
4988 E->getType()->isHLSLResourceRecordArray()) {
4989 LV = CGM.getHLSLRuntime().emitResourceArraySubscriptExpr(E, CGF&: *this);
4990 } else if (E->getType().getAddressSpace() == LangAS::hlsl_constant) {
4991 LV = CGM.getHLSLRuntime().emitBufferArraySubscriptExpr(E, CGF&: *this,
4992 EmitIdxAfterBase);
4993 }
4994 if (LV.has_value())
4995 return *LV;
4996 }
4997
4998 // All the other cases basically behave like simple offsetting.
4999
5000 // Handle the extvector case we ignored above.
5001 if (isa<ExtVectorElementExpr>(Val: E->getBase())) {
5002 LValue LV = EmitLValue(E: E->getBase());
5003 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5004 Address Addr = EmitExtVectorElementLValue(LV);
5005
5006 QualType EltType = LV.getType()->castAs<VectorType>()->getElementType();
5007 Addr = emitArraySubscriptGEP(CGF&: *this, addr: Addr, indices: Idx, eltType: EltType, /*inbounds*/ true,
5008 signedIndices: SignedIndices, loc: E->getExprLoc());
5009 return MakeAddrLValue(Addr, T: EltType, BaseInfo: LV.getBaseInfo(),
5010 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: EltType));
5011 }
5012
5013 LValueBaseInfo EltBaseInfo;
5014 TBAAAccessInfo EltTBAAInfo;
5015 Address Addr = Address::invalid();
5016 if (const VariableArrayType *vla =
5017 getContext().getAsVariableArrayType(T: E->getType())) {
5018 // The base must be a pointer, which is not an aggregate. Emit
5019 // it. It needs to be emitted first in case it's what captures
5020 // the VLA bounds.
5021 Addr = EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
5022 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5023
5024 // The element count here is the total number of non-VLA elements.
5025 llvm::Value *numElements = getVLASize(vla).NumElts;
5026
5027 // Effectively, the multiply by the VLA size is part of the GEP.
5028 // GEP indexes are signed, and scaling an index isn't permitted to
5029 // signed-overflow, so we use the same semantics for our explicit
5030 // multiply. We suppress this if overflow is not undefined behavior.
5031 if (getLangOpts().PointerOverflowDefined) {
5032 Idx = Builder.CreateMul(LHS: Idx, RHS: numElements);
5033 } else {
5034 Idx = Builder.CreateNSWMul(LHS: Idx, RHS: numElements);
5035 }
5036
5037 Addr = emitArraySubscriptGEP(CGF&: *this, addr: Addr, indices: Idx, eltType: vla->getElementType(),
5038 inbounds: !getLangOpts().PointerOverflowDefined,
5039 signedIndices: SignedIndices, loc: E->getExprLoc());
5040
5041 } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){
5042 // Indexing over an interface, as in "NSString *P; P[4];"
5043
5044 // Emit the base pointer.
5045 Addr = EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
5046 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5047
5048 CharUnits InterfaceSize = getContext().getTypeSizeInChars(T: OIT);
5049 llvm::Value *InterfaceSizeVal =
5050 llvm::ConstantInt::get(Ty: Idx->getType(), V: InterfaceSize.getQuantity());
5051
5052 llvm::Value *ScaledIdx = Builder.CreateMul(LHS: Idx, RHS: InterfaceSizeVal);
5053
5054 // We don't necessarily build correct LLVM struct types for ObjC
5055 // interfaces, so we can't rely on GEP to do this scaling
5056 // correctly, so we need to cast to i8*. FIXME: is this actually
5057 // true? A lot of other things in the fragile ABI would break...
5058 llvm::Type *OrigBaseElemTy = Addr.getElementType();
5059
5060 // Do the GEP.
5061 CharUnits EltAlign =
5062 getArrayElementAlign(arrayAlign: Addr.getAlignment(), idx: Idx, eltSize: InterfaceSize);
5063 llvm::Value *EltPtr =
5064 emitArraySubscriptGEP(CGF&: *this, elemType: Int8Ty, ptr: Addr.emitRawPointer(CGF&: *this),
5065 indices: ScaledIdx, inbounds: false, signedIndices: SignedIndices, loc: E->getExprLoc());
5066 Addr = Address(EltPtr, OrigBaseElemTy, EltAlign);
5067 } else if (const Expr *Array = isSimpleArrayDecayOperand(E: E->getBase())) {
5068 // If this is A[i] where A is an array, the frontend will have decayed the
5069 // base to be a ArrayToPointerDecay implicit cast. While correct, it is
5070 // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
5071 // "gep x, i" here. Emit one "gep A, 0, i".
5072 assert(Array->getType()->isArrayType() &&
5073 "Array to pointer decay must have array source type!");
5074 LValue ArrayLV;
5075 // For simple multidimensional array indexing, set the 'accessed' flag for
5076 // better bounds-checking of the base expression.
5077 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Array))
5078 ArrayLV = EmitArraySubscriptExpr(E: ASE, /*Accessed*/ true);
5079 else
5080 ArrayLV = EmitLValue(E: Array);
5081 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5082
5083 if (SanOpts.has(K: SanitizerKind::ArrayBounds))
5084 EmitCountedByBoundsChecking(ArrayExpr: Array, ArrayType: Array->getType(), ArrayInst: ArrayLV.getAddress(),
5085 IndexType: E->getIdx()->getType(), IndexVal: Idx, Accessed,
5086 /*FlexibleArray=*/true);
5087
5088 // Propagate the alignment from the array itself to the result.
5089 QualType arrayType = Array->getType();
5090 Addr = emitArraySubscriptGEP(
5091 CGF&: *this, addr: ArrayLV.getAddress(), indices: {CGM.getSize(numChars: CharUnits::Zero()), Idx},
5092 eltType: E->getType(), inbounds: !getLangOpts().PointerOverflowDefined, signedIndices: SignedIndices,
5093 loc: E->getExprLoc(), arrayType: &arrayType, Base: E->getBase());
5094 EltBaseInfo = ArrayLV.getBaseInfo();
5095 if (!CGM.getCodeGenOpts().NewStructPathTBAA) {
5096 // Since CodeGenTBAA::getTypeInfoHelper only handles array types for
5097 // new struct path TBAA, we must a use a plain access.
5098 EltTBAAInfo = CGM.getTBAAInfoForSubobject(Base: ArrayLV, AccessType: E->getType());
5099 } else if (ArrayLV.getTBAAInfo().isMayAlias()) {
5100 EltTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
5101 } else if (ArrayLV.getTBAAInfo().isIncomplete()) {
5102 // The array element is complete, even if the array is not.
5103 EltTBAAInfo = CGM.getTBAAAccessInfo(AccessType: E->getType());
5104 } else {
5105 // The TBAA access info from the array (base) lvalue is ordinary. We will
5106 // adapt it to create access info for the element.
5107 EltTBAAInfo = ArrayLV.getTBAAInfo();
5108
5109 // We retain the TBAA struct path (BaseType and Offset members) from the
5110 // array. In the TBAA representation, we map any array access to the
5111 // element at index 0, as the index is generally a runtime value. This
5112 // element has the same offset in the base type as the array itself.
5113 // If the array lvalue had no base type, there is no point trying to
5114 // generate one, since an array itself is not a valid base type.
5115
5116 // We also retain the access type from the base lvalue, but the access
5117 // size must be updated to the size of an individual element.
5118 EltTBAAInfo.Size =
5119 getContext().getTypeSizeInChars(T: E->getType()).getQuantity();
5120 }
5121 } else {
5122 // The base must be a pointer; emit it with an estimate of its alignment.
5123 Address BaseAddr =
5124 EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
5125 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5126 QualType ptrType = E->getBase()->getType();
5127 Addr = emitArraySubscriptGEP(CGF&: *this, addr: BaseAddr, indices: Idx, eltType: E->getType(),
5128 inbounds: !getLangOpts().PointerOverflowDefined,
5129 signedIndices: SignedIndices, loc: E->getExprLoc(), arrayType: &ptrType,
5130 Base: E->getBase());
5131
5132 if (SanOpts.has(K: SanitizerKind::ArrayBounds)) {
5133 StructFieldAccess Visitor;
5134 const Expr *Base = Visitor.Visit(S: E->getBase());
5135
5136 if (const auto *CE = dyn_cast_if_present<CastExpr>(Val: Base);
5137 CE && CE->getCastKind() == CK_LValueToRValue)
5138 EmitCountedByBoundsChecking(ArrayExpr: CE, ArrayType: ptrType, ArrayInst: Address::invalid(),
5139 IndexType: E->getIdx()->getType(), IndexVal: Idx, Accessed,
5140 /*FlexibleArray=*/false);
5141 }
5142 }
5143
5144 LValue LV = MakeAddrLValue(Addr, T: E->getType(), BaseInfo: EltBaseInfo, TBAAInfo: EltTBAAInfo);
5145
5146 if (getLangOpts().ObjC &&
5147 getLangOpts().getGC() != LangOptions::NonGC) {
5148 LV.setNonGC(!E->isOBJCGCCandidate(Ctx&: getContext()));
5149 setObjCGCLValueClass(Ctx: getContext(), E, LV);
5150 }
5151 return LV;
5152}
5153
5154llvm::Value *CodeGenFunction::EmitMatrixIndexExpr(const Expr *E) {
5155 llvm::Value *Idx = EmitScalarExpr(E);
5156 if (Idx->getType() == IntPtrTy)
5157 return Idx;
5158 bool IsSigned = E->getType()->isSignedIntegerOrEnumerationType();
5159 return Builder.CreateIntCast(V: Idx, DestTy: IntPtrTy, isSigned: IsSigned);
5160}
5161
5162LValue CodeGenFunction::EmitMatrixSingleSubscriptExpr(
5163 const MatrixSingleSubscriptExpr *E) {
5164 LValue Base = EmitLValue(E: E->getBase());
5165 llvm::Value *RowIdx = EmitMatrixIndexExpr(E: E->getRowIdx());
5166
5167 RawAddress MatAddr = Base.getAddress();
5168 if (getLangOpts().HLSL &&
5169 E->getBase()->getType().getAddressSpace() == LangAS::hlsl_constant)
5170 MatAddr = CGM.getHLSLRuntime().createBufferMatrixTempAddress(
5171 LV: Base, Loc: E->getExprLoc(), CGF&: *this);
5172
5173 return LValue::MakeMatrixRow(Addr: MaybeConvertMatrixAddress(Addr: MatAddr, CGF&: *this),
5174 RowIdx, MatrixTy: E->getBase()->getType(),
5175 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5176}
5177
5178LValue CodeGenFunction::EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E) {
5179 assert(
5180 !E->isIncomplete() &&
5181 "incomplete matrix subscript expressions should be rejected during Sema");
5182 LValue Base = EmitLValue(E: E->getBase());
5183
5184 // Extend or truncate the index type to 32 or 64-bits if needed.
5185 llvm::Value *RowIdx = EmitMatrixIndexExpr(E: E->getRowIdx());
5186 llvm::Value *ColIdx = EmitMatrixIndexExpr(E: E->getColumnIdx());
5187 llvm::MatrixBuilder MB(Builder);
5188 const auto *MatrixTy = E->getBase()->getType()->castAs<ConstantMatrixType>();
5189 unsigned NumCols = MatrixTy->getNumColumns();
5190 unsigned NumRows = MatrixTy->getNumRows();
5191 bool IsMatrixRowMajor = getLangOpts().getDefaultMatrixMemoryLayout() ==
5192 LangOptions::MatrixMemoryLayout::MatrixRowMajor;
5193 llvm::Value *FinalIdx =
5194 MB.CreateIndex(RowIdx, ColumnIdx: ColIdx, NumRows, NumCols, IsMatrixRowMajor);
5195
5196 return LValue::MakeMatrixElt(
5197 matAddress: MaybeConvertMatrixAddress(Addr: Base.getAddress(), CGF&: *this), Idx: FinalIdx,
5198 type: E->getBase()->getType(), BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5199}
5200
5201static Address emitOMPArraySectionBase(CodeGenFunction &CGF, const Expr *Base,
5202 LValueBaseInfo &BaseInfo,
5203 TBAAAccessInfo &TBAAInfo,
5204 QualType BaseTy, QualType ElTy,
5205 bool IsLowerBound) {
5206 LValue BaseLVal;
5207 if (auto *ASE = dyn_cast<ArraySectionExpr>(Val: Base->IgnoreParenImpCasts())) {
5208 BaseLVal = CGF.EmitArraySectionExpr(E: ASE, IsLowerBound);
5209 if (BaseTy->isArrayType()) {
5210 Address Addr = BaseLVal.getAddress();
5211 BaseInfo = BaseLVal.getBaseInfo();
5212
5213 // If the array type was an incomplete type, we need to make sure
5214 // the decay ends up being the right type.
5215 llvm::Type *NewTy = CGF.ConvertType(T: BaseTy);
5216 Addr = Addr.withElementType(ElemTy: NewTy);
5217
5218 // Note that VLA pointers are always decayed, so we don't need to do
5219 // anything here.
5220 if (!BaseTy->isVariableArrayType()) {
5221 assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
5222 "Expected pointer to array");
5223 Addr = CGF.Builder.CreateConstArrayGEP(Addr, Index: 0, Name: "arraydecay");
5224 }
5225
5226 return Addr.withElementType(ElemTy: CGF.ConvertTypeForMem(T: ElTy));
5227 }
5228 LValueBaseInfo TypeBaseInfo;
5229 TBAAAccessInfo TypeTBAAInfo;
5230 CharUnits Align =
5231 CGF.CGM.getNaturalTypeAlignment(T: ElTy, BaseInfo: &TypeBaseInfo, TBAAInfo: &TypeTBAAInfo);
5232 BaseInfo.mergeForCast(Info: TypeBaseInfo);
5233 TBAAInfo = CGF.CGM.mergeTBAAInfoForCast(SourceInfo: TBAAInfo, TargetInfo: TypeTBAAInfo);
5234 return Address(CGF.Builder.CreateLoad(Addr: BaseLVal.getAddress()),
5235 CGF.ConvertTypeForMem(T: ElTy), Align);
5236 }
5237 return CGF.EmitPointerWithAlignment(E: Base, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
5238}
5239
5240LValue CodeGenFunction::EmitArraySectionExpr(const ArraySectionExpr *E,
5241 bool IsLowerBound) {
5242
5243 assert(!E->isOpenACCArraySection() &&
5244 "OpenACC Array section codegen not implemented");
5245
5246 QualType BaseTy = ArraySectionExpr::getBaseOriginalType(Base: E->getBase());
5247 QualType ResultExprTy;
5248 if (auto *AT = getContext().getAsArrayType(T: BaseTy))
5249 ResultExprTy = AT->getElementType();
5250 else
5251 ResultExprTy = BaseTy->getPointeeType();
5252 llvm::Value *Idx = nullptr;
5253 if (IsLowerBound || E->getColonLocFirst().isInvalid()) {
5254 // Requesting lower bound or upper bound, but without provided length and
5255 // without ':' symbol for the default length -> length = 1.
5256 // Idx = LowerBound ?: 0;
5257 if (auto *LowerBound = E->getLowerBound()) {
5258 Idx = Builder.CreateIntCast(
5259 V: EmitScalarExpr(E: LowerBound), DestTy: IntPtrTy,
5260 isSigned: LowerBound->getType()->hasSignedIntegerRepresentation());
5261 } else
5262 Idx = llvm::ConstantInt::getNullValue(Ty: IntPtrTy);
5263 } else {
5264 // Try to emit length or lower bound as constant. If this is possible, 1
5265 // is subtracted from constant length or lower bound. Otherwise, emit LLVM
5266 // IR (LB + Len) - 1.
5267 auto &C = CGM.getContext();
5268 auto *Length = E->getLength();
5269 llvm::APSInt ConstLength;
5270 if (Length) {
5271 // Idx = LowerBound + Length - 1;
5272 if (std::optional<llvm::APSInt> CL = Length->getIntegerConstantExpr(Ctx: C)) {
5273 ConstLength = CL->zextOrTrunc(width: PointerWidthInBits);
5274 Length = nullptr;
5275 }
5276 auto *LowerBound = E->getLowerBound();
5277 llvm::APSInt ConstLowerBound(PointerWidthInBits, /*isUnsigned=*/false);
5278 if (LowerBound) {
5279 if (std::optional<llvm::APSInt> LB =
5280 LowerBound->getIntegerConstantExpr(Ctx: C)) {
5281 ConstLowerBound = LB->zextOrTrunc(width: PointerWidthInBits);
5282 LowerBound = nullptr;
5283 }
5284 }
5285 if (!Length)
5286 --ConstLength;
5287 else if (!LowerBound)
5288 --ConstLowerBound;
5289
5290 if (Length || LowerBound) {
5291 auto *LowerBoundVal =
5292 LowerBound
5293 ? Builder.CreateIntCast(
5294 V: EmitScalarExpr(E: LowerBound), DestTy: IntPtrTy,
5295 isSigned: LowerBound->getType()->hasSignedIntegerRepresentation())
5296 : llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLowerBound);
5297 auto *LengthVal =
5298 Length
5299 ? Builder.CreateIntCast(
5300 V: EmitScalarExpr(E: Length), DestTy: IntPtrTy,
5301 isSigned: Length->getType()->hasSignedIntegerRepresentation())
5302 : llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLength);
5303 Idx = Builder.CreateAdd(LHS: LowerBoundVal, RHS: LengthVal, Name: "lb_add_len",
5304 /*HasNUW=*/false,
5305 HasNSW: !getLangOpts().PointerOverflowDefined);
5306 if (Length && LowerBound) {
5307 Idx = Builder.CreateSub(
5308 LHS: Idx, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, /*V=*/1), Name: "idx_sub_1",
5309 /*HasNUW=*/false, HasNSW: !getLangOpts().PointerOverflowDefined);
5310 }
5311 } else
5312 Idx = llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLength + ConstLowerBound);
5313 } else {
5314 // Idx = ArraySize - 1;
5315 QualType ArrayTy = BaseTy->isPointerType()
5316 ? E->getBase()->IgnoreParenImpCasts()->getType()
5317 : BaseTy;
5318 if (auto *VAT = C.getAsVariableArrayType(T: ArrayTy)) {
5319 Length = VAT->getSizeExpr();
5320 if (std::optional<llvm::APSInt> L = Length->getIntegerConstantExpr(Ctx: C)) {
5321 ConstLength = *L;
5322 Length = nullptr;
5323 }
5324 } else {
5325 auto *CAT = C.getAsConstantArrayType(T: ArrayTy);
5326 assert(CAT && "unexpected type for array initializer");
5327 ConstLength = CAT->getSize();
5328 }
5329 if (Length) {
5330 auto *LengthVal = Builder.CreateIntCast(
5331 V: EmitScalarExpr(E: Length), DestTy: IntPtrTy,
5332 isSigned: Length->getType()->hasSignedIntegerRepresentation());
5333 Idx = Builder.CreateSub(
5334 LHS: LengthVal, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, /*V=*/1), Name: "len_sub_1",
5335 /*HasNUW=*/false, HasNSW: !getLangOpts().PointerOverflowDefined);
5336 } else {
5337 ConstLength = ConstLength.zextOrTrunc(width: PointerWidthInBits);
5338 --ConstLength;
5339 Idx = llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLength);
5340 }
5341 }
5342 }
5343 assert(Idx);
5344
5345 Address EltPtr = Address::invalid();
5346 LValueBaseInfo BaseInfo;
5347 TBAAAccessInfo TBAAInfo;
5348 if (auto *VLA = getContext().getAsVariableArrayType(T: ResultExprTy)) {
5349 // The base must be a pointer, which is not an aggregate. Emit
5350 // it. It needs to be emitted first in case it's what captures
5351 // the VLA bounds.
5352 Address Base =
5353 emitOMPArraySectionBase(CGF&: *this, Base: E->getBase(), BaseInfo, TBAAInfo,
5354 BaseTy, ElTy: VLA->getElementType(), IsLowerBound);
5355 // The element count here is the total number of non-VLA elements.
5356 llvm::Value *NumElements = getVLASize(vla: VLA).NumElts;
5357
5358 // Effectively, the multiply by the VLA size is part of the GEP.
5359 // GEP indexes are signed, and scaling an index isn't permitted to
5360 // signed-overflow, so we use the same semantics for our explicit
5361 // multiply. We suppress this if overflow is not undefined behavior.
5362 if (getLangOpts().PointerOverflowDefined)
5363 Idx = Builder.CreateMul(LHS: Idx, RHS: NumElements);
5364 else
5365 Idx = Builder.CreateNSWMul(LHS: Idx, RHS: NumElements);
5366 EltPtr = emitArraySubscriptGEP(CGF&: *this, addr: Base, indices: Idx, eltType: VLA->getElementType(),
5367 inbounds: !getLangOpts().PointerOverflowDefined,
5368 /*signedIndices=*/false, loc: E->getExprLoc());
5369 } else if (const Expr *Array = isSimpleArrayDecayOperand(E: E->getBase())) {
5370 // If this is A[i] where A is an array, the frontend will have decayed the
5371 // base to be a ArrayToPointerDecay implicit cast. While correct, it is
5372 // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
5373 // "gep x, i" here. Emit one "gep A, 0, i".
5374 assert(Array->getType()->isArrayType() &&
5375 "Array to pointer decay must have array source type!");
5376 LValue ArrayLV;
5377 // For simple multidimensional array indexing, set the 'accessed' flag for
5378 // better bounds-checking of the base expression.
5379 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Array))
5380 ArrayLV = EmitArraySubscriptExpr(E: ASE, /*Accessed*/ true);
5381 else
5382 ArrayLV = EmitLValue(E: Array);
5383
5384 // Propagate the alignment from the array itself to the result.
5385 EltPtr = emitArraySubscriptGEP(
5386 CGF&: *this, addr: ArrayLV.getAddress(), indices: {CGM.getSize(numChars: CharUnits::Zero()), Idx},
5387 eltType: ResultExprTy, inbounds: !getLangOpts().PointerOverflowDefined,
5388 /*signedIndices=*/false, loc: E->getExprLoc());
5389 BaseInfo = ArrayLV.getBaseInfo();
5390 TBAAInfo = CGM.getTBAAInfoForSubobject(Base: ArrayLV, AccessType: ResultExprTy);
5391 } else {
5392 Address Base =
5393 emitOMPArraySectionBase(CGF&: *this, Base: E->getBase(), BaseInfo, TBAAInfo, BaseTy,
5394 ElTy: ResultExprTy, IsLowerBound);
5395 EltPtr = emitArraySubscriptGEP(CGF&: *this, addr: Base, indices: Idx, eltType: ResultExprTy,
5396 inbounds: !getLangOpts().PointerOverflowDefined,
5397 /*signedIndices=*/false, loc: E->getExprLoc());
5398 }
5399
5400 return MakeAddrLValue(Addr: EltPtr, T: ResultExprTy, BaseInfo, TBAAInfo);
5401}
5402
5403LValue CodeGenFunction::
5404EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
5405 // Emit the base vector as an l-value.
5406 LValue Base;
5407
5408 // ExtVectorElementExpr's base can either be a vector or pointer to vector.
5409 if (E->isArrow()) {
5410 // If it is a pointer to a vector, emit the address and form an lvalue with
5411 // it.
5412 LValueBaseInfo BaseInfo;
5413 TBAAAccessInfo TBAAInfo;
5414 Address Ptr = EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
5415 const auto *PT = E->getBase()->getType()->castAs<PointerType>();
5416 Base = MakeAddrLValue(Addr: Ptr, T: PT->getPointeeType(), BaseInfo, TBAAInfo);
5417 Base.getQuals().removeObjCGCAttr();
5418 } else if (E->getBase()->isGLValue()) {
5419 // Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
5420 // emit the base as an lvalue.
5421 assert(E->getBase()->getType()->isVectorType());
5422 Base = EmitLValue(E: E->getBase());
5423 } else {
5424 // Otherwise, the base is a normal rvalue (as in (V+V).x), emit it as such.
5425 assert(E->getBase()->getType()->isVectorType() &&
5426 "Result must be a vector");
5427 llvm::Value *Vec = EmitScalarExpr(E: E->getBase());
5428
5429 // Store the vector to memory (because LValue wants an address).
5430 Address VecMem = CreateMemTemp(Ty: E->getBase()->getType());
5431 // need to zero extend an hlsl boolean vector to store it back to memory
5432 QualType Ty = E->getBase()->getType();
5433 llvm::Type *LTy = convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Vec->getType());
5434 if (LTy->getScalarSizeInBits() > Vec->getType()->getScalarSizeInBits())
5435 Vec = Builder.CreateZExt(V: Vec, DestTy: LTy);
5436 Builder.CreateStore(Val: Vec, Addr: VecMem);
5437 Base = MakeAddrLValue(Addr: VecMem, T: Ty, Source: AlignmentSource::Decl);
5438 }
5439
5440 QualType type =
5441 E->getType().withCVRQualifiers(CVR: Base.getQuals().getCVRQualifiers());
5442
5443 // Encode the element access list into a vector of unsigned indices.
5444 SmallVector<uint32_t, 4> Indices;
5445 E->getEncodedElementAccess(Elts&: Indices);
5446
5447 if (Base.isSimple()) {
5448 llvm::Constant *CV =
5449 llvm::ConstantDataVector::get(Context&: getLLVMContext(), Elts: Indices);
5450 return LValue::MakeExtVectorElt(Addr: Base.getAddress(), Elts: CV, type,
5451 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5452 }
5453
5454 if (Base.isMatrixRow()) {
5455 if (auto *RowIdx =
5456 llvm::dyn_cast<llvm::ConstantInt>(Val: Base.getMatrixRowIdx())) {
5457 llvm::SmallVector<llvm::Constant *> MatIndices;
5458 QualType MatTy = Base.getType();
5459 const ConstantMatrixType *MT = MatTy->castAs<ConstantMatrixType>();
5460 unsigned NumCols = Indices.size();
5461 unsigned NumRows = MT->getNumRows();
5462 unsigned Row = RowIdx->getZExtValue();
5463 QualType VecQT = E->getBase()->getType();
5464 if (NumCols != MT->getNumColumns()) {
5465 const auto *EVT = VecQT->getAs<ExtVectorType>();
5466 QualType ElemQT = EVT->getElementType();
5467 VecQT = getContext().getExtVectorType(VectorType: ElemQT, NumElts: NumCols);
5468 }
5469 for (unsigned C = 0; C < NumCols; ++C) {
5470 unsigned Col = Indices[C];
5471 unsigned Linear = Col * NumRows + Row;
5472 MatIndices.push_back(Elt: llvm::ConstantInt::get(Ty: Int32Ty, V: Linear));
5473 }
5474
5475 llvm::Constant *ConstIdxs = llvm::ConstantVector::get(V: MatIndices);
5476 return LValue::MakeExtVectorElt(Addr: Base.getMatrixAddress(), Elts: ConstIdxs, type: VecQT,
5477 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5478 }
5479 llvm::Constant *Cols =
5480 llvm::ConstantDataVector::get(Context&: getLLVMContext(), Elts: Indices);
5481 // Note: intentionally not using E.getType() so we can reuse isMatrixRow()
5482 // implementations in EmitLoadOfLValue & EmitStoreThroughLValue and don't
5483 // need the LValue to have its own number of rows and columns when the
5484 // type is a vector.
5485 return LValue::MakeMatrixRowSwizzle(
5486 MatAddr: Base.getMatrixAddress(), RowIdx: Base.getMatrixRowIdx(), Cols, MatrixTy: Base.getType(),
5487 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5488 }
5489
5490 assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
5491
5492 llvm::Constant *BaseElts = Base.getExtVectorElts();
5493 SmallVector<llvm::Constant *, 4> CElts;
5494
5495 for (unsigned Index : Indices)
5496 CElts.push_back(Elt: BaseElts->getAggregateElement(Elt: Index));
5497 llvm::Constant *CV = llvm::ConstantVector::get(V: CElts);
5498 return LValue::MakeExtVectorElt(Addr: Base.getExtVectorAddress(), Elts: CV, type,
5499 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5500}
5501
5502bool CodeGenFunction::isUnderlyingBasePointerConstantNull(const Expr *E) {
5503 const Expr *UnderlyingBaseExpr = E->IgnoreParens();
5504 while (auto *BaseMemberExpr = dyn_cast<MemberExpr>(Val: UnderlyingBaseExpr))
5505 UnderlyingBaseExpr = BaseMemberExpr->getBase()->IgnoreParens();
5506 return getContext().isSentinelNullExpr(E: UnderlyingBaseExpr);
5507}
5508
5509LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
5510 if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(CGF&: *this, ME: E)) {
5511 EmitIgnoredExpr(E: E->getBase());
5512 return EmitDeclRefLValue(E: DRE);
5513 }
5514 if (getLangOpts().HLSL &&
5515 E->getType().getAddressSpace() == LangAS::hlsl_constant) {
5516 // We have an HLSL buffer - emit using HLSL's layout rules.
5517 return CGM.getHLSLRuntime().emitBufferMemberExpr(CGF&: *this, E);
5518 }
5519
5520 Expr *BaseExpr = E->getBase();
5521 // Check whether the underlying base pointer is a constant null.
5522 // If so, we do not set inbounds flag for GEP to avoid breaking some
5523 // old-style offsetof idioms.
5524 bool IsInBounds = !getLangOpts().PointerOverflowDefined &&
5525 !isUnderlyingBasePointerConstantNull(E: BaseExpr);
5526 // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar.
5527 LValue BaseLV;
5528 if (E->isArrow()) {
5529 LValueBaseInfo BaseInfo;
5530 TBAAAccessInfo TBAAInfo;
5531 Address Addr = EmitPointerWithAlignment(E: BaseExpr, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
5532 QualType PtrTy = BaseExpr->getType()->getPointeeType();
5533 SanitizerSet SkippedChecks;
5534 bool IsBaseCXXThis = IsWrappedCXXThis(Obj: BaseExpr);
5535 if (IsBaseCXXThis)
5536 SkippedChecks.set(K: SanitizerKind::Alignment, Value: true);
5537 if (IsBaseCXXThis || isa<DeclRefExpr>(Val: BaseExpr))
5538 SkippedChecks.set(K: SanitizerKind::Null, Value: true);
5539 EmitTypeCheck(TCK: TCK_MemberAccess, Loc: E->getExprLoc(), Addr, Type: PtrTy,
5540 /*Alignment=*/CharUnits::Zero(), SkippedChecks);
5541 BaseLV = MakeAddrLValue(Addr, T: PtrTy, BaseInfo, TBAAInfo);
5542 } else
5543 BaseLV = EmitCheckedLValue(E: BaseExpr, TCK: TCK_MemberAccess);
5544
5545 NamedDecl *ND = E->getMemberDecl();
5546 if (auto *Field = dyn_cast<FieldDecl>(Val: ND)) {
5547 LValue LV = EmitLValueForField(Base: BaseLV, Field, IsInBounds);
5548 setObjCGCLValueClass(Ctx: getContext(), E, LV);
5549 if (getLangOpts().OpenMP) {
5550 // If the member was explicitly marked as nontemporal, mark it as
5551 // nontemporal. If the base lvalue is marked as nontemporal, mark access
5552 // to children as nontemporal too.
5553 if ((IsWrappedCXXThis(Obj: BaseExpr) &&
5554 CGM.getOpenMPRuntime().isNontemporalDecl(VD: Field)) ||
5555 BaseLV.isNontemporal())
5556 LV.setNontemporal(/*Value=*/true);
5557 }
5558 return LV;
5559 }
5560
5561 if (const auto *FD = dyn_cast<FunctionDecl>(Val: ND))
5562 return EmitFunctionDeclLValue(CGF&: *this, E, GD: FD);
5563
5564 llvm_unreachable("Unhandled member declaration!");
5565}
5566
5567/// Given that we are currently emitting a lambda, emit an l-value for
5568/// one of its members.
5569///
5570LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field,
5571 llvm::Value *ThisValue) {
5572 bool HasExplicitObjectParameter = false;
5573 const auto *MD = dyn_cast_if_present<CXXMethodDecl>(Val: CurCodeDecl);
5574 if (MD) {
5575 HasExplicitObjectParameter = MD->isExplicitObjectMemberFunction();
5576 assert(MD->getParent()->isLambda());
5577 assert(MD->getParent() == Field->getParent());
5578 }
5579 LValue LambdaLV;
5580 if (HasExplicitObjectParameter) {
5581 const VarDecl *D = cast<CXXMethodDecl>(Val: CurCodeDecl)->getParamDecl(i: 0);
5582 auto It = LocalDeclMap.find(Val: D);
5583 assert(It != LocalDeclMap.end() && "explicit parameter not loaded?");
5584 Address AddrOfExplicitObject = It->getSecond();
5585 if (D->getType()->isReferenceType())
5586 LambdaLV = EmitLoadOfReferenceLValue(RefAddr: AddrOfExplicitObject, RefTy: D->getType(),
5587 Source: AlignmentSource::Decl);
5588 else
5589 LambdaLV = MakeAddrLValue(Addr: AddrOfExplicitObject,
5590 T: D->getType().getNonReferenceType());
5591
5592 // Make sure we have an lvalue to the lambda itself and not a derived class.
5593 auto *ThisTy = D->getType().getNonReferenceType()->getAsCXXRecordDecl();
5594 auto *LambdaTy = cast<CXXRecordDecl>(Val: Field->getParent());
5595 if (ThisTy != LambdaTy) {
5596 const CXXCastPath &BasePathArray = getContext().LambdaCastPaths.at(Val: MD);
5597 Address Base = GetAddressOfBaseClass(
5598 Value: LambdaLV.getAddress(), Derived: ThisTy, PathBegin: BasePathArray.begin(),
5599 PathEnd: BasePathArray.end(), /*NullCheckValue=*/false, Loc: SourceLocation());
5600 CanQualType T = getContext().getCanonicalTagType(TD: LambdaTy);
5601 LambdaLV = MakeAddrLValue(Addr: Base, T);
5602 }
5603 } else {
5604 CanQualType LambdaTagType =
5605 getContext().getCanonicalTagType(TD: Field->getParent());
5606 LambdaLV = MakeNaturalAlignAddrLValue(V: ThisValue, T: LambdaTagType);
5607 }
5608 return EmitLValueForField(Base: LambdaLV, Field);
5609}
5610
5611LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field) {
5612 return EmitLValueForLambdaField(Field, ThisValue: CXXABIThisValue);
5613}
5614
5615/// Get the field index in the debug info. The debug info structure/union
5616/// will ignore the unnamed bitfields.
5617unsigned CodeGenFunction::getDebugInfoFIndex(const RecordDecl *Rec,
5618 unsigned FieldIndex) {
5619 unsigned I = 0, Skipped = 0;
5620
5621 for (auto *F : Rec->getDefinition()->fields()) {
5622 if (I == FieldIndex)
5623 break;
5624 if (F->isUnnamedBitField())
5625 Skipped++;
5626 I++;
5627 }
5628
5629 return FieldIndex - Skipped;
5630}
5631
5632/// Get the address of a zero-sized field within a record. The resulting
5633/// address doesn't necessarily have the right type.
5634static Address emitAddrOfZeroSizeField(CodeGenFunction &CGF, Address Base,
5635 const FieldDecl *Field,
5636 bool IsInBounds) {
5637 CharUnits Offset = CGF.getContext().toCharUnitsFromBits(
5638 BitSize: CGF.getContext().getFieldOffset(FD: Field));
5639 if (Offset.isZero())
5640 return Base;
5641 Base = Base.withElementType(ElemTy: CGF.Int8Ty);
5642 if (!IsInBounds)
5643 return CGF.Builder.CreateConstByteGEP(Addr: Base, Offset);
5644 return CGF.Builder.CreateConstInBoundsByteGEP(Addr: Base, Offset);
5645}
5646
5647/// Drill down to the storage of a field without walking into reference types,
5648/// and without respect for pointer field protection.
5649///
5650/// The resulting address doesn't necessarily have the right type.
5651static Address emitRawAddrOfFieldStorage(CodeGenFunction &CGF, Address base,
5652 const FieldDecl *field,
5653 bool IsInBounds) {
5654 if (isEmptyFieldForLayout(Context: CGF.getContext(), FD: field))
5655 return emitAddrOfZeroSizeField(CGF, Base: base, Field: field, IsInBounds);
5656
5657 const RecordDecl *rec = field->getParent();
5658
5659 unsigned idx =
5660 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(FD: field);
5661 llvm::Type *StructType =
5662 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMType();
5663
5664 if (CGF.getLangOpts().EmitStructuredGEP)
5665 return RawAddress(
5666 CGF.Builder.CreateStructuredGEP(BaseType: StructType, PtrBase: base.emitRawPointer(CGF),
5667 Indices: {CGF.Builder.getSize(N: idx)}),
5668 base.getElementType(), base.getAlignment());
5669
5670 if (!IsInBounds)
5671 return CGF.Builder.CreateConstGEP2_32(Addr: base, Idx0: 0, Idx1: idx, Name: field->getName());
5672
5673 return CGF.Builder.CreateStructGEP(Addr: base, Index: idx, Name: field->getName());
5674}
5675
5676/// Drill down to the storage of a field without walking into reference types,
5677/// wrapping the address in an llvm.protected.field.ptr intrinsic for the
5678/// pointer field protection feature if necessary.
5679///
5680/// The resulting address doesn't necessarily have the right type.
5681static Address emitAddrOfFieldStorage(CodeGenFunction &CGF, Address base,
5682 const FieldDecl *field, bool IsInBounds) {
5683 Address Addr = emitRawAddrOfFieldStorage(CGF, base, field, IsInBounds);
5684
5685 if (!CGF.getContext().isPFPField(Field: field))
5686 return Addr;
5687
5688 return CGF.EmitAddressOfPFPField(RecordPtr: base, FieldPtr: Addr, Field: field);
5689}
5690
5691static Address emitPreserveStructAccess(CodeGenFunction &CGF, LValue base,
5692 Address addr, const FieldDecl *field) {
5693 const RecordDecl *rec = field->getParent();
5694 llvm::DIType *DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(
5695 Ty: base.getType(), Loc: rec->getLocation());
5696
5697 unsigned idx =
5698 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(FD: field);
5699
5700 return CGF.Builder.CreatePreserveStructAccessIndex(
5701 Addr: addr, Index: idx, FieldIndex: CGF.getDebugInfoFIndex(Rec: rec, FieldIndex: field->getFieldIndex()), DbgInfo);
5702}
5703
5704static bool hasAnyVptr(const QualType Type, const ASTContext &Context) {
5705 const auto *RD = Type.getTypePtr()->getAsCXXRecordDecl();
5706 if (!RD)
5707 return false;
5708
5709 if (RD->isDynamicClass())
5710 return true;
5711
5712 for (const auto &Base : RD->bases())
5713 if (hasAnyVptr(Type: Base.getType(), Context))
5714 return true;
5715
5716 for (const FieldDecl *Field : RD->fields())
5717 if (hasAnyVptr(Type: Field->getType(), Context))
5718 return true;
5719
5720 return false;
5721}
5722
5723LValue CodeGenFunction::EmitLValueForField(LValue base, const FieldDecl *field,
5724 bool IsInBounds) {
5725 LValueBaseInfo BaseInfo = base.getBaseInfo();
5726
5727 if (field->isBitField()) {
5728 const CGRecordLayout &RL =
5729 CGM.getTypes().getCGRecordLayout(field->getParent());
5730 const CGBitFieldInfo &Info = RL.getBitFieldInfo(FD: field);
5731 const bool UseVolatile = isAAPCS(TargetInfo: CGM.getTarget()) &&
5732 CGM.getCodeGenOpts().AAPCSBitfieldWidth &&
5733 Info.VolatileStorageSize != 0 &&
5734 field->getType()
5735 .withCVRQualifiers(CVR: base.getVRQualifiers())
5736 .isVolatileQualified();
5737 Address Addr = base.getAddress();
5738 unsigned Idx = RL.getLLVMFieldNo(FD: field);
5739 const RecordDecl *rec = field->getParent();
5740 if (hasBPFPreserveStaticOffset(D: rec))
5741 Addr = wrapWithBPFPreserveStaticOffset(CGF&: *this, Addr);
5742 if (!UseVolatile) {
5743 if (!IsInPreservedAIRegion &&
5744 (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
5745 if (Idx != 0) {
5746 // For structs, we GEP to the field that the record layout suggests.
5747 if (!IsInBounds)
5748 Addr = Builder.CreateConstGEP2_32(Addr, Idx0: 0, Idx1: Idx, Name: field->getName());
5749 else
5750 Addr = Builder.CreateStructGEP(Addr, Index: Idx, Name: field->getName());
5751 }
5752 } else {
5753 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType(
5754 Ty: getContext().getCanonicalTagType(TD: rec), L: rec->getLocation());
5755 Addr = Builder.CreatePreserveStructAccessIndex(
5756 Addr, Index: Idx, FieldIndex: getDebugInfoFIndex(Rec: rec, FieldIndex: field->getFieldIndex()),
5757 DbgInfo);
5758 }
5759 }
5760 const unsigned SS =
5761 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
5762 // Get the access type.
5763 llvm::Type *FieldIntTy = llvm::Type::getIntNTy(C&: getLLVMContext(), N: SS);
5764 Addr = Addr.withElementType(ElemTy: FieldIntTy);
5765 if (UseVolatile) {
5766 const unsigned VolatileOffset = Info.VolatileStorageOffset.getQuantity();
5767 if (VolatileOffset)
5768 Addr = Builder.CreateConstInBoundsGEP(Addr, Index: VolatileOffset);
5769 }
5770
5771 QualType fieldType =
5772 field->getType().withCVRQualifiers(CVR: base.getVRQualifiers());
5773 // TODO: Support TBAA for bit fields.
5774 LValueBaseInfo FieldBaseInfo(BaseInfo.getAlignmentSource());
5775 return LValue::MakeBitfield(Addr, Info, type: fieldType, BaseInfo: FieldBaseInfo,
5776 TBAAInfo: TBAAAccessInfo());
5777 }
5778
5779 // Fields of may-alias structures are may-alias themselves.
5780 // FIXME: this should get propagated down through anonymous structs
5781 // and unions.
5782 QualType FieldType = field->getType();
5783 const RecordDecl *rec = field->getParent();
5784 AlignmentSource BaseAlignSource = BaseInfo.getAlignmentSource();
5785 LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(Source: BaseAlignSource));
5786 TBAAAccessInfo FieldTBAAInfo;
5787 if (base.getTBAAInfo().isMayAlias() ||
5788 rec->hasAttr<MayAliasAttr>() || FieldType->isVectorType()) {
5789 FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
5790 } else if (rec->isUnion()) {
5791 // TODO: Support TBAA for unions.
5792 FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
5793 } else {
5794 // If no base type been assigned for the base access, then try to generate
5795 // one for this base lvalue.
5796 FieldTBAAInfo = base.getTBAAInfo();
5797 if (!FieldTBAAInfo.BaseType) {
5798 FieldTBAAInfo.BaseType = CGM.getTBAABaseTypeInfo(QTy: base.getType());
5799 assert(!FieldTBAAInfo.Offset &&
5800 "Nonzero offset for an access with no base type!");
5801 }
5802
5803 // Adjust offset to be relative to the base type.
5804 const ASTRecordLayout &Layout =
5805 getContext().getASTRecordLayout(D: field->getParent());
5806 unsigned CharWidth = getContext().getCharWidth();
5807 if (FieldTBAAInfo.BaseType)
5808 FieldTBAAInfo.Offset +=
5809 Layout.getFieldOffset(FieldNo: field->getFieldIndex()) / CharWidth;
5810
5811 // Update the final access type and size.
5812 FieldTBAAInfo.AccessType = CGM.getTBAATypeInfo(QTy: FieldType);
5813 FieldTBAAInfo.Size =
5814 getContext().getTypeSizeInChars(T: FieldType).getQuantity();
5815 }
5816
5817 Address addr = base.getAddress();
5818 if (hasBPFPreserveStaticOffset(D: rec))
5819 addr = wrapWithBPFPreserveStaticOffset(CGF&: *this, Addr&: addr);
5820 if (auto *ClassDef = dyn_cast<CXXRecordDecl>(Val: rec)) {
5821 if (CGM.getCodeGenOpts().StrictVTablePointers &&
5822 ClassDef->isDynamicClass()) {
5823 // Getting to any field of dynamic object requires stripping dynamic
5824 // information provided by invariant.group. This is because accessing
5825 // fields may leak the real address of dynamic object, which could result
5826 // in miscompilation when leaked pointer would be compared.
5827 auto *stripped =
5828 Builder.CreateStripInvariantGroup(Ptr: addr.emitRawPointer(CGF&: *this));
5829 addr = Address(stripped, addr.getElementType(), addr.getAlignment());
5830 }
5831 }
5832
5833 unsigned RecordCVR = base.getVRQualifiers();
5834 if (rec->isUnion()) {
5835 // For unions, there is no pointer adjustment.
5836 if (CGM.getCodeGenOpts().StrictVTablePointers &&
5837 hasAnyVptr(Type: FieldType, Context: getContext()))
5838 // Because unions can easily skip invariant.barriers, we need to add
5839 // a barrier every time CXXRecord field with vptr is referenced.
5840 addr = Builder.CreateLaunderInvariantGroup(Addr: addr);
5841
5842 if (IsInPreservedAIRegion ||
5843 (getDebugInfo() && rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
5844 // Remember the original union field index
5845 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(Ty: base.getType(),
5846 Loc: rec->getLocation());
5847 addr =
5848 Address(Builder.CreatePreserveUnionAccessIndex(
5849 Base: addr.emitRawPointer(CGF&: *this),
5850 FieldIndex: getDebugInfoFIndex(Rec: rec, FieldIndex: field->getFieldIndex()), DbgInfo),
5851 addr.getElementType(), addr.getAlignment());
5852 }
5853
5854 if (FieldType->isReferenceType())
5855 addr = addr.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: FieldType));
5856 } else {
5857 if (!IsInPreservedAIRegion &&
5858 (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>()))
5859 // For structs, we GEP to the field that the record layout suggests.
5860 addr = emitAddrOfFieldStorage(CGF&: *this, base: addr, field, IsInBounds);
5861 else
5862 // Remember the original struct field index
5863 addr = emitPreserveStructAccess(CGF&: *this, base, addr, field);
5864 }
5865
5866 // If this is a reference field, load the reference right now.
5867 if (FieldType->isReferenceType()) {
5868 LValue RefLVal =
5869 MakeAddrLValue(Addr: addr, T: FieldType, BaseInfo: FieldBaseInfo, TBAAInfo: FieldTBAAInfo);
5870 if (RecordCVR & Qualifiers::Volatile)
5871 RefLVal.getQuals().addVolatile();
5872 addr = EmitLoadOfReference(RefLVal, PointeeBaseInfo: &FieldBaseInfo, PointeeTBAAInfo: &FieldTBAAInfo);
5873
5874 // Qualifiers on the struct don't apply to the referencee.
5875 RecordCVR = 0;
5876 FieldType = FieldType->getPointeeType();
5877 }
5878
5879 // Make sure that the address is pointing to the right type. This is critical
5880 // for both unions and structs.
5881 addr = addr.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: FieldType));
5882
5883 if (field->hasAttr<AnnotateAttr>())
5884 addr = EmitFieldAnnotations(D: field, V: addr);
5885
5886 LValue LV = MakeAddrLValue(Addr: addr, T: FieldType, BaseInfo: FieldBaseInfo, TBAAInfo: FieldTBAAInfo);
5887 LV.getQuals().addCVRQualifiers(mask: RecordCVR);
5888
5889 // __weak attribute on a field is ignored.
5890 if (LV.getQuals().getObjCGCAttr() == Qualifiers::Weak)
5891 LV.getQuals().removeObjCGCAttr();
5892
5893 return LV;
5894}
5895
5896LValue
5897CodeGenFunction::EmitLValueForFieldInitialization(LValue Base,
5898 const FieldDecl *Field) {
5899 QualType FieldType = Field->getType();
5900
5901 if (!FieldType->isReferenceType())
5902 return EmitLValueForField(base: Base, field: Field);
5903
5904 Address V = emitAddrOfFieldStorage(
5905 CGF&: *this, base: Base.getAddress(), field: Field,
5906 /*IsInBounds=*/!getLangOpts().PointerOverflowDefined);
5907
5908 // Make sure that the address is pointing to the right type.
5909 llvm::Type *llvmType = ConvertTypeForMem(T: FieldType);
5910 V = V.withElementType(ElemTy: llvmType);
5911
5912 // TODO: Generate TBAA information that describes this access as a structure
5913 // member access and not just an access to an object of the field's type. This
5914 // should be similar to what we do in EmitLValueForField().
5915 LValueBaseInfo BaseInfo = Base.getBaseInfo();
5916 AlignmentSource FieldAlignSource = BaseInfo.getAlignmentSource();
5917 LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(Source: FieldAlignSource));
5918 return MakeAddrLValue(Addr: V, T: FieldType, BaseInfo: FieldBaseInfo,
5919 TBAAInfo: CGM.getTBAAInfoForSubobject(Base, AccessType: FieldType));
5920}
5921
5922LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr *E){
5923 if (E->isFileScope()) {
5924 ConstantAddress GlobalPtr = CGM.GetAddrOfConstantCompoundLiteral(E);
5925 return MakeAddrLValue(Addr: GlobalPtr, T: E->getType(), Source: AlignmentSource::Decl);
5926 }
5927 if (E->getType()->isVariablyModifiedType())
5928 // make sure to emit the VLA size.
5929 EmitVariablyModifiedType(Ty: E->getType());
5930
5931 Address DeclPtr = CreateMemTemp(Ty: E->getType(), Name: ".compoundliteral");
5932 const Expr *InitExpr = E->getInitializer();
5933 LValue Result = MakeAddrLValue(Addr: DeclPtr, T: E->getType(), Source: AlignmentSource::Decl);
5934
5935 EmitAnyExprToMem(E: InitExpr, Location: DeclPtr, Quals: E->getType().getQualifiers(),
5936 /*Init*/ IsInit: true);
5937
5938 // Block-scope compound literals are destroyed at the end of the enclosing
5939 // scope in C.
5940 if (!getLangOpts().CPlusPlus)
5941 if (QualType::DestructionKind DtorKind = E->getType().isDestructedType())
5942 pushLifetimeExtendedDestroy(kind: getCleanupKind(kind: DtorKind), addr: DeclPtr,
5943 type: E->getType(), destroyer: getDestroyer(destructionKind: DtorKind),
5944 useEHCleanupForArray: DtorKind & EHCleanup);
5945
5946 return Result;
5947}
5948
5949LValue CodeGenFunction::EmitInitListLValue(const InitListExpr *E) {
5950 if (!E->isGLValue())
5951 // Initializing an aggregate temporary in C++11: T{...}.
5952 return EmitAggExprToLValue(E);
5953
5954 // An lvalue initializer list must be initializing a reference.
5955 assert(E->isTransparent() && "non-transparent glvalue init list");
5956 return EmitLValue(E: E->getInit(Init: 0));
5957}
5958
5959/// Emit the operand of a glvalue conditional operator. This is either a glvalue
5960/// or a (possibly-parenthesized) throw-expression. If this is a throw, no
5961/// LValue is returned and the current block has been terminated.
5962static std::optional<LValue> EmitLValueOrThrowExpression(CodeGenFunction &CGF,
5963 const Expr *Operand) {
5964 if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Val: Operand->IgnoreParens())) {
5965 CGF.EmitCXXThrowExpr(E: ThrowExpr, /*KeepInsertionPoint*/false);
5966 return std::nullopt;
5967 }
5968
5969 return CGF.EmitLValue(E: Operand);
5970}
5971
5972namespace {
5973// Handle the case where the condition is a constant evaluatable simple integer,
5974// which means we don't have to separately handle the true/false blocks.
5975std::optional<LValue> HandleConditionalOperatorLValueSimpleCase(
5976 CodeGenFunction &CGF, const AbstractConditionalOperator *E) {
5977 const Expr *condExpr = E->getCond();
5978 bool CondExprBool;
5979 if (CGF.ConstantFoldsToSimpleInteger(Cond: condExpr, Result&: CondExprBool)) {
5980 const Expr *Live = E->getTrueExpr(), *Dead = E->getFalseExpr();
5981 if (!CondExprBool)
5982 std::swap(a&: Live, b&: Dead);
5983
5984 if (!CGF.ContainsLabel(S: Dead)) {
5985 // If the true case is live, we need to track its region.
5986 CGF.incrementProfileCounter(ExecSkip: CondExprBool ? CGF.UseExecPath
5987 : CGF.UseSkipPath,
5988 S: E, /*UseBoth=*/true);
5989 CGF.markStmtMaybeUsed(S: Dead);
5990 // If a throw expression we emit it and return an undefined lvalue
5991 // because it can't be used.
5992 if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Val: Live->IgnoreParens())) {
5993 CGF.EmitCXXThrowExpr(E: ThrowExpr);
5994 llvm::Type *ElemTy = CGF.ConvertType(T: Dead->getType());
5995 llvm::Type *Ty = CGF.DefaultPtrTy;
5996 return CGF.MakeAddrLValue(
5997 Addr: Address(llvm::UndefValue::get(T: Ty), ElemTy, CharUnits::One()),
5998 T: Dead->getType());
5999 }
6000 return CGF.EmitLValue(E: Live);
6001 }
6002 }
6003 return std::nullopt;
6004}
6005struct ConditionalInfo {
6006 llvm::BasicBlock *lhsBlock, *rhsBlock;
6007 std::optional<LValue> LHS, RHS;
6008};
6009
6010// Create and generate the 3 blocks for a conditional operator.
6011// Leaves the 'current block' in the continuation basic block.
6012template<typename FuncTy>
6013ConditionalInfo EmitConditionalBlocks(CodeGenFunction &CGF,
6014 const AbstractConditionalOperator *E,
6015 const FuncTy &BranchGenFunc) {
6016 ConditionalInfo Info{.lhsBlock: CGF.createBasicBlock(name: "cond.true"),
6017 .rhsBlock: CGF.createBasicBlock(name: "cond.false"), .LHS: std::nullopt,
6018 .RHS: std::nullopt};
6019 llvm::BasicBlock *endBlock = CGF.createBasicBlock(name: "cond.end");
6020
6021 CodeGenFunction::ConditionalEvaluation eval(CGF);
6022 CGF.EmitBranchOnBoolExpr(Cond: E->getCond(), TrueBlock: Info.lhsBlock, FalseBlock: Info.rhsBlock,
6023 TrueCount: CGF.getProfileCount(S: E));
6024
6025 // Any temporaries created here are conditional.
6026 CGF.EmitBlock(BB: Info.lhsBlock);
6027 CGF.incrementProfileCounter(ExecSkip: CGF.UseExecPath, S: E);
6028 eval.begin(CGF);
6029 Info.LHS = BranchGenFunc(CGF, E->getTrueExpr());
6030 eval.end(CGF);
6031 Info.lhsBlock = CGF.Builder.GetInsertBlock();
6032
6033 if (Info.LHS)
6034 CGF.Builder.CreateBr(Dest: endBlock);
6035
6036 // Any temporaries created here are conditional.
6037 CGF.EmitBlock(BB: Info.rhsBlock);
6038 CGF.incrementProfileCounter(ExecSkip: CGF.UseSkipPath, S: E);
6039 eval.begin(CGF);
6040 Info.RHS = BranchGenFunc(CGF, E->getFalseExpr());
6041 eval.end(CGF);
6042 Info.rhsBlock = CGF.Builder.GetInsertBlock();
6043 CGF.EmitBlock(BB: endBlock);
6044
6045 return Info;
6046}
6047} // namespace
6048
6049void CodeGenFunction::EmitIgnoredConditionalOperator(
6050 const AbstractConditionalOperator *E) {
6051 if (!E->isGLValue()) {
6052 // ?: here should be an aggregate.
6053 assert(hasAggregateEvaluationKind(E->getType()) &&
6054 "Unexpected conditional operator!");
6055 return (void)EmitAggExprToLValue(E);
6056 }
6057
6058 OpaqueValueMapping binding(*this, E);
6059 if (HandleConditionalOperatorLValueSimpleCase(CGF&: *this, E))
6060 return;
6061
6062 EmitConditionalBlocks(CGF&: *this, E, BranchGenFunc: [](CodeGenFunction &CGF, const Expr *E) {
6063 CGF.EmitIgnoredExpr(E);
6064 return LValue{};
6065 });
6066}
6067LValue CodeGenFunction::EmitConditionalOperatorLValue(
6068 const AbstractConditionalOperator *expr) {
6069 if (!expr->isGLValue()) {
6070 // ?: here should be an aggregate.
6071 assert(hasAggregateEvaluationKind(expr->getType()) &&
6072 "Unexpected conditional operator!");
6073 return EmitAggExprToLValue(E: expr);
6074 }
6075
6076 OpaqueValueMapping binding(*this, expr);
6077 if (std::optional<LValue> Res =
6078 HandleConditionalOperatorLValueSimpleCase(CGF&: *this, E: expr))
6079 return *Res;
6080
6081 ConditionalInfo Info = EmitConditionalBlocks(
6082 CGF&: *this, E: expr, BranchGenFunc: [](CodeGenFunction &CGF, const Expr *E) {
6083 return EmitLValueOrThrowExpression(CGF, Operand: E);
6084 });
6085
6086 if ((Info.LHS && !Info.LHS->isSimple()) ||
6087 (Info.RHS && !Info.RHS->isSimple()))
6088 return EmitUnsupportedLValue(E: expr, Name: "conditional operator");
6089
6090 if (Info.LHS && Info.RHS) {
6091 Address lhsAddr = Info.LHS->getAddress();
6092 Address rhsAddr = Info.RHS->getAddress();
6093 Address result = mergeAddressesInConditionalExpr(
6094 LHS: lhsAddr, RHS: rhsAddr, LHSBlock: Info.lhsBlock, RHSBlock: Info.rhsBlock,
6095 MergeBlock: Builder.GetInsertBlock(), MergedType: expr->getType());
6096 AlignmentSource alignSource =
6097 std::max(a: Info.LHS->getBaseInfo().getAlignmentSource(),
6098 b: Info.RHS->getBaseInfo().getAlignmentSource());
6099 TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForConditionalOperator(
6100 InfoA: Info.LHS->getTBAAInfo(), InfoB: Info.RHS->getTBAAInfo());
6101 return MakeAddrLValue(Addr: result, T: expr->getType(), BaseInfo: LValueBaseInfo(alignSource),
6102 TBAAInfo);
6103 } else {
6104 assert((Info.LHS || Info.RHS) &&
6105 "both operands of glvalue conditional are throw-expressions?");
6106 return Info.LHS ? *Info.LHS : *Info.RHS;
6107 }
6108}
6109
6110/// EmitCastLValue - Casts are never lvalues unless that cast is to a reference
6111/// type. If the cast is to a reference, we can have the usual lvalue result,
6112/// otherwise if a cast is needed by the code generator in an lvalue context,
6113/// then it must mean that we need the address of an aggregate in order to
6114/// access one of its members. This can happen for all the reasons that casts
6115/// are permitted with aggregate result, including noop aggregate casts, and
6116/// cast from scalar to union.
6117LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
6118 llvm::scope_exit RestoreCurCast([this, Prev = CurCast] { CurCast = Prev; });
6119 CurCast = E;
6120 switch (E->getCastKind()) {
6121 case CK_ToVoid:
6122 case CK_BitCast:
6123 case CK_LValueToRValueBitCast:
6124 case CK_ArrayToPointerDecay:
6125 case CK_FunctionToPointerDecay:
6126 case CK_NullToMemberPointer:
6127 case CK_NullToPointer:
6128 case CK_IntegralToPointer:
6129 case CK_PointerToIntegral:
6130 case CK_PointerToBoolean:
6131 case CK_IntegralCast:
6132 case CK_BooleanToSignedIntegral:
6133 case CK_IntegralToBoolean:
6134 case CK_IntegralToFloating:
6135 case CK_FloatingToIntegral:
6136 case CK_FloatingToBoolean:
6137 case CK_FloatingCast:
6138 case CK_FloatingRealToComplex:
6139 case CK_FloatingComplexToReal:
6140 case CK_FloatingComplexToBoolean:
6141 case CK_FloatingComplexCast:
6142 case CK_FloatingComplexToIntegralComplex:
6143 case CK_IntegralRealToComplex:
6144 case CK_IntegralComplexToReal:
6145 case CK_IntegralComplexToBoolean:
6146 case CK_IntegralComplexCast:
6147 case CK_IntegralComplexToFloatingComplex:
6148 case CK_DerivedToBaseMemberPointer:
6149 case CK_BaseToDerivedMemberPointer:
6150 case CK_MemberPointerToBoolean:
6151 case CK_ReinterpretMemberPointer:
6152 case CK_AnyPointerToBlockPointerCast:
6153 case CK_ARCProduceObject:
6154 case CK_ARCConsumeObject:
6155 case CK_ARCReclaimReturnedObject:
6156 case CK_ARCExtendBlockObject:
6157 case CK_CopyAndAutoreleaseBlockObject:
6158 case CK_IntToOCLSampler:
6159 case CK_FloatingToFixedPoint:
6160 case CK_FixedPointToFloating:
6161 case CK_FixedPointCast:
6162 case CK_FixedPointToBoolean:
6163 case CK_FixedPointToIntegral:
6164 case CK_IntegralToFixedPoint:
6165 case CK_MatrixCast:
6166 case CK_HLSLVectorTruncation:
6167 case CK_HLSLMatrixTruncation:
6168 case CK_HLSLArrayRValue:
6169 case CK_HLSLElementwiseCast:
6170 case CK_HLSLAggregateSplatCast:
6171 return EmitUnsupportedLValue(E, Name: "unexpected cast lvalue");
6172
6173 case CK_Dependent:
6174 llvm_unreachable("dependent cast kind in IR gen!");
6175
6176 case CK_BuiltinFnToFnPtr:
6177 llvm_unreachable("builtin functions are handled elsewhere");
6178
6179 // These are never l-values; just use the aggregate emission code.
6180 case CK_NonAtomicToAtomic:
6181 case CK_AtomicToNonAtomic:
6182 return EmitAggExprToLValue(E);
6183
6184 case CK_Dynamic: {
6185 LValue LV = EmitLValue(E: E->getSubExpr());
6186 Address V = LV.getAddress();
6187 const auto *DCE = cast<CXXDynamicCastExpr>(Val: E);
6188 return MakeNaturalAlignRawAddrLValue(V: EmitDynamicCast(V, DCE), T: E->getType());
6189 }
6190
6191 case CK_ConstructorConversion:
6192 case CK_UserDefinedConversion:
6193 case CK_CPointerToObjCPointerCast:
6194 case CK_BlockPointerToObjCPointerCast:
6195 case CK_LValueToRValue:
6196 return EmitLValue(E: E->getSubExpr());
6197
6198 case CK_NoOp: {
6199 // CK_NoOp can model a qualification conversion, which can remove an array
6200 // bound and change the IR type.
6201 // FIXME: Once pointee types are removed from IR, remove this.
6202 LValue LV = EmitLValue(E: E->getSubExpr());
6203 // Propagate the volatile qualifer to LValue, if exist in E.
6204 if (E->changesVolatileQualification())
6205 LV.getQuals() = E->getType().getQualifiers();
6206 if (LV.isSimple()) {
6207 Address V = LV.getAddress();
6208 if (V.isValid()) {
6209 llvm::Type *T = ConvertTypeForMem(T: E->getType());
6210 if (V.getElementType() != T)
6211 LV.setAddress(V.withElementType(ElemTy: T));
6212 }
6213 }
6214 return LV;
6215 }
6216
6217 case CK_UncheckedDerivedToBase:
6218 case CK_DerivedToBase: {
6219 auto *DerivedClassDecl = E->getSubExpr()->getType()->castAsCXXRecordDecl();
6220 LValue LV = EmitLValue(E: E->getSubExpr());
6221 Address This = LV.getAddress();
6222
6223 // Perform the derived-to-base conversion
6224 Address Base = GetAddressOfBaseClass(
6225 Value: This, Derived: DerivedClassDecl, PathBegin: E->path_begin(), PathEnd: E->path_end(),
6226 /*NullCheckValue=*/false, Loc: E->getExprLoc());
6227
6228 // TODO: Support accesses to members of base classes in TBAA. For now, we
6229 // conservatively pretend that the complete object is of the base class
6230 // type.
6231 return MakeAddrLValue(Addr: Base, T: E->getType(), BaseInfo: LV.getBaseInfo(),
6232 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
6233 }
6234 case CK_ToUnion:
6235 return EmitAggExprToLValue(E);
6236 case CK_BaseToDerived: {
6237 auto *DerivedClassDecl = E->getType()->castAsCXXRecordDecl();
6238 LValue LV = EmitLValue(E: E->getSubExpr());
6239
6240 // Perform the base-to-derived conversion
6241 Address Derived = GetAddressOfDerivedClass(
6242 Value: LV.getAddress(), Derived: DerivedClassDecl, PathBegin: E->path_begin(), PathEnd: E->path_end(),
6243 /*NullCheckValue=*/false);
6244
6245 // C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is
6246 // performed and the object is not of the derived type.
6247 if (sanitizePerformTypeCheck())
6248 EmitTypeCheck(TCK: TCK_DowncastReference, Loc: E->getExprLoc(), Addr: Derived,
6249 Type: E->getType());
6250
6251 if (SanOpts.has(K: SanitizerKind::CFIDerivedCast))
6252 EmitVTablePtrCheckForCast(T: E->getType(), Derived,
6253 /*MayBeNull=*/false, TCK: CFITCK_DerivedCast,
6254 Loc: E->getBeginLoc());
6255
6256 return MakeAddrLValue(Addr: Derived, T: E->getType(), BaseInfo: LV.getBaseInfo(),
6257 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
6258 }
6259 case CK_LValueBitCast: {
6260 // This must be a reinterpret_cast (or c-style equivalent).
6261 const auto *CE = cast<ExplicitCastExpr>(Val: E);
6262
6263 CGM.EmitExplicitCastExprType(E: CE, CGF: this);
6264 LValue LV = EmitLValue(E: E->getSubExpr());
6265 Address V = LV.getAddress().withElementType(
6266 ElemTy: ConvertTypeForMem(T: CE->getTypeAsWritten()->getPointeeType()));
6267
6268 if (SanOpts.has(K: SanitizerKind::CFIUnrelatedCast))
6269 EmitVTablePtrCheckForCast(T: E->getType(), Derived: V,
6270 /*MayBeNull=*/false, TCK: CFITCK_UnrelatedCast,
6271 Loc: E->getBeginLoc());
6272
6273 return MakeAddrLValue(Addr: V, T: E->getType(), BaseInfo: LV.getBaseInfo(),
6274 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
6275 }
6276 case CK_AddressSpaceConversion: {
6277 LValue LV = EmitLValue(E: E->getSubExpr());
6278 QualType DestTy = getContext().getPointerType(T: E->getType());
6279 llvm::Value *V =
6280 performAddrSpaceCast(Src: LV.getPointer(CGF&: *this), DestTy: ConvertType(T: DestTy));
6281 return MakeAddrLValue(Addr: Address(V, ConvertTypeForMem(T: E->getType()),
6282 LV.getAddress().getAlignment()),
6283 T: E->getType(), BaseInfo: LV.getBaseInfo(), TBAAInfo: LV.getTBAAInfo());
6284 }
6285 case CK_ObjCObjectLValueCast: {
6286 LValue LV = EmitLValue(E: E->getSubExpr());
6287 Address V = LV.getAddress().withElementType(ElemTy: ConvertType(T: E->getType()));
6288 return MakeAddrLValue(Addr: V, T: E->getType(), BaseInfo: LV.getBaseInfo(),
6289 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
6290 }
6291 case CK_ZeroToOCLOpaqueType:
6292 llvm_unreachable("NULL to OpenCL opaque type lvalue cast is not valid");
6293
6294 case CK_VectorSplat: {
6295 // LValue results of vector splats are only supported in HLSL.
6296 if (!getLangOpts().HLSL)
6297 return EmitUnsupportedLValue(E, Name: "unexpected cast lvalue");
6298 return EmitLValue(E: E->getSubExpr());
6299 }
6300 }
6301
6302 llvm_unreachable("Unhandled lvalue cast kind?");
6303}
6304
6305LValue CodeGenFunction::EmitOpaqueValueLValue(const OpaqueValueExpr *e) {
6306 assert(OpaqueValueMappingData::shouldBindAsLValue(e));
6307 return getOrCreateOpaqueLValueMapping(e);
6308}
6309
6310std::pair<LValue, LValue>
6311CodeGenFunction::EmitHLSLOutArgLValues(const HLSLOutArgExpr *E, QualType Ty) {
6312 // Emitting the casted temporary through an opaque value.
6313 LValue BaseLV = EmitLValue(E: E->getArgLValue());
6314 OpaqueValueMappingData::bind(CGF&: *this, ov: E->getOpaqueArgLValue(), lv: BaseLV);
6315
6316 QualType ExprTy = E->getType();
6317 Address OutTemp = CreateIRTempWithoutCast(Ty: ExprTy);
6318 LValue TempLV = MakeAddrLValue(Addr: OutTemp, T: ExprTy);
6319
6320 if (E->isInOut())
6321 EmitInitializationToLValue(E: E->getCastedTemporary()->getSourceExpr(),
6322 LV: TempLV);
6323
6324 OpaqueValueMappingData::bind(CGF&: *this, ov: E->getCastedTemporary(), lv: TempLV);
6325 return std::make_pair(x&: BaseLV, y&: TempLV);
6326}
6327
6328LValue CodeGenFunction::EmitHLSLOutArgExpr(const HLSLOutArgExpr *E,
6329 CallArgList &Args, QualType Ty) {
6330
6331 auto [BaseLV, TempLV] = EmitHLSLOutArgLValues(E, Ty);
6332
6333 llvm::Value *Addr = TempLV.getAddress().getBasePointer();
6334 llvm::Type *ElTy = ConvertTypeForMem(T: TempLV.getType());
6335
6336 EmitLifetimeStart(Addr);
6337
6338 Address TmpAddr(Addr, ElTy, TempLV.getAlignment());
6339 Args.addWriteback(srcLV: BaseLV, temporary: TmpAddr, toUse: nullptr, writebackExpr: E->getWritebackCast());
6340 Args.add(rvalue: RValue::get(Addr: TmpAddr, CGF&: *this), type: Ty);
6341 return TempLV;
6342}
6343
6344LValue
6345CodeGenFunction::getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e) {
6346 assert(OpaqueValueMapping::shouldBindAsLValue(e));
6347
6348 llvm::DenseMap<const OpaqueValueExpr*,LValue>::iterator
6349 it = OpaqueLValues.find(Val: e);
6350
6351 if (it != OpaqueLValues.end())
6352 return it->second;
6353
6354 assert(e->isUnique() && "LValue for a nonunique OVE hasn't been emitted");
6355 return EmitLValue(E: e->getSourceExpr());
6356}
6357
6358RValue
6359CodeGenFunction::getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e) {
6360 assert(!OpaqueValueMapping::shouldBindAsLValue(e));
6361
6362 llvm::DenseMap<const OpaqueValueExpr*,RValue>::iterator
6363 it = OpaqueRValues.find(Val: e);
6364
6365 if (it != OpaqueRValues.end())
6366 return it->second;
6367
6368 assert(e->isUnique() && "RValue for a nonunique OVE hasn't been emitted");
6369 return EmitAnyExpr(E: e->getSourceExpr());
6370}
6371
6372bool CodeGenFunction::isOpaqueValueEmitted(const OpaqueValueExpr *E) {
6373 if (OpaqueValueMapping::shouldBindAsLValue(expr: E))
6374 return OpaqueLValues.contains(Val: E);
6375 return OpaqueRValues.contains(Val: E);
6376}
6377
6378RValue CodeGenFunction::EmitRValueForField(LValue LV,
6379 const FieldDecl *FD,
6380 SourceLocation Loc) {
6381 QualType FT = FD->getType();
6382 LValue FieldLV = EmitLValueForField(base: LV, field: FD);
6383 switch (getEvaluationKind(T: FT)) {
6384 case TEK_Complex:
6385 return RValue::getComplex(C: EmitLoadOfComplex(src: FieldLV, loc: Loc));
6386 case TEK_Aggregate:
6387 return FieldLV.asAggregateRValue();
6388 case TEK_Scalar:
6389 // This routine is used to load fields one-by-one to perform a copy, so
6390 // don't load reference fields.
6391 if (FD->getType()->isReferenceType())
6392 return RValue::get(V: FieldLV.getPointer(CGF&: *this));
6393 // Call EmitLoadOfScalar except when the lvalue is a bitfield to emit a
6394 // primitive load.
6395 if (FieldLV.isBitField())
6396 return EmitLoadOfLValue(LV: FieldLV, Loc);
6397 return RValue::get(V: EmitLoadOfScalar(lvalue: FieldLV, Loc));
6398 }
6399 llvm_unreachable("bad evaluation kind");
6400}
6401
6402//===--------------------------------------------------------------------===//
6403// Expression Emission
6404//===--------------------------------------------------------------------===//
6405
6406RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
6407 ReturnValueSlot ReturnValue,
6408 llvm::CallBase **CallOrInvoke) {
6409 llvm::CallBase *CallOrInvokeStorage;
6410 if (!CallOrInvoke) {
6411 CallOrInvoke = &CallOrInvokeStorage;
6412 }
6413
6414 llvm::scope_exit AddCoroElideSafeOnExit([&] {
6415 if (E->isCoroElideSafe()) {
6416 auto *I = *CallOrInvoke;
6417 if (I)
6418 I->addFnAttr(Kind: llvm::Attribute::CoroElideSafe);
6419 }
6420 });
6421
6422 // Builtins never have block type.
6423 if (E->getCallee()->getType()->isBlockPointerType())
6424 return EmitBlockCallExpr(E, ReturnValue, CallOrInvoke);
6425
6426 if (const auto *CE = dyn_cast<CXXMemberCallExpr>(Val: E))
6427 return EmitCXXMemberCallExpr(E: CE, ReturnValue, CallOrInvoke);
6428
6429 if (const auto *CE = dyn_cast<CUDAKernelCallExpr>(Val: E))
6430 return EmitCUDAKernelCallExpr(E: CE, ReturnValue, CallOrInvoke);
6431
6432 // A CXXOperatorCallExpr is created even for explicit object methods, but
6433 // these should be treated like static function call.
6434 if (const auto *CE = dyn_cast<CXXOperatorCallExpr>(Val: E))
6435 if (const auto *MD =
6436 dyn_cast_if_present<CXXMethodDecl>(Val: CE->getCalleeDecl());
6437 MD && MD->isImplicitObjectMemberFunction())
6438 return EmitCXXOperatorMemberCallExpr(E: CE, MD, ReturnValue, CallOrInvoke);
6439
6440 CGCallee callee = EmitCallee(E: E->getCallee());
6441
6442 if (callee.isBuiltin()) {
6443 return EmitBuiltinExpr(GD: callee.getBuiltinDecl(), BuiltinID: callee.getBuiltinID(),
6444 E, ReturnValue);
6445 }
6446
6447 if (callee.isPseudoDestructor()) {
6448 return EmitCXXPseudoDestructorExpr(E: callee.getPseudoDestructorExpr());
6449 }
6450
6451 return EmitCall(FnType: E->getCallee()->getType(), Callee: callee, E, ReturnValue,
6452 /*Chain=*/nullptr, CallOrInvoke);
6453}
6454
6455/// Emit a CallExpr without considering whether it might be a subclass.
6456RValue CodeGenFunction::EmitSimpleCallExpr(const CallExpr *E,
6457 ReturnValueSlot ReturnValue,
6458 llvm::CallBase **CallOrInvoke) {
6459 CGCallee Callee = EmitCallee(E: E->getCallee());
6460 return EmitCall(FnType: E->getCallee()->getType(), Callee, E, ReturnValue,
6461 /*Chain=*/nullptr, CallOrInvoke);
6462}
6463
6464// Detect the unusual situation where an inline version is shadowed by a
6465// non-inline version. In that case we should pick the external one
6466// everywhere. That's GCC behavior too.
6467static bool OnlyHasInlineBuiltinDeclaration(const FunctionDecl *FD) {
6468 for (const FunctionDecl *PD = FD; PD; PD = PD->getPreviousDecl())
6469 if (!PD->isInlineBuiltinDeclaration())
6470 return false;
6471 return true;
6472}
6473
6474static CGCallee EmitDirectCallee(CodeGenFunction &CGF, GlobalDecl GD) {
6475 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
6476
6477 if (auto builtinID = FD->getBuiltinID()) {
6478 std::string NoBuiltinFD = ("no-builtin-" + FD->getName()).str();
6479 std::string NoBuiltins = "no-builtins";
6480
6481 StringRef Ident = CGF.CGM.getMangledName(GD);
6482 std::string FDInlineName = (Ident + ".inline").str();
6483
6484 bool IsPredefinedLibFunction =
6485 CGF.getContext().BuiltinInfo.isPredefinedLibFunction(ID: builtinID);
6486 bool HasAttributeNoBuiltin =
6487 CGF.CurFn->getAttributes().hasFnAttr(Kind: NoBuiltinFD) ||
6488 CGF.CurFn->getAttributes().hasFnAttr(Kind: NoBuiltins);
6489
6490 // When directing calling an inline builtin, call it through it's mangled
6491 // name to make it clear it's not the actual builtin.
6492 if (CGF.CurFn->getName() != FDInlineName &&
6493 OnlyHasInlineBuiltinDeclaration(FD)) {
6494 llvm::Constant *CalleePtr = CGF.CGM.getRawFunctionPointer(GD);
6495 llvm::Function *Fn = llvm::cast<llvm::Function>(Val: CalleePtr);
6496 llvm::Module *M = Fn->getParent();
6497 llvm::Function *Clone = M->getFunction(Name: FDInlineName);
6498 if (!Clone) {
6499 Clone = llvm::Function::Create(Ty: Fn->getFunctionType(),
6500 Linkage: llvm::GlobalValue::InternalLinkage,
6501 AddrSpace: Fn->getAddressSpace(), N: FDInlineName, M);
6502 Clone->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
6503 }
6504 return CGCallee::forDirect(functionPtr: Clone, abstractInfo: GD);
6505 }
6506
6507 // Replaceable builtins provide their own implementation of a builtin. If we
6508 // are in an inline builtin implementation, avoid trivial infinite
6509 // recursion. Honor __attribute__((no_builtin("foo"))) or
6510 // __attribute__((no_builtin)) on the current function unless foo is
6511 // not a predefined library function which means we must generate the
6512 // builtin no matter what.
6513 else if (!IsPredefinedLibFunction || !HasAttributeNoBuiltin)
6514 return CGCallee::forBuiltin(builtinID, builtinDecl: FD);
6515 }
6516
6517 llvm::Constant *CalleePtr = CGF.CGM.getRawFunctionPointer(GD);
6518 if (CGF.CGM.getLangOpts().CUDA && !CGF.CGM.getLangOpts().CUDAIsDevice &&
6519 FD->hasAttr<CUDAGlobalAttr>())
6520 CalleePtr = CGF.CGM.getCUDARuntime().getKernelStub(
6521 Handle: cast<llvm::GlobalValue>(Val: CalleePtr->stripPointerCasts()));
6522
6523 return CGCallee::forDirect(functionPtr: CalleePtr, abstractInfo: GD);
6524}
6525
6526static GlobalDecl getGlobalDeclForDirectCall(const FunctionDecl *FD) {
6527 if (DeviceKernelAttr::isOpenCLSpelling(A: FD->getAttr<DeviceKernelAttr>()))
6528 return GlobalDecl(FD, KernelReferenceKind::Stub);
6529 return GlobalDecl(FD);
6530}
6531
6532CGCallee CodeGenFunction::EmitCallee(const Expr *E) {
6533 E = E->IgnoreParens();
6534
6535 // Look through function-to-pointer decay.
6536 if (auto ICE = dyn_cast<ImplicitCastExpr>(Val: E)) {
6537 if (ICE->getCastKind() == CK_FunctionToPointerDecay ||
6538 ICE->getCastKind() == CK_BuiltinFnToFnPtr) {
6539 return EmitCallee(E: ICE->getSubExpr());
6540 }
6541
6542 // Try to remember the original __ptrauth qualifier for loads of
6543 // function pointers.
6544 if (ICE->getCastKind() == CK_LValueToRValue) {
6545 const Expr *SubExpr = ICE->getSubExpr();
6546 if (const auto *PtrType = SubExpr->getType()->getAs<PointerType>()) {
6547 std::pair<llvm::Value *, CGPointerAuthInfo> Result =
6548 EmitOrigPointerRValue(E);
6549
6550 QualType FunctionType = PtrType->getPointeeType();
6551 assert(FunctionType->isFunctionType());
6552
6553 GlobalDecl GD;
6554 if (const auto *VD =
6555 dyn_cast_or_null<VarDecl>(Val: E->getReferencedDeclOfCallee())) {
6556 GD = GlobalDecl(VD);
6557 }
6558 CGCalleeInfo CalleeInfo(FunctionType->getAs<FunctionProtoType>(), GD);
6559 CGCallee Callee(CalleeInfo, Result.first, Result.second);
6560 return Callee;
6561 }
6562 }
6563
6564 // Resolve direct calls.
6565 } else if (auto DRE = dyn_cast<DeclRefExpr>(Val: E)) {
6566 if (auto FD = dyn_cast<FunctionDecl>(Val: DRE->getDecl())) {
6567 return EmitDirectCallee(CGF&: *this, GD: getGlobalDeclForDirectCall(FD));
6568 }
6569 } else if (auto ME = dyn_cast<MemberExpr>(Val: E)) {
6570 if (auto FD = dyn_cast<FunctionDecl>(Val: ME->getMemberDecl())) {
6571 EmitIgnoredExpr(E: ME->getBase());
6572 return EmitDirectCallee(CGF&: *this, GD: FD);
6573 }
6574
6575 // Look through template substitutions.
6576 } else if (auto NTTP = dyn_cast<SubstNonTypeTemplateParmExpr>(Val: E)) {
6577 return EmitCallee(E: NTTP->getReplacement());
6578
6579 // Treat pseudo-destructor calls differently.
6580 } else if (auto PDE = dyn_cast<CXXPseudoDestructorExpr>(Val: E)) {
6581 return CGCallee::forPseudoDestructor(E: PDE);
6582 }
6583
6584 // Otherwise, we have an indirect reference.
6585 llvm::Value *calleePtr;
6586 QualType functionType;
6587 if (auto ptrType = E->getType()->getAs<PointerType>()) {
6588 calleePtr = EmitScalarExpr(E);
6589 functionType = ptrType->getPointeeType();
6590 } else {
6591 functionType = E->getType();
6592 calleePtr = EmitLValue(E, IsKnownNonNull: KnownNonNull).getPointer(CGF&: *this);
6593 }
6594 assert(functionType->isFunctionType());
6595
6596 GlobalDecl GD;
6597 if (const auto *VD =
6598 dyn_cast_or_null<VarDecl>(Val: E->getReferencedDeclOfCallee()))
6599 GD = GlobalDecl(VD);
6600
6601 CGCalleeInfo calleeInfo(functionType->getAs<FunctionProtoType>(), GD);
6602 CGPointerAuthInfo pointerAuth = CGM.getFunctionPointerAuthInfo(T: functionType);
6603 CGCallee callee(calleeInfo, calleePtr, pointerAuth);
6604 return callee;
6605}
6606
6607LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
6608 // Comma expressions just emit their LHS then their RHS as an l-value.
6609 if (E->getOpcode() == BO_Comma) {
6610 EmitIgnoredExpr(E: E->getLHS());
6611 EnsureInsertPoint();
6612 return EmitLValue(E: E->getRHS());
6613 }
6614
6615 if (E->getOpcode() == BO_PtrMemD ||
6616 E->getOpcode() == BO_PtrMemI)
6617 return EmitPointerToDataMemberBinaryExpr(E);
6618
6619 assert(E->getOpcode() == BO_Assign && "unexpected binary l-value");
6620
6621 // Create a Key Instructions source location atom group that covers both
6622 // LHS and RHS expressions. Nested RHS expressions may get subsequently
6623 // separately grouped (1 below):
6624 //
6625 // 1. `a = b = c` -> Two atoms.
6626 // 2. `x = new(1)` -> One atom (for both addr store and value store).
6627 // 3. Complex and agg assignment -> One atom.
6628 ApplyAtomGroup Grp(getDebugInfo());
6629
6630 // Note that in all of these cases, __block variables need the RHS
6631 // evaluated first just in case the variable gets moved by the RHS.
6632
6633 switch (getEvaluationKind(T: E->getType())) {
6634 case TEK_Scalar: {
6635 if (PointerAuthQualifier PtrAuth =
6636 E->getLHS()->getType().getPointerAuth()) {
6637 LValue LV = EmitCheckedLValue(E: E->getLHS(), TCK: TCK_Store);
6638 LValue CopiedLV = LV;
6639 CopiedLV.getQuals().removePointerAuth();
6640 llvm::Value *RV =
6641 EmitPointerAuthQualify(Qualifier: PtrAuth, PointerExpr: E->getRHS(), StorageAddress: CopiedLV.getAddress());
6642 EmitNullabilityCheck(LHS: CopiedLV, RHS: RV, Loc: E->getExprLoc());
6643 EmitStoreThroughLValue(Src: RValue::get(V: RV), Dst: CopiedLV);
6644 return LV;
6645 }
6646
6647 switch (E->getLHS()->getType().getObjCLifetime()) {
6648 case Qualifiers::OCL_Strong:
6649 return EmitARCStoreStrong(e: E, /*ignored*/ false).first;
6650
6651 case Qualifiers::OCL_Autoreleasing:
6652 return EmitARCStoreAutoreleasing(e: E).first;
6653
6654 // No reason to do any of these differently.
6655 case Qualifiers::OCL_None:
6656 case Qualifiers::OCL_ExplicitNone:
6657 case Qualifiers::OCL_Weak:
6658 break;
6659 }
6660
6661 // TODO: Can we de-duplicate this code with the corresponding code in
6662 // CGExprScalar, similar to the way EmitCompoundAssignmentLValue works?
6663 RValue RV;
6664 llvm::Value *Previous = nullptr;
6665 QualType SrcType = E->getRHS()->getType();
6666 // Check if LHS is a bitfield, if RHS contains an implicit cast expression
6667 // we want to extract that value and potentially (if the bitfield sanitizer
6668 // is enabled) use it to check for an implicit conversion.
6669 if (E->getLHS()->refersToBitField()) {
6670 llvm::Value *RHS =
6671 EmitWithOriginalRHSBitfieldAssignment(E, Previous: &Previous, SrcType: &SrcType);
6672 RV = RValue::get(V: RHS);
6673 } else
6674 RV = EmitAnyExpr(E: E->getRHS());
6675
6676 LValue LV = EmitCheckedLValue(E: E->getLHS(), TCK: TCK_Store);
6677
6678 if (RV.isScalar())
6679 EmitNullabilityCheck(LHS: LV, RHS: RV.getScalarVal(), Loc: E->getExprLoc());
6680
6681 if (LV.isBitField()) {
6682 llvm::Value *Result = nullptr;
6683 // If bitfield sanitizers are enabled we want to use the result
6684 // to check whether a truncation or sign change has occurred.
6685 if (SanOpts.has(K: SanitizerKind::ImplicitBitfieldConversion))
6686 EmitStoreThroughBitfieldLValue(Src: RV, Dst: LV, Result: &Result);
6687 else
6688 EmitStoreThroughBitfieldLValue(Src: RV, Dst: LV);
6689
6690 // If the expression contained an implicit conversion, make sure
6691 // to use the value before the scalar conversion.
6692 llvm::Value *Src = Previous ? Previous : RV.getScalarVal();
6693 QualType DstType = E->getLHS()->getType();
6694 EmitBitfieldConversionCheck(Src, SrcType, Dst: Result, DstType,
6695 Info: LV.getBitFieldInfo(), Loc: E->getExprLoc());
6696 } else
6697 EmitStoreThroughLValue(Src: RV, Dst: LV);
6698
6699 if (getLangOpts().OpenMP)
6700 CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF&: *this,
6701 LHS: E->getLHS());
6702 return LV;
6703 }
6704
6705 case TEK_Complex:
6706 return EmitComplexAssignmentLValue(E);
6707
6708 case TEK_Aggregate:
6709 // If the lang opt is HLSL and the LHS is a constant array
6710 // then we are performing a copy assignment and call a special
6711 // function because EmitAggExprToLValue emits to a temporary LValue
6712 if (getLangOpts().HLSL && E->getLHS()->getType()->isConstantArrayType())
6713 return EmitHLSLArrayAssignLValue(E);
6714
6715 return EmitAggExprToLValue(E);
6716 }
6717 llvm_unreachable("bad evaluation kind");
6718}
6719
6720// This function implements trivial copy assignment for HLSL's
6721// assignable constant arrays.
6722LValue CodeGenFunction::EmitHLSLArrayAssignLValue(const BinaryOperator *E) {
6723 // Don't emit an LValue for the RHS because it might not be an LValue
6724 LValue LHS = EmitLValue(E: E->getLHS());
6725
6726 // If the RHS is a global resource array, copy all individual resources
6727 // into LHS.
6728 if (E->getRHS()->getType()->isHLSLResourceRecordArray())
6729 if (CGM.getHLSLRuntime().emitResourceArrayCopy(LHS, RHSExpr: E->getRHS(), CGF&: *this))
6730 return LHS;
6731
6732 // In C the RHS of an assignment operator is an RValue.
6733 // EmitAggregateAssign takes an LValue for the RHS. Instead we can call
6734 // EmitInitializationToLValue to emit an RValue into an LValue.
6735 EmitInitializationToLValue(E: E->getRHS(), LV: LHS);
6736 return LHS;
6737}
6738
6739LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E,
6740 llvm::CallBase **CallOrInvoke) {
6741 RValue RV = EmitCallExpr(E, ReturnValue: ReturnValueSlot(), CallOrInvoke);
6742
6743 if (!RV.isScalar())
6744 return MakeAddrLValue(Addr: RV.getAggregateAddress(), T: E->getType(),
6745 Source: AlignmentSource::Decl);
6746
6747 assert(E->getCallReturnType(getContext())->isReferenceType() &&
6748 "Can't have a scalar return unless the return type is a "
6749 "reference type!");
6750
6751 return MakeNaturalAlignPointeeAddrLValue(V: RV.getScalarVal(), T: E->getType());
6752}
6753
6754LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {
6755 // FIXME: This shouldn't require another copy.
6756 return EmitAggExprToLValue(E);
6757}
6758
6759LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
6760 assert(E->getType()->getAsCXXRecordDecl()->hasTrivialDestructor()
6761 && "binding l-value to type which needs a temporary");
6762 AggValueSlot Slot = CreateAggTemp(T: E->getType());
6763 EmitCXXConstructExpr(E, Dest: Slot);
6764 return MakeAddrLValue(Addr: Slot.getAddress(), T: E->getType(), Source: AlignmentSource::Decl);
6765}
6766
6767LValue
6768CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
6769 return MakeNaturalAlignRawAddrLValue(V: EmitCXXTypeidExpr(E), T: E->getType());
6770}
6771
6772Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) {
6773 return CGM.GetAddrOfMSGuidDecl(GD: E->getGuidDecl())
6774 .withElementType(ElemTy: ConvertType(T: E->getType()));
6775}
6776
6777LValue CodeGenFunction::EmitCXXUuidofLValue(const CXXUuidofExpr *E) {
6778 return MakeAddrLValue(Addr: EmitCXXUuidofExpr(E), T: E->getType(),
6779 Source: AlignmentSource::Decl);
6780}
6781
6782LValue
6783CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) {
6784 AggValueSlot Slot = CreateAggTemp(T: E->getType(), Name: "temp.lvalue");
6785 Slot.setExternallyDestructed();
6786 EmitAggExpr(E: E->getSubExpr(), AS: Slot);
6787 EmitCXXTemporary(Temporary: E->getTemporary(), TempType: E->getType(), Ptr: Slot.getAddress());
6788 return MakeAddrLValue(Addr: Slot.getAddress(), T: E->getType(), Source: AlignmentSource::Decl);
6789}
6790
6791LValue CodeGenFunction::EmitObjCMessageExprLValue(const ObjCMessageExpr *E) {
6792 RValue RV = EmitObjCMessageExpr(E);
6793
6794 if (!RV.isScalar())
6795 return MakeAddrLValue(Addr: RV.getAggregateAddress(), T: E->getType(),
6796 Source: AlignmentSource::Decl);
6797
6798 assert(E->getMethodDecl()->getReturnType()->isReferenceType() &&
6799 "Can't have a scalar return unless the return type is a "
6800 "reference type!");
6801
6802 return MakeNaturalAlignPointeeAddrLValue(V: RV.getScalarVal(), T: E->getType());
6803}
6804
6805LValue CodeGenFunction::EmitObjCSelectorLValue(const ObjCSelectorExpr *E) {
6806 Address V =
6807 CGM.getObjCRuntime().GetAddrOfSelector(CGF&: *this, Sel: E->getSelector());
6808 return MakeAddrLValue(Addr: V, T: E->getType(), Source: AlignmentSource::Decl);
6809}
6810
6811llvm::Value *CodeGenFunction::EmitIvarOffset(const ObjCInterfaceDecl *Interface,
6812 const ObjCIvarDecl *Ivar) {
6813 return CGM.getObjCRuntime().EmitIvarOffset(CGF&: *this, Interface, Ivar);
6814}
6815
6816llvm::Value *
6817CodeGenFunction::EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,
6818 const ObjCIvarDecl *Ivar) {
6819 llvm::Value *OffsetValue = EmitIvarOffset(Interface, Ivar);
6820 QualType PointerDiffType = getContext().getPointerDiffType();
6821 return Builder.CreateZExtOrTrunc(V: OffsetValue,
6822 DestTy: getTypes().ConvertType(T: PointerDiffType));
6823}
6824
6825LValue CodeGenFunction::EmitLValueForIvar(QualType ObjectTy,
6826 llvm::Value *BaseValue,
6827 const ObjCIvarDecl *Ivar,
6828 unsigned CVRQualifiers) {
6829 return CGM.getObjCRuntime().EmitObjCValueForIvar(CGF&: *this, ObjectTy, BaseValue,
6830 Ivar, CVRQualifiers);
6831}
6832
6833LValue CodeGenFunction::EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E) {
6834 // FIXME: A lot of the code below could be shared with EmitMemberExpr.
6835 llvm::Value *BaseValue = nullptr;
6836 const Expr *BaseExpr = E->getBase();
6837 Qualifiers BaseQuals;
6838 QualType ObjectTy;
6839 if (E->isArrow()) {
6840 BaseValue = EmitScalarExpr(E: BaseExpr);
6841 ObjectTy = BaseExpr->getType()->getPointeeType();
6842 BaseQuals = ObjectTy.getQualifiers();
6843 } else {
6844 LValue BaseLV = EmitLValue(E: BaseExpr);
6845 BaseValue = BaseLV.getPointer(CGF&: *this);
6846 ObjectTy = BaseExpr->getType();
6847 BaseQuals = ObjectTy.getQualifiers();
6848 }
6849
6850 LValue LV =
6851 EmitLValueForIvar(ObjectTy, BaseValue, Ivar: E->getDecl(),
6852 CVRQualifiers: BaseQuals.getCVRQualifiers());
6853 setObjCGCLValueClass(Ctx: getContext(), E, LV);
6854 return LV;
6855}
6856
6857LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {
6858 // Can only get l-value for message expression returning aggregate type
6859 RValue RV = EmitAnyExprToTemp(E);
6860 return MakeAddrLValue(Addr: RV.getAggregateAddress(), T: E->getType(),
6861 Source: AlignmentSource::Decl);
6862}
6863
6864RValue CodeGenFunction::EmitCall(QualType CalleeType,
6865 const CGCallee &OrigCallee, const CallExpr *E,
6866 ReturnValueSlot ReturnValue,
6867 llvm::Value *Chain,
6868 llvm::CallBase **CallOrInvoke,
6869 CGFunctionInfo const **ResolvedFnInfo) {
6870 // Get the actual function type. The callee type will always be a pointer to
6871 // function type or a block pointer type.
6872 assert(CalleeType->isFunctionPointerType() &&
6873 "Call must have function pointer type!");
6874
6875 const Decl *TargetDecl =
6876 OrigCallee.getAbstractInfo().getCalleeDecl().getDecl();
6877
6878 assert((!isa_and_present<FunctionDecl>(TargetDecl) ||
6879 !cast<FunctionDecl>(TargetDecl)->isImmediateFunction()) &&
6880 "trying to emit a call to an immediate function");
6881
6882 CalleeType = getContext().getCanonicalType(T: CalleeType);
6883
6884 auto PointeeType = cast<PointerType>(Val&: CalleeType)->getPointeeType();
6885
6886 CGCallee Callee = OrigCallee;
6887
6888 bool CFIUnchecked = CalleeType->hasPointeeToCFIUncheckedCalleeFunctionType();
6889
6890 if (SanOpts.has(K: SanitizerKind::Function) &&
6891 (!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl)) &&
6892 !isa<FunctionNoProtoType>(Val: PointeeType) && !CFIUnchecked) {
6893 if (llvm::Constant *PrefixSig =
6894 CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM)) {
6895 auto CheckOrdinal = SanitizerKind::SO_Function;
6896 auto CheckHandler = SanitizerHandler::FunctionTypeMismatch;
6897 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
6898 auto *TypeHash = getUBSanFunctionTypeHash(T: PointeeType);
6899
6900 llvm::Type *PrefixSigType = PrefixSig->getType();
6901 llvm::StructType *PrefixStructTy = llvm::StructType::get(
6902 Context&: CGM.getLLVMContext(), Elements: {PrefixSigType, Int32Ty}, /*isPacked=*/true);
6903
6904 llvm::Value *CalleePtr = Callee.getFunctionPointer();
6905 if (CGM.getCodeGenOpts().PointerAuth.FunctionPointers) {
6906 // Use raw pointer since we are using the callee pointer as data here.
6907 Address Addr =
6908 Address(CalleePtr, CalleePtr->getType(),
6909 CharUnits::fromQuantity(
6910 Quantity: CalleePtr->getPointerAlignment(DL: CGM.getDataLayout())),
6911 Callee.getPointerAuthInfo(), nullptr);
6912 CalleePtr = Addr.emitRawPointer(CGF&: *this);
6913 }
6914
6915 // On 32-bit Arm, the low bit of a function pointer indicates whether
6916 // it's using the Arm or Thumb instruction set. The actual first
6917 // instruction lives at the same address either way, so we must clear
6918 // that low bit before using the function address to find the prefix
6919 // structure.
6920 //
6921 // This applies to both Arm and Thumb target triples, because
6922 // either one could be used in an interworking context where it
6923 // might be passed function pointers of both types.
6924 llvm::Value *AlignedCalleePtr;
6925 if (CGM.getTriple().isARM() || CGM.getTriple().isThumb()) {
6926 AlignedCalleePtr = Builder.CreateIntrinsic(
6927 RetTy: CalleePtr->getType(), ID: llvm::Intrinsic::ptrmask,
6928 Args: {CalleePtr, llvm::ConstantInt::getSigned(Ty: IntPtrTy, V: ~1)});
6929 } else {
6930 AlignedCalleePtr = CalleePtr;
6931 }
6932
6933 llvm::Value *CalleePrefixStruct = AlignedCalleePtr;
6934 llvm::Value *CalleeSigPtr =
6935 Builder.CreateConstGEP2_32(Ty: PrefixStructTy, Ptr: CalleePrefixStruct, Idx0: -1, Idx1: 0);
6936 llvm::Value *CalleeSig =
6937 Builder.CreateAlignedLoad(Ty: PrefixSigType, Addr: CalleeSigPtr, Align: getIntAlign());
6938 llvm::Value *CalleeSigMatch = Builder.CreateICmpEQ(LHS: CalleeSig, RHS: PrefixSig);
6939
6940 llvm::BasicBlock *Cont = createBasicBlock(name: "cont");
6941 llvm::BasicBlock *TypeCheck = createBasicBlock(name: "typecheck");
6942 Builder.CreateCondBr(Cond: CalleeSigMatch, True: TypeCheck, False: Cont);
6943
6944 EmitBlock(BB: TypeCheck);
6945 llvm::Value *CalleeTypeHash = Builder.CreateAlignedLoad(
6946 Ty: Int32Ty,
6947 Addr: Builder.CreateConstGEP2_32(Ty: PrefixStructTy, Ptr: CalleePrefixStruct, Idx0: -1, Idx1: 1),
6948 Align: getPointerAlign());
6949 llvm::Value *CalleeTypeHashMatch =
6950 Builder.CreateICmpEQ(LHS: CalleeTypeHash, RHS: TypeHash);
6951 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc: E->getBeginLoc()),
6952 EmitCheckTypeDescriptor(T: CalleeType)};
6953 EmitCheck(Checked: std::make_pair(x&: CalleeTypeHashMatch, y&: CheckOrdinal), CheckHandler,
6954 StaticArgs: StaticData, DynamicArgs: {CalleePtr});
6955
6956 Builder.CreateBr(Dest: Cont);
6957 EmitBlock(BB: Cont);
6958 }
6959 }
6960
6961 const auto *FnType = cast<FunctionType>(Val&: PointeeType);
6962
6963 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
6964 FD && DeviceKernelAttr::isOpenCLSpelling(A: FD->getAttr<DeviceKernelAttr>()))
6965 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FnType);
6966
6967 // If we are checking indirect calls and this call is indirect, check that the
6968 // function pointer is a member of the bit set for the function type.
6969 if (SanOpts.has(K: SanitizerKind::CFIICall) &&
6970 (!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl)) && !CFIUnchecked) {
6971 auto CheckOrdinal = SanitizerKind::SO_CFIICall;
6972 auto CheckHandler = SanitizerHandler::CFICheckFail;
6973 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
6974 EmitSanitizerStatReport(SSK: llvm::SanStat_CFI_ICall);
6975
6976 llvm::Metadata *MD =
6977 CGM.CreateMetadataIdentifierForFnType(T: QualType(FnType, 0));
6978
6979 llvm::Value *TypeId = llvm::MetadataAsValue::get(Context&: getLLVMContext(), MD);
6980
6981 llvm::Value *CalleePtr = Callee.getFunctionPointer();
6982 llvm::Value *TypeTest = Builder.CreateCall(
6983 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::type_test), Args: {CalleePtr, TypeId});
6984
6985 auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
6986 llvm::Constant *StaticData[] = {
6987 llvm::ConstantInt::get(Ty: Int8Ty, V: CFITCK_ICall),
6988 EmitCheckSourceLocation(Loc: E->getBeginLoc()),
6989 EmitCheckTypeDescriptor(T: QualType(FnType, 0)),
6990 };
6991 if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) {
6992 EmitCfiSlowPathCheck(Ordinal: CheckOrdinal, Cond: TypeTest, TypeId: CrossDsoTypeId, Ptr: CalleePtr,
6993 StaticArgs: StaticData);
6994 } else {
6995 EmitCheck(Checked: std::make_pair(x&: TypeTest, y&: CheckOrdinal), CheckHandler,
6996 StaticArgs: StaticData, DynamicArgs: {CalleePtr, llvm::UndefValue::get(T: IntPtrTy)});
6997 }
6998 }
6999
7000 CallArgList Args;
7001 if (Chain)
7002 Args.add(rvalue: RValue::get(V: Chain), type: CGM.getContext().VoidPtrTy);
7003
7004 // C++17 requires that we evaluate arguments to a call using assignment syntax
7005 // right-to-left, and that we evaluate arguments to certain other operators
7006 // left-to-right. Note that we allow this to override the order dictated by
7007 // the calling convention on the MS ABI, which means that parameter
7008 // destruction order is not necessarily reverse construction order.
7009 // FIXME: Revisit this based on C++ committee response to unimplementability.
7010 EvaluationOrder Order = EvaluationOrder::Default;
7011 bool StaticOperator = false;
7012 if (auto *OCE = dyn_cast<CXXOperatorCallExpr>(Val: E)) {
7013 if (OCE->isAssignmentOp())
7014 Order = EvaluationOrder::ForceRightToLeft;
7015 else {
7016 switch (OCE->getOperator()) {
7017 case OO_LessLess:
7018 case OO_GreaterGreater:
7019 case OO_AmpAmp:
7020 case OO_PipePipe:
7021 case OO_Comma:
7022 case OO_ArrowStar:
7023 Order = EvaluationOrder::ForceLeftToRight;
7024 break;
7025 default:
7026 break;
7027 }
7028 }
7029
7030 if (const auto *MD =
7031 dyn_cast_if_present<CXXMethodDecl>(Val: OCE->getCalleeDecl());
7032 MD && MD->isStatic())
7033 StaticOperator = true;
7034 }
7035
7036 auto Arguments = E->arguments();
7037 if (StaticOperator) {
7038 // If we're calling a static operator, we need to emit the object argument
7039 // and ignore it.
7040 EmitIgnoredExpr(E: E->getArg(Arg: 0));
7041 Arguments = drop_begin(RangeOrContainer&: Arguments, N: 1);
7042 }
7043 EmitCallArgs(Args, Prototype: dyn_cast<FunctionProtoType>(Val: FnType), ArgRange: Arguments,
7044 AC: E->getDirectCallee(), /*ParamsToSkip=*/0, Order);
7045
7046 const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(
7047 Args, Ty: FnType, /*ChainCall=*/Chain);
7048
7049 if (ResolvedFnInfo)
7050 *ResolvedFnInfo = &FnInfo;
7051
7052 // HIP function pointer contains kernel handle when it is used in triple
7053 // chevron. The kernel stub needs to be loaded from kernel handle and used
7054 // as callee.
7055 if (CGM.getLangOpts().HIP && !CGM.getLangOpts().CUDAIsDevice &&
7056 isa<CUDAKernelCallExpr>(Val: E) &&
7057 (!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl))) {
7058 llvm::Value *Handle = Callee.getFunctionPointer();
7059 auto *Stub = Builder.CreateLoad(
7060 Addr: Address(Handle, Handle->getType(), CGM.getPointerAlign()));
7061 Callee.setFunctionPointer(Stub);
7062 }
7063
7064 // Insert function pointer lookup if this is a target call
7065 //
7066 // This is used for the indirect function case, virtual function case is
7067 // handled in ItaniumCXXABI.cpp
7068 if (getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU() &&
7069 (!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl))) {
7070 const Expr *CalleeExpr = E->getCallee()->IgnoreParenImpCasts();
7071 const DeclRefExpr *DRE = nullptr;
7072 while (CalleeExpr) {
7073 if ((DRE = dyn_cast<DeclRefExpr>(Val: CalleeExpr)))
7074 break;
7075 if (const auto *ME = dyn_cast<MemberExpr>(Val: CalleeExpr))
7076 CalleeExpr = ME->getBase()->IgnoreParenImpCasts();
7077 else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: CalleeExpr))
7078 CalleeExpr = ASE->getBase()->IgnoreParenImpCasts();
7079 else
7080 break;
7081 }
7082
7083 const auto *VD = DRE ? dyn_cast<VarDecl>(Val: DRE->getDecl()) : nullptr;
7084 if (VD && VD->hasAttr<OMPTargetIndirectCallAttr>()) {
7085 auto *PtrTy = CGM.VoidPtrTy;
7086 llvm::Type *RtlFnArgs[] = {PtrTy};
7087 llvm::FunctionCallee DeviceRtlFn = CGM.CreateRuntimeFunction(
7088 Ty: llvm::FunctionType::get(Result: PtrTy, Params: RtlFnArgs, isVarArg: false),
7089 Name: "__llvm_omp_indirect_call_lookup");
7090 llvm::Value *Func = Callee.getFunctionPointer();
7091 llvm::Type *BackupTy = Func->getType();
7092 Func = Builder.CreatePointerBitCastOrAddrSpaceCast(V: Func, DestTy: PtrTy);
7093 Func = EmitRuntimeCall(callee: DeviceRtlFn, args: {Func});
7094 Func = Builder.CreatePointerBitCastOrAddrSpaceCast(V: Func, DestTy: BackupTy);
7095 Callee.setFunctionPointer(Func);
7096 }
7097 }
7098
7099 llvm::CallBase *LocalCallOrInvoke = nullptr;
7100 RValue Call = EmitCall(CallInfo: FnInfo, Callee, ReturnValue, Args, CallOrInvoke: &LocalCallOrInvoke,
7101 IsMustTail: E == MustTailCall, Loc: E->getExprLoc());
7102
7103 if (auto *CalleeDecl = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
7104 if (CalleeDecl->hasAttr<RestrictAttr>() ||
7105 CalleeDecl->hasAttr<MallocSpanAttr>() ||
7106 CalleeDecl->hasAttr<AllocSizeAttr>()) {
7107 // Function has 'malloc' (aka. 'restrict') or 'alloc_size' attribute.
7108 if (SanOpts.has(K: SanitizerKind::AllocToken)) {
7109 // Set !alloc_token metadata.
7110 EmitAllocToken(CB: LocalCallOrInvoke, E);
7111 }
7112 }
7113 }
7114 if (CallOrInvoke)
7115 *CallOrInvoke = LocalCallOrInvoke;
7116
7117 return Call;
7118}
7119
7120LValue CodeGenFunction::
7121EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) {
7122 Address BaseAddr = Address::invalid();
7123 if (E->getOpcode() == BO_PtrMemI) {
7124 BaseAddr = EmitPointerWithAlignment(E: E->getLHS());
7125 } else {
7126 BaseAddr = EmitLValue(E: E->getLHS()).getAddress();
7127 }
7128
7129 llvm::Value *OffsetV = EmitScalarExpr(E: E->getRHS());
7130 const auto *MPT = E->getRHS()->getType()->castAs<MemberPointerType>();
7131
7132 LValueBaseInfo BaseInfo;
7133 TBAAAccessInfo TBAAInfo;
7134 bool IsInBounds = !getLangOpts().PointerOverflowDefined &&
7135 !isUnderlyingBasePointerConstantNull(E: E->getLHS());
7136 Address MemberAddr = EmitCXXMemberDataPointerAddress(
7137 E, base: BaseAddr, memberPtr: OffsetV, memberPtrType: MPT, IsInBounds, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
7138
7139 return MakeAddrLValue(Addr: MemberAddr, T: MPT->getPointeeType(), BaseInfo, TBAAInfo);
7140}
7141
7142/// Given the address of a temporary variable, produce an r-value of
7143/// its type.
7144RValue CodeGenFunction::convertTempToRValue(Address addr,
7145 QualType type,
7146 SourceLocation loc) {
7147 LValue lvalue = MakeAddrLValue(Addr: addr, T: type, Source: AlignmentSource::Decl);
7148 switch (getEvaluationKind(T: type)) {
7149 case TEK_Complex:
7150 return RValue::getComplex(C: EmitLoadOfComplex(src: lvalue, loc));
7151 case TEK_Aggregate:
7152 return lvalue.asAggregateRValue();
7153 case TEK_Scalar:
7154 return RValue::get(V: EmitLoadOfScalar(lvalue, Loc: loc));
7155 }
7156 llvm_unreachable("bad evaluation kind");
7157}
7158
7159void CodeGenFunction::SetFPAccuracy(llvm::Value *Val, float Accuracy) {
7160 assert(Val->getType()->isFPOrFPVectorTy());
7161 if (Accuracy == 0.0 || !isa<llvm::Instruction>(Val))
7162 return;
7163
7164 llvm::MDBuilder MDHelper(getLLVMContext());
7165 llvm::MDNode *Node = MDHelper.createFPMath(Accuracy);
7166
7167 cast<llvm::Instruction>(Val)->setMetadata(KindID: llvm::LLVMContext::MD_fpmath, Node);
7168}
7169
7170void CodeGenFunction::SetSqrtFPAccuracy(llvm::Value *Val) {
7171 llvm::Type *EltTy = Val->getType()->getScalarType();
7172 if (!EltTy->isFloatTy() && !EltTy->isHalfTy())
7173 return;
7174
7175 if ((getLangOpts().OpenCL &&
7176 !CGM.getCodeGenOpts().OpenCLCorrectlyRoundedDivSqrt) ||
7177 (getLangOpts().HIP && getLangOpts().CUDAIsDevice &&
7178 !CGM.getCodeGenOpts().HIPCorrectlyRoundedDivSqrt)) {
7179 // OpenCL v1.1 s7.4: minimum accuracy of single precision sqrt is 3 ulp.
7180 // OpenCL v3.0 s7.4: minimum accuracy of half precision sqrt is 1.5 ulp.
7181 //
7182 // OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt
7183 // build option allows an application to specify that single precision
7184 // floating-point divide (x/y and 1/x) and sqrt used in the program
7185 // source are correctly rounded.
7186 //
7187 // TODO: CUDA has a prec-sqrt flag
7188 SetFPAccuracy(Val, Accuracy: EltTy->isFloatTy() ? 3.0f : 1.5f);
7189 }
7190}
7191
7192void CodeGenFunction::SetDivFPAccuracy(llvm::Value *Val) {
7193 llvm::Type *EltTy = Val->getType()->getScalarType();
7194 if (!EltTy->isFloatTy() && !EltTy->isHalfTy())
7195 return;
7196
7197 if ((getLangOpts().OpenCL &&
7198 !CGM.getCodeGenOpts().OpenCLCorrectlyRoundedDivSqrt) ||
7199 (getLangOpts().HIP && getLangOpts().CUDAIsDevice &&
7200 !CGM.getCodeGenOpts().HIPCorrectlyRoundedDivSqrt)) {
7201 // OpenCL v1.1 s7.4: minimum accuracy of single precision / is 2.5 ulp.
7202 // OpenCL v3.0 s7.4: minimum accuracy of half precision / is 1 ulp.
7203 //
7204 // OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt
7205 // build option allows an application to specify that single precision
7206 // floating-point divide (x/y and 1/x) and sqrt used in the program
7207 // source are correctly rounded.
7208 //
7209 // TODO: CUDA has a prec-div flag
7210 SetFPAccuracy(Val, Accuracy: EltTy->isFloatTy() ? 2.5f : 1.f);
7211 }
7212}
7213
7214namespace {
7215 struct LValueOrRValue {
7216 LValue LV;
7217 RValue RV;
7218 };
7219}
7220
7221static LValueOrRValue emitPseudoObjectExpr(CodeGenFunction &CGF,
7222 const PseudoObjectExpr *E,
7223 bool forLValue,
7224 AggValueSlot slot) {
7225 SmallVector<CodeGenFunction::OpaqueValueMappingData, 4> opaques;
7226
7227 // Find the result expression, if any.
7228 const Expr *resultExpr = E->getResultExpr();
7229 LValueOrRValue result;
7230
7231 for (PseudoObjectExpr::const_semantics_iterator
7232 i = E->semantics_begin(), e = E->semantics_end(); i != e; ++i) {
7233 const Expr *semantic = *i;
7234
7235 // If this semantic expression is an opaque value, bind it
7236 // to the result of its source expression.
7237 if (const auto *ov = dyn_cast<OpaqueValueExpr>(Val: semantic)) {
7238 // Skip unique OVEs.
7239 if (ov->isUnique()) {
7240 assert(ov != resultExpr &&
7241 "A unique OVE cannot be used as the result expression");
7242 continue;
7243 }
7244
7245 // If this is the result expression, we may need to evaluate
7246 // directly into the slot.
7247 typedef CodeGenFunction::OpaqueValueMappingData OVMA;
7248 OVMA opaqueData;
7249 if (ov == resultExpr && ov->isPRValue() && !forLValue &&
7250 CodeGenFunction::hasAggregateEvaluationKind(T: ov->getType())) {
7251 CGF.EmitAggExpr(E: ov->getSourceExpr(), AS: slot);
7252 LValue LV = CGF.MakeAddrLValue(Addr: slot.getAddress(), T: ov->getType(),
7253 Source: AlignmentSource::Decl);
7254 opaqueData = OVMA::bind(CGF, ov, lv: LV);
7255 result.RV = slot.asRValue();
7256
7257 // Otherwise, emit as normal.
7258 } else {
7259 opaqueData = OVMA::bind(CGF, ov, e: ov->getSourceExpr());
7260
7261 // If this is the result, also evaluate the result now.
7262 if (ov == resultExpr) {
7263 if (forLValue)
7264 result.LV = CGF.EmitLValue(E: ov);
7265 else
7266 result.RV = CGF.EmitAnyExpr(E: ov, aggSlot: slot);
7267 }
7268 }
7269
7270 opaques.push_back(Elt: opaqueData);
7271
7272 // Otherwise, if the expression is the result, evaluate it
7273 // and remember the result.
7274 } else if (semantic == resultExpr) {
7275 if (forLValue)
7276 result.LV = CGF.EmitLValue(E: semantic);
7277 else
7278 result.RV = CGF.EmitAnyExpr(E: semantic, aggSlot: slot);
7279
7280 // Otherwise, evaluate the expression in an ignored context.
7281 } else {
7282 CGF.EmitIgnoredExpr(E: semantic);
7283 }
7284 }
7285
7286 // Unbind all the opaques now.
7287 for (CodeGenFunction::OpaqueValueMappingData &opaque : opaques)
7288 opaque.unbind(CGF);
7289
7290 return result;
7291}
7292
7293RValue CodeGenFunction::EmitPseudoObjectRValue(const PseudoObjectExpr *E,
7294 AggValueSlot slot) {
7295 return emitPseudoObjectExpr(CGF&: *this, E, forLValue: false, slot).RV;
7296}
7297
7298LValue CodeGenFunction::EmitPseudoObjectLValue(const PseudoObjectExpr *E) {
7299 return emitPseudoObjectExpr(CGF&: *this, E, forLValue: true, slot: AggValueSlot::ignored()).LV;
7300}
7301
7302void CodeGenFunction::FlattenAccessAndTypeLValue(
7303 LValue Val, SmallVectorImpl<LValue> &AccessList) {
7304
7305 llvm::SmallVector<
7306 std::tuple<LValue, QualType, llvm::SmallVector<llvm::Value *, 4>>, 16>
7307 WorkList;
7308 llvm::IntegerType *IdxTy = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: 32);
7309 WorkList.push_back(Elt: {Val, Val.getType(), {llvm::ConstantInt::get(Ty: IdxTy, V: 0)}});
7310
7311 while (!WorkList.empty()) {
7312 auto [LVal, T, IdxList] = WorkList.pop_back_val();
7313 T = T.getCanonicalType().getUnqualifiedType();
7314 if (const auto *CAT = dyn_cast<ConstantArrayType>(Val&: T)) {
7315 uint64_t Size = CAT->getZExtSize();
7316 for (int64_t I = Size - 1; I > -1; I--) {
7317 llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList;
7318 IdxListCopy.push_back(Elt: llvm::ConstantInt::get(Ty: IdxTy, V: I));
7319 WorkList.emplace_back(Args&: LVal, Args: CAT->getElementType(), Args&: IdxListCopy);
7320 }
7321 } else if (const auto *RT = dyn_cast<RecordType>(Val&: T)) {
7322 const RecordDecl *Record = RT->getDecl()->getDefinitionOrSelf();
7323 assert(!Record->isUnion() && "Union types not supported in flat cast.");
7324
7325 const CXXRecordDecl *CXXD = dyn_cast<CXXRecordDecl>(Val: Record);
7326
7327 llvm::SmallVector<
7328 std::tuple<LValue, QualType, llvm::SmallVector<llvm::Value *, 4>>, 16>
7329 ReverseList;
7330 if (CXXD && CXXD->isStandardLayout())
7331 Record = CXXD->getStandardLayoutBaseWithFields();
7332
7333 // deal with potential base classes
7334 if (CXXD && !CXXD->isStandardLayout()) {
7335 if (CXXD->getNumBases() > 0) {
7336 assert(CXXD->getNumBases() == 1 &&
7337 "HLSL doesn't support multiple inheritance.");
7338 auto Base = CXXD->bases_begin();
7339 llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList;
7340 IdxListCopy.push_back(Elt: llvm::ConstantInt::get(
7341 Ty: IdxTy, V: 0)); // base struct should be at index zero
7342 ReverseList.emplace_back(Args&: LVal, Args: Base->getType(), Args&: IdxListCopy);
7343 }
7344 }
7345
7346 const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(Record);
7347
7348 llvm::Type *LLVMT = ConvertTypeForMem(T);
7349 CharUnits Align = getContext().getTypeAlignInChars(T);
7350 LValue RLValue;
7351 bool createdGEP = false;
7352 for (auto *FD : Record->fields()) {
7353 if (FD->isBitField()) {
7354 if (FD->isUnnamedBitField())
7355 continue;
7356 if (!createdGEP) {
7357 createdGEP = true;
7358 Address GEP = Builder.CreateInBoundsGEP(Addr: LVal.getAddress(), IdxList,
7359 ElementType: LLVMT, Align, Name: "gep");
7360 RLValue = MakeAddrLValue(Addr: GEP, T);
7361 }
7362 LValue FieldLVal = EmitLValueForField(base: RLValue, field: FD, IsInBounds: true);
7363 ReverseList.push_back(Elt: {FieldLVal, FD->getType(), {}});
7364 } else {
7365 llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList;
7366 IdxListCopy.push_back(
7367 Elt: llvm::ConstantInt::get(Ty: IdxTy, V: Layout.getLLVMFieldNo(FD)));
7368 ReverseList.emplace_back(Args&: LVal, Args: FD->getType(), Args&: IdxListCopy);
7369 }
7370 }
7371
7372 std::reverse(first: ReverseList.begin(), last: ReverseList.end());
7373 llvm::append_range(C&: WorkList, R&: ReverseList);
7374 } else if (const auto *VT = dyn_cast<VectorType>(Val&: T)) {
7375 llvm::Type *LLVMT = ConvertTypeForMem(T);
7376 CharUnits Align = getContext().getTypeAlignInChars(T);
7377 Address GEP = Builder.CreateInBoundsGEP(Addr: LVal.getAddress(), IdxList, ElementType: LLVMT,
7378 Align, Name: "vector.gep");
7379 LValue Base = MakeAddrLValue(Addr: GEP, T);
7380 for (unsigned I = 0, E = VT->getNumElements(); I < E; I++) {
7381 llvm::Constant *Idx = llvm::ConstantInt::get(Ty: IdxTy, V: I);
7382 LValue LV =
7383 LValue::MakeVectorElt(vecAddress: Base.getAddress(), Idx, type: VT->getElementType(),
7384 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
7385 AccessList.emplace_back(Args&: LV);
7386 }
7387 } else if (const auto *MT = dyn_cast<ConstantMatrixType>(Val&: T)) {
7388 // Matrices are represented as flat arrays in memory, but has a vector
7389 // value type. So we use ConvertMatrixAddress to convert the address from
7390 // array to vector, and extract elements similar to the vector case above.
7391 // The matrix elements are iterated over in row-major order regardless of
7392 // the memory layout of the matrix.
7393 llvm::Type *LLVMT = ConvertTypeForMem(T);
7394 CharUnits Align = getContext().getTypeAlignInChars(T);
7395 Address GEP = Builder.CreateInBoundsGEP(Addr: LVal.getAddress(), IdxList, ElementType: LLVMT,
7396 Align, Name: "matrix.gep");
7397 LValue Base = MakeAddrLValue(Addr: GEP, T);
7398 Address MatAddr = MaybeConvertMatrixAddress(Addr: Base.getAddress(), CGF&: *this);
7399 unsigned NumRows = MT->getNumRows();
7400 unsigned NumCols = MT->getNumColumns();
7401 bool IsMatrixRowMajor = getLangOpts().getDefaultMatrixMemoryLayout() ==
7402 LangOptions::MatrixMemoryLayout::MatrixRowMajor;
7403 llvm::MatrixBuilder MB(Builder);
7404 for (unsigned Row = 0; Row < MT->getNumRows(); Row++) {
7405 for (unsigned Col = 0; Col < MT->getNumColumns(); Col++) {
7406 llvm::Value *RowIdx = llvm::ConstantInt::get(Ty: IdxTy, V: Row);
7407 llvm::Value *ColIdx = llvm::ConstantInt::get(Ty: IdxTy, V: Col);
7408 llvm::Value *Idx = MB.CreateIndex(RowIdx, ColumnIdx: ColIdx, NumRows, NumCols,
7409 IsMatrixRowMajor);
7410 LValue LV =
7411 LValue::MakeMatrixElt(matAddress: MatAddr, Idx, type: MT->getElementType(),
7412 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
7413 AccessList.emplace_back(Args&: LV);
7414 }
7415 }
7416 } else { // a scalar/builtin type
7417 if (!IdxList.empty()) {
7418 llvm::Type *LLVMT = ConvertTypeForMem(T);
7419 CharUnits Align = getContext().getTypeAlignInChars(T);
7420 Address GEP = Builder.CreateInBoundsGEP(Addr: LVal.getAddress(), IdxList,
7421 ElementType: LLVMT, Align, Name: "gep");
7422 AccessList.emplace_back(Args: MakeAddrLValue(Addr: GEP, T));
7423 } else // must be a bitfield we already created an lvalue for
7424 AccessList.emplace_back(Args&: LVal);
7425 }
7426 }
7427}
7428