1//===--- CGExpr.cpp - Emit LLVM Code from Expressions ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Expr nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ABIInfoImpl.h"
14#include "CGCUDARuntime.h"
15#include "CGCXXABI.h"
16#include "CGCall.h"
17#include "CGCleanup.h"
18#include "CGDebugInfo.h"
19#include "CGHLSLRuntime.h"
20#include "CGObjCRuntime.h"
21#include "CGOpenMPRuntime.h"
22#include "CGRecordLayout.h"
23#include "CodeGenFunction.h"
24#include "CodeGenModule.h"
25#include "CodeGenPGO.h"
26#include "ConstantEmitter.h"
27#include "TargetInfo.h"
28#include "clang/AST/ASTContext.h"
29#include "clang/AST/ASTLambda.h"
30#include "clang/AST/Attr.h"
31#include "clang/AST/DeclObjC.h"
32#include "clang/AST/Expr.h"
33#include "clang/AST/InferAlloc.h"
34#include "clang/AST/MatrixUtils.h"
35#include "clang/AST/NSAPI.h"
36#include "clang/AST/ParentMapContext.h"
37#include "clang/AST/StmtVisitor.h"
38#include "clang/Basic/Builtins.h"
39#include "clang/Basic/CodeGenOptions.h"
40#include "clang/Basic/Module.h"
41#include "clang/Basic/SourceManager.h"
42#include "clang/CodeGenUtils/ExprUtils.h"
43#include "clang/CodeGenUtils/RecordLayoutUtils.h"
44#include "clang/CodeGenUtils/TargetUtils.h"
45#include "llvm/ADT/STLExtras.h"
46#include "llvm/ADT/ScopeExit.h"
47#include "llvm/ADT/StringExtras.h"
48#include "llvm/IR/Constants.h"
49#include "llvm/IR/DataLayout.h"
50#include "llvm/IR/Intrinsics.h"
51#include "llvm/IR/IntrinsicsWebAssembly.h"
52#include "llvm/IR/LLVMContext.h"
53#include "llvm/IR/MDBuilder.h"
54#include "llvm/IR/MatrixBuilder.h"
55#include "llvm/Support/ConvertUTF.h"
56#include "llvm/Support/Endian.h"
57#include "llvm/Support/MathExtras.h"
58#include "llvm/Support/Path.h"
59#include "llvm/Support/SaveAndRestore.h"
60#include "llvm/Support/xxhash.h"
61#include "llvm/Transforms/Utils/SanitizerStats.h"
62
63#include <numeric>
64#include <optional>
65#include <string>
66
67using namespace clang;
68using namespace CodeGen;
69
70namespace clang {
71// TODO: consider deprecating ClSanitizeGuardChecks; functionality is subsumed
72// by -fsanitize-skip-hot-cutoff
73llvm::cl::opt<bool> ClSanitizeGuardChecks(
74 "ubsan-guard-checks",
75 llvm::cl::desc("Guard UBSAN checks with `llvm.allow.ubsan.check()`."));
76
77} // namespace clang
78
79//===--------------------------------------------------------------------===//
80// Defines for metadata
81//===--------------------------------------------------------------------===//
82
83// Those values are crucial to be the SAME as in ubsan runtime library.
84enum VariableTypeDescriptorKind : uint16_t {
85 /// An integer type.
86 TK_Integer = 0x0000,
87 /// A floating-point type.
88 TK_Float = 0x0001,
89 /// An _BitInt(N) type.
90 TK_BitInt = 0x0002,
91 /// Any other type. The value representation is unspecified.
92 TK_Unknown = 0xffff
93};
94
95//===--------------------------------------------------------------------===//
96// Miscellaneous Helper Methods
97//===--------------------------------------------------------------------===//
98
99static llvm::StringRef GetUBSanTrapForHandler(SanitizerHandler ID) {
100 switch (ID) {
101#define SANITIZER_CHECK(Enum, Name, Version, Msg) \
102 case SanitizerHandler::Enum: \
103 return Msg;
104 LIST_SANITIZER_CHECKS
105#undef SANITIZER_CHECK
106 }
107 llvm_unreachable("unhandled switch case");
108}
109
110/// CreateTempAlloca - This creates a alloca and inserts it into the entry
111/// block.
112RawAddress
113CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits Align,
114 const Twine &Name,
115 llvm::Value *ArraySize) {
116 if (getLangOpts().EmitLogicalPointer) {
117 auto Alloca = Builder.CreateStructuredAlloca(BaseType: Ty, Name);
118 return RawAddress(Alloca, Ty, Align, KnownNonNull);
119 }
120
121 auto *Alloca = CreateTempAlloca(Ty, Name, ArraySize);
122 Alloca->setAlignment(Align.getAsAlign());
123 return RawAddress(Alloca, Ty, Align, KnownNonNull);
124}
125
126RawAddress CodeGenFunction::MaybeCastStackAddressSpace(RawAddress Alloca,
127 LangAS DestLangAS,
128 llvm::Value *ArraySize) {
129
130 llvm::Value *V = Alloca.getPointer();
131 // Alloca always returns a pointer in alloca address space, which may
132 // be different from the type defined by the language. For example,
133 // in C++ the auto variables are in the default address space. Therefore
134 // cast alloca to the default address space when necessary.
135
136 unsigned DestAddrSpace = getContext().getTargetAddressSpace(AS: DestLangAS);
137 if (DestAddrSpace != Alloca.getAddressSpace()) {
138 llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
139 // When ArraySize is nullptr, alloca is inserted at AllocaInsertPt,
140 // otherwise alloca is inserted at the current insertion point of the
141 // builder.
142 if (!ArraySize)
143 Builder.SetInsertPoint(getPostAllocaInsertPoint());
144 V = performAddrSpaceCast(Src: V, DestTy: Builder.getPtrTy(AddrSpace: DestAddrSpace));
145 }
146
147 return RawAddress(V, Alloca.getElementType(), Alloca.getAlignment(),
148 KnownNonNull);
149}
150
151RawAddress CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, LangAS DestLangAS,
152 CharUnits Align, const Twine &Name,
153 llvm::Value *ArraySize,
154 RawAddress *AllocaAddr) {
155 RawAddress Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize);
156 if (AllocaAddr)
157 *AllocaAddr = Alloca;
158 return MaybeCastStackAddressSpace(Alloca, DestLangAS, ArraySize);
159}
160
161/// CreateTempAlloca - This creates an alloca and inserts it into the entry
162/// block if \p ArraySize is nullptr, otherwise inserts it at the current
163/// insertion point of the builder.
164llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
165 const Twine &Name,
166 llvm::Value *ArraySize) {
167 llvm::AllocaInst *Alloca;
168 if (ArraySize)
169 Alloca = Builder.CreateAlloca(Ty, ArraySize, Name);
170 else
171 Alloca =
172 new llvm::AllocaInst(Ty, CGM.getDataLayout().getAllocaAddrSpace(),
173 ArraySize, Name, AllocaInsertPt->getIterator());
174 if (SanOpts.Mask & SanitizerKind::Address) {
175 Alloca->addAnnotationMetadata(Annotations: {"alloca_name_altered", Name.str()});
176 }
177 if (Allocas) {
178 Allocas->Add(I: Alloca);
179 }
180 return Alloca;
181}
182
183/// CreateDefaultAlignTempAlloca - This creates an alloca with the
184/// default alignment of the corresponding LLVM type, which is *not*
185/// guaranteed to be related in any way to the expected alignment of
186/// an AST type that might have been lowered to Ty.
187RawAddress CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty,
188 const Twine &Name) {
189 CharUnits Align =
190 CharUnits::fromQuantity(Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty));
191 return CreateTempAlloca(Ty, DestLangAS: LangAS::Default, Align, Name);
192}
193
194RawAddress CodeGenFunction::CreateIRTempWithoutCast(QualType Ty,
195 const Twine &Name) {
196 CharUnits Align = getContext().getTypeAlignInChars(T: Ty);
197 return CreateTempAllocaWithoutCast(Ty: ConvertType(T: Ty), Align, Name, ArraySize: nullptr);
198}
199
200RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name,
201 RawAddress *Alloca) {
202 // FIXME: Should we prefer the preferred type alignment here?
203 return CreateMemTemp(T: Ty, Align: getContext().getTypeAlignInChars(T: Ty), Name, Alloca);
204}
205
206RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align,
207 const Twine &Name,
208 RawAddress *Alloca) {
209 RawAddress Result =
210 CreateTempAlloca(Ty: ConvertTypeForMem(T: Ty), DestLangAS: Ty.getAddressSpace(), Align, Name,
211 /*ArraySize=*/nullptr, AllocaAddr: Alloca);
212
213 if (Ty->isConstantMatrixType()) {
214 auto *ArrayTy = cast<llvm::ArrayType>(Val: Result.getElementType());
215 auto *ArrayElementTy = ArrayTy->getElementType();
216 auto ArrayElements = ArrayTy->getNumElements();
217 if (getContext().getLangOpts().HLSL) {
218 auto *VectorTy = cast<llvm::FixedVectorType>(Val: ArrayElementTy);
219 ArrayElementTy = VectorTy->getElementType();
220 ArrayElements *= VectorTy->getNumElements();
221 }
222 auto *VectorTy = llvm::FixedVectorType::get(ElementType: ArrayElementTy, NumElts: ArrayElements);
223
224 Result = Address(Result.getPointer(), VectorTy, Result.getAlignment(),
225 KnownNonNull);
226 }
227 return Result;
228}
229
230RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
231 CharUnits Align,
232 const Twine &Name) {
233 return CreateTempAllocaWithoutCast(Ty: ConvertTypeForMem(T: Ty), Align, Name);
234}
235
236RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
237 const Twine &Name) {
238 return CreateMemTempWithoutCast(Ty, Align: getContext().getTypeAlignInChars(T: Ty),
239 Name);
240}
241
242/// EvaluateExprAsBool - Perform the usual unary conversions on the specified
243/// expression and compare the result against zero, returning an Int1Ty value.
244llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
245 PGO->setCurrentStmt(E);
246 if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {
247 llvm::Value *MemPtr = EmitScalarExpr(E);
248 return CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF&: *this, MemPtr, MPT);
249 }
250
251 QualType BoolTy = getContext().BoolTy;
252 SourceLocation Loc = E->getExprLoc();
253 CGFPOptionsRAII FPOptsRAII(*this, E);
254 if (!E->getType()->isAnyComplexType())
255 return EmitScalarConversion(Src: EmitScalarExpr(E), SrcTy: E->getType(), DstTy: BoolTy, Loc);
256
257 return EmitComplexToScalarConversion(Src: EmitComplexExpr(E), SrcTy: E->getType(), DstTy: BoolTy,
258 Loc);
259}
260
261/// EmitIgnoredExpr - Emit code to compute the specified expression,
262/// ignoring the result.
263void CodeGenFunction::EmitIgnoredExpr(const Expr *E) {
264 if (E->isPRValue())
265 return (void)EmitAnyExpr(E, aggSlot: AggValueSlot::ignored(), ignoreResult: true);
266
267 // if this is a bitfield-resulting conditional operator, we can special case
268 // emit this. The normal 'EmitLValue' version of this is particularly
269 // difficult to codegen for, since creating a single "LValue" for two
270 // different sized arguments here is not particularly doable.
271 if (const auto *CondOp = dyn_cast<AbstractConditionalOperator>(
272 Val: E->IgnoreParenNoopCasts(Ctx: getContext()))) {
273 if (CondOp->getObjectKind() == OK_BitField)
274 return EmitIgnoredConditionalOperator(E: CondOp);
275 }
276
277 // Just emit it as an l-value and drop the result.
278 EmitLValue(E);
279}
280
281/// EmitAnyExpr - Emit code to compute the specified expression which
282/// can have any type. The result is returned as an RValue struct.
283/// If this is an aggregate expression, AggSlot indicates where the
284/// result should be returned.
285RValue CodeGenFunction::EmitAnyExpr(const Expr *E,
286 AggValueSlot aggSlot,
287 bool ignoreResult) {
288 switch (getEvaluationKind(T: E->getType())) {
289 case TEK_Scalar:
290 return RValue::get(V: EmitScalarExpr(E, IgnoreResultAssign: ignoreResult));
291 case TEK_Complex:
292 return RValue::getComplex(C: EmitComplexExpr(E, IgnoreReal: ignoreResult, IgnoreImag: ignoreResult));
293 case TEK_Aggregate:
294 if (!ignoreResult && aggSlot.isIgnored())
295 aggSlot = CreateAggTemp(T: E->getType().getUnqualifiedType(), Name: "agg-temp");
296 EmitAggExpr(E, AS: aggSlot);
297 return aggSlot.asRValue();
298 }
299 llvm_unreachable("bad evaluation kind");
300}
301
302/// EmitAnyExprToTemp - Similar to EmitAnyExpr(), however, the result will
303/// always be accessible even if no aggregate location is provided.
304RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E) {
305 AggValueSlot AggSlot = AggValueSlot::ignored();
306
307 if (hasAggregateEvaluationKind(T: E->getType()))
308 AggSlot = CreateAggTemp(T: E->getType(), Name: "agg.tmp");
309 return EmitAnyExpr(E, aggSlot: AggSlot);
310}
311
312/// EmitAnyExprToMem - Evaluate an expression into a given memory
313/// location.
314void CodeGenFunction::EmitAnyExprToMem(const Expr *E,
315 Address Location,
316 Qualifiers Quals,
317 bool IsInit) {
318 // FIXME: This function should take an LValue as an argument.
319 switch (getEvaluationKind(T: E->getType())) {
320 case TEK_Complex:
321 EmitComplexExprIntoLValue(E, dest: MakeAddrLValue(Addr: Location, T: E->getType()),
322 /*isInit*/ false);
323 return;
324
325 case TEK_Aggregate: {
326 EmitAggExpr(E, AS: AggValueSlot::forAddr(addr: Location, quals: Quals,
327 isDestructed: AggValueSlot::IsDestructed_t(IsInit),
328 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
329 isAliased: AggValueSlot::IsAliased_t(!IsInit),
330 mayOverlap: AggValueSlot::MayOverlap));
331 return;
332 }
333
334 case TEK_Scalar: {
335 RValue RV = RValue::get(V: EmitScalarExpr(E, /*Ignore*/ IgnoreResultAssign: false));
336 LValue LV = MakeAddrLValue(Addr: Location, T: E->getType());
337 EmitStoreThroughLValue(Src: RV, Dst: LV);
338 return;
339 }
340 }
341 llvm_unreachable("bad evaluation kind");
342}
343
344void CodeGenFunction::EmitInitializationToLValue(
345 const Expr *E, LValue LV, AggValueSlot::IsZeroed_t IsZeroed) {
346 QualType Type = LV.getType();
347 switch (getEvaluationKind(T: Type)) {
348 case TEK_Complex:
349 EmitComplexExprIntoLValue(E, dest: LV, /*isInit*/ true);
350 return;
351 case TEK_Aggregate:
352 EmitAggExpr(E, AS: AggValueSlot::forLValue(LV, isDestructed: AggValueSlot::IsDestructed,
353 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
354 isAliased: AggValueSlot::IsNotAliased,
355 mayOverlap: AggValueSlot::MayOverlap, isZeroed: IsZeroed));
356 return;
357 case TEK_Scalar:
358 if (LV.isSimple())
359 EmitScalarInit(init: E, /*D=*/nullptr, lvalue: LV, /*Captured=*/capturedByInit: false);
360 else
361 EmitStoreThroughLValue(Src: RValue::get(V: EmitScalarExpr(E)), Dst: LV);
362 return;
363 }
364 llvm_unreachable("bad evaluation kind");
365}
366
367static void
368pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M,
369 const Expr *E, Address ReferenceTemporary) {
370 // Objective-C++ ARC:
371 // If we are binding a reference to a temporary that has ownership, we
372 // need to perform retain/release operations on the temporary.
373 //
374 // FIXME: This should be looking at E, not M.
375 if (auto Lifetime = M->getType().getObjCLifetime()) {
376 switch (Lifetime) {
377 case Qualifiers::OCL_None:
378 case Qualifiers::OCL_ExplicitNone:
379 // Carry on to normal cleanup handling.
380 break;
381
382 case Qualifiers::OCL_Autoreleasing:
383 // Nothing to do; cleaned up by an autorelease pool.
384 return;
385
386 case Qualifiers::OCL_Strong:
387 case Qualifiers::OCL_Weak:
388 switch (StorageDuration Duration = M->getStorageDuration()) {
389 case SD_Static:
390 // Note: we intentionally do not register a cleanup to release
391 // the object on program termination.
392 return;
393
394 case SD_Thread:
395 // FIXME: We should probably register a cleanup in this case.
396 return;
397
398 case SD_Automatic:
399 case SD_FullExpression:
400 CodeGenFunction::Destroyer *Destroy;
401 CleanupKind CleanupKind;
402 if (Lifetime == Qualifiers::OCL_Strong) {
403 const ValueDecl *VD = M->getExtendingDecl();
404 bool Precise = isa_and_nonnull<VarDecl>(Val: VD) &&
405 VD->hasAttr<ObjCPreciseLifetimeAttr>();
406 CleanupKind = CGF.getARCCleanupKind();
407 Destroy = Precise ? &CodeGenFunction::destroyARCStrongPrecise
408 : &CodeGenFunction::destroyARCStrongImprecise;
409 } else {
410 // __weak objects always get EH cleanups; otherwise, exceptions
411 // could cause really nasty crashes instead of mere leaks.
412 CleanupKind = NormalAndEHCleanup;
413 Destroy = &CodeGenFunction::destroyARCWeak;
414 }
415 if (Duration == SD_FullExpression)
416 CGF.pushDestroy(kind: CleanupKind, addr: ReferenceTemporary,
417 type: M->getType(), destroyer: *Destroy,
418 useEHCleanupForArray: CleanupKind & EHCleanup);
419 else
420 CGF.pushLifetimeExtendedDestroy(kind: CleanupKind, addr: ReferenceTemporary,
421 type: M->getType(),
422 destroyer: *Destroy, useEHCleanupForArray: CleanupKind & EHCleanup);
423 return;
424
425 case SD_Dynamic:
426 llvm_unreachable("temporary cannot have dynamic storage duration");
427 }
428 llvm_unreachable("unknown storage duration");
429 }
430 }
431
432 QualType::DestructionKind DK = E->getType().isDestructedType();
433 if (DK != QualType::DK_none) {
434 switch (M->getStorageDuration()) {
435 case SD_Static:
436 case SD_Thread: {
437 CXXDestructorDecl *ReferenceTemporaryDtor = nullptr;
438 if (const auto *ClassDecl =
439 E->getType()->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
440 ClassDecl && !ClassDecl->hasTrivialDestructor())
441 // Get the destructor for the reference temporary.
442 ReferenceTemporaryDtor = ClassDecl->getDestructor();
443
444 if (!ReferenceTemporaryDtor)
445 return;
446
447 // Like in `EmitDeclDestroy`, destructors that return `this` need a helper
448 // if the target does not tolerate the mismatch (e.g. WebAssembly).
449 bool CanRegisterDestructor =
450 !CGF.CGM.getCXXABI().HasThisReturn(
451 GD: GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete)) ||
452 CGF.CGM.getCXXABI().canCallMismatchedFunctionType();
453
454 llvm::FunctionCallee CleanupFn;
455 llvm::Constant *CleanupArg;
456 if (E->getType()->isArrayType() || !CanRegisterDestructor) {
457 CleanupFn = CodeGenFunction(CGF.CGM).generateDestroyHelper(
458 addr: ReferenceTemporary, type: E->getType(), destroyer: CodeGenFunction::destroyCXXObject,
459 useEHCleanupForArray: CGF.getLangOpts().Exceptions,
460 VD: dyn_cast_or_null<VarDecl>(Val: M->getExtendingDecl()));
461 CleanupArg = llvm::Constant::getNullValue(Ty: CGF.Int8PtrTy);
462 } else {
463 CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor(
464 GD: GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete));
465 CleanupArg =
466 cast<llvm::Constant>(Val: ReferenceTemporary.emitRawPointer(CGF));
467 }
468 CGF.CGM.getCXXABI().registerGlobalDtor(
469 CGF, D: *cast<VarDecl>(Val: M->getExtendingDecl()), Dtor: CleanupFn, Addr: CleanupArg);
470 } break;
471 case SD_FullExpression:
472 CGF.pushDestroy(dtorKind: DK, addr: ReferenceTemporary, type: E->getType());
473 break;
474 case SD_Automatic:
475 CGF.pushLifetimeExtendedDestroy(dtorKind: DK, addr: ReferenceTemporary, type: E->getType());
476 break;
477 case SD_Dynamic:
478 llvm_unreachable("temporary cannot have dynamic storage duration");
479 }
480 }
481}
482
483static RawAddress createReferenceTemporary(CodeGenFunction &CGF,
484 const MaterializeTemporaryExpr *M,
485 const Expr *Inner,
486 RawAddress *Alloca = nullptr) {
487 switch (M->getStorageDuration()) {
488 case SD_FullExpression:
489 case SD_Automatic: {
490 // If we have a constant temporary array or record try to promote it into a
491 // constant global under the same rules a normal constant would've been
492 // promoted. This is easier on the optimizer and generally emits fewer
493 // instructions.
494 QualType Ty = Inner->getType();
495 if (CGF.CGM.getCodeGenOpts().MergeAllConstants &&
496 (Ty->isArrayType() || Ty->isRecordType()) &&
497 Ty.isConstantStorage(Ctx: CGF.getContext(), ExcludeCtor: true, ExcludeDtor: false))
498 if (auto Init = ConstantEmitter(CGF).tryEmitAbstract(E: Inner, T: Ty)) {
499 auto AS = CGF.CGM.GetGlobalConstantAddressSpace();
500 auto *GV = new llvm::GlobalVariable(
501 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
502 llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp", nullptr,
503 llvm::GlobalValue::NotThreadLocal,
504 CGF.getContext().getTargetAddressSpace(AS));
505 CharUnits alignment = CGF.getContext().getTypeAlignInChars(T: Ty);
506 GV->setAlignment(alignment.getAsAlign());
507 llvm::Constant *C = GV;
508 if (AS != Ty.getAddressSpace())
509 C = CGF.CGM.performAddrSpaceCast(
510 Src: GV, DestTy: llvm::PointerType::get(C&: CGF.getLLVMContext(),
511 AddressSpace: CGF.getContext().getTargetAddressSpace(
512 AS: Ty.getAddressSpace())));
513 // FIXME: Should we put the new global into a COMDAT?
514 return RawAddress(C, GV->getValueType(), alignment);
515 }
516 return CGF.CreateMemTemp(Ty, Name: "ref.tmp", Alloca);
517 }
518 case SD_Thread:
519 case SD_Static:
520 return CGF.CGM.GetAddrOfGlobalTemporary(E: M, Inner);
521
522 case SD_Dynamic:
523 llvm_unreachable("temporary can't have dynamic storage duration");
524 }
525 llvm_unreachable("unknown storage duration");
526}
527
528LValue CodeGenFunction::
529EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
530 const Expr *E = M->getSubExpr();
531
532 assert((!M->getExtendingDecl() || !isa<VarDecl>(M->getExtendingDecl()) ||
533 !cast<VarDecl>(M->getExtendingDecl())->isARCPseudoStrong()) &&
534 "Reference should never be pseudo-strong!");
535
536 // FIXME: ideally this would use EmitAnyExprToMem, however, we cannot do so
537 // as that will cause the lifetime adjustment to be lost for ARC
538 auto ownership = M->getType().getObjCLifetime();
539 if (ownership != Qualifiers::OCL_None &&
540 ownership != Qualifiers::OCL_ExplicitNone) {
541 RawAddress Object = createReferenceTemporary(CGF&: *this, M, Inner: E);
542 if (auto *Var = dyn_cast<llvm::GlobalVariable>(Val: Object.getPointer())) {
543 llvm::Type *Ty = ConvertTypeForMem(T: E->getType());
544 Object = Object.withElementType(ElemTy: Ty);
545
546 // createReferenceTemporary will promote the temporary to a global with a
547 // constant initializer if it can. It can only do this to a value of
548 // ARC-manageable type if the value is global and therefore "immune" to
549 // ref-counting operations. Therefore we have no need to emit either a
550 // dynamic initialization or a cleanup and we can just return the address
551 // of the temporary.
552 if (Var->hasInitializer())
553 return MakeAddrLValue(Addr: Object, T: M->getType(), Source: AlignmentSource::Decl);
554
555 Var->setInitializer(CGM.EmitNullConstant(T: E->getType()));
556 }
557 LValue RefTempDst = MakeAddrLValue(Addr: Object, T: M->getType(),
558 Source: AlignmentSource::Decl);
559
560 switch (getEvaluationKind(T: E->getType())) {
561 default: llvm_unreachable("expected scalar or aggregate expression");
562 case TEK_Scalar:
563 EmitScalarInit(init: E, D: M->getExtendingDecl(), lvalue: RefTempDst, capturedByInit: false);
564 break;
565 case TEK_Aggregate: {
566 EmitAggExpr(E, AS: AggValueSlot::forAddr(addr: Object,
567 quals: E->getType().getQualifiers(),
568 isDestructed: AggValueSlot::IsDestructed,
569 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
570 isAliased: AggValueSlot::IsNotAliased,
571 mayOverlap: AggValueSlot::DoesNotOverlap));
572 break;
573 }
574 }
575
576 pushTemporaryCleanup(CGF&: *this, M, E, ReferenceTemporary: Object);
577 return RefTempDst;
578 }
579
580 SmallVector<const Expr *, 2> CommaLHSs;
581 SmallVector<SubobjectAdjustment, 2> Adjustments;
582 E = E->skipRValueSubobjectAdjustments(CommaLHS&: CommaLHSs, Adjustments);
583
584 for (const auto &Ignored : CommaLHSs)
585 EmitIgnoredExpr(E: Ignored);
586
587 if (const auto *opaque = dyn_cast<OpaqueValueExpr>(Val: E)) {
588 if (opaque->getType()->isRecordType()) {
589 assert(Adjustments.empty());
590 return EmitOpaqueValueLValue(e: opaque);
591 }
592 }
593
594 // Create and initialize the reference temporary.
595 RawAddress Alloca = Address::invalid();
596 RawAddress Object = createReferenceTemporary(CGF&: *this, M, Inner: E, Alloca: &Alloca);
597 if (auto *Var = dyn_cast<llvm::GlobalVariable>(
598 Val: Object.getPointer()->stripPointerCasts())) {
599 llvm::Type *TemporaryType = ConvertTypeForMem(T: E->getType());
600 Object = Object.withElementType(ElemTy: TemporaryType);
601 // If the temporary is a global and has a constant initializer or is a
602 // constant temporary that we promoted to a global, we may have already
603 // initialized it.
604 if (!Var->hasInitializer()) {
605 Var->setInitializer(CGM.EmitNullConstant(T: E->getType()));
606 QualType RefType = M->getType().withoutLocalFastQualifiers();
607 if (RefType.getPointerAuth()) {
608 // Use the qualifier of the reference temporary to sign the pointer.
609 LValue LV = MakeRawAddrLValue(V: Object.getPointer(), T: RefType,
610 Alignment: Object.getAlignment());
611 EmitScalarInit(init: E, D: M->getExtendingDecl(), lvalue: LV, capturedByInit: false);
612 } else {
613 EmitAnyExprToMem(E, Location: Object, Quals: Qualifiers(), /*IsInit*/ true);
614 }
615 }
616 } else {
617 switch (M->getStorageDuration()) {
618 case SD_Automatic:
619 if (EmitLifetimeStart(Addr: Alloca.getPointer())) {
620 pushCleanupAfterFullExpr<CallLifetimeEnd>(Kind: NormalEHLifetimeMarker,
621 A: Alloca);
622 }
623 break;
624
625 case SD_FullExpression: {
626 if (!ShouldEmitLifetimeMarkers)
627 break;
628
629 // Avoid creating a conditional cleanup just to hold an llvm.lifetime.end
630 // marker. Instead, start the lifetime of a conditional temporary earlier
631 // so that it's unconditional. Don't do this with sanitizers which need
632 // more precise lifetime marks. However when inside an "await.suspend"
633 // block, we should always avoid conditional cleanup because it creates
634 // boolean marker that lives across await_suspend, which can destroy coro
635 // frame.
636 ConditionalEvaluation *OldConditional = nullptr;
637 CGBuilderTy::InsertPoint OldIP;
638 if (isInConditionalBranch() && !E->getType().isDestructedType() &&
639 ((!SanOpts.has(K: SanitizerKind::HWAddress) &&
640 !SanOpts.has(K: SanitizerKind::Memory) &&
641 !SanOpts.has(K: SanitizerKind::MemtagStack) &&
642 !CGM.getCodeGenOpts().SanitizeAddressUseAfterScope) ||
643 inSuspendBlock())) {
644 OldConditional = OutermostConditional;
645 OutermostConditional = nullptr;
646
647 OldIP = Builder.saveIP();
648 llvm::BasicBlock *Block = OldConditional->getStartingBlock();
649 Builder.restoreIP(IP: Block->back().getIterator());
650 }
651
652 if (EmitLifetimeStart(Addr: Alloca.getPointer())) {
653 pushFullExprCleanup<CallLifetimeEnd>(kind: NormalEHLifetimeMarker, A: Alloca);
654 }
655
656 if (OldConditional) {
657 OutermostConditional = OldConditional;
658 Builder.restoreIP(IP: OldIP);
659 }
660 break;
661 }
662
663 default:
664 break;
665 }
666 EmitAnyExprToMem(E, Location: Object, Quals: Qualifiers(), /*IsInit*/true);
667 }
668 pushTemporaryCleanup(CGF&: *this, M, E, ReferenceTemporary: Object);
669
670 // Perform derived-to-base casts and/or field accesses, to get from the
671 // temporary object we created (and, potentially, for which we extended
672 // the lifetime) to the subobject we're binding the reference to.
673 for (SubobjectAdjustment &Adjustment : llvm::reverse(C&: Adjustments)) {
674 switch (Adjustment.Kind) {
675 case SubobjectAdjustment::DerivedToBaseAdjustment:
676 Object =
677 GetAddressOfBaseClass(Value: Object, Derived: Adjustment.DerivedToBase.DerivedClass,
678 PathBegin: Adjustment.DerivedToBase.BasePath->path_begin(),
679 PathEnd: Adjustment.DerivedToBase.BasePath->path_end(),
680 /*NullCheckValue=*/ false, Loc: E->getExprLoc());
681 break;
682
683 case SubobjectAdjustment::FieldAdjustment: {
684 LValue LV = MakeAddrLValue(Addr: Object, T: E->getType(), Source: AlignmentSource::Decl);
685 LV = EmitLValueForField(Base: LV, Field: Adjustment.Field);
686 assert(LV.isSimple() &&
687 "materialized temporary field is not a simple lvalue");
688 Object = LV.getAddress();
689 break;
690 }
691
692 case SubobjectAdjustment::MemberPointerAdjustment: {
693 llvm::Value *Ptr = EmitScalarExpr(E: Adjustment.Ptr.RHS);
694 Object = EmitCXXMemberDataPointerAddress(
695 E, base: Object, memberPtr: Ptr, memberPtrType: Adjustment.Ptr.MPT, /*IsInBounds=*/true);
696 break;
697 }
698 }
699 }
700
701 return MakeAddrLValue(Addr: Object, T: M->getType(), Source: AlignmentSource::Decl);
702}
703
704RValue
705CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E) {
706 // Emit the expression as an lvalue.
707 LValue LV = EmitLValue(E);
708 assert(LV.isSimple());
709 llvm::Value *Value = LV.getPointer(CGF&: *this);
710
711 if (sanitizePerformTypeCheck() && !E->getType()->isFunctionType()) {
712 // C++11 [dcl.ref]p5 (as amended by core issue 453):
713 // If a glvalue to which a reference is directly bound designates neither
714 // an existing object or function of an appropriate type nor a region of
715 // storage of suitable size and alignment to contain an object of the
716 // reference's type, the behavior is undefined.
717 QualType Ty = E->getType();
718 EmitTypeCheck(TCK: TCK_ReferenceBinding, Loc: E->getExprLoc(), V: Value, Type: Ty);
719 }
720
721 return RValue::get(V: Value);
722}
723
724
725/// getAccessedFieldNo - Given an encoded value and a result number, return the
726/// input field number being accessed.
727unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx,
728 const llvm::Constant *Elts) {
729 return cast<llvm::ConstantInt>(Val: Elts->getAggregateElement(Elt: Idx))
730 ->getZExtValue();
731}
732
733static llvm::Value *emitHashMix(CGBuilderTy &Builder, llvm::Value *Acc,
734 llvm::Value *Ptr) {
735 llvm::Value *A0 =
736 Builder.CreateMul(LHS: Ptr, RHS: Builder.getInt64(C: 0xbf58476d1ce4e5b9u));
737 llvm::Value *A1 =
738 Builder.CreateXor(LHS: A0, RHS: Builder.CreateLShr(LHS: A0, RHS: Builder.getInt64(C: 31)));
739 return Builder.CreateXor(LHS: Acc, RHS: A1);
740}
741
742bool CodeGenFunction::isNullPointerAllowed(TypeCheckKind TCK) {
743 return TCK == TCK_DowncastPointer || TCK == TCK_Upcast ||
744 TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation;
745}
746
747bool CodeGenFunction::isVptrCheckRequired(TypeCheckKind TCK, QualType Ty) {
748 CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
749 return (RD && RD->hasDefinition() && RD->isDynamicClass()) &&
750 (TCK == TCK_MemberAccess || TCK == TCK_MemberCall ||
751 TCK == TCK_DowncastPointer || TCK == TCK_DowncastReference ||
752 TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation);
753}
754
755bool CodeGenFunction::sanitizePerformTypeCheck() const {
756 return SanOpts.has(K: SanitizerKind::Null) ||
757 SanOpts.has(K: SanitizerKind::Alignment) ||
758 SanOpts.has(K: SanitizerKind::ObjectSize) ||
759 SanOpts.has(K: SanitizerKind::Vptr);
760}
761
762void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
763 llvm::Value *Ptr, QualType Ty,
764 CharUnits Alignment,
765 SanitizerSet SkippedChecks,
766 llvm::Value *ArraySize) {
767 if (!sanitizePerformTypeCheck())
768 return;
769
770 // Don't check pointers outside the default address space. The null check
771 // isn't correct, the object-size check isn't supported by LLVM, and we can't
772 // communicate the addresses to the runtime handler for the vptr check.
773 if (Ptr->getType()->getPointerAddressSpace())
774 return;
775
776 // Don't check pointers to volatile data. The behavior here is implementation-
777 // defined.
778 if (Ty.isVolatileQualified())
779 return;
780
781 // Quickly determine whether we have a pointer to an alloca. It's possible
782 // to skip null checks, and some alignment checks, for these pointers. This
783 // can reduce compile-time significantly.
784 auto PtrToAlloca = dyn_cast<llvm::AllocaInst>(Val: Ptr->stripPointerCasts());
785
786 llvm::Value *IsNonNull = nullptr;
787 bool IsGuaranteedNonNull =
788 SkippedChecks.has(K: SanitizerKind::Null) || PtrToAlloca;
789
790 llvm::BasicBlock *Done = nullptr;
791 bool DoneViaNullSanitize = false;
792
793 {
794 auto CheckHandler = SanitizerHandler::TypeMismatch;
795 SanitizerDebugLocation SanScope(this,
796 {SanitizerKind::SO_Null,
797 SanitizerKind::SO_ObjectSize,
798 SanitizerKind::SO_Alignment},
799 CheckHandler);
800
801 SmallVector<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>, 3>
802 Checks;
803
804 llvm::Value *True = llvm::ConstantInt::getTrue(Context&: getLLVMContext());
805 bool AllowNullPointers = isNullPointerAllowed(TCK);
806 if ((SanOpts.has(K: SanitizerKind::Null) || AllowNullPointers) &&
807 !IsGuaranteedNonNull) {
808 // The glvalue must not be an empty glvalue.
809 IsNonNull = Builder.CreateIsNotNull(Arg: Ptr);
810
811 // The IR builder can constant-fold the null check if the pointer points
812 // to a constant.
813 IsGuaranteedNonNull = IsNonNull == True;
814
815 // Skip the null check if the pointer is known to be non-null.
816 if (!IsGuaranteedNonNull) {
817 if (AllowNullPointers) {
818 // When performing pointer casts, it's OK if the value is null.
819 // Skip the remaining checks in that case.
820 Done = createBasicBlock(name: "null");
821 DoneViaNullSanitize = true;
822 llvm::BasicBlock *Rest = createBasicBlock(name: "not.null");
823 Builder.CreateCondBr(Cond: IsNonNull, True: Rest, False: Done);
824 EmitBlock(BB: Rest);
825 } else {
826 Checks.push_back(Elt: std::make_pair(x&: IsNonNull, y: SanitizerKind::SO_Null));
827 }
828 }
829 }
830
831 if (SanOpts.has(K: SanitizerKind::ObjectSize) &&
832 !SkippedChecks.has(K: SanitizerKind::ObjectSize) &&
833 !Ty->isIncompleteType()) {
834 uint64_t TySize = CGM.getMinimumObjectSize(Ty).getQuantity();
835 llvm::Value *Size = llvm::ConstantInt::get(Ty: IntPtrTy, V: TySize);
836 if (ArraySize)
837 Size = Builder.CreateMul(LHS: Size, RHS: ArraySize);
838
839 // Degenerate case: new X[0] does not need an objectsize check.
840 llvm::Constant *ConstantSize = dyn_cast<llvm::Constant>(Val: Size);
841 if (!ConstantSize || !ConstantSize->isNullValue()) {
842 // The glvalue must refer to a large enough storage region.
843 // FIXME: If Address Sanitizer is enabled, insert dynamic
844 // instrumentation
845 // to check this.
846 // FIXME: Get object address space
847 llvm::Type *Tys[2] = {IntPtrTy, Int8PtrTy};
848 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::objectsize, Tys);
849 llvm::Value *Min = Builder.getFalse();
850 llvm::Value *NullIsUnknown = Builder.getFalse();
851 llvm::Value *Dynamic = Builder.getFalse();
852 llvm::Value *LargeEnough = Builder.CreateICmpUGE(
853 LHS: Builder.CreateCall(Callee: F, Args: {Ptr, Min, NullIsUnknown, Dynamic}), RHS: Size);
854 Checks.push_back(
855 Elt: std::make_pair(x&: LargeEnough, y: SanitizerKind::SO_ObjectSize));
856 }
857 }
858
859 llvm::MaybeAlign AlignVal;
860 llvm::Value *PtrAsInt = nullptr;
861
862 if (SanOpts.has(K: SanitizerKind::Alignment) &&
863 !SkippedChecks.has(K: SanitizerKind::Alignment)) {
864 AlignVal = Alignment.getAsMaybeAlign();
865 if (!Ty->isIncompleteType() && !AlignVal)
866 AlignVal = CGM.getNaturalTypeAlignment(T: Ty, BaseInfo: nullptr, TBAAInfo: nullptr,
867 /*ForPointeeType=*/forPointeeType: true)
868 .getAsMaybeAlign();
869
870 // The glvalue must be suitably aligned.
871 if (AlignVal && *AlignVal > llvm::Align(1) &&
872 (!PtrToAlloca || PtrToAlloca->getAlign() < *AlignVal)) {
873 PtrAsInt = Builder.CreatePtrToInt(V: Ptr, DestTy: IntPtrTy);
874 llvm::Value *Align = Builder.CreateAnd(
875 LHS: PtrAsInt, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, V: AlignVal->value() - 1));
876 llvm::Value *Aligned =
877 Builder.CreateICmpEQ(LHS: Align, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, V: 0));
878 if (Aligned != True)
879 Checks.push_back(
880 Elt: std::make_pair(x&: Aligned, y: SanitizerKind::SO_Alignment));
881 }
882 }
883
884 if (Checks.size() > 0) {
885 llvm::Constant *StaticData[] = {
886 EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(T: Ty),
887 llvm::ConstantInt::get(Ty: Int8Ty, V: AlignVal ? llvm::Log2(A: *AlignVal) : 1),
888 llvm::ConstantInt::get(Ty: Int8Ty, V: TCK)};
889 EmitCheck(Checked: Checks, Check: CheckHandler, StaticArgs: StaticData, DynamicArgs: PtrAsInt ? PtrAsInt : Ptr);
890 }
891 }
892
893 // If possible, check that the vptr indicates that there is a subobject of
894 // type Ty at offset zero within this object.
895 //
896 // C++11 [basic.life]p5,6:
897 // [For storage which does not refer to an object within its lifetime]
898 // The program has undefined behavior if:
899 // -- the [pointer or glvalue] is used to access a non-static data member
900 // or call a non-static member function
901 if (SanOpts.has(K: SanitizerKind::Vptr) &&
902 !SkippedChecks.has(K: SanitizerKind::Vptr) && isVptrCheckRequired(TCK, Ty)) {
903 SanitizerDebugLocation SanScope(this, {SanitizerKind::SO_Vptr},
904 SanitizerHandler::DynamicTypeCacheMiss);
905
906 // Ensure that the pointer is non-null before loading it. If there is no
907 // compile-time guarantee, reuse the run-time null check or emit a new one.
908 if (!IsGuaranteedNonNull) {
909 if (!IsNonNull)
910 IsNonNull = Builder.CreateIsNotNull(Arg: Ptr);
911 if (!Done)
912 Done = createBasicBlock(name: "vptr.null");
913 llvm::BasicBlock *VptrNotNull = createBasicBlock(name: "vptr.not.null");
914 Builder.CreateCondBr(Cond: IsNonNull, True: VptrNotNull, False: Done);
915 EmitBlock(BB: VptrNotNull);
916 }
917
918 // Compute a deterministic hash of the mangled name of the type.
919 SmallString<64> MangledName;
920 llvm::raw_svector_ostream Out(MangledName);
921 CGM.getCXXABI().getMangleContext().mangleCXXRTTI(T: Ty.getUnqualifiedType(),
922 Out);
923
924 // Contained in NoSanitizeList based on the mangled type.
925 if (!CGM.getContext().getNoSanitizeList().containsType(Mask: SanitizerKind::Vptr,
926 MangledTypeName: Out.str())) {
927 // Load the vptr, and mix it with TypeHash.
928 llvm::Value *TypeHash =
929 llvm::ConstantInt::get(Ty: Int64Ty, V: xxh3_64bits(data: Out.str()));
930
931 llvm::Type *VPtrTy = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: 0);
932 Address VPtrAddr(Ptr, IntPtrTy, getPointerAlign());
933 llvm::Value *VPtrVal = GetVTablePtr(This: VPtrAddr, VTableTy: VPtrTy,
934 VTableClass: Ty->getAsCXXRecordDecl(),
935 AuthMode: VTableAuthMode::UnsafeUbsanStrip);
936 VPtrVal = Builder.CreateBitOrPointerCast(V: VPtrVal, DestTy: IntPtrTy);
937
938 llvm::Value *Hash =
939 emitHashMix(Builder, Acc: TypeHash, Ptr: Builder.CreateZExt(V: VPtrVal, DestTy: Int64Ty));
940 Hash = Builder.CreateTrunc(V: Hash, DestTy: IntPtrTy);
941
942 // Look the hash up in our cache.
943 const int CacheSize = 128;
944 llvm::Type *HashTable = llvm::ArrayType::get(ElementType: IntPtrTy, NumElements: CacheSize);
945 llvm::Value *Cache = CGM.CreateRuntimeVariable(Ty: HashTable,
946 Name: "__ubsan_vptr_type_cache");
947 llvm::Value *Slot = Builder.CreateAnd(LHS: Hash,
948 RHS: llvm::ConstantInt::get(Ty: IntPtrTy,
949 V: CacheSize-1));
950 llvm::Value *Indices[] = { Builder.getInt32(C: 0), Slot };
951 llvm::Value *CacheVal = Builder.CreateAlignedLoad(
952 Ty: IntPtrTy, Addr: Builder.CreateInBoundsGEP(Ty: HashTable, Ptr: Cache, IdxList: Indices),
953 Align: getPointerAlign());
954
955 // If the hash isn't in the cache, call a runtime handler to perform the
956 // hard work of checking whether the vptr is for an object of the right
957 // type. This will either fill in the cache and return, or produce a
958 // diagnostic.
959 llvm::Value *EqualHash = Builder.CreateICmpEQ(LHS: CacheVal, RHS: Hash);
960 llvm::Constant *StaticData[] = {
961 EmitCheckSourceLocation(Loc),
962 EmitCheckTypeDescriptor(T: Ty),
963 CGM.GetAddrOfRTTIDescriptor(Ty: Ty.getUnqualifiedType()),
964 llvm::ConstantInt::get(Ty: Int8Ty, V: TCK)
965 };
966 llvm::Value *DynamicData[] = { Ptr, Hash };
967 EmitCheck(Checked: std::make_pair(x&: EqualHash, y: SanitizerKind::SO_Vptr),
968 Check: SanitizerHandler::DynamicTypeCacheMiss, StaticArgs: StaticData,
969 DynamicArgs: DynamicData);
970 }
971 }
972
973 if (Done) {
974 SanitizerDebugLocation SanScope(
975 this,
976 {DoneViaNullSanitize ? SanitizerKind::SO_Null : SanitizerKind::SO_Vptr},
977 DoneViaNullSanitize ? SanitizerHandler::TypeMismatch
978 : SanitizerHandler::DynamicTypeCacheMiss);
979 Builder.CreateBr(Dest: Done);
980 EmitBlock(BB: Done);
981 }
982}
983
984llvm::Value *CodeGenFunction::LoadPassedObjectSize(const Expr *E,
985 QualType EltTy) {
986 ASTContext &C = getContext();
987 uint64_t EltSize = C.getTypeSizeInChars(T: EltTy).getQuantity();
988 if (!EltSize)
989 return nullptr;
990
991 auto *ArrayDeclRef = dyn_cast<DeclRefExpr>(Val: E->IgnoreParenImpCasts());
992 if (!ArrayDeclRef)
993 return nullptr;
994
995 auto *ParamDecl = dyn_cast<ParmVarDecl>(Val: ArrayDeclRef->getDecl());
996 if (!ParamDecl)
997 return nullptr;
998
999 auto *POSAttr = ParamDecl->getAttr<PassObjectSizeAttr>();
1000 if (!POSAttr)
1001 return nullptr;
1002
1003 // Don't load the size if it's a lower bound.
1004 int POSType = POSAttr->getType();
1005 if (POSType != 0 && POSType != 1)
1006 return nullptr;
1007
1008 // Find the implicit size parameter.
1009 auto PassedSizeIt = SizeArguments.find(Val: ParamDecl);
1010 if (PassedSizeIt == SizeArguments.end())
1011 return nullptr;
1012
1013 const ImplicitParamDecl *PassedSizeDecl = PassedSizeIt->second;
1014 assert(LocalDeclMap.count(PassedSizeDecl) && "Passed size not loadable");
1015 Address AddrOfSize = LocalDeclMap.find(Val: PassedSizeDecl)->second;
1016 llvm::Value *SizeInBytes = EmitLoadOfScalar(Addr: AddrOfSize, /*Volatile=*/false,
1017 Ty: C.getSizeType(), Loc: E->getExprLoc());
1018 llvm::Value *SizeOfElement =
1019 llvm::ConstantInt::get(Ty: SizeInBytes->getType(), V: EltSize);
1020 return Builder.CreateUDiv(LHS: SizeInBytes, RHS: SizeOfElement);
1021}
1022
1023/// If Base is known to point to the start of an array, return the length of
1024/// that array. Return 0 if the length cannot be determined.
1025static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF,
1026 const Expr *Base,
1027 QualType &IndexedType,
1028 LangOptions::StrictFlexArraysLevelKind
1029 StrictFlexArraysLevel) {
1030 // For the vector indexing extension, the bound is the number of elements.
1031 if (const VectorType *VT = Base->getType()->getAs<VectorType>()) {
1032 IndexedType = Base->getType();
1033 return CGF.Builder.getInt32(C: VT->getNumElements());
1034 }
1035
1036 Base = Base->IgnoreParens();
1037
1038 if (const auto *CE = dyn_cast<CastExpr>(Val: Base)) {
1039 if (CE->getCastKind() == CK_ArrayToPointerDecay &&
1040 !CE->getSubExpr()->isFlexibleArrayMemberLike(Context: CGF.getContext(),
1041 StrictFlexArraysLevel)) {
1042 CodeGenFunction::SanitizerScope SanScope(&CGF);
1043
1044 IndexedType = CE->getSubExpr()->getType();
1045 const ArrayType *AT = IndexedType->castAsArrayTypeUnsafe();
1046 if (const auto *CAT = dyn_cast<ConstantArrayType>(Val: AT))
1047 return CGF.Builder.getInt(AI: CAT->getSize());
1048
1049 if (const auto *VAT = dyn_cast<VariableArrayType>(Val: AT))
1050 return CGF.getVLASize(vla: VAT).NumElts;
1051 // Ignore pass_object_size here. It's not applicable on decayed pointers.
1052 }
1053 }
1054
1055 CodeGenFunction::SanitizerScope SanScope(&CGF);
1056
1057 QualType EltTy{Base->getType()->getPointeeOrArrayElementType(), 0};
1058 if (llvm::Value *POS = CGF.LoadPassedObjectSize(E: Base, EltTy)) {
1059 IndexedType = Base->getType();
1060 return POS;
1061 }
1062
1063 return nullptr;
1064}
1065
1066/// Returns true if \p Field is reachable from \p RD either as a direct field or
1067/// through a chain of nested record fields (including anonymous
1068/// structs/unions). This mirrors the path that getFieldOffsetInBits walks, and
1069/// is used to identify the right anchor expression in Base.
1070static bool RecordContainsField(const RecordDecl *RD, const FieldDecl *Field) {
1071 for (const FieldDecl *FD : RD->fields()) {
1072 if (FD == Field)
1073 return true;
1074 QualType Ty = FD->getType();
1075 if (Ty->isRecordType())
1076 if (RecordContainsField(RD: Ty->getAsRecordDecl(), Field))
1077 return true;
1078 }
1079 return false;
1080}
1081
1082namespace {
1083
1084/// \p StructAccessBase returns the base \p Expr of a field access. It returns
1085/// either a \p DeclRefExpr, representing the base pointer to the struct, i.e.:
1086///
1087/// p in p-> a.b.c
1088///
1089/// or a \p MemberExpr, if the \p MemberExpr has the \p RecordDecl we're
1090/// looking for:
1091///
1092/// struct s {
1093/// struct s *ptr;
1094/// int count;
1095/// char array[] __attribute__((counted_by(count)));
1096/// };
1097///
1098/// If we have an expression like \p p->ptr->array[index], we want the
1099/// \p MemberExpr for \p p->ptr instead of \p p.
1100class StructAccessBase
1101 : public ConstStmtVisitor<StructAccessBase, const Expr *> {
1102 /// The count field we're navigating to. We stop at the innermost expression
1103 /// whose struct type transitively contains this field, so that
1104 /// getFieldOffsetInBits can navigate from that struct down to it.
1105 const FieldDecl *CountDecl;
1106
1107 /// Returns true if E's record type (or pointee record type) transitively
1108 /// contains CountDecl. Handles both direct containment and nested structs,
1109 /// so we don't need a pre-computed RD from the caller.
1110 bool IsExpectedRecordDecl(const Expr *E) const {
1111 QualType Ty = E->getType();
1112 if (Ty->isPointerType())
1113 Ty = Ty->getPointeeType();
1114 const RecordDecl *RD = Ty->getAsRecordDecl();
1115 return RD && RecordContainsField(RD, Field: CountDecl);
1116 }
1117
1118public:
1119 StructAccessBase(const FieldDecl *CountDecl) : CountDecl(CountDecl) {}
1120
1121 //===--------------------------------------------------------------------===//
1122 // Visitor Methods
1123 //===--------------------------------------------------------------------===//
1124
1125 // NOTE: If we build C++ support for counted_by, then we'll have to handle
1126 // horrors like this:
1127 //
1128 // struct S {
1129 // int x, y;
1130 // int blah[] __attribute__((counted_by(x)));
1131 // } s;
1132 //
1133 // int foo(int index, int val) {
1134 // int (S::*IHatePMDs)[] = &S::blah;
1135 // (s.*IHatePMDs)[index] = val;
1136 // }
1137
1138 const Expr *Visit(const Expr *E) {
1139 return ConstStmtVisitor<StructAccessBase, const Expr *>::Visit(S: E);
1140 }
1141
1142 const Expr *VisitStmt(const Stmt *S) { return nullptr; }
1143
1144 // These are the types we expect to return (in order of most to least
1145 // likely):
1146 //
1147 // 1. DeclRefExpr - This is the expression for the base of the structure.
1148 // It's exactly what we want to build an access to the \p counted_by
1149 // field.
1150 // 2. MemberExpr - This is the expression that has the same \p RecordDecl
1151 // as the flexble array member's lexical enclosing \p RecordDecl. This
1152 // allows us to catch things like: "p->p->array"
1153 // 3. CompoundLiteralExpr - This is for people who create something
1154 // heretical like (struct foo has a flexible array member):
1155 //
1156 // (struct foo){ 1, 2 }.blah[idx];
1157 const Expr *VisitDeclRefExpr(const DeclRefExpr *E) {
1158 return IsExpectedRecordDecl(E) ? E : nullptr;
1159 }
1160 const Expr *VisitMemberExpr(const MemberExpr *E) {
1161 if (IsExpectedRecordDecl(E) && E->isArrow())
1162 return E;
1163 const Expr *Res = Visit(E: E->getBase());
1164 return !Res && IsExpectedRecordDecl(E) ? E : Res;
1165 }
1166 const Expr *VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) {
1167 return IsExpectedRecordDecl(E) ? E : nullptr;
1168 }
1169 const Expr *VisitCallExpr(const CallExpr *E) {
1170 return IsExpectedRecordDecl(E) ? E : nullptr;
1171 }
1172
1173 const Expr *VisitArraySubscriptExpr(const ArraySubscriptExpr *E) {
1174 if (IsExpectedRecordDecl(E))
1175 return E;
1176 return Visit(E: E->getBase());
1177 }
1178 const Expr *VisitCastExpr(const CastExpr *E) {
1179 if (E->getCastKind() == CK_LValueToRValue)
1180 return IsExpectedRecordDecl(E) ? E : nullptr;
1181 return Visit(E: E->getSubExpr());
1182 }
1183 const Expr *VisitParenExpr(const ParenExpr *E) {
1184 return Visit(E: E->getSubExpr());
1185 }
1186 const Expr *VisitUnaryAddrOf(const UnaryOperator *E) {
1187 return Visit(E: E->getSubExpr());
1188 }
1189 const Expr *VisitUnaryDeref(const UnaryOperator *E) {
1190 return Visit(E: E->getSubExpr());
1191 }
1192};
1193
1194} // end anonymous namespace
1195
1196/// The offset of a field from the beginning of the record.
1197static bool getFieldOffsetInBits(CodeGenFunction &CGF, const RecordDecl *RD,
1198 const FieldDecl *Field, int64_t &Offset) {
1199 ASTContext &Ctx = CGF.getContext();
1200 const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(D: RD);
1201 unsigned FieldNo = 0;
1202
1203 for (const FieldDecl *FD : RD->fields()) {
1204 if (FD == Field) {
1205 Offset += Layout.getFieldOffset(FieldNo);
1206 return true;
1207 }
1208
1209 QualType Ty = FD->getType();
1210 if (Ty->isRecordType())
1211 if (getFieldOffsetInBits(CGF, RD: Ty->getAsRecordDecl(), Field, Offset)) {
1212 Offset += Layout.getFieldOffset(FieldNo);
1213 return true;
1214 }
1215
1216 if (!RD->isUnion())
1217 ++FieldNo;
1218 }
1219
1220 return false;
1221}
1222
1223llvm::Value *CodeGenFunction::GetCountedByFieldExprGEP(
1224 const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl) {
1225 // Walk Base to find the deepest sub-expression whose struct type transitively
1226 // contains CountDecl. This is our GEP anchor — getFieldOffsetInBits then
1227 // computes the offset from that struct down to CountDecl, handling any
1228 // intermediate nesting without requiring us to pre-compute a RecordDecl from
1229 // Base's type or from CountDecl's parent chain.
1230 const Expr *StructBase = StructAccessBase(CountDecl).Visit(E: Base);
1231 if (!StructBase || StructBase->HasSideEffects(Ctx: getContext()))
1232 return nullptr;
1233
1234 // Derive the record type from the anchor expression itself.
1235 QualType StructTy = StructBase->getType();
1236 if (StructTy->isPointerType())
1237 StructTy = StructTy->getPointeeType();
1238 const RecordDecl *RD = StructTy->getAsRecordDecl();
1239 if (!RD)
1240 return nullptr;
1241
1242 // A bit-field count has no byte offset of its own to load from.
1243 if (CountDecl->isBitField())
1244 return nullptr;
1245
1246 // Struct GEP indices can't reach a count inside a union, because the union
1247 // lowers to its largest member and the indices would walk that one instead.
1248 int64_t Offset = 0;
1249 if (!getFieldOffsetInBits(CGF&: *this, RD, Field: CountDecl, Offset))
1250 return nullptr;
1251
1252 llvm::Value *Res = nullptr;
1253 if (StructBase->getType()->isPointerType()) {
1254 LValueBaseInfo BaseInfo;
1255 TBAAAccessInfo TBAAInfo;
1256 Address Addr = EmitPointerWithAlignment(Addr: StructBase, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
1257 Res = Addr.emitRawPointer(CGF&: *this);
1258 } else if (StructBase->isLValue()) {
1259 LValue LV = EmitLValue(E: StructBase);
1260 Address Addr = LV.getAddress();
1261 Res = Addr.emitRawPointer(CGF&: *this);
1262 } else {
1263 return nullptr;
1264 }
1265
1266 return Builder.CreateInBoundsGEP(
1267 Ty: Int8Ty, Ptr: Res, IdxList: Builder.getSize(N: getContext().toCharUnitsFromBits(BitSize: Offset)),
1268 Name: "counted_by.gep");
1269}
1270
1271/// This method is typically called in contexts where we can't generate
1272/// side-effects, like in __builtin_dynamic_object_size. When finding
1273/// expressions, only choose those that have either already been emitted or can
1274/// be loaded without side-effects.
1275///
1276/// - \p FAMDecl: the \p Decl for the flexible array member. It may not be
1277/// within the top-level struct.
1278/// - \p CountDecl: must be within the same non-anonymous struct as \p FAMDecl.
1279llvm::Value *CodeGenFunction::EmitLoadOfCountedByField(
1280 const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl) {
1281 if (llvm::Value *GEP = GetCountedByFieldExprGEP(Base, FAMDecl, CountDecl))
1282 return Builder.CreateAlignedLoad(Ty: ConvertType(T: CountDecl->getType()), Addr: GEP,
1283 Align: getIntAlign(), Name: "counted_by.load");
1284 return nullptr;
1285}
1286
1287void CodeGenFunction::EmitBoundsCheck(const Expr *ArrayExpr,
1288 const Expr *ArrayExprBase,
1289 llvm::Value *IndexVal, QualType IndexType,
1290 bool Accessed) {
1291 assert(SanOpts.has(SanitizerKind::ArrayBounds) &&
1292 "should not be called unless adding bounds checks");
1293 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
1294 getLangOpts().getStrictFlexArraysLevel();
1295 QualType ArrayExprBaseType;
1296 llvm::Value *BoundsVal = getArrayIndexingBound(
1297 CGF&: *this, Base: ArrayExprBase, IndexedType&: ArrayExprBaseType, StrictFlexArraysLevel);
1298
1299 EmitBoundsCheckImpl(ArrayExpr, ArrayBaseType: ArrayExprBaseType, IndexVal, IndexType,
1300 BoundsVal, BoundsType: getContext().getSizeType(), Accessed);
1301}
1302
1303void CodeGenFunction::EmitBoundsCheckImpl(const Expr *ArrayExpr,
1304 QualType ArrayBaseType,
1305 llvm::Value *IndexVal,
1306 QualType IndexType,
1307 llvm::Value *BoundsVal,
1308 QualType BoundsType, bool Accessed) {
1309 if (!BoundsVal)
1310 return;
1311
1312 auto CheckKind = SanitizerKind::SO_ArrayBounds;
1313 auto CheckHandler = SanitizerHandler::OutOfBounds;
1314 SanitizerDebugLocation SanScope(this, {CheckKind}, CheckHandler);
1315
1316 // All hail the C implicit type conversion rules!!!
1317 bool IndexSigned = IndexType->isSignedIntegerOrEnumerationType();
1318 bool BoundsSigned = BoundsType->isSignedIntegerOrEnumerationType();
1319
1320 const ASTContext &Ctx = getContext();
1321 llvm::Type *Ty = ConvertType(
1322 T: Ctx.getTypeSize(T: IndexType) >= Ctx.getTypeSize(T: BoundsType) ? IndexType
1323 : BoundsType);
1324
1325 llvm::Value *IndexInst = Builder.CreateIntCast(V: IndexVal, DestTy: Ty, isSigned: IndexSigned);
1326 llvm::Value *BoundsInst = Builder.CreateIntCast(V: BoundsVal, DestTy: Ty, isSigned: false);
1327
1328 llvm::Constant *StaticData[] = {
1329 EmitCheckSourceLocation(Loc: ArrayExpr->getExprLoc()),
1330 EmitCheckTypeDescriptor(T: ArrayBaseType),
1331 EmitCheckTypeDescriptor(T: IndexType),
1332 };
1333
1334 llvm::Value *Check = Accessed ? Builder.CreateICmpULT(LHS: IndexInst, RHS: BoundsInst)
1335 : Builder.CreateICmpULE(LHS: IndexInst, RHS: BoundsInst);
1336
1337 if (BoundsSigned) {
1338 // Don't allow a negative bounds.
1339 llvm::Value *Cmp = Builder.CreateICmpSGT(
1340 LHS: BoundsVal, RHS: llvm::ConstantInt::get(Ty: BoundsVal->getType(), V: 0));
1341 Check = Builder.CreateAnd(LHS: Cmp, RHS: Check);
1342 }
1343
1344 EmitCheck(Checked: std::make_pair(x&: Check, y&: CheckKind), Check: CheckHandler, StaticArgs: StaticData,
1345 DynamicArgs: IndexInst);
1346}
1347
1348/// Returns the qualified name of \p D for use as the function name in
1349/// !alloc_token metadata, or "" if \p D is null or not a NamedDecl.
1350static SmallString<64> getAllocTokenFunctionName(const Decl *D,
1351 const ASTContext &Ctx) {
1352 SmallString<64> Name;
1353 const auto *ND = dyn_cast_if_present<NamedDecl>(Val: D);
1354 if (!ND)
1355 return Name;
1356 PrintingPolicy Policy(Ctx.getLangOpts());
1357 // Do not use name lookup to decide whether to print inline namespaces.
1358 Policy.SuppressInlineNamespace =
1359 llvm::to_underlying(E: PrintingPolicy::SuppressInlineNamespaceMode::All);
1360 // Do not print file paths for unnamed types, e.g. in template arguments.
1361 Policy.AnonymousTagNameStyle =
1362 llvm::to_underlying(E: PrintingPolicy::AnonymousTagMode::Plain);
1363 llvm::raw_svector_ostream OS(Name);
1364 ND->printQualifiedName(OS, Policy);
1365 return Name;
1366}
1367
1368llvm::MDNode *CodeGenFunction::buildAllocToken(QualType AllocType) {
1369 std::optional<llvm::AllocTokenMetadata> ATMD;
1370 if (!AllocType.isNull())
1371 ATMD = infer_alloc::getAllocTokenMetadata(T: AllocType, Ctx: getContext());
1372
1373 const llvm::AllocTokenMode Mode =
1374 getLangOpts().AllocTokenMode.value_or(u: llvm::DefaultAllocTokenMode);
1375 if (Mode == llvm::AllocTokenMode::TypeFuncHash ||
1376 Mode == llvm::AllocTokenMode::TypeFuncHashPointerSplit) {
1377 // Even if the type cannot be inferred, emit metadata for a function-based
1378 // token. This allows distinguishing allocations that would otherwise all
1379 // share the fallback token.
1380 if (!ATMD)
1381 ATMD = llvm::AllocTokenMetadata{.TypeName: {}, .ContainsPointer: false};
1382 // Use the outermost non-closure function, i.e. allocations in lambdas,
1383 // blocks, and captured statements use the enclosing function. Allocations
1384 // outside of any function (e.g. global initializers) use "".
1385 ATMD->FunctionName = getAllocTokenFunctionName(D: CurFuncDecl, Ctx: getContext());
1386 }
1387 if (!ATMD)
1388 return nullptr;
1389
1390 llvm::MDBuilder MDB(getLLVMContext());
1391 auto *TypeNameMD = MDB.createString(Str: ATMD->TypeName);
1392 auto *ContainsPtrC = Builder.getInt1(V: ATMD->ContainsPointer);
1393 auto *ContainsPtrMD = MDB.createConstant(C: ContainsPtrC);
1394
1395 // Format: !{<type-name>, <contains-pointer>[, <function-name>]}
1396 SmallVector<llvm::Metadata *, 3> Ops = {TypeNameMD, ContainsPtrMD};
1397 if (ATMD->FunctionName)
1398 Ops.push_back(Elt: MDB.createString(Str: *ATMD->FunctionName));
1399 return llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Ops);
1400}
1401
1402void CodeGenFunction::EmitAllocToken(llvm::CallBase *CB, QualType AllocType) {
1403 assert(SanOpts.has(SanitizerKind::AllocToken) &&
1404 "Only needed with -fsanitize=alloc-token");
1405 CB->setMetadata(KindID: llvm::LLVMContext::MD_alloc_token,
1406 Node: buildAllocToken(AllocType));
1407}
1408
1409llvm::MDNode *CodeGenFunction::buildAllocToken(const CallExpr *E) {
1410 // C has no placement new, so there is nothing to collect.
1411 if (!getLangOpts().CPlusPlus)
1412 return buildAllocToken(
1413 AllocType: infer_alloc::inferPossibleType(E, Ctx: getContext(), CastE: CurCast));
1414 // The placement news of the enclosing function body are collected once, on
1415 // the first allocation call that is not already typed by a sizeof.
1416 auto PlacementNewType = [this](const CallExpr *Call) -> QualType {
1417 if (!AllocTokenPlacementTypes)
1418 AllocTokenPlacementTypes = infer_alloc::collectPlacementNewTypes(
1419 Body: CurCodeDecl ? CurCodeDecl->getBody() : nullptr);
1420 return AllocTokenPlacementTypes->lookup(Val: Call);
1421 };
1422 return buildAllocToken(AllocType: infer_alloc::inferPossibleType(
1423 E, Ctx: getContext(), CastE: CurCast, PlacementNewType));
1424}
1425
1426void CodeGenFunction::EmitAllocToken(llvm::CallBase *CB, const CallExpr *E) {
1427 assert(SanOpts.has(SanitizerKind::AllocToken) &&
1428 "Only needed with -fsanitize=alloc-token");
1429 if (llvm::MDNode *MDN = buildAllocToken(E))
1430 CB->setMetadata(KindID: llvm::LLVMContext::MD_alloc_token, Node: MDN);
1431}
1432
1433CodeGenFunction::ComplexPairTy CodeGenFunction::
1434EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
1435 bool isInc, bool isPre) {
1436 ComplexPairTy InVal = EmitLoadOfComplex(src: LV, loc: E->getExprLoc());
1437
1438 llvm::Value *NextVal;
1439 if (isa<llvm::IntegerType>(Val: InVal.first->getType())) {
1440 uint64_t AmountVal = isInc ? 1 : -1;
1441 NextVal = llvm::ConstantInt::get(Ty: InVal.first->getType(), V: AmountVal, IsSigned: true);
1442
1443 // Add the inc/dec to the real part.
1444 NextVal = Builder.CreateAdd(LHS: InVal.first, RHS: NextVal, Name: isInc ? "inc" : "dec");
1445 } else {
1446 QualType ElemTy = E->getType()->castAs<ComplexType>()->getElementType();
1447 llvm::APFloat FVal(getContext().getFloatTypeSemantics(T: ElemTy), 1);
1448 if (!isInc)
1449 FVal.changeSign();
1450 NextVal = llvm::ConstantFP::get(Context&: getLLVMContext(), V: FVal);
1451
1452 // Add the inc/dec to the real part.
1453 NextVal = Builder.CreateFAdd(L: InVal.first, R: NextVal, Name: isInc ? "inc" : "dec");
1454 }
1455
1456 ComplexPairTy IncVal(NextVal, InVal.second);
1457
1458 // Store the updated result through the lvalue.
1459 EmitStoreOfComplex(V: IncVal, dest: LV, /*init*/ isInit: false);
1460 if (getLangOpts().OpenMP)
1461 CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF&: *this,
1462 LHS: E->getSubExpr());
1463
1464 // If this is a postinc, return the value read from memory, otherwise use the
1465 // updated value.
1466 return isPre ? IncVal : InVal;
1467}
1468
1469void CodeGenModule::EmitExplicitCastExprType(const ExplicitCastExpr *E,
1470 CodeGenFunction *CGF) {
1471 // Bind VLAs in the cast type.
1472 if (CGF && E->getType()->isVariablyModifiedType())
1473 CGF->EmitVariablyModifiedType(Ty: E->getType());
1474
1475 if (CGDebugInfo *DI = getModuleDebugInfo())
1476 DI->EmitExplicitCastType(Ty: E->getType());
1477}
1478
1479//===----------------------------------------------------------------------===//
1480// LValue Expression Emission
1481//===----------------------------------------------------------------------===//
1482
1483static CharUnits getArrayElementAlign(CharUnits arrayAlign, llvm::Value *idx,
1484 CharUnits eltSize) {
1485 // If we have a constant index, we can use the exact offset of the
1486 // element we're accessing.
1487 if (auto *constantIdx = dyn_cast<llvm::ConstantInt>(Val: idx)) {
1488 CharUnits offset = constantIdx->getZExtValue() * eltSize;
1489 return arrayAlign.alignmentAtOffset(offset);
1490 }
1491
1492 // Otherwise, use the worst-case alignment for any element.
1493 return arrayAlign.alignmentOfArrayElement(elementSize: eltSize);
1494}
1495
1496/// Emit pointer + index arithmetic.
1497static Address emitPointerArithmetic(CodeGenFunction &CGF,
1498 const BinaryOperator *BO,
1499 LValueBaseInfo *BaseInfo,
1500 TBAAAccessInfo *TBAAInfo,
1501 KnownNonNull_t IsKnownNonNull) {
1502 assert(BO->isAdditiveOp() && "Expect an addition or subtraction.");
1503 Expr *pointerOperand = BO->getLHS();
1504 Expr *indexOperand = BO->getRHS();
1505 bool isSubtraction = BO->getOpcode() == BO_Sub;
1506
1507 Address BaseAddr = Address::invalid();
1508 llvm::Value *index = nullptr;
1509 // In a subtraction, the LHS is always the pointer.
1510 // Note: do not change the evaluation order.
1511 if (!isSubtraction && !pointerOperand->getType()->isAnyPointerType()) {
1512 std::swap(a&: pointerOperand, b&: indexOperand);
1513 index = CGF.EmitScalarExpr(E: indexOperand);
1514 BaseAddr = CGF.EmitPointerWithAlignment(Addr: pointerOperand, BaseInfo, TBAAInfo,
1515 IsKnownNonNull: NotKnownNonNull);
1516 } else {
1517 BaseAddr = CGF.EmitPointerWithAlignment(Addr: pointerOperand, BaseInfo, TBAAInfo,
1518 IsKnownNonNull: NotKnownNonNull);
1519 index = CGF.EmitScalarExpr(E: indexOperand);
1520 }
1521
1522 llvm::Value *pointer = BaseAddr.getBasePointer();
1523 llvm::Value *Res = CGF.EmitPointerArithmetic(
1524 BO, pointerOperand, pointer, indexOperand, index, isSubtraction);
1525 QualType PointeeTy = BO->getType()->getPointeeType();
1526 CharUnits Align =
1527 getArrayElementAlign(arrayAlign: BaseAddr.getAlignment(), idx: index,
1528 eltSize: CGF.getContext().getTypeSizeInChars(T: PointeeTy));
1529 return Address(Res, CGF.ConvertTypeForMem(T: PointeeTy), Align,
1530 CGF.CGM.getPointerAuthInfoForPointeeType(type: PointeeTy),
1531 /*Offset=*/nullptr, IsKnownNonNull);
1532}
1533
1534static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo,
1535 TBAAAccessInfo *TBAAInfo,
1536 KnownNonNull_t IsKnownNonNull,
1537 CodeGenFunction &CGF) {
1538 // We allow this with ObjC object pointers because of fragile ABIs.
1539 assert(E->getType()->isPointerType() ||
1540 E->getType()->isObjCObjectPointerType());
1541 E = E->IgnoreParens();
1542
1543 // Casts:
1544 if (const CastExpr *CE = dyn_cast<CastExpr>(Val: E)) {
1545 if (const auto *ECE = dyn_cast<ExplicitCastExpr>(Val: CE))
1546 CGF.CGM.EmitExplicitCastExprType(E: ECE, CGF: &CGF);
1547
1548 switch (CE->getCastKind()) {
1549 // Non-converting casts (but not C's implicit conversion from void*).
1550 case CK_BitCast:
1551 case CK_NoOp:
1552 case CK_AddressSpaceConversion:
1553 if (auto PtrTy = CE->getSubExpr()->getType()->getAs<PointerType>()) {
1554 if (PtrTy->getPointeeType()->isVoidType())
1555 break;
1556
1557 LValueBaseInfo InnerBaseInfo;
1558 TBAAAccessInfo InnerTBAAInfo;
1559 Address Addr = CGF.EmitPointerWithAlignment(
1560 Addr: CE->getSubExpr(), BaseInfo: &InnerBaseInfo, TBAAInfo: &InnerTBAAInfo, IsKnownNonNull);
1561 if (BaseInfo) *BaseInfo = InnerBaseInfo;
1562 if (TBAAInfo) *TBAAInfo = InnerTBAAInfo;
1563
1564 if (isa<ExplicitCastExpr>(Val: CE)) {
1565 LValueBaseInfo TargetTypeBaseInfo;
1566 TBAAAccessInfo TargetTypeTBAAInfo;
1567 CharUnits Align = CGF.CGM.getNaturalPointeeTypeAlignment(
1568 T: E->getType(), BaseInfo: &TargetTypeBaseInfo, TBAAInfo: &TargetTypeTBAAInfo);
1569 if (TBAAInfo)
1570 *TBAAInfo =
1571 CGF.CGM.mergeTBAAInfoForCast(SourceInfo: *TBAAInfo, TargetInfo: TargetTypeTBAAInfo);
1572 // If the source l-value is opaque, honor the alignment of the
1573 // casted-to type.
1574 if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) {
1575 if (BaseInfo)
1576 BaseInfo->mergeForCast(Info: TargetTypeBaseInfo);
1577 Addr.setAlignment(Align);
1578 }
1579 }
1580
1581 if (CGF.SanOpts.has(K: SanitizerKind::CFIUnrelatedCast) &&
1582 CE->getCastKind() == CK_BitCast) {
1583 if (auto PT = E->getType()->getAs<PointerType>())
1584 CGF.EmitVTablePtrCheckForCast(T: PT->getPointeeType(), Derived: Addr,
1585 /*MayBeNull=*/true,
1586 TCK: CodeGenFunction::CFITCK_UnrelatedCast,
1587 Loc: CE->getBeginLoc());
1588 }
1589
1590 llvm::Type *ElemTy =
1591 CGF.ConvertTypeForMem(T: E->getType()->getPointeeType());
1592 Addr = Addr.withElementType(ElemTy);
1593 if (CE->getCastKind() == CK_AddressSpaceConversion)
1594 Addr = CGF.Builder.CreateAddrSpaceCast(
1595 Addr, Ty: CGF.ConvertType(T: E->getType()), ElementTy: ElemTy);
1596
1597 return CGF.authPointerToPointerCast(Ptr: Addr, SourceType: CE->getSubExpr()->getType(),
1598 DestType: CE->getType());
1599 }
1600 break;
1601
1602 // Array-to-pointer decay.
1603 case CK_ArrayToPointerDecay:
1604 return CGF.EmitArrayToPointerDecay(Array: CE->getSubExpr(), BaseInfo, TBAAInfo);
1605
1606 // Derived-to-base conversions.
1607 case CK_UncheckedDerivedToBase:
1608 case CK_DerivedToBase: {
1609 // TODO: Support accesses to members of base classes in TBAA. For now, we
1610 // conservatively pretend that the complete object is of the base class
1611 // type.
1612 if (TBAAInfo)
1613 *TBAAInfo = CGF.CGM.getTBAAAccessInfo(AccessType: E->getType());
1614 Address Addr = CGF.EmitPointerWithAlignment(
1615 Addr: CE->getSubExpr(), BaseInfo, TBAAInfo: nullptr,
1616 IsKnownNonNull: (KnownNonNull_t)(IsKnownNonNull ||
1617 CE->getCastKind() == CK_UncheckedDerivedToBase));
1618 auto Derived = CE->getSubExpr()->getType()->getPointeeCXXRecordDecl();
1619 return CGF.GetAddressOfBaseClass(
1620 Value: Addr, Derived, PathBegin: CE->path_begin(), PathEnd: CE->path_end(),
1621 NullCheckValue: CGF.ShouldNullCheckClassCastValue(Cast: CE), Loc: CE->getExprLoc());
1622 }
1623
1624 // TODO: Is there any reason to treat base-to-derived conversions
1625 // specially?
1626 default:
1627 break;
1628 }
1629 }
1630
1631 // Unary &.
1632 if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(Val: E)) {
1633 if (UO->getOpcode() == UO_AddrOf) {
1634 LValue LV = CGF.EmitLValue(E: UO->getSubExpr(), IsKnownNonNull);
1635 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1636 if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1637 return LV.getAddress();
1638 }
1639 }
1640
1641 // std::addressof and variants.
1642 if (auto *Call = dyn_cast<CallExpr>(Val: E)) {
1643 switch (Call->getBuiltinCallee()) {
1644 default:
1645 break;
1646 case Builtin::BIaddressof:
1647 case Builtin::BI__addressof:
1648 case Builtin::BI__builtin_addressof: {
1649 LValue LV = CGF.EmitLValue(E: Call->getArg(Arg: 0), IsKnownNonNull);
1650 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1651 if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1652 return LV.getAddress();
1653 }
1654 }
1655 }
1656
1657 // Pointer arithmetic: pointer +/- index.
1658 if (auto *BO = dyn_cast<BinaryOperator>(Val: E)) {
1659 if (BO->isAdditiveOp())
1660 return emitPointerArithmetic(CGF, BO, BaseInfo, TBAAInfo, IsKnownNonNull);
1661 }
1662
1663 // TODO: conditional operators, comma.
1664
1665 // Otherwise, use the alignment of the type.
1666 return CGF.makeNaturalAddressForPointer(
1667 Ptr: CGF.EmitScalarExpr(E), T: E->getType()->getPointeeType(), Alignment: CharUnits(),
1668 /*ForPointeeType=*/true, BaseInfo, TBAAInfo, IsKnownNonNull);
1669}
1670
1671/// EmitPointerWithAlignment - Given an expression of pointer type, try to
1672/// derive a more accurate bound on the alignment of the pointer.
1673Address CodeGenFunction::EmitPointerWithAlignment(
1674 const Expr *E, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo,
1675 KnownNonNull_t IsKnownNonNull) {
1676 Address Addr =
1677 ::EmitPointerWithAlignment(E, BaseInfo, TBAAInfo, IsKnownNonNull, CGF&: *this);
1678 if (IsKnownNonNull && !Addr.isKnownNonNull())
1679 Addr.setKnownNonNull();
1680 return Addr;
1681}
1682
1683llvm::Value *CodeGenFunction::EmitNonNullRValueCheck(RValue RV, QualType T) {
1684 llvm::Value *V = RV.getScalarVal();
1685 if (auto MPT = T->getAs<MemberPointerType>())
1686 return CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF&: *this, MemPtr: V, MPT);
1687 return Builder.CreateICmpNE(LHS: V, RHS: llvm::Constant::getNullValue(Ty: V->getType()));
1688}
1689
1690RValue CodeGenFunction::GetUndefRValue(QualType Ty) {
1691 if (Ty->isVoidType())
1692 return RValue::get(V: nullptr);
1693
1694 switch (getEvaluationKind(T: Ty)) {
1695 case TEK_Complex: {
1696 llvm::Type *EltTy =
1697 ConvertType(T: Ty->castAs<ComplexType>()->getElementType());
1698 llvm::Value *U = llvm::UndefValue::get(T: EltTy);
1699 return RValue::getComplex(C: std::make_pair(x&: U, y&: U));
1700 }
1701
1702 // If this is a use of an undefined aggregate type, the aggregate must have an
1703 // identifiable address. Just because the contents of the value are undefined
1704 // doesn't mean that the address can't be taken and compared.
1705 case TEK_Aggregate: {
1706 Address DestPtr = CreateMemTemp(Ty, Name: "undef.agg.tmp");
1707 return RValue::getAggregate(addr: DestPtr);
1708 }
1709
1710 case TEK_Scalar:
1711 return RValue::get(V: llvm::UndefValue::get(T: ConvertType(T: Ty)));
1712 }
1713 llvm_unreachable("bad evaluation kind");
1714}
1715
1716RValue CodeGenFunction::EmitUnsupportedRValue(const Expr *E,
1717 const char *Name) {
1718 ErrorUnsupported(S: E, Type: Name);
1719 return GetUndefRValue(Ty: E->getType());
1720}
1721
1722LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E,
1723 const char *Name) {
1724 ErrorUnsupported(S: E, Type: Name);
1725 llvm::Type *ElTy = ConvertType(T: E->getType());
1726 llvm::Type *Ty = DefaultPtrTy;
1727 return MakeAddrLValue(
1728 Addr: Address(llvm::UndefValue::get(T: Ty), ElTy, CharUnits::One()), T: E->getType());
1729}
1730
1731bool CodeGenFunction::IsWrappedCXXThis(const Expr *Obj) {
1732 const Expr *Base = Obj;
1733 while (!isa<CXXThisExpr>(Val: Base)) {
1734 // The result of a dynamic_cast can be null.
1735 if (isa<CXXDynamicCastExpr>(Val: Base))
1736 return false;
1737
1738 if (const auto *CE = dyn_cast<CastExpr>(Val: Base)) {
1739 Base = CE->getSubExpr();
1740 } else if (const auto *PE = dyn_cast<ParenExpr>(Val: Base)) {
1741 Base = PE->getSubExpr();
1742 } else if (const auto *UO = dyn_cast<UnaryOperator>(Val: Base)) {
1743 if (UO->getOpcode() == UO_Extension)
1744 Base = UO->getSubExpr();
1745 else
1746 return false;
1747 } else {
1748 return false;
1749 }
1750 }
1751 return true;
1752}
1753
1754LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {
1755 LValue LV;
1756 if (SanOpts.has(K: SanitizerKind::ArrayBounds) && isa<ArraySubscriptExpr>(Val: E))
1757 LV = EmitArraySubscriptExpr(E: cast<ArraySubscriptExpr>(Val: E), /*Accessed*/true);
1758 else
1759 LV = EmitLValue(E);
1760 if (!isa<DeclRefExpr>(Val: E) && !LV.isBitField() && LV.isSimple()) {
1761 SanitizerSet SkippedChecks;
1762 if (const auto *ME = dyn_cast<MemberExpr>(Val: E)) {
1763 bool IsBaseCXXThis = IsWrappedCXXThis(Obj: ME->getBase());
1764 if (IsBaseCXXThis)
1765 SkippedChecks.set(K: SanitizerKind::Alignment, Value: true);
1766 if (IsBaseCXXThis || isa<DeclRefExpr>(Val: ME->getBase()))
1767 SkippedChecks.set(K: SanitizerKind::Null, Value: true);
1768 }
1769 EmitTypeCheck(TCK, Loc: E->getExprLoc(), LV, Type: E->getType(), SkippedChecks);
1770 }
1771 return LV;
1772}
1773
1774/// EmitLValue - Emit code to compute a designator that specifies the location
1775/// of the expression.
1776///
1777/// This can return one of two things: a simple address or a bitfield reference.
1778/// In either case, the LLVM Value* in the LValue structure is guaranteed to be
1779/// an LLVM pointer type.
1780///
1781/// If this returns a bitfield reference, nothing about the pointee type of the
1782/// LLVM value is known: For example, it may not be a pointer to an integer.
1783///
1784/// If this returns a normal address, and if the lvalue's C type is fixed size,
1785/// this method guarantees that the returned pointer type will point to an LLVM
1786/// type of the same size of the lvalue's type. If the lvalue has a variable
1787/// length type, this is not possible.
1788///
1789LValue CodeGenFunction::EmitLValue(const Expr *E,
1790 KnownNonNull_t IsKnownNonNull) {
1791 // Running with sufficient stack space to avoid deeply nested expressions
1792 // cause a stack overflow.
1793 LValue LV;
1794 CGM.runWithSufficientStackSpace(
1795 Loc: E->getExprLoc(), Fn: [&] { LV = EmitLValueHelper(E, IsKnownNonNull); });
1796
1797 if (IsKnownNonNull && !LV.isKnownNonNull())
1798 LV.setKnownNonNull();
1799 return LV;
1800}
1801
1802LValue CodeGenFunction::EmitLValueHelper(const Expr *E,
1803 KnownNonNull_t IsKnownNonNull) {
1804 ApplyDebugLocation DL(*this, E);
1805 switch (E->getStmtClass()) {
1806 default: return EmitUnsupportedLValue(E, Name: "l-value expression");
1807
1808 case Expr::ObjCPropertyRefExprClass:
1809 llvm_unreachable("cannot emit a property reference directly");
1810
1811 case Expr::ObjCSelectorExprClass:
1812 return EmitObjCSelectorLValue(E: cast<ObjCSelectorExpr>(Val: E));
1813 case Expr::ObjCIsaExprClass:
1814 return EmitObjCIsaExpr(E: cast<ObjCIsaExpr>(Val: E));
1815 case Expr::BinaryOperatorClass:
1816 return EmitBinaryOperatorLValue(E: cast<BinaryOperator>(Val: E));
1817 case Expr::CompoundAssignOperatorClass: {
1818 QualType Ty = E->getType();
1819 if (const AtomicType *AT = Ty->getAs<AtomicType>())
1820 Ty = AT->getValueType();
1821 if (!Ty->isAnyComplexType())
1822 return EmitCompoundAssignmentLValue(E: cast<CompoundAssignOperator>(Val: E));
1823 return EmitComplexCompoundAssignmentLValue(E: cast<CompoundAssignOperator>(Val: E));
1824 }
1825 case Expr::CallExprClass:
1826 case Expr::CXXMemberCallExprClass:
1827 case Expr::CXXOperatorCallExprClass:
1828 case Expr::UserDefinedLiteralClass:
1829 return EmitCallExprLValue(E: cast<CallExpr>(Val: E));
1830 case Expr::CXXRewrittenBinaryOperatorClass:
1831 return EmitLValue(E: cast<CXXRewrittenBinaryOperator>(Val: E)->getSemanticForm(),
1832 IsKnownNonNull);
1833 case Expr::VAArgExprClass:
1834 return EmitVAArgExprLValue(E: cast<VAArgExpr>(Val: E));
1835 case Expr::DeclRefExprClass:
1836 return EmitDeclRefLValue(E: cast<DeclRefExpr>(Val: E));
1837 case Expr::ConstantExprClass: {
1838 const ConstantExpr *CE = cast<ConstantExpr>(Val: E);
1839 if (llvm::Value *Result = ConstantEmitter(*this).tryEmitConstantExpr(CE))
1840 return MakeNaturalAlignPointeeAddrLValue(V: Result, T: CE->getType());
1841 return EmitLValue(E: cast<ConstantExpr>(Val: E)->getSubExpr(), IsKnownNonNull);
1842 }
1843 case Expr::ParenExprClass:
1844 return EmitLValue(E: cast<ParenExpr>(Val: E)->getSubExpr(), IsKnownNonNull);
1845 case Expr::GenericSelectionExprClass:
1846 return EmitLValue(E: cast<GenericSelectionExpr>(Val: E)->getResultExpr(),
1847 IsKnownNonNull);
1848 case Expr::PredefinedExprClass:
1849 return EmitPredefinedLValue(E: cast<PredefinedExpr>(Val: E));
1850 case Expr::StringLiteralClass:
1851 return EmitStringLiteralLValue(E: cast<StringLiteral>(Val: E));
1852 case Expr::ObjCEncodeExprClass:
1853 return EmitObjCEncodeExprLValue(E: cast<ObjCEncodeExpr>(Val: E));
1854 case Expr::PseudoObjectExprClass:
1855 return EmitPseudoObjectLValue(e: cast<PseudoObjectExpr>(Val: E));
1856 case Expr::InitListExprClass:
1857 return EmitInitListLValue(E: cast<InitListExpr>(Val: E));
1858 case Expr::CXXTemporaryObjectExprClass:
1859 case Expr::CXXConstructExprClass:
1860 return EmitCXXConstructLValue(E: cast<CXXConstructExpr>(Val: E));
1861 case Expr::CXXBindTemporaryExprClass:
1862 return EmitCXXBindTemporaryLValue(E: cast<CXXBindTemporaryExpr>(Val: E));
1863 case Expr::CXXUuidofExprClass:
1864 return EmitCXXUuidofLValue(E: cast<CXXUuidofExpr>(Val: E));
1865 case Expr::LambdaExprClass:
1866 return EmitAggExprToLValue(E);
1867
1868 case Expr::ExprWithCleanupsClass: {
1869 const auto *cleanups = cast<ExprWithCleanups>(Val: E);
1870 RunCleanupsScope Scope(*this);
1871 LValue LV = EmitLValue(E: cleanups->getSubExpr(), IsKnownNonNull);
1872 if (LV.isSimple()) {
1873 // Defend against branches out of gnu statement expressions surrounded by
1874 // cleanups.
1875 Address Addr = LV.getAddress();
1876 llvm::Value *V = Addr.getBasePointer();
1877 Scope.ForceCleanup(ValuesToReload: {&V});
1878 Addr.replaceBasePointer(P: V);
1879 return LValue::MakeAddr(Addr, type: LV.getType(), Context&: getContext(),
1880 BaseInfo: LV.getBaseInfo(), TBAAInfo: LV.getTBAAInfo());
1881 }
1882 // FIXME: Is it possible to create an ExprWithCleanups that produces a
1883 // bitfield lvalue or some other non-simple lvalue?
1884 return LV;
1885 }
1886
1887 case Expr::CXXDefaultArgExprClass: {
1888 auto *DAE = cast<CXXDefaultArgExpr>(Val: E);
1889 CXXDefaultArgExprScope Scope(*this, DAE);
1890 return EmitLValue(E: DAE->getExpr(), IsKnownNonNull);
1891 }
1892 case Expr::CXXDefaultInitExprClass: {
1893 auto *DIE = cast<CXXDefaultInitExpr>(Val: E);
1894 CXXDefaultInitExprScope Scope(*this, DIE);
1895 return EmitLValue(E: DIE->getExpr(), IsKnownNonNull);
1896 }
1897 case Expr::CXXTypeidExprClass:
1898 return EmitCXXTypeidLValue(E: cast<CXXTypeidExpr>(Val: E));
1899
1900 case Expr::ObjCMessageExprClass:
1901 return EmitObjCMessageExprLValue(E: cast<ObjCMessageExpr>(Val: E));
1902 case Expr::ObjCIvarRefExprClass:
1903 return EmitObjCIvarRefLValue(E: cast<ObjCIvarRefExpr>(Val: E));
1904 case Expr::StmtExprClass:
1905 return EmitStmtExprLValue(E: cast<StmtExpr>(Val: E));
1906 case Expr::UnaryOperatorClass:
1907 return EmitUnaryOpLValue(E: cast<UnaryOperator>(Val: E));
1908 case Expr::ArraySubscriptExprClass:
1909 return EmitArraySubscriptExpr(E: cast<ArraySubscriptExpr>(Val: E));
1910 case Expr::MatrixSingleSubscriptExprClass:
1911 return EmitMatrixSingleSubscriptExpr(E: cast<MatrixSingleSubscriptExpr>(Val: E));
1912 case Expr::MatrixSubscriptExprClass:
1913 return EmitMatrixSubscriptExpr(E: cast<MatrixSubscriptExpr>(Val: E));
1914 case Expr::ArraySectionExprClass:
1915 return EmitArraySectionExpr(E: cast<ArraySectionExpr>(Val: E));
1916 case Expr::ExtVectorElementExprClass:
1917 return EmitExtVectorElementExpr(E: cast<ExtVectorElementExpr>(Val: E));
1918 case Expr::MatrixElementExprClass:
1919 return EmitMatrixElementExpr(E: cast<MatrixElementExpr>(Val: E));
1920 case Expr::CXXThisExprClass:
1921 return MakeAddrLValue(Addr: LoadCXXThisAddress(), T: E->getType());
1922 case Expr::MemberExprClass:
1923 return EmitMemberExpr(E: cast<MemberExpr>(Val: E));
1924 case Expr::CompoundLiteralExprClass:
1925 return EmitCompoundLiteralLValue(E: cast<CompoundLiteralExpr>(Val: E));
1926 case Expr::ConditionalOperatorClass:
1927 return EmitConditionalOperatorLValue(E: cast<ConditionalOperator>(Val: E));
1928 case Expr::BinaryConditionalOperatorClass:
1929 return EmitConditionalOperatorLValue(E: cast<BinaryConditionalOperator>(Val: E));
1930 case Expr::ChooseExprClass:
1931 return EmitLValue(E: cast<ChooseExpr>(Val: E)->getChosenSubExpr(), IsKnownNonNull);
1932 case Expr::OpaqueValueExprClass:
1933 return EmitOpaqueValueLValue(e: cast<OpaqueValueExpr>(Val: E));
1934 case Expr::SubstNonTypeTemplateParmExprClass:
1935 return EmitLValue(E: cast<SubstNonTypeTemplateParmExpr>(Val: E)->getReplacement(),
1936 IsKnownNonNull);
1937 case Expr::ImplicitCastExprClass:
1938 case Expr::CStyleCastExprClass:
1939 case Expr::CXXFunctionalCastExprClass:
1940 case Expr::CXXStaticCastExprClass:
1941 case Expr::CXXDynamicCastExprClass:
1942 case Expr::CXXReinterpretCastExprClass:
1943 case Expr::CXXConstCastExprClass:
1944 case Expr::CXXAddrspaceCastExprClass:
1945 case Expr::ObjCBridgedCastExprClass:
1946 return EmitCastLValue(E: cast<CastExpr>(Val: E));
1947
1948 case Expr::MaterializeTemporaryExprClass:
1949 return EmitMaterializeTemporaryExpr(M: cast<MaterializeTemporaryExpr>(Val: E));
1950
1951 case Expr::CoawaitExprClass:
1952 return EmitCoawaitLValue(E: cast<CoawaitExpr>(Val: E));
1953 case Expr::CoyieldExprClass:
1954 return EmitCoyieldLValue(E: cast<CoyieldExpr>(Val: E));
1955 case Expr::PackIndexingExprClass:
1956 return EmitLValue(E: cast<PackIndexingExpr>(Val: E)->getSelectedExpr());
1957 case Expr::HLSLOutArgExprClass:
1958 llvm_unreachable("cannot emit a HLSL out argument directly");
1959 }
1960}
1961
1962/// Try to emit a reference to the given value without producing it as
1963/// an l-value. This is just an optimization, but it avoids us needing
1964/// to emit global copies of variables if they're named without triggering
1965/// a formal use in a context where we can't emit a direct reference to them,
1966/// for instance if a block or lambda or a member of a local class uses a
1967/// const int variable or constexpr variable from an enclosing function.
1968CodeGenFunction::ConstantEmission
1969CodeGenFunction::tryEmitAsConstant(const DeclRefExpr *RefExpr) {
1970 const ValueDecl *Value = RefExpr->getDecl();
1971
1972 // The value needs to be an enum constant or a constant variable.
1973 CodeGenUtils::ConstantEmissionKind CEK;
1974 if (isa<ParmVarDecl>(Val: Value)) {
1975 CEK = CodeGenUtils::CEK_None;
1976 } else if (const auto *var = dyn_cast<VarDecl>(Val: Value)) {
1977 CEK = CodeGenUtils::checkVarTypeForConstantEmission(Type: var->getType());
1978 } else if (isa<EnumConstantDecl>(Val: Value)) {
1979 CEK = CodeGenUtils::CEK_AsValueOnly;
1980 } else {
1981 CEK = CodeGenUtils::CEK_None;
1982 }
1983 if (CEK == CodeGenUtils::CEK_None)
1984 return ConstantEmission();
1985
1986 Expr::EvalResult result;
1987 bool resultIsReference;
1988 QualType resultType;
1989
1990 // It's best to evaluate all the way as an r-value if that's permitted.
1991 if (CEK != CodeGenUtils::CEK_AsReferenceOnly &&
1992 RefExpr->EvaluateAsRValue(Result&: result, Ctx: getContext())) {
1993 resultIsReference = false;
1994 resultType = RefExpr->getType().getUnqualifiedType();
1995
1996 // Otherwise, try to evaluate as an l-value.
1997 } else if (CEK != CodeGenUtils::CEK_AsValueOnly &&
1998 RefExpr->EvaluateAsLValue(Result&: result, Ctx: getContext())) {
1999 resultIsReference = true;
2000 resultType = Value->getType();
2001
2002 // Failure.
2003 } else {
2004 return ConstantEmission();
2005 }
2006
2007 // In any case, if the initializer has side-effects, abandon ship.
2008 if (result.HasSideEffects)
2009 return ConstantEmission();
2010
2011 // In CUDA/HIP device compilation, a lambda may capture a reference variable
2012 // referencing a global host variable by copy. In this case the lambda should
2013 // make a copy of the value of the global host variable. The DRE of the
2014 // captured reference variable cannot be emitted as load from the host
2015 // global variable as compile time constant, since the host variable is not
2016 // accessible on device. The DRE of the captured reference variable has to be
2017 // loaded from captures.
2018 if (CGM.getLangOpts().CUDAIsDevice && result.Val.isLValue() &&
2019 RefExpr->refersToEnclosingVariableOrCapture()) {
2020 auto *MD = dyn_cast_or_null<CXXMethodDecl>(Val: CurCodeDecl);
2021 if (isLambdaMethod(DC: MD) && MD->getOverloadedOperator() == OO_Call) {
2022 const APValue::LValueBase &base = result.Val.getLValueBase();
2023 if (const ValueDecl *D = base.dyn_cast<const ValueDecl *>()) {
2024 if (const VarDecl *VD = dyn_cast<const VarDecl>(Val: D)) {
2025 if (!VD->hasAttr<CUDADeviceAttr>()) {
2026 return ConstantEmission();
2027 }
2028 }
2029 }
2030 }
2031 }
2032
2033 // Emit as a constant.
2034 llvm::Constant *C = ConstantEmitter(*this).emitAbstract(
2035 loc: RefExpr->getLocation(), value: result.Val, T: resultType);
2036
2037 // Make sure we emit a debug reference to the global variable.
2038 // This should probably fire even for
2039 if (isa<VarDecl>(Val: Value)) {
2040 if (!getContext().DeclMustBeEmitted(D: cast<VarDecl>(Val: Value)))
2041 EmitDeclRefExprDbgValue(E: RefExpr, Init: result.Val);
2042 } else {
2043 assert(isa<EnumConstantDecl>(Value));
2044 EmitDeclRefExprDbgValue(E: RefExpr, Init: result.Val);
2045 }
2046
2047 // If we emitted a reference constant, we need to dereference that.
2048 if (resultIsReference)
2049 return ConstantEmission::forReference(C);
2050
2051 return ConstantEmission::forValue(C);
2052}
2053
2054static DeclRefExpr *tryToConvertMemberExprToDeclRefExpr(CodeGenFunction &CGF,
2055 const MemberExpr *ME) {
2056 if (auto *VD = dyn_cast<VarDecl>(Val: ME->getMemberDecl())) {
2057 // Try to emit static variable member expressions as DREs.
2058 return DeclRefExpr::Create(
2059 Context: CGF.getContext(), QualifierLoc: NestedNameSpecifierLoc(), TemplateKWLoc: SourceLocation(), D: VD,
2060 /*RefersToEnclosingVariableOrCapture=*/false, NameLoc: ME->getExprLoc(),
2061 T: ME->getType(), VK: ME->getValueKind(), FoundD: nullptr, TemplateArgs: nullptr, NOUR: ME->isNonOdrUse());
2062 }
2063 return nullptr;
2064}
2065
2066CodeGenFunction::ConstantEmission
2067CodeGenFunction::tryEmitAsConstant(const MemberExpr *ME) {
2068 if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(CGF&: *this, ME))
2069 return tryEmitAsConstant(RefExpr: DRE);
2070 return ConstantEmission();
2071}
2072
2073llvm::Value *CodeGenFunction::emitScalarConstant(
2074 const CodeGenFunction::ConstantEmission &Constant, Expr *E) {
2075 assert(Constant && "not a constant");
2076 if (Constant.isReference())
2077 return EmitLoadOfLValue(V: Constant.getReferenceLValue(CGF&: *this, RefExpr: E),
2078 Loc: E->getExprLoc())
2079 .getScalarVal();
2080 return Constant.getValue();
2081}
2082
2083llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue,
2084 SourceLocation Loc) {
2085 return EmitLoadOfScalar(Addr: lvalue.getAddress(), Volatile: lvalue.isVolatile(),
2086 Ty: lvalue.getType(), Loc, BaseInfo: lvalue.getBaseInfo(),
2087 TBAAInfo: lvalue.getTBAAInfo(), isNontemporal: lvalue.isNontemporal());
2088}
2089
2090// This method SHOULD NOT be extended to support additional types, like BitInt
2091// types, without an opt-in bool controlled by a CodeGenOptions setting (like
2092// -fstrict-bool) and a new UBSan check (like SanitizerKind::Bool) as breaking
2093// that assumption would lead to memory corruption. See link for examples of how
2094// having a bool that has a value different from 0 or 1 in memory can lead to
2095// memory corruption.
2096// https://discourse.llvm.org/t/defining-what-happens-when-a-bool-isn-t-0-or-1/86778
2097static bool getRangeForType(CodeGenFunction &CGF, QualType Ty, llvm::APInt &Min,
2098 llvm::APInt &End, bool StrictEnums, bool StrictBool,
2099 bool IsBool) {
2100 const auto *ED = Ty->getAsEnumDecl();
2101 bool IsRegularCPlusPlusEnum =
2102 CGF.getLangOpts().CPlusPlus && StrictEnums && ED && !ED->isFixed();
2103 if (!IsBool && !IsRegularCPlusPlusEnum)
2104 return false;
2105
2106 if (IsBool) {
2107 if (!StrictBool)
2108 return false;
2109 Min = llvm::APInt(CGF.getContext().getTypeSize(T: Ty), 0);
2110 End = llvm::APInt(CGF.getContext().getTypeSize(T: Ty), 2);
2111 } else {
2112 ED->getValueRange(Max&: End, Min);
2113 }
2114 return true;
2115}
2116
2117llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) {
2118 llvm::APInt Min, End;
2119 bool IsBool = Ty->hasBooleanRepresentation() && !Ty->isVectorType();
2120 bool StrictBoolEnabled = CGM.getCodeGenOpts().getLoadBoolFromMem() ==
2121 CodeGenOptions::BoolFromMem::Strict;
2122 if (!getRangeForType(CGF&: *this, Ty, Min, End,
2123 /*StrictEnums=*/CGM.getCodeGenOpts().StrictEnums,
2124 /*StrictBool=*/StrictBoolEnabled, /*IsBool=*/IsBool))
2125 return nullptr;
2126
2127 llvm::MDBuilder MDHelper(getLLVMContext());
2128 return MDHelper.createRange(Lo: Min, Hi: End);
2129}
2130
2131void CodeGenFunction::maybeAttachRangeForLoad(llvm::LoadInst *Load, QualType Ty,
2132 SourceLocation Loc) {
2133 if (EmitScalarRangeCheck(Value: Load, Ty, Loc)) {
2134 // In order to prevent the optimizer from throwing away the check, don't
2135 // attach range metadata to the load.
2136 } else if (CGM.getCodeGenOpts().isOptimizedBuild()) {
2137 if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty)) {
2138 Load->setMetadata(KindID: llvm::LLVMContext::MD_range, Node: RangeInfo);
2139 Load->setMetadata(KindID: llvm::LLVMContext::MD_noundef,
2140 Node: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: {}));
2141 }
2142 }
2143}
2144
2145bool CodeGenFunction::EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
2146 SourceLocation Loc) {
2147 bool HasBoolCheck = SanOpts.has(K: SanitizerKind::Bool);
2148 bool HasEnumCheck = SanOpts.has(K: SanitizerKind::Enum);
2149 if (!HasBoolCheck && !HasEnumCheck)
2150 return false;
2151
2152 bool IsBool = (Ty->hasBooleanRepresentation() && !Ty->isVectorType()) ||
2153 NSAPI(CGM.getContext()).isObjCBOOLType(T: Ty);
2154 bool NeedsBoolCheck = HasBoolCheck && IsBool;
2155 bool NeedsEnumCheck = HasEnumCheck && Ty->isEnumeralType();
2156 if (!NeedsBoolCheck && !NeedsEnumCheck)
2157 return false;
2158
2159 // Single-bit booleans don't need to be checked. Special-case this to avoid
2160 // a bit width mismatch when handling bitfield values. This is handled by
2161 // EmitFromMemory for the non-bitfield case.
2162 if (IsBool &&
2163 cast<llvm::IntegerType>(Val: Value->getType())->getBitWidth() == 1)
2164 return false;
2165
2166 if (NeedsEnumCheck &&
2167 getContext().isTypeIgnoredBySanitizer(Mask: SanitizerKind::Enum, Ty))
2168 return false;
2169
2170 llvm::APInt Min, End;
2171 if (!getRangeForType(CGF&: *this, Ty, Min, End, /*StrictEnums=*/true,
2172 /*StrictBool=*/true, IsBool))
2173 return true;
2174
2175 SanitizerKind::SanitizerOrdinal Kind =
2176 NeedsEnumCheck ? SanitizerKind::SO_Enum : SanitizerKind::SO_Bool;
2177
2178 auto &Ctx = getLLVMContext();
2179 auto CheckHandler = SanitizerHandler::LoadInvalidValue;
2180 SanitizerDebugLocation SanScope(this, {Kind}, CheckHandler);
2181 llvm::Value *Check;
2182 --End;
2183 if (!Min) {
2184 Check = Builder.CreateICmpULE(LHS: Value, RHS: llvm::ConstantInt::get(Context&: Ctx, V: End));
2185 } else {
2186 llvm::Value *Upper =
2187 Builder.CreateICmpSLE(LHS: Value, RHS: llvm::ConstantInt::get(Context&: Ctx, V: End));
2188 llvm::Value *Lower =
2189 Builder.CreateICmpSGE(LHS: Value, RHS: llvm::ConstantInt::get(Context&: Ctx, V: Min));
2190 Check = Builder.CreateAnd(LHS: Upper, RHS: Lower);
2191 }
2192 llvm::Constant *StaticArgs[] = {EmitCheckSourceLocation(Loc),
2193 EmitCheckTypeDescriptor(T: Ty)};
2194 EmitCheck(Checked: std::make_pair(x&: Check, y&: Kind), Check: CheckHandler, StaticArgs, DynamicArgs: Value);
2195 return true;
2196}
2197
2198llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
2199 QualType Ty,
2200 SourceLocation Loc,
2201 LValueBaseInfo BaseInfo,
2202 TBAAAccessInfo TBAAInfo,
2203 bool isNontemporal) {
2204 if (auto *GV = dyn_cast<llvm::GlobalValue>(Val: Addr.getBasePointer()))
2205 if (GV->isThreadLocal())
2206 Addr = Addr.withPointer(NewPointer: Builder.CreateThreadLocalAddress(Ptr: GV),
2207 IsKnownNonNull: NotKnownNonNull);
2208
2209 if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
2210 // Boolean vectors use `iN` as storage type.
2211 if (ClangVecTy->isPackedVectorBoolType(ctx: getContext())) {
2212 llvm::Type *ValTy = ConvertType(T: Ty);
2213 unsigned ValNumElems =
2214 cast<llvm::FixedVectorType>(Val: ValTy)->getNumElements();
2215 // Load the `iP` storage object (P is the padded vector size).
2216 auto *RawIntV = Builder.CreateLoad(Addr, IsVolatile: Volatile, Name: "load_bits");
2217 const auto *RawIntTy = RawIntV->getType();
2218 assert(RawIntTy->isIntegerTy() && "compressed iN storage for bitvectors");
2219 // Bitcast iP --> <P x i1>.
2220 auto *PaddedVecTy = llvm::FixedVectorType::get(
2221 ElementType: Builder.getInt1Ty(), NumElts: RawIntTy->getPrimitiveSizeInBits());
2222 llvm::Value *V = Builder.CreateBitCast(V: RawIntV, DestTy: PaddedVecTy);
2223 // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
2224 V = emitBoolVecConversion(SrcVec: V, NumElementsDst: ValNumElems, Name: "extractvec");
2225
2226 return EmitFromMemory(Value: V, Ty);
2227 }
2228
2229 // Handles vectors of sizes that are likely to be expanded to a larger size
2230 // to optimize performance.
2231 auto *VTy = cast<llvm::FixedVectorType>(Val: Addr.getElementType());
2232 auto *NewVecTy =
2233 CGM.getABIInfo().getOptimalVectorMemoryType(T: VTy, Opt: getLangOpts());
2234
2235 if (VTy != NewVecTy) {
2236 Address Cast = Addr.withElementType(ElemTy: NewVecTy);
2237 llvm::Value *V = Builder.CreateLoad(Addr: Cast, IsVolatile: Volatile, Name: "loadVecN");
2238 unsigned OldNumElements = VTy->getNumElements();
2239 SmallVector<int, 16> Mask(OldNumElements);
2240 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
2241 V = Builder.CreateShuffleVector(V, Mask, Name: "extractVec");
2242 return EmitFromMemory(Value: V, Ty);
2243 }
2244 }
2245
2246 // Atomic operations have to be done on integral types.
2247 LValue AtomicLValue =
2248 LValue::MakeAddr(Addr, type: Ty, Context&: getContext(), BaseInfo, TBAAInfo);
2249 if (Ty->isAtomicType() || LValueIsSuitableForInlineAtomic(Src: AtomicLValue)) {
2250 return EmitAtomicLoad(LV: AtomicLValue, SL: Loc).getScalarVal();
2251 }
2252
2253 Addr =
2254 Addr.withElementType(ElemTy: convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Addr.getElementType()));
2255
2256 llvm::LoadInst *Load = Builder.CreateLoad(Addr, IsVolatile: Volatile);
2257 if (isNontemporal) {
2258 llvm::MDNode *Node = llvm::MDNode::get(
2259 Context&: Load->getContext(), MDs: llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: 1)));
2260 Load->setMetadata(KindID: llvm::LLVMContext::MD_nontemporal, Node);
2261 }
2262
2263 CGM.DecorateInstructionWithTBAA(Inst: Load, TBAAInfo);
2264
2265 maybeAttachRangeForLoad(Load, Ty, Loc);
2266
2267 return EmitFromMemory(Value: Load, Ty);
2268}
2269
2270/// Converts a scalar value from its primary IR type (as returned
2271/// by ConvertType) to its load/store type (as returned by
2272/// convertTypeForLoadStore).
2273llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) {
2274 if (auto *AtomicTy = Ty->getAs<AtomicType>())
2275 Ty = AtomicTy->getValueType();
2276
2277 if (Ty->isExtVectorBoolType() || Ty->isConstantMatrixBoolType()) {
2278 llvm::Type *StoreTy = convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Value->getType());
2279
2280 if (Value->getType() == StoreTy)
2281 return Value;
2282
2283 if (StoreTy->isVectorTy() && StoreTy->getScalarSizeInBits() >
2284 Value->getType()->getScalarSizeInBits())
2285 return Builder.CreateZExt(V: Value, DestTy: StoreTy);
2286
2287 // Expand to the memory bit width.
2288 unsigned MemNumElems = StoreTy->getPrimitiveSizeInBits();
2289 // <N x i1> --> <P x i1>.
2290 Value = emitBoolVecConversion(SrcVec: Value, NumElementsDst: MemNumElems, Name: "insertvec");
2291 // <P x i1> --> iP.
2292 Value = Builder.CreateBitCast(V: Value, DestTy: StoreTy);
2293 }
2294
2295 if (Ty->hasBooleanRepresentation() || Ty->isBitIntType()) {
2296 llvm::Type *StoreTy = convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Value->getType());
2297 bool Signed = Ty->isSignedIntegerOrEnumerationType();
2298 return Builder.CreateIntCast(V: Value, DestTy: StoreTy, isSigned: Signed, Name: "storedv");
2299 }
2300
2301 return Value;
2302}
2303
2304/// Converts a scalar value from its load/store type (as returned
2305/// by convertTypeForLoadStore) to its primary IR type (as returned
2306/// by ConvertType).
2307llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
2308 if (auto *AtomicTy = Ty->getAs<AtomicType>())
2309 Ty = AtomicTy->getValueType();
2310
2311 if (Ty->isPackedVectorBoolType(ctx: getContext())) {
2312 const auto *RawIntTy = Value->getType();
2313
2314 // Bitcast iP --> <P x i1>.
2315 auto *PaddedVecTy = llvm::FixedVectorType::get(
2316 ElementType: Builder.getInt1Ty(), NumElts: RawIntTy->getPrimitiveSizeInBits());
2317 auto *V = Builder.CreateBitCast(V: Value, DestTy: PaddedVecTy);
2318 // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
2319 llvm::Type *ValTy = ConvertType(T: Ty);
2320 unsigned ValNumElems = cast<llvm::FixedVectorType>(Val: ValTy)->getNumElements();
2321 return emitBoolVecConversion(SrcVec: V, NumElementsDst: ValNumElems, Name: "extractvec");
2322 }
2323
2324 llvm::Type *ResTy = ConvertType(T: Ty);
2325 bool HasBoolRep = Ty->hasBooleanRepresentation() ||
2326 Ty->isExtVectorBoolType() || Ty->isConstantMatrixBoolType();
2327 if (HasBoolRep && CGM.getCodeGenOpts().isConvertingBoolWithCmp0()) {
2328 return Builder.CreateICmpNE(
2329 LHS: Value, RHS: llvm::Constant::getNullValue(Ty: Value->getType()), Name: "loadedv");
2330 }
2331 if (HasBoolRep || Ty->isBitIntType())
2332 return Builder.CreateTrunc(V: Value, DestTy: ResTy, Name: "loadedv");
2333
2334 return Value;
2335}
2336
2337// Convert the pointer of \p Addr to a pointer to a vector (the value type of
2338// MatrixType), if it points to a array (the memory type of MatrixType).
2339static RawAddress MaybeConvertMatrixAddress(RawAddress Addr,
2340 CodeGenFunction &CGF,
2341 bool IsVector = true) {
2342 auto *ArrayTy = dyn_cast<llvm::ArrayType>(Val: Addr.getElementType());
2343 if (ArrayTy && IsVector) {
2344 auto ArrayElements = ArrayTy->getNumElements();
2345 auto *ArrayElementTy = ArrayTy->getElementType();
2346 if (CGF.getContext().getLangOpts().HLSL) {
2347 auto *VectorTy = cast<llvm::FixedVectorType>(Val: ArrayElementTy);
2348 ArrayElementTy = VectorTy->getElementType();
2349 ArrayElements *= VectorTy->getNumElements();
2350 }
2351 auto *VectorTy = llvm::FixedVectorType::get(ElementType: ArrayElementTy, NumElts: ArrayElements);
2352
2353 return Addr.withElementType(ElemTy: VectorTy);
2354 }
2355 auto *VectorTy = dyn_cast<llvm::VectorType>(Val: Addr.getElementType());
2356 if (VectorTy && !IsVector) {
2357 auto *ArrayTy = llvm::ArrayType::get(
2358 ElementType: VectorTy->getElementType(),
2359 NumElements: cast<llvm::FixedVectorType>(Val: VectorTy)->getNumElements());
2360
2361 return Addr.withElementType(ElemTy: ArrayTy);
2362 }
2363
2364 return Addr;
2365}
2366
2367// Emit a store of a matrix LValue. This may require casting the original
2368// pointer to memory address (ArrayType) to a pointer to the value type
2369// (VectorType).
2370static void EmitStoreOfMatrixScalar(llvm::Value *value, LValue lvalue,
2371 bool isInit, CodeGenFunction &CGF) {
2372 if (CGF.getLangOpts().HLSL &&
2373 isMatrixRowMajor(LangOpts: CGF.getLangOpts(), T: lvalue.getType())) {
2374 const auto *MatrixTy = lvalue.getType()->castAs<ConstantMatrixType>();
2375 llvm::MatrixBuilder MB(CGF.Builder);
2376 value = MB.CreateColumnMajorToRowMajorTransform(
2377 Matrix: value, Rows: MatrixTy->getNumRows(), Columns: MatrixTy->getNumColumns());
2378 }
2379 Address Addr = MaybeConvertMatrixAddress(Addr: lvalue.getAddress(), CGF,
2380 IsVector: value->getType()->isVectorTy());
2381 CGF.EmitStoreOfScalar(Value: value, Addr, Volatile: lvalue.isVolatile(), Ty: lvalue.getType(),
2382 BaseInfo: lvalue.getBaseInfo(), TBAAInfo: lvalue.getTBAAInfo(), isInit,
2383 isNontemporal: lvalue.isNontemporal());
2384}
2385
2386LValue CodeGenFunction::EmitMatrixElementExpr(const MatrixElementExpr *E) {
2387 LValue Base;
2388 if (E->getBase()->isGLValue())
2389 Base = EmitLValue(E: E->getBase());
2390 else {
2391 assert(E->getBase()->getType()->isConstantMatrixType() &&
2392 "Result must be a Constant Matrix");
2393 llvm::Value *Mat = EmitScalarExpr(E: E->getBase());
2394 Address MatMem = CreateMemTemp(Ty: E->getBase()->getType());
2395 QualType Ty = E->getBase()->getType();
2396 Base = MakeAddrLValue(Addr: MatMem, T: Ty, Source: AlignmentSource::Decl);
2397 EmitStoreOfMatrixScalar(value: Mat, lvalue: Base, /*isInit=*/true, CGF&: *this);
2398 }
2399 QualType ResultType =
2400 E->getType().withCVRQualifiers(CVR: Base.getQuals().getCVRQualifiers());
2401
2402 // Encode the element access list into a vector of unsigned indices.
2403 // getEncodedElementAccess returns row-major linearized indices.
2404 SmallVector<uint32_t, 4> Indices;
2405 E->getEncodedElementAccess(Elts&: Indices);
2406
2407 // getEncodedElementAccess returns row-major linearized indices
2408 // If the matrix memory layout is column-major, convert indices
2409 // to column-major indices.
2410 bool IsRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: E->getBase()->getType());
2411 if (!IsRowMajor) {
2412 const auto *MT = E->getBase()->getType()->castAs<ConstantMatrixType>();
2413 unsigned NumCols = MT->getNumColumns();
2414 for (uint32_t &Idx : Indices) {
2415 // Decompose row-major index: Row = Idx / NumCols, Col = Idx % NumCols
2416 unsigned Row = Idx / NumCols;
2417 unsigned Col = Idx % NumCols;
2418 // Re-linearize as column-major
2419 Idx = MT->getColumnMajorFlattenedIndex(Row, Column: Col);
2420 }
2421 }
2422
2423 if (Base.isSimple()) {
2424 RawAddress MatAddr = Base.getAddress();
2425 if (getLangOpts().HLSL &&
2426 E->getBase()->getType().getAddressSpace() == LangAS::hlsl_constant)
2427 MatAddr = CGM.getHLSLRuntime().createBufferMatrixTempAddress(LV: Base, CGF&: *this);
2428
2429 llvm::Constant *CV =
2430 llvm::ConstantDataVector::get(Context&: getLLVMContext(), Elts: Indices);
2431 return LValue::MakeExtVectorElt(Addr: MaybeConvertMatrixAddress(Addr: MatAddr, CGF&: *this),
2432 Elts: CV, type: ResultType, BaseInfo: Base.getBaseInfo(),
2433 TBAAInfo: TBAAAccessInfo());
2434 }
2435 assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
2436
2437 llvm::Constant *BaseElts = Base.getExtVectorElts();
2438 SmallVector<llvm::Constant *, 4> CElts;
2439
2440 for (unsigned Index : Indices)
2441 CElts.push_back(Elt: BaseElts->getAggregateElement(Elt: Index));
2442 llvm::Constant *CV = llvm::ConstantVector::get(V: CElts);
2443
2444 return LValue::MakeExtVectorElt(
2445 Addr: MaybeConvertMatrixAddress(Addr: Base.getExtVectorAddress(), CGF&: *this), Elts: CV,
2446 type: ResultType, BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
2447}
2448
2449void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr,
2450 bool Volatile, QualType Ty,
2451 LValueBaseInfo BaseInfo,
2452 TBAAAccessInfo TBAAInfo,
2453 bool isInit, bool isNontemporal) {
2454 if (auto *GV = dyn_cast<llvm::GlobalValue>(Val: Addr.getBasePointer()))
2455 if (GV->isThreadLocal())
2456 Addr = Addr.withPointer(NewPointer: Builder.CreateThreadLocalAddress(Ptr: GV),
2457 IsKnownNonNull: NotKnownNonNull);
2458
2459 // Handles vectors of sizes that are likely to be expanded to a larger size
2460 // to optimize performance.
2461 llvm::Type *SrcTy = Value->getType();
2462 if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
2463 if (auto *VecTy = dyn_cast<llvm::FixedVectorType>(Val: SrcTy)) {
2464 auto *NewVecTy =
2465 CGM.getABIInfo().getOptimalVectorMemoryType(T: VecTy, Opt: getLangOpts());
2466 if (!ClangVecTy->isPackedVectorBoolType(ctx: getContext()) &&
2467 VecTy != NewVecTy) {
2468 SmallVector<int, 16> Mask(NewVecTy->getNumElements(),
2469 VecTy->getNumElements());
2470 std::iota(first: Mask.begin(), last: Mask.begin() + VecTy->getNumElements(), value: 0);
2471 // Use undef instead of poison for the padding lanes, to make sure no
2472 // padding bits are poisoned, which may break coercion.
2473 Value = Builder.CreateShuffleVector(V1: Value, V2: llvm::UndefValue::get(T: VecTy),
2474 Mask, Name: "extractVec");
2475 SrcTy = NewVecTy;
2476 }
2477 if (Addr.getElementType() != SrcTy)
2478 Addr = Addr.withElementType(ElemTy: SrcTy);
2479 }
2480 }
2481
2482 Value = EmitToMemory(Value, Ty);
2483
2484 LValue AtomicLValue =
2485 LValue::MakeAddr(Addr, type: Ty, Context&: getContext(), BaseInfo, TBAAInfo);
2486 if (Ty->isAtomicType() ||
2487 (!isInit && LValueIsSuitableForInlineAtomic(Src: AtomicLValue))) {
2488 EmitAtomicStore(rvalue: RValue::get(V: Value), lvalue: AtomicLValue, isInit);
2489 return;
2490 }
2491
2492 llvm::StoreInst *Store = Builder.CreateStore(Val: Value, Addr, IsVolatile: Volatile);
2493 addInstToCurrentSourceAtom(KeyInstruction: Store, Backup: Value);
2494
2495 if (isNontemporal) {
2496 llvm::MDNode *Node =
2497 llvm::MDNode::get(Context&: Store->getContext(),
2498 MDs: llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: 1)));
2499 Store->setMetadata(KindID: llvm::LLVMContext::MD_nontemporal, Node);
2500 }
2501
2502 CGM.DecorateInstructionWithTBAA(Inst: Store, TBAAInfo);
2503}
2504
2505void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
2506 bool isInit) {
2507 if (lvalue.getType()->isConstantMatrixType()) {
2508 EmitStoreOfMatrixScalar(value, lvalue, isInit, CGF&: *this);
2509 return;
2510 }
2511
2512 EmitStoreOfScalar(Value: value, Addr: lvalue.getAddress(), Volatile: lvalue.isVolatile(),
2513 Ty: lvalue.getType(), BaseInfo: lvalue.getBaseInfo(),
2514 TBAAInfo: lvalue.getTBAAInfo(), isInit, isNontemporal: lvalue.isNontemporal());
2515}
2516
2517// Emit a load of a LValue of matrix type. This may require casting the pointer
2518// to memory address (ArrayType) to a pointer to the value type (VectorType).
2519static RValue EmitLoadOfMatrixLValue(LValue LV, SourceLocation Loc,
2520 CodeGenFunction &CGF) {
2521 assert(LV.getType()->isConstantMatrixType());
2522 RawAddress DestAddr = LV.getAddress();
2523
2524 // HLSL constant buffers may pad matrix layouts, so copy elements into a
2525 // non-padded local alloca before loading.
2526 if (CGF.getLangOpts().HLSL &&
2527 LV.getType().getAddressSpace() == LangAS::hlsl_constant)
2528 DestAddr = CGF.CGM.getHLSLRuntime().createBufferMatrixTempAddress(LV, CGF);
2529
2530 Address Addr = MaybeConvertMatrixAddress(Addr: DestAddr, CGF);
2531 LV.setAddress(Addr);
2532 llvm::Value *Value = CGF.EmitLoadOfScalar(lvalue: LV, Loc);
2533 if (CGF.getLangOpts().HLSL &&
2534 isMatrixRowMajor(LangOpts: CGF.getLangOpts(), T: LV.getType())) {
2535 const auto *MatrixTy = LV.getType()->castAs<ConstantMatrixType>();
2536 llvm::MatrixBuilder MB(CGF.Builder);
2537 Value = MB.CreateRowMajorToColumnMajorTransform(
2538 Matrix: Value, Rows: MatrixTy->getNumRows(), Columns: MatrixTy->getNumColumns());
2539 }
2540 return RValue::get(V: Value);
2541}
2542
2543RValue CodeGenFunction::EmitLoadOfAnyValue(LValue LV, AggValueSlot Slot,
2544 SourceLocation Loc) {
2545 QualType Ty = LV.getType();
2546 switch (getEvaluationKind(T: Ty)) {
2547 case TEK_Scalar:
2548 return EmitLoadOfLValue(V: LV, Loc);
2549 case TEK_Complex:
2550 return RValue::getComplex(C: EmitLoadOfComplex(src: LV, loc: Loc));
2551 case TEK_Aggregate:
2552 EmitAggFinalDestCopy(Type: Ty, Dest: Slot, Src: LV, SrcKind: EVK_NonRValue);
2553 return Slot.asRValue();
2554 }
2555 llvm_unreachable("bad evaluation kind");
2556}
2557
2558/// EmitLoadOfLValue - Given an expression that represents a value lvalue, this
2559/// method emits the address of the lvalue, then loads the result as an rvalue,
2560/// returning the rvalue.
2561RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) {
2562 // Load from __ptrauth.
2563 if (PointerAuthQualifier PtrAuth = LV.getQuals().getPointerAuth()) {
2564 LV.getQuals().removePointerAuth();
2565 llvm::Value *Value = EmitLoadOfLValue(LV, Loc).getScalarVal();
2566 return RValue::get(V: EmitPointerAuthUnqualify(Qualifier: PtrAuth, Pointer: Value, PointerType: LV.getType(),
2567 StorageAddress: LV.getAddress(),
2568 /*known nonnull*/ IsKnownNonNull: false));
2569 }
2570
2571 if (LV.isObjCWeak()) {
2572 // load of a __weak object.
2573 Address AddrWeakObj = LV.getAddress();
2574 return RValue::get(V: CGM.getObjCRuntime().EmitObjCWeakRead(CGF&: *this,
2575 AddrWeakObj));
2576 }
2577 if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) {
2578 // In MRC mode, we do a load+autorelease.
2579 if (!getLangOpts().ObjCAutoRefCount) {
2580 return RValue::get(V: EmitARCLoadWeak(addr: LV.getAddress()));
2581 }
2582
2583 // In ARC mode, we load retained and then consume the value.
2584 llvm::Value *Object = EmitARCLoadWeakRetained(addr: LV.getAddress());
2585 Object = EmitObjCConsumeObject(T: LV.getType(), Ptr: Object);
2586 return RValue::get(V: Object);
2587 }
2588
2589 if (LV.isSimple()) {
2590 assert(!LV.getType()->isFunctionType());
2591
2592 if (LV.getType()->isConstantMatrixType())
2593 return EmitLoadOfMatrixLValue(LV, Loc, CGF&: *this);
2594
2595 // Everything needs a load.
2596 return RValue::get(V: EmitLoadOfScalar(lvalue: LV, Loc));
2597 }
2598
2599 if (LV.isVectorElt()) {
2600 llvm::LoadInst *Load = Builder.CreateLoad(Addr: LV.getVectorAddress(),
2601 IsVolatile: LV.isVolatileQualified());
2602 llvm::Value *Elt =
2603 Builder.CreateExtractElement(Vec: Load, Idx: LV.getVectorIdx(), Name: "vecext");
2604 return RValue::get(V: EmitFromMemory(Value: Elt, Ty: LV.getType()));
2605 }
2606
2607 // If this is a reference to a subset of the elements of a vector, either
2608 // shuffle the input or extract/insert them as appropriate.
2609 if (LV.isExtVectorElt()) {
2610 return EmitLoadOfExtVectorElementLValue(V: LV);
2611 }
2612
2613 // Global Register variables always invoke intrinsics
2614 if (LV.isGlobalReg())
2615 return EmitLoadOfGlobalRegLValue(LV);
2616
2617 if (LV.isMatrixElt()) {
2618 llvm::Value *Idx = LV.getMatrixIdx();
2619 QualType EltTy = LV.getType();
2620 if (const auto *MatTy = EltTy->getAs<ConstantMatrixType>()) {
2621 EltTy = MatTy->getElementType();
2622 if (CGM.getCodeGenOpts().isOptimizedBuild()) {
2623 llvm::MatrixBuilder MB(Builder);
2624 MB.CreateIndexAssumption(Idx, NumElements: MatTy->getNumElementsFlattened());
2625 }
2626 }
2627 llvm::LoadInst *Load =
2628 Builder.CreateLoad(Addr: LV.getMatrixAddress(), IsVolatile: LV.isVolatileQualified());
2629 llvm::Value *Elt = Builder.CreateExtractElement(Vec: Load, Idx, Name: "matrixext");
2630 return RValue::get(V: EmitFromMemory(Value: Elt, Ty: EltTy));
2631 }
2632 if (LV.isMatrixRow()) {
2633 QualType MatTy = LV.getType();
2634 const ConstantMatrixType *MT = MatTy->castAs<ConstantMatrixType>();
2635
2636 unsigned NumRows = MT->getNumRows();
2637 unsigned NumCols = MT->getNumColumns();
2638 unsigned NumLanes = NumCols;
2639 llvm::Value *MatrixVec = EmitLoadOfScalar(lvalue: LV, Loc);
2640 llvm::Value *Row = LV.getMatrixRowIdx();
2641 llvm::Type *ElemTy = ConvertType(T: MT->getElementType());
2642 llvm::Constant *ColConstsIndices = nullptr;
2643 llvm::MatrixBuilder MB(Builder);
2644
2645 if (LV.isMatrixRowSwizzle()) {
2646 ColConstsIndices = LV.getMatrixRowElts();
2647 NumLanes = llvm::cast<llvm::FixedVectorType>(Val: ColConstsIndices->getType())
2648 ->getNumElements();
2649 }
2650
2651 llvm::Type *RowTy = llvm::FixedVectorType::get(ElementType: ElemTy, NumElts: NumLanes);
2652 llvm::Value *Result = llvm::PoisonValue::get(T: RowTy); // <NumLanes x T>
2653
2654 for (unsigned Col = 0; Col < NumLanes; ++Col) {
2655 llvm::Value *ColIdx;
2656 if (ColConstsIndices)
2657 ColIdx = ColConstsIndices->getAggregateElement(Elt: Col);
2658 else
2659 ColIdx = llvm::ConstantInt::get(Ty: Row->getType(), V: Col);
2660 bool IsMatrixRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: MatTy);
2661 llvm::Value *EltIndex =
2662 MB.CreateIndex(RowIdx: Row, ColumnIdx: ColIdx, NumRows, NumCols, IsMatrixRowMajor);
2663 llvm::Value *Elt = Builder.CreateExtractElement(Vec: MatrixVec, Idx: EltIndex);
2664 llvm::Value *Lane = llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: Col);
2665 Result = Builder.CreateInsertElement(Vec: Result, NewElt: Elt, Idx: Lane);
2666 }
2667
2668 return RValue::get(V: Result);
2669 }
2670
2671 assert(LV.isBitField() && "Unknown LValue type!");
2672 return EmitLoadOfBitfieldLValue(LV, Loc);
2673}
2674
2675RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV,
2676 SourceLocation Loc) {
2677 const CGBitFieldInfo &Info = LV.getBitFieldInfo();
2678
2679 // Get the output type.
2680 llvm::Type *ResLTy = ConvertType(T: LV.getType());
2681
2682 Address Ptr = LV.getBitFieldAddress();
2683 llvm::Value *Val =
2684 Builder.CreateLoad(Addr: Ptr, IsVolatile: LV.isVolatileQualified(), Name: "bf.load");
2685
2686 bool UseVolatile = LV.isVolatileQualified() &&
2687 Info.VolatileStorageSize != 0 &&
2688 CodeGenUtils::isAAPCS(TargetInfo: CGM.getTarget());
2689 const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
2690 const unsigned StorageSize =
2691 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
2692 if (Info.IsSigned) {
2693 assert(static_cast<unsigned>(Offset + Info.Size) <= StorageSize);
2694 unsigned HighBits = StorageSize - Offset - Info.Size;
2695 if (HighBits)
2696 Val = Builder.CreateShl(LHS: Val, RHS: HighBits, Name: "bf.shl");
2697 if (Offset + HighBits)
2698 Val = Builder.CreateAShr(LHS: Val, RHS: Offset + HighBits, Name: "bf.ashr");
2699 } else {
2700 if (Offset)
2701 Val = Builder.CreateLShr(LHS: Val, RHS: Offset, Name: "bf.lshr");
2702 if (static_cast<unsigned>(Offset) + Info.Size < StorageSize)
2703 Val = Builder.CreateAnd(
2704 LHS: Val, RHS: llvm::APInt::getLowBitsSet(numBits: StorageSize, loBitsSet: Info.Size), Name: "bf.clear");
2705 }
2706 Val = Builder.CreateIntCast(V: Val, DestTy: ResLTy, isSigned: Info.IsSigned, Name: "bf.cast");
2707 EmitScalarRangeCheck(Value: Val, Ty: LV.getType(), Loc);
2708 return RValue::get(V: Val);
2709}
2710
2711// If this is a reference to a subset of the elements of a vector, create an
2712// appropriate shufflevector.
2713RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {
2714 llvm::Value *Vec = Builder.CreateLoad(Addr: LV.getExtVectorAddress(),
2715 IsVolatile: LV.isVolatileQualified());
2716
2717 // HLSL allows treating scalars as one-element vectors. Converting the scalar
2718 // IR value to a vector here allows the rest of codegen to behave as normal.
2719 if (getLangOpts().HLSL && !Vec->getType()->isVectorTy()) {
2720 llvm::Type *DstTy = llvm::FixedVectorType::get(ElementType: Vec->getType(), NumElts: 1);
2721 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: CGM.Int64Ty);
2722 Vec = Builder.CreateInsertElement(VecTy: DstTy, NewElt: Vec, Idx: Zero, Name: "cast.splat");
2723 }
2724
2725 const llvm::Constant *Elts = LV.getExtVectorElts();
2726
2727 // If the result of the expression is a non-vector type, we must be extracting
2728 // a single element. Just codegen as an extractelement.
2729 const VectorType *ExprVT = LV.getType()->getAs<VectorType>();
2730 if (!ExprVT) {
2731 unsigned InIdx = getAccessedFieldNo(Idx: 0, Elts);
2732 llvm::Value *Elt = llvm::ConstantInt::get(Ty: SizeTy, V: InIdx);
2733
2734 llvm::Value *Element = Builder.CreateExtractElement(Vec, Idx: Elt);
2735
2736 llvm::Type *LVTy = ConvertType(T: LV.getType());
2737 if (Element->getType()->getPrimitiveSizeInBits() >
2738 LVTy->getPrimitiveSizeInBits()) {
2739 if (LV.getType()->hasBooleanRepresentation() &&
2740 CGM.getCodeGenOpts().isConvertingBoolWithCmp0())
2741 Element = Builder.CreateICmpNE(
2742 LHS: Element, RHS: llvm::Constant::getNullValue(Ty: Element->getType()));
2743 else
2744 Element = Builder.CreateTrunc(V: Element, DestTy: LVTy);
2745 }
2746
2747 return RValue::get(V: Element);
2748 }
2749
2750 // Always use shuffle vector to try to retain the original program structure
2751 unsigned NumResultElts = ExprVT->getNumElements();
2752
2753 SmallVector<int, 4> Mask;
2754 for (unsigned i = 0; i != NumResultElts; ++i)
2755 Mask.push_back(Elt: getAccessedFieldNo(Idx: i, Elts));
2756
2757 Vec = Builder.CreateShuffleVector(V: Vec, Mask);
2758
2759 if (LV.getType()->isExtVectorBoolType()) {
2760 if (CGM.getCodeGenOpts().isConvertingBoolWithCmp0())
2761 Vec = Builder.CreateICmpNE(LHS: Vec,
2762 RHS: llvm::Constant::getNullValue(Ty: Vec->getType()));
2763 else
2764 Vec = Builder.CreateTrunc(V: Vec, DestTy: ConvertType(T: LV.getType()), Name: "truncv");
2765 }
2766
2767 return RValue::get(V: Vec);
2768}
2769
2770/// Generates lvalue for partial ext_vector access.
2771Address CodeGenFunction::EmitExtVectorElementLValue(LValue LV) {
2772 Address VectorAddress = LV.getExtVectorAddress();
2773 QualType EQT = LV.getType()->castAs<VectorType>()->getElementType();
2774 llvm::Type *VectorElementTy = CGM.getTypes().ConvertType(T: EQT);
2775
2776 Address CastToPointerElement = VectorAddress.withElementType(ElemTy: VectorElementTy);
2777
2778 const llvm::Constant *Elts = LV.getExtVectorElts();
2779 unsigned ix = getAccessedFieldNo(Idx: 0, Elts);
2780
2781 Address VectorBasePtrPlusIx =
2782 Builder.CreateConstInBoundsGEP(Addr: CastToPointerElement, Index: ix,
2783 Name: "vector.elt");
2784
2785 return VectorBasePtrPlusIx;
2786}
2787
2788/// Load of global named registers are always calls to intrinsics.
2789RValue CodeGenFunction::EmitLoadOfGlobalRegLValue(LValue LV) {
2790 assert((LV.getType()->isIntegerType() || LV.getType()->isPointerType()) &&
2791 "Bad type for register variable");
2792 llvm::MDNode *RegName = cast<llvm::MDNode>(
2793 Val: cast<llvm::MetadataAsValue>(Val: LV.getGlobalReg())->getMetadata());
2794
2795 // We accept integer and pointer types only
2796 llvm::Type *OrigTy = CGM.getTypes().ConvertType(T: LV.getType());
2797 llvm::Type *Ty = OrigTy;
2798 if (OrigTy->isPointerTy())
2799 Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
2800 llvm::Type *Types[] = { Ty };
2801
2802 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::read_register, Tys: Types);
2803 llvm::Value *Call = Builder.CreateCall(
2804 Callee: F, Args: llvm::MetadataAsValue::get(Context&: Ty->getContext(), MD: RegName));
2805 if (OrigTy->isPointerTy())
2806 Call = Builder.CreateIntToPtr(V: Call, DestTy: OrigTy);
2807 return RValue::get(V: Call);
2808}
2809
2810/// EmitStoreThroughLValue - Store the specified rvalue into the specified
2811/// lvalue, where both are guaranteed to the have the same type, and that type
2812/// is 'Ty'.
2813void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
2814 bool isInit) {
2815 if (!Dst.isSimple()) {
2816 if (Dst.isVectorElt()) {
2817 if (getLangOpts().HLSL) {
2818 // HLSL allows direct access to vector elements, so storing to
2819 // individual elements of a vector through VectorElt is handled as
2820 // separate store instructions.
2821 Address DstAddr = Dst.getVectorAddress();
2822 llvm::Type *DestAddrTy = DstAddr.getElementType();
2823 llvm::Type *ElemTy = DestAddrTy->getScalarType();
2824 CharUnits ElemAlign = CharUnits::fromQuantity(
2825 Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty: ElemTy));
2826
2827 assert(ElemTy->getScalarSizeInBits() >= 8 &&
2828 "vector element type must be at least byte-sized");
2829
2830 llvm::Value *Val = Src.getScalarVal();
2831 if (Val->getType()->getPrimitiveSizeInBits() <
2832 ElemTy->getScalarSizeInBits())
2833 Val = Builder.CreateZExt(V: Val, DestTy: ElemTy->getScalarType());
2834
2835 llvm::Value *Idx = Dst.getVectorIdx();
2836 llvm::Value *Zero = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
2837 Address DstElemAddr =
2838 Builder.CreateGEP(Addr: DstAddr, IdxList: {Zero, Idx}, ElementType: DestAddrTy, Align: ElemAlign);
2839 Builder.CreateStore(Val, Addr: DstElemAddr, IsVolatile: Dst.isVolatileQualified());
2840 return;
2841 }
2842
2843 // Read/modify/write the vector, inserting the new element.
2844 llvm::Value *Vec = Builder.CreateLoad(Addr: Dst.getVectorAddress(),
2845 IsVolatile: Dst.isVolatileQualified());
2846 llvm::Type *VecTy = Vec->getType();
2847 llvm::Value *SrcVal = Src.getScalarVal();
2848
2849 if (VecTy->isVectorTy() && SrcVal->getType()->getPrimitiveSizeInBits() <
2850 VecTy->getScalarSizeInBits())
2851 SrcVal = Builder.CreateZExt(V: SrcVal, DestTy: VecTy->getScalarType());
2852
2853 auto *IRStoreTy = dyn_cast<llvm::IntegerType>(Val: Vec->getType());
2854 if (IRStoreTy) {
2855 auto *IRVecTy = llvm::FixedVectorType::get(
2856 ElementType: Builder.getInt1Ty(), NumElts: IRStoreTy->getPrimitiveSizeInBits());
2857 Vec = Builder.CreateBitCast(V: Vec, DestTy: IRVecTy);
2858 // iN --> <N x i1>.
2859 }
2860
2861 // Allow inserting `<1 x T>` into an `<N x T>`. It can happen with scalar
2862 // types which are mapped to vector LLVM IR types (e.g. for implementing
2863 // an ABI).
2864 if (auto *EltTy = dyn_cast<llvm::FixedVectorType>(Val: SrcVal->getType());
2865 EltTy && EltTy->getNumElements() == 1)
2866 SrcVal = Builder.CreateBitCast(V: SrcVal, DestTy: EltTy->getElementType());
2867
2868 Vec = Builder.CreateInsertElement(Vec, NewElt: SrcVal, Idx: Dst.getVectorIdx(),
2869 Name: "vecins");
2870 if (IRStoreTy) {
2871 // <N x i1> --> <iN>.
2872 Vec = Builder.CreateBitCast(V: Vec, DestTy: IRStoreTy);
2873 }
2874
2875 auto *I = Builder.CreateStore(Val: Vec, Addr: Dst.getVectorAddress(),
2876 IsVolatile: Dst.isVolatileQualified());
2877 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Vec);
2878 return;
2879 }
2880
2881 // If this is an update of extended vector elements, insert them as
2882 // appropriate.
2883 if (Dst.isExtVectorElt())
2884 return EmitStoreThroughExtVectorComponentLValue(Src, Dst);
2885
2886 if (Dst.isGlobalReg())
2887 return EmitStoreThroughGlobalRegLValue(Src, Dst);
2888
2889 if (Dst.isMatrixElt()) {
2890 if (getLangOpts().HLSL) {
2891 // HLSL allows direct access to matrix elements, so storing to
2892 // individual elements of a matrix through MatrixElt is handled as
2893 // separate store instructions.
2894 Address DstAddr = Dst.getMatrixAddress();
2895 llvm::Type *DestAddrTy = DstAddr.getElementType();
2896 llvm::Type *ElemTy = DestAddrTy->getScalarType();
2897 CharUnits ElemAlign = CharUnits::fromQuantity(
2898 Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty: ElemTy));
2899
2900 assert(ElemTy->getScalarSizeInBits() >= 8 &&
2901 "matrix element type must be at least byte-sized");
2902
2903 llvm::Value *Val = Src.getScalarVal();
2904 if (Val->getType()->getPrimitiveSizeInBits() <
2905 ElemTy->getScalarSizeInBits())
2906 Val = Builder.CreateZExt(V: Val, DestTy: ElemTy->getScalarType());
2907
2908 llvm::Value *Idx = Dst.getMatrixIdx();
2909 llvm::Value *Zero = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
2910 Address DstElemAddr =
2911 Builder.CreateGEP(Addr: DstAddr, IdxList: {Zero, Idx}, ElementType: DestAddrTy, Align: ElemAlign);
2912 Builder.CreateStore(Val, Addr: DstElemAddr, IsVolatile: Dst.isVolatileQualified());
2913 return;
2914 }
2915
2916 llvm::Value *Idx = Dst.getMatrixIdx();
2917 if (CGM.getCodeGenOpts().isOptimizedBuild()) {
2918 const auto *const MatTy = Dst.getType()->castAs<ConstantMatrixType>();
2919 llvm::MatrixBuilder MB(Builder);
2920 MB.CreateIndexAssumption(Idx, NumElements: MatTy->getNumElementsFlattened());
2921 }
2922 llvm::Instruction *Load = Builder.CreateLoad(Addr: Dst.getMatrixAddress());
2923 llvm::Value *InsertVal = Src.getScalarVal();
2924 llvm::Value *Vec =
2925 Builder.CreateInsertElement(Vec: Load, NewElt: InsertVal, Idx, Name: "matins");
2926 auto *I = Builder.CreateStore(Val: Vec, Addr: Dst.getMatrixAddress(),
2927 IsVolatile: Dst.isVolatileQualified());
2928 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Vec);
2929 return;
2930 }
2931 if (Dst.isMatrixRow()) {
2932 // NOTE: Since there are no other languages that implement matrix single
2933 // subscripting, the logic here is specific to HLSL which allows
2934 // per-element stores to rows of matrices.
2935 assert(getLangOpts().HLSL &&
2936 "Store through matrix row LValues is only implemented for HLSL!");
2937 QualType MatTy = Dst.getType();
2938 const ConstantMatrixType *MT = MatTy->castAs<ConstantMatrixType>();
2939
2940 unsigned NumRows = MT->getNumRows();
2941 unsigned NumCols = MT->getNumColumns();
2942 unsigned NumLanes = NumCols;
2943
2944 Address DstAddr = Dst.getMatrixAddress();
2945 llvm::Type *DestAddrTy = DstAddr.getElementType();
2946 llvm::Type *ElemTy = DestAddrTy->getScalarType();
2947 CharUnits ElemAlign =
2948 CharUnits::fromQuantity(Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty: ElemTy));
2949
2950 assert(ElemTy->getScalarSizeInBits() >= 8 &&
2951 "matrix element type must be at least byte-sized");
2952
2953 llvm::Value *RowVal = Src.getScalarVal();
2954 if (RowVal->getType()->getScalarType()->getPrimitiveSizeInBits() <
2955 ElemTy->getScalarSizeInBits()) {
2956 auto *RowValVecTy = cast<llvm::FixedVectorType>(Val: RowVal->getType());
2957 llvm::Type *StorageElmTy = llvm::FixedVectorType::get(
2958 ElementType: ElemTy->getScalarType(), NumElts: RowValVecTy->getNumElements());
2959 RowVal = Builder.CreateZExt(V: RowVal, DestTy: StorageElmTy);
2960 }
2961
2962 llvm::MatrixBuilder MB(Builder);
2963
2964 llvm::Constant *ColConstsIndices = nullptr;
2965 if (Dst.isMatrixRowSwizzle()) {
2966 ColConstsIndices = Dst.getMatrixRowElts();
2967 NumLanes =
2968 llvm::cast<llvm::FixedVectorType>(Val: ColConstsIndices->getType())
2969 ->getNumElements();
2970 }
2971
2972 llvm::Value *Row = Dst.getMatrixRowIdx();
2973 for (unsigned Col = 0; Col < NumLanes; ++Col) {
2974 llvm::Value *ColIdx;
2975 if (ColConstsIndices)
2976 ColIdx = ColConstsIndices->getAggregateElement(Elt: Col);
2977 else
2978 ColIdx = llvm::ConstantInt::get(Ty: Row->getType(), V: Col);
2979 bool IsMatrixRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: Dst.getType());
2980 llvm::Value *EltIndex =
2981 MB.CreateIndex(RowIdx: Row, ColumnIdx: ColIdx, NumRows, NumCols, IsMatrixRowMajor);
2982 llvm::Value *Lane = llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: Col);
2983 llvm::Value *Zero = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
2984 llvm::Value *NewElt = Builder.CreateExtractElement(Vec: RowVal, Idx: Lane);
2985 Address DstElemAddr =
2986 Builder.CreateGEP(Addr: DstAddr, IdxList: {Zero, EltIndex}, ElementType: DestAddrTy, Align: ElemAlign);
2987 Builder.CreateStore(Val: NewElt, Addr: DstElemAddr, IsVolatile: Dst.isVolatileQualified());
2988 }
2989
2990 return;
2991 }
2992
2993 assert(Dst.isBitField() && "Unknown LValue type");
2994 return EmitStoreThroughBitfieldLValue(Src, Dst);
2995 }
2996
2997 // Handle __ptrauth qualification by re-signing the value.
2998 if (PointerAuthQualifier PointerAuth = Dst.getQuals().getPointerAuth()) {
2999 Src = RValue::get(V: EmitPointerAuthQualify(Qualifier: PointerAuth, Pointer: Src.getScalarVal(),
3000 ValueType: Dst.getType(), StorageAddress: Dst.getAddress(),
3001 /*known nonnull*/ IsKnownNonNull: false));
3002 }
3003
3004 // There's special magic for assigning into an ARC-qualified l-value.
3005 if (Qualifiers::ObjCLifetime Lifetime = Dst.getQuals().getObjCLifetime()) {
3006 switch (Lifetime) {
3007 case Qualifiers::OCL_None:
3008 llvm_unreachable("present but none");
3009
3010 case Qualifiers::OCL_ExplicitNone:
3011 // nothing special
3012 break;
3013
3014 case Qualifiers::OCL_Strong:
3015 if (isInit) {
3016 Src = RValue::get(V: EmitARCRetain(type: Dst.getType(), value: Src.getScalarVal()));
3017 break;
3018 }
3019 EmitARCStoreStrong(lvalue: Dst, value: Src.getScalarVal(), /*ignore*/ resultIgnored: true);
3020 return;
3021
3022 case Qualifiers::OCL_Weak:
3023 if (isInit)
3024 // Initialize and then skip the primitive store.
3025 EmitARCInitWeak(addr: Dst.getAddress(), value: Src.getScalarVal());
3026 else
3027 EmitARCStoreWeak(addr: Dst.getAddress(), value: Src.getScalarVal(),
3028 /*ignore*/ ignored: true);
3029 return;
3030
3031 case Qualifiers::OCL_Autoreleasing:
3032 Src = RValue::get(V: EmitObjCExtendObjectLifetime(T: Dst.getType(),
3033 Ptr: Src.getScalarVal()));
3034 // fall into the normal path
3035 break;
3036 }
3037 }
3038
3039 if (Dst.isObjCWeak() && !Dst.isNonGC()) {
3040 // load of a __weak object.
3041 Address LvalueDst = Dst.getAddress();
3042 llvm::Value *src = Src.getScalarVal();
3043 CGM.getObjCRuntime().EmitObjCWeakAssign(CGF&: *this, src, dest: LvalueDst);
3044 return;
3045 }
3046
3047 if (Dst.isObjCStrong() && !Dst.isNonGC()) {
3048 // load of a __strong object.
3049 Address LvalueDst = Dst.getAddress();
3050 llvm::Value *src = Src.getScalarVal();
3051 if (Dst.isObjCIvar()) {
3052 assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");
3053 llvm::Type *ResultType = IntPtrTy;
3054 Address dst = EmitPointerWithAlignment(E: Dst.getBaseIvarExp());
3055 llvm::Value *RHS = dst.emitRawPointer(CGF&: *this);
3056 RHS = Builder.CreatePtrToInt(V: RHS, DestTy: ResultType, Name: "sub.ptr.rhs.cast");
3057 llvm::Value *LHS = Builder.CreatePtrToInt(V: LvalueDst.emitRawPointer(CGF&: *this),
3058 DestTy: ResultType, Name: "sub.ptr.lhs.cast");
3059 llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, Name: "ivar.offset");
3060 CGM.getObjCRuntime().EmitObjCIvarAssign(CGF&: *this, src, dest: dst, ivarOffset: BytesBetween);
3061 } else if (Dst.isGlobalObjCRef()) {
3062 CGM.getObjCRuntime().EmitObjCGlobalAssign(CGF&: *this, src, dest: LvalueDst,
3063 threadlocal: Dst.isThreadLocalRef());
3064 }
3065 else
3066 CGM.getObjCRuntime().EmitObjCStrongCastAssign(CGF&: *this, src, dest: LvalueDst);
3067 return;
3068 }
3069
3070 assert(Src.isScalar() && "Can't emit an agg store with this method");
3071 EmitStoreOfScalar(value: Src.getScalarVal(), lvalue: Dst, isInit);
3072}
3073
3074void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
3075 llvm::Value **Result) {
3076 const CGBitFieldInfo &Info = Dst.getBitFieldInfo();
3077 llvm::Type *ResLTy = convertTypeForLoadStore(ASTTy: Dst.getType());
3078 Address Ptr = Dst.getBitFieldAddress();
3079
3080 // Get the source value, truncated to the width of the bit-field.
3081 llvm::Value *SrcVal = Src.getScalarVal();
3082
3083 // Cast the source to the storage type and shift it into place.
3084 SrcVal = Builder.CreateIntCast(V: SrcVal, DestTy: Ptr.getElementType(),
3085 /*isSigned=*/false);
3086 llvm::Value *MaskedVal = SrcVal;
3087
3088 const bool UseVolatile =
3089 CGM.getCodeGenOpts().AAPCSBitfieldWidth && Dst.isVolatileQualified() &&
3090 Info.VolatileStorageSize != 0 && CodeGenUtils::isAAPCS(TargetInfo: CGM.getTarget());
3091 const unsigned StorageSize =
3092 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
3093 const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
3094 // See if there are other bits in the bitfield's storage we'll need to load
3095 // and mask together with source before storing.
3096 if (StorageSize != Info.Size) {
3097 assert(StorageSize > Info.Size && "Invalid bitfield size.");
3098 llvm::Value *Val =
3099 Builder.CreateLoad(Addr: Ptr, IsVolatile: Dst.isVolatileQualified(), Name: "bf.load");
3100
3101 // Mask the source value as needed.
3102 if (!Dst.getType()->hasBooleanRepresentation())
3103 SrcVal = Builder.CreateAnd(
3104 LHS: SrcVal, RHS: llvm::APInt::getLowBitsSet(numBits: StorageSize, loBitsSet: Info.Size),
3105 Name: "bf.value");
3106 MaskedVal = SrcVal;
3107 if (Offset)
3108 SrcVal = Builder.CreateShl(LHS: SrcVal, RHS: Offset, Name: "bf.shl");
3109
3110 // Mask out the original value.
3111 Val = Builder.CreateAnd(
3112 LHS: Val, RHS: ~llvm::APInt::getBitsSet(numBits: StorageSize, loBit: Offset, hiBit: Offset + Info.Size),
3113 Name: "bf.clear");
3114
3115 // Or together the unchanged values and the source value.
3116 SrcVal = Builder.CreateOr(LHS: Val, RHS: SrcVal, Name: "bf.set");
3117 } else {
3118 assert(Offset == 0);
3119 // According to the AACPS:
3120 // When a volatile bit-field is written, and its container does not overlap
3121 // with any non-bit-field member, its container must be read exactly once
3122 // and written exactly once using the access width appropriate to the type
3123 // of the container. The two accesses are not atomic.
3124 if (Dst.isVolatileQualified() && CodeGenUtils::isAAPCS(TargetInfo: CGM.getTarget()) &&
3125 CGM.getCodeGenOpts().ForceAAPCSBitfieldLoad)
3126 Builder.CreateLoad(Addr: Ptr, IsVolatile: true, Name: "bf.load");
3127 }
3128
3129 // Write the new value back out.
3130 auto *I = Builder.CreateStore(Val: SrcVal, Addr: Ptr, IsVolatile: Dst.isVolatileQualified());
3131 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: SrcVal);
3132
3133 // Return the new value of the bit-field, if requested.
3134 if (Result) {
3135 llvm::Value *ResultVal = MaskedVal;
3136
3137 // Sign extend the value if needed.
3138 if (Info.IsSigned) {
3139 assert(Info.Size <= StorageSize);
3140 unsigned HighBits = StorageSize - Info.Size;
3141 if (HighBits) {
3142 ResultVal = Builder.CreateShl(LHS: ResultVal, RHS: HighBits, Name: "bf.result.shl");
3143 ResultVal = Builder.CreateAShr(LHS: ResultVal, RHS: HighBits, Name: "bf.result.ashr");
3144 }
3145 }
3146
3147 ResultVal = Builder.CreateIntCast(V: ResultVal, DestTy: ResLTy, isSigned: Info.IsSigned,
3148 Name: "bf.result.cast");
3149 *Result = EmitFromMemory(Value: ResultVal, Ty: Dst.getType());
3150 }
3151}
3152
3153void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
3154 LValue Dst) {
3155 llvm::Value *SrcVal = Src.getScalarVal();
3156 Address DstAddr = Dst.getExtVectorAddress();
3157 const llvm::Constant *Elts = Dst.getExtVectorElts();
3158 if (DstAddr.getElementType()->getScalarSizeInBits() >
3159 SrcVal->getType()->getScalarSizeInBits())
3160 SrcVal = Builder.CreateZExt(
3161 V: SrcVal, DestTy: convertTypeForLoadStore(ASTTy: Dst.getType(), LLVMTy: SrcVal->getType()));
3162
3163 if (getLangOpts().HLSL) {
3164 llvm::Type *DestAddrTy = DstAddr.getElementType();
3165 // HLSL allows storing to scalar values through ExtVector component LValues.
3166 // To support this we need to handle the case where the destination address
3167 // is a scalar.
3168 if (!DestAddrTy->isVectorTy()) {
3169 assert(!Dst.getType()->isVectorType() &&
3170 "this should only occur for non-vector l-values");
3171 Builder.CreateStore(Val: SrcVal, Addr: DstAddr, IsVolatile: Dst.isVolatileQualified());
3172 return;
3173 }
3174
3175 // HLSL allows direct access to vector elements, so storing to individual
3176 // elements of a vector through ExtVector is handled as separate store
3177 // instructions.
3178 // If we are updating multiple elements, Dst and Src are vectors; for
3179 // a single element update they are scalars.
3180 const VectorType *VTy = Dst.getType()->getAs<VectorType>();
3181 unsigned NumSrcElts = VTy ? VTy->getNumElements() : 1;
3182 CharUnits ElemAlign = CharUnits::fromQuantity(
3183 Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty: DestAddrTy->getScalarType()));
3184 llvm::Value *Zero = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
3185
3186 for (unsigned I = 0; I != NumSrcElts; ++I) {
3187 llvm::Value *Val = VTy ? Builder.CreateExtractElement(
3188 Vec: SrcVal, Idx: llvm::ConstantInt::get(Ty: Int32Ty, V: I))
3189 : SrcVal;
3190 unsigned FieldNo = getAccessedFieldNo(Idx: I, Elts);
3191 Address DstElemAddr = Address::invalid();
3192 if (FieldNo == 0)
3193 DstElemAddr = DstAddr.withAlignment(NewAlignment: ElemAlign);
3194 else
3195 DstElemAddr = Builder.CreateGEP(
3196 Addr: DstAddr, IdxList: {Zero, llvm::ConstantInt::get(Ty: Int32Ty, V: FieldNo)},
3197 ElementType: DestAddrTy, Align: ElemAlign);
3198 Builder.CreateStore(Val, Addr: DstElemAddr, IsVolatile: Dst.isVolatileQualified());
3199 }
3200 return;
3201 }
3202
3203 // This access turns into a read/modify/write of the vector. Load the input
3204 // value now.
3205 llvm::Value *Vec = Builder.CreateLoad(Addr: DstAddr, IsVolatile: Dst.isVolatileQualified());
3206 llvm::Type *VecTy = Vec->getType();
3207
3208 if (const VectorType *VTy = Dst.getType()->getAs<VectorType>()) {
3209 unsigned NumSrcElts = VTy->getNumElements();
3210 unsigned NumDstElts = cast<llvm::FixedVectorType>(Val: VecTy)->getNumElements();
3211 if (NumDstElts == NumSrcElts) {
3212 // Use shuffle vector is the src and destination are the same number of
3213 // elements and restore the vector mask since it is on the side it will be
3214 // stored.
3215 SmallVector<int, 4> Mask(NumDstElts);
3216 for (unsigned i = 0; i != NumSrcElts; ++i)
3217 Mask[getAccessedFieldNo(Idx: i, Elts)] = i;
3218
3219 Vec = Builder.CreateShuffleVector(V: SrcVal, Mask);
3220 } else if (NumDstElts > NumSrcElts) {
3221 // Extended the source vector to the same length and then shuffle it
3222 // into the destination.
3223 // FIXME: since we're shuffling with undef, can we just use the indices
3224 // into that? This could be simpler.
3225 SmallVector<int, 4> ExtMask;
3226 for (unsigned i = 0; i != NumSrcElts; ++i)
3227 ExtMask.push_back(Elt: i);
3228 ExtMask.resize(N: NumDstElts, NV: -1);
3229 llvm::Value *ExtSrcVal = Builder.CreateShuffleVector(V: SrcVal, Mask: ExtMask);
3230 // build identity
3231 SmallVector<int, 4> Mask;
3232 for (unsigned i = 0; i != NumDstElts; ++i)
3233 Mask.push_back(Elt: i);
3234
3235 // When the vector size is odd and .odd or .hi is used, the last element
3236 // of the Elts constant array will be one past the size of the vector.
3237 // Ignore the last element here, if it is greater than the mask size.
3238 if (getAccessedFieldNo(Idx: NumSrcElts - 1, Elts) == Mask.size())
3239 NumSrcElts--;
3240
3241 // modify when what gets shuffled in
3242 for (unsigned i = 0; i != NumSrcElts; ++i)
3243 Mask[getAccessedFieldNo(Idx: i, Elts)] = i + NumDstElts;
3244 Vec = Builder.CreateShuffleVector(V1: Vec, V2: ExtSrcVal, Mask);
3245 } else {
3246 // We should never shorten the vector
3247 llvm_unreachable("unexpected shorten vector length");
3248 }
3249 } else {
3250 // If the Src is a scalar (not a vector), and the target is a vector it must
3251 // be updating one element.
3252 unsigned InIdx = getAccessedFieldNo(Idx: 0, Elts);
3253 llvm::Value *Elt = llvm::ConstantInt::get(Ty: SizeTy, V: InIdx);
3254
3255 Vec = Builder.CreateInsertElement(Vec, NewElt: SrcVal, Idx: Elt);
3256 }
3257
3258 Builder.CreateStore(Val: Vec, Addr: Dst.getExtVectorAddress(),
3259 IsVolatile: Dst.isVolatileQualified());
3260}
3261
3262/// Store of global named registers are always calls to intrinsics.
3263void CodeGenFunction::EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst) {
3264 assert((Dst.getType()->isIntegerType() || Dst.getType()->isPointerType()) &&
3265 "Bad type for register variable");
3266 llvm::MDNode *RegName = cast<llvm::MDNode>(
3267 Val: cast<llvm::MetadataAsValue>(Val: Dst.getGlobalReg())->getMetadata());
3268 assert(RegName && "Register LValue is not metadata");
3269
3270 // We accept integer and pointer types only
3271 llvm::Type *OrigTy = CGM.getTypes().ConvertType(T: Dst.getType());
3272 llvm::Type *Ty = OrigTy;
3273 if (OrigTy->isPointerTy())
3274 Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
3275 llvm::Type *Types[] = { Ty };
3276
3277 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::write_register, Tys: Types);
3278 llvm::Value *Value = Src.getScalarVal();
3279 if (OrigTy->isPointerTy())
3280 Value = Builder.CreatePtrToInt(V: Value, DestTy: Ty);
3281 Builder.CreateCall(
3282 Callee: F, Args: {llvm::MetadataAsValue::get(Context&: Ty->getContext(), MD: RegName), Value});
3283}
3284
3285// setObjCGCLValueClass - sets class of the lvalue for the purpose of
3286// generating write-barries API. It is currently a global, ivar,
3287// or neither.
3288static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
3289 LValue &LV,
3290 bool IsMemberAccess=false) {
3291 if (Ctx.getLangOpts().getGC() == LangOptions::NonGC)
3292 return;
3293
3294 if (isa<ObjCIvarRefExpr>(Val: E)) {
3295 QualType ExpTy = E->getType();
3296 if (IsMemberAccess && ExpTy->isPointerType()) {
3297 // If ivar is a structure pointer, assigning to field of
3298 // this struct follows gcc's behavior and makes it a non-ivar
3299 // writer-barrier conservatively.
3300 ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
3301 if (ExpTy->isRecordType()) {
3302 LV.setObjCIvar(false);
3303 return;
3304 }
3305 }
3306 LV.setObjCIvar(true);
3307 auto *Exp = cast<ObjCIvarRefExpr>(Val: const_cast<Expr *>(E));
3308 LV.setBaseIvarExp(Exp->getBase());
3309 LV.setObjCArray(E->getType()->isArrayType());
3310 return;
3311 }
3312
3313 if (const auto *Exp = dyn_cast<DeclRefExpr>(Val: E)) {
3314 if (const auto *VD = dyn_cast<VarDecl>(Val: Exp->getDecl())) {
3315 if (VD->hasGlobalStorage()) {
3316 LV.setGlobalObjCRef(true);
3317 LV.setThreadLocalRef(VD->getTLSKind() != VarDecl::TLS_None);
3318 }
3319 }
3320 LV.setObjCArray(E->getType()->isArrayType());
3321 return;
3322 }
3323
3324 if (const auto *Exp = dyn_cast<UnaryOperator>(Val: E)) {
3325 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3326 return;
3327 }
3328
3329 if (const auto *Exp = dyn_cast<ParenExpr>(Val: E)) {
3330 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3331 if (LV.isObjCIvar()) {
3332 // If cast is to a structure pointer, follow gcc's behavior and make it
3333 // a non-ivar write-barrier.
3334 QualType ExpTy = E->getType();
3335 if (ExpTy->isPointerType())
3336 ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
3337 if (ExpTy->isRecordType())
3338 LV.setObjCIvar(false);
3339 }
3340 return;
3341 }
3342
3343 if (const auto *Exp = dyn_cast<GenericSelectionExpr>(Val: E)) {
3344 setObjCGCLValueClass(Ctx, E: Exp->getResultExpr(), LV);
3345 return;
3346 }
3347
3348 if (const auto *Exp = dyn_cast<ImplicitCastExpr>(Val: E)) {
3349 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3350 return;
3351 }
3352
3353 if (const auto *Exp = dyn_cast<CStyleCastExpr>(Val: E)) {
3354 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3355 return;
3356 }
3357
3358 if (const auto *Exp = dyn_cast<ObjCBridgedCastExpr>(Val: E)) {
3359 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3360 return;
3361 }
3362
3363 if (const auto *Exp = dyn_cast<ArraySubscriptExpr>(Val: E)) {
3364 setObjCGCLValueClass(Ctx, E: Exp->getBase(), LV);
3365 if (LV.isObjCIvar() && !LV.isObjCArray())
3366 // Using array syntax to assigning to what an ivar points to is not
3367 // same as assigning to the ivar itself. {id *Names;} Names[i] = 0;
3368 LV.setObjCIvar(false);
3369 else if (LV.isGlobalObjCRef() && !LV.isObjCArray())
3370 // Using array syntax to assigning to what global points to is not
3371 // same as assigning to the global itself. {id *G;} G[i] = 0;
3372 LV.setGlobalObjCRef(false);
3373 return;
3374 }
3375
3376 if (const auto *Exp = dyn_cast<MemberExpr>(Val: E)) {
3377 setObjCGCLValueClass(Ctx, E: Exp->getBase(), LV, IsMemberAccess: true);
3378 // We don't know if member is an 'ivar', but this flag is looked at
3379 // only in the context of LV.isObjCIvar().
3380 LV.setObjCArray(E->getType()->isArrayType());
3381 return;
3382 }
3383}
3384
3385static LValue EmitThreadPrivateVarDeclLValue(
3386 CodeGenFunction &CGF, const VarDecl *VD, QualType T, Address Addr,
3387 llvm::Type *RealVarTy, SourceLocation Loc) {
3388 if (CGF.CGM.getLangOpts().OpenMPIRBuilder)
3389 Addr = CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(
3390 CGF, VD, VDAddr: Addr, Loc);
3391 else
3392 Addr =
3393 CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, VDAddr: Addr, Loc);
3394
3395 Addr = Addr.withElementType(ElemTy: RealVarTy);
3396 return CGF.MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3397}
3398
3399static Address emitDeclTargetVarDeclLValue(CodeGenFunction &CGF,
3400 const VarDecl *VD, QualType T) {
3401 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
3402 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
3403 // Always return an invalid address for MT_Local, and also for
3404 // MT_To/MT_Enter when unified memory is not enabled. These use direct
3405 // access (global exists in device image). Otherwise, return a valid
3406 // address.
3407 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Local ||
3408 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
3409 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
3410 !CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory()))
3411 return Address::invalid();
3412 assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
3413 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
3414 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
3415 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) &&
3416 "Expected link clause OR to clause with unified memory enabled.");
3417 QualType PtrTy = CGF.getContext().getPointerType(T: VD->getType());
3418 Address Addr = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
3419 return CGF.EmitLoadOfPointer(Ptr: Addr, PtrTy: PtrTy->castAs<PointerType>());
3420}
3421
3422Address
3423CodeGenFunction::EmitLoadOfReference(LValue RefLVal,
3424 LValueBaseInfo *PointeeBaseInfo,
3425 TBAAAccessInfo *PointeeTBAAInfo) {
3426 llvm::LoadInst *Load =
3427 Builder.CreateLoad(Addr: RefLVal.getAddress(), IsVolatile: RefLVal.isVolatile());
3428 CGM.DecorateInstructionWithTBAA(Inst: Load, TBAAInfo: RefLVal.getTBAAInfo());
3429 QualType PTy = RefLVal.getType()->getPointeeType();
3430 CharUnits Align = CGM.getNaturalTypeAlignment(
3431 T: PTy, BaseInfo: PointeeBaseInfo, TBAAInfo: PointeeTBAAInfo, /*ForPointeeType=*/forPointeeType: true);
3432 if (!PTy->isIncompleteType()) {
3433 llvm::LLVMContext &Ctx = getLLVMContext();
3434 llvm::MDBuilder MDB(Ctx);
3435 // Emit !nonnull metadata
3436 if (CGM.getTypes().getTargetAddressSpace(T: PTy) == 0 &&
3437 !CGM.getCodeGenOpts().NullPointerIsValid)
3438 Load->setMetadata(KindID: llvm::LLVMContext::MD_nonnull,
3439 Node: llvm::MDNode::get(Context&: Ctx, MDs: {}));
3440 // Emit !align metadata
3441 if (PTy->isObjectType()) {
3442 auto AlignVal = Align.getQuantity();
3443 if (AlignVal > 1) {
3444 Load->setMetadata(
3445 KindID: llvm::LLVMContext::MD_align,
3446 Node: llvm::MDNode::get(Context&: Ctx, MDs: MDB.createConstant(C: llvm::ConstantInt::get(
3447 Ty: Builder.getInt64Ty(), V: AlignVal))));
3448 }
3449 }
3450 }
3451 return makeNaturalAddressForPointer(Ptr: Load, T: PTy, Alignment: Align,
3452 /*ForPointeeType=*/true, BaseInfo: PointeeBaseInfo,
3453 TBAAInfo: PointeeTBAAInfo);
3454}
3455
3456LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) {
3457 LValueBaseInfo PointeeBaseInfo;
3458 TBAAAccessInfo PointeeTBAAInfo;
3459 Address PointeeAddr = EmitLoadOfReference(RefLVal, PointeeBaseInfo: &PointeeBaseInfo,
3460 PointeeTBAAInfo: &PointeeTBAAInfo);
3461 return MakeAddrLValue(Addr: PointeeAddr, T: RefLVal.getType()->getPointeeType(),
3462 BaseInfo: PointeeBaseInfo, TBAAInfo: PointeeTBAAInfo);
3463}
3464
3465Address CodeGenFunction::EmitLoadOfPointer(Address Ptr,
3466 const PointerType *PtrTy,
3467 LValueBaseInfo *BaseInfo,
3468 TBAAAccessInfo *TBAAInfo) {
3469 llvm::Value *Addr = Builder.CreateLoad(Addr: Ptr);
3470 return makeNaturalAddressForPointer(Ptr: Addr, T: PtrTy->getPointeeType(),
3471 Alignment: CharUnits(), /*ForPointeeType=*/true,
3472 BaseInfo, TBAAInfo);
3473}
3474
3475LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr,
3476 const PointerType *PtrTy) {
3477 LValueBaseInfo BaseInfo;
3478 TBAAAccessInfo TBAAInfo;
3479 Address Addr = EmitLoadOfPointer(Ptr: PtrAddr, PtrTy, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
3480 return MakeAddrLValue(Addr, T: PtrTy->getPointeeType(), BaseInfo, TBAAInfo);
3481}
3482
3483static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
3484 const Expr *E, const VarDecl *VD) {
3485 QualType T = E->getType();
3486
3487 // If it's thread_local, emit a call to its wrapper function instead.
3488 if (VD->getTLSKind() == VarDecl::TLS_Dynamic &&
3489 CGF.CGM.getCXXABI().usesThreadWrapperFunction(VD))
3490 return CGF.CGM.getCXXABI().EmitThreadLocalVarDeclLValue(CGF, VD, LValType: T);
3491 // Check if the variable is marked as declare target with link clause in
3492 // device codegen.
3493 if (CGF.getLangOpts().OpenMPIsTargetDevice) {
3494 Address Addr = emitDeclTargetVarDeclLValue(CGF, VD, T);
3495 if (Addr.isValid())
3496 return CGF.MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3497 }
3498
3499 // Global HLSL resource arrays initialized on access; create a temporary with
3500 // the initialized global resource array.
3501 if (CGF.getLangOpts().HLSL && VD->getType()->isHLSLResourceRecordArray()) {
3502 std::optional<LValue> LV =
3503 CGF.CGM.getHLSLRuntime().emitGlobalResourceArrayAsLValue(CGF, ArrayDecl: VD);
3504 if (LV.has_value())
3505 return LV.value();
3506 }
3507
3508 llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(D: VD);
3509
3510 if (VD->getTLSKind() != VarDecl::TLS_None)
3511 V = CGF.Builder.CreateThreadLocalAddress(Ptr: V);
3512
3513 llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(T: VD->getType());
3514 CharUnits Alignment = CGF.getContext().getDeclAlign(D: VD);
3515 Address Addr(V, RealVarTy, Alignment);
3516 // Emit reference to the private copy of the variable if it is an OpenMP
3517 // threadprivate variable.
3518 if (CGF.getLangOpts().OpenMP && !CGF.getLangOpts().OpenMPSimd &&
3519 VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
3520 return EmitThreadPrivateVarDeclLValue(CGF, VD, T, Addr, RealVarTy,
3521 Loc: E->getExprLoc());
3522 }
3523 LValue LV = VD->getType()->isReferenceType() ?
3524 CGF.EmitLoadOfReferenceLValue(RefAddr: Addr, RefTy: VD->getType(),
3525 Source: AlignmentSource::Decl) :
3526 CGF.MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3527 setObjCGCLValueClass(Ctx: CGF.getContext(), E, LV);
3528 return LV;
3529}
3530
3531llvm::Constant *CodeGenModule::getRawFunctionPointer(GlobalDecl GD,
3532 llvm::Type *Ty) {
3533 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
3534 if (FD->hasAttr<WeakRefAttr>()) {
3535 ConstantAddress aliasee = GetWeakRefReference(VD: FD);
3536 return aliasee.getPointer();
3537 }
3538
3539 llvm::Constant *V = GetAddrOfFunction(GD, Ty);
3540 return V;
3541}
3542
3543static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF, const Expr *E,
3544 GlobalDecl GD) {
3545 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
3546 llvm::Constant *V = CGF.CGM.getFunctionPointer(GD);
3547 QualType ETy = E->getType();
3548 if (ETy->isCFIUncheckedCalleeFunctionType()) {
3549 if (auto *GV = dyn_cast<llvm::GlobalValue>(Val: V))
3550 V = llvm::NoCFIValue::get(GV);
3551 }
3552 CharUnits Alignment = CGF.getContext().getDeclAlign(D: FD);
3553 return CGF.MakeAddrLValue(V, T: ETy, Alignment, Source: AlignmentSource::Decl);
3554}
3555
3556static LValue EmitCapturedFieldLValue(CodeGenFunction &CGF, const FieldDecl *FD,
3557 llvm::Value *ThisValue) {
3558
3559 return CGF.EmitLValueForLambdaField(Field: FD, ThisValue);
3560}
3561
3562/// Named Registers are named metadata pointing to the register name
3563/// which will be read from/written to as an argument to the intrinsic
3564/// @llvm.read/write_register.
3565/// So far, only the name is being passed down, but other options such as
3566/// register type, allocation type or even optimization options could be
3567/// passed down via the metadata node.
3568static LValue EmitGlobalNamedRegister(const VarDecl *VD, CodeGenModule &CGM) {
3569 SmallString<64> Name("llvm.named.register.");
3570 AsmLabelAttr *Asm = VD->getAttr<AsmLabelAttr>();
3571 assert(Asm->getLabel().size() < 64-Name.size() &&
3572 "Register name too big");
3573 Name.append(RHS: Asm->getLabel());
3574 llvm::NamedMDNode *M =
3575 CGM.getModule().getOrInsertNamedMetadata(Name);
3576 if (M->getNumOperands() == 0) {
3577 llvm::MDString *Str = llvm::MDString::get(Context&: CGM.getLLVMContext(),
3578 Str: Asm->getLabel());
3579 llvm::Metadata *Ops[] = {Str};
3580 M->addOperand(M: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Ops));
3581 }
3582
3583 CharUnits Alignment = CGM.getContext().getDeclAlign(D: VD);
3584
3585 llvm::Value *Ptr =
3586 llvm::MetadataAsValue::get(Context&: CGM.getLLVMContext(), MD: M->getOperand(i: 0));
3587 return LValue::MakeGlobalReg(V: Ptr, alignment: Alignment, type: VD->getType());
3588}
3589
3590/// Determine whether we can emit a reference to \p VD from the current
3591/// context, despite not necessarily having seen an odr-use of the variable in
3592/// this context.
3593static bool canEmitSpuriousReferenceToVariable(CodeGenFunction &CGF,
3594 const DeclRefExpr *E,
3595 const VarDecl *VD) {
3596 // For a variable declared in an enclosing scope, do not emit a spurious
3597 // reference even if we have a capture, as that will emit an unwarranted
3598 // reference to our capture state, and will likely generate worse code than
3599 // emitting a local copy.
3600 if (E->refersToEnclosingVariableOrCapture())
3601 return false;
3602
3603 // For a local declaration declared in this function, we can always reference
3604 // it even if we don't have an odr-use.
3605 if (VD->hasLocalStorage()) {
3606 return VD->getDeclContext() ==
3607 dyn_cast_or_null<DeclContext>(Val: CGF.CurCodeDecl);
3608 }
3609
3610 // For a global declaration, we can emit a reference to it if we know
3611 // for sure that we are able to emit a definition of it.
3612 VD = VD->getDefinition(C&: CGF.getContext());
3613 if (!VD)
3614 return false;
3615
3616 // Don't emit a spurious reference if it might be to a variable that only
3617 // exists on a different device / target.
3618 // FIXME: This is unnecessarily broad. Check whether this would actually be a
3619 // cross-target reference.
3620 if (CGF.getLangOpts().OpenMP || CGF.getLangOpts().CUDA ||
3621 CGF.getLangOpts().OpenCL) {
3622 return false;
3623 }
3624
3625 // We can emit a spurious reference only if the linkage implies that we'll
3626 // be emitting a non-interposable symbol that will be retained until link
3627 // time.
3628 switch (CGF.CGM.getLLVMLinkageVarDefinition(VD)) {
3629 case llvm::GlobalValue::ExternalLinkage:
3630 case llvm::GlobalValue::LinkOnceODRLinkage:
3631 case llvm::GlobalValue::WeakODRLinkage:
3632 case llvm::GlobalValue::InternalLinkage:
3633 case llvm::GlobalValue::PrivateLinkage:
3634 return true;
3635 default:
3636 return false;
3637 }
3638}
3639
3640/// Emit an LValue for a structured binding captured in an OpenMP region.
3641/// Handles extracting individual bindings from the captured decomposed
3642/// declaration (struct fields, array elements, etc.).
3643LValue CodeGenFunction::EmitOMPCapturedBindingLValue(const BindingDecl *BD) {
3644 assert(CapturedStmtInfo && "Expected to be inside a captured region");
3645 assert(CapturedStmtInfo->getKind() == CapturedRegionKind::CR_OpenMP &&
3646 "Expected OpenMP captured region");
3647 assert(CGM.getLangOpts().OpenMP && "Expected OpenMP to be enabled");
3648
3649 if (auto It = LocalDeclMap.find(Val: BD->getCanonicalDecl());
3650 It != LocalDeclMap.end())
3651 return MakeAddrLValue(Addr: It->second, T: BD->getType());
3652
3653 const auto *DD = cast<VarDecl>(Val: BD->getDecomposedDecl());
3654
3655 // Use getNonReferenceType() because we need the actual object type, not the
3656 // reference type. DeclRefExpr with VK_LValue requires a non-reference type
3657 // (AST invariant). EmitDeclRefLValue will load any reference for us.
3658 QualType DREType = DD->getType().getNonReferenceType();
3659 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(DD),
3660 /*RefersToEnclosingVariableOrCapture=*/true, DREType,
3661 VK_LValue, SourceLocation());
3662 LValue BaseLVal = EmitDeclRefLValue(E: &DRE);
3663
3664 // Ensure the Address has the correct element type for DD's type.
3665 // EmitDeclRefLValue might return an address with a different element type
3666 // if reference unwrapping occurred.
3667 Address BaseAddr = BaseLVal.getAddress();
3668 QualType DDType = DD->getType();
3669 llvm::Type *ExpectedTy = CGM.getTypes().ConvertTypeForMem(T: DDType);
3670 if (BaseAddr.getElementType() != ExpectedTy)
3671 BaseAddr = BaseAddr.withElementType(ElemTy: ExpectedTy);
3672
3673 // Now emit the binding expression (array subscript, member access, etc.)
3674 // by temporarily installing the decomposed storage address, then routing
3675 // through EmitLValue for the binding expression.
3676 Expr *BindingExpr = BD->getBinding();
3677 auto It = LocalDeclMap.find(Val: DD);
3678 bool WasMapped = It != LocalDeclMap.end();
3679 Address SavedAddr = WasMapped ? It->second : Address::invalid();
3680 Address MapAddr = BaseAddr;
3681 if (DD->getType()->isReferenceType()) {
3682 RawAddress RefSlot = CreateMemTemp(Ty: DD->getType(), Name: "omp.binding.ref");
3683 Builder.CreateStore(Val: BaseAddr.emitRawPointer(CGF&: *this), Addr: RefSlot);
3684 MapAddr = RefSlot;
3685 }
3686 if (WasMapped)
3687 It->second = MapAddr;
3688 else
3689 LocalDeclMap.insert(KV: {DD, MapAddr});
3690 llvm::scope_exit Guard([&] {
3691 if (WasMapped) {
3692 auto RestoreIt = LocalDeclMap.find(Val: DD);
3693 assert(RestoreIt != LocalDeclMap.end() && "DD should still be in map");
3694 RestoreIt->second = SavedAddr;
3695 } else {
3696 LocalDeclMap.erase(Val: DD);
3697 }
3698 });
3699
3700 return EmitLValue(E: BindingExpr);
3701}
3702
3703LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
3704 const NamedDecl *ND = E->getDecl();
3705 QualType T = E->getType();
3706
3707 assert(E->isNonOdrUse() != NOUR_Unevaluated &&
3708 "should not emit an unevaluated operand");
3709
3710 if (const auto *VD = dyn_cast<VarDecl>(Val: ND)) {
3711 // Global Named registers access via intrinsics only
3712 if (VD->getStorageClass() == SC_Register &&
3713 VD->hasAttr<AsmLabelAttr>() && !VD->isLocalVarDecl())
3714 return EmitGlobalNamedRegister(VD, CGM);
3715
3716 // If this DeclRefExpr does not constitute an odr-use of the variable,
3717 // we're not permitted to emit a reference to it in general, and it might
3718 // not be captured if capture would be necessary for a use. Emit the
3719 // constant value directly instead.
3720 if (E->isNonOdrUse() == NOUR_Constant &&
3721 (VD->getType()->isReferenceType() ||
3722 !canEmitSpuriousReferenceToVariable(CGF&: *this, E, VD))) {
3723 VD->getAnyInitializer(D&: VD);
3724 llvm::Constant *Val = ConstantEmitter(*this).emitAbstract(
3725 loc: E->getLocation(), value: *VD->evaluateValue(), T: VD->getType());
3726 assert(Val && "failed to emit constant expression");
3727
3728 Address Addr = Address::invalid();
3729 if (!VD->getType()->isReferenceType()) {
3730 // Spill the constant value to a global.
3731 Addr = CGM.createUnnamedGlobalFrom(D: *VD, Constant: Val,
3732 Align: getContext().getDeclAlign(D: VD));
3733 llvm::Type *VarTy = getTypes().ConvertTypeForMem(T: VD->getType());
3734 auto *PTy = llvm::PointerType::get(
3735 C&: getLLVMContext(), AddressSpace: getTypes().getTargetAddressSpace(T: VD->getType()));
3736 Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty: PTy, ElementTy: VarTy);
3737 } else {
3738 // Should we be using the alignment of the constant pointer we emitted?
3739 CharUnits Alignment =
3740 CGM.getNaturalTypeAlignment(T: E->getType(),
3741 /* BaseInfo= */ nullptr,
3742 /* TBAAInfo= */ nullptr,
3743 /* forPointeeType= */ true);
3744 Addr = makeNaturalAddressForPointer(Ptr: Val, T, Alignment);
3745 }
3746 return MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3747 }
3748
3749 // FIXME: Handle other kinds of non-odr-use DeclRefExprs.
3750
3751 // Check for captured variables.
3752 if (E->refersToEnclosingVariableOrCapture()) {
3753 VD = VD->getCanonicalDecl();
3754 if (auto *FD = LambdaCaptureFields.lookup(Val: VD))
3755 return EmitCapturedFieldLValue(CGF&: *this, FD, ThisValue: CXXABIThisValue);
3756 if (CapturedStmtInfo) {
3757 auto I = LocalDeclMap.find(Val: VD);
3758 if (I != LocalDeclMap.end()) {
3759 LValue CapLVal;
3760 if (VD->getType()->isReferenceType())
3761 CapLVal = EmitLoadOfReferenceLValue(RefAddr: I->second, RefTy: VD->getType(),
3762 Source: AlignmentSource::Decl);
3763 else
3764 CapLVal = MakeAddrLValue(Addr: I->second, T);
3765 // Mark lvalue as nontemporal if the variable is marked as nontemporal
3766 // in simd context.
3767 if (getLangOpts().OpenMP &&
3768 CGM.getOpenMPRuntime().isNontemporalDecl(VD))
3769 CapLVal.setNontemporal(/*Value=*/true);
3770 return CapLVal;
3771 }
3772 LValue CapLVal =
3773 EmitCapturedFieldLValue(CGF&: *this, FD: CapturedStmtInfo->lookup(VD),
3774 ThisValue: CapturedStmtInfo->getContextValue());
3775 Address LValueAddress = CapLVal.getAddress();
3776 CapLVal = MakeAddrLValue(Addr: Address(LValueAddress.emitRawPointer(CGF&: *this),
3777 LValueAddress.getElementType(),
3778 getContext().getDeclAlign(D: VD)),
3779 T: CapLVal.getType(),
3780 BaseInfo: LValueBaseInfo(AlignmentSource::Decl),
3781 TBAAInfo: CapLVal.getTBAAInfo());
3782 // Mark lvalue as nontemporal if the variable is marked as nontemporal
3783 // in simd context.
3784 if (getLangOpts().OpenMP &&
3785 CGM.getOpenMPRuntime().isNontemporalDecl(VD))
3786 CapLVal.setNontemporal(/*Value=*/true);
3787 return CapLVal;
3788 }
3789
3790 assert(isa<BlockDecl>(CurCodeDecl));
3791 Address addr = GetAddrOfBlockDecl(var: VD);
3792 return MakeAddrLValue(Addr: addr, T, Source: AlignmentSource::Decl);
3793 }
3794 }
3795
3796 // FIXME: We should be able to assert this for FunctionDecls as well!
3797 // FIXME: We should be able to assert this for all DeclRefExprs, not just
3798 // those with a valid source location.
3799 assert((ND->isUsed(false) || !isa<VarDecl>(ND) || E->isNonOdrUse() ||
3800 !E->getLocation().isValid()) &&
3801 "Should not use decl without marking it used!");
3802
3803 if (ND->hasAttr<WeakRefAttr>()) {
3804 const auto *VD = cast<ValueDecl>(Val: ND);
3805 ConstantAddress Aliasee = CGM.GetWeakRefReference(VD);
3806 return MakeAddrLValue(Addr: Aliasee, T, Source: AlignmentSource::Decl);
3807 }
3808
3809 if (const auto *VD = dyn_cast<VarDecl>(Val: ND)) {
3810 // Check if this is a global variable.
3811 if (VD->hasLinkage() || VD->isStaticDataMember())
3812 return EmitGlobalVarDeclLValue(CGF&: *this, E, VD);
3813
3814 Address addr = Address::invalid();
3815
3816 // The variable should generally be present in the local decl map.
3817 auto iter = LocalDeclMap.find(Val: VD);
3818 if (iter != LocalDeclMap.end()) {
3819 addr = iter->second;
3820
3821 // Otherwise, it might be static local we haven't emitted yet for
3822 // some reason; most likely, because it's in an outer function.
3823 } else if (VD->isStaticLocal()) {
3824 llvm::Constant *var = CGM.getOrCreateStaticVarDecl(
3825 D: *VD, Linkage: CGM.getLLVMLinkageVarDefinition(VD));
3826 addr = Address(
3827 var, ConvertTypeForMem(T: VD->getType()), getContext().getDeclAlign(D: VD));
3828
3829 // No other cases for now.
3830 } else {
3831 llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?");
3832 }
3833
3834 // Handle threadlocal function locals.
3835 if (VD->getTLSKind() != VarDecl::TLS_None)
3836 addr = addr.withPointer(
3837 NewPointer: Builder.CreateThreadLocalAddress(Ptr: addr.getBasePointer()),
3838 IsKnownNonNull: NotKnownNonNull);
3839
3840 // Check for OpenMP threadprivate variables.
3841 if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd &&
3842 VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
3843 return EmitThreadPrivateVarDeclLValue(
3844 CGF&: *this, VD, T, Addr: addr, RealVarTy: getTypes().ConvertTypeForMem(T: VD->getType()),
3845 Loc: E->getExprLoc());
3846 }
3847
3848 // Drill into block byref variables.
3849 bool isBlockByref = VD->isEscapingByref();
3850 if (isBlockByref) {
3851 addr = emitBlockByrefAddress(baseAddr: addr, V: VD);
3852 }
3853
3854 // Drill into reference types.
3855 LValue LV = VD->getType()->isReferenceType() ?
3856 EmitLoadOfReferenceLValue(RefAddr: addr, RefTy: VD->getType(), Source: AlignmentSource::Decl) :
3857 MakeAddrLValue(Addr: addr, T, Source: AlignmentSource::Decl);
3858
3859 bool isLocalStorage = VD->hasLocalStorage();
3860
3861 bool NonGCable = isLocalStorage &&
3862 !VD->getType()->isReferenceType() &&
3863 !isBlockByref;
3864 if (NonGCable) {
3865 LV.getQuals().removeObjCGCAttr();
3866 LV.setNonGC(true);
3867 }
3868
3869 bool isImpreciseLifetime =
3870 (isLocalStorage && !VD->hasAttr<ObjCPreciseLifetimeAttr>());
3871 if (isImpreciseLifetime)
3872 LV.setARCPreciseLifetime(ARCImpreciseLifetime);
3873 setObjCGCLValueClass(Ctx: getContext(), E, LV);
3874 return LV;
3875 }
3876
3877 if (const auto *FD = dyn_cast<FunctionDecl>(Val: ND))
3878 return EmitFunctionDeclLValue(CGF&: *this, E, GD: FD);
3879
3880 // FIXME: While we're emitting a binding from an enclosing scope, all other
3881 // DeclRefExprs we see should be implicitly treated as if they also refer to
3882 // an enclosing scope.
3883 if (const auto *BD = dyn_cast<BindingDecl>(Val: ND)) {
3884 if (E->refersToEnclosingVariableOrCapture()) {
3885 auto ApplyNontemporal = [&](LValue LV) {
3886 if (getLangOpts().OpenMP &&
3887 CGM.getOpenMPRuntime().isNontemporalDecl(VD: BD))
3888 LV.setNontemporal(/*Value=*/true);
3889 return LV;
3890 };
3891
3892 // Try direct lookup first.
3893 auto It = LocalDeclMap.find(Val: BD->getCanonicalDecl());
3894 if (It != LocalDeclMap.end()) {
3895 return ApplyNontemporal(
3896 MakeAddrLValue(Addr: It->second, T: E->getType(), Source: AlignmentSource::Decl));
3897 }
3898
3899 // OpenMP case: binding was captured via its decomposed decl.
3900 if (CapturedStmtInfo &&
3901 CapturedStmtInfo->getKind() == CapturedRegionKind::CR_OpenMP &&
3902 CGM.getLangOpts().OpenMP) {
3903 auto NameIt = OMPPrivatizedBindings.find(
3904 Val: cast<BindingDecl>(Val: BD->getCanonicalDecl()));
3905 if (NameIt != OMPPrivatizedBindings.end()) {
3906 return ApplyNontemporal(MakeAddrLValue(Addr: NameIt->second, T: E->getType(),
3907 Source: AlignmentSource::Decl));
3908 }
3909 return ApplyNontemporal(EmitOMPCapturedBindingLValue(BD));
3910 }
3911 // Non-OpenMP case: lambda capture.
3912 auto *FD = LambdaCaptureFields.lookup(Val: BD);
3913 return EmitCapturedFieldLValue(CGF&: *this, FD, ThisValue: CXXABIThisValue);
3914 }
3915 // Suppress debug location updates when visiting the binding, since the
3916 // binding may emit instructions that would otherwise be associated with the
3917 // binding itself, rather than the expression referencing the binding. (this
3918 // leads to jumpy debug stepping behavior where the location/debugger jump
3919 // back to the binding declaration, then back to the expression referencing
3920 // the binding)
3921 DisableDebugLocationUpdates D(*this);
3922 return EmitLValue(E: BD->getBinding(), IsKnownNonNull: NotKnownNonNull);
3923 }
3924
3925 // We can form DeclRefExprs naming GUID declarations when reconstituting
3926 // non-type template parameters into expressions.
3927 if (const auto *GD = dyn_cast<MSGuidDecl>(Val: ND))
3928 return MakeAddrLValue(Addr: CGM.GetAddrOfMSGuidDecl(GD), T,
3929 Source: AlignmentSource::Decl);
3930
3931 if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(Val: ND)) {
3932 ConstantAddress ATPO = CGM.GetAddrOfTemplateParamObject(TPO);
3933 auto AS = getLangASFromTargetAS(TargetAS: ATPO.getAddressSpace());
3934
3935 if (AS != T.getAddressSpace()) {
3936 auto TargetAS = getContext().getTargetAddressSpace(AS: T.getAddressSpace());
3937 llvm::Type *PtrTy =
3938 llvm::PointerType::get(C&: CGM.getLLVMContext(), AddressSpace: TargetAS);
3939 llvm::Constant *ASC = CGM.performAddrSpaceCast(Src: ATPO.getPointer(), DestTy: PtrTy);
3940 ATPO = ConstantAddress(ASC, ATPO.getElementType(), ATPO.getAlignment());
3941 }
3942
3943 return MakeAddrLValue(Addr: ATPO, T, Source: AlignmentSource::Decl);
3944 }
3945
3946 llvm_unreachable("Unhandled DeclRefExpr");
3947}
3948
3949LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
3950 // __extension__ doesn't affect lvalue-ness.
3951 if (E->getOpcode() == UO_Extension)
3952 return EmitLValue(E: E->getSubExpr());
3953
3954 QualType ExprTy = getContext().getCanonicalType(T: E->getSubExpr()->getType());
3955 switch (E->getOpcode()) {
3956 default: llvm_unreachable("Unknown unary operator lvalue!");
3957 case UO_Deref: {
3958 QualType T = E->getSubExpr()->getType()->getPointeeType();
3959 assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
3960
3961 LValueBaseInfo BaseInfo;
3962 TBAAAccessInfo TBAAInfo;
3963 Address Addr = EmitPointerWithAlignment(E: E->getSubExpr(), BaseInfo: &BaseInfo,
3964 TBAAInfo: &TBAAInfo);
3965 LValue LV = MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
3966 LV.getQuals().setAddressSpace(ExprTy.getAddressSpace());
3967
3968 // We should not generate __weak write barrier on indirect reference
3969 // of a pointer to object; as in void foo (__weak id *param); *param = 0;
3970 // But, we continue to generate __strong write barrier on indirect write
3971 // into a pointer to object.
3972 if (getLangOpts().ObjC &&
3973 getLangOpts().getGC() != LangOptions::NonGC &&
3974 LV.isObjCWeak())
3975 LV.setNonGC(!E->isOBJCGCCandidate(Ctx&: getContext()));
3976 return LV;
3977 }
3978 case UO_Real:
3979 case UO_Imag: {
3980 LValue LV = EmitLValue(E: E->getSubExpr());
3981 assert(LV.isSimple() && "real/imag on non-ordinary l-value");
3982
3983 // __real is valid on scalars. This is a faster way of testing that.
3984 // __imag can only produce an rvalue on scalars.
3985 if (E->getOpcode() == UO_Real &&
3986 !LV.getAddress().getElementType()->isStructTy()) {
3987 assert(E->getSubExpr()->getType()->isArithmeticType());
3988 return LV;
3989 }
3990
3991 QualType T = ExprTy->castAs<ComplexType>()->getElementType();
3992
3993 Address Component =
3994 (E->getOpcode() == UO_Real
3995 ? emitAddrOfRealComponent(complex: LV.getAddress(), complexType: LV.getType())
3996 : emitAddrOfImagComponent(complex: LV.getAddress(), complexType: LV.getType()));
3997 LValue ElemLV = MakeAddrLValue(Addr: Component, T, BaseInfo: LV.getBaseInfo(),
3998 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: T));
3999 ElemLV.getQuals().addQualifiers(Q: LV.getQuals());
4000 return ElemLV;
4001 }
4002 case UO_PreInc:
4003 case UO_PreDec: {
4004 LValue LV = EmitLValue(E: E->getSubExpr());
4005 bool isInc = E->getOpcode() == UO_PreInc;
4006
4007 if (E->getType()->isAnyComplexType())
4008 EmitComplexPrePostIncDec(E, LV, isInc, isPre: true/*isPre*/);
4009 else
4010 EmitScalarPrePostIncDec(E, LV, isInc, isPre: true/*isPre*/);
4011 return LV;
4012 }
4013 }
4014}
4015
4016LValue CodeGenFunction::EmitStringLiteralLValue(const StringLiteral *E) {
4017 return MakeAddrLValue(Addr: CGM.GetAddrOfConstantStringFromLiteral(S: E),
4018 T: E->getType(), Source: AlignmentSource::Decl);
4019}
4020
4021LValue CodeGenFunction::EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E) {
4022 return MakeAddrLValue(Addr: CGM.GetAddrOfConstantStringFromObjCEncode(E),
4023 T: E->getType(), Source: AlignmentSource::Decl);
4024}
4025
4026LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
4027 auto SL = E->getFunctionName();
4028 assert(SL != nullptr && "No StringLiteral name in PredefinedExpr");
4029 StringRef FnName = CurFn->getName();
4030 FnName.consume_front(Prefix: "\01");
4031 StringRef NameItems[] = {
4032 PredefinedExpr::getIdentKindName(IK: E->getIdentKind()), FnName};
4033 std::string GVName = llvm::join(Begin: NameItems, End: NameItems + 2, Separator: ".");
4034 if (auto *BD = dyn_cast_or_null<BlockDecl>(Val: CurCodeDecl)) {
4035 std::string Name = std::string(SL->getString());
4036 if (!Name.empty()) {
4037 unsigned Discriminator =
4038 CGM.getCXXABI().getMangleContext().getBlockId(BD, Local: true);
4039 if (Discriminator)
4040 Name += "_" + Twine(Discriminator + 1).str();
4041 auto C = CGM.GetAddrOfConstantCString(Str: Name, GlobalName: GVName);
4042 return MakeAddrLValue(Addr: C, T: E->getType(), Source: AlignmentSource::Decl);
4043 } else {
4044 auto C = CGM.GetAddrOfConstantCString(Str: std::string(FnName), GlobalName: GVName);
4045 return MakeAddrLValue(Addr: C, T: E->getType(), Source: AlignmentSource::Decl);
4046 }
4047 }
4048 auto C = CGM.GetAddrOfConstantStringFromLiteral(S: SL, Name: GVName);
4049 return MakeAddrLValue(Addr: C, T: E->getType(), Source: AlignmentSource::Decl);
4050}
4051
4052/// Emit a type description suitable for use by a runtime sanitizer library. The
4053/// format of a type descriptor is
4054///
4055/// \code
4056/// { i16 TypeKind, i16 TypeInfo }
4057/// \endcode
4058///
4059/// followed by an array of i8 containing the type name with extra information
4060/// for BitInt. TypeKind is TK_Integer(0) for an integer, TK_Float(1) for a
4061/// floating point value, TK_BitInt(2) for BitInt and TK_Unknown(0xFFFF) for
4062/// anything else.
4063llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) {
4064 // Only emit each type's descriptor once.
4065 if (llvm::Constant *C = CGM.getTypeDescriptorFromMap(Ty: T))
4066 return C;
4067
4068 uint16_t TypeKind = TK_Unknown;
4069 uint16_t TypeInfo = 0;
4070 bool IsBitInt = false;
4071
4072 // isIntegerType() never holds for scoped enums, and getAs<BitIntType> can't
4073 // see through the EnumType node to a __BitInt underlying type even for
4074 // unscoped enums.
4075 QualType UnderlyingT = T;
4076 if (const EnumType *ET = T->getAs<EnumType>())
4077 if (ET->getDecl()->isComplete())
4078 UnderlyingT = ET->getDecl()->getIntegerType();
4079
4080 if (UnderlyingT->isIntegerType()) {
4081 TypeKind = TK_Integer;
4082 TypeInfo = (llvm::Log2_32(Value: getContext().getTypeSize(T: UnderlyingT)) << 1) |
4083 (UnderlyingT->isSignedIntegerType() ? 1 : 0);
4084 // Follow suggestion from discussion of issue 64100.
4085 // So we can write the exact amount of bits in TypeName after '\0'
4086 // making it <diagnostic-like type name>.'\0'.<32-bit width>.
4087 if (UnderlyingT->isSignedIntegerType() &&
4088 UnderlyingT->getAs<BitIntType>()) {
4089 // Do a sanity checks as we are using 32-bit type to store bit length.
4090 assert(getContext().getTypeSize(UnderlyingT) > 0 &&
4091 " non positive amount of bits in __BitInt type");
4092 assert(getContext().getTypeSize(UnderlyingT) <= 0xFFFFFFFF &&
4093 " too many bits in __BitInt type");
4094
4095 // Redefine TypeKind with the actual __BitInt type if we have signed
4096 // BitInt.
4097 TypeKind = TK_BitInt;
4098 IsBitInt = true;
4099 }
4100 } else if (UnderlyingT->isFloatingType()) {
4101 TypeKind = TK_Float;
4102 TypeInfo = getContext().getTypeSize(T: UnderlyingT);
4103 }
4104
4105 // Format the type name as if for a diagnostic, including quotes and
4106 // optionally an 'aka'.
4107 SmallString<32> Buffer;
4108 CGM.getDiags().ConvertArgToString(Kind: DiagnosticsEngine::ak_qualtype,
4109 Val: (intptr_t)T.getAsOpaquePtr(), Modifier: StringRef(),
4110 Argument: StringRef(), PrevArgs: {}, Output&: Buffer, QualTypeVals: {});
4111
4112 if (IsBitInt) {
4113 // The Structure is: 0 to end the string, 32 bit unsigned integer in target
4114 // endianness, zero.
4115 char S[6] = {'\0', '\0', '\0', '\0', '\0', '\0'};
4116 const auto *EIT = UnderlyingT->castAs<BitIntType>();
4117 uint32_t Bits = EIT->getNumBits();
4118 llvm::support::endian::write32(P: S + 1, V: Bits,
4119 E: getTarget().isBigEndian()
4120 ? llvm::endianness::big
4121 : llvm::endianness::little);
4122 StringRef Str = StringRef(S, sizeof(S) / sizeof(decltype(S[0])));
4123 Buffer.append(RHS: Str);
4124 }
4125
4126 llvm::Constant *Components[] = {
4127 Builder.getInt16(C: TypeKind), Builder.getInt16(C: TypeInfo),
4128 llvm::ConstantDataArray::getString(Context&: getLLVMContext(), Initializer: Buffer)
4129 };
4130 llvm::Constant *Descriptor = llvm::ConstantStruct::getAnon(V: Components);
4131
4132 auto *GV = new llvm::GlobalVariable(
4133 CGM.getModule(), Descriptor->getType(),
4134 /*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage, Descriptor);
4135 GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4136 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV);
4137
4138 // Remember the descriptor for this type.
4139 CGM.setTypeDescriptorInMap(Ty: T, C: GV);
4140
4141 return GV;
4142}
4143
4144llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) {
4145 llvm::Type *TargetTy = IntPtrTy;
4146
4147 if (V->getType() == TargetTy)
4148 return V;
4149
4150 // Floating-point types which fit into intptr_t are bitcast to integers
4151 // and then passed directly (after zero-extension, if necessary).
4152 if (V->getType()->isFloatingPointTy()) {
4153 unsigned Bits = V->getType()->getPrimitiveSizeInBits().getFixedValue();
4154 if (Bits <= TargetTy->getIntegerBitWidth())
4155 V = Builder.CreateBitCast(V, DestTy: llvm::Type::getIntNTy(C&: getLLVMContext(),
4156 N: Bits));
4157 }
4158
4159 // Integers which fit in intptr_t are zero-extended and passed directly.
4160 if (V->getType()->isIntegerTy() &&
4161 V->getType()->getIntegerBitWidth() <= TargetTy->getIntegerBitWidth())
4162 return Builder.CreateZExt(V, DestTy: TargetTy);
4163
4164 // Pointers are passed directly, everything else is passed by address.
4165 if (!V->getType()->isPointerTy()) {
4166 RawAddress Ptr = CreateDefaultAlignTempAlloca(Ty: V->getType());
4167 Builder.CreateStore(Val: V, Addr: Ptr);
4168 V = Ptr.getPointer();
4169 }
4170 return Builder.CreatePtrToInt(V, DestTy: TargetTy);
4171}
4172
4173/// Emit a representation of a SourceLocation for passing to a handler
4174/// in a sanitizer runtime library. The format for this data is:
4175/// \code
4176/// struct SourceLocation {
4177/// const char *Filename;
4178/// int32_t Line, Column;
4179/// };
4180/// \endcode
4181/// For an invalid SourceLocation, the Filename pointer is null.
4182llvm::Constant *CodeGenFunction::EmitCheckSourceLocation(SourceLocation Loc) {
4183 llvm::Constant *Filename;
4184 int Line, Column;
4185
4186 PresumedLoc PLoc = getContext().getSourceManager().getPresumedLoc(Loc);
4187 if (PLoc.isValid()) {
4188 StringRef FilenameString = PLoc.getFilename();
4189
4190 int PathComponentsToStrip =
4191 CGM.getCodeGenOpts().EmitCheckPathComponentsToStrip;
4192 if (PathComponentsToStrip < 0) {
4193 assert(PathComponentsToStrip != INT_MIN);
4194 int PathComponentsToKeep = -PathComponentsToStrip;
4195 auto I = llvm::sys::path::rbegin(path: FilenameString);
4196 auto E = llvm::sys::path::rend(path: FilenameString);
4197 while (I != E && --PathComponentsToKeep)
4198 ++I;
4199
4200 FilenameString = FilenameString.substr(Start: I - E);
4201 } else if (PathComponentsToStrip > 0) {
4202 auto I = llvm::sys::path::begin(path: FilenameString);
4203 auto E = llvm::sys::path::end(path: FilenameString);
4204 while (I != E && PathComponentsToStrip--)
4205 ++I;
4206
4207 if (I != E)
4208 FilenameString =
4209 FilenameString.substr(Start: I - llvm::sys::path::begin(path: FilenameString));
4210 else
4211 FilenameString = llvm::sys::path::filename(path: FilenameString);
4212 }
4213
4214 auto FilenameGV =
4215 CGM.GetAddrOfConstantCString(Str: std::string(FilenameString), GlobalName: ".src");
4216 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(
4217 GV: cast<llvm::GlobalVariable>(
4218 Val: FilenameGV.getPointer()->stripPointerCasts()));
4219 Filename = FilenameGV.getPointer();
4220 Line = PLoc.getLine();
4221 Column = PLoc.getColumn();
4222 } else {
4223 Filename = llvm::Constant::getNullValue(Ty: Int8PtrTy);
4224 Line = Column = 0;
4225 }
4226
4227 llvm::Constant *Data[] = {Filename, Builder.getInt32(C: Line),
4228 Builder.getInt32(C: Column)};
4229
4230 return llvm::ConstantStruct::getAnon(V: Data);
4231}
4232
4233namespace {
4234/// Specify under what conditions this check can be recovered
4235enum class CheckRecoverableKind {
4236 /// Always terminate program execution if this check fails.
4237 Unrecoverable,
4238 /// Check supports recovering, runtime has both fatal (noreturn) and
4239 /// non-fatal handlers for this check.
4240 Recoverable,
4241 /// Runtime conditionally aborts, always need to support recovery.
4242 AlwaysRecoverable
4243};
4244}
4245
4246static CheckRecoverableKind
4247getRecoverableKind(SanitizerKind::SanitizerOrdinal Ordinal) {
4248 if (Ordinal == SanitizerKind::SO_Vptr)
4249 return CheckRecoverableKind::AlwaysRecoverable;
4250 else if (Ordinal == SanitizerKind::SO_Return ||
4251 Ordinal == SanitizerKind::SO_Unreachable)
4252 return CheckRecoverableKind::Unrecoverable;
4253 else
4254 return CheckRecoverableKind::Recoverable;
4255}
4256
4257namespace {
4258struct SanitizerHandlerInfo {
4259 char const *const Name;
4260 unsigned Version;
4261};
4262}
4263
4264const SanitizerHandlerInfo SanitizerHandlers[] = {
4265#define SANITIZER_CHECK(Enum, Name, Version, Msg) {#Name, Version},
4266 LIST_SANITIZER_CHECKS
4267#undef SANITIZER_CHECK
4268};
4269
4270static void emitCheckHandlerCall(CodeGenFunction &CGF,
4271 llvm::FunctionType *FnType,
4272 ArrayRef<llvm::Value *> FnArgs,
4273 SanitizerHandler CheckHandler,
4274 CheckRecoverableKind RecoverKind, bool IsFatal,
4275 llvm::BasicBlock *ContBB, bool NoMerge) {
4276 assert(IsFatal || RecoverKind != CheckRecoverableKind::Unrecoverable);
4277 std::optional<ApplyDebugLocation> DL;
4278 if (!CGF.Builder.getCurrentDebugLocation()) {
4279 // Ensure that the call has at least an artificial debug location.
4280 DL.emplace(args&: CGF, args: SourceLocation());
4281 }
4282 bool NeedsAbortSuffix =
4283 IsFatal && RecoverKind != CheckRecoverableKind::Unrecoverable;
4284 bool MinimalRuntime = CGF.CGM.getCodeGenOpts().SanitizeMinimalRuntime;
4285 bool HandlerPreserveAllRegs =
4286 CGF.CGM.getCodeGenOpts().SanitizeHandlerPreserveAllRegs;
4287 const SanitizerHandlerInfo &CheckInfo = SanitizerHandlers[CheckHandler];
4288 const StringRef CheckName = CheckInfo.Name;
4289 std::string FnName = "__ubsan_handle_" + CheckName.str();
4290 if (CheckInfo.Version && !MinimalRuntime)
4291 FnName += "_v" + llvm::utostr(X: CheckInfo.Version);
4292 if (MinimalRuntime)
4293 FnName += "_minimal";
4294 if (NeedsAbortSuffix)
4295 FnName += "_abort";
4296 if (HandlerPreserveAllRegs && !NeedsAbortSuffix)
4297 FnName += "_preserve";
4298 bool MayReturn =
4299 !IsFatal || RecoverKind == CheckRecoverableKind::AlwaysRecoverable;
4300
4301 llvm::AttrBuilder B(CGF.getLLVMContext());
4302 if (!MayReturn) {
4303 B.addAttribute(Val: llvm::Attribute::NoReturn)
4304 .addAttribute(Val: llvm::Attribute::NoUnwind);
4305 }
4306 B.addUWTableAttr(Kind: llvm::UWTableKind::Default);
4307
4308 llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(
4309 Ty: FnType, Name: FnName,
4310 ExtraAttrs: llvm::AttributeList::get(C&: CGF.getLLVMContext(),
4311 Index: llvm::AttributeList::FunctionIndex, B),
4312 /*Local=*/true);
4313 llvm::CallInst *HandlerCall = CGF.EmitNounwindRuntimeCall(callee: Fn, args: FnArgs);
4314 NoMerge = NoMerge || !CGF.CGM.getCodeGenOpts().isOptimizedBuild() ||
4315 (CGF.CurCodeDecl && CGF.CurCodeDecl->hasAttr<OptimizeNoneAttr>());
4316 if (NoMerge)
4317 HandlerCall->addFnAttr(Kind: llvm::Attribute::NoMerge);
4318 if (HandlerPreserveAllRegs && !NeedsAbortSuffix) {
4319 // N.B. there is also a clang::CallingConv which is not what we want here.
4320 HandlerCall->setCallingConv(llvm::CallingConv::PreserveAll);
4321 }
4322 if (!MayReturn) {
4323 HandlerCall->setDoesNotReturn();
4324 CGF.Builder.CreateUnreachable();
4325 } else {
4326 CGF.Builder.CreateBr(Dest: ContBB);
4327 }
4328}
4329
4330void CodeGenFunction::EmitCheck(
4331 ArrayRef<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>> Checked,
4332 SanitizerHandler CheckHandler, ArrayRef<llvm::Constant *> StaticArgs,
4333 ArrayRef<llvm::Value *> DynamicArgs, const TrapReason *TR) {
4334 assert(IsSanitizerScope);
4335 assert(Checked.size() > 0);
4336 assert(CheckHandler >= 0 &&
4337 size_t(CheckHandler) < std::size(SanitizerHandlers));
4338 const StringRef CheckName = SanitizerHandlers[CheckHandler].Name;
4339
4340 llvm::Value *FatalCond = nullptr;
4341 llvm::Value *RecoverableCond = nullptr;
4342 llvm::Value *TrapCond = nullptr;
4343 bool NoMerge = false;
4344 // Expand checks into:
4345 // (Check1 || !allow_ubsan_check) && (Check2 || !allow_ubsan_check) ...
4346 // We need separate allow_ubsan_check intrinsics because they have separately
4347 // specified cutoffs.
4348 // This expression looks expensive but will be simplified after
4349 // LowerAllowCheckPass.
4350 for (auto &[Check, Ord] : Checked) {
4351 llvm::Value *GuardedCheck = Check;
4352 if (ClSanitizeGuardChecks ||
4353 (CGM.getCodeGenOpts().SanitizeSkipHotCutoffs[Ord] > 0)) {
4354 llvm::Value *Allow = Builder.CreateCall(
4355 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::allow_ubsan_check),
4356 Args: llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: Ord));
4357 GuardedCheck = Builder.CreateOr(LHS: Check, RHS: Builder.CreateNot(V: Allow));
4358 }
4359
4360 // -fsanitize-trap= overrides -fsanitize-recover=.
4361 llvm::Value *&Cond = CGM.getCodeGenOpts().SanitizeTrap.has(O: Ord) ? TrapCond
4362 : CGM.getCodeGenOpts().SanitizeRecover.has(O: Ord)
4363 ? RecoverableCond
4364 : FatalCond;
4365 Cond = Cond ? Builder.CreateAnd(LHS: Cond, RHS: GuardedCheck) : GuardedCheck;
4366
4367 if (!CGM.getCodeGenOpts().SanitizeMergeHandlers.has(O: Ord))
4368 NoMerge = true;
4369 }
4370
4371 if (TrapCond)
4372 EmitTrapCheck(Checked: TrapCond, CheckHandlerID: CheckHandler, NoMerge, TR);
4373 if (!FatalCond && !RecoverableCond)
4374 return;
4375
4376 llvm::Value *JointCond;
4377 if (FatalCond && RecoverableCond)
4378 JointCond = Builder.CreateAnd(LHS: FatalCond, RHS: RecoverableCond);
4379 else
4380 JointCond = FatalCond ? FatalCond : RecoverableCond;
4381 assert(JointCond);
4382
4383 CheckRecoverableKind RecoverKind = getRecoverableKind(Ordinal: Checked[0].second);
4384 assert(SanOpts.has(Checked[0].second));
4385#ifndef NDEBUG
4386 for (int i = 1, n = Checked.size(); i < n; ++i) {
4387 assert(RecoverKind == getRecoverableKind(Checked[i].second) &&
4388 "All recoverable kinds in a single check must be same!");
4389 assert(SanOpts.has(Checked[i].second));
4390 }
4391#endif
4392
4393 llvm::BasicBlock *Cont = createBasicBlock(name: "cont");
4394 llvm::BasicBlock *Handlers = createBasicBlock(name: "handler." + CheckName);
4395 llvm::Instruction *Branch = Builder.CreateCondBr(Cond: JointCond, True: Cont, False: Handlers);
4396 // Give hint that we very much don't expect to execute the handler
4397 llvm::MDBuilder MDHelper(getLLVMContext());
4398 llvm::MDNode *Node = MDHelper.createLikelyBranchWeights();
4399 Branch->setMetadata(KindID: llvm::LLVMContext::MD_prof, Node);
4400 EmitBlock(BB: Handlers);
4401
4402 // Clear arguments for the MinimalRuntime handler.
4403 if (CGM.getCodeGenOpts().SanitizeMinimalRuntime) {
4404 StaticArgs = {};
4405 DynamicArgs = {};
4406 }
4407
4408 // Handler functions take an i8* pointing to the (handler-specific) static
4409 // information block, followed by a sequence of intptr_t arguments
4410 // representing operand values.
4411 SmallVector<llvm::Value *, 4> Args;
4412 SmallVector<llvm::Type *, 4> ArgTypes;
4413
4414 Args.reserve(N: DynamicArgs.size() + 1);
4415 ArgTypes.reserve(N: DynamicArgs.size() + 1);
4416
4417 // Emit handler arguments and create handler function type.
4418 if (!StaticArgs.empty()) {
4419 llvm::Constant *Info = llvm::ConstantStruct::getAnon(V: StaticArgs);
4420 auto *InfoPtr = new llvm::GlobalVariable(
4421 CGM.getModule(), Info->getType(),
4422 // Non-constant global is used in a handler to deduplicate reports.
4423 // TODO: change deduplication logic and make it constant.
4424 /*isConstant=*/false, llvm::GlobalVariable::PrivateLinkage, Info, "",
4425 nullptr, llvm::GlobalVariable::NotThreadLocal,
4426 CGM.getDataLayout().getDefaultGlobalsAddressSpace());
4427 InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4428 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV: InfoPtr);
4429 Args.push_back(Elt: Builder.CreateAddrSpaceCast(V: InfoPtr, DestTy: CGM.VoidPtrTy));
4430 ArgTypes.push_back(Elt: CGM.VoidPtrTy);
4431 }
4432
4433 for (llvm::Value *DynamicArg : DynamicArgs) {
4434 Args.push_back(Elt: EmitCheckValue(V: DynamicArg));
4435 ArgTypes.push_back(Elt: IntPtrTy);
4436 }
4437
4438 llvm::FunctionType *FnType =
4439 llvm::FunctionType::get(Result: CGM.VoidTy, Params: ArgTypes, isVarArg: false);
4440
4441 if (!FatalCond || !RecoverableCond) {
4442 // Simple case: we need to generate a single handler call, either
4443 // fatal, or non-fatal.
4444 emitCheckHandlerCall(CGF&: *this, FnType, FnArgs: Args, CheckHandler, RecoverKind,
4445 IsFatal: (FatalCond != nullptr), ContBB: Cont, NoMerge);
4446 } else {
4447 // Emit two handler calls: first one for set of unrecoverable checks,
4448 // another one for recoverable.
4449 llvm::BasicBlock *NonFatalHandlerBB =
4450 createBasicBlock(name: "non_fatal." + CheckName);
4451 llvm::BasicBlock *FatalHandlerBB = createBasicBlock(name: "fatal." + CheckName);
4452 Builder.CreateCondBr(Cond: FatalCond, True: NonFatalHandlerBB, False: FatalHandlerBB);
4453 EmitBlock(BB: FatalHandlerBB);
4454 emitCheckHandlerCall(CGF&: *this, FnType, FnArgs: Args, CheckHandler, RecoverKind, IsFatal: true,
4455 ContBB: NonFatalHandlerBB, NoMerge);
4456 EmitBlock(BB: NonFatalHandlerBB);
4457 emitCheckHandlerCall(CGF&: *this, FnType, FnArgs: Args, CheckHandler, RecoverKind, IsFatal: false,
4458 ContBB: Cont, NoMerge);
4459 }
4460
4461 EmitBlock(BB: Cont);
4462}
4463
4464void CodeGenFunction::EmitCfiSlowPathCheck(
4465 SanitizerKind::SanitizerOrdinal Ordinal, llvm::Value *Cond,
4466 llvm::ConstantInt *TypeId, llvm::Value *Ptr,
4467 ArrayRef<llvm::Constant *> StaticArgs) {
4468 llvm::BasicBlock *Cont = createBasicBlock(name: "cfi.cont");
4469
4470 llvm::BasicBlock *CheckBB = createBasicBlock(name: "cfi.slowpath");
4471 llvm::CondBrInst *BI = Builder.CreateCondBr(Cond, True: Cont, False: CheckBB);
4472
4473 llvm::MDBuilder MDHelper(getLLVMContext());
4474 llvm::MDNode *Node = MDHelper.createLikelyBranchWeights();
4475 BI->setMetadata(KindID: llvm::LLVMContext::MD_prof, Node);
4476
4477 EmitBlock(BB: CheckBB);
4478
4479 bool WithDiag = !CGM.getCodeGenOpts().SanitizeTrap.has(O: Ordinal);
4480
4481 llvm::CallInst *CheckCall;
4482 llvm::FunctionCallee SlowPathFn;
4483 if (WithDiag) {
4484 llvm::Constant *Info = llvm::ConstantStruct::getAnon(V: StaticArgs);
4485 auto *InfoPtr =
4486 new llvm::GlobalVariable(CGM.getModule(), Info->getType(), false,
4487 llvm::GlobalVariable::PrivateLinkage, Info);
4488 InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4489 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV: InfoPtr);
4490
4491 SlowPathFn = CGM.getModule().getOrInsertFunction(
4492 Name: "__cfi_slowpath_diag",
4493 T: llvm::FunctionType::get(Result: VoidTy, Params: {Int64Ty, Int8PtrTy, Int8PtrTy},
4494 isVarArg: false));
4495 CheckCall = Builder.CreateCall(Callee: SlowPathFn, Args: {TypeId, Ptr, InfoPtr});
4496 } else {
4497 SlowPathFn = CGM.getModule().getOrInsertFunction(
4498 Name: "__cfi_slowpath",
4499 T: llvm::FunctionType::get(Result: VoidTy, Params: {Int64Ty, Int8PtrTy}, isVarArg: false));
4500 CheckCall = Builder.CreateCall(Callee: SlowPathFn, Args: {TypeId, Ptr});
4501 }
4502
4503 CGM.setDSOLocal(
4504 cast<llvm::GlobalValue>(Val: SlowPathFn.getCallee()->stripPointerCasts()));
4505 CheckCall->setDoesNotThrow();
4506
4507 EmitBlock(BB: Cont);
4508}
4509
4510// Emit a stub for __cfi_check function so that the linker knows about this
4511// symbol in LTO mode.
4512void CodeGenFunction::EmitCfiCheckStub() {
4513 llvm::Module *M = &CGM.getModule();
4514 ASTContext &C = getContext();
4515 QualType QInt64Ty = C.getIntTypeForBitwidth(DestWidth: 64, Signed: false);
4516
4517 auto *ArgCallsiteTypeId =
4518 ImplicitParamDecl::Create(C, T: QInt64Ty, ParamKind: ImplicitParamKind::Other);
4519 auto *ArgAddr =
4520 ImplicitParamDecl::Create(C, T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
4521 auto *ArgCFICheckFailData =
4522 ImplicitParamDecl::Create(C, T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
4523 FunctionArgList FnArgs{ArgCallsiteTypeId, ArgAddr, ArgCFICheckFailData};
4524 const CGFunctionInfo &FI =
4525 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: FnArgs);
4526
4527 llvm::Function *F = llvm::Function::Create(
4528 Ty: llvm::FunctionType::get(Result: VoidTy, Params: {Int64Ty, VoidPtrTy, VoidPtrTy}, isVarArg: false),
4529 Linkage: llvm::GlobalValue::WeakAnyLinkage, N: "__cfi_check", M);
4530 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F, /*IsThunk=*/false);
4531 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F);
4532 F->setAlignment(llvm::Align(4096));
4533 CGM.setDSOLocal(F);
4534
4535 llvm::LLVMContext &Ctx = M->getContext();
4536 llvm::BasicBlock *BB = llvm::BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: F);
4537 // CrossDSOCFI pass is not executed if there is no executable code.
4538 SmallVector<llvm::Value*> Args{F->getArg(i: 2), F->getArg(i: 1)};
4539 llvm::CallInst::Create(Func: M->getFunction(Name: "__cfi_check_fail"), Args, NameStr: "", InsertBefore: BB);
4540 llvm::ReturnInst::Create(C&: Ctx, retVal: nullptr, InsertBefore: BB);
4541}
4542
4543// This function is basically a switch over the CFI failure kind, which is
4544// extracted from CFICheckFailData (1st function argument). Each case is either
4545// llvm.trap or a call to one of the two runtime handlers, based on
4546// -fsanitize-trap and -fsanitize-recover settings. Default case (invalid
4547// failure kind) traps, but this should really never happen. CFICheckFailData
4548// can be nullptr if the calling module has -fsanitize-trap behavior for this
4549// check kind; in this case __cfi_check_fail traps as well.
4550void CodeGenFunction::EmitCfiCheckFail() {
4551 auto CheckHandler = SanitizerHandler::CFICheckFail;
4552 // TODO: the SanitizerKind is not yet determined for this check (and might
4553 // not even be available, if Data == nullptr). However, we still want to
4554 // annotate the instrumentation. We approximate this by using all the CFI
4555 // kinds.
4556 SanitizerDebugLocation SanScope(
4557 this,
4558 {SanitizerKind::SO_CFIVCall, SanitizerKind::SO_CFINVCall,
4559 SanitizerKind::SO_CFIDerivedCast, SanitizerKind::SO_CFIUnrelatedCast,
4560 SanitizerKind::SO_CFIICall},
4561 CheckHandler);
4562 auto *ArgData = ImplicitParamDecl::Create(
4563 C&: getContext(), T: getContext().VoidPtrTy, ParamKind: ImplicitParamKind::Other);
4564 auto *ArgAddr = ImplicitParamDecl::Create(
4565 C&: getContext(), T: getContext().VoidPtrTy, ParamKind: ImplicitParamKind::Other);
4566
4567 FunctionArgList Args{ArgData, ArgAddr};
4568 const CGFunctionInfo &FI =
4569 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: getContext().VoidTy, args: Args);
4570
4571 llvm::Function *F = llvm::Function::Create(
4572 Ty: llvm::FunctionType::get(Result: VoidTy, Params: {VoidPtrTy, VoidPtrTy}, isVarArg: false),
4573 Linkage: llvm::GlobalValue::WeakODRLinkage, N: "__cfi_check_fail", M: &CGM.getModule());
4574
4575 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F, /*IsThunk=*/false);
4576 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F);
4577 F->setVisibility(llvm::GlobalValue::HiddenVisibility);
4578
4579 StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidTy, Fn: F, FnInfo: FI, Args,
4580 Loc: SourceLocation());
4581
4582 ApplyDebugLocation ADL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
4583
4584 // This function is not affected by NoSanitizeList. This function does
4585 // not have a source location, but "src:*" would still apply. Revert any
4586 // changes to SanOpts made in StartFunction.
4587 SanOpts = CGM.getLangOpts().Sanitize;
4588
4589 llvm::Value *Data =
4590 EmitLoadOfScalar(Addr: GetAddrOfLocalVar(VD: ArgData), /*Volatile=*/false,
4591 Ty: CGM.getContext().VoidPtrTy, Loc: ArgData->getLocation());
4592 llvm::Value *Addr =
4593 EmitLoadOfScalar(Addr: GetAddrOfLocalVar(VD: ArgAddr), /*Volatile=*/false,
4594 Ty: CGM.getContext().VoidPtrTy, Loc: ArgAddr->getLocation());
4595
4596 // Data == nullptr means the calling module has trap behaviour for this check.
4597 llvm::Value *DataIsNotNullPtr =
4598 Builder.CreateICmpNE(LHS: Data, RHS: llvm::ConstantPointerNull::get(T: Int8PtrTy));
4599 // TODO: since there is no data, we don't know the CheckKind, and therefore
4600 // cannot inspect CGM.getCodeGenOpts().SanitizeMergeHandlers. We default to
4601 // NoMerge = false. Users can disable merging by disabling optimization.
4602 EmitTrapCheck(Checked: DataIsNotNullPtr, CheckHandlerID: SanitizerHandler::CFICheckFail,
4603 /*NoMerge=*/false);
4604
4605 llvm::StructType *SourceLocationTy =
4606 llvm::StructType::get(elt1: VoidPtrTy, elts: Int32Ty, elts: Int32Ty);
4607 llvm::StructType *CfiCheckFailDataTy =
4608 llvm::StructType::get(elt1: Int8Ty, elts: SourceLocationTy, elts: VoidPtrTy);
4609
4610 llvm::Value *V = Builder.CreateConstGEP2_32(
4611 Ty: CfiCheckFailDataTy, Ptr: Builder.CreatePointerCast(V: Data, DestTy: DefaultPtrTy), Idx0: 0, Idx1: 0);
4612
4613 Address CheckKindAddr(V, Int8Ty, getIntAlign());
4614 llvm::Value *CheckKind = Builder.CreateLoad(Addr: CheckKindAddr);
4615
4616 llvm::Value *AllVtables = llvm::MetadataAsValue::get(
4617 Context&: CGM.getLLVMContext(),
4618 MD: llvm::MDString::get(Context&: CGM.getLLVMContext(), Str: "all-vtables"));
4619 llvm::Value *ValidVtable = Builder.CreateZExt(
4620 V: Builder.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::type_test),
4621 Args: {Addr, AllVtables}),
4622 DestTy: IntPtrTy);
4623
4624 const std::pair<int, SanitizerKind::SanitizerOrdinal> CheckKinds[] = {
4625 {CFITCK_VCall, SanitizerKind::SO_CFIVCall},
4626 {CFITCK_NVCall, SanitizerKind::SO_CFINVCall},
4627 {CFITCK_DerivedCast, SanitizerKind::SO_CFIDerivedCast},
4628 {CFITCK_UnrelatedCast, SanitizerKind::SO_CFIUnrelatedCast},
4629 {CFITCK_ICall, SanitizerKind::SO_CFIICall}};
4630
4631 for (auto CheckKindOrdinalPair : CheckKinds) {
4632 int Kind = CheckKindOrdinalPair.first;
4633 SanitizerKind::SanitizerOrdinal Ordinal = CheckKindOrdinalPair.second;
4634
4635 // TODO: we could apply SanitizerAnnotateDebugInfo(Ordinal) instead of
4636 // relying on the SanitizerScope with all CFI ordinals
4637
4638 llvm::Value *Cond =
4639 Builder.CreateICmpNE(LHS: CheckKind, RHS: llvm::ConstantInt::get(Ty: Int8Ty, V: Kind));
4640 if (CGM.getLangOpts().Sanitize.has(O: Ordinal))
4641 EmitCheck(Checked: std::make_pair(x&: Cond, y&: Ordinal), CheckHandler: SanitizerHandler::CFICheckFail,
4642 StaticArgs: {}, DynamicArgs: {Data, Addr, ValidVtable});
4643 else
4644 // TODO: we can't rely on CGM.getCodeGenOpts().SanitizeMergeHandlers.
4645 // Although the compiler allows SanitizeMergeHandlers to be set
4646 // independently of CGM.getLangOpts().Sanitize, Driver/SanitizerArgs.cpp
4647 // requires that SanitizeMergeHandlers is a subset of Sanitize.
4648 EmitTrapCheck(Checked: Cond, CheckHandlerID: CheckHandler, /*NoMerge=*/false);
4649 }
4650
4651 FinishFunction();
4652 // The only reference to this function will be created during LTO link.
4653 // Make sure it survives until then.
4654 CGM.addUsedGlobal(GV: F);
4655}
4656
4657void CodeGenFunction::EmitUnreachable(SourceLocation Loc) {
4658 if (SanOpts.has(K: SanitizerKind::Unreachable)) {
4659 auto CheckOrdinal = SanitizerKind::SO_Unreachable;
4660 auto CheckHandler = SanitizerHandler::BuiltinUnreachable;
4661 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
4662 EmitCheck(Checked: std::make_pair(x: static_cast<llvm::Value *>(Builder.getFalse()),
4663 y&: CheckOrdinal),
4664 CheckHandler, StaticArgs: EmitCheckSourceLocation(Loc), DynamicArgs: {});
4665 }
4666 Builder.CreateUnreachable();
4667}
4668
4669void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked,
4670 SanitizerHandler CheckHandlerID,
4671 bool NoMerge, const TrapReason *TR) {
4672 llvm::BasicBlock *Cont = createBasicBlock(name: "cont");
4673
4674 // If we're optimizing, collapse all calls to trap down to just one per
4675 // check-type per function to save on code size.
4676 if ((int)TrapBBs.size() <= CheckHandlerID)
4677 TrapBBs.resize(N: CheckHandlerID + 1);
4678
4679 llvm::BasicBlock *&TrapBB = TrapBBs[CheckHandlerID];
4680
4681 llvm::DILocation *TrapLocation = Builder.getCurrentDebugLocation();
4682 llvm::StringRef TrapMessage;
4683 llvm::StringRef TrapCategory;
4684 auto DebugTrapReasonKind = CGM.getCodeGenOpts().getSanitizeDebugTrapReasons();
4685 if (TR && !TR->isEmpty() &&
4686 DebugTrapReasonKind ==
4687 CodeGenOptions::SanitizeDebugTrapReasonKind::Detailed) {
4688 TrapMessage = TR->getMessage();
4689 TrapCategory = TR->getCategory();
4690 } else {
4691 TrapMessage = GetUBSanTrapForHandler(ID: CheckHandlerID);
4692 TrapCategory = "Undefined Behavior Sanitizer";
4693 }
4694
4695 if (getDebugInfo() && !TrapMessage.empty() &&
4696 DebugTrapReasonKind !=
4697 CodeGenOptions::SanitizeDebugTrapReasonKind::None &&
4698 TrapLocation) {
4699 TrapLocation = getDebugInfo()->CreateTrapFailureMessageFor(
4700 TrapLocation, Category: TrapCategory, FailureMsg: TrapMessage);
4701 }
4702
4703 NoMerge = NoMerge || !CGM.getCodeGenOpts().isOptimizedBuild() ||
4704 (CurCodeDecl && CurCodeDecl->hasAttr<OptimizeNoneAttr>());
4705
4706 llvm::MDBuilder MDHelper(getLLVMContext());
4707 if (TrapBB && !NoMerge) {
4708 auto Call = TrapBB->begin();
4709 assert(isa<llvm::CallInst>(Call) && "Expected call in trap BB");
4710
4711 Call->applyMergedLocation(LocA: Call->getDebugLoc(), LocB: TrapLocation);
4712
4713 Builder.CreateCondBr(Cond: Checked, True: Cont, False: TrapBB,
4714 BranchWeights: MDHelper.createLikelyBranchWeights());
4715 } else {
4716 TrapBB = createBasicBlock(name: "trap");
4717 Builder.CreateCondBr(Cond: Checked, True: Cont, False: TrapBB,
4718 BranchWeights: MDHelper.createLikelyBranchWeights());
4719 EmitBlock(BB: TrapBB);
4720
4721 ApplyDebugLocation applyTrapDI(*this, TrapLocation);
4722
4723 llvm::CallInst *TrapCall;
4724 if (CGM.getCodeGenOpts().SanitizeTrapLoop)
4725 TrapCall =
4726 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::looptrap));
4727 else
4728 TrapCall = Builder.CreateCall(
4729 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::ubsantrap),
4730 Args: llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: CheckHandlerID));
4731
4732 if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
4733 auto A = llvm::Attribute::get(Context&: getLLVMContext(), Kind: "trap-func-name",
4734 Val: CGM.getCodeGenOpts().TrapFuncName);
4735 TrapCall->addFnAttr(Attr: A);
4736 }
4737 if (NoMerge)
4738 TrapCall->addFnAttr(Kind: llvm::Attribute::NoMerge);
4739 Builder.CreateUnreachable();
4740 }
4741
4742 EmitBlock(BB: Cont);
4743}
4744
4745llvm::CallInst *CodeGenFunction::EmitTrapCall(llvm::Intrinsic::ID IntrID,
4746 bool EnsureInsertPoint) {
4747 llvm::Function *TrapIntrinsic = CGM.getIntrinsic(IID: IntrID);
4748 llvm::CallInst *TrapCall = Builder.CreateCall(Callee: TrapIntrinsic);
4749
4750 if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
4751 auto A = llvm::Attribute::get(Context&: getLLVMContext(), Kind: "trap-func-name",
4752 Val: CGM.getCodeGenOpts().TrapFuncName);
4753 TrapCall->addFnAttr(Attr: A);
4754 }
4755
4756 if (InNoMergeAttributedStmt)
4757 TrapCall->addFnAttr(Kind: llvm::Attribute::NoMerge);
4758 if (TrapIntrinsic->doesNotReturn()) {
4759 Builder.CreateUnreachable();
4760 if (EnsureInsertPoint)
4761 EmitBlock(BB: createBasicBlock());
4762 else
4763 Builder.ClearInsertionPoint();
4764 }
4765 return TrapCall;
4766}
4767
4768void CodeGenFunction::EmitTrapCallAndMakeUnreachable() {
4769 EmitTrapCall(IntrID: llvm::Intrinsic::trap, /*EnsureInsertPoint=*/false);
4770 if (HaveInsertPoint()) {
4771 Builder.CreateUnreachable();
4772 Builder.ClearInsertionPoint();
4773 }
4774}
4775
4776Address CodeGenFunction::EmitArrayToPointerDecay(const Expr *E,
4777 LValueBaseInfo *BaseInfo,
4778 TBAAAccessInfo *TBAAInfo) {
4779 assert(E->getType()->isArrayType() &&
4780 "Array to pointer decay must have array source type!");
4781
4782 // Expressions of array type can't be bitfields or vector elements.
4783 LValue LV = EmitLValue(E);
4784 Address Addr = LV.getAddress();
4785
4786 // If the array type was an incomplete type, we need to make sure
4787 // the decay ends up being the right type.
4788 llvm::Type *NewTy = ConvertType(T: E->getType());
4789 Addr = Addr.withElementType(ElemTy: NewTy);
4790
4791 // Note that VLA pointers are always decayed, so we don't need to do
4792 // anything here.
4793 if (!E->getType()->isVariableArrayType()) {
4794 assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
4795 "Expected pointer to array");
4796
4797 if (getLangOpts().EmitLogicalPointer) {
4798 // Array-to-pointer decay for an SGEP is a no-op as we don't do any
4799 // logical indexing. See #179951 for some additional context.
4800 auto *SGEP =
4801 Builder.CreateStructuredGEP(BaseType: NewTy, PtrBase: Addr.emitRawPointer(CGF&: *this), Indices: {});
4802 Addr = Address(SGEP, NewTy, Addr.getAlignment(), Addr.isKnownNonNull());
4803 } else {
4804 Addr = Builder.CreateConstArrayGEP(Addr, Index: 0, Name: "arraydecay");
4805 }
4806 }
4807
4808 // The result of this decay conversion points to an array element within the
4809 // base lvalue. However, since TBAA currently does not support representing
4810 // accesses to elements of member arrays, we conservatively represent accesses
4811 // to the pointee object as if it had no any base lvalue specified.
4812 // TODO: Support TBAA for member arrays.
4813 QualType EltType = E->getType()->castAsArrayTypeUnsafe()->getElementType();
4814 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
4815 if (TBAAInfo) *TBAAInfo = CGM.getTBAAAccessInfo(AccessType: EltType);
4816
4817 return Addr.withElementType(ElemTy: ConvertTypeForMem(T: EltType));
4818}
4819
4820/// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an
4821/// array to pointer, return the array subexpression.
4822static const Expr *isSimpleArrayDecayOperand(const Expr *E) {
4823 // If this isn't just an array->pointer decay, bail out.
4824 const auto *CE = dyn_cast<CastExpr>(Val: E);
4825 if (!CE || CE->getCastKind() != CK_ArrayToPointerDecay)
4826 return nullptr;
4827
4828 // If this is a decay from variable width array, bail out.
4829 const Expr *SubExpr = CE->getSubExpr();
4830 if (SubExpr->getType()->isVariableArrayType())
4831 return nullptr;
4832
4833 return SubExpr;
4834}
4835
4836static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF,
4837 llvm::Type *elemType,
4838 llvm::Value *ptr,
4839 ArrayRef<llvm::Value*> indices,
4840 bool inbounds,
4841 bool signedIndices,
4842 SourceLocation loc,
4843 const llvm::Twine &name = "arrayidx") {
4844 if (inbounds && CGF.getLangOpts().EmitLogicalPointer)
4845 return CGF.Builder.CreateStructuredGEP(BaseType: elemType, PtrBase: ptr, Indices: indices);
4846
4847 if (inbounds) {
4848 return CGF.EmitCheckedInBoundsGEP(ElemTy: elemType, Ptr: ptr, IdxList: indices, SignedIndices: signedIndices,
4849 IsSubtraction: CodeGenFunction::NotSubtraction, Loc: loc,
4850 Name: name);
4851 } else {
4852 return CGF.Builder.CreateGEP(Ty: elemType, Ptr: ptr, IdxList: indices, Name: name);
4853 }
4854}
4855
4856static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
4857 ArrayRef<llvm::Value *> indices,
4858 llvm::Type *arrayType,
4859 llvm::Type *elementType, bool inbounds,
4860 bool signedIndices, SourceLocation loc,
4861 CharUnits align,
4862 const llvm::Twine &name = "arrayidx") {
4863 if (inbounds && CGF.getLangOpts().EmitLogicalPointer)
4864 return RawAddress(CGF.Builder.CreateStructuredGEP(BaseType: arrayType,
4865 PtrBase: addr.emitRawPointer(CGF),
4866 Indices: indices.drop_front()),
4867 elementType, align);
4868
4869 if (inbounds) {
4870 return CGF.EmitCheckedInBoundsGEP(Addr: addr, IdxList: indices, elementType, SignedIndices: signedIndices,
4871 IsSubtraction: CodeGenFunction::NotSubtraction, Loc: loc,
4872 Align: align, Name: name);
4873 } else {
4874 return CGF.Builder.CreateGEP(Addr: addr, IdxList: indices, ElementType: elementType, Align: align, Name: name);
4875 }
4876}
4877
4878static bool hasBPFPreserveStaticOffset(const RecordDecl *D) {
4879 return D && D->hasAttr<BPFPreserveStaticOffsetAttr>();
4880}
4881
4882static bool hasBPFPreserveStaticOffset(const Expr *E) {
4883 if (!E)
4884 return false;
4885 QualType PointeeType = E->getType()->getPointeeType();
4886 if (PointeeType.isNull())
4887 return false;
4888 if (const auto *BaseDecl = PointeeType->getAsRecordDecl())
4889 return hasBPFPreserveStaticOffset(D: BaseDecl);
4890 return false;
4891}
4892
4893// Wraps Addr with a call to llvm.preserve.static.offset intrinsic.
4894static Address wrapWithBPFPreserveStaticOffset(CodeGenFunction &CGF,
4895 Address &Addr) {
4896 if (!CGF.getTarget().getTriple().isBPF() || CGF.IsInPreserveFieldInfo)
4897 return Addr;
4898
4899 llvm::Function *Fn =
4900 CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::preserve_static_offset);
4901 llvm::CallInst *Call = CGF.Builder.CreateCall(Callee: Fn, Args: {Addr.emitRawPointer(CGF)});
4902 return Address(Call, Addr.getElementType(), Addr.getAlignment());
4903}
4904
4905/// Given an array base, check whether its member access belongs to a record
4906/// with preserve_access_index attribute or not.
4907static bool IsPreserveAIArrayBase(CodeGenFunction &CGF, const Expr *ArrayBase) {
4908 if (!ArrayBase || !CGF.getDebugInfo())
4909 return false;
4910
4911 // Only support base as either a MemberExpr or DeclRefExpr.
4912 // DeclRefExpr to cover cases like:
4913 // struct s { int a; int b[10]; };
4914 // struct s *p;
4915 // p[1].a
4916 // p[1] will generate a DeclRefExpr and p[1].a is a MemberExpr.
4917 // p->b[5] is a MemberExpr example.
4918 const Expr *E = ArrayBase->IgnoreParenImpCasts();
4919 // In p->b[1][2], the base of [2] is the subscript p->b[1]. Step through
4920 // subscripts to the member p->b.
4921 while (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: E))
4922 E = ASE->getBase()->IgnoreParenImpCasts();
4923 if (const auto *ME = dyn_cast<MemberExpr>(Val: E))
4924 return ME->getMemberDecl()->hasAttr<BPFPreserveAccessIndexAttr>();
4925
4926 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
4927 const auto *VarDef = dyn_cast<VarDecl>(Val: DRE->getDecl());
4928 if (!VarDef)
4929 return false;
4930
4931 const auto *PtrT = VarDef->getType()->getAs<PointerType>();
4932 if (!PtrT)
4933 return false;
4934
4935 const auto *PointeeT = PtrT->getPointeeType()
4936 ->getUnqualifiedDesugaredType();
4937 if (const auto *RecT = dyn_cast<RecordType>(Val: PointeeT))
4938 return RecT->getDecl()
4939 ->getMostRecentDecl()
4940 ->hasAttr<BPFPreserveAccessIndexAttr>();
4941 return false;
4942 }
4943
4944 return false;
4945}
4946
4947static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
4948 ArrayRef<llvm::Value *> indices,
4949 QualType eltType, bool inbounds,
4950 bool signedIndices, SourceLocation loc,
4951 QualType *arrayType = nullptr,
4952 const Expr *Base = nullptr,
4953 const llvm::Twine &name = "arrayidx") {
4954 // All the indices except that last must be zero.
4955#ifndef NDEBUG
4956 for (auto *idx : indices.drop_back())
4957 assert(isa<llvm::ConstantInt>(idx) &&
4958 cast<llvm::ConstantInt>(idx)->isZero());
4959#endif
4960
4961 // Determine the element size of the statically-sized base. This is
4962 // the thing that the indices are expressed in terms of.
4963 if (auto vla = CGF.getContext().getAsVariableArrayType(T: eltType)) {
4964 eltType = CodeGenUtils::getFixedSizeElementType(Ctx: CGF.getContext(), VLA: vla);
4965 }
4966
4967 // We can use that to compute the best alignment of the element.
4968 CharUnits eltSize = CGF.getContext().getTypeSizeInChars(T: eltType);
4969 CharUnits eltAlign =
4970 getArrayElementAlign(arrayAlign: addr.getAlignment(), idx: indices.back(), eltSize);
4971
4972 if (hasBPFPreserveStaticOffset(E: Base))
4973 addr = wrapWithBPFPreserveStaticOffset(CGF, Addr&: addr);
4974
4975 llvm::Value *eltPtr;
4976 auto LastIndex = dyn_cast<llvm::ConstantInt>(Val: indices.back());
4977 if (!LastIndex ||
4978 (!CGF.IsInPreservedAIRegion && !IsPreserveAIArrayBase(CGF, ArrayBase: Base))) {
4979 addr = emitArraySubscriptGEP(CGF, addr, indices,
4980 arrayType: arrayType ? CGF.ConvertTypeForMem(T: *arrayType)
4981 : nullptr,
4982 elementType: CGF.ConvertTypeForMem(T: eltType), inbounds,
4983 signedIndices, loc, align: eltAlign, name);
4984 return addr;
4985 } else {
4986 // Remember the original array subscript for bpf target
4987 unsigned idx = LastIndex->getZExtValue();
4988 llvm::DIType *DbgInfo = nullptr;
4989 if (arrayType)
4990 DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(Ty: *arrayType, Loc: loc);
4991 eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(
4992 ElTy: addr.getElementType(), Base: addr.emitRawPointer(CGF), Dimension: indices.size() - 1,
4993 LastIndex: idx, DbgInfo);
4994 }
4995
4996 return Address(eltPtr, CGF.ConvertTypeForMem(T: eltType), eltAlign);
4997}
4998
4999namespace {
5000
5001/// StructFieldAccess is a simple visitor class to grab the first l-value to
5002/// r-value cast Expr.
5003struct StructFieldAccess
5004 : public ConstStmtVisitor<StructFieldAccess, const Expr *> {
5005 const Expr *VisitCastExpr(const CastExpr *E) {
5006 if (E->getCastKind() == CK_LValueToRValue)
5007 return E;
5008 return Visit(S: E->getSubExpr());
5009 }
5010 const Expr *VisitParenExpr(const ParenExpr *E) {
5011 return Visit(S: E->getSubExpr());
5012 }
5013};
5014
5015} // end anonymous namespace
5016
5017/// Returns the relative offset difference between \p FD1 and \p FD2.
5018/// \code
5019/// offsetof(struct foo, FD1) - offsetof(struct foo, FD2)
5020/// \endcode
5021/// Both fields must be within the same struct.
5022static std::optional<int64_t> getOffsetDifferenceInBits(CodeGenFunction &CGF,
5023 const FieldDecl *FD1,
5024 const FieldDecl *FD2) {
5025 const RecordDecl *FD1OuterRec =
5026 FD1->getParent()->getOuterLexicalRecordContext();
5027 const RecordDecl *FD2OuterRec =
5028 FD2->getParent()->getOuterLexicalRecordContext();
5029
5030 if (FD1OuterRec != FD2OuterRec)
5031 // Fields must be within the same RecordDecl.
5032 return std::optional<int64_t>();
5033
5034 int64_t FD1Offset = 0;
5035 if (!getFieldOffsetInBits(CGF, RD: FD1OuterRec, Field: FD1, Offset&: FD1Offset))
5036 return std::optional<int64_t>();
5037
5038 int64_t FD2Offset = 0;
5039 if (!getFieldOffsetInBits(CGF, RD: FD2OuterRec, Field: FD2, Offset&: FD2Offset))
5040 return std::optional<int64_t>();
5041
5042 return std::make_optional<int64_t>(t: FD1Offset - FD2Offset);
5043}
5044
5045/// Convert a '__sized_by' byte-count bound to an element-count bound so it can
5046/// be compared against an element index. \p BoundsVal is the loaded byte count
5047/// and \p PointeeTy is the pointer's pointee type. Returns \p BoundsVal
5048/// unchanged when no scaling is needed, otherwise returns a new `llvm::Value*`
5049/// that is element count.
5050static llvm::Value *convertSizedByBoundToElementCount(CodeGenFunction &CGF,
5051 llvm::Value *BoundsVal,
5052 QualType PointeeTy,
5053 bool CountSigned) {
5054 assert(BoundsVal->getType()->isIntegerTy());
5055 assert(!PointeeTy.isNull() && "pointee type is never null");
5056 assert(!PointeeTy->isFunctionType() &&
5057 "Sema guarantees a '__sized_by' pointee is a non-function type");
5058
5059 if (PointeeTy->isIncompleteType()) {
5060 // Sema enforces that only 'void' can be subscripted here (GNU extension,
5061 // 1-byte stride), so the byte count already equals the element count.
5062 assert(PointeeTy->isVoidType() && "expected a 'void' incomplete pointee");
5063 return BoundsVal;
5064 }
5065
5066 CharUnits ElemSize = CGF.getContext().getTypeSizeInChars(T: PointeeTy);
5067 if (ElemSize <= CharUnits::One())
5068 return BoundsVal;
5069
5070 int64_t ElemSizeQ = ElemSize.getQuantity();
5071 unsigned CountWidth = BoundsVal->getType()->getIntegerBitWidth();
5072
5073 // The divisor must be representable in the count field's type.
5074 bool ElemSizeFits =
5075 CountSigned ? llvm::isIntN(N: CountWidth, x: ElemSizeQ)
5076 : llvm::isUIntN(N: CountWidth, x: static_cast<uint64_t>(ElemSizeQ));
5077 if (!ElemSizeFits)
5078 // No whole element fits in any representable byte count. Use a bound of 0
5079 // to always trap.
5080 // FIXME: Sema should just reject this (#223525).
5081 return llvm::ConstantInt::get(Ty: BoundsVal->getType(), V: 0);
5082
5083 llvm::Value *ElemSizeV =
5084 llvm::ConstantInt::get(Ty: BoundsVal->getType(), V: ElemSizeQ);
5085 // Use signed division for a signed count field so a negative byte count stays
5086 // non-positive and is still rejected by the negative-bounds guard in
5087 // EmitBoundsCheckImpl (unsigned division would turn it into a large positive
5088 // count).
5089 return CountSigned ? CGF.Builder.CreateSDiv(LHS: BoundsVal, RHS: ElemSizeV)
5090 : CGF.Builder.CreateUDiv(LHS: BoundsVal, RHS: ElemSizeV);
5091}
5092
5093/// EmitCountedByBoundsChecking - If the array being accessed has a "counted_by"
5094/// attribute, generate bounds checking code. The "count" field is at the top
5095/// level of the struct or in an anonymous struct, that's also at the top level.
5096/// Future expansions may allow the "count" to reside at any place in the
5097/// struct, but the value of "counted_by" will be a "simple" path to the count,
5098/// i.e. "a.b.count", so we shouldn't need the full force of EmitLValue or
5099/// similar to emit the correct GEP.
5100void CodeGenFunction::EmitCountedByBoundsChecking(
5101 const Expr *ArrayExpr, QualType ArrayType, Address ArrayInst,
5102 QualType IndexType, llvm::Value *IndexVal, bool Accessed,
5103 bool FlexibleArray) {
5104 const auto *ME = dyn_cast<MemberExpr>(Val: ArrayExpr->IgnoreImpCasts());
5105 if (!ME || !ME->getMemberDecl()->getType()->isCountAttributedType())
5106 return;
5107
5108 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
5109 getLangOpts().getStrictFlexArraysLevel();
5110 if (FlexibleArray &&
5111 !ME->isFlexibleArrayMemberLike(Context: getContext(), StrictFlexArraysLevel))
5112 return;
5113
5114 const FieldDecl *FD = cast<FieldDecl>(Val: ME->getMemberDecl());
5115 const FieldDecl *CountFD = FD->findCountedByField();
5116 if (!CountFD)
5117 return;
5118
5119 if (std::optional<int64_t> Diff =
5120 getOffsetDifferenceInBits(CGF&: *this, FD1: CountFD, FD2: FD)) {
5121 if (!ArrayInst.isValid()) {
5122 // An invalid Address indicates we're checking a pointer array access.
5123 // Emit the checked L-Value here.
5124 LValue LV = EmitCheckedLValue(E: ArrayExpr, TCK: TCK_MemberAccess);
5125 ArrayInst = LV.getAddress();
5126 }
5127
5128 // FIXME: The 'static_cast' is necessary, otherwise the result turns into a
5129 // uint64_t, which messes things up if we have a negative offset difference.
5130 Diff = *Diff / static_cast<int64_t>(CGM.getContext().getCharWidth());
5131
5132 // Create a GEP with the byte offset between the counted object and the
5133 // count and use that to load the count value.
5134 Address CountAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
5135 Addr: ArrayInst, Ty: Int8PtrTy, ElementTy: Int8Ty);
5136
5137 llvm::Type *BoundsType = ConvertType(T: CountFD->getType());
5138 llvm::Value *BoundsVal =
5139 Builder.CreateInBoundsGEP(Ty: Int8Ty, Ptr: CountAddr.emitRawPointer(CGF&: *this),
5140 IdxList: Builder.getInt32(C: *Diff), Name: ".counted_by.gep");
5141 BoundsVal = Builder.CreateAlignedLoad(Ty: BoundsType, Addr: BoundsVal, Align: getIntAlign(),
5142 Name: ".counted_by.load");
5143
5144 const auto *CountAttributedTy = FD->getType()->getAs<CountAttributedType>();
5145 assert(CountAttributedTy && "expected FD to have a CountAttributedType");
5146
5147 // For the '_or_null' variants a null pointer describes no accessible
5148 // memory, so treat the bound as 0 when the pointer is null; any access then
5149 // traps.
5150 if (CountAttributedTy->isOrNull()) {
5151 // Load the pointer from its address rather than re-emitting the
5152 // member expression, which would re-evaluate a side-effecting base.
5153 llvm::Value *Ptr = Builder.CreateLoad(Addr: ArrayInst);
5154 llvm::Value *IsNull = Builder.CreateIsNull(Arg: Ptr);
5155 BoundsVal = Builder.CreateSelect(
5156 C: IsNull, True: llvm::ConstantInt::get(Ty: BoundsType, V: 0), False: BoundsVal);
5157 }
5158
5159 // For '__sized_by' the loaded bound is a byte count. Convert it to an
5160 // element count by dividing by the element size so the check can compare
5161 // the (element) index directly. '__counted_by' already counts elements so
5162 // needs no special handling.
5163 if (CountAttributedTy->isCountInBytes())
5164 BoundsVal = convertSizedByBoundToElementCount(
5165 CGF&: *this, BoundsVal, PointeeTy: ArrayType->getPointeeType(),
5166 CountSigned: CountFD->getType()->isSignedIntegerOrEnumerationType());
5167
5168 // Now emit the bounds checking.
5169 EmitBoundsCheckImpl(ArrayExpr, ArrayBaseType: ArrayType, IndexVal, IndexType, BoundsVal,
5170 BoundsType: CountFD->getType(), Accessed);
5171 }
5172}
5173
5174LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
5175 bool Accessed) {
5176 // The index must always be an integer, which is not an aggregate. Emit it
5177 // in lexical order (this complexity is, sadly, required by C++17).
5178 llvm::Value *IdxPre =
5179 (E->getLHS() == E->getIdx()) ? EmitScalarExpr(E: E->getIdx()) : nullptr;
5180 bool SignedIndices = false;
5181 auto EmitIdxAfterBase = [&, IdxPre](bool Promote) -> llvm::Value * {
5182 auto *Idx = IdxPre;
5183 if (E->getLHS() != E->getIdx()) {
5184 assert(E->getRHS() == E->getIdx() && "index was neither LHS nor RHS");
5185 Idx = EmitScalarExpr(E: E->getIdx());
5186 }
5187
5188 QualType IdxTy = E->getIdx()->getType();
5189 bool IdxSigned = IdxTy->isSignedIntegerOrEnumerationType();
5190 SignedIndices |= IdxSigned;
5191
5192 if (SanOpts.has(K: SanitizerKind::ArrayBounds))
5193 EmitBoundsCheck(ArrayExpr: E, ArrayExprBase: E->getBase(), IndexVal: Idx, IndexType: IdxTy, Accessed);
5194
5195 // Extend or truncate the index type to 32 or 64-bits.
5196 if (Promote && Idx->getType() != IntPtrTy)
5197 Idx = Builder.CreateIntCast(V: Idx, DestTy: IntPtrTy, isSigned: IdxSigned, Name: "idxprom");
5198
5199 return Idx;
5200 };
5201 IdxPre = nullptr;
5202
5203 // If the base is a vector type, then we are forming a vector element lvalue
5204 // with this subscript.
5205 if (E->getBase()->getType()->isSubscriptableVectorType() &&
5206 !isa<ExtVectorElementExpr>(Val: E->getBase())) {
5207 // Emit the vector as an lvalue to get its address.
5208 LValue LHS = EmitLValue(E: E->getBase());
5209 auto *Idx = EmitIdxAfterBase(/*Promote*/false);
5210 assert(LHS.isSimple() && "Can only subscript lvalue vectors here!");
5211 return LValue::MakeVectorElt(vecAddress: LHS.getAddress(), Idx, type: E->getBase()->getType(),
5212 BaseInfo: LHS.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5213 }
5214
5215 // The HLSL runtime handles subscript expressions on global resource arrays
5216 // and objects with HLSL buffer layouts.
5217 if (getLangOpts().HLSL) {
5218 std::optional<LValue> LV;
5219 if (E->getType()->isHLSLResourceRecord() ||
5220 E->getType()->isHLSLResourceRecordArray()) {
5221 LV = CGM.getHLSLRuntime().emitResourceArraySubscriptExpr(E, CGF&: *this);
5222 } else if (E->getType().getAddressSpace() == LangAS::hlsl_constant) {
5223 LV = CGM.getHLSLRuntime().emitBufferArraySubscriptExpr(E, CGF&: *this,
5224 EmitIdxAfterBase);
5225 }
5226 if (LV.has_value())
5227 return *LV;
5228 }
5229
5230 // All the other cases basically behave like simple offsetting.
5231
5232 // Handle the extvector case we ignored above.
5233 if (isa<ExtVectorElementExpr>(Val: E->getBase())) {
5234 LValue LV = EmitLValue(E: E->getBase());
5235 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5236 Address Addr = EmitExtVectorElementLValue(LV);
5237
5238 QualType EltType = LV.getType()->castAs<VectorType>()->getElementType();
5239 Addr = emitArraySubscriptGEP(CGF&: *this, addr: Addr, indices: Idx, eltType: EltType, /*inbounds*/ true,
5240 signedIndices: SignedIndices, loc: E->getExprLoc());
5241 return MakeAddrLValue(Addr, T: EltType, BaseInfo: LV.getBaseInfo(),
5242 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: EltType));
5243 }
5244
5245 LValueBaseInfo EltBaseInfo;
5246 TBAAAccessInfo EltTBAAInfo;
5247 Address Addr = Address::invalid();
5248 if (const VariableArrayType *vla =
5249 getContext().getAsVariableArrayType(T: E->getType())) {
5250 // The base must be a pointer, which is not an aggregate. Emit
5251 // it. It needs to be emitted first in case it's what captures
5252 // the VLA bounds.
5253 Addr = EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
5254 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5255
5256 // The element count here is the total number of non-VLA elements.
5257 llvm::Value *numElements = getVLASize(vla).NumElts;
5258
5259 // Effectively, the multiply by the VLA size is part of the GEP.
5260 // GEP indexes are signed, and scaling an index isn't permitted to
5261 // signed-overflow, so we use the same semantics for our explicit
5262 // multiply. We suppress this if overflow is not undefined behavior.
5263 if (getLangOpts().PointerOverflowDefined) {
5264 Idx = Builder.CreateMul(LHS: Idx, RHS: numElements);
5265 } else {
5266 Idx = Builder.CreateNSWMul(LHS: Idx, RHS: numElements);
5267 }
5268
5269 Addr = emitArraySubscriptGEP(CGF&: *this, addr: Addr, indices: Idx, eltType: vla->getElementType(),
5270 inbounds: !getLangOpts().PointerOverflowDefined,
5271 signedIndices: SignedIndices, loc: E->getExprLoc());
5272
5273 } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){
5274 // Indexing over an interface, as in "NSString *P; P[4];"
5275
5276 // Emit the base pointer.
5277 Addr = EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
5278 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5279
5280 CharUnits InterfaceSize = getContext().getTypeSizeInChars(T: OIT);
5281 llvm::Value *InterfaceSizeVal =
5282 llvm::ConstantInt::get(Ty: Idx->getType(), V: InterfaceSize.getQuantity());
5283
5284 llvm::Value *ScaledIdx = Builder.CreateMul(LHS: Idx, RHS: InterfaceSizeVal);
5285
5286 // We don't necessarily build correct LLVM struct types for ObjC
5287 // interfaces, so we can't rely on GEP to do this scaling
5288 // correctly, so we need to cast to i8*. FIXME: is this actually
5289 // true? A lot of other things in the fragile ABI would break...
5290 llvm::Type *OrigBaseElemTy = Addr.getElementType();
5291
5292 // Do the GEP.
5293 CharUnits EltAlign =
5294 getArrayElementAlign(arrayAlign: Addr.getAlignment(), idx: Idx, eltSize: InterfaceSize);
5295 llvm::Value *EltPtr =
5296 emitArraySubscriptGEP(CGF&: *this, elemType: Int8Ty, ptr: Addr.emitRawPointer(CGF&: *this),
5297 indices: ScaledIdx, inbounds: false, signedIndices: SignedIndices, loc: E->getExprLoc());
5298 Addr = Address(EltPtr, OrigBaseElemTy, EltAlign);
5299 } else if (const Expr *Array = isSimpleArrayDecayOperand(E: E->getBase())) {
5300 // If this is A[i] where A is an array, the frontend will have decayed the
5301 // base to be a ArrayToPointerDecay implicit cast. While correct, it is
5302 // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
5303 // "gep x, i" here. Emit one "gep A, 0, i".
5304 assert(Array->getType()->isArrayType() &&
5305 "Array to pointer decay must have array source type!");
5306 LValue ArrayLV;
5307 // For simple multidimensional array indexing, set the 'accessed' flag for
5308 // better bounds-checking of the base expression.
5309 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Array))
5310 ArrayLV = EmitArraySubscriptExpr(E: ASE, /*Accessed*/ true);
5311 else
5312 ArrayLV = EmitLValue(E: Array);
5313 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5314
5315 if (SanOpts.has(K: SanitizerKind::ArrayBounds))
5316 EmitCountedByBoundsChecking(ArrayExpr: Array, ArrayType: Array->getType(), ArrayInst: ArrayLV.getAddress(),
5317 IndexType: E->getIdx()->getType(), IndexVal: Idx, Accessed,
5318 /*FlexibleArray=*/true);
5319
5320 // Propagate the alignment from the array itself to the result.
5321 QualType arrayType = Array->getType();
5322 Addr = emitArraySubscriptGEP(
5323 CGF&: *this, addr: ArrayLV.getAddress(), indices: {CGM.getSize(numChars: CharUnits::Zero()), Idx},
5324 eltType: E->getType(), inbounds: !getLangOpts().PointerOverflowDefined, signedIndices: SignedIndices,
5325 loc: E->getExprLoc(), arrayType: &arrayType, Base: E->getBase());
5326 EltBaseInfo = ArrayLV.getBaseInfo();
5327 if (!CGM.getCodeGenOpts().NewStructPathTBAA) {
5328 // Since CodeGenTBAA::getTypeInfoHelper only handles array types for
5329 // new struct path TBAA, we must a use a plain access.
5330 EltTBAAInfo = CGM.getTBAAInfoForSubobject(Base: ArrayLV, AccessType: E->getType());
5331 } else if (ArrayLV.getTBAAInfo().isMayAlias()) {
5332 EltTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
5333 } else if (ArrayLV.getTBAAInfo().isIncomplete()) {
5334 // The array element is complete, even if the array is not.
5335 EltTBAAInfo = CGM.getTBAAAccessInfo(AccessType: E->getType());
5336 } else {
5337 // The TBAA access info from the array (base) lvalue is ordinary. We will
5338 // adapt it to create access info for the element.
5339 EltTBAAInfo = ArrayLV.getTBAAInfo();
5340
5341 // We retain the TBAA struct path (BaseType and Offset members) from the
5342 // array. In the TBAA representation, we map any array access to the
5343 // element at index 0, as the index is generally a runtime value. This
5344 // element has the same offset in the base type as the array itself.
5345 // If the array lvalue had no base type, there is no point trying to
5346 // generate one, since an array itself is not a valid base type.
5347
5348 // We also retain the access type from the base lvalue, but the access
5349 // size must be updated to the size of an individual element.
5350 EltTBAAInfo.Size =
5351 getContext().getTypeSizeInChars(T: E->getType()).getQuantity();
5352 }
5353 } else {
5354 // Find the load of a 'counted_by' pointer field (e.g. 'p->buf' in
5355 // 'p->buf[i]') whose count bounds this access.
5356 const CastExpr *CountedByLoad = nullptr;
5357 const MemberExpr *CountedByME = nullptr;
5358 if (SanOpts.has(K: SanitizerKind::ArrayBounds)) {
5359 StructFieldAccess Visitor;
5360 if (const auto *CE =
5361 dyn_cast_if_present<CastExpr>(Val: Visitor.Visit(S: E->getBase()));
5362 CE && CE->getCastKind() == CK_LValueToRValue) {
5363 const auto *ME = dyn_cast<MemberExpr>(Val: CE->getSubExpr());
5364 if (ME && ME->getMemberDecl()->getType()->isCountAttributedType()) {
5365 CountedByLoad = CE;
5366 CountedByME = ME;
5367 }
5368 }
5369 }
5370
5371 // The bounds check locates the count relative to the pointer field's
5372 // address. Record the field's l-value while the base is emitted, so that
5373 // the struct base (which may have side-effects, e.g. 'get()->buf[i]' or
5374 // 'p++->buf[i]') isn't evaluated a second time for the check. The base
5375 // itself is emitted exactly as it is without the sanitizer, so the load
5376 // keeps its debug location, '__ptrauth' authentication, 'align_value'
5377 // assumption, and so on.
5378 LoadedLValueCapture FieldCapture{.E: CountedByME, .LV: std::nullopt};
5379 Address BaseAddr = Address::invalid();
5380 {
5381 llvm::SaveAndRestore SavedCapture(CurLoadedLValueCapture,
5382 CountedByME ? &FieldCapture : nullptr);
5383 // The base must be a pointer; emit it with an estimate of its alignment.
5384 BaseAddr =
5385 EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
5386 }
5387
5388 Address CountedByFieldAddr =
5389 FieldCapture.LV ? FieldCapture.LV->getAddress() : Address::invalid();
5390
5391 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5392 QualType ptrType = E->getBase()->getType();
5393 Addr = emitArraySubscriptGEP(CGF&: *this, addr: BaseAddr, indices: Idx, eltType: E->getType(),
5394 inbounds: !getLangOpts().PointerOverflowDefined,
5395 signedIndices: SignedIndices, loc: E->getExprLoc(), arrayType: &ptrType,
5396 Base: E->getBase());
5397
5398 // If the field wasn't loaded through ScalarExprEmitter (e.g. it was
5399 // constant folded), EmitCountedByBoundsChecking re-emits the field's
5400 // l-value. That only recomputes the field's address, so it's safe unless
5401 // the struct base has side-effects. In that case skip the check rather
5402 // than evaluating the base twice.
5403 if (CountedByME && (CountedByFieldAddr.isValid() ||
5404 !CountedByME->getBase()->HasSideEffects(Ctx: getContext())))
5405 EmitCountedByBoundsChecking(ArrayExpr: CountedByLoad, ArrayType: ptrType, ArrayInst: CountedByFieldAddr,
5406 IndexType: E->getIdx()->getType(), IndexVal: Idx, Accessed,
5407 /*FlexibleArray=*/false);
5408 }
5409
5410 LValue LV = MakeAddrLValue(Addr, T: E->getType(), BaseInfo: EltBaseInfo, TBAAInfo: EltTBAAInfo);
5411
5412 if (getLangOpts().ObjC &&
5413 getLangOpts().getGC() != LangOptions::NonGC) {
5414 LV.setNonGC(!E->isOBJCGCCandidate(Ctx&: getContext()));
5415 setObjCGCLValueClass(Ctx: getContext(), E, LV);
5416 }
5417 return LV;
5418}
5419
5420llvm::Value *CodeGenFunction::EmitMatrixIndexExpr(const Expr *E) {
5421 llvm::Value *Idx = EmitScalarExpr(E);
5422 if (Idx->getType() == IntPtrTy)
5423 return Idx;
5424 bool IsSigned = E->getType()->isSignedIntegerOrEnumerationType();
5425 return Builder.CreateIntCast(V: Idx, DestTy: IntPtrTy, isSigned: IsSigned);
5426}
5427
5428LValue CodeGenFunction::EmitMatrixSingleSubscriptExpr(
5429 const MatrixSingleSubscriptExpr *E) {
5430 LValue Base = EmitLValue(E: E->getBase());
5431 llvm::Value *RowIdx = EmitMatrixIndexExpr(E: E->getRowIdx());
5432
5433 RawAddress MatAddr = Base.getAddress();
5434 if (getLangOpts().HLSL &&
5435 E->getBase()->getType().getAddressSpace() == LangAS::hlsl_constant)
5436 MatAddr = CGM.getHLSLRuntime().createBufferMatrixTempAddress(LV: Base, CGF&: *this);
5437
5438 return LValue::MakeMatrixRow(Addr: MaybeConvertMatrixAddress(Addr: MatAddr, CGF&: *this),
5439 RowIdx, MatrixTy: E->getBase()->getType(),
5440 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5441}
5442
5443LValue CodeGenFunction::EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E) {
5444 assert(
5445 !E->isIncomplete() &&
5446 "incomplete matrix subscript expressions should be rejected during Sema");
5447 LValue Base = EmitLValue(E: E->getBase());
5448
5449 // Extend or truncate the index type to 32 or 64-bits if needed.
5450 llvm::Value *RowIdx = EmitMatrixIndexExpr(E: E->getRowIdx());
5451 llvm::Value *ColIdx = EmitMatrixIndexExpr(E: E->getColumnIdx());
5452 llvm::MatrixBuilder MB(Builder);
5453 const auto *MatrixTy = E->getBase()->getType()->castAs<ConstantMatrixType>();
5454 unsigned NumCols = MatrixTy->getNumColumns();
5455 unsigned NumRows = MatrixTy->getNumRows();
5456 bool IsMatrixRowMajor =
5457 isMatrixRowMajor(LangOpts: getLangOpts(), T: E->getBase()->getType());
5458 llvm::Value *FinalIdx =
5459 MB.CreateIndex(RowIdx, ColumnIdx: ColIdx, NumRows, NumCols, IsMatrixRowMajor);
5460
5461 return LValue::MakeMatrixElt(
5462 matAddress: MaybeConvertMatrixAddress(Addr: Base.getAddress(), CGF&: *this), Idx: FinalIdx,
5463 type: E->getBase()->getType(), BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5464}
5465
5466static Address emitOMPArraySectionBase(CodeGenFunction &CGF, const Expr *Base,
5467 LValueBaseInfo &BaseInfo,
5468 TBAAAccessInfo &TBAAInfo,
5469 QualType BaseTy, QualType ElTy,
5470 bool IsLowerBound) {
5471 LValue BaseLVal;
5472 if (auto *ASE = dyn_cast<ArraySectionExpr>(Val: Base->IgnoreParenImpCasts())) {
5473 BaseLVal = CGF.EmitArraySectionExpr(E: ASE, IsLowerBound);
5474 if (BaseTy->isArrayType()) {
5475 Address Addr = BaseLVal.getAddress();
5476 BaseInfo = BaseLVal.getBaseInfo();
5477
5478 // If the array type was an incomplete type, we need to make sure
5479 // the decay ends up being the right type.
5480 llvm::Type *NewTy = CGF.ConvertType(T: BaseTy);
5481 Addr = Addr.withElementType(ElemTy: NewTy);
5482
5483 // Note that VLA pointers are always decayed, so we don't need to do
5484 // anything here.
5485 if (!BaseTy->isVariableArrayType()) {
5486 assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
5487 "Expected pointer to array");
5488 Addr = CGF.Builder.CreateConstArrayGEP(Addr, Index: 0, Name: "arraydecay");
5489 }
5490
5491 return Addr.withElementType(ElemTy: CGF.ConvertTypeForMem(T: ElTy));
5492 }
5493 LValueBaseInfo TypeBaseInfo;
5494 TBAAAccessInfo TypeTBAAInfo;
5495 CharUnits Align =
5496 CGF.CGM.getNaturalTypeAlignment(T: ElTy, BaseInfo: &TypeBaseInfo, TBAAInfo: &TypeTBAAInfo);
5497 BaseInfo.mergeForCast(Info: TypeBaseInfo);
5498 TBAAInfo = CGF.CGM.mergeTBAAInfoForCast(SourceInfo: TBAAInfo, TargetInfo: TypeTBAAInfo);
5499 return Address(CGF.Builder.CreateLoad(Addr: BaseLVal.getAddress()),
5500 CGF.ConvertTypeForMem(T: ElTy), Align);
5501 }
5502 return CGF.EmitPointerWithAlignment(E: Base, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
5503}
5504
5505LValue CodeGenFunction::EmitArraySectionExpr(const ArraySectionExpr *E,
5506 bool IsLowerBound) {
5507
5508 assert(!E->isOpenACCArraySection() &&
5509 "OpenACC Array section codegen not implemented");
5510
5511 QualType BaseTy = ArraySectionExpr::getBaseOriginalType(Base: E->getBase());
5512 QualType ResultExprTy;
5513 if (auto *AT = getContext().getAsArrayType(T: BaseTy))
5514 ResultExprTy = AT->getElementType();
5515 else
5516 ResultExprTy = BaseTy->getPointeeType();
5517 llvm::Value *Idx = nullptr;
5518 if (IsLowerBound || E->getColonLocFirst().isInvalid()) {
5519 // Requesting lower bound or upper bound, but without provided length and
5520 // without ':' symbol for the default length -> length = 1.
5521 // Idx = LowerBound ?: 0;
5522 if (auto *LowerBound = E->getLowerBound()) {
5523 Idx = Builder.CreateIntCast(
5524 V: EmitScalarExpr(E: LowerBound), DestTy: IntPtrTy,
5525 isSigned: LowerBound->getType()->hasSignedIntegerRepresentation());
5526 } else
5527 Idx = llvm::ConstantInt::getNullValue(Ty: IntPtrTy);
5528 } else {
5529 // Try to emit length or lower bound as constant. If this is possible, 1
5530 // is subtracted from constant length or lower bound. Otherwise, emit LLVM
5531 // IR (LB + Len) - 1.
5532 auto &C = CGM.getContext();
5533 auto *Length = E->getLength();
5534 llvm::APSInt ConstLength;
5535 if (Length) {
5536 // Idx = LowerBound + Length - 1;
5537 if (std::optional<llvm::APSInt> CL = Length->getIntegerConstantExpr(Ctx: C)) {
5538 ConstLength = CL->zextOrTrunc(width: PointerWidthInBits);
5539 Length = nullptr;
5540 }
5541 auto *LowerBound = E->getLowerBound();
5542 llvm::APSInt ConstLowerBound(PointerWidthInBits, /*isUnsigned=*/false);
5543 if (LowerBound) {
5544 if (std::optional<llvm::APSInt> LB =
5545 LowerBound->getIntegerConstantExpr(Ctx: C)) {
5546 ConstLowerBound = LB->zextOrTrunc(width: PointerWidthInBits);
5547 LowerBound = nullptr;
5548 }
5549 }
5550 if (!Length)
5551 --ConstLength;
5552 else if (!LowerBound)
5553 --ConstLowerBound;
5554
5555 if (Length || LowerBound) {
5556 auto *LowerBoundVal =
5557 LowerBound
5558 ? Builder.CreateIntCast(
5559 V: EmitScalarExpr(E: LowerBound), DestTy: IntPtrTy,
5560 isSigned: LowerBound->getType()->hasSignedIntegerRepresentation())
5561 : llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLowerBound);
5562 auto *LengthVal =
5563 Length
5564 ? Builder.CreateIntCast(
5565 V: EmitScalarExpr(E: Length), DestTy: IntPtrTy,
5566 isSigned: Length->getType()->hasSignedIntegerRepresentation())
5567 : llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLength);
5568 Idx = Builder.CreateAdd(LHS: LowerBoundVal, RHS: LengthVal, Name: "lb_add_len",
5569 /*HasNUW=*/false,
5570 HasNSW: !getLangOpts().PointerOverflowDefined);
5571 if (Length && LowerBound) {
5572 Idx = Builder.CreateSub(
5573 LHS: Idx, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, /*V=*/1), Name: "idx_sub_1",
5574 /*HasNUW=*/false, HasNSW: !getLangOpts().PointerOverflowDefined);
5575 }
5576 } else
5577 Idx = llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLength + ConstLowerBound);
5578 } else {
5579 // Idx = ArraySize - 1;
5580 QualType ArrayTy = BaseTy->isPointerType()
5581 ? E->getBase()->IgnoreParenImpCasts()->getType()
5582 : BaseTy;
5583 if (auto *VAT = C.getAsVariableArrayType(T: ArrayTy)) {
5584 Length = VAT->getSizeExpr();
5585 if (std::optional<llvm::APSInt> L = Length->getIntegerConstantExpr(Ctx: C)) {
5586 ConstLength = *L;
5587 Length = nullptr;
5588 }
5589 } else {
5590 auto *CAT = C.getAsConstantArrayType(T: ArrayTy);
5591 assert(CAT && "unexpected type for array initializer");
5592 ConstLength = CAT->getSize();
5593 }
5594 if (Length) {
5595 auto *LengthVal = Builder.CreateIntCast(
5596 V: EmitScalarExpr(E: Length), DestTy: IntPtrTy,
5597 isSigned: Length->getType()->hasSignedIntegerRepresentation());
5598 Idx = Builder.CreateSub(
5599 LHS: LengthVal, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, /*V=*/1), Name: "len_sub_1",
5600 /*HasNUW=*/false, HasNSW: !getLangOpts().PointerOverflowDefined);
5601 } else {
5602 ConstLength = ConstLength.zextOrTrunc(width: PointerWidthInBits);
5603 --ConstLength;
5604 Idx = llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLength);
5605 }
5606 }
5607 }
5608 assert(Idx);
5609
5610 Address EltPtr = Address::invalid();
5611 LValueBaseInfo BaseInfo;
5612 TBAAAccessInfo TBAAInfo;
5613 if (auto *VLA = getContext().getAsVariableArrayType(T: ResultExprTy)) {
5614 // The base must be a pointer, which is not an aggregate. Emit
5615 // it. It needs to be emitted first in case it's what captures
5616 // the VLA bounds.
5617 Address Base =
5618 emitOMPArraySectionBase(CGF&: *this, Base: E->getBase(), BaseInfo, TBAAInfo,
5619 BaseTy, ElTy: VLA->getElementType(), IsLowerBound);
5620 // The element count here is the total number of non-VLA elements.
5621 llvm::Value *NumElements = getVLASize(vla: VLA).NumElts;
5622
5623 // Effectively, the multiply by the VLA size is part of the GEP.
5624 // GEP indexes are signed, and scaling an index isn't permitted to
5625 // signed-overflow, so we use the same semantics for our explicit
5626 // multiply. We suppress this if overflow is not undefined behavior.
5627 if (getLangOpts().PointerOverflowDefined)
5628 Idx = Builder.CreateMul(LHS: Idx, RHS: NumElements);
5629 else
5630 Idx = Builder.CreateNSWMul(LHS: Idx, RHS: NumElements);
5631 EltPtr = emitArraySubscriptGEP(CGF&: *this, addr: Base, indices: Idx, eltType: VLA->getElementType(),
5632 inbounds: !getLangOpts().PointerOverflowDefined,
5633 /*signedIndices=*/false, loc: E->getExprLoc());
5634 } else if (const Expr *Array = isSimpleArrayDecayOperand(E: E->getBase())) {
5635 // If this is A[i] where A is an array, the frontend will have decayed the
5636 // base to be a ArrayToPointerDecay implicit cast. While correct, it is
5637 // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
5638 // "gep x, i" here. Emit one "gep A, 0, i".
5639 assert(Array->getType()->isArrayType() &&
5640 "Array to pointer decay must have array source type!");
5641 LValue ArrayLV;
5642 // For simple multidimensional array indexing, set the 'accessed' flag for
5643 // better bounds-checking of the base expression.
5644 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Array))
5645 ArrayLV = EmitArraySubscriptExpr(E: ASE, /*Accessed*/ true);
5646 else
5647 ArrayLV = EmitLValue(E: Array);
5648
5649 // Propagate the alignment from the array itself to the result.
5650 EltPtr = emitArraySubscriptGEP(
5651 CGF&: *this, addr: ArrayLV.getAddress(), indices: {CGM.getSize(numChars: CharUnits::Zero()), Idx},
5652 eltType: ResultExprTy, inbounds: !getLangOpts().PointerOverflowDefined,
5653 /*signedIndices=*/false, loc: E->getExprLoc());
5654 BaseInfo = ArrayLV.getBaseInfo();
5655 TBAAInfo = CGM.getTBAAInfoForSubobject(Base: ArrayLV, AccessType: ResultExprTy);
5656 } else {
5657 Address Base =
5658 emitOMPArraySectionBase(CGF&: *this, Base: E->getBase(), BaseInfo, TBAAInfo, BaseTy,
5659 ElTy: ResultExprTy, IsLowerBound);
5660 EltPtr = emitArraySubscriptGEP(CGF&: *this, addr: Base, indices: Idx, eltType: ResultExprTy,
5661 inbounds: !getLangOpts().PointerOverflowDefined,
5662 /*signedIndices=*/false, loc: E->getExprLoc());
5663 }
5664
5665 return MakeAddrLValue(Addr: EltPtr, T: ResultExprTy, BaseInfo, TBAAInfo);
5666}
5667
5668LValue CodeGenFunction::
5669EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
5670 // Emit the base vector as an l-value.
5671 LValue Base;
5672
5673 // ExtVectorElementExpr's base can either be a vector or pointer to vector.
5674 if (E->isArrow()) {
5675 // If it is a pointer to a vector, emit the address and form an lvalue with
5676 // it.
5677 LValueBaseInfo BaseInfo;
5678 TBAAAccessInfo TBAAInfo;
5679 Address Ptr = EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
5680 const auto *PT = E->getBase()->getType()->castAs<PointerType>();
5681 Base = MakeAddrLValue(Addr: Ptr, T: PT->getPointeeType(), BaseInfo, TBAAInfo);
5682 Base.getQuals().removeObjCGCAttr();
5683 } else if (E->getBase()->isGLValue()) {
5684 // Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
5685 // emit the base as an lvalue.
5686 assert(E->getBase()->getType()->isVectorType());
5687 Base = EmitLValue(E: E->getBase());
5688 } else {
5689 // Otherwise, the base is a normal rvalue (as in (V+V).x), emit it as such.
5690 assert(E->getBase()->getType()->isVectorType() &&
5691 "Result must be a vector");
5692 llvm::Value *Vec = EmitScalarExpr(E: E->getBase());
5693
5694 // Store the vector to memory (because LValue wants an address).
5695 Address VecMem = CreateMemTemp(Ty: E->getBase()->getType());
5696 // need to zero extend an hlsl boolean vector to store it back to memory
5697 QualType Ty = E->getBase()->getType();
5698 llvm::Type *LTy = convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Vec->getType());
5699 if (LTy->getScalarSizeInBits() > Vec->getType()->getScalarSizeInBits())
5700 Vec = Builder.CreateZExt(V: Vec, DestTy: LTy);
5701 Builder.CreateStore(Val: Vec, Addr: VecMem);
5702 Base = MakeAddrLValue(Addr: VecMem, T: Ty, Source: AlignmentSource::Decl);
5703 }
5704
5705 QualType type =
5706 E->getType().withCVRQualifiers(CVR: Base.getQuals().getCVRQualifiers());
5707
5708 // Encode the element access list into a vector of unsigned indices.
5709 SmallVector<uint32_t, 4> Indices;
5710 E->getEncodedElementAccess(Elts&: Indices);
5711
5712 if (Base.isSimple()) {
5713 llvm::Constant *CV =
5714 llvm::ConstantDataVector::get(Context&: getLLVMContext(), Elts: Indices);
5715 return LValue::MakeExtVectorElt(Addr: Base.getAddress(), Elts: CV, type,
5716 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5717 }
5718
5719 if (Base.isMatrixRow()) {
5720 if (auto *RowIdx =
5721 llvm::dyn_cast<llvm::ConstantInt>(Val: Base.getMatrixRowIdx())) {
5722 llvm::SmallVector<llvm::Constant *> MatIndices;
5723 QualType MatTy = Base.getType();
5724 const ConstantMatrixType *MT = MatTy->castAs<ConstantMatrixType>();
5725 unsigned NumCols = Indices.size();
5726 unsigned NumRows = MT->getNumRows();
5727 unsigned Row = RowIdx->getZExtValue();
5728 QualType VecQT = E->getBase()->getType();
5729 if (NumCols != MT->getNumColumns()) {
5730 const auto *EVT = VecQT->getAs<ExtVectorType>();
5731 QualType ElemQT = EVT->getElementType();
5732 VecQT = getContext().getExtVectorType(VectorType: ElemQT, NumElts: NumCols);
5733 }
5734 for (unsigned C = 0; C < NumCols; ++C) {
5735 unsigned Col = Indices[C];
5736 unsigned Linear = Col * NumRows + Row;
5737 MatIndices.push_back(Elt: llvm::ConstantInt::get(Ty: Int32Ty, V: Linear));
5738 }
5739
5740 llvm::Constant *ConstIdxs = llvm::ConstantVector::get(V: MatIndices);
5741 return LValue::MakeExtVectorElt(Addr: Base.getMatrixAddress(), Elts: ConstIdxs, type: VecQT,
5742 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5743 }
5744 llvm::Constant *Cols =
5745 llvm::ConstantDataVector::get(Context&: getLLVMContext(), Elts: Indices);
5746 // Note: intentionally not using E.getType() so we can reuse isMatrixRow()
5747 // implementations in EmitLoadOfLValue & EmitStoreThroughLValue and don't
5748 // need the LValue to have its own number of rows and columns when the
5749 // type is a vector.
5750 return LValue::MakeMatrixRowSwizzle(
5751 MatAddr: Base.getMatrixAddress(), RowIdx: Base.getMatrixRowIdx(), Cols, MatrixTy: Base.getType(),
5752 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5753 }
5754
5755 assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
5756
5757 llvm::Constant *BaseElts = Base.getExtVectorElts();
5758 SmallVector<llvm::Constant *, 4> CElts;
5759
5760 for (unsigned Index : Indices)
5761 CElts.push_back(Elt: BaseElts->getAggregateElement(Elt: Index));
5762 llvm::Constant *CV = llvm::ConstantVector::get(V: CElts);
5763 return LValue::MakeExtVectorElt(Addr: Base.getExtVectorAddress(), Elts: CV, type,
5764 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5765}
5766
5767bool CodeGenFunction::isUnderlyingBasePointerConstantNull(const Expr *E) {
5768 const Expr *UnderlyingBaseExpr = E->IgnoreParens();
5769 while (auto *BaseMemberExpr = dyn_cast<MemberExpr>(Val: UnderlyingBaseExpr))
5770 UnderlyingBaseExpr = BaseMemberExpr->getBase()->IgnoreParens();
5771 return getContext().isSentinelNullExpr(E: UnderlyingBaseExpr);
5772}
5773
5774LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
5775 if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(CGF&: *this, ME: E)) {
5776 EmitIgnoredExpr(E: E->getBase());
5777 return EmitDeclRefLValue(E: DRE);
5778 }
5779
5780 if (getLangOpts().HLSL) {
5781 QualType QT = E->getType();
5782 if (QT.getAddressSpace() == LangAS::hlsl_constant)
5783 return CGM.getHLSLRuntime().emitBufferMemberExpr(CGF&: *this, E);
5784
5785 if (QT->isHLSLResourceRecord() || QT->isHLSLResourceRecordArray()) {
5786 std::optional<LValue> LV;
5787 LV = CGM.getHLSLRuntime().emitResourceMemberExpr(CGF&: *this, E);
5788 if (LV.has_value())
5789 return *LV;
5790 }
5791 }
5792
5793 Expr *BaseExpr = E->getBase();
5794 // Check whether the underlying base pointer is a constant null.
5795 // If so, we do not set inbounds flag for GEP to avoid breaking some
5796 // old-style offsetof idioms.
5797 bool IsInBounds = !getLangOpts().PointerOverflowDefined &&
5798 !isUnderlyingBasePointerConstantNull(E: BaseExpr);
5799 // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar.
5800 LValue BaseLV;
5801 if (E->isArrow()) {
5802 LValueBaseInfo BaseInfo;
5803 TBAAAccessInfo TBAAInfo;
5804 Address Addr = EmitPointerWithAlignment(E: BaseExpr, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
5805 QualType PtrTy = BaseExpr->getType()->getPointeeType();
5806 SanitizerSet SkippedChecks;
5807 bool IsBaseCXXThis = IsWrappedCXXThis(Obj: BaseExpr);
5808 if (IsBaseCXXThis)
5809 SkippedChecks.set(K: SanitizerKind::Alignment, Value: true);
5810 if (IsBaseCXXThis || isa<DeclRefExpr>(Val: BaseExpr))
5811 SkippedChecks.set(K: SanitizerKind::Null, Value: true);
5812 EmitTypeCheck(TCK: TCK_MemberAccess, Loc: E->getExprLoc(), Addr, Type: PtrTy,
5813 /*Alignment=*/CharUnits::Zero(), SkippedChecks);
5814 BaseLV = MakeAddrLValue(Addr, T: PtrTy, BaseInfo, TBAAInfo);
5815 } else
5816 BaseLV = EmitCheckedLValue(E: BaseExpr, TCK: TCK_MemberAccess);
5817
5818 NamedDecl *ND = E->getMemberDecl();
5819 if (auto *Field = dyn_cast<FieldDecl>(Val: ND)) {
5820 LValue LV = EmitLValueForField(Base: BaseLV, Field, IsInBounds);
5821 setObjCGCLValueClass(Ctx: getContext(), E, LV);
5822 if (getLangOpts().OpenMP) {
5823 // If the member was explicitly marked as nontemporal, mark it as
5824 // nontemporal. If the base lvalue is marked as nontemporal, mark access
5825 // to children as nontemporal too.
5826 if ((IsWrappedCXXThis(Obj: BaseExpr) &&
5827 CGM.getOpenMPRuntime().isNontemporalDecl(VD: Field)) ||
5828 BaseLV.isNontemporal())
5829 LV.setNontemporal(/*Value=*/true);
5830 }
5831 return LV;
5832 }
5833
5834 if (const auto *FD = dyn_cast<FunctionDecl>(Val: ND))
5835 return EmitFunctionDeclLValue(CGF&: *this, E, GD: FD);
5836
5837 llvm_unreachable("Unhandled member declaration!");
5838}
5839
5840/// Given that we are currently emitting a lambda, emit an l-value for
5841/// one of its members.
5842///
5843LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field,
5844 llvm::Value *ThisValue) {
5845 bool HasExplicitObjectParameter = false;
5846 const auto *MD = dyn_cast_if_present<CXXMethodDecl>(Val: CurCodeDecl);
5847 if (MD) {
5848 HasExplicitObjectParameter = MD->isExplicitObjectMemberFunction();
5849 assert(MD->getParent()->isLambda());
5850 assert(MD->getParent() == Field->getParent());
5851 }
5852 LValue LambdaLV;
5853 if (HasExplicitObjectParameter) {
5854 const VarDecl *D = cast<CXXMethodDecl>(Val: CurCodeDecl)->getParamDecl(i: 0);
5855 auto It = LocalDeclMap.find(Val: D);
5856 assert(It != LocalDeclMap.end() && "explicit parameter not loaded?");
5857 Address AddrOfExplicitObject = It->getSecond();
5858 if (D->getType()->isReferenceType())
5859 LambdaLV = EmitLoadOfReferenceLValue(RefAddr: AddrOfExplicitObject, RefTy: D->getType(),
5860 Source: AlignmentSource::Decl);
5861 else
5862 LambdaLV = MakeAddrLValue(Addr: AddrOfExplicitObject,
5863 T: D->getType().getNonReferenceType());
5864
5865 // Make sure we have an lvalue to the lambda itself and not a derived class.
5866 auto *ThisTy = D->getType().getNonReferenceType()->getAsCXXRecordDecl();
5867 auto *LambdaTy = cast<CXXRecordDecl>(Val: Field->getParent());
5868 if (ThisTy != LambdaTy) {
5869 const CXXCastPath &BasePathArray = getContext().LambdaCastPaths.at(Val: MD);
5870 Address Base = GetAddressOfBaseClass(
5871 Value: LambdaLV.getAddress(), Derived: ThisTy, PathBegin: BasePathArray.begin(),
5872 PathEnd: BasePathArray.end(), /*NullCheckValue=*/false, Loc: SourceLocation());
5873 CanQualType T = getContext().getCanonicalTagType(TD: LambdaTy);
5874 LambdaLV = MakeAddrLValue(Addr: Base, T);
5875 }
5876 } else {
5877 CanQualType LambdaTagType =
5878 getContext().getCanonicalTagType(TD: Field->getParent());
5879 LambdaLV = MakeNaturalAlignAddrLValue(V: ThisValue, T: LambdaTagType);
5880 }
5881 return EmitLValueForField(Base: LambdaLV, Field);
5882}
5883
5884LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field) {
5885 return EmitLValueForLambdaField(Field, ThisValue: CXXABIThisValue);
5886}
5887
5888/// Get the address of a zero-sized field within a record. The resulting
5889/// address doesn't necessarily have the right type.
5890static Address emitAddrOfZeroSizeField(CodeGenFunction &CGF, Address Base,
5891 const FieldDecl *Field,
5892 bool IsInBounds) {
5893 CharUnits Offset = CGF.getContext().toCharUnitsFromBits(
5894 BitSize: CGF.getContext().getFieldOffset(FD: Field));
5895 if (Offset.isZero())
5896 return Base;
5897 Base = Base.withElementType(ElemTy: CGF.Int8Ty);
5898 if (!IsInBounds)
5899 return CGF.Builder.CreateConstByteGEP(Addr: Base, Offset);
5900 return CGF.Builder.CreateConstInBoundsByteGEP(Addr: Base, Offset);
5901}
5902
5903/// Drill down to the storage of a field without walking into reference types,
5904/// and without respect for pointer field protection.
5905///
5906/// The resulting address doesn't necessarily have the right type.
5907static Address emitRawAddrOfFieldStorage(CodeGenFunction &CGF, Address base,
5908 const FieldDecl *field,
5909 bool IsInBounds) {
5910 if (CodeGenUtils::isEmptyFieldForLayout(Ctx: CGF.getContext(), FD: field))
5911 return emitAddrOfZeroSizeField(CGF, Base: base, Field: field, IsInBounds);
5912
5913 const RecordDecl *rec = field->getParent();
5914
5915 unsigned idx =
5916 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(FD: field);
5917 llvm::Type *StructType =
5918 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMType();
5919
5920 if (CGF.getLangOpts().EmitLogicalPointer)
5921 return RawAddress(
5922 CGF.Builder.CreateStructuredGEP(BaseType: StructType, PtrBase: base.emitRawPointer(CGF),
5923 Indices: {CGF.Builder.getSize(N: idx)}),
5924 base.getElementType(), base.getAlignment());
5925
5926 if (!IsInBounds)
5927 return CGF.Builder.CreateConstGEP2_32(Addr: base, Idx0: 0, Idx1: idx, Name: field->getName());
5928
5929 return CGF.Builder.CreateStructGEP(Addr: base, Index: idx, Name: field->getName());
5930}
5931
5932/// Drill down to the storage of a field without walking into reference types,
5933/// wrapping the address in an llvm.protected.field.ptr intrinsic for the
5934/// pointer field protection feature if necessary.
5935///
5936/// The resulting address doesn't necessarily have the right type.
5937static Address emitAddrOfFieldStorage(CodeGenFunction &CGF, Address base,
5938 const FieldDecl *field, bool IsInBounds) {
5939 Address Addr = emitRawAddrOfFieldStorage(CGF, base, field, IsInBounds);
5940
5941 if (!CGF.getContext().isPFPField(Field: field))
5942 return Addr;
5943
5944 return CGF.EmitAddressOfPFPField(RecordPtr: base, FieldPtr: Addr, Field: field);
5945}
5946
5947static Address emitPreserveStructAccess(CodeGenFunction &CGF, LValue base,
5948 Address addr, const FieldDecl *field) {
5949 const RecordDecl *rec = field->getParent();
5950 auto [DbgInfo, DIIndex] =
5951 CGF.getDebugInfo()->getOrCreatePreserveAccessInfo(Ty: base.getType(), Field: field);
5952
5953 unsigned idx =
5954 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(FD: field);
5955
5956 return CGF.Builder.CreatePreserveStructAccessIndex(Addr: addr, Index: idx, FieldIndex: DIIndex,
5957 DbgInfo);
5958}
5959
5960static bool hasAnyVptr(const QualType Type, const ASTContext &Context) {
5961 const auto *RD = Type.getTypePtr()->getAsCXXRecordDecl();
5962 if (!RD)
5963 return false;
5964
5965 if (RD->isDynamicClass())
5966 return true;
5967
5968 for (const auto &Base : RD->bases())
5969 if (hasAnyVptr(Type: Base.getType(), Context))
5970 return true;
5971
5972 for (const FieldDecl *Field : RD->fields())
5973 if (hasAnyVptr(Type: Field->getType(), Context))
5974 return true;
5975
5976 return false;
5977}
5978
5979LValue CodeGenFunction::EmitLValueForField(LValue base, const FieldDecl *field,
5980 bool IsInBounds) {
5981 LValueBaseInfo BaseInfo = base.getBaseInfo();
5982
5983 if (field->isBitField()) {
5984 const CGRecordLayout &RL =
5985 CGM.getTypes().getCGRecordLayout(field->getParent());
5986 const CGBitFieldInfo &Info = RL.getBitFieldInfo(FD: field);
5987 const bool UseVolatile = CodeGenUtils::isAAPCS(TargetInfo: CGM.getTarget()) &&
5988 CGM.getCodeGenOpts().AAPCSBitfieldWidth &&
5989 Info.VolatileStorageSize != 0 &&
5990 field->getType()
5991 .withCVRQualifiers(CVR: base.getVRQualifiers())
5992 .isVolatileQualified();
5993 Address Addr = base.getAddress();
5994 unsigned Idx = RL.getLLVMFieldNo(FD: field);
5995 const RecordDecl *rec = field->getParent();
5996 if (hasBPFPreserveStaticOffset(D: rec))
5997 Addr = wrapWithBPFPreserveStaticOffset(CGF&: *this, Addr);
5998 if (!UseVolatile) {
5999 if (!IsInPreservedAIRegion &&
6000 (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
6001 if (Idx != 0) {
6002 // For structs, we GEP to the field that the record layout suggests.
6003 if (!IsInBounds)
6004 Addr = Builder.CreateConstGEP2_32(Addr, Idx0: 0, Idx1: Idx, Name: field->getName());
6005 else
6006 Addr = Builder.CreateStructGEP(Addr, Index: Idx, Name: field->getName());
6007 }
6008 } else {
6009 auto [DbgInfo, DIIndex] = getDebugInfo()->getOrCreatePreserveAccessInfo(
6010 Ty: getContext().getCanonicalTagType(TD: rec), Field: field);
6011 Addr = Builder.CreatePreserveStructAccessIndex(Addr, Index: Idx, FieldIndex: DIIndex,
6012 DbgInfo);
6013 }
6014 }
6015 const unsigned SS =
6016 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
6017 // Get the access type.
6018 llvm::Type *FieldIntTy = llvm::Type::getIntNTy(C&: getLLVMContext(), N: SS);
6019 Addr = Addr.withElementType(ElemTy: FieldIntTy);
6020 if (UseVolatile) {
6021 const unsigned VolatileOffset = Info.VolatileStorageOffset.getQuantity();
6022 if (VolatileOffset)
6023 Addr = Builder.CreateConstInBoundsGEP(Addr, Index: VolatileOffset);
6024 }
6025
6026 QualType fieldType =
6027 field->getType().withCVRQualifiers(CVR: base.getVRQualifiers());
6028 // TODO: Support TBAA for bit fields.
6029 LValueBaseInfo FieldBaseInfo(BaseInfo.getAlignmentSource());
6030 return LValue::MakeBitfield(Addr, Info, type: fieldType, BaseInfo: FieldBaseInfo,
6031 TBAAInfo: TBAAAccessInfo());
6032 }
6033
6034 // Fields of may-alias structures are may-alias themselves.
6035 // FIXME: this should get propagated down through anonymous structs
6036 // and unions.
6037 QualType FieldType = field->getType();
6038 const RecordDecl *rec = field->getParent();
6039 AlignmentSource BaseAlignSource = BaseInfo.getAlignmentSource();
6040 LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(Source: BaseAlignSource));
6041 TBAAAccessInfo FieldTBAAInfo;
6042 if (base.getTBAAInfo().isMayAlias() ||
6043 rec->hasAttr<MayAliasAttr>() || FieldType->isVectorType()) {
6044 FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
6045 } else if (rec->isUnion()) {
6046 // TODO: Support TBAA for unions.
6047 FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
6048 } else {
6049 // If no base type been assigned for the base access, then try to generate
6050 // one for this base lvalue.
6051 FieldTBAAInfo = base.getTBAAInfo();
6052 if (!FieldTBAAInfo.BaseType) {
6053 FieldTBAAInfo.BaseType = CGM.getTBAABaseTypeInfo(QTy: base.getType());
6054 assert(!FieldTBAAInfo.Offset &&
6055 "Nonzero offset for an access with no base type!");
6056 }
6057
6058 // Adjust offset to be relative to the base type.
6059 const ASTRecordLayout &Layout =
6060 getContext().getASTRecordLayout(D: field->getParent());
6061 unsigned CharWidth = getContext().getCharWidth();
6062 if (FieldTBAAInfo.BaseType)
6063 FieldTBAAInfo.Offset +=
6064 Layout.getFieldOffset(FieldNo: field->getFieldIndex()) / CharWidth;
6065
6066 // Update the final access type and size.
6067 FieldTBAAInfo.AccessType = CGM.getTBAATypeInfo(QTy: FieldType);
6068 FieldTBAAInfo.Size =
6069 getContext().getTypeSizeInChars(T: FieldType).getQuantity();
6070 }
6071
6072 Address addr = base.getAddress();
6073 if (hasBPFPreserveStaticOffset(D: rec))
6074 addr = wrapWithBPFPreserveStaticOffset(CGF&: *this, Addr&: addr);
6075
6076 unsigned RecordCVR = base.getVRQualifiers();
6077 if (rec->isUnion()) {
6078 // For unions, there is no pointer adjustment.
6079 if (CGM.getCodeGenOpts().StrictVTablePointers &&
6080 hasAnyVptr(Type: FieldType, Context: getContext()))
6081 // Because unions can easily skip invariant.barriers, we need to add
6082 // a barrier every time CXXRecord field with vptr is referenced.
6083 addr = Builder.CreateLaunderInvariantGroup(Addr: addr);
6084
6085 if (IsInPreservedAIRegion ||
6086 (getDebugInfo() && rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
6087 // Remember the original union field index
6088 auto [DbgInfo, DIIndex] =
6089 getDebugInfo()->getOrCreatePreserveAccessInfo(Ty: base.getType(), Field: field);
6090 addr = Address(Builder.CreatePreserveUnionAccessIndex(
6091 Base: addr.emitRawPointer(CGF&: *this), FieldIndex: DIIndex, DbgInfo),
6092 addr.getElementType(), addr.getAlignment());
6093 }
6094
6095 if (FieldType->isReferenceType())
6096 addr = addr.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: FieldType));
6097 } else {
6098 if (!IsInPreservedAIRegion &&
6099 (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>()))
6100 // For structs, we GEP to the field that the record layout suggests.
6101 addr = emitAddrOfFieldStorage(CGF&: *this, base: addr, field, IsInBounds);
6102 else
6103 // Remember the original struct field index
6104 addr = emitPreserveStructAccess(CGF&: *this, base, addr, field);
6105 }
6106
6107 // If this is a reference field, load the reference right now.
6108 if (FieldType->isReferenceType()) {
6109 LValue RefLVal =
6110 MakeAddrLValue(Addr: addr, T: FieldType, BaseInfo: FieldBaseInfo, TBAAInfo: FieldTBAAInfo);
6111 if (RecordCVR & Qualifiers::Volatile)
6112 RefLVal.getQuals().addVolatile();
6113 addr = EmitLoadOfReference(RefLVal, PointeeBaseInfo: &FieldBaseInfo, PointeeTBAAInfo: &FieldTBAAInfo);
6114
6115 // Qualifiers on the struct don't apply to the referencee.
6116 RecordCVR = 0;
6117 FieldType = FieldType->getPointeeType();
6118 }
6119
6120 // Make sure that the address is pointing to the right type. This is critical
6121 // for both unions and structs.
6122 addr = addr.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: FieldType));
6123
6124 if (field->hasAttr<AnnotateAttr>())
6125 addr = EmitFieldAnnotations(D: field, V: addr);
6126
6127 LValue LV = MakeAddrLValue(Addr: addr, T: FieldType, BaseInfo: FieldBaseInfo, TBAAInfo: FieldTBAAInfo);
6128 LV.getQuals().addCVRQualifiers(mask: RecordCVR);
6129
6130 // __weak attribute on a field is ignored.
6131 if (LV.getQuals().getObjCGCAttr() == Qualifiers::Weak)
6132 LV.getQuals().removeObjCGCAttr();
6133
6134 return LV;
6135}
6136
6137LValue
6138CodeGenFunction::EmitLValueForFieldInitialization(LValue Base,
6139 const FieldDecl *Field) {
6140 QualType FieldType = Field->getType();
6141
6142 if (!FieldType->isReferenceType())
6143 return EmitLValueForField(base: Base, field: Field);
6144
6145 Address V = emitAddrOfFieldStorage(
6146 CGF&: *this, base: Base.getAddress(), field: Field,
6147 /*IsInBounds=*/!getLangOpts().PointerOverflowDefined);
6148
6149 // Make sure that the address is pointing to the right type.
6150 llvm::Type *llvmType = ConvertTypeForMem(T: FieldType);
6151 V = V.withElementType(ElemTy: llvmType);
6152
6153 // TODO: Generate TBAA information that describes this access as a structure
6154 // member access and not just an access to an object of the field's type. This
6155 // should be similar to what we do in EmitLValueForField().
6156 LValueBaseInfo BaseInfo = Base.getBaseInfo();
6157 AlignmentSource FieldAlignSource = BaseInfo.getAlignmentSource();
6158 LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(Source: FieldAlignSource));
6159 return MakeAddrLValue(Addr: V, T: FieldType, BaseInfo: FieldBaseInfo,
6160 TBAAInfo: CGM.getTBAAInfoForSubobject(Base, AccessType: FieldType));
6161}
6162
6163LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr *E){
6164 if (E->isFileScope()) {
6165 ConstantAddress GlobalPtr = CGM.GetAddrOfConstantCompoundLiteral(E);
6166 return MakeAddrLValue(Addr: GlobalPtr, T: E->getType(), Source: AlignmentSource::Decl);
6167 }
6168 if (E->getType()->isVariablyModifiedType())
6169 // make sure to emit the VLA size.
6170 EmitVariablyModifiedType(Ty: E->getType());
6171
6172 Address DeclPtr = CreateMemTempWithoutCast(Ty: E->getType(), Name: ".compoundliteral");
6173 const Expr *InitExpr = E->getInitializer();
6174 LValue Result = MakeAddrLValue(Addr: DeclPtr, T: E->getType(), Source: AlignmentSource::Decl);
6175
6176 if (!getLangOpts().CPlusPlus) {
6177 if (HaveInsertPoint() && !hasLabelBeenSeenInCurrentScope() &&
6178 EmitLifetimeStart(Addr: DeclPtr.getBasePointer()))
6179 pushCleanupAfterFullExpr<CallLifetimeEnd>(Kind: NormalEHLifetimeMarker,
6180 A: DeclPtr);
6181 }
6182
6183 EmitAnyExprToMem(E: InitExpr, Location: DeclPtr, Quals: E->getType().getQualifiers(),
6184 /*Init*/ IsInit: true);
6185
6186 // Block-scope compound literals are destroyed at the end of the enclosing
6187 // scope in C.
6188 if (!getLangOpts().CPlusPlus)
6189 if (QualType::DestructionKind DtorKind = E->getType().isDestructedType())
6190 pushLifetimeExtendedDestroy(dtorKind: DtorKind, addr: DeclPtr, type: E->getType());
6191
6192 return Result;
6193}
6194
6195LValue CodeGenFunction::EmitInitListLValue(const InitListExpr *E) {
6196 if (!E->isGLValue())
6197 // Initializing an aggregate temporary in C++11: T{...}.
6198 return EmitAggExprToLValue(E);
6199
6200 // An lvalue initializer list must be initializing a reference.
6201 assert(E->isTransparent() && "non-transparent glvalue init list");
6202 return EmitLValue(E: E->getInit(Init: 0));
6203}
6204
6205/// Emit the operand of a glvalue conditional operator. This is either a glvalue
6206/// or a (possibly-parenthesized) throw-expression. If this is a throw, no
6207/// LValue is returned and the current block has been terminated.
6208static std::optional<LValue> EmitLValueOrThrowExpression(CodeGenFunction &CGF,
6209 const Expr *Operand) {
6210 if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Val: Operand->IgnoreParens())) {
6211 CGF.EmitCXXThrowExpr(E: ThrowExpr, /*KeepInsertionPoint*/false);
6212 return std::nullopt;
6213 }
6214
6215 return CGF.EmitLValue(E: Operand);
6216}
6217
6218namespace {
6219// Handle the case where the condition is a constant evaluatable simple integer,
6220// which means we don't have to separately handle the true/false blocks.
6221std::optional<LValue> HandleConditionalOperatorLValueSimpleCase(
6222 CodeGenFunction &CGF, const AbstractConditionalOperator *E) {
6223 const Expr *condExpr = E->getCond();
6224 bool CondExprBool;
6225 if (CGF.ConstantFoldsToSimpleInteger(Cond: condExpr, Result&: CondExprBool)) {
6226 const Expr *Live = E->getTrueExpr(), *Dead = E->getFalseExpr();
6227 if (!CondExprBool)
6228 std::swap(a&: Live, b&: Dead);
6229
6230 if (!CGF.ContainsLabel(S: Dead)) {
6231 // If the true case is live, we need to track its region.
6232 CGF.incrementProfileCounter(ExecSkip: CondExprBool ? CGF.UseExecPath
6233 : CGF.UseSkipPath,
6234 S: E, /*UseBoth=*/true);
6235 CGF.markStmtMaybeUsed(S: Dead);
6236 // If a throw expression we emit it and return an undefined lvalue
6237 // because it can't be used.
6238 if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Val: Live->IgnoreParens())) {
6239 CGF.EmitCXXThrowExpr(E: ThrowExpr);
6240 llvm::Type *ElemTy = CGF.ConvertType(T: Dead->getType());
6241 llvm::Type *Ty = CGF.DefaultPtrTy;
6242 return CGF.MakeAddrLValue(
6243 Addr: Address(llvm::UndefValue::get(T: Ty), ElemTy, CharUnits::One()),
6244 T: Dead->getType());
6245 }
6246 return CGF.EmitLValue(E: Live);
6247 }
6248 }
6249 return std::nullopt;
6250}
6251struct ConditionalInfo {
6252 llvm::BasicBlock *lhsBlock, *rhsBlock;
6253 std::optional<LValue> LHS, RHS;
6254};
6255
6256// Create and generate the 3 blocks for a conditional operator.
6257// Leaves the 'current block' in the continuation basic block.
6258template<typename FuncTy>
6259ConditionalInfo EmitConditionalBlocks(CodeGenFunction &CGF,
6260 const AbstractConditionalOperator *E,
6261 const FuncTy &BranchGenFunc) {
6262 ConditionalInfo Info{.lhsBlock: CGF.createBasicBlock(name: "cond.true"),
6263 .rhsBlock: CGF.createBasicBlock(name: "cond.false"), .LHS: std::nullopt,
6264 .RHS: std::nullopt};
6265 llvm::BasicBlock *endBlock = CGF.createBasicBlock(name: "cond.end");
6266
6267 CodeGenFunction::ConditionalEvaluation eval(CGF);
6268 CGF.EmitBranchOnBoolExpr(Cond: E->getCond(), TrueBlock: Info.lhsBlock, FalseBlock: Info.rhsBlock,
6269 TrueCount: CGF.getProfileCount(S: E));
6270
6271 // Any temporaries created here are conditional.
6272 CGF.EmitBlock(BB: Info.lhsBlock);
6273 CGF.incrementProfileCounter(ExecSkip: CGF.UseExecPath, S: E);
6274 eval.begin(CGF);
6275 Info.LHS = BranchGenFunc(CGF, E->getTrueExpr());
6276 eval.end(CGF);
6277 Info.lhsBlock = CGF.Builder.GetInsertBlock();
6278
6279 if (Info.LHS)
6280 CGF.Builder.CreateBr(Dest: endBlock);
6281
6282 // Any temporaries created here are conditional.
6283 CGF.EmitBlock(BB: Info.rhsBlock);
6284 CGF.incrementProfileCounter(ExecSkip: CGF.UseSkipPath, S: E);
6285 eval.begin(CGF);
6286 Info.RHS = BranchGenFunc(CGF, E->getFalseExpr());
6287 eval.end(CGF);
6288 Info.rhsBlock = CGF.Builder.GetInsertBlock();
6289 CGF.EmitBlock(BB: endBlock);
6290
6291 return Info;
6292}
6293} // namespace
6294
6295void CodeGenFunction::EmitIgnoredConditionalOperator(
6296 const AbstractConditionalOperator *E) {
6297 if (!E->isGLValue()) {
6298 // ?: here should be an aggregate.
6299 assert(hasAggregateEvaluationKind(E->getType()) &&
6300 "Unexpected conditional operator!");
6301 return (void)EmitAggExprToLValue(E);
6302 }
6303
6304 OpaqueValueMapping binding(*this, E);
6305 if (HandleConditionalOperatorLValueSimpleCase(CGF&: *this, E))
6306 return;
6307
6308 EmitConditionalBlocks(CGF&: *this, E, BranchGenFunc: [](CodeGenFunction &CGF, const Expr *E) {
6309 CGF.EmitIgnoredExpr(E);
6310 return LValue{};
6311 });
6312}
6313LValue CodeGenFunction::EmitConditionalOperatorLValue(
6314 const AbstractConditionalOperator *expr) {
6315 if (!expr->isGLValue()) {
6316 // ?: here should be an aggregate.
6317 assert(hasAggregateEvaluationKind(expr->getType()) &&
6318 "Unexpected conditional operator!");
6319 return EmitAggExprToLValue(E: expr);
6320 }
6321
6322 OpaqueValueMapping binding(*this, expr);
6323 if (std::optional<LValue> Res =
6324 HandleConditionalOperatorLValueSimpleCase(CGF&: *this, E: expr))
6325 return *Res;
6326
6327 ConditionalInfo Info = EmitConditionalBlocks(
6328 CGF&: *this, E: expr, BranchGenFunc: [](CodeGenFunction &CGF, const Expr *E) {
6329 return EmitLValueOrThrowExpression(CGF, Operand: E);
6330 });
6331
6332 if ((Info.LHS && !Info.LHS->isSimple()) ||
6333 (Info.RHS && !Info.RHS->isSimple()))
6334 return EmitUnsupportedLValue(E: expr, Name: "conditional operator");
6335
6336 if (Info.LHS && Info.RHS) {
6337 Address lhsAddr = Info.LHS->getAddress();
6338 Address rhsAddr = Info.RHS->getAddress();
6339 Address result = mergeAddressesInConditionalExpr(
6340 LHS: lhsAddr, RHS: rhsAddr, LHSBlock: Info.lhsBlock, RHSBlock: Info.rhsBlock,
6341 MergeBlock: Builder.GetInsertBlock(), MergedType: expr->getType());
6342 AlignmentSource alignSource =
6343 std::max(a: Info.LHS->getBaseInfo().getAlignmentSource(),
6344 b: Info.RHS->getBaseInfo().getAlignmentSource());
6345 TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForConditionalOperator(
6346 InfoA: Info.LHS->getTBAAInfo(), InfoB: Info.RHS->getTBAAInfo());
6347 return MakeAddrLValue(Addr: result, T: expr->getType(), BaseInfo: LValueBaseInfo(alignSource),
6348 TBAAInfo);
6349 } else {
6350 assert((Info.LHS || Info.RHS) &&
6351 "both operands of glvalue conditional are throw-expressions?");
6352 return Info.LHS ? *Info.LHS : *Info.RHS;
6353 }
6354}
6355
6356/// EmitCastLValue - Casts are never lvalues unless that cast is to a reference
6357/// type. If the cast is to a reference, we can have the usual lvalue result,
6358/// otherwise if a cast is needed by the code generator in an lvalue context,
6359/// then it must mean that we need the address of an aggregate in order to
6360/// access one of its members. This can happen for all the reasons that casts
6361/// are permitted with aggregate result, including noop aggregate casts, and
6362/// cast from scalar to union.
6363LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
6364 llvm::scope_exit RestoreCurCast([this, Prev = CurCast] { CurCast = Prev; });
6365 CurCast = E;
6366 switch (E->getCastKind()) {
6367 case CK_ToVoid:
6368 case CK_BitCast:
6369 case CK_LValueToRValueBitCast:
6370 case CK_ArrayToPointerDecay:
6371 case CK_FunctionToPointerDecay:
6372 case CK_NullToMemberPointer:
6373 case CK_NullToPointer:
6374 case CK_IntegralToPointer:
6375 case CK_PointerToIntegral:
6376 case CK_PointerToBoolean:
6377 case CK_IntegralCast:
6378 case CK_BooleanToSignedIntegral:
6379 case CK_IntegralToBoolean:
6380 case CK_IntegralToFloating:
6381 case CK_FloatingToIntegral:
6382 case CK_FloatingToBoolean:
6383 case CK_FloatingCast:
6384 case CK_FloatingRealToComplex:
6385 case CK_FloatingComplexToReal:
6386 case CK_FloatingComplexToBoolean:
6387 case CK_FloatingComplexCast:
6388 case CK_FloatingComplexToIntegralComplex:
6389 case CK_IntegralRealToComplex:
6390 case CK_IntegralComplexToReal:
6391 case CK_IntegralComplexToBoolean:
6392 case CK_IntegralComplexCast:
6393 case CK_IntegralComplexToFloatingComplex:
6394 case CK_DerivedToBaseMemberPointer:
6395 case CK_BaseToDerivedMemberPointer:
6396 case CK_MemberPointerToBoolean:
6397 case CK_ReinterpretMemberPointer:
6398 case CK_AnyPointerToBlockPointerCast:
6399 case CK_ARCProduceObject:
6400 case CK_ARCConsumeObject:
6401 case CK_ARCReclaimReturnedObject:
6402 case CK_ARCExtendBlockObject:
6403 case CK_CopyAndAutoreleaseBlockObject:
6404 case CK_IntToOCLSampler:
6405 case CK_FloatingToFixedPoint:
6406 case CK_FixedPointToFloating:
6407 case CK_FixedPointCast:
6408 case CK_FixedPointToBoolean:
6409 case CK_FixedPointToIntegral:
6410 case CK_IntegralToFixedPoint:
6411 case CK_MatrixCast:
6412 case CK_HLSLVectorTruncation:
6413 case CK_HLSLMatrixTruncation:
6414 case CK_HLSLArrayRValue:
6415 case CK_HLSLElementwiseCast:
6416 case CK_HLSLAggregateSplatCast:
6417 return EmitUnsupportedLValue(E, Name: "unexpected cast lvalue");
6418
6419 case CK_Dependent:
6420 llvm_unreachable("dependent cast kind in IR gen!");
6421
6422 case CK_BuiltinFnToFnPtr:
6423 llvm_unreachable("builtin functions are handled elsewhere");
6424
6425 // These are never l-values; just use the aggregate emission code.
6426 case CK_NonAtomicToAtomic:
6427 case CK_AtomicToNonAtomic:
6428 return EmitAggExprToLValue(E);
6429
6430 case CK_Dynamic: {
6431 LValue LV = EmitLValue(E: E->getSubExpr());
6432 Address V = LV.getAddress();
6433 const auto *DCE = cast<CXXDynamicCastExpr>(Val: E);
6434 return MakeNaturalAlignRawAddrLValue(V: EmitDynamicCast(V, DCE), T: E->getType());
6435 }
6436
6437 case CK_ConstructorConversion:
6438 case CK_UserDefinedConversion:
6439 case CK_CPointerToObjCPointerCast:
6440 case CK_BlockPointerToObjCPointerCast:
6441 case CK_LValueToRValue:
6442 return EmitLValue(E: E->getSubExpr());
6443
6444 case CK_NoOp: {
6445 // CK_NoOp can model a qualification conversion, which can remove an array
6446 // bound and change the IR type.
6447 // FIXME: Once pointee types are removed from IR, remove this.
6448 LValue LV = EmitLValue(E: E->getSubExpr());
6449 // Propagate the volatile qualifer to LValue, if exist in E.
6450 if (E->changesVolatileQualification())
6451 LV.getQuals() = E->getType().getQualifiers();
6452 if (LV.isSimple()) {
6453 Address V = LV.getAddress();
6454 if (V.isValid()) {
6455 llvm::Type *T = ConvertTypeForMem(T: E->getType());
6456 if (V.getElementType() != T)
6457 LV.setAddress(V.withElementType(ElemTy: T));
6458 }
6459 }
6460 return LV;
6461 }
6462
6463 case CK_UncheckedDerivedToBase:
6464 case CK_DerivedToBase: {
6465 auto *DerivedClassDecl = E->getSubExpr()->getType()->castAsCXXRecordDecl();
6466 LValue LV = EmitLValue(E: E->getSubExpr());
6467 Address This = LV.getAddress();
6468
6469 // Perform the derived-to-base conversion
6470 Address Base = GetAddressOfBaseClass(
6471 Value: This, Derived: DerivedClassDecl, PathBegin: E->path_begin(), PathEnd: E->path_end(),
6472 /*NullCheckValue=*/false, Loc: E->getExprLoc());
6473
6474 // TODO: Support accesses to members of base classes in TBAA. For now, we
6475 // conservatively pretend that the complete object is of the base class
6476 // type.
6477 return MakeAddrLValue(Addr: Base, T: E->getType(), BaseInfo: LV.getBaseInfo(),
6478 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
6479 }
6480 case CK_ToUnion:
6481 return EmitAggExprToLValue(E);
6482 case CK_BaseToDerived: {
6483 auto *DerivedClassDecl = E->getType()->castAsCXXRecordDecl();
6484 LValue LV = EmitLValue(E: E->getSubExpr());
6485
6486 // Perform the base-to-derived conversion
6487 Address Derived = GetAddressOfDerivedClass(
6488 Value: LV.getAddress(), Derived: DerivedClassDecl, PathBegin: E->path_begin(), PathEnd: E->path_end(),
6489 /*NullCheckValue=*/false);
6490
6491 // C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is
6492 // performed and the object is not of the derived type.
6493 if (sanitizePerformTypeCheck())
6494 EmitTypeCheck(TCK: TCK_DowncastReference, Loc: E->getExprLoc(), Addr: Derived,
6495 Type: E->getType());
6496
6497 if (SanOpts.has(K: SanitizerKind::CFIDerivedCast))
6498 EmitVTablePtrCheckForCast(T: E->getType(), Derived,
6499 /*MayBeNull=*/false, TCK: CFITCK_DerivedCast,
6500 Loc: E->getBeginLoc());
6501
6502 return MakeAddrLValue(Addr: Derived, T: E->getType(), BaseInfo: LV.getBaseInfo(),
6503 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
6504 }
6505 case CK_LValueBitCast: {
6506 // This must be a reinterpret_cast (or c-style equivalent).
6507 const auto *CE = cast<ExplicitCastExpr>(Val: E);
6508
6509 CGM.EmitExplicitCastExprType(E: CE, CGF: this);
6510 LValue LV = EmitLValue(E: E->getSubExpr());
6511 Address V = LV.getAddress().withElementType(
6512 ElemTy: ConvertTypeForMem(T: CE->getTypeAsWritten()->getPointeeType()));
6513
6514 if (SanOpts.has(K: SanitizerKind::CFIUnrelatedCast))
6515 EmitVTablePtrCheckForCast(T: E->getType(), Derived: V,
6516 /*MayBeNull=*/false, TCK: CFITCK_UnrelatedCast,
6517 Loc: E->getBeginLoc());
6518
6519 return MakeAddrLValue(Addr: V, T: E->getType(), BaseInfo: LV.getBaseInfo(),
6520 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
6521 }
6522 case CK_AddressSpaceConversion: {
6523 LValue LV = EmitLValue(E: E->getSubExpr());
6524 QualType DestTy = getContext().getPointerType(T: E->getType());
6525 llvm::Value *V =
6526 performAddrSpaceCast(Src: LV.getPointer(CGF&: *this), DestTy: ConvertType(T: DestTy));
6527 return MakeAddrLValue(Addr: Address(V, ConvertTypeForMem(T: E->getType()),
6528 LV.getAddress().getAlignment()),
6529 T: E->getType(), BaseInfo: LV.getBaseInfo(), TBAAInfo: LV.getTBAAInfo());
6530 }
6531 case CK_ObjCObjectLValueCast: {
6532 LValue LV = EmitLValue(E: E->getSubExpr());
6533 Address V = LV.getAddress().withElementType(ElemTy: ConvertType(T: E->getType()));
6534 return MakeAddrLValue(Addr: V, T: E->getType(), BaseInfo: LV.getBaseInfo(),
6535 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
6536 }
6537 case CK_ZeroToOCLOpaqueType:
6538 llvm_unreachable("NULL to OpenCL opaque type lvalue cast is not valid");
6539
6540 case CK_VectorSplat: {
6541 // LValue results of vector splats are only supported in HLSL.
6542 if (!getLangOpts().HLSL)
6543 return EmitUnsupportedLValue(E, Name: "unexpected cast lvalue");
6544 return EmitLValue(E: E->getSubExpr());
6545 }
6546 }
6547
6548 llvm_unreachable("Unhandled lvalue cast kind?");
6549}
6550
6551LValue CodeGenFunction::EmitOpaqueValueLValue(const OpaqueValueExpr *e) {
6552 assert(OpaqueValueMappingData::shouldBindAsLValue(e));
6553 return getOrCreateOpaqueLValueMapping(e);
6554}
6555
6556std::pair<LValue, LValue>
6557CodeGenFunction::EmitHLSLOutArgLValues(const HLSLOutArgExpr *E, QualType Ty) {
6558 // Emitting the casted temporary through an opaque value.
6559 LValue BaseLV = EmitLValue(E: E->getArgLValue());
6560 OpaqueValueMappingData::bind(CGF&: *this, ov: E->getOpaqueArgLValue(), lv: BaseLV);
6561
6562 QualType ExprTy = E->getType();
6563 Address OutTemp = CreateIRTempWithoutCast(Ty: ExprTy);
6564 LValue TempLV = MakeAddrLValue(Addr: OutTemp, T: ExprTy);
6565
6566 // Start the lifetime before the copy-in so that the temporary is live when
6567 // the initial value is written. This ensures the store is within the
6568 // lifetime and is not killed by a store undef inserted at lifetime.start.
6569 EmitLifetimeStart(Addr: OutTemp.getBasePointer());
6570
6571 if (E->isInOut())
6572 EmitInitializationToLValue(E: E->getCastedTemporary()->getSourceExpr(),
6573 LV: TempLV);
6574
6575 OpaqueValueMappingData::bind(CGF&: *this, ov: E->getCastedTemporary(), lv: TempLV);
6576 return std::make_pair(x&: BaseLV, y&: TempLV);
6577}
6578
6579LValue CodeGenFunction::EmitHLSLOutArgExpr(const HLSLOutArgExpr *E,
6580 CallArgList &Args, QualType Ty,
6581 bool IsImplicit) {
6582
6583 auto [BaseLV, TempLV] = EmitHLSLOutArgLValues(E, Ty);
6584
6585 llvm::Value *Addr = TempLV.getAddress().getBasePointer();
6586 llvm::Type *ElTy = ConvertTypeForMem(T: TempLV.getType());
6587
6588 Address TmpAddr(Addr, ElTy, TempLV.getAlignment());
6589 Args.addWriteback(srcLV: BaseLV, temporary: TmpAddr, toUse: nullptr, writebackExpr: E->getWritebackCast());
6590 if (!IsImplicit)
6591 Args.add(rvalue: RValue::get(Addr: TmpAddr, CGF&: *this), type: Ty);
6592 return TempLV;
6593}
6594
6595LValue
6596CodeGenFunction::getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e) {
6597 assert(OpaqueValueMapping::shouldBindAsLValue(e));
6598
6599 llvm::DenseMap<const OpaqueValueExpr*,LValue>::iterator
6600 it = OpaqueLValues.find(Val: e);
6601
6602 if (it != OpaqueLValues.end())
6603 return it->second;
6604
6605 assert(e->isUnique() && "LValue for a nonunique OVE hasn't been emitted");
6606 return EmitLValue(E: e->getSourceExpr());
6607}
6608
6609RValue
6610CodeGenFunction::getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e) {
6611 assert(!OpaqueValueMapping::shouldBindAsLValue(e));
6612
6613 llvm::DenseMap<const OpaqueValueExpr*,RValue>::iterator
6614 it = OpaqueRValues.find(Val: e);
6615
6616 if (it != OpaqueRValues.end())
6617 return it->second;
6618
6619 assert(e->isUnique() && "RValue for a nonunique OVE hasn't been emitted");
6620 return EmitAnyExpr(E: e->getSourceExpr());
6621}
6622
6623bool CodeGenFunction::isOpaqueValueEmitted(const OpaqueValueExpr *E) {
6624 if (OpaqueValueMapping::shouldBindAsLValue(expr: E))
6625 return OpaqueLValues.contains(Val: E);
6626 return OpaqueRValues.contains(Val: E);
6627}
6628
6629RValue CodeGenFunction::EmitRValueForField(LValue LV,
6630 const FieldDecl *FD,
6631 SourceLocation Loc) {
6632 QualType FT = FD->getType();
6633 LValue FieldLV = EmitLValueForField(base: LV, field: FD);
6634 switch (getEvaluationKind(T: FT)) {
6635 case TEK_Complex:
6636 return RValue::getComplex(C: EmitLoadOfComplex(src: FieldLV, loc: Loc));
6637 case TEK_Aggregate:
6638 return FieldLV.asAggregateRValue();
6639 case TEK_Scalar:
6640 // This routine is used to load fields one-by-one to perform a copy, so
6641 // don't load reference fields.
6642 if (FD->getType()->isReferenceType())
6643 return RValue::get(V: FieldLV.getPointer(CGF&: *this));
6644 // Call EmitLoadOfScalar except when the lvalue is a bitfield to emit a
6645 // primitive load.
6646 if (FieldLV.isBitField())
6647 return EmitLoadOfLValue(LV: FieldLV, Loc);
6648 return RValue::get(V: EmitLoadOfScalar(lvalue: FieldLV, Loc));
6649 }
6650 llvm_unreachable("bad evaluation kind");
6651}
6652
6653//===--------------------------------------------------------------------===//
6654// Expression Emission
6655//===--------------------------------------------------------------------===//
6656
6657RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
6658 ReturnValueSlot ReturnValue,
6659 llvm::CallBase **CallOrInvoke) {
6660 llvm::CallBase *CallOrInvokeStorage;
6661 if (!CallOrInvoke) {
6662 CallOrInvoke = &CallOrInvokeStorage;
6663 }
6664
6665 llvm::scope_exit AddCoroElideSafeOnExit([&] {
6666 if (E->isCoroElideSafe()) {
6667 auto *I = *CallOrInvoke;
6668 if (I)
6669 I->addFnAttr(Kind: llvm::Attribute::CoroElideSafe);
6670 }
6671 });
6672
6673 // Builtins never have block type.
6674 if (E->getCallee()->getType()->isBlockPointerType())
6675 return EmitBlockCallExpr(E, ReturnValue, CallOrInvoke);
6676
6677 if (const auto *CE = dyn_cast<CXXMemberCallExpr>(Val: E))
6678 return EmitCXXMemberCallExpr(E: CE, ReturnValue, CallOrInvoke);
6679
6680 if (const auto *CE = dyn_cast<CUDAKernelCallExpr>(Val: E))
6681 return EmitCUDAKernelCallExpr(E: CE, ReturnValue, CallOrInvoke);
6682
6683 // A CXXOperatorCallExpr is created even for explicit object methods, but
6684 // these should be treated like static function call.
6685 if (const auto *CE = dyn_cast<CXXOperatorCallExpr>(Val: E))
6686 if (const auto *MD =
6687 dyn_cast_if_present<CXXMethodDecl>(Val: CE->getCalleeDecl());
6688 MD && MD->isImplicitObjectMemberFunction())
6689 return EmitCXXOperatorMemberCallExpr(E: CE, MD, ReturnValue, CallOrInvoke);
6690
6691 CGCallee callee = EmitCallee(E: E->getCallee());
6692
6693 if (callee.isBuiltin()) {
6694 return EmitBuiltinExpr(GD: callee.getBuiltinDecl(), BuiltinID: callee.getBuiltinID(),
6695 E, ReturnValue);
6696 }
6697
6698 if (callee.isPseudoDestructor()) {
6699 return EmitCXXPseudoDestructorExpr(E: callee.getPseudoDestructorExpr());
6700 }
6701
6702 return EmitCall(FnType: E->getCallee()->getType(), Callee: callee, E, ReturnValue,
6703 /*Chain=*/nullptr, CallOrInvoke);
6704}
6705
6706/// Emit a CallExpr without considering whether it might be a subclass.
6707RValue CodeGenFunction::EmitSimpleCallExpr(const CallExpr *E,
6708 ReturnValueSlot ReturnValue,
6709 llvm::CallBase **CallOrInvoke) {
6710 CGCallee Callee = EmitCallee(E: E->getCallee());
6711 return EmitCall(FnType: E->getCallee()->getType(), Callee, E, ReturnValue,
6712 /*Chain=*/nullptr, CallOrInvoke);
6713}
6714
6715static CGCallee EmitDirectCallee(CodeGenFunction &CGF, GlobalDecl GD) {
6716 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
6717
6718 if (auto builtinID = FD->getBuiltinID()) {
6719 std::string NoBuiltinFD = ("no-builtin-" + FD->getName()).str();
6720 std::string NoBuiltins = "no-builtins";
6721
6722 StringRef Ident = CGF.CGM.getMangledName(GD);
6723 std::string FDInlineName = (Ident + ".inline").str();
6724
6725 bool IsPredefinedLibFunction =
6726 CGF.getContext().BuiltinInfo.isPredefinedLibFunction(ID: builtinID);
6727 bool HasAttributeNoBuiltin =
6728 CGF.CurFn->getAttributes().hasFnAttr(Kind: NoBuiltinFD) ||
6729 CGF.CurFn->getAttributes().hasFnAttr(Kind: NoBuiltins);
6730
6731 // When directing calling an inline builtin, call it through it's mangled
6732 // name to make it clear it's not the actual builtin.
6733 if (CGF.CurFn->getName() != FDInlineName &&
6734 CodeGenUtils::onlyHasInlineBuiltinDeclaration(FD)) {
6735 llvm::Constant *CalleePtr = CGF.CGM.getRawFunctionPointer(GD);
6736 llvm::Function *Fn = llvm::cast<llvm::Function>(Val: CalleePtr);
6737 llvm::Module *M = Fn->getParent();
6738 llvm::Function *Clone = M->getFunction(Name: FDInlineName);
6739 if (!Clone) {
6740 Clone = llvm::Function::Create(Ty: Fn->getFunctionType(),
6741 Linkage: llvm::GlobalValue::InternalLinkage,
6742 AddrSpace: Fn->getAddressSpace(), N: FDInlineName, M);
6743 Clone->setCallingConv(Fn->getCallingConv());
6744 Clone->setAttributes(Fn->getAttributes());
6745 Clone->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
6746 }
6747 return CGCallee::forDirect(functionPtr: Clone, abstractInfo: GD);
6748 }
6749
6750 // Replaceable builtins provide their own implementation of a builtin. If we
6751 // are in an inline builtin implementation, avoid trivial infinite
6752 // recursion. Honor __attribute__((no_builtin("foo"))) or
6753 // __attribute__((no_builtin)) on the current function unless foo is
6754 // not a predefined library function which means we must generate the
6755 // builtin no matter what.
6756 else if (!IsPredefinedLibFunction || !HasAttributeNoBuiltin)
6757 return CGCallee::forBuiltin(builtinID, builtinDecl: FD);
6758 }
6759
6760 llvm::Constant *CalleePtr = CGF.CGM.getRawFunctionPointer(GD);
6761 if (CGF.CGM.getLangOpts().CUDA && !CGF.CGM.getLangOpts().CUDAIsDevice &&
6762 FD->hasAttr<CUDAGlobalAttr>())
6763 CalleePtr = CGF.CGM.getCUDARuntime().getKernelStub(
6764 Handle: cast<llvm::GlobalValue>(Val: CalleePtr->stripPointerCasts()));
6765
6766 return CGCallee::forDirect(functionPtr: CalleePtr, abstractInfo: GD);
6767}
6768
6769static GlobalDecl getGlobalDeclForDirectCall(const FunctionDecl *FD) {
6770 if (DeviceKernelAttr::isOpenCLSpelling(A: FD->getAttr<DeviceKernelAttr>()))
6771 return GlobalDecl(FD, KernelReferenceKind::Stub);
6772 return GlobalDecl(FD);
6773}
6774
6775CGCallee CodeGenFunction::EmitCallee(const Expr *E) {
6776 E = E->IgnoreParens();
6777
6778 // A WebAssembly funcref is an opaque reference type and llvm only accepts
6779 // function pointers as the call target. To make an indirect call through a
6780 // reference type, first use the llvm.wasm.funcref.to_ptr intrinsic to make a
6781 // fake function pointer to it. The backend lowers the resulting indirect call
6782 // to a table.set into a single element dummy table + call_indirect 0.
6783 auto ConvertFuncrefToPtr = [&](llvm::Value *CalleePtr) -> llvm::Value * {
6784 if (auto *TET = dyn_cast<llvm::TargetExtType>(Val: CalleePtr->getType());
6785 TET && TET->getName() == "wasm.funcref") {
6786 llvm::Function *ToPtr =
6787 CGM.getIntrinsic(IID: llvm::Intrinsic::wasm_funcref_to_ptr);
6788 return Builder.CreateCall(Callee: ToPtr, Args: {CalleePtr});
6789 }
6790 return CalleePtr;
6791 };
6792
6793 // Look through function-to-pointer decay.
6794 if (auto ICE = dyn_cast<ImplicitCastExpr>(Val: E)) {
6795 if (ICE->getCastKind() == CK_FunctionToPointerDecay ||
6796 ICE->getCastKind() == CK_BuiltinFnToFnPtr) {
6797 return EmitCallee(E: ICE->getSubExpr());
6798 }
6799
6800 // Try to remember the original __ptrauth qualifier for loads of
6801 // function pointers.
6802 if (ICE->getCastKind() == CK_LValueToRValue) {
6803 const Expr *SubExpr = ICE->getSubExpr();
6804 if (const auto *PtrType = SubExpr->getType()->getAs<PointerType>()) {
6805 std::pair<llvm::Value *, CGPointerAuthInfo> Result =
6806 EmitOrigPointerRValue(E);
6807
6808 QualType FunctionType = PtrType->getPointeeType();
6809 assert(FunctionType->isFunctionType());
6810
6811 GlobalDecl GD;
6812 if (const auto *VD =
6813 dyn_cast_or_null<VarDecl>(Val: E->getReferencedDeclOfCallee())) {
6814 GD = GlobalDecl(VD);
6815 }
6816 CGCalleeInfo CalleeInfo(FunctionType->castAs<clang::FunctionType>(),
6817 GD);
6818 CGCallee Callee(CalleeInfo, ConvertFuncrefToPtr(Result.first),
6819 Result.second);
6820 return Callee;
6821 }
6822 }
6823
6824 // Resolve direct calls.
6825 } else if (auto DRE = dyn_cast<DeclRefExpr>(Val: E)) {
6826 if (auto FD = dyn_cast<FunctionDecl>(Val: DRE->getDecl())) {
6827 return EmitDirectCallee(CGF&: *this, GD: getGlobalDeclForDirectCall(FD));
6828 }
6829 } else if (auto ME = dyn_cast<MemberExpr>(Val: E)) {
6830 if (auto FD = dyn_cast<FunctionDecl>(Val: ME->getMemberDecl())) {
6831 EmitIgnoredExpr(E: ME->getBase());
6832 return EmitDirectCallee(CGF&: *this, GD: FD);
6833 }
6834
6835 // Look through template substitutions.
6836 } else if (auto NTTP = dyn_cast<SubstNonTypeTemplateParmExpr>(Val: E)) {
6837 return EmitCallee(E: NTTP->getReplacement());
6838
6839 // Treat pseudo-destructor calls differently.
6840 } else if (auto PDE = dyn_cast<CXXPseudoDestructorExpr>(Val: E)) {
6841 return CGCallee::forPseudoDestructor(E: PDE);
6842 }
6843
6844 // Otherwise, we have an indirect reference.
6845 llvm::Value *calleePtr;
6846 QualType functionType;
6847 if (auto ptrType = E->getType()->getAs<PointerType>()) {
6848 calleePtr = EmitScalarExpr(E);
6849 functionType = ptrType->getPointeeType();
6850 } else {
6851 functionType = E->getType();
6852 calleePtr = EmitLValue(E, IsKnownNonNull: KnownNonNull).getPointer(CGF&: *this);
6853 }
6854 assert(functionType->isFunctionType());
6855
6856 GlobalDecl GD;
6857 if (const auto *VD =
6858 dyn_cast_or_null<VarDecl>(Val: E->getReferencedDeclOfCallee()))
6859 GD = GlobalDecl(VD);
6860
6861 CGCalleeInfo calleeInfo(functionType->castAs<clang::FunctionType>(), GD);
6862 CGPointerAuthInfo pointerAuth = CGM.getFunctionPointerAuthInfo(T: functionType);
6863 CGCallee callee(calleeInfo, ConvertFuncrefToPtr(calleePtr), pointerAuth);
6864 return callee;
6865}
6866
6867LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
6868 // Comma expressions just emit their LHS then their RHS as an l-value.
6869 if (E->getOpcode() == BO_Comma) {
6870 EmitIgnoredExpr(E: E->getLHS());
6871 EnsureInsertPoint();
6872 return EmitLValue(E: E->getRHS());
6873 }
6874
6875 if (E->getOpcode() == BO_PtrMemD ||
6876 E->getOpcode() == BO_PtrMemI)
6877 return EmitPointerToDataMemberBinaryExpr(E);
6878
6879 assert(E->getOpcode() == BO_Assign && "unexpected binary l-value");
6880
6881 // Create a Key Instructions source location atom group that covers both
6882 // LHS and RHS expressions. Nested RHS expressions may get subsequently
6883 // separately grouped (1 below):
6884 //
6885 // 1. `a = b = c` -> Two atoms.
6886 // 2. `x = new(1)` -> One atom (for both addr store and value store).
6887 // 3. Complex and agg assignment -> One atom.
6888 ApplyAtomGroup Grp(getDebugInfo());
6889
6890 // Note that in all of these cases, __block variables need the RHS
6891 // evaluated first just in case the variable gets moved by the RHS.
6892
6893 switch (getEvaluationKind(T: E->getType())) {
6894 case TEK_Scalar: {
6895 if (PointerAuthQualifier PtrAuth =
6896 E->getLHS()->getType().getPointerAuth()) {
6897 LValue LV = EmitCheckedLValue(E: E->getLHS(), TCK: TCK_Store);
6898 LValue CopiedLV = LV;
6899 CopiedLV.getQuals().removePointerAuth();
6900 llvm::Value *RV =
6901 EmitPointerAuthQualify(Qualifier: PtrAuth, PointerExpr: E->getRHS(), StorageAddress: CopiedLV.getAddress());
6902 EmitNullabilityCheck(LHS: CopiedLV, RHS: RV, Loc: E->getExprLoc());
6903 EmitStoreThroughLValue(Src: RValue::get(V: RV), Dst: CopiedLV);
6904 return LV;
6905 }
6906
6907 switch (E->getLHS()->getType().getObjCLifetime()) {
6908 case Qualifiers::OCL_Strong:
6909 return EmitARCStoreStrong(e: E, /*ignored*/ false).first;
6910
6911 case Qualifiers::OCL_Autoreleasing:
6912 return EmitARCStoreAutoreleasing(e: E).first;
6913
6914 // No reason to do any of these differently.
6915 case Qualifiers::OCL_None:
6916 case Qualifiers::OCL_ExplicitNone:
6917 case Qualifiers::OCL_Weak:
6918 break;
6919 }
6920
6921 // TODO: Can we de-duplicate this code with the corresponding code in
6922 // CGExprScalar, similar to the way EmitCompoundAssignmentLValue works?
6923 RValue RV;
6924 llvm::Value *Previous = nullptr;
6925 QualType SrcType = E->getRHS()->getType();
6926 // Check if LHS is a bitfield, if RHS contains an implicit cast expression
6927 // we want to extract that value and potentially (if the bitfield sanitizer
6928 // is enabled) use it to check for an implicit conversion.
6929 if (E->getLHS()->refersToBitField()) {
6930 llvm::Value *RHS =
6931 EmitWithOriginalRHSBitfieldAssignment(E, Previous: &Previous, SrcType: &SrcType);
6932 RV = RValue::get(V: RHS);
6933 } else
6934 RV = EmitAnyExpr(E: E->getRHS());
6935
6936 LValue LV = EmitCheckedLValue(E: E->getLHS(), TCK: TCK_Store);
6937
6938 if (RV.isScalar())
6939 EmitNullabilityCheck(LHS: LV, RHS: RV.getScalarVal(), Loc: E->getExprLoc());
6940
6941 if (LV.isBitField()) {
6942 llvm::Value *Result = nullptr;
6943 // If bitfield sanitizers are enabled we want to use the result
6944 // to check whether a truncation or sign change has occurred.
6945 if (SanOpts.has(K: SanitizerKind::ImplicitBitfieldConversion))
6946 EmitStoreThroughBitfieldLValue(Src: RV, Dst: LV, Result: &Result);
6947 else
6948 EmitStoreThroughBitfieldLValue(Src: RV, Dst: LV);
6949
6950 // If the expression contained an implicit conversion, make sure
6951 // to use the value before the scalar conversion.
6952 llvm::Value *Src = Previous ? Previous : RV.getScalarVal();
6953 QualType DstType = E->getLHS()->getType();
6954 EmitBitfieldConversionCheck(Src, SrcType, Dst: Result, DstType,
6955 Info: LV.getBitFieldInfo(), Loc: E->getExprLoc());
6956 } else
6957 EmitStoreThroughLValue(Src: RV, Dst: LV);
6958
6959 if (getLangOpts().OpenMP)
6960 CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF&: *this,
6961 LHS: E->getLHS());
6962 return LV;
6963 }
6964
6965 case TEK_Complex:
6966 return EmitComplexAssignmentLValue(E);
6967
6968 case TEK_Aggregate:
6969 // If the lang opt is HLSL and the LHS is a constant array
6970 // then we are performing a copy assignment and call a special
6971 // function because EmitAggExprToLValue emits to a temporary LValue
6972 if (getLangOpts().HLSL && E->getLHS()->getType()->isConstantArrayType())
6973 return EmitHLSLArrayAssignLValue(E);
6974
6975 return EmitAggExprToLValue(E);
6976 }
6977 llvm_unreachable("bad evaluation kind");
6978}
6979
6980// This function implements trivial copy assignment for HLSL's
6981// assignable constant arrays.
6982LValue CodeGenFunction::EmitHLSLArrayAssignLValue(const BinaryOperator *E) {
6983 // Don't emit an LValue for the RHS because it might not be an LValue
6984 LValue LHS = EmitLValue(E: E->getLHS());
6985
6986 // If the RHS is a global resource array, copy all individual resources
6987 // into LHS.
6988 if (E->getRHS()->getType()->isHLSLResourceRecordArray()) {
6989 AggValueSlot Slot = AggValueSlot::forAddr(
6990 addr: LHS.getAddress(), quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed_t(true),
6991 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsAliased_t(false),
6992 mayOverlap: AggValueSlot::DoesNotOverlap);
6993 if (CGM.getHLSLRuntime().emitGlobalResourceArray(CGF&: *this, E: E->getRHS(), DestSlot&: Slot))
6994 return LHS;
6995 }
6996
6997 // In C the RHS of an assignment operator is an RValue.
6998 // EmitAggregateAssign takes an LValue for the RHS. Instead we can call
6999 // EmitInitializationToLValue to emit an RValue into an LValue.
7000 EmitInitializationToLValue(E: E->getRHS(), LV: LHS);
7001 return LHS;
7002}
7003
7004LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E,
7005 llvm::CallBase **CallOrInvoke) {
7006 RValue RV = EmitCallExpr(E, ReturnValue: ReturnValueSlot(), CallOrInvoke);
7007
7008 if (!RV.isScalar())
7009 return MakeAddrLValue(Addr: RV.getAggregateAddress(), T: E->getType(),
7010 Source: AlignmentSource::Decl);
7011
7012 assert(E->getCallReturnType(getContext())->isReferenceType() &&
7013 "Can't have a scalar return unless the return type is a "
7014 "reference type!");
7015
7016 return MakeNaturalAlignPointeeAddrLValue(V: RV.getScalarVal(), T: E->getType());
7017}
7018
7019LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {
7020 // FIXME: This shouldn't require another copy.
7021 return EmitAggExprToLValue(E);
7022}
7023
7024LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
7025 assert(E->getType()->getAsCXXRecordDecl()->hasTrivialDestructor()
7026 && "binding l-value to type which needs a temporary");
7027 AggValueSlot Slot = CreateAggTemp(T: E->getType());
7028 EmitCXXConstructExpr(E, Dest: Slot);
7029 return MakeAddrLValue(Addr: Slot.getAddress(), T: E->getType(), Source: AlignmentSource::Decl);
7030}
7031
7032LValue
7033CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
7034 return MakeNaturalAlignRawAddrLValue(V: EmitCXXTypeidExpr(E), T: E->getType());
7035}
7036
7037Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) {
7038 return CGM.GetAddrOfMSGuidDecl(GD: E->getGuidDecl())
7039 .withElementType(ElemTy: ConvertType(T: E->getType()));
7040}
7041
7042LValue CodeGenFunction::EmitCXXUuidofLValue(const CXXUuidofExpr *E) {
7043 return MakeAddrLValue(Addr: EmitCXXUuidofExpr(E), T: E->getType(),
7044 Source: AlignmentSource::Decl);
7045}
7046
7047LValue
7048CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) {
7049 AggValueSlot Slot = CreateAggTemp(T: E->getType(), Name: "temp.lvalue");
7050 Slot.setExternallyDestructed();
7051 EmitAggExpr(E: E->getSubExpr(), AS: Slot);
7052 EmitCXXTemporary(Temporary: E->getTemporary(), TempType: E->getType(), Ptr: Slot.getAddress());
7053 return MakeAddrLValue(Addr: Slot.getAddress(), T: E->getType(), Source: AlignmentSource::Decl);
7054}
7055
7056LValue CodeGenFunction::EmitObjCMessageExprLValue(const ObjCMessageExpr *E) {
7057 RValue RV = EmitObjCMessageExpr(E);
7058
7059 if (!RV.isScalar())
7060 return MakeAddrLValue(Addr: RV.getAggregateAddress(), T: E->getType(),
7061 Source: AlignmentSource::Decl);
7062
7063 assert(E->getMethodDecl()->getReturnType()->isReferenceType() &&
7064 "Can't have a scalar return unless the return type is a "
7065 "reference type!");
7066
7067 return MakeNaturalAlignPointeeAddrLValue(V: RV.getScalarVal(), T: E->getType());
7068}
7069
7070LValue CodeGenFunction::EmitObjCSelectorLValue(const ObjCSelectorExpr *E) {
7071 Address V =
7072 CGM.getObjCRuntime().GetAddrOfSelector(CGF&: *this, Sel: E->getSelector());
7073 return MakeAddrLValue(Addr: V, T: E->getType(), Source: AlignmentSource::Decl);
7074}
7075
7076llvm::Value *CodeGenFunction::EmitIvarOffset(const ObjCInterfaceDecl *Interface,
7077 const ObjCIvarDecl *Ivar) {
7078 return CGM.getObjCRuntime().EmitIvarOffset(CGF&: *this, Interface, Ivar);
7079}
7080
7081llvm::Value *
7082CodeGenFunction::EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,
7083 const ObjCIvarDecl *Ivar) {
7084 llvm::Value *OffsetValue = EmitIvarOffset(Interface, Ivar);
7085 QualType PointerDiffType = getContext().getPointerDiffType();
7086 return Builder.CreateZExtOrTrunc(V: OffsetValue,
7087 DestTy: getTypes().ConvertType(T: PointerDiffType));
7088}
7089
7090LValue CodeGenFunction::EmitLValueForIvar(QualType ObjectTy,
7091 llvm::Value *BaseValue,
7092 const ObjCIvarDecl *Ivar,
7093 unsigned CVRQualifiers) {
7094 return CGM.getObjCRuntime().EmitObjCValueForIvar(CGF&: *this, ObjectTy, BaseValue,
7095 Ivar, CVRQualifiers);
7096}
7097
7098LValue CodeGenFunction::EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E) {
7099 // FIXME: A lot of the code below could be shared with EmitMemberExpr.
7100 llvm::Value *BaseValue = nullptr;
7101 const Expr *BaseExpr = E->getBase();
7102 Qualifiers BaseQuals;
7103 QualType ObjectTy;
7104 if (E->isArrow()) {
7105 BaseValue = EmitScalarExpr(E: BaseExpr);
7106 ObjectTy = BaseExpr->getType()->getPointeeType();
7107 BaseQuals = ObjectTy.getQualifiers();
7108 } else {
7109 LValue BaseLV = EmitLValue(E: BaseExpr);
7110 BaseValue = BaseLV.getPointer(CGF&: *this);
7111 ObjectTy = BaseExpr->getType();
7112 BaseQuals = ObjectTy.getQualifiers();
7113 }
7114
7115 LValue LV =
7116 EmitLValueForIvar(ObjectTy, BaseValue, Ivar: E->getDecl(),
7117 CVRQualifiers: BaseQuals.getCVRQualifiers());
7118 setObjCGCLValueClass(Ctx: getContext(), E, LV);
7119 return LV;
7120}
7121
7122LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {
7123 // Can only get l-value for message expression returning aggregate type
7124 RValue RV = EmitAnyExprToTemp(E);
7125 return MakeAddrLValue(Addr: RV.getAggregateAddress(), T: E->getType(),
7126 Source: AlignmentSource::Decl);
7127}
7128
7129RValue CodeGenFunction::EmitCall(QualType CalleeType,
7130 const CGCallee &OrigCallee, const CallExpr *E,
7131 ReturnValueSlot ReturnValue,
7132 llvm::Value *Chain,
7133 llvm::CallBase **CallOrInvoke,
7134 CGFunctionInfo const **ResolvedFnInfo) {
7135 // Get the actual function type. The callee type will always be a pointer to
7136 // function type or a block pointer type.
7137 assert(CalleeType->isFunctionPointerType() &&
7138 "Call must have function pointer type!");
7139
7140 const Decl *TargetDecl =
7141 OrigCallee.getAbstractInfo().getCalleeDecl().getDecl();
7142
7143 assert((!isa_and_present<FunctionDecl>(TargetDecl) ||
7144 !cast<FunctionDecl>(TargetDecl)->isImmediateFunction()) &&
7145 "trying to emit a call to an immediate function");
7146
7147 CalleeType = getContext().getCanonicalType(T: CalleeType);
7148
7149 auto PointeeType = cast<PointerType>(Val&: CalleeType)->getPointeeType();
7150
7151 CGCallee Callee = OrigCallee;
7152
7153 bool CFIUnchecked = CalleeType->hasPointeeToCFIUncheckedCalleeFunctionType();
7154
7155 if (SanOpts.has(K: SanitizerKind::Function) &&
7156 (!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl)) &&
7157 !isa<FunctionNoProtoType>(Val: PointeeType) && !CFIUnchecked) {
7158 if (llvm::Constant *PrefixSig =
7159 CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM)) {
7160 auto CheckOrdinal = SanitizerKind::SO_Function;
7161 auto CheckHandler = SanitizerHandler::FunctionTypeMismatch;
7162 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
7163 auto *TypeHash = getUBSanFunctionTypeHash(T: PointeeType);
7164
7165 llvm::Type *PrefixSigType = PrefixSig->getType();
7166 llvm::StructType *PrefixStructTy = llvm::StructType::get(
7167 Context&: CGM.getLLVMContext(), Elements: {PrefixSigType, Int32Ty}, /*isPacked=*/true);
7168
7169 llvm::Value *CalleePtr = Callee.getFunctionPointer();
7170 if (CGM.getCodeGenOpts().PointerAuth.FunctionPointers) {
7171 // Use raw pointer since we are using the callee pointer as data here.
7172 Address Addr =
7173 Address(CalleePtr, CalleePtr->getType(),
7174 CharUnits::fromQuantity(
7175 Quantity: CalleePtr->getPointerAlignment(DL: CGM.getDataLayout())),
7176 Callee.getPointerAuthInfo(), nullptr);
7177 CalleePtr = Addr.emitRawPointer(CGF&: *this);
7178 }
7179
7180 // On 32-bit Arm, the low bit of a function pointer indicates whether
7181 // it's using the Arm or Thumb instruction set. The actual first
7182 // instruction lives at the same address either way, so we must clear
7183 // that low bit before using the function address to find the prefix
7184 // structure.
7185 //
7186 // This applies to both Arm and Thumb target triples, because
7187 // either one could be used in an interworking context where it
7188 // might be passed function pointers of both types.
7189 llvm::Value *AlignedCalleePtr;
7190 if (CGM.getTriple().isARM() || CGM.getTriple().isThumb()) {
7191 AlignedCalleePtr = Builder.CreateIntrinsic(
7192 RetTy: CalleePtr->getType(), ID: llvm::Intrinsic::ptrmask,
7193 Args: {CalleePtr, llvm::ConstantInt::getSigned(Ty: IntPtrTy, V: ~1)});
7194 } else {
7195 AlignedCalleePtr = CalleePtr;
7196 }
7197
7198 llvm::Value *CalleePrefixStruct = AlignedCalleePtr;
7199 llvm::Value *CalleeSigPtr =
7200 Builder.CreateConstGEP2_32(Ty: PrefixStructTy, Ptr: CalleePrefixStruct, Idx0: -1, Idx1: 0);
7201 llvm::Value *CalleeSig =
7202 Builder.CreateAlignedLoad(Ty: PrefixSigType, Addr: CalleeSigPtr, Align: getIntAlign());
7203 llvm::Value *CalleeSigMatch = Builder.CreateICmpEQ(LHS: CalleeSig, RHS: PrefixSig);
7204
7205 llvm::BasicBlock *Cont = createBasicBlock(name: "cont");
7206 llvm::BasicBlock *TypeCheck = createBasicBlock(name: "typecheck");
7207 Builder.CreateCondBr(Cond: CalleeSigMatch, True: TypeCheck, False: Cont);
7208
7209 EmitBlock(BB: TypeCheck);
7210 llvm::Value *CalleeTypeHash = Builder.CreateAlignedLoad(
7211 Ty: Int32Ty,
7212 Addr: Builder.CreateConstGEP2_32(Ty: PrefixStructTy, Ptr: CalleePrefixStruct, Idx0: -1, Idx1: 1),
7213 Align: getPointerAlign());
7214 llvm::Value *CalleeTypeHashMatch =
7215 Builder.CreateICmpEQ(LHS: CalleeTypeHash, RHS: TypeHash);
7216 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc: E->getBeginLoc()),
7217 EmitCheckTypeDescriptor(T: CalleeType)};
7218 EmitCheck(Checked: std::make_pair(x&: CalleeTypeHashMatch, y&: CheckOrdinal), CheckHandler,
7219 StaticArgs: StaticData, DynamicArgs: {CalleePtr});
7220
7221 Builder.CreateBr(Dest: Cont);
7222 EmitBlock(BB: Cont);
7223 }
7224 }
7225
7226 const auto *FnType = cast<FunctionType>(Val&: PointeeType);
7227
7228 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
7229 FD && DeviceKernelAttr::isOpenCLSpelling(A: FD->getAttr<DeviceKernelAttr>()))
7230 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FnType);
7231
7232 // If we are checking indirect calls and this call is indirect, check that the
7233 // function pointer is a member of the bit set for the function type.
7234 if (SanOpts.has(K: SanitizerKind::CFIICall) &&
7235 (!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl)) && !CFIUnchecked) {
7236 auto CheckOrdinal = SanitizerKind::SO_CFIICall;
7237 auto CheckHandler = SanitizerHandler::CFICheckFail;
7238 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
7239 EmitSanitizerStatReport(SSK: llvm::SanStat_CFI_ICall);
7240
7241 llvm::Metadata *MD =
7242 CGM.CreateMetadataIdentifierForCFIFnType(T: QualType(FnType, 0));
7243
7244 llvm::Value *TypeId = llvm::MetadataAsValue::get(Context&: getLLVMContext(), MD);
7245
7246 llvm::Value *CalleePtr = Callee.getFunctionPointer();
7247 llvm::Value *TypeTest = Builder.CreateCall(
7248 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::type_test), Args: {CalleePtr, TypeId});
7249
7250 auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
7251 llvm::Constant *StaticData[] = {
7252 llvm::ConstantInt::get(Ty: Int8Ty, V: CFITCK_ICall),
7253 EmitCheckSourceLocation(Loc: E->getBeginLoc()),
7254 EmitCheckTypeDescriptor(T: QualType(FnType, 0)),
7255 };
7256 if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) {
7257 EmitCfiSlowPathCheck(Ordinal: CheckOrdinal, Cond: TypeTest, TypeId: CrossDsoTypeId, Ptr: CalleePtr,
7258 StaticArgs: StaticData);
7259 } else {
7260 EmitCheck(Checked: std::make_pair(x&: TypeTest, y&: CheckOrdinal), CheckHandler,
7261 StaticArgs: StaticData, DynamicArgs: {CalleePtr, llvm::UndefValue::get(T: IntPtrTy)});
7262 }
7263 }
7264
7265 CallArgList Args;
7266 if (Chain)
7267 Args.add(rvalue: RValue::get(V: Chain), type: CGM.getContext().VoidPtrTy);
7268
7269 EvaluationOrder Order = CodeGenUtils::getCallArgEvaluationOrder(E);
7270 bool StaticOperator = false;
7271 if (auto *OCE = dyn_cast<CXXOperatorCallExpr>(Val: E)) {
7272 if (const auto *MD =
7273 dyn_cast_if_present<CXXMethodDecl>(Val: OCE->getCalleeDecl());
7274 MD && MD->isStatic())
7275 StaticOperator = true;
7276 }
7277
7278 auto Arguments = E->arguments();
7279 if (StaticOperator) {
7280 // If we're calling a static operator, we need to emit the object argument
7281 // and ignore it.
7282 EmitIgnoredExpr(E: E->getArg(Arg: 0));
7283 Arguments = drop_begin(RangeOrContainer&: Arguments, N: 1);
7284 }
7285 EmitCallArgs(Args, Prototype: dyn_cast<FunctionProtoType>(Val: FnType), ArgRange: Arguments,
7286 AC: E->getDirectCallee(), /*ParamsToSkip=*/0, Order);
7287
7288 const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(
7289 Args, Ty: FnType, /*ChainCall=*/Chain, ABIInfoFD: getCurrentFunctionDecl());
7290
7291 if (ResolvedFnInfo)
7292 *ResolvedFnInfo = &FnInfo;
7293
7294 // HIP function pointer contains kernel handle when it is used in triple
7295 // chevron. The kernel stub needs to be loaded from kernel handle and used
7296 // as callee.
7297 if (CGM.getLangOpts().HIP && !CGM.getLangOpts().CUDAIsDevice &&
7298 isa<CUDAKernelCallExpr>(Val: E) &&
7299 (!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl))) {
7300 llvm::Value *Handle = Callee.getFunctionPointer();
7301 auto *Stub = Builder.CreateLoad(
7302 Addr: Address(Handle, Handle->getType(), CGM.getPointerAlign()));
7303 Callee.setFunctionPointer(Stub);
7304 }
7305
7306 // Insert function pointer lookup if this is a target call
7307 //
7308 // This is used for the indirect function case, virtual function case is
7309 // handled in ItaniumCXXABI.cpp
7310 if (getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU() &&
7311 (!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl))) {
7312 const Expr *CalleeExpr = E->getCallee()->IgnoreParenImpCasts();
7313 const DeclRefExpr *DRE = nullptr;
7314 while (CalleeExpr) {
7315 if ((DRE = dyn_cast<DeclRefExpr>(Val: CalleeExpr)))
7316 break;
7317 if (const auto *ME = dyn_cast<MemberExpr>(Val: CalleeExpr))
7318 CalleeExpr = ME->getBase()->IgnoreParenImpCasts();
7319 else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: CalleeExpr))
7320 CalleeExpr = ASE->getBase()->IgnoreParenImpCasts();
7321 else
7322 break;
7323 }
7324
7325 const auto *VD = DRE ? dyn_cast<VarDecl>(Val: DRE->getDecl()) : nullptr;
7326 if (VD && VD->hasAttr<OMPTargetIndirectCallAttr>()) {
7327 auto *FuncPtrTy = llvm::PointerType::get(
7328 C&: CGM.getLLVMContext(), AddressSpace: CGM.getDataLayout().getProgramAddressSpace());
7329 llvm::Type *RtlFnArgs[] = {FuncPtrTy};
7330 llvm::FunctionCallee DeviceRtlFn = CGM.CreateRuntimeFunction(
7331 Ty: llvm::FunctionType::get(Result: FuncPtrTy, Params: RtlFnArgs, isVarArg: false),
7332 Name: "__llvm_omp_indirect_call_lookup");
7333 llvm::Value *Func = Callee.getFunctionPointer();
7334 llvm::Type *BackupTy = Func->getType();
7335 Func = Builder.CreatePointerBitCastOrAddrSpaceCast(V: Func, DestTy: FuncPtrTy);
7336 Func = EmitRuntimeCall(callee: DeviceRtlFn, args: {Func});
7337 Func = Builder.CreatePointerBitCastOrAddrSpaceCast(V: Func, DestTy: BackupTy);
7338 Callee.setFunctionPointer(Func);
7339 }
7340 }
7341
7342 llvm::CallBase *LocalCallOrInvoke = nullptr;
7343 RValue Call = EmitCall(CallInfo: FnInfo, Callee, ReturnValue, Args, CallOrInvoke: &LocalCallOrInvoke,
7344 IsMustTail: E == MustTailCall, Loc: E->getExprLoc());
7345
7346 if (auto *CalleeDecl = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
7347 if (CalleeDecl->hasAttr<RestrictAttr>() ||
7348 CalleeDecl->hasAttr<MallocSpanAttr>() ||
7349 CalleeDecl->hasAttr<AllocSizeAttr>()) {
7350 // Function has 'malloc' (aka. 'restrict') or 'alloc_size' attribute.
7351 if (SanOpts.has(K: SanitizerKind::AllocToken)) {
7352 // Set !alloc_token metadata.
7353 EmitAllocToken(CB: LocalCallOrInvoke, E);
7354 }
7355 }
7356 }
7357 if (CallOrInvoke)
7358 *CallOrInvoke = LocalCallOrInvoke;
7359
7360 return Call;
7361}
7362
7363LValue CodeGenFunction::
7364EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) {
7365 Address BaseAddr = Address::invalid();
7366 if (E->getOpcode() == BO_PtrMemI) {
7367 BaseAddr = EmitPointerWithAlignment(E: E->getLHS());
7368 } else {
7369 BaseAddr = EmitLValue(E: E->getLHS()).getAddress();
7370 }
7371
7372 llvm::Value *OffsetV = EmitScalarExpr(E: E->getRHS());
7373 const auto *MPT = E->getRHS()->getType()->castAs<MemberPointerType>();
7374
7375 LValueBaseInfo BaseInfo;
7376 TBAAAccessInfo TBAAInfo;
7377 bool IsInBounds = !getLangOpts().PointerOverflowDefined &&
7378 !isUnderlyingBasePointerConstantNull(E: E->getLHS());
7379 Address MemberAddr = EmitCXXMemberDataPointerAddress(
7380 E, base: BaseAddr, memberPtr: OffsetV, memberPtrType: MPT, IsInBounds, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
7381
7382 return MakeAddrLValue(Addr: MemberAddr, T: MPT->getPointeeType(), BaseInfo, TBAAInfo);
7383}
7384
7385/// Given the address of a temporary variable, produce an r-value of
7386/// its type.
7387RValue CodeGenFunction::convertTempToRValue(Address addr,
7388 QualType type,
7389 SourceLocation loc) {
7390 LValue lvalue = MakeAddrLValue(Addr: addr, T: type, Source: AlignmentSource::Decl);
7391 switch (getEvaluationKind(T: type)) {
7392 case TEK_Complex:
7393 return RValue::getComplex(C: EmitLoadOfComplex(src: lvalue, loc));
7394 case TEK_Aggregate:
7395 return lvalue.asAggregateRValue();
7396 case TEK_Scalar:
7397 return RValue::get(V: EmitLoadOfScalar(lvalue, Loc: loc));
7398 }
7399 llvm_unreachable("bad evaluation kind");
7400}
7401
7402void CodeGenFunction::SetFPAccuracy(llvm::Value *Val, float Accuracy) {
7403 assert(Val->getType()->isFPOrFPVectorTy());
7404 if (Accuracy == 0.0 || !isa<llvm::Instruction>(Val))
7405 return;
7406
7407 llvm::MDBuilder MDHelper(getLLVMContext());
7408 llvm::MDNode *Node = MDHelper.createFPMath(Accuracy);
7409
7410 cast<llvm::Instruction>(Val)->setMetadata(KindID: llvm::LLVMContext::MD_fpmath, Node);
7411}
7412
7413void CodeGenFunction::SetSqrtFPAccuracy(llvm::Value *Val) {
7414 llvm::Type *EltTy = Val->getType()->getScalarType();
7415 if (!EltTy->isFloatTy() && !EltTy->isHalfTy())
7416 return;
7417
7418 if ((getLangOpts().OpenCL &&
7419 !CGM.getCodeGenOpts().OpenCLCorrectlyRoundedDivSqrt) ||
7420 (getLangOpts().HIP && getLangOpts().CUDAIsDevice &&
7421 !CGM.getCodeGenOpts().HIPCorrectlyRoundedDivSqrt)) {
7422 // OpenCL v1.1 s7.4: minimum accuracy of single precision sqrt is 3 ulp.
7423 // OpenCL v3.0 s7.4: minimum accuracy of half precision sqrt is 1.5 ulp.
7424 //
7425 // OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt
7426 // build option allows an application to specify that single precision
7427 // floating-point divide (x/y and 1/x) and sqrt used in the program
7428 // source are correctly rounded.
7429 //
7430 // TODO: CUDA has a prec-sqrt flag
7431 SetFPAccuracy(Val, Accuracy: EltTy->isFloatTy() ? 3.0f : 1.5f);
7432 }
7433}
7434
7435void CodeGenFunction::SetDivFPAccuracy(llvm::Value *Val) {
7436 llvm::Type *EltTy = Val->getType()->getScalarType();
7437 if (!EltTy->isFloatTy() && !EltTy->isHalfTy())
7438 return;
7439
7440 if ((getLangOpts().OpenCL &&
7441 !CGM.getCodeGenOpts().OpenCLCorrectlyRoundedDivSqrt) ||
7442 (getLangOpts().HIP && getLangOpts().CUDAIsDevice &&
7443 !CGM.getCodeGenOpts().HIPCorrectlyRoundedDivSqrt)) {
7444 // OpenCL v1.1 s7.4: minimum accuracy of single precision / is 2.5 ulp.
7445 // OpenCL v3.0 s7.4: minimum accuracy of half precision / is 1 ulp.
7446 //
7447 // OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt
7448 // build option allows an application to specify that single precision
7449 // floating-point divide (x/y and 1/x) and sqrt used in the program
7450 // source are correctly rounded.
7451 //
7452 // TODO: CUDA has a prec-div flag
7453 SetFPAccuracy(Val, Accuracy: EltTy->isFloatTy() ? 2.5f : 1.f);
7454 }
7455}
7456
7457namespace {
7458 struct LValueOrRValue {
7459 LValue LV;
7460 RValue RV;
7461 };
7462}
7463
7464static LValueOrRValue emitPseudoObjectExpr(CodeGenFunction &CGF,
7465 const PseudoObjectExpr *E,
7466 bool forLValue,
7467 AggValueSlot slot) {
7468 SmallVector<CodeGenFunction::OpaqueValueMappingData, 4> opaques;
7469
7470 // Find the result expression, if any.
7471 const Expr *resultExpr = E->getResultExpr();
7472 LValueOrRValue result;
7473
7474 // Without debug info, e.g., in a nodebug function, an enumerator of a BPF
7475 // preserve_access_index enum cannot be relocated, so use its value.
7476 if (!forLValue && CGF.getTarget().getTriple().isBPF() &&
7477 (!CGF.CGM.getCodeGenOpts().hasReducedDebugInfo() || !CGF.getDebugInfo()))
7478 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: E->getSyntacticForm());
7479 DRE && isa<EnumConstantDecl>(Val: DRE->getDecl())) {
7480 result.RV = RValue::get(V: CGF.EmitScalarExpr(E: DRE));
7481 return result;
7482 }
7483
7484 for (PseudoObjectExpr::const_semantics_iterator
7485 i = E->semantics_begin(), e = E->semantics_end(); i != e; ++i) {
7486 const Expr *semantic = *i;
7487
7488 // If this semantic expression is an opaque value, bind it
7489 // to the result of its source expression.
7490 if (const auto *ov = dyn_cast<OpaqueValueExpr>(Val: semantic)) {
7491 // Skip unique OVEs.
7492 if (ov->isUnique()) {
7493 assert(ov != resultExpr &&
7494 "A unique OVE cannot be used as the result expression");
7495 continue;
7496 }
7497
7498 // If this is the result expression, we may need to evaluate
7499 // directly into the slot.
7500 typedef CodeGenFunction::OpaqueValueMappingData OVMA;
7501 OVMA opaqueData;
7502 if (ov == resultExpr && ov->isPRValue() && !forLValue &&
7503 CodeGenFunction::hasAggregateEvaluationKind(T: ov->getType())) {
7504 CGF.EmitAggExpr(E: ov->getSourceExpr(), AS: slot);
7505 LValue LV = CGF.MakeAddrLValue(Addr: slot.getAddress(), T: ov->getType(),
7506 Source: AlignmentSource::Decl);
7507 opaqueData = OVMA::bind(CGF, ov, lv: LV);
7508 result.RV = slot.asRValue();
7509
7510 // Otherwise, emit as normal.
7511 } else {
7512 opaqueData = OVMA::bind(CGF, ov, e: ov->getSourceExpr());
7513
7514 // If this is the result, also evaluate the result now.
7515 if (ov == resultExpr) {
7516 if (forLValue)
7517 result.LV = CGF.EmitLValue(E: ov);
7518 else
7519 result.RV = CGF.EmitAnyExpr(E: ov, aggSlot: slot);
7520 }
7521 }
7522
7523 opaques.push_back(Elt: opaqueData);
7524
7525 // Otherwise, if the expression is the result, evaluate it
7526 // and remember the result.
7527 } else if (semantic == resultExpr) {
7528 if (forLValue)
7529 result.LV = CGF.EmitLValue(E: semantic);
7530 else
7531 result.RV = CGF.EmitAnyExpr(E: semantic, aggSlot: slot);
7532
7533 // Otherwise, evaluate the expression in an ignored context.
7534 } else {
7535 CGF.EmitIgnoredExpr(E: semantic);
7536 }
7537 }
7538
7539 // Unbind all the opaques now.
7540 for (CodeGenFunction::OpaqueValueMappingData &opaque : opaques)
7541 opaque.unbind(CGF);
7542
7543 return result;
7544}
7545
7546RValue CodeGenFunction::EmitPseudoObjectRValue(const PseudoObjectExpr *E,
7547 AggValueSlot slot) {
7548 return emitPseudoObjectExpr(CGF&: *this, E, forLValue: false, slot).RV;
7549}
7550
7551LValue CodeGenFunction::EmitPseudoObjectLValue(const PseudoObjectExpr *E) {
7552 return emitPseudoObjectExpr(CGF&: *this, E, forLValue: true, slot: AggValueSlot::ignored()).LV;
7553}
7554
7555void CodeGenFunction::FlattenAccessAndTypeLValue(
7556 LValue Val, SmallVectorImpl<LValue> &AccessList) {
7557
7558 llvm::SmallVector<
7559 std::tuple<LValue, QualType, llvm::SmallVector<llvm::Value *, 4>>, 16>
7560 WorkList;
7561 llvm::IntegerType *IdxTy = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: 32);
7562 WorkList.push_back(Elt: {Val, Val.getType(), {llvm::ConstantInt::get(Ty: IdxTy, V: 0)}});
7563
7564 while (!WorkList.empty()) {
7565 auto [LVal, T, IdxList] = WorkList.pop_back_val();
7566 T = T.getCanonicalType().getUnqualifiedType();
7567 if (const auto *CAT = dyn_cast<ConstantArrayType>(Val&: T)) {
7568 uint64_t Size = CAT->getZExtSize();
7569 for (int64_t I = Size - 1; I > -1; I--) {
7570 llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList;
7571 IdxListCopy.push_back(Elt: llvm::ConstantInt::get(Ty: IdxTy, V: I));
7572 WorkList.emplace_back(Args&: LVal, Args: CAT->getElementType(), Args&: IdxListCopy);
7573 }
7574 } else if (const auto *RT = dyn_cast<RecordType>(Val&: T)) {
7575 const RecordDecl *Record = RT->getDecl()->getDefinitionOrSelf();
7576 assert(!Record->isUnion() && "Union types not supported in flat cast.");
7577
7578 const CXXRecordDecl *CXXD = dyn_cast<CXXRecordDecl>(Val: Record);
7579
7580 llvm::SmallVector<
7581 std::tuple<LValue, QualType, llvm::SmallVector<llvm::Value *, 4>>, 16>
7582 ReverseList;
7583 if (CXXD && CXXD->isStandardLayout())
7584 Record = CXXD->getStandardLayoutBaseWithFields();
7585
7586 // deal with potential base classes
7587 if (CXXD && !CXXD->isStandardLayout()) {
7588 if (CXXD->getNumBases() > 0) {
7589 assert(CXXD->getNumBases() == 1 &&
7590 "HLSL doesn't support multiple inheritance.");
7591 auto Base = CXXD->bases_begin();
7592 llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList;
7593 IdxListCopy.push_back(Elt: llvm::ConstantInt::get(
7594 Ty: IdxTy, V: 0)); // base struct should be at index zero
7595 ReverseList.emplace_back(Args&: LVal, Args: Base->getType(), Args&: IdxListCopy);
7596 }
7597 }
7598
7599 const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(Record);
7600
7601 llvm::Type *LLVMT = ConvertTypeForMem(T);
7602 CharUnits Align = getContext().getTypeAlignInChars(T);
7603 LValue RLValue;
7604 bool createdGEP = false;
7605 for (auto *FD : Record->fields()) {
7606 if (FD->isBitField()) {
7607 if (FD->isUnnamedBitField())
7608 continue;
7609 if (!createdGEP) {
7610 createdGEP = true;
7611 Address GEP = Builder.CreateInBoundsGEP(Addr: LVal.getAddress(), IdxList,
7612 ElementType: LLVMT, Align, Name: "gep");
7613 RLValue = MakeAddrLValue(Addr: GEP, T);
7614 }
7615 LValue FieldLVal = EmitLValueForField(base: RLValue, field: FD, IsInBounds: true);
7616 ReverseList.push_back(Elt: {FieldLVal, FD->getType(), {}});
7617 } else {
7618 llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList;
7619 IdxListCopy.push_back(
7620 Elt: llvm::ConstantInt::get(Ty: IdxTy, V: Layout.getLLVMFieldNo(FD)));
7621 ReverseList.emplace_back(Args&: LVal, Args: FD->getType(), Args&: IdxListCopy);
7622 }
7623 }
7624
7625 std::reverse(first: ReverseList.begin(), last: ReverseList.end());
7626 llvm::append_range(C&: WorkList, R&: ReverseList);
7627 } else if (const auto *VT = dyn_cast<VectorType>(Val&: T)) {
7628 llvm::Type *LLVMT = ConvertTypeForMem(T);
7629 CharUnits Align = getContext().getTypeAlignInChars(T);
7630 Address GEP = Builder.CreateInBoundsGEP(Addr: LVal.getAddress(), IdxList, ElementType: LLVMT,
7631 Align, Name: "vector.gep");
7632 LValue Base = MakeAddrLValue(Addr: GEP, T);
7633 for (unsigned I = 0, E = VT->getNumElements(); I < E; I++) {
7634 llvm::Constant *Idx = llvm::ConstantInt::get(Ty: IdxTy, V: I);
7635 LValue LV =
7636 LValue::MakeVectorElt(vecAddress: Base.getAddress(), Idx, type: VT->getElementType(),
7637 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
7638 AccessList.emplace_back(Args&: LV);
7639 }
7640 } else if (const auto *MT = dyn_cast<ConstantMatrixType>(Val&: T)) {
7641 // Matrices are represented as flat arrays in memory, but has a vector
7642 // value type. So we use ConvertMatrixAddress to convert the address from
7643 // array to vector, and extract elements similar to the vector case above.
7644 // The matrix elements are iterated over in row-major order regardless of
7645 // the memory layout of the matrix.
7646 llvm::Type *LLVMT = ConvertTypeForMem(T);
7647 CharUnits Align = getContext().getTypeAlignInChars(T);
7648 Address GEP = Builder.CreateInBoundsGEP(Addr: LVal.getAddress(), IdxList, ElementType: LLVMT,
7649 Align, Name: "matrix.gep");
7650 LValue Base = MakeAddrLValue(Addr: GEP, T);
7651 Address MatAddr = MaybeConvertMatrixAddress(Addr: Base.getAddress(), CGF&: *this);
7652 unsigned NumRows = MT->getNumRows();
7653 unsigned NumCols = MT->getNumColumns();
7654 bool IsMatrixRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T);
7655 llvm::MatrixBuilder MB(Builder);
7656 for (unsigned Row = 0; Row < MT->getNumRows(); Row++) {
7657 for (unsigned Col = 0; Col < MT->getNumColumns(); Col++) {
7658 llvm::Value *RowIdx = llvm::ConstantInt::get(Ty: IdxTy, V: Row);
7659 llvm::Value *ColIdx = llvm::ConstantInt::get(Ty: IdxTy, V: Col);
7660 llvm::Value *Idx = MB.CreateIndex(RowIdx, ColumnIdx: ColIdx, NumRows, NumCols,
7661 IsMatrixRowMajor);
7662 LValue LV =
7663 LValue::MakeMatrixElt(matAddress: MatAddr, Idx, type: MT->getElementType(),
7664 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
7665 AccessList.emplace_back(Args&: LV);
7666 }
7667 }
7668 } else { // a scalar/builtin type
7669 if (!IdxList.empty()) {
7670 llvm::Type *LLVMT = ConvertTypeForMem(T);
7671 CharUnits Align = getContext().getTypeAlignInChars(T);
7672 Address GEP = Builder.CreateInBoundsGEP(Addr: LVal.getAddress(), IdxList,
7673 ElementType: LLVMT, Align, Name: "gep");
7674 AccessList.emplace_back(Args: MakeAddrLValue(Addr: GEP, T));
7675 } else // must be a bitfield we already created an lvalue for
7676 AccessList.emplace_back(Args&: LVal);
7677 }
7678 }
7679}
7680