1//===--- CGExpr.cpp - Emit LLVM Code from Expressions ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Expr nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ABIInfoImpl.h"
14#include "CGCUDARuntime.h"
15#include "CGCXXABI.h"
16#include "CGCall.h"
17#include "CGCleanup.h"
18#include "CGDebugInfo.h"
19#include "CGHLSLRuntime.h"
20#include "CGObjCRuntime.h"
21#include "CGOpenMPRuntime.h"
22#include "CGRecordLayout.h"
23#include "CodeGenFunction.h"
24#include "CodeGenModule.h"
25#include "CodeGenPGO.h"
26#include "ConstantEmitter.h"
27#include "TargetInfo.h"
28#include "clang/AST/ASTContext.h"
29#include "clang/AST/ASTLambda.h"
30#include "clang/AST/Attr.h"
31#include "clang/AST/DeclObjC.h"
32#include "clang/AST/Expr.h"
33#include "clang/AST/InferAlloc.h"
34#include "clang/AST/MatrixUtils.h"
35#include "clang/AST/NSAPI.h"
36#include "clang/AST/ParentMapContext.h"
37#include "clang/AST/StmtVisitor.h"
38#include "clang/Basic/Builtins.h"
39#include "clang/Basic/CodeGenOptions.h"
40#include "clang/Basic/Module.h"
41#include "clang/Basic/SourceManager.h"
42#include "clang/CodeGenUtils/CodeGenUtils.h"
43#include "llvm/ADT/STLExtras.h"
44#include "llvm/ADT/ScopeExit.h"
45#include "llvm/ADT/StringExtras.h"
46#include "llvm/IR/Constants.h"
47#include "llvm/IR/DataLayout.h"
48#include "llvm/IR/Intrinsics.h"
49#include "llvm/IR/IntrinsicsWebAssembly.h"
50#include "llvm/IR/LLVMContext.h"
51#include "llvm/IR/MDBuilder.h"
52#include "llvm/IR/MatrixBuilder.h"
53#include "llvm/Support/ConvertUTF.h"
54#include "llvm/Support/Endian.h"
55#include "llvm/Support/MathExtras.h"
56#include "llvm/Support/Path.h"
57#include "llvm/Support/xxhash.h"
58#include "llvm/Transforms/Utils/SanitizerStats.h"
59
60#include <numeric>
61#include <optional>
62#include <string>
63
64using namespace clang;
65using namespace CodeGen;
66
67namespace clang {
68// TODO: consider deprecating ClSanitizeGuardChecks; functionality is subsumed
69// by -fsanitize-skip-hot-cutoff
70llvm::cl::opt<bool> ClSanitizeGuardChecks(
71 "ubsan-guard-checks", llvm::cl::Optional,
72 llvm::cl::desc("Guard UBSAN checks with `llvm.allow.ubsan.check()`."));
73
74} // namespace clang
75
76//===--------------------------------------------------------------------===//
77// Defines for metadata
78//===--------------------------------------------------------------------===//
79
80// Those values are crucial to be the SAME as in ubsan runtime library.
81enum VariableTypeDescriptorKind : uint16_t {
82 /// An integer type.
83 TK_Integer = 0x0000,
84 /// A floating-point type.
85 TK_Float = 0x0001,
86 /// An _BitInt(N) type.
87 TK_BitInt = 0x0002,
88 /// Any other type. The value representation is unspecified.
89 TK_Unknown = 0xffff
90};
91
92//===--------------------------------------------------------------------===//
93// Miscellaneous Helper Methods
94//===--------------------------------------------------------------------===//
95
96static llvm::StringRef GetUBSanTrapForHandler(SanitizerHandler ID) {
97 switch (ID) {
98#define SANITIZER_CHECK(Enum, Name, Version, Msg) \
99 case SanitizerHandler::Enum: \
100 return Msg;
101 LIST_SANITIZER_CHECKS
102#undef SANITIZER_CHECK
103 }
104 llvm_unreachable("unhandled switch case");
105}
106
107/// CreateTempAlloca - This creates a alloca and inserts it into the entry
108/// block.
109RawAddress
110CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits Align,
111 const Twine &Name,
112 llvm::Value *ArraySize) {
113 if (getLangOpts().EmitLogicalPointer) {
114 auto Alloca = Builder.CreateStructuredAlloca(BaseType: Ty, Name);
115 return RawAddress(Alloca, Ty, Align, KnownNonNull);
116 }
117
118 auto *Alloca = CreateTempAlloca(Ty, Name, ArraySize);
119 Alloca->setAlignment(Align.getAsAlign());
120 return RawAddress(Alloca, Ty, Align, KnownNonNull);
121}
122
123RawAddress CodeGenFunction::MaybeCastStackAddressSpace(RawAddress Alloca,
124 LangAS DestLangAS,
125 llvm::Value *ArraySize) {
126
127 llvm::Value *V = Alloca.getPointer();
128 // Alloca always returns a pointer in alloca address space, which may
129 // be different from the type defined by the language. For example,
130 // in C++ the auto variables are in the default address space. Therefore
131 // cast alloca to the default address space when necessary.
132
133 unsigned DestAddrSpace = getContext().getTargetAddressSpace(AS: DestLangAS);
134 if (DestAddrSpace != Alloca.getAddressSpace()) {
135 llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
136 // When ArraySize is nullptr, alloca is inserted at AllocaInsertPt,
137 // otherwise alloca is inserted at the current insertion point of the
138 // builder.
139 if (!ArraySize)
140 Builder.SetInsertPoint(getPostAllocaInsertPoint());
141 V = performAddrSpaceCast(Src: V, DestTy: Builder.getPtrTy(AddrSpace: DestAddrSpace));
142 }
143
144 return RawAddress(V, Alloca.getElementType(), Alloca.getAlignment(),
145 KnownNonNull);
146}
147
148RawAddress CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, LangAS DestLangAS,
149 CharUnits Align, const Twine &Name,
150 llvm::Value *ArraySize,
151 RawAddress *AllocaAddr) {
152 RawAddress Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize);
153 if (AllocaAddr)
154 *AllocaAddr = Alloca;
155 return MaybeCastStackAddressSpace(Alloca, DestLangAS, ArraySize);
156}
157
158/// CreateTempAlloca - This creates an alloca and inserts it into the entry
159/// block if \p ArraySize is nullptr, otherwise inserts it at the current
160/// insertion point of the builder.
161llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
162 const Twine &Name,
163 llvm::Value *ArraySize) {
164 llvm::AllocaInst *Alloca;
165 if (ArraySize)
166 Alloca = Builder.CreateAlloca(Ty, ArraySize, Name);
167 else
168 Alloca =
169 new llvm::AllocaInst(Ty, CGM.getDataLayout().getAllocaAddrSpace(),
170 ArraySize, Name, AllocaInsertPt->getIterator());
171 if (SanOpts.Mask & SanitizerKind::Address) {
172 Alloca->addAnnotationMetadata(Annotations: {"alloca_name_altered", Name.str()});
173 }
174 if (Allocas) {
175 Allocas->Add(I: Alloca);
176 }
177 return Alloca;
178}
179
180/// CreateDefaultAlignTempAlloca - This creates an alloca with the
181/// default alignment of the corresponding LLVM type, which is *not*
182/// guaranteed to be related in any way to the expected alignment of
183/// an AST type that might have been lowered to Ty.
184RawAddress CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty,
185 const Twine &Name) {
186 CharUnits Align =
187 CharUnits::fromQuantity(Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty));
188 return CreateTempAlloca(Ty, DestLangAS: LangAS::Default, Align, Name);
189}
190
191RawAddress CodeGenFunction::CreateIRTempWithoutCast(QualType Ty,
192 const Twine &Name) {
193 CharUnits Align = getContext().getTypeAlignInChars(T: Ty);
194 return CreateTempAllocaWithoutCast(Ty: ConvertType(T: Ty), Align, Name, ArraySize: nullptr);
195}
196
197RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name,
198 RawAddress *Alloca) {
199 // FIXME: Should we prefer the preferred type alignment here?
200 return CreateMemTemp(T: Ty, Align: getContext().getTypeAlignInChars(T: Ty), Name, Alloca);
201}
202
203RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align,
204 const Twine &Name,
205 RawAddress *Alloca) {
206 RawAddress Result =
207 CreateTempAlloca(Ty: ConvertTypeForMem(T: Ty), DestLangAS: Ty.getAddressSpace(), Align, Name,
208 /*ArraySize=*/nullptr, AllocaAddr: Alloca);
209
210 if (Ty->isConstantMatrixType()) {
211 auto *ArrayTy = cast<llvm::ArrayType>(Val: Result.getElementType());
212 auto *ArrayElementTy = ArrayTy->getElementType();
213 auto ArrayElements = ArrayTy->getNumElements();
214 if (getContext().getLangOpts().HLSL) {
215 auto *VectorTy = cast<llvm::FixedVectorType>(Val: ArrayElementTy);
216 ArrayElementTy = VectorTy->getElementType();
217 ArrayElements *= VectorTy->getNumElements();
218 }
219 auto *VectorTy = llvm::FixedVectorType::get(ElementType: ArrayElementTy, NumElts: ArrayElements);
220
221 Result = Address(Result.getPointer(), VectorTy, Result.getAlignment(),
222 KnownNonNull);
223 }
224 return Result;
225}
226
227RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
228 CharUnits Align,
229 const Twine &Name) {
230 return CreateTempAllocaWithoutCast(Ty: ConvertTypeForMem(T: Ty), Align, Name);
231}
232
233RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
234 const Twine &Name) {
235 return CreateMemTempWithoutCast(Ty, Align: getContext().getTypeAlignInChars(T: Ty),
236 Name);
237}
238
239/// EvaluateExprAsBool - Perform the usual unary conversions on the specified
240/// expression and compare the result against zero, returning an Int1Ty value.
241llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
242 PGO->setCurrentStmt(E);
243 if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {
244 llvm::Value *MemPtr = EmitScalarExpr(E);
245 return CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF&: *this, MemPtr, MPT);
246 }
247
248 QualType BoolTy = getContext().BoolTy;
249 SourceLocation Loc = E->getExprLoc();
250 CGFPOptionsRAII FPOptsRAII(*this, E);
251 if (!E->getType()->isAnyComplexType())
252 return EmitScalarConversion(Src: EmitScalarExpr(E), SrcTy: E->getType(), DstTy: BoolTy, Loc);
253
254 return EmitComplexToScalarConversion(Src: EmitComplexExpr(E), SrcTy: E->getType(), DstTy: BoolTy,
255 Loc);
256}
257
258/// EmitIgnoredExpr - Emit code to compute the specified expression,
259/// ignoring the result.
260void CodeGenFunction::EmitIgnoredExpr(const Expr *E) {
261 if (E->isPRValue())
262 return (void)EmitAnyExpr(E, aggSlot: AggValueSlot::ignored(), ignoreResult: true);
263
264 // if this is a bitfield-resulting conditional operator, we can special case
265 // emit this. The normal 'EmitLValue' version of this is particularly
266 // difficult to codegen for, since creating a single "LValue" for two
267 // different sized arguments here is not particularly doable.
268 if (const auto *CondOp = dyn_cast<AbstractConditionalOperator>(
269 Val: E->IgnoreParenNoopCasts(Ctx: getContext()))) {
270 if (CondOp->getObjectKind() == OK_BitField)
271 return EmitIgnoredConditionalOperator(E: CondOp);
272 }
273
274 // Just emit it as an l-value and drop the result.
275 EmitLValue(E);
276}
277
278/// EmitAnyExpr - Emit code to compute the specified expression which
279/// can have any type. The result is returned as an RValue struct.
280/// If this is an aggregate expression, AggSlot indicates where the
281/// result should be returned.
282RValue CodeGenFunction::EmitAnyExpr(const Expr *E,
283 AggValueSlot aggSlot,
284 bool ignoreResult) {
285 switch (getEvaluationKind(T: E->getType())) {
286 case TEK_Scalar:
287 return RValue::get(V: EmitScalarExpr(E, IgnoreResultAssign: ignoreResult));
288 case TEK_Complex:
289 return RValue::getComplex(C: EmitComplexExpr(E, IgnoreReal: ignoreResult, IgnoreImag: ignoreResult));
290 case TEK_Aggregate:
291 if (!ignoreResult && aggSlot.isIgnored())
292 aggSlot = CreateAggTemp(T: E->getType().getUnqualifiedType(), Name: "agg-temp");
293 EmitAggExpr(E, AS: aggSlot);
294 return aggSlot.asRValue();
295 }
296 llvm_unreachable("bad evaluation kind");
297}
298
299/// EmitAnyExprToTemp - Similar to EmitAnyExpr(), however, the result will
300/// always be accessible even if no aggregate location is provided.
301RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E) {
302 AggValueSlot AggSlot = AggValueSlot::ignored();
303
304 if (hasAggregateEvaluationKind(T: E->getType()))
305 AggSlot = CreateAggTemp(T: E->getType(), Name: "agg.tmp");
306 return EmitAnyExpr(E, aggSlot: AggSlot);
307}
308
309/// EmitAnyExprToMem - Evaluate an expression into a given memory
310/// location.
311void CodeGenFunction::EmitAnyExprToMem(const Expr *E,
312 Address Location,
313 Qualifiers Quals,
314 bool IsInit) {
315 // FIXME: This function should take an LValue as an argument.
316 switch (getEvaluationKind(T: E->getType())) {
317 case TEK_Complex:
318 EmitComplexExprIntoLValue(E, dest: MakeAddrLValue(Addr: Location, T: E->getType()),
319 /*isInit*/ false);
320 return;
321
322 case TEK_Aggregate: {
323 EmitAggExpr(E, AS: AggValueSlot::forAddr(addr: Location, quals: Quals,
324 isDestructed: AggValueSlot::IsDestructed_t(IsInit),
325 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
326 isAliased: AggValueSlot::IsAliased_t(!IsInit),
327 mayOverlap: AggValueSlot::MayOverlap));
328 return;
329 }
330
331 case TEK_Scalar: {
332 RValue RV = RValue::get(V: EmitScalarExpr(E, /*Ignore*/ IgnoreResultAssign: false));
333 LValue LV = MakeAddrLValue(Addr: Location, T: E->getType());
334 EmitStoreThroughLValue(Src: RV, Dst: LV);
335 return;
336 }
337 }
338 llvm_unreachable("bad evaluation kind");
339}
340
341void CodeGenFunction::EmitInitializationToLValue(
342 const Expr *E, LValue LV, AggValueSlot::IsZeroed_t IsZeroed) {
343 QualType Type = LV.getType();
344 switch (getEvaluationKind(T: Type)) {
345 case TEK_Complex:
346 EmitComplexExprIntoLValue(E, dest: LV, /*isInit*/ true);
347 return;
348 case TEK_Aggregate:
349 EmitAggExpr(E, AS: AggValueSlot::forLValue(LV, isDestructed: AggValueSlot::IsDestructed,
350 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
351 isAliased: AggValueSlot::IsNotAliased,
352 mayOverlap: AggValueSlot::MayOverlap, isZeroed: IsZeroed));
353 return;
354 case TEK_Scalar:
355 if (LV.isSimple())
356 EmitScalarInit(init: E, /*D=*/nullptr, lvalue: LV, /*Captured=*/capturedByInit: false);
357 else
358 EmitStoreThroughLValue(Src: RValue::get(V: EmitScalarExpr(E)), Dst: LV);
359 return;
360 }
361 llvm_unreachable("bad evaluation kind");
362}
363
364static void
365pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M,
366 const Expr *E, Address ReferenceTemporary) {
367 // Objective-C++ ARC:
368 // If we are binding a reference to a temporary that has ownership, we
369 // need to perform retain/release operations on the temporary.
370 //
371 // FIXME: This should be looking at E, not M.
372 if (auto Lifetime = M->getType().getObjCLifetime()) {
373 switch (Lifetime) {
374 case Qualifiers::OCL_None:
375 case Qualifiers::OCL_ExplicitNone:
376 // Carry on to normal cleanup handling.
377 break;
378
379 case Qualifiers::OCL_Autoreleasing:
380 // Nothing to do; cleaned up by an autorelease pool.
381 return;
382
383 case Qualifiers::OCL_Strong:
384 case Qualifiers::OCL_Weak:
385 switch (StorageDuration Duration = M->getStorageDuration()) {
386 case SD_Static:
387 // Note: we intentionally do not register a cleanup to release
388 // the object on program termination.
389 return;
390
391 case SD_Thread:
392 // FIXME: We should probably register a cleanup in this case.
393 return;
394
395 case SD_Automatic:
396 case SD_FullExpression:
397 CodeGenFunction::Destroyer *Destroy;
398 CleanupKind CleanupKind;
399 if (Lifetime == Qualifiers::OCL_Strong) {
400 const ValueDecl *VD = M->getExtendingDecl();
401 bool Precise = isa_and_nonnull<VarDecl>(Val: VD) &&
402 VD->hasAttr<ObjCPreciseLifetimeAttr>();
403 CleanupKind = CGF.getARCCleanupKind();
404 Destroy = Precise ? &CodeGenFunction::destroyARCStrongPrecise
405 : &CodeGenFunction::destroyARCStrongImprecise;
406 } else {
407 // __weak objects always get EH cleanups; otherwise, exceptions
408 // could cause really nasty crashes instead of mere leaks.
409 CleanupKind = NormalAndEHCleanup;
410 Destroy = &CodeGenFunction::destroyARCWeak;
411 }
412 if (Duration == SD_FullExpression)
413 CGF.pushDestroy(kind: CleanupKind, addr: ReferenceTemporary,
414 type: M->getType(), destroyer: *Destroy,
415 useEHCleanupForArray: CleanupKind & EHCleanup);
416 else
417 CGF.pushLifetimeExtendedDestroy(kind: CleanupKind, addr: ReferenceTemporary,
418 type: M->getType(),
419 destroyer: *Destroy, useEHCleanupForArray: CleanupKind & EHCleanup);
420 return;
421
422 case SD_Dynamic:
423 llvm_unreachable("temporary cannot have dynamic storage duration");
424 }
425 llvm_unreachable("unknown storage duration");
426 }
427 }
428
429 QualType::DestructionKind DK = E->getType().isDestructedType();
430 if (DK != QualType::DK_none) {
431 switch (M->getStorageDuration()) {
432 case SD_Static:
433 case SD_Thread: {
434 CXXDestructorDecl *ReferenceTemporaryDtor = nullptr;
435 if (const auto *ClassDecl =
436 E->getType()->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
437 ClassDecl && !ClassDecl->hasTrivialDestructor())
438 // Get the destructor for the reference temporary.
439 ReferenceTemporaryDtor = ClassDecl->getDestructor();
440
441 if (!ReferenceTemporaryDtor)
442 return;
443
444 llvm::FunctionCallee CleanupFn;
445 llvm::Constant *CleanupArg;
446 if (E->getType()->isArrayType()) {
447 CleanupFn = CodeGenFunction(CGF.CGM).generateDestroyHelper(
448 addr: ReferenceTemporary, type: E->getType(), destroyer: CodeGenFunction::destroyCXXObject,
449 useEHCleanupForArray: CGF.getLangOpts().Exceptions,
450 VD: dyn_cast_or_null<VarDecl>(Val: M->getExtendingDecl()));
451 CleanupArg = llvm::Constant::getNullValue(Ty: CGF.Int8PtrTy);
452 } else {
453 CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor(
454 GD: GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete));
455 CleanupArg =
456 cast<llvm::Constant>(Val: ReferenceTemporary.emitRawPointer(CGF));
457 }
458 CGF.CGM.getCXXABI().registerGlobalDtor(
459 CGF, D: *cast<VarDecl>(Val: M->getExtendingDecl()), Dtor: CleanupFn, Addr: CleanupArg);
460 } break;
461 case SD_FullExpression:
462 CGF.pushDestroy(dtorKind: DK, addr: ReferenceTemporary, type: E->getType());
463 break;
464 case SD_Automatic:
465 CGF.pushLifetimeExtendedDestroy(dtorKind: DK, addr: ReferenceTemporary, type: E->getType());
466 break;
467 case SD_Dynamic:
468 llvm_unreachable("temporary cannot have dynamic storage duration");
469 }
470 }
471}
472
473static RawAddress createReferenceTemporary(CodeGenFunction &CGF,
474 const MaterializeTemporaryExpr *M,
475 const Expr *Inner,
476 RawAddress *Alloca = nullptr) {
477 switch (M->getStorageDuration()) {
478 case SD_FullExpression:
479 case SD_Automatic: {
480 // If we have a constant temporary array or record try to promote it into a
481 // constant global under the same rules a normal constant would've been
482 // promoted. This is easier on the optimizer and generally emits fewer
483 // instructions.
484 QualType Ty = Inner->getType();
485 if (CGF.CGM.getCodeGenOpts().MergeAllConstants &&
486 (Ty->isArrayType() || Ty->isRecordType()) &&
487 Ty.isConstantStorage(Ctx: CGF.getContext(), ExcludeCtor: true, ExcludeDtor: false))
488 if (auto Init = ConstantEmitter(CGF).tryEmitAbstract(E: Inner, T: Ty)) {
489 auto AS = CGF.CGM.GetGlobalConstantAddressSpace();
490 auto *GV = new llvm::GlobalVariable(
491 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
492 llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp", nullptr,
493 llvm::GlobalValue::NotThreadLocal,
494 CGF.getContext().getTargetAddressSpace(AS));
495 CharUnits alignment = CGF.getContext().getTypeAlignInChars(T: Ty);
496 GV->setAlignment(alignment.getAsAlign());
497 llvm::Constant *C = GV;
498 if (AS != Ty.getAddressSpace())
499 C = CGF.CGM.performAddrSpaceCast(
500 Src: GV, DestTy: llvm::PointerType::get(C&: CGF.getLLVMContext(),
501 AddressSpace: CGF.getContext().getTargetAddressSpace(
502 AS: Ty.getAddressSpace())));
503 // FIXME: Should we put the new global into a COMDAT?
504 return RawAddress(C, GV->getValueType(), alignment);
505 }
506 return CGF.CreateMemTemp(Ty, Name: "ref.tmp", Alloca);
507 }
508 case SD_Thread:
509 case SD_Static:
510 return CGF.CGM.GetAddrOfGlobalTemporary(E: M, Inner);
511
512 case SD_Dynamic:
513 llvm_unreachable("temporary can't have dynamic storage duration");
514 }
515 llvm_unreachable("unknown storage duration");
516}
517
518LValue CodeGenFunction::
519EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
520 const Expr *E = M->getSubExpr();
521
522 assert((!M->getExtendingDecl() || !isa<VarDecl>(M->getExtendingDecl()) ||
523 !cast<VarDecl>(M->getExtendingDecl())->isARCPseudoStrong()) &&
524 "Reference should never be pseudo-strong!");
525
526 // FIXME: ideally this would use EmitAnyExprToMem, however, we cannot do so
527 // as that will cause the lifetime adjustment to be lost for ARC
528 auto ownership = M->getType().getObjCLifetime();
529 if (ownership != Qualifiers::OCL_None &&
530 ownership != Qualifiers::OCL_ExplicitNone) {
531 RawAddress Object = createReferenceTemporary(CGF&: *this, M, Inner: E);
532 if (auto *Var = dyn_cast<llvm::GlobalVariable>(Val: Object.getPointer())) {
533 llvm::Type *Ty = ConvertTypeForMem(T: E->getType());
534 Object = Object.withElementType(ElemTy: Ty);
535
536 // createReferenceTemporary will promote the temporary to a global with a
537 // constant initializer if it can. It can only do this to a value of
538 // ARC-manageable type if the value is global and therefore "immune" to
539 // ref-counting operations. Therefore we have no need to emit either a
540 // dynamic initialization or a cleanup and we can just return the address
541 // of the temporary.
542 if (Var->hasInitializer())
543 return MakeAddrLValue(Addr: Object, T: M->getType(), Source: AlignmentSource::Decl);
544
545 Var->setInitializer(CGM.EmitNullConstant(T: E->getType()));
546 }
547 LValue RefTempDst = MakeAddrLValue(Addr: Object, T: M->getType(),
548 Source: AlignmentSource::Decl);
549
550 switch (getEvaluationKind(T: E->getType())) {
551 default: llvm_unreachable("expected scalar or aggregate expression");
552 case TEK_Scalar:
553 EmitScalarInit(init: E, D: M->getExtendingDecl(), lvalue: RefTempDst, capturedByInit: false);
554 break;
555 case TEK_Aggregate: {
556 EmitAggExpr(E, AS: AggValueSlot::forAddr(addr: Object,
557 quals: E->getType().getQualifiers(),
558 isDestructed: AggValueSlot::IsDestructed,
559 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
560 isAliased: AggValueSlot::IsNotAliased,
561 mayOverlap: AggValueSlot::DoesNotOverlap));
562 break;
563 }
564 }
565
566 pushTemporaryCleanup(CGF&: *this, M, E, ReferenceTemporary: Object);
567 return RefTempDst;
568 }
569
570 SmallVector<const Expr *, 2> CommaLHSs;
571 SmallVector<SubobjectAdjustment, 2> Adjustments;
572 E = E->skipRValueSubobjectAdjustments(CommaLHS&: CommaLHSs, Adjustments);
573
574 for (const auto &Ignored : CommaLHSs)
575 EmitIgnoredExpr(E: Ignored);
576
577 if (const auto *opaque = dyn_cast<OpaqueValueExpr>(Val: E)) {
578 if (opaque->getType()->isRecordType()) {
579 assert(Adjustments.empty());
580 return EmitOpaqueValueLValue(e: opaque);
581 }
582 }
583
584 // Create and initialize the reference temporary.
585 RawAddress Alloca = Address::invalid();
586 RawAddress Object = createReferenceTemporary(CGF&: *this, M, Inner: E, Alloca: &Alloca);
587 if (auto *Var = dyn_cast<llvm::GlobalVariable>(
588 Val: Object.getPointer()->stripPointerCasts())) {
589 llvm::Type *TemporaryType = ConvertTypeForMem(T: E->getType());
590 Object = Object.withElementType(ElemTy: TemporaryType);
591 // If the temporary is a global and has a constant initializer or is a
592 // constant temporary that we promoted to a global, we may have already
593 // initialized it.
594 if (!Var->hasInitializer()) {
595 Var->setInitializer(CGM.EmitNullConstant(T: E->getType()));
596 QualType RefType = M->getType().withoutLocalFastQualifiers();
597 if (RefType.getPointerAuth()) {
598 // Use the qualifier of the reference temporary to sign the pointer.
599 LValue LV = MakeRawAddrLValue(V: Object.getPointer(), T: RefType,
600 Alignment: Object.getAlignment());
601 EmitScalarInit(init: E, D: M->getExtendingDecl(), lvalue: LV, capturedByInit: false);
602 } else {
603 EmitAnyExprToMem(E, Location: Object, Quals: Qualifiers(), /*IsInit*/ true);
604 }
605 }
606 } else {
607 switch (M->getStorageDuration()) {
608 case SD_Automatic:
609 if (EmitLifetimeStart(Addr: Alloca.getPointer())) {
610 pushCleanupAfterFullExpr<CallLifetimeEnd>(Kind: NormalEHLifetimeMarker,
611 A: Alloca);
612 }
613 break;
614
615 case SD_FullExpression: {
616 if (!ShouldEmitLifetimeMarkers)
617 break;
618
619 // Avoid creating a conditional cleanup just to hold an llvm.lifetime.end
620 // marker. Instead, start the lifetime of a conditional temporary earlier
621 // so that it's unconditional. Don't do this with sanitizers which need
622 // more precise lifetime marks. However when inside an "await.suspend"
623 // block, we should always avoid conditional cleanup because it creates
624 // boolean marker that lives across await_suspend, which can destroy coro
625 // frame.
626 ConditionalEvaluation *OldConditional = nullptr;
627 CGBuilderTy::InsertPoint OldIP;
628 if (isInConditionalBranch() && !E->getType().isDestructedType() &&
629 ((!SanOpts.has(K: SanitizerKind::HWAddress) &&
630 !SanOpts.has(K: SanitizerKind::Memory) &&
631 !SanOpts.has(K: SanitizerKind::MemtagStack) &&
632 !CGM.getCodeGenOpts().SanitizeAddressUseAfterScope) ||
633 inSuspendBlock())) {
634 OldConditional = OutermostConditional;
635 OutermostConditional = nullptr;
636
637 OldIP = Builder.saveIP();
638 llvm::BasicBlock *Block = OldConditional->getStartingBlock();
639 Builder.restoreIP(IP: CGBuilderTy::InsertPoint(
640 Block, llvm::BasicBlock::iterator(Block->back())));
641 }
642
643 if (EmitLifetimeStart(Addr: Alloca.getPointer())) {
644 pushFullExprCleanup<CallLifetimeEnd>(kind: NormalEHLifetimeMarker, A: Alloca);
645 }
646
647 if (OldConditional) {
648 OutermostConditional = OldConditional;
649 Builder.restoreIP(IP: OldIP);
650 }
651 break;
652 }
653
654 default:
655 break;
656 }
657 EmitAnyExprToMem(E, Location: Object, Quals: Qualifiers(), /*IsInit*/true);
658 }
659 pushTemporaryCleanup(CGF&: *this, M, E, ReferenceTemporary: Object);
660
661 // Perform derived-to-base casts and/or field accesses, to get from the
662 // temporary object we created (and, potentially, for which we extended
663 // the lifetime) to the subobject we're binding the reference to.
664 for (SubobjectAdjustment &Adjustment : llvm::reverse(C&: Adjustments)) {
665 switch (Adjustment.Kind) {
666 case SubobjectAdjustment::DerivedToBaseAdjustment:
667 Object =
668 GetAddressOfBaseClass(Value: Object, Derived: Adjustment.DerivedToBase.DerivedClass,
669 PathBegin: Adjustment.DerivedToBase.BasePath->path_begin(),
670 PathEnd: Adjustment.DerivedToBase.BasePath->path_end(),
671 /*NullCheckValue=*/ false, Loc: E->getExprLoc());
672 break;
673
674 case SubobjectAdjustment::FieldAdjustment: {
675 LValue LV = MakeAddrLValue(Addr: Object, T: E->getType(), Source: AlignmentSource::Decl);
676 LV = EmitLValueForField(Base: LV, Field: Adjustment.Field);
677 assert(LV.isSimple() &&
678 "materialized temporary field is not a simple lvalue");
679 Object = LV.getAddress();
680 break;
681 }
682
683 case SubobjectAdjustment::MemberPointerAdjustment: {
684 llvm::Value *Ptr = EmitScalarExpr(E: Adjustment.Ptr.RHS);
685 Object = EmitCXXMemberDataPointerAddress(
686 E, base: Object, memberPtr: Ptr, memberPtrType: Adjustment.Ptr.MPT, /*IsInBounds=*/true);
687 break;
688 }
689 }
690 }
691
692 return MakeAddrLValue(Addr: Object, T: M->getType(), Source: AlignmentSource::Decl);
693}
694
695RValue
696CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E) {
697 // Emit the expression as an lvalue.
698 LValue LV = EmitLValue(E);
699 assert(LV.isSimple());
700 llvm::Value *Value = LV.getPointer(CGF&: *this);
701
702 if (sanitizePerformTypeCheck() && !E->getType()->isFunctionType()) {
703 // C++11 [dcl.ref]p5 (as amended by core issue 453):
704 // If a glvalue to which a reference is directly bound designates neither
705 // an existing object or function of an appropriate type nor a region of
706 // storage of suitable size and alignment to contain an object of the
707 // reference's type, the behavior is undefined.
708 QualType Ty = E->getType();
709 EmitTypeCheck(TCK: TCK_ReferenceBinding, Loc: E->getExprLoc(), V: Value, Type: Ty);
710 }
711
712 return RValue::get(V: Value);
713}
714
715
716/// getAccessedFieldNo - Given an encoded value and a result number, return the
717/// input field number being accessed.
718unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx,
719 const llvm::Constant *Elts) {
720 return cast<llvm::ConstantInt>(Val: Elts->getAggregateElement(Elt: Idx))
721 ->getZExtValue();
722}
723
724static llvm::Value *emitHashMix(CGBuilderTy &Builder, llvm::Value *Acc,
725 llvm::Value *Ptr) {
726 llvm::Value *A0 =
727 Builder.CreateMul(LHS: Ptr, RHS: Builder.getInt64(C: 0xbf58476d1ce4e5b9u));
728 llvm::Value *A1 =
729 Builder.CreateXor(LHS: A0, RHS: Builder.CreateLShr(LHS: A0, RHS: Builder.getInt64(C: 31)));
730 return Builder.CreateXor(LHS: Acc, RHS: A1);
731}
732
733bool CodeGenFunction::isNullPointerAllowed(TypeCheckKind TCK) {
734 return TCK == TCK_DowncastPointer || TCK == TCK_Upcast ||
735 TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation;
736}
737
738bool CodeGenFunction::isVptrCheckRequired(TypeCheckKind TCK, QualType Ty) {
739 CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
740 return (RD && RD->hasDefinition() && RD->isDynamicClass()) &&
741 (TCK == TCK_MemberAccess || TCK == TCK_MemberCall ||
742 TCK == TCK_DowncastPointer || TCK == TCK_DowncastReference ||
743 TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation);
744}
745
746bool CodeGenFunction::sanitizePerformTypeCheck() const {
747 return SanOpts.has(K: SanitizerKind::Null) ||
748 SanOpts.has(K: SanitizerKind::Alignment) ||
749 SanOpts.has(K: SanitizerKind::ObjectSize) ||
750 SanOpts.has(K: SanitizerKind::Vptr);
751}
752
753void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
754 llvm::Value *Ptr, QualType Ty,
755 CharUnits Alignment,
756 SanitizerSet SkippedChecks,
757 llvm::Value *ArraySize) {
758 if (!sanitizePerformTypeCheck())
759 return;
760
761 // Don't check pointers outside the default address space. The null check
762 // isn't correct, the object-size check isn't supported by LLVM, and we can't
763 // communicate the addresses to the runtime handler for the vptr check.
764 if (Ptr->getType()->getPointerAddressSpace())
765 return;
766
767 // Don't check pointers to volatile data. The behavior here is implementation-
768 // defined.
769 if (Ty.isVolatileQualified())
770 return;
771
772 // Quickly determine whether we have a pointer to an alloca. It's possible
773 // to skip null checks, and some alignment checks, for these pointers. This
774 // can reduce compile-time significantly.
775 auto PtrToAlloca = dyn_cast<llvm::AllocaInst>(Val: Ptr->stripPointerCasts());
776
777 llvm::Value *IsNonNull = nullptr;
778 bool IsGuaranteedNonNull =
779 SkippedChecks.has(K: SanitizerKind::Null) || PtrToAlloca;
780
781 llvm::BasicBlock *Done = nullptr;
782 bool DoneViaNullSanitize = false;
783
784 {
785 auto CheckHandler = SanitizerHandler::TypeMismatch;
786 SanitizerDebugLocation SanScope(this,
787 {SanitizerKind::SO_Null,
788 SanitizerKind::SO_ObjectSize,
789 SanitizerKind::SO_Alignment},
790 CheckHandler);
791
792 SmallVector<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>, 3>
793 Checks;
794
795 llvm::Value *True = llvm::ConstantInt::getTrue(Context&: getLLVMContext());
796 bool AllowNullPointers = isNullPointerAllowed(TCK);
797 if ((SanOpts.has(K: SanitizerKind::Null) || AllowNullPointers) &&
798 !IsGuaranteedNonNull) {
799 // The glvalue must not be an empty glvalue.
800 IsNonNull = Builder.CreateIsNotNull(Arg: Ptr);
801
802 // The IR builder can constant-fold the null check if the pointer points
803 // to a constant.
804 IsGuaranteedNonNull = IsNonNull == True;
805
806 // Skip the null check if the pointer is known to be non-null.
807 if (!IsGuaranteedNonNull) {
808 if (AllowNullPointers) {
809 // When performing pointer casts, it's OK if the value is null.
810 // Skip the remaining checks in that case.
811 Done = createBasicBlock(name: "null");
812 DoneViaNullSanitize = true;
813 llvm::BasicBlock *Rest = createBasicBlock(name: "not.null");
814 Builder.CreateCondBr(Cond: IsNonNull, True: Rest, False: Done);
815 EmitBlock(BB: Rest);
816 } else {
817 Checks.push_back(Elt: std::make_pair(x&: IsNonNull, y: SanitizerKind::SO_Null));
818 }
819 }
820 }
821
822 if (SanOpts.has(K: SanitizerKind::ObjectSize) &&
823 !SkippedChecks.has(K: SanitizerKind::ObjectSize) &&
824 !Ty->isIncompleteType()) {
825 uint64_t TySize = CGM.getMinimumObjectSize(Ty).getQuantity();
826 llvm::Value *Size = llvm::ConstantInt::get(Ty: IntPtrTy, V: TySize);
827 if (ArraySize)
828 Size = Builder.CreateMul(LHS: Size, RHS: ArraySize);
829
830 // Degenerate case: new X[0] does not need an objectsize check.
831 llvm::Constant *ConstantSize = dyn_cast<llvm::Constant>(Val: Size);
832 if (!ConstantSize || !ConstantSize->isNullValue()) {
833 // The glvalue must refer to a large enough storage region.
834 // FIXME: If Address Sanitizer is enabled, insert dynamic
835 // instrumentation
836 // to check this.
837 // FIXME: Get object address space
838 llvm::Type *Tys[2] = {IntPtrTy, Int8PtrTy};
839 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::objectsize, Tys);
840 llvm::Value *Min = Builder.getFalse();
841 llvm::Value *NullIsUnknown = Builder.getFalse();
842 llvm::Value *Dynamic = Builder.getFalse();
843 llvm::Value *LargeEnough = Builder.CreateICmpUGE(
844 LHS: Builder.CreateCall(Callee: F, Args: {Ptr, Min, NullIsUnknown, Dynamic}), RHS: Size);
845 Checks.push_back(
846 Elt: std::make_pair(x&: LargeEnough, y: SanitizerKind::SO_ObjectSize));
847 }
848 }
849
850 llvm::MaybeAlign AlignVal;
851 llvm::Value *PtrAsInt = nullptr;
852
853 if (SanOpts.has(K: SanitizerKind::Alignment) &&
854 !SkippedChecks.has(K: SanitizerKind::Alignment)) {
855 AlignVal = Alignment.getAsMaybeAlign();
856 if (!Ty->isIncompleteType() && !AlignVal)
857 AlignVal = CGM.getNaturalTypeAlignment(T: Ty, BaseInfo: nullptr, TBAAInfo: nullptr,
858 /*ForPointeeType=*/forPointeeType: true)
859 .getAsMaybeAlign();
860
861 // The glvalue must be suitably aligned.
862 if (AlignVal && *AlignVal > llvm::Align(1) &&
863 (!PtrToAlloca || PtrToAlloca->getAlign() < *AlignVal)) {
864 PtrAsInt = Builder.CreatePtrToInt(V: Ptr, DestTy: IntPtrTy);
865 llvm::Value *Align = Builder.CreateAnd(
866 LHS: PtrAsInt, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, V: AlignVal->value() - 1));
867 llvm::Value *Aligned =
868 Builder.CreateICmpEQ(LHS: Align, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, V: 0));
869 if (Aligned != True)
870 Checks.push_back(
871 Elt: std::make_pair(x&: Aligned, y: SanitizerKind::SO_Alignment));
872 }
873 }
874
875 if (Checks.size() > 0) {
876 llvm::Constant *StaticData[] = {
877 EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(T: Ty),
878 llvm::ConstantInt::get(Ty: Int8Ty, V: AlignVal ? llvm::Log2(A: *AlignVal) : 1),
879 llvm::ConstantInt::get(Ty: Int8Ty, V: TCK)};
880 EmitCheck(Checked: Checks, Check: CheckHandler, StaticArgs: StaticData, DynamicArgs: PtrAsInt ? PtrAsInt : Ptr);
881 }
882 }
883
884 // If possible, check that the vptr indicates that there is a subobject of
885 // type Ty at offset zero within this object.
886 //
887 // C++11 [basic.life]p5,6:
888 // [For storage which does not refer to an object within its lifetime]
889 // The program has undefined behavior if:
890 // -- the [pointer or glvalue] is used to access a non-static data member
891 // or call a non-static member function
892 if (SanOpts.has(K: SanitizerKind::Vptr) &&
893 !SkippedChecks.has(K: SanitizerKind::Vptr) && isVptrCheckRequired(TCK, Ty)) {
894 SanitizerDebugLocation SanScope(this, {SanitizerKind::SO_Vptr},
895 SanitizerHandler::DynamicTypeCacheMiss);
896
897 // Ensure that the pointer is non-null before loading it. If there is no
898 // compile-time guarantee, reuse the run-time null check or emit a new one.
899 if (!IsGuaranteedNonNull) {
900 if (!IsNonNull)
901 IsNonNull = Builder.CreateIsNotNull(Arg: Ptr);
902 if (!Done)
903 Done = createBasicBlock(name: "vptr.null");
904 llvm::BasicBlock *VptrNotNull = createBasicBlock(name: "vptr.not.null");
905 Builder.CreateCondBr(Cond: IsNonNull, True: VptrNotNull, False: Done);
906 EmitBlock(BB: VptrNotNull);
907 }
908
909 // Compute a deterministic hash of the mangled name of the type.
910 SmallString<64> MangledName;
911 llvm::raw_svector_ostream Out(MangledName);
912 CGM.getCXXABI().getMangleContext().mangleCXXRTTI(T: Ty.getUnqualifiedType(),
913 Out);
914
915 // Contained in NoSanitizeList based on the mangled type.
916 if (!CGM.getContext().getNoSanitizeList().containsType(Mask: SanitizerKind::Vptr,
917 MangledTypeName: Out.str())) {
918 // Load the vptr, and mix it with TypeHash.
919 llvm::Value *TypeHash =
920 llvm::ConstantInt::get(Ty: Int64Ty, V: xxh3_64bits(data: Out.str()));
921
922 llvm::Type *VPtrTy = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: 0);
923 Address VPtrAddr(Ptr, IntPtrTy, getPointerAlign());
924 llvm::Value *VPtrVal = GetVTablePtr(This: VPtrAddr, VTableTy: VPtrTy,
925 VTableClass: Ty->getAsCXXRecordDecl(),
926 AuthMode: VTableAuthMode::UnsafeUbsanStrip);
927 VPtrVal = Builder.CreateBitOrPointerCast(V: VPtrVal, DestTy: IntPtrTy);
928
929 llvm::Value *Hash =
930 emitHashMix(Builder, Acc: TypeHash, Ptr: Builder.CreateZExt(V: VPtrVal, DestTy: Int64Ty));
931 Hash = Builder.CreateTrunc(V: Hash, DestTy: IntPtrTy);
932
933 // Look the hash up in our cache.
934 const int CacheSize = 128;
935 llvm::Type *HashTable = llvm::ArrayType::get(ElementType: IntPtrTy, NumElements: CacheSize);
936 llvm::Value *Cache = CGM.CreateRuntimeVariable(Ty: HashTable,
937 Name: "__ubsan_vptr_type_cache");
938 llvm::Value *Slot = Builder.CreateAnd(LHS: Hash,
939 RHS: llvm::ConstantInt::get(Ty: IntPtrTy,
940 V: CacheSize-1));
941 llvm::Value *Indices[] = { Builder.getInt32(C: 0), Slot };
942 llvm::Value *CacheVal = Builder.CreateAlignedLoad(
943 Ty: IntPtrTy, Addr: Builder.CreateInBoundsGEP(Ty: HashTable, Ptr: Cache, IdxList: Indices),
944 Align: getPointerAlign());
945
946 // If the hash isn't in the cache, call a runtime handler to perform the
947 // hard work of checking whether the vptr is for an object of the right
948 // type. This will either fill in the cache and return, or produce a
949 // diagnostic.
950 llvm::Value *EqualHash = Builder.CreateICmpEQ(LHS: CacheVal, RHS: Hash);
951 llvm::Constant *StaticData[] = {
952 EmitCheckSourceLocation(Loc),
953 EmitCheckTypeDescriptor(T: Ty),
954 CGM.GetAddrOfRTTIDescriptor(Ty: Ty.getUnqualifiedType()),
955 llvm::ConstantInt::get(Ty: Int8Ty, V: TCK)
956 };
957 llvm::Value *DynamicData[] = { Ptr, Hash };
958 EmitCheck(Checked: std::make_pair(x&: EqualHash, y: SanitizerKind::SO_Vptr),
959 Check: SanitizerHandler::DynamicTypeCacheMiss, StaticArgs: StaticData,
960 DynamicArgs: DynamicData);
961 }
962 }
963
964 if (Done) {
965 SanitizerDebugLocation SanScope(
966 this,
967 {DoneViaNullSanitize ? SanitizerKind::SO_Null : SanitizerKind::SO_Vptr},
968 DoneViaNullSanitize ? SanitizerHandler::TypeMismatch
969 : SanitizerHandler::DynamicTypeCacheMiss);
970 Builder.CreateBr(Dest: Done);
971 EmitBlock(BB: Done);
972 }
973}
974
975llvm::Value *CodeGenFunction::LoadPassedObjectSize(const Expr *E,
976 QualType EltTy) {
977 ASTContext &C = getContext();
978 uint64_t EltSize = C.getTypeSizeInChars(T: EltTy).getQuantity();
979 if (!EltSize)
980 return nullptr;
981
982 auto *ArrayDeclRef = dyn_cast<DeclRefExpr>(Val: E->IgnoreParenImpCasts());
983 if (!ArrayDeclRef)
984 return nullptr;
985
986 auto *ParamDecl = dyn_cast<ParmVarDecl>(Val: ArrayDeclRef->getDecl());
987 if (!ParamDecl)
988 return nullptr;
989
990 auto *POSAttr = ParamDecl->getAttr<PassObjectSizeAttr>();
991 if (!POSAttr)
992 return nullptr;
993
994 // Don't load the size if it's a lower bound.
995 int POSType = POSAttr->getType();
996 if (POSType != 0 && POSType != 1)
997 return nullptr;
998
999 // Find the implicit size parameter.
1000 auto PassedSizeIt = SizeArguments.find(Val: ParamDecl);
1001 if (PassedSizeIt == SizeArguments.end())
1002 return nullptr;
1003
1004 const ImplicitParamDecl *PassedSizeDecl = PassedSizeIt->second;
1005 assert(LocalDeclMap.count(PassedSizeDecl) && "Passed size not loadable");
1006 Address AddrOfSize = LocalDeclMap.find(Val: PassedSizeDecl)->second;
1007 llvm::Value *SizeInBytes = EmitLoadOfScalar(Addr: AddrOfSize, /*Volatile=*/false,
1008 Ty: C.getSizeType(), Loc: E->getExprLoc());
1009 llvm::Value *SizeOfElement =
1010 llvm::ConstantInt::get(Ty: SizeInBytes->getType(), V: EltSize);
1011 return Builder.CreateUDiv(LHS: SizeInBytes, RHS: SizeOfElement);
1012}
1013
1014/// If Base is known to point to the start of an array, return the length of
1015/// that array. Return 0 if the length cannot be determined.
1016static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF,
1017 const Expr *Base,
1018 QualType &IndexedType,
1019 LangOptions::StrictFlexArraysLevelKind
1020 StrictFlexArraysLevel) {
1021 // For the vector indexing extension, the bound is the number of elements.
1022 if (const VectorType *VT = Base->getType()->getAs<VectorType>()) {
1023 IndexedType = Base->getType();
1024 return CGF.Builder.getInt32(C: VT->getNumElements());
1025 }
1026
1027 Base = Base->IgnoreParens();
1028
1029 if (const auto *CE = dyn_cast<CastExpr>(Val: Base)) {
1030 if (CE->getCastKind() == CK_ArrayToPointerDecay &&
1031 !CE->getSubExpr()->isFlexibleArrayMemberLike(Context: CGF.getContext(),
1032 StrictFlexArraysLevel)) {
1033 CodeGenFunction::SanitizerScope SanScope(&CGF);
1034
1035 IndexedType = CE->getSubExpr()->getType();
1036 const ArrayType *AT = IndexedType->castAsArrayTypeUnsafe();
1037 if (const auto *CAT = dyn_cast<ConstantArrayType>(Val: AT))
1038 return CGF.Builder.getInt(AI: CAT->getSize());
1039
1040 if (const auto *VAT = dyn_cast<VariableArrayType>(Val: AT))
1041 return CGF.getVLASize(vla: VAT).NumElts;
1042 // Ignore pass_object_size here. It's not applicable on decayed pointers.
1043 }
1044 }
1045
1046 CodeGenFunction::SanitizerScope SanScope(&CGF);
1047
1048 QualType EltTy{Base->getType()->getPointeeOrArrayElementType(), 0};
1049 if (llvm::Value *POS = CGF.LoadPassedObjectSize(E: Base, EltTy)) {
1050 IndexedType = Base->getType();
1051 return POS;
1052 }
1053
1054 return nullptr;
1055}
1056
1057/// Returns true if \p Field is reachable from \p RD either as a direct field or
1058/// through a chain of nested record fields (including anonymous
1059/// structs/unions). This mirrors the GEP path that getGEPIndicesToField builds,
1060/// and is used to identify the right anchor expression in Base.
1061static bool RecordContainsField(const RecordDecl *RD, const FieldDecl *Field) {
1062 for (const FieldDecl *FD : RD->fields()) {
1063 if (FD == Field)
1064 return true;
1065 QualType Ty = FD->getType();
1066 if (Ty->isRecordType())
1067 if (RecordContainsField(RD: Ty->getAsRecordDecl(), Field))
1068 return true;
1069 }
1070 return false;
1071}
1072
1073namespace {
1074
1075/// \p StructAccessBase returns the base \p Expr of a field access. It returns
1076/// either a \p DeclRefExpr, representing the base pointer to the struct, i.e.:
1077///
1078/// p in p-> a.b.c
1079///
1080/// or a \p MemberExpr, if the \p MemberExpr has the \p RecordDecl we're
1081/// looking for:
1082///
1083/// struct s {
1084/// struct s *ptr;
1085/// int count;
1086/// char array[] __attribute__((counted_by(count)));
1087/// };
1088///
1089/// If we have an expression like \p p->ptr->array[index], we want the
1090/// \p MemberExpr for \p p->ptr instead of \p p.
1091class StructAccessBase
1092 : public ConstStmtVisitor<StructAccessBase, const Expr *> {
1093 /// The count field we're navigating to. We stop at the innermost expression
1094 /// whose struct type transitively contains this field, so that
1095 /// getGEPIndicesToField can navigate from that struct down to it.
1096 const FieldDecl *CountDecl;
1097
1098 /// Returns true if E's record type (or pointee record type) transitively
1099 /// contains CountDecl. Handles both direct containment and nested structs,
1100 /// so we don't need a pre-computed RD from the caller.
1101 bool IsExpectedRecordDecl(const Expr *E) const {
1102 QualType Ty = E->getType();
1103 if (Ty->isPointerType())
1104 Ty = Ty->getPointeeType();
1105 const RecordDecl *RD = Ty->getAsRecordDecl();
1106 return RD && RecordContainsField(RD, Field: CountDecl);
1107 }
1108
1109public:
1110 StructAccessBase(const FieldDecl *CountDecl) : CountDecl(CountDecl) {}
1111
1112 //===--------------------------------------------------------------------===//
1113 // Visitor Methods
1114 //===--------------------------------------------------------------------===//
1115
1116 // NOTE: If we build C++ support for counted_by, then we'll have to handle
1117 // horrors like this:
1118 //
1119 // struct S {
1120 // int x, y;
1121 // int blah[] __attribute__((counted_by(x)));
1122 // } s;
1123 //
1124 // int foo(int index, int val) {
1125 // int (S::*IHatePMDs)[] = &S::blah;
1126 // (s.*IHatePMDs)[index] = val;
1127 // }
1128
1129 const Expr *Visit(const Expr *E) {
1130 return ConstStmtVisitor<StructAccessBase, const Expr *>::Visit(S: E);
1131 }
1132
1133 const Expr *VisitStmt(const Stmt *S) { return nullptr; }
1134
1135 // These are the types we expect to return (in order of most to least
1136 // likely):
1137 //
1138 // 1. DeclRefExpr - This is the expression for the base of the structure.
1139 // It's exactly what we want to build an access to the \p counted_by
1140 // field.
1141 // 2. MemberExpr - This is the expression that has the same \p RecordDecl
1142 // as the flexble array member's lexical enclosing \p RecordDecl. This
1143 // allows us to catch things like: "p->p->array"
1144 // 3. CompoundLiteralExpr - This is for people who create something
1145 // heretical like (struct foo has a flexible array member):
1146 //
1147 // (struct foo){ 1, 2 }.blah[idx];
1148 const Expr *VisitDeclRefExpr(const DeclRefExpr *E) {
1149 return IsExpectedRecordDecl(E) ? E : nullptr;
1150 }
1151 const Expr *VisitMemberExpr(const MemberExpr *E) {
1152 if (IsExpectedRecordDecl(E) && E->isArrow())
1153 return E;
1154 const Expr *Res = Visit(E: E->getBase());
1155 return !Res && IsExpectedRecordDecl(E) ? E : Res;
1156 }
1157 const Expr *VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) {
1158 return IsExpectedRecordDecl(E) ? E : nullptr;
1159 }
1160 const Expr *VisitCallExpr(const CallExpr *E) {
1161 return IsExpectedRecordDecl(E) ? E : nullptr;
1162 }
1163
1164 const Expr *VisitArraySubscriptExpr(const ArraySubscriptExpr *E) {
1165 if (IsExpectedRecordDecl(E))
1166 return E;
1167 return Visit(E: E->getBase());
1168 }
1169 const Expr *VisitCastExpr(const CastExpr *E) {
1170 if (E->getCastKind() == CK_LValueToRValue)
1171 return IsExpectedRecordDecl(E) ? E : nullptr;
1172 return Visit(E: E->getSubExpr());
1173 }
1174 const Expr *VisitParenExpr(const ParenExpr *E) {
1175 return Visit(E: E->getSubExpr());
1176 }
1177 const Expr *VisitUnaryAddrOf(const UnaryOperator *E) {
1178 return Visit(E: E->getSubExpr());
1179 }
1180 const Expr *VisitUnaryDeref(const UnaryOperator *E) {
1181 return Visit(E: E->getSubExpr());
1182 }
1183};
1184
1185} // end anonymous namespace
1186
1187using RecIndicesTy = SmallVector<llvm::Value *, 8>;
1188
1189static bool getGEPIndicesToField(CodeGenFunction &CGF, const RecordDecl *RD,
1190 const FieldDecl *Field,
1191 RecIndicesTy &Indices) {
1192 const CGRecordLayout &Layout = CGF.CGM.getTypes().getCGRecordLayout(RD);
1193 int64_t FieldNo = -1;
1194 for (const FieldDecl *FD : RD->fields()) {
1195 if (!Layout.containsFieldDecl(FD))
1196 // This could happen if the field has a struct type that's empty. I don't
1197 // know why either.
1198 continue;
1199
1200 FieldNo = Layout.getLLVMFieldNo(FD);
1201 if (FD == Field) {
1202 Indices.emplace_back(Args: CGF.Builder.getInt32(C: FieldNo));
1203 return true;
1204 }
1205
1206 QualType Ty = FD->getType();
1207 if (Ty->isRecordType()) {
1208 if (getGEPIndicesToField(CGF, RD: Ty->getAsRecordDecl(), Field, Indices)) {
1209 if (RD->isUnion())
1210 FieldNo = 0;
1211 Indices.emplace_back(Args: CGF.Builder.getInt32(C: FieldNo));
1212 return true;
1213 }
1214 }
1215 }
1216
1217 return false;
1218}
1219
1220llvm::Value *CodeGenFunction::GetCountedByFieldExprGEP(
1221 const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl) {
1222 // Walk Base to find the deepest sub-expression whose struct type transitively
1223 // contains CountDecl. This is our GEP anchor — getGEPIndicesToField then
1224 // builds the field indices from that struct down to CountDecl, handling any
1225 // intermediate nesting without requiring us to pre-compute a RecordDecl from
1226 // Base's type or from CountDecl's parent chain.
1227 const Expr *StructBase = StructAccessBase(CountDecl).Visit(E: Base);
1228 if (!StructBase || StructBase->HasSideEffects(Ctx: getContext()))
1229 return nullptr;
1230
1231 // Derive the record type from the anchor expression itself.
1232 QualType StructTy = StructBase->getType();
1233 if (StructTy->isPointerType())
1234 StructTy = StructTy->getPointeeType();
1235 const RecordDecl *RD = StructTy->getAsRecordDecl();
1236 if (!RD)
1237 return nullptr;
1238
1239 llvm::Value *Res = nullptr;
1240 if (StructBase->getType()->isPointerType()) {
1241 LValueBaseInfo BaseInfo;
1242 TBAAAccessInfo TBAAInfo;
1243 Address Addr = EmitPointerWithAlignment(Addr: StructBase, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
1244 Res = Addr.emitRawPointer(CGF&: *this);
1245 } else if (StructBase->isLValue()) {
1246 LValue LV = EmitLValue(E: StructBase);
1247 Address Addr = LV.getAddress();
1248 Res = Addr.emitRawPointer(CGF&: *this);
1249 } else {
1250 return nullptr;
1251 }
1252
1253 RecIndicesTy Indices;
1254 getGEPIndicesToField(CGF&: *this, RD, Field: CountDecl, Indices);
1255 if (Indices.empty())
1256 return nullptr;
1257
1258 Indices.push_back(Elt: Builder.getInt32(C: 0));
1259 CanQualType T = CGM.getContext().getCanonicalTagType(TD: RD);
1260 return Builder.CreateInBoundsGEP(Ty: ConvertType(T), Ptr: Res,
1261 IdxList: RecIndicesTy(llvm::reverse(C&: Indices)),
1262 Name: "counted_by.gep");
1263}
1264
1265/// This method is typically called in contexts where we can't generate
1266/// side-effects, like in __builtin_dynamic_object_size. When finding
1267/// expressions, only choose those that have either already been emitted or can
1268/// be loaded without side-effects.
1269///
1270/// - \p FAMDecl: the \p Decl for the flexible array member. It may not be
1271/// within the top-level struct.
1272/// - \p CountDecl: must be within the same non-anonymous struct as \p FAMDecl.
1273llvm::Value *CodeGenFunction::EmitLoadOfCountedByField(
1274 const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl) {
1275 if (llvm::Value *GEP = GetCountedByFieldExprGEP(Base, FAMDecl, CountDecl))
1276 return Builder.CreateAlignedLoad(Ty: ConvertType(T: CountDecl->getType()), Addr: GEP,
1277 Align: getIntAlign(), Name: "counted_by.load");
1278 return nullptr;
1279}
1280
1281void CodeGenFunction::EmitBoundsCheck(const Expr *ArrayExpr,
1282 const Expr *ArrayExprBase,
1283 llvm::Value *IndexVal, QualType IndexType,
1284 bool Accessed) {
1285 assert(SanOpts.has(SanitizerKind::ArrayBounds) &&
1286 "should not be called unless adding bounds checks");
1287 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
1288 getLangOpts().getStrictFlexArraysLevel();
1289 QualType ArrayExprBaseType;
1290 llvm::Value *BoundsVal = getArrayIndexingBound(
1291 CGF&: *this, Base: ArrayExprBase, IndexedType&: ArrayExprBaseType, StrictFlexArraysLevel);
1292
1293 EmitBoundsCheckImpl(ArrayExpr, ArrayBaseType: ArrayExprBaseType, IndexVal, IndexType,
1294 BoundsVal, BoundsType: getContext().getSizeType(), Accessed);
1295}
1296
1297void CodeGenFunction::EmitBoundsCheckImpl(const Expr *ArrayExpr,
1298 QualType ArrayBaseType,
1299 llvm::Value *IndexVal,
1300 QualType IndexType,
1301 llvm::Value *BoundsVal,
1302 QualType BoundsType, bool Accessed) {
1303 if (!BoundsVal)
1304 return;
1305
1306 auto CheckKind = SanitizerKind::SO_ArrayBounds;
1307 auto CheckHandler = SanitizerHandler::OutOfBounds;
1308 SanitizerDebugLocation SanScope(this, {CheckKind}, CheckHandler);
1309
1310 // All hail the C implicit type conversion rules!!!
1311 bool IndexSigned = IndexType->isSignedIntegerOrEnumerationType();
1312 bool BoundsSigned = BoundsType->isSignedIntegerOrEnumerationType();
1313
1314 const ASTContext &Ctx = getContext();
1315 llvm::Type *Ty = ConvertType(
1316 T: Ctx.getTypeSize(T: IndexType) >= Ctx.getTypeSize(T: BoundsType) ? IndexType
1317 : BoundsType);
1318
1319 llvm::Value *IndexInst = Builder.CreateIntCast(V: IndexVal, DestTy: Ty, isSigned: IndexSigned);
1320 llvm::Value *BoundsInst = Builder.CreateIntCast(V: BoundsVal, DestTy: Ty, isSigned: false);
1321
1322 llvm::Constant *StaticData[] = {
1323 EmitCheckSourceLocation(Loc: ArrayExpr->getExprLoc()),
1324 EmitCheckTypeDescriptor(T: ArrayBaseType),
1325 EmitCheckTypeDescriptor(T: IndexType),
1326 };
1327
1328 llvm::Value *Check = Accessed ? Builder.CreateICmpULT(LHS: IndexInst, RHS: BoundsInst)
1329 : Builder.CreateICmpULE(LHS: IndexInst, RHS: BoundsInst);
1330
1331 if (BoundsSigned) {
1332 // Don't allow a negative bounds.
1333 llvm::Value *Cmp = Builder.CreateICmpSGT(
1334 LHS: BoundsVal, RHS: llvm::ConstantInt::get(Ty: BoundsVal->getType(), V: 0));
1335 Check = Builder.CreateAnd(LHS: Cmp, RHS: Check);
1336 }
1337
1338 EmitCheck(Checked: std::make_pair(x&: Check, y&: CheckKind), Check: CheckHandler, StaticArgs: StaticData,
1339 DynamicArgs: IndexInst);
1340}
1341
1342llvm::MDNode *CodeGenFunction::buildAllocToken(QualType AllocType) {
1343 auto ATMD = infer_alloc::getAllocTokenMetadata(T: AllocType, Ctx: getContext());
1344 if (!ATMD)
1345 return nullptr;
1346
1347 llvm::MDBuilder MDB(getLLVMContext());
1348 auto *TypeNameMD = MDB.createString(Str: ATMD->TypeName);
1349 auto *ContainsPtrC = Builder.getInt1(V: ATMD->ContainsPointer);
1350 auto *ContainsPtrMD = MDB.createConstant(C: ContainsPtrC);
1351
1352 // Format: !{<type-name>, <contains-pointer>}
1353 return llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: {TypeNameMD, ContainsPtrMD});
1354}
1355
1356void CodeGenFunction::EmitAllocToken(llvm::CallBase *CB, QualType AllocType) {
1357 assert(SanOpts.has(SanitizerKind::AllocToken) &&
1358 "Only needed with -fsanitize=alloc-token");
1359 CB->setMetadata(KindID: llvm::LLVMContext::MD_alloc_token,
1360 Node: buildAllocToken(AllocType));
1361}
1362
1363llvm::MDNode *CodeGenFunction::buildAllocToken(const CallExpr *E) {
1364 QualType AllocType = infer_alloc::inferPossibleType(E, Ctx: getContext(), CastE: CurCast);
1365 if (!AllocType.isNull())
1366 return buildAllocToken(AllocType);
1367 return nullptr;
1368}
1369
1370void CodeGenFunction::EmitAllocToken(llvm::CallBase *CB, const CallExpr *E) {
1371 assert(SanOpts.has(SanitizerKind::AllocToken) &&
1372 "Only needed with -fsanitize=alloc-token");
1373 if (llvm::MDNode *MDN = buildAllocToken(E))
1374 CB->setMetadata(KindID: llvm::LLVMContext::MD_alloc_token, Node: MDN);
1375}
1376
1377CodeGenFunction::ComplexPairTy CodeGenFunction::
1378EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
1379 bool isInc, bool isPre) {
1380 ComplexPairTy InVal = EmitLoadOfComplex(src: LV, loc: E->getExprLoc());
1381
1382 llvm::Value *NextVal;
1383 if (isa<llvm::IntegerType>(Val: InVal.first->getType())) {
1384 uint64_t AmountVal = isInc ? 1 : -1;
1385 NextVal = llvm::ConstantInt::get(Ty: InVal.first->getType(), V: AmountVal, IsSigned: true);
1386
1387 // Add the inc/dec to the real part.
1388 NextVal = Builder.CreateAdd(LHS: InVal.first, RHS: NextVal, Name: isInc ? "inc" : "dec");
1389 } else {
1390 QualType ElemTy = E->getType()->castAs<ComplexType>()->getElementType();
1391 llvm::APFloat FVal(getContext().getFloatTypeSemantics(T: ElemTy), 1);
1392 if (!isInc)
1393 FVal.changeSign();
1394 NextVal = llvm::ConstantFP::get(Context&: getLLVMContext(), V: FVal);
1395
1396 // Add the inc/dec to the real part.
1397 NextVal = Builder.CreateFAdd(L: InVal.first, R: NextVal, Name: isInc ? "inc" : "dec");
1398 }
1399
1400 ComplexPairTy IncVal(NextVal, InVal.second);
1401
1402 // Store the updated result through the lvalue.
1403 EmitStoreOfComplex(V: IncVal, dest: LV, /*init*/ isInit: false);
1404 if (getLangOpts().OpenMP)
1405 CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF&: *this,
1406 LHS: E->getSubExpr());
1407
1408 // If this is a postinc, return the value read from memory, otherwise use the
1409 // updated value.
1410 return isPre ? IncVal : InVal;
1411}
1412
1413void CodeGenModule::EmitExplicitCastExprType(const ExplicitCastExpr *E,
1414 CodeGenFunction *CGF) {
1415 // Bind VLAs in the cast type.
1416 if (CGF && E->getType()->isVariablyModifiedType())
1417 CGF->EmitVariablyModifiedType(Ty: E->getType());
1418
1419 if (CGDebugInfo *DI = getModuleDebugInfo())
1420 DI->EmitExplicitCastType(Ty: E->getType());
1421}
1422
1423//===----------------------------------------------------------------------===//
1424// LValue Expression Emission
1425//===----------------------------------------------------------------------===//
1426
1427static CharUnits getArrayElementAlign(CharUnits arrayAlign, llvm::Value *idx,
1428 CharUnits eltSize) {
1429 // If we have a constant index, we can use the exact offset of the
1430 // element we're accessing.
1431 if (auto *constantIdx = dyn_cast<llvm::ConstantInt>(Val: idx)) {
1432 CharUnits offset = constantIdx->getZExtValue() * eltSize;
1433 return arrayAlign.alignmentAtOffset(offset);
1434 }
1435
1436 // Otherwise, use the worst-case alignment for any element.
1437 return arrayAlign.alignmentOfArrayElement(elementSize: eltSize);
1438}
1439
1440/// Emit pointer + index arithmetic.
1441static Address emitPointerArithmetic(CodeGenFunction &CGF,
1442 const BinaryOperator *BO,
1443 LValueBaseInfo *BaseInfo,
1444 TBAAAccessInfo *TBAAInfo,
1445 KnownNonNull_t IsKnownNonNull) {
1446 assert(BO->isAdditiveOp() && "Expect an addition or subtraction.");
1447 Expr *pointerOperand = BO->getLHS();
1448 Expr *indexOperand = BO->getRHS();
1449 bool isSubtraction = BO->getOpcode() == BO_Sub;
1450
1451 Address BaseAddr = Address::invalid();
1452 llvm::Value *index = nullptr;
1453 // In a subtraction, the LHS is always the pointer.
1454 // Note: do not change the evaluation order.
1455 if (!isSubtraction && !pointerOperand->getType()->isAnyPointerType()) {
1456 std::swap(a&: pointerOperand, b&: indexOperand);
1457 index = CGF.EmitScalarExpr(E: indexOperand);
1458 BaseAddr = CGF.EmitPointerWithAlignment(Addr: pointerOperand, BaseInfo, TBAAInfo,
1459 IsKnownNonNull: NotKnownNonNull);
1460 } else {
1461 BaseAddr = CGF.EmitPointerWithAlignment(Addr: pointerOperand, BaseInfo, TBAAInfo,
1462 IsKnownNonNull: NotKnownNonNull);
1463 index = CGF.EmitScalarExpr(E: indexOperand);
1464 }
1465
1466 llvm::Value *pointer = BaseAddr.getBasePointer();
1467 llvm::Value *Res = CGF.EmitPointerArithmetic(
1468 BO, pointerOperand, pointer, indexOperand, index, isSubtraction);
1469 QualType PointeeTy = BO->getType()->getPointeeType();
1470 CharUnits Align =
1471 getArrayElementAlign(arrayAlign: BaseAddr.getAlignment(), idx: index,
1472 eltSize: CGF.getContext().getTypeSizeInChars(T: PointeeTy));
1473 return Address(Res, CGF.ConvertTypeForMem(T: PointeeTy), Align,
1474 CGF.CGM.getPointerAuthInfoForPointeeType(type: PointeeTy),
1475 /*Offset=*/nullptr, IsKnownNonNull);
1476}
1477
1478static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo,
1479 TBAAAccessInfo *TBAAInfo,
1480 KnownNonNull_t IsKnownNonNull,
1481 CodeGenFunction &CGF) {
1482 // We allow this with ObjC object pointers because of fragile ABIs.
1483 assert(E->getType()->isPointerType() ||
1484 E->getType()->isObjCObjectPointerType());
1485 E = E->IgnoreParens();
1486
1487 // Casts:
1488 if (const CastExpr *CE = dyn_cast<CastExpr>(Val: E)) {
1489 if (const auto *ECE = dyn_cast<ExplicitCastExpr>(Val: CE))
1490 CGF.CGM.EmitExplicitCastExprType(E: ECE, CGF: &CGF);
1491
1492 switch (CE->getCastKind()) {
1493 // Non-converting casts (but not C's implicit conversion from void*).
1494 case CK_BitCast:
1495 case CK_NoOp:
1496 case CK_AddressSpaceConversion:
1497 if (auto PtrTy = CE->getSubExpr()->getType()->getAs<PointerType>()) {
1498 if (PtrTy->getPointeeType()->isVoidType())
1499 break;
1500
1501 LValueBaseInfo InnerBaseInfo;
1502 TBAAAccessInfo InnerTBAAInfo;
1503 Address Addr = CGF.EmitPointerWithAlignment(
1504 Addr: CE->getSubExpr(), BaseInfo: &InnerBaseInfo, TBAAInfo: &InnerTBAAInfo, IsKnownNonNull);
1505 if (BaseInfo) *BaseInfo = InnerBaseInfo;
1506 if (TBAAInfo) *TBAAInfo = InnerTBAAInfo;
1507
1508 if (isa<ExplicitCastExpr>(Val: CE)) {
1509 LValueBaseInfo TargetTypeBaseInfo;
1510 TBAAAccessInfo TargetTypeTBAAInfo;
1511 CharUnits Align = CGF.CGM.getNaturalPointeeTypeAlignment(
1512 T: E->getType(), BaseInfo: &TargetTypeBaseInfo, TBAAInfo: &TargetTypeTBAAInfo);
1513 if (TBAAInfo)
1514 *TBAAInfo =
1515 CGF.CGM.mergeTBAAInfoForCast(SourceInfo: *TBAAInfo, TargetInfo: TargetTypeTBAAInfo);
1516 // If the source l-value is opaque, honor the alignment of the
1517 // casted-to type.
1518 if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) {
1519 if (BaseInfo)
1520 BaseInfo->mergeForCast(Info: TargetTypeBaseInfo);
1521 Addr.setAlignment(Align);
1522 }
1523 }
1524
1525 if (CGF.SanOpts.has(K: SanitizerKind::CFIUnrelatedCast) &&
1526 CE->getCastKind() == CK_BitCast) {
1527 if (auto PT = E->getType()->getAs<PointerType>())
1528 CGF.EmitVTablePtrCheckForCast(T: PT->getPointeeType(), Derived: Addr,
1529 /*MayBeNull=*/true,
1530 TCK: CodeGenFunction::CFITCK_UnrelatedCast,
1531 Loc: CE->getBeginLoc());
1532 }
1533
1534 llvm::Type *ElemTy =
1535 CGF.ConvertTypeForMem(T: E->getType()->getPointeeType());
1536 Addr = Addr.withElementType(ElemTy);
1537 if (CE->getCastKind() == CK_AddressSpaceConversion)
1538 Addr = CGF.Builder.CreateAddrSpaceCast(
1539 Addr, Ty: CGF.ConvertType(T: E->getType()), ElementTy: ElemTy);
1540
1541 return CGF.authPointerToPointerCast(Ptr: Addr, SourceType: CE->getSubExpr()->getType(),
1542 DestType: CE->getType());
1543 }
1544 break;
1545
1546 // Array-to-pointer decay.
1547 case CK_ArrayToPointerDecay:
1548 return CGF.EmitArrayToPointerDecay(Array: CE->getSubExpr(), BaseInfo, TBAAInfo);
1549
1550 // Derived-to-base conversions.
1551 case CK_UncheckedDerivedToBase:
1552 case CK_DerivedToBase: {
1553 // TODO: Support accesses to members of base classes in TBAA. For now, we
1554 // conservatively pretend that the complete object is of the base class
1555 // type.
1556 if (TBAAInfo)
1557 *TBAAInfo = CGF.CGM.getTBAAAccessInfo(AccessType: E->getType());
1558 Address Addr = CGF.EmitPointerWithAlignment(
1559 Addr: CE->getSubExpr(), BaseInfo, TBAAInfo: nullptr,
1560 IsKnownNonNull: (KnownNonNull_t)(IsKnownNonNull ||
1561 CE->getCastKind() == CK_UncheckedDerivedToBase));
1562 auto Derived = CE->getSubExpr()->getType()->getPointeeCXXRecordDecl();
1563 return CGF.GetAddressOfBaseClass(
1564 Value: Addr, Derived, PathBegin: CE->path_begin(), PathEnd: CE->path_end(),
1565 NullCheckValue: CGF.ShouldNullCheckClassCastValue(Cast: CE), Loc: CE->getExprLoc());
1566 }
1567
1568 // TODO: Is there any reason to treat base-to-derived conversions
1569 // specially?
1570 default:
1571 break;
1572 }
1573 }
1574
1575 // Unary &.
1576 if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(Val: E)) {
1577 if (UO->getOpcode() == UO_AddrOf) {
1578 LValue LV = CGF.EmitLValue(E: UO->getSubExpr(), IsKnownNonNull);
1579 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1580 if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1581 return LV.getAddress();
1582 }
1583 }
1584
1585 // std::addressof and variants.
1586 if (auto *Call = dyn_cast<CallExpr>(Val: E)) {
1587 switch (Call->getBuiltinCallee()) {
1588 default:
1589 break;
1590 case Builtin::BIaddressof:
1591 case Builtin::BI__addressof:
1592 case Builtin::BI__builtin_addressof: {
1593 LValue LV = CGF.EmitLValue(E: Call->getArg(Arg: 0), IsKnownNonNull);
1594 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1595 if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1596 return LV.getAddress();
1597 }
1598 }
1599 }
1600
1601 // Pointer arithmetic: pointer +/- index.
1602 if (auto *BO = dyn_cast<BinaryOperator>(Val: E)) {
1603 if (BO->isAdditiveOp())
1604 return emitPointerArithmetic(CGF, BO, BaseInfo, TBAAInfo, IsKnownNonNull);
1605 }
1606
1607 // TODO: conditional operators, comma.
1608
1609 // Otherwise, use the alignment of the type.
1610 return CGF.makeNaturalAddressForPointer(
1611 Ptr: CGF.EmitScalarExpr(E), T: E->getType()->getPointeeType(), Alignment: CharUnits(),
1612 /*ForPointeeType=*/true, BaseInfo, TBAAInfo, IsKnownNonNull);
1613}
1614
1615/// EmitPointerWithAlignment - Given an expression of pointer type, try to
1616/// derive a more accurate bound on the alignment of the pointer.
1617Address CodeGenFunction::EmitPointerWithAlignment(
1618 const Expr *E, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo,
1619 KnownNonNull_t IsKnownNonNull) {
1620 Address Addr =
1621 ::EmitPointerWithAlignment(E, BaseInfo, TBAAInfo, IsKnownNonNull, CGF&: *this);
1622 if (IsKnownNonNull && !Addr.isKnownNonNull())
1623 Addr.setKnownNonNull();
1624 return Addr;
1625}
1626
1627llvm::Value *CodeGenFunction::EmitNonNullRValueCheck(RValue RV, QualType T) {
1628 llvm::Value *V = RV.getScalarVal();
1629 if (auto MPT = T->getAs<MemberPointerType>())
1630 return CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF&: *this, MemPtr: V, MPT);
1631 return Builder.CreateICmpNE(LHS: V, RHS: llvm::Constant::getNullValue(Ty: V->getType()));
1632}
1633
1634RValue CodeGenFunction::GetUndefRValue(QualType Ty) {
1635 if (Ty->isVoidType())
1636 return RValue::get(V: nullptr);
1637
1638 switch (getEvaluationKind(T: Ty)) {
1639 case TEK_Complex: {
1640 llvm::Type *EltTy =
1641 ConvertType(T: Ty->castAs<ComplexType>()->getElementType());
1642 llvm::Value *U = llvm::UndefValue::get(T: EltTy);
1643 return RValue::getComplex(C: std::make_pair(x&: U, y&: U));
1644 }
1645
1646 // If this is a use of an undefined aggregate type, the aggregate must have an
1647 // identifiable address. Just because the contents of the value are undefined
1648 // doesn't mean that the address can't be taken and compared.
1649 case TEK_Aggregate: {
1650 Address DestPtr = CreateMemTemp(Ty, Name: "undef.agg.tmp");
1651 return RValue::getAggregate(addr: DestPtr);
1652 }
1653
1654 case TEK_Scalar:
1655 return RValue::get(V: llvm::UndefValue::get(T: ConvertType(T: Ty)));
1656 }
1657 llvm_unreachable("bad evaluation kind");
1658}
1659
1660RValue CodeGenFunction::EmitUnsupportedRValue(const Expr *E,
1661 const char *Name) {
1662 ErrorUnsupported(S: E, Type: Name);
1663 return GetUndefRValue(Ty: E->getType());
1664}
1665
1666LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E,
1667 const char *Name) {
1668 ErrorUnsupported(S: E, Type: Name);
1669 llvm::Type *ElTy = ConvertType(T: E->getType());
1670 llvm::Type *Ty = DefaultPtrTy;
1671 return MakeAddrLValue(
1672 Addr: Address(llvm::UndefValue::get(T: Ty), ElTy, CharUnits::One()), T: E->getType());
1673}
1674
1675bool CodeGenFunction::IsWrappedCXXThis(const Expr *Obj) {
1676 const Expr *Base = Obj;
1677 while (!isa<CXXThisExpr>(Val: Base)) {
1678 // The result of a dynamic_cast can be null.
1679 if (isa<CXXDynamicCastExpr>(Val: Base))
1680 return false;
1681
1682 if (const auto *CE = dyn_cast<CastExpr>(Val: Base)) {
1683 Base = CE->getSubExpr();
1684 } else if (const auto *PE = dyn_cast<ParenExpr>(Val: Base)) {
1685 Base = PE->getSubExpr();
1686 } else if (const auto *UO = dyn_cast<UnaryOperator>(Val: Base)) {
1687 if (UO->getOpcode() == UO_Extension)
1688 Base = UO->getSubExpr();
1689 else
1690 return false;
1691 } else {
1692 return false;
1693 }
1694 }
1695 return true;
1696}
1697
1698LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {
1699 LValue LV;
1700 if (SanOpts.has(K: SanitizerKind::ArrayBounds) && isa<ArraySubscriptExpr>(Val: E))
1701 LV = EmitArraySubscriptExpr(E: cast<ArraySubscriptExpr>(Val: E), /*Accessed*/true);
1702 else
1703 LV = EmitLValue(E);
1704 if (!isa<DeclRefExpr>(Val: E) && !LV.isBitField() && LV.isSimple()) {
1705 SanitizerSet SkippedChecks;
1706 if (const auto *ME = dyn_cast<MemberExpr>(Val: E)) {
1707 bool IsBaseCXXThis = IsWrappedCXXThis(Obj: ME->getBase());
1708 if (IsBaseCXXThis)
1709 SkippedChecks.set(K: SanitizerKind::Alignment, Value: true);
1710 if (IsBaseCXXThis || isa<DeclRefExpr>(Val: ME->getBase()))
1711 SkippedChecks.set(K: SanitizerKind::Null, Value: true);
1712 }
1713 EmitTypeCheck(TCK, Loc: E->getExprLoc(), LV, Type: E->getType(), SkippedChecks);
1714 }
1715 return LV;
1716}
1717
1718/// EmitLValue - Emit code to compute a designator that specifies the location
1719/// of the expression.
1720///
1721/// This can return one of two things: a simple address or a bitfield reference.
1722/// In either case, the LLVM Value* in the LValue structure is guaranteed to be
1723/// an LLVM pointer type.
1724///
1725/// If this returns a bitfield reference, nothing about the pointee type of the
1726/// LLVM value is known: For example, it may not be a pointer to an integer.
1727///
1728/// If this returns a normal address, and if the lvalue's C type is fixed size,
1729/// this method guarantees that the returned pointer type will point to an LLVM
1730/// type of the same size of the lvalue's type. If the lvalue has a variable
1731/// length type, this is not possible.
1732///
1733LValue CodeGenFunction::EmitLValue(const Expr *E,
1734 KnownNonNull_t IsKnownNonNull) {
1735 // Running with sufficient stack space to avoid deeply nested expressions
1736 // cause a stack overflow.
1737 LValue LV;
1738 CGM.runWithSufficientStackSpace(
1739 Loc: E->getExprLoc(), Fn: [&] { LV = EmitLValueHelper(E, IsKnownNonNull); });
1740
1741 if (IsKnownNonNull && !LV.isKnownNonNull())
1742 LV.setKnownNonNull();
1743 return LV;
1744}
1745
1746LValue CodeGenFunction::EmitLValueHelper(const Expr *E,
1747 KnownNonNull_t IsKnownNonNull) {
1748 ApplyDebugLocation DL(*this, E);
1749 switch (E->getStmtClass()) {
1750 default: return EmitUnsupportedLValue(E, Name: "l-value expression");
1751
1752 case Expr::ObjCPropertyRefExprClass:
1753 llvm_unreachable("cannot emit a property reference directly");
1754
1755 case Expr::ObjCSelectorExprClass:
1756 return EmitObjCSelectorLValue(E: cast<ObjCSelectorExpr>(Val: E));
1757 case Expr::ObjCIsaExprClass:
1758 return EmitObjCIsaExpr(E: cast<ObjCIsaExpr>(Val: E));
1759 case Expr::BinaryOperatorClass:
1760 return EmitBinaryOperatorLValue(E: cast<BinaryOperator>(Val: E));
1761 case Expr::CompoundAssignOperatorClass: {
1762 QualType Ty = E->getType();
1763 if (const AtomicType *AT = Ty->getAs<AtomicType>())
1764 Ty = AT->getValueType();
1765 if (!Ty->isAnyComplexType())
1766 return EmitCompoundAssignmentLValue(E: cast<CompoundAssignOperator>(Val: E));
1767 return EmitComplexCompoundAssignmentLValue(E: cast<CompoundAssignOperator>(Val: E));
1768 }
1769 case Expr::CallExprClass:
1770 case Expr::CXXMemberCallExprClass:
1771 case Expr::CXXOperatorCallExprClass:
1772 case Expr::UserDefinedLiteralClass:
1773 return EmitCallExprLValue(E: cast<CallExpr>(Val: E));
1774 case Expr::CXXRewrittenBinaryOperatorClass:
1775 return EmitLValue(E: cast<CXXRewrittenBinaryOperator>(Val: E)->getSemanticForm(),
1776 IsKnownNonNull);
1777 case Expr::VAArgExprClass:
1778 return EmitVAArgExprLValue(E: cast<VAArgExpr>(Val: E));
1779 case Expr::DeclRefExprClass:
1780 return EmitDeclRefLValue(E: cast<DeclRefExpr>(Val: E));
1781 case Expr::ConstantExprClass: {
1782 const ConstantExpr *CE = cast<ConstantExpr>(Val: E);
1783 if (llvm::Value *Result = ConstantEmitter(*this).tryEmitConstantExpr(CE))
1784 return MakeNaturalAlignPointeeAddrLValue(V: Result, T: CE->getType());
1785 return EmitLValue(E: cast<ConstantExpr>(Val: E)->getSubExpr(), IsKnownNonNull);
1786 }
1787 case Expr::ParenExprClass:
1788 return EmitLValue(E: cast<ParenExpr>(Val: E)->getSubExpr(), IsKnownNonNull);
1789 case Expr::GenericSelectionExprClass:
1790 return EmitLValue(E: cast<GenericSelectionExpr>(Val: E)->getResultExpr(),
1791 IsKnownNonNull);
1792 case Expr::PredefinedExprClass:
1793 return EmitPredefinedLValue(E: cast<PredefinedExpr>(Val: E));
1794 case Expr::StringLiteralClass:
1795 return EmitStringLiteralLValue(E: cast<StringLiteral>(Val: E));
1796 case Expr::ObjCEncodeExprClass:
1797 return EmitObjCEncodeExprLValue(E: cast<ObjCEncodeExpr>(Val: E));
1798 case Expr::PseudoObjectExprClass:
1799 return EmitPseudoObjectLValue(e: cast<PseudoObjectExpr>(Val: E));
1800 case Expr::InitListExprClass:
1801 return EmitInitListLValue(E: cast<InitListExpr>(Val: E));
1802 case Expr::CXXTemporaryObjectExprClass:
1803 case Expr::CXXConstructExprClass:
1804 return EmitCXXConstructLValue(E: cast<CXXConstructExpr>(Val: E));
1805 case Expr::CXXBindTemporaryExprClass:
1806 return EmitCXXBindTemporaryLValue(E: cast<CXXBindTemporaryExpr>(Val: E));
1807 case Expr::CXXUuidofExprClass:
1808 return EmitCXXUuidofLValue(E: cast<CXXUuidofExpr>(Val: E));
1809 case Expr::LambdaExprClass:
1810 return EmitAggExprToLValue(E);
1811
1812 case Expr::ExprWithCleanupsClass: {
1813 const auto *cleanups = cast<ExprWithCleanups>(Val: E);
1814 RunCleanupsScope Scope(*this);
1815 LValue LV = EmitLValue(E: cleanups->getSubExpr(), IsKnownNonNull);
1816 if (LV.isSimple()) {
1817 // Defend against branches out of gnu statement expressions surrounded by
1818 // cleanups.
1819 Address Addr = LV.getAddress();
1820 llvm::Value *V = Addr.getBasePointer();
1821 Scope.ForceCleanup(ValuesToReload: {&V});
1822 Addr.replaceBasePointer(P: V);
1823 return LValue::MakeAddr(Addr, type: LV.getType(), Context&: getContext(),
1824 BaseInfo: LV.getBaseInfo(), TBAAInfo: LV.getTBAAInfo());
1825 }
1826 // FIXME: Is it possible to create an ExprWithCleanups that produces a
1827 // bitfield lvalue or some other non-simple lvalue?
1828 return LV;
1829 }
1830
1831 case Expr::CXXDefaultArgExprClass: {
1832 auto *DAE = cast<CXXDefaultArgExpr>(Val: E);
1833 CXXDefaultArgExprScope Scope(*this, DAE);
1834 return EmitLValue(E: DAE->getExpr(), IsKnownNonNull);
1835 }
1836 case Expr::CXXDefaultInitExprClass: {
1837 auto *DIE = cast<CXXDefaultInitExpr>(Val: E);
1838 CXXDefaultInitExprScope Scope(*this, DIE);
1839 return EmitLValue(E: DIE->getExpr(), IsKnownNonNull);
1840 }
1841 case Expr::CXXTypeidExprClass:
1842 return EmitCXXTypeidLValue(E: cast<CXXTypeidExpr>(Val: E));
1843
1844 case Expr::ObjCMessageExprClass:
1845 return EmitObjCMessageExprLValue(E: cast<ObjCMessageExpr>(Val: E));
1846 case Expr::ObjCIvarRefExprClass:
1847 return EmitObjCIvarRefLValue(E: cast<ObjCIvarRefExpr>(Val: E));
1848 case Expr::StmtExprClass:
1849 return EmitStmtExprLValue(E: cast<StmtExpr>(Val: E));
1850 case Expr::UnaryOperatorClass:
1851 return EmitUnaryOpLValue(E: cast<UnaryOperator>(Val: E));
1852 case Expr::ArraySubscriptExprClass:
1853 return EmitArraySubscriptExpr(E: cast<ArraySubscriptExpr>(Val: E));
1854 case Expr::MatrixSingleSubscriptExprClass:
1855 return EmitMatrixSingleSubscriptExpr(E: cast<MatrixSingleSubscriptExpr>(Val: E));
1856 case Expr::MatrixSubscriptExprClass:
1857 return EmitMatrixSubscriptExpr(E: cast<MatrixSubscriptExpr>(Val: E));
1858 case Expr::ArraySectionExprClass:
1859 return EmitArraySectionExpr(E: cast<ArraySectionExpr>(Val: E));
1860 case Expr::ExtVectorElementExprClass:
1861 return EmitExtVectorElementExpr(E: cast<ExtVectorElementExpr>(Val: E));
1862 case Expr::MatrixElementExprClass:
1863 return EmitMatrixElementExpr(E: cast<MatrixElementExpr>(Val: E));
1864 case Expr::CXXThisExprClass:
1865 return MakeAddrLValue(Addr: LoadCXXThisAddress(), T: E->getType());
1866 case Expr::MemberExprClass:
1867 return EmitMemberExpr(E: cast<MemberExpr>(Val: E));
1868 case Expr::CompoundLiteralExprClass:
1869 return EmitCompoundLiteralLValue(E: cast<CompoundLiteralExpr>(Val: E));
1870 case Expr::ConditionalOperatorClass:
1871 return EmitConditionalOperatorLValue(E: cast<ConditionalOperator>(Val: E));
1872 case Expr::BinaryConditionalOperatorClass:
1873 return EmitConditionalOperatorLValue(E: cast<BinaryConditionalOperator>(Val: E));
1874 case Expr::ChooseExprClass:
1875 return EmitLValue(E: cast<ChooseExpr>(Val: E)->getChosenSubExpr(), IsKnownNonNull);
1876 case Expr::OpaqueValueExprClass:
1877 return EmitOpaqueValueLValue(e: cast<OpaqueValueExpr>(Val: E));
1878 case Expr::SubstNonTypeTemplateParmExprClass:
1879 return EmitLValue(E: cast<SubstNonTypeTemplateParmExpr>(Val: E)->getReplacement(),
1880 IsKnownNonNull);
1881 case Expr::ImplicitCastExprClass:
1882 case Expr::CStyleCastExprClass:
1883 case Expr::CXXFunctionalCastExprClass:
1884 case Expr::CXXStaticCastExprClass:
1885 case Expr::CXXDynamicCastExprClass:
1886 case Expr::CXXReinterpretCastExprClass:
1887 case Expr::CXXConstCastExprClass:
1888 case Expr::CXXAddrspaceCastExprClass:
1889 case Expr::ObjCBridgedCastExprClass:
1890 return EmitCastLValue(E: cast<CastExpr>(Val: E));
1891
1892 case Expr::MaterializeTemporaryExprClass:
1893 return EmitMaterializeTemporaryExpr(M: cast<MaterializeTemporaryExpr>(Val: E));
1894
1895 case Expr::CoawaitExprClass:
1896 return EmitCoawaitLValue(E: cast<CoawaitExpr>(Val: E));
1897 case Expr::CoyieldExprClass:
1898 return EmitCoyieldLValue(E: cast<CoyieldExpr>(Val: E));
1899 case Expr::PackIndexingExprClass:
1900 return EmitLValue(E: cast<PackIndexingExpr>(Val: E)->getSelectedExpr());
1901 case Expr::HLSLOutArgExprClass:
1902 llvm_unreachable("cannot emit a HLSL out argument directly");
1903 }
1904}
1905
1906/// Given an object of the given canonical type, can we safely copy a
1907/// value out of it based on its initializer?
1908static bool isConstantEmittableObjectType(QualType type) {
1909 assert(type.isCanonical());
1910 assert(!type->isReferenceType());
1911
1912 // Must be const-qualified but non-volatile.
1913 Qualifiers qs = type.getLocalQualifiers();
1914 if (!qs.hasConst() || qs.hasVolatile()) return false;
1915
1916 // Otherwise, all object types satisfy this except C++ classes with
1917 // mutable subobjects or non-trivial copy/destroy behavior.
1918 if (const auto *RT = dyn_cast<RecordType>(Val&: type))
1919 if (const auto *RD = dyn_cast<CXXRecordDecl>(Val: RT->getDecl())) {
1920 RD = RD->getDefinitionOrSelf();
1921 if (RD->hasMutableFields() || !RD->isTrivial())
1922 return false;
1923 }
1924
1925 return true;
1926}
1927
1928/// Can we constant-emit a load of a reference to a variable of the
1929/// given type? This is different from predicates like
1930/// Decl::mightBeUsableInConstantExpressions because we do want it to apply
1931/// in situations that don't necessarily satisfy the language's rules
1932/// for this (e.g. C++'s ODR-use rules). For example, we want to able
1933/// to do this with const float variables even if those variables
1934/// aren't marked 'constexpr'.
1935enum ConstantEmissionKind {
1936 CEK_None,
1937 CEK_AsReferenceOnly,
1938 CEK_AsValueOrReference,
1939 CEK_AsValueOnly
1940};
1941static ConstantEmissionKind checkVarTypeForConstantEmission(QualType type) {
1942 type = type.getCanonicalType();
1943 if (const auto *ref = dyn_cast<ReferenceType>(Val&: type)) {
1944 if (isConstantEmittableObjectType(type: ref->getPointeeType()))
1945 return CEK_AsValueOrReference;
1946 return CEK_AsReferenceOnly;
1947 }
1948 if (isConstantEmittableObjectType(type))
1949 return CEK_AsValueOnly;
1950 return CEK_None;
1951}
1952
1953/// Try to emit a reference to the given value without producing it as
1954/// an l-value. This is just an optimization, but it avoids us needing
1955/// to emit global copies of variables if they're named without triggering
1956/// a formal use in a context where we can't emit a direct reference to them,
1957/// for instance if a block or lambda or a member of a local class uses a
1958/// const int variable or constexpr variable from an enclosing function.
1959CodeGenFunction::ConstantEmission
1960CodeGenFunction::tryEmitAsConstant(const DeclRefExpr *RefExpr) {
1961 const ValueDecl *Value = RefExpr->getDecl();
1962
1963 // The value needs to be an enum constant or a constant variable.
1964 ConstantEmissionKind CEK;
1965 if (isa<ParmVarDecl>(Val: Value)) {
1966 CEK = CEK_None;
1967 } else if (const auto *var = dyn_cast<VarDecl>(Val: Value)) {
1968 CEK = checkVarTypeForConstantEmission(type: var->getType());
1969 } else if (isa<EnumConstantDecl>(Val: Value)) {
1970 CEK = CEK_AsValueOnly;
1971 } else {
1972 CEK = CEK_None;
1973 }
1974 if (CEK == CEK_None) return ConstantEmission();
1975
1976 Expr::EvalResult result;
1977 bool resultIsReference;
1978 QualType resultType;
1979
1980 // It's best to evaluate all the way as an r-value if that's permitted.
1981 if (CEK != CEK_AsReferenceOnly &&
1982 RefExpr->EvaluateAsRValue(Result&: result, Ctx: getContext())) {
1983 resultIsReference = false;
1984 resultType = RefExpr->getType().getUnqualifiedType();
1985
1986 // Otherwise, try to evaluate as an l-value.
1987 } else if (CEK != CEK_AsValueOnly &&
1988 RefExpr->EvaluateAsLValue(Result&: result, Ctx: getContext())) {
1989 resultIsReference = true;
1990 resultType = Value->getType();
1991
1992 // Failure.
1993 } else {
1994 return ConstantEmission();
1995 }
1996
1997 // In any case, if the initializer has side-effects, abandon ship.
1998 if (result.HasSideEffects)
1999 return ConstantEmission();
2000
2001 // In CUDA/HIP device compilation, a lambda may capture a reference variable
2002 // referencing a global host variable by copy. In this case the lambda should
2003 // make a copy of the value of the global host variable. The DRE of the
2004 // captured reference variable cannot be emitted as load from the host
2005 // global variable as compile time constant, since the host variable is not
2006 // accessible on device. The DRE of the captured reference variable has to be
2007 // loaded from captures.
2008 if (CGM.getLangOpts().CUDAIsDevice && result.Val.isLValue() &&
2009 RefExpr->refersToEnclosingVariableOrCapture()) {
2010 auto *MD = dyn_cast_or_null<CXXMethodDecl>(Val: CurCodeDecl);
2011 if (isLambdaMethod(DC: MD) && MD->getOverloadedOperator() == OO_Call) {
2012 const APValue::LValueBase &base = result.Val.getLValueBase();
2013 if (const ValueDecl *D = base.dyn_cast<const ValueDecl *>()) {
2014 if (const VarDecl *VD = dyn_cast<const VarDecl>(Val: D)) {
2015 if (!VD->hasAttr<CUDADeviceAttr>()) {
2016 return ConstantEmission();
2017 }
2018 }
2019 }
2020 }
2021 }
2022
2023 // Emit as a constant.
2024 llvm::Constant *C = ConstantEmitter(*this).emitAbstract(
2025 loc: RefExpr->getLocation(), value: result.Val, T: resultType);
2026
2027 // Make sure we emit a debug reference to the global variable.
2028 // This should probably fire even for
2029 if (isa<VarDecl>(Val: Value)) {
2030 if (!getContext().DeclMustBeEmitted(D: cast<VarDecl>(Val: Value)))
2031 EmitDeclRefExprDbgValue(E: RefExpr, Init: result.Val);
2032 } else {
2033 assert(isa<EnumConstantDecl>(Value));
2034 EmitDeclRefExprDbgValue(E: RefExpr, Init: result.Val);
2035 }
2036
2037 // If we emitted a reference constant, we need to dereference that.
2038 if (resultIsReference)
2039 return ConstantEmission::forReference(C);
2040
2041 return ConstantEmission::forValue(C);
2042}
2043
2044static DeclRefExpr *tryToConvertMemberExprToDeclRefExpr(CodeGenFunction &CGF,
2045 const MemberExpr *ME) {
2046 if (auto *VD = dyn_cast<VarDecl>(Val: ME->getMemberDecl())) {
2047 // Try to emit static variable member expressions as DREs.
2048 return DeclRefExpr::Create(
2049 Context: CGF.getContext(), QualifierLoc: NestedNameSpecifierLoc(), TemplateKWLoc: SourceLocation(), D: VD,
2050 /*RefersToEnclosingVariableOrCapture=*/false, NameLoc: ME->getExprLoc(),
2051 T: ME->getType(), VK: ME->getValueKind(), FoundD: nullptr, TemplateArgs: nullptr, NOUR: ME->isNonOdrUse());
2052 }
2053 return nullptr;
2054}
2055
2056CodeGenFunction::ConstantEmission
2057CodeGenFunction::tryEmitAsConstant(const MemberExpr *ME) {
2058 if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(CGF&: *this, ME))
2059 return tryEmitAsConstant(RefExpr: DRE);
2060 return ConstantEmission();
2061}
2062
2063llvm::Value *CodeGenFunction::emitScalarConstant(
2064 const CodeGenFunction::ConstantEmission &Constant, Expr *E) {
2065 assert(Constant && "not a constant");
2066 if (Constant.isReference())
2067 return EmitLoadOfLValue(V: Constant.getReferenceLValue(CGF&: *this, RefExpr: E),
2068 Loc: E->getExprLoc())
2069 .getScalarVal();
2070 return Constant.getValue();
2071}
2072
2073llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue,
2074 SourceLocation Loc) {
2075 return EmitLoadOfScalar(Addr: lvalue.getAddress(), Volatile: lvalue.isVolatile(),
2076 Ty: lvalue.getType(), Loc, BaseInfo: lvalue.getBaseInfo(),
2077 TBAAInfo: lvalue.getTBAAInfo(), isNontemporal: lvalue.isNontemporal());
2078}
2079
2080// This method SHOULD NOT be extended to support additional types, like BitInt
2081// types, without an opt-in bool controlled by a CodeGenOptions setting (like
2082// -fstrict-bool) and a new UBSan check (like SanitizerKind::Bool) as breaking
2083// that assumption would lead to memory corruption. See link for examples of how
2084// having a bool that has a value different from 0 or 1 in memory can lead to
2085// memory corruption.
2086// https://discourse.llvm.org/t/defining-what-happens-when-a-bool-isn-t-0-or-1/86778
2087static bool getRangeForType(CodeGenFunction &CGF, QualType Ty, llvm::APInt &Min,
2088 llvm::APInt &End, bool StrictEnums, bool StrictBool,
2089 bool IsBool) {
2090 const auto *ED = Ty->getAsEnumDecl();
2091 bool IsRegularCPlusPlusEnum =
2092 CGF.getLangOpts().CPlusPlus && StrictEnums && ED && !ED->isFixed();
2093 if (!IsBool && !IsRegularCPlusPlusEnum)
2094 return false;
2095
2096 if (IsBool) {
2097 if (!StrictBool)
2098 return false;
2099 Min = llvm::APInt(CGF.getContext().getTypeSize(T: Ty), 0);
2100 End = llvm::APInt(CGF.getContext().getTypeSize(T: Ty), 2);
2101 } else {
2102 ED->getValueRange(Max&: End, Min);
2103 }
2104 return true;
2105}
2106
2107llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) {
2108 llvm::APInt Min, End;
2109 bool IsBool = Ty->hasBooleanRepresentation() && !Ty->isVectorType();
2110 bool StrictBoolEnabled = CGM.getCodeGenOpts().getLoadBoolFromMem() ==
2111 CodeGenOptions::BoolFromMem::Strict;
2112 if (!getRangeForType(CGF&: *this, Ty, Min, End,
2113 /*StrictEnums=*/CGM.getCodeGenOpts().StrictEnums,
2114 /*StrictBool=*/StrictBoolEnabled, /*IsBool=*/IsBool))
2115 return nullptr;
2116
2117 llvm::MDBuilder MDHelper(getLLVMContext());
2118 return MDHelper.createRange(Lo: Min, Hi: End);
2119}
2120
2121void CodeGenFunction::maybeAttachRangeForLoad(llvm::LoadInst *Load, QualType Ty,
2122 SourceLocation Loc) {
2123 if (EmitScalarRangeCheck(Value: Load, Ty, Loc)) {
2124 // In order to prevent the optimizer from throwing away the check, don't
2125 // attach range metadata to the load.
2126 } else if (CGM.getCodeGenOpts().isOptimizedBuild()) {
2127 if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty)) {
2128 Load->setMetadata(KindID: llvm::LLVMContext::MD_range, Node: RangeInfo);
2129 Load->setMetadata(KindID: llvm::LLVMContext::MD_noundef,
2130 Node: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: {}));
2131 }
2132 }
2133}
2134
2135bool CodeGenFunction::EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
2136 SourceLocation Loc) {
2137 bool HasBoolCheck = SanOpts.has(K: SanitizerKind::Bool);
2138 bool HasEnumCheck = SanOpts.has(K: SanitizerKind::Enum);
2139 if (!HasBoolCheck && !HasEnumCheck)
2140 return false;
2141
2142 bool IsBool = (Ty->hasBooleanRepresentation() && !Ty->isVectorType()) ||
2143 NSAPI(CGM.getContext()).isObjCBOOLType(T: Ty);
2144 bool NeedsBoolCheck = HasBoolCheck && IsBool;
2145 bool NeedsEnumCheck = HasEnumCheck && Ty->isEnumeralType();
2146 if (!NeedsBoolCheck && !NeedsEnumCheck)
2147 return false;
2148
2149 // Single-bit booleans don't need to be checked. Special-case this to avoid
2150 // a bit width mismatch when handling bitfield values. This is handled by
2151 // EmitFromMemory for the non-bitfield case.
2152 if (IsBool &&
2153 cast<llvm::IntegerType>(Val: Value->getType())->getBitWidth() == 1)
2154 return false;
2155
2156 if (NeedsEnumCheck &&
2157 getContext().isTypeIgnoredBySanitizer(Mask: SanitizerKind::Enum, Ty))
2158 return false;
2159
2160 llvm::APInt Min, End;
2161 if (!getRangeForType(CGF&: *this, Ty, Min, End, /*StrictEnums=*/true,
2162 /*StrictBool=*/true, IsBool))
2163 return true;
2164
2165 SanitizerKind::SanitizerOrdinal Kind =
2166 NeedsEnumCheck ? SanitizerKind::SO_Enum : SanitizerKind::SO_Bool;
2167
2168 auto &Ctx = getLLVMContext();
2169 auto CheckHandler = SanitizerHandler::LoadInvalidValue;
2170 SanitizerDebugLocation SanScope(this, {Kind}, CheckHandler);
2171 llvm::Value *Check;
2172 --End;
2173 if (!Min) {
2174 Check = Builder.CreateICmpULE(LHS: Value, RHS: llvm::ConstantInt::get(Context&: Ctx, V: End));
2175 } else {
2176 llvm::Value *Upper =
2177 Builder.CreateICmpSLE(LHS: Value, RHS: llvm::ConstantInt::get(Context&: Ctx, V: End));
2178 llvm::Value *Lower =
2179 Builder.CreateICmpSGE(LHS: Value, RHS: llvm::ConstantInt::get(Context&: Ctx, V: Min));
2180 Check = Builder.CreateAnd(LHS: Upper, RHS: Lower);
2181 }
2182 llvm::Constant *StaticArgs[] = {EmitCheckSourceLocation(Loc),
2183 EmitCheckTypeDescriptor(T: Ty)};
2184 EmitCheck(Checked: std::make_pair(x&: Check, y&: Kind), Check: CheckHandler, StaticArgs, DynamicArgs: Value);
2185 return true;
2186}
2187
2188llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
2189 QualType Ty,
2190 SourceLocation Loc,
2191 LValueBaseInfo BaseInfo,
2192 TBAAAccessInfo TBAAInfo,
2193 bool isNontemporal) {
2194 if (auto *GV = dyn_cast<llvm::GlobalValue>(Val: Addr.getBasePointer()))
2195 if (GV->isThreadLocal())
2196 Addr = Addr.withPointer(NewPointer: Builder.CreateThreadLocalAddress(Ptr: GV),
2197 IsKnownNonNull: NotKnownNonNull);
2198
2199 if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
2200 // Boolean vectors use `iN` as storage type.
2201 if (ClangVecTy->isPackedVectorBoolType(ctx: getContext())) {
2202 llvm::Type *ValTy = ConvertType(T: Ty);
2203 unsigned ValNumElems =
2204 cast<llvm::FixedVectorType>(Val: ValTy)->getNumElements();
2205 // Load the `iP` storage object (P is the padded vector size).
2206 auto *RawIntV = Builder.CreateLoad(Addr, IsVolatile: Volatile, Name: "load_bits");
2207 const auto *RawIntTy = RawIntV->getType();
2208 assert(RawIntTy->isIntegerTy() && "compressed iN storage for bitvectors");
2209 // Bitcast iP --> <P x i1>.
2210 auto *PaddedVecTy = llvm::FixedVectorType::get(
2211 ElementType: Builder.getInt1Ty(), NumElts: RawIntTy->getPrimitiveSizeInBits());
2212 llvm::Value *V = Builder.CreateBitCast(V: RawIntV, DestTy: PaddedVecTy);
2213 // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
2214 V = emitBoolVecConversion(SrcVec: V, NumElementsDst: ValNumElems, Name: "extractvec");
2215
2216 return EmitFromMemory(Value: V, Ty);
2217 }
2218
2219 // Handles vectors of sizes that are likely to be expanded to a larger size
2220 // to optimize performance.
2221 auto *VTy = cast<llvm::FixedVectorType>(Val: Addr.getElementType());
2222 auto *NewVecTy =
2223 CGM.getABIInfo().getOptimalVectorMemoryType(T: VTy, Opt: getLangOpts());
2224
2225 if (VTy != NewVecTy) {
2226 Address Cast = Addr.withElementType(ElemTy: NewVecTy);
2227 llvm::Value *V = Builder.CreateLoad(Addr: Cast, IsVolatile: Volatile, Name: "loadVecN");
2228 unsigned OldNumElements = VTy->getNumElements();
2229 SmallVector<int, 16> Mask(OldNumElements);
2230 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
2231 V = Builder.CreateShuffleVector(V, Mask, Name: "extractVec");
2232 return EmitFromMemory(Value: V, Ty);
2233 }
2234 }
2235
2236 // Atomic operations have to be done on integral types.
2237 LValue AtomicLValue =
2238 LValue::MakeAddr(Addr, type: Ty, Context&: getContext(), BaseInfo, TBAAInfo);
2239 if (Ty->isAtomicType() || LValueIsSuitableForInlineAtomic(Src: AtomicLValue)) {
2240 return EmitAtomicLoad(LV: AtomicLValue, SL: Loc).getScalarVal();
2241 }
2242
2243 Addr =
2244 Addr.withElementType(ElemTy: convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Addr.getElementType()));
2245
2246 llvm::LoadInst *Load = Builder.CreateLoad(Addr, IsVolatile: Volatile);
2247 if (isNontemporal) {
2248 llvm::MDNode *Node = llvm::MDNode::get(
2249 Context&: Load->getContext(), MDs: llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: 1)));
2250 Load->setMetadata(KindID: llvm::LLVMContext::MD_nontemporal, Node);
2251 }
2252
2253 CGM.DecorateInstructionWithTBAA(Inst: Load, TBAAInfo);
2254
2255 maybeAttachRangeForLoad(Load, Ty, Loc);
2256
2257 return EmitFromMemory(Value: Load, Ty);
2258}
2259
2260/// Converts a scalar value from its primary IR type (as returned
2261/// by ConvertType) to its load/store type (as returned by
2262/// convertTypeForLoadStore).
2263llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) {
2264 if (auto *AtomicTy = Ty->getAs<AtomicType>())
2265 Ty = AtomicTy->getValueType();
2266
2267 if (Ty->isExtVectorBoolType() || Ty->isConstantMatrixBoolType()) {
2268 llvm::Type *StoreTy = convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Value->getType());
2269
2270 if (Value->getType() == StoreTy)
2271 return Value;
2272
2273 if (StoreTy->isVectorTy() && StoreTy->getScalarSizeInBits() >
2274 Value->getType()->getScalarSizeInBits())
2275 return Builder.CreateZExt(V: Value, DestTy: StoreTy);
2276
2277 // Expand to the memory bit width.
2278 unsigned MemNumElems = StoreTy->getPrimitiveSizeInBits();
2279 // <N x i1> --> <P x i1>.
2280 Value = emitBoolVecConversion(SrcVec: Value, NumElementsDst: MemNumElems, Name: "insertvec");
2281 // <P x i1> --> iP.
2282 Value = Builder.CreateBitCast(V: Value, DestTy: StoreTy);
2283 }
2284
2285 if (Ty->hasBooleanRepresentation() || Ty->isBitIntType()) {
2286 llvm::Type *StoreTy = convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Value->getType());
2287 bool Signed = Ty->isSignedIntegerOrEnumerationType();
2288 return Builder.CreateIntCast(V: Value, DestTy: StoreTy, isSigned: Signed, Name: "storedv");
2289 }
2290
2291 return Value;
2292}
2293
2294/// Converts a scalar value from its load/store type (as returned
2295/// by convertTypeForLoadStore) to its primary IR type (as returned
2296/// by ConvertType).
2297llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
2298 if (auto *AtomicTy = Ty->getAs<AtomicType>())
2299 Ty = AtomicTy->getValueType();
2300
2301 if (Ty->isPackedVectorBoolType(ctx: getContext())) {
2302 const auto *RawIntTy = Value->getType();
2303
2304 // Bitcast iP --> <P x i1>.
2305 auto *PaddedVecTy = llvm::FixedVectorType::get(
2306 ElementType: Builder.getInt1Ty(), NumElts: RawIntTy->getPrimitiveSizeInBits());
2307 auto *V = Builder.CreateBitCast(V: Value, DestTy: PaddedVecTy);
2308 // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
2309 llvm::Type *ValTy = ConvertType(T: Ty);
2310 unsigned ValNumElems = cast<llvm::FixedVectorType>(Val: ValTy)->getNumElements();
2311 return emitBoolVecConversion(SrcVec: V, NumElementsDst: ValNumElems, Name: "extractvec");
2312 }
2313
2314 llvm::Type *ResTy = ConvertType(T: Ty);
2315 bool HasBoolRep = Ty->hasBooleanRepresentation() || Ty->isExtVectorBoolType();
2316 if (HasBoolRep && CGM.getCodeGenOpts().isConvertingBoolWithCmp0()) {
2317 return Builder.CreateICmpNE(
2318 LHS: Value, RHS: llvm::Constant::getNullValue(Ty: Value->getType()), Name: "loadedv");
2319 }
2320 if (HasBoolRep || Ty->isBitIntType())
2321 return Builder.CreateTrunc(V: Value, DestTy: ResTy, Name: "loadedv");
2322
2323 return Value;
2324}
2325
2326// Convert the pointer of \p Addr to a pointer to a vector (the value type of
2327// MatrixType), if it points to a array (the memory type of MatrixType).
2328static RawAddress MaybeConvertMatrixAddress(RawAddress Addr,
2329 CodeGenFunction &CGF,
2330 bool IsVector = true) {
2331 auto *ArrayTy = dyn_cast<llvm::ArrayType>(Val: Addr.getElementType());
2332 if (ArrayTy && IsVector) {
2333 auto ArrayElements = ArrayTy->getNumElements();
2334 auto *ArrayElementTy = ArrayTy->getElementType();
2335 if (CGF.getContext().getLangOpts().HLSL) {
2336 auto *VectorTy = cast<llvm::FixedVectorType>(Val: ArrayElementTy);
2337 ArrayElementTy = VectorTy->getElementType();
2338 ArrayElements *= VectorTy->getNumElements();
2339 }
2340 auto *VectorTy = llvm::FixedVectorType::get(ElementType: ArrayElementTy, NumElts: ArrayElements);
2341
2342 return Addr.withElementType(ElemTy: VectorTy);
2343 }
2344 auto *VectorTy = dyn_cast<llvm::VectorType>(Val: Addr.getElementType());
2345 if (VectorTy && !IsVector) {
2346 auto *ArrayTy = llvm::ArrayType::get(
2347 ElementType: VectorTy->getElementType(),
2348 NumElements: cast<llvm::FixedVectorType>(Val: VectorTy)->getNumElements());
2349
2350 return Addr.withElementType(ElemTy: ArrayTy);
2351 }
2352
2353 return Addr;
2354}
2355
2356LValue CodeGenFunction::EmitMatrixElementExpr(const MatrixElementExpr *E) {
2357 LValue Base;
2358 if (E->getBase()->isGLValue())
2359 Base = EmitLValue(E: E->getBase());
2360 else {
2361 assert(E->getBase()->getType()->isConstantMatrixType() &&
2362 "Result must be a Constant Matrix");
2363 llvm::Value *Mat = EmitScalarExpr(E: E->getBase());
2364 Address MatMem = CreateMemTemp(Ty: E->getBase()->getType());
2365 QualType Ty = E->getBase()->getType();
2366 llvm::Type *LTy = convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Mat->getType());
2367 if (LTy->getScalarSizeInBits() > Mat->getType()->getScalarSizeInBits())
2368 Mat = Builder.CreateZExt(V: Mat, DestTy: LTy);
2369 Builder.CreateStore(Val: Mat, Addr: MatMem);
2370 Base = MakeAddrLValue(Addr: MatMem, T: Ty, Source: AlignmentSource::Decl);
2371 }
2372 QualType ResultType =
2373 E->getType().withCVRQualifiers(CVR: Base.getQuals().getCVRQualifiers());
2374
2375 // Encode the element access list into a vector of unsigned indices.
2376 // getEncodedElementAccess returns row-major linearized indices.
2377 SmallVector<uint32_t, 4> Indices;
2378 E->getEncodedElementAccess(Elts&: Indices);
2379
2380 // getEncodedElementAccess returns row-major linearized indices
2381 // If the matrix memory layout is column-major, convert indices
2382 // to column-major indices.
2383 bool IsRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: E->getBase()->getType());
2384 if (!IsRowMajor) {
2385 const auto *MT = E->getBase()->getType()->castAs<ConstantMatrixType>();
2386 unsigned NumCols = MT->getNumColumns();
2387 for (uint32_t &Idx : Indices) {
2388 // Decompose row-major index: Row = Idx / NumCols, Col = Idx % NumCols
2389 unsigned Row = Idx / NumCols;
2390 unsigned Col = Idx % NumCols;
2391 // Re-linearize as column-major
2392 Idx = MT->getColumnMajorFlattenedIndex(Row, Column: Col);
2393 }
2394 }
2395
2396 if (Base.isSimple()) {
2397 RawAddress MatAddr = Base.getAddress();
2398 if (getLangOpts().HLSL &&
2399 E->getBase()->getType().getAddressSpace() == LangAS::hlsl_constant)
2400 MatAddr = CGM.getHLSLRuntime().createBufferMatrixTempAddress(LV: Base, CGF&: *this);
2401
2402 llvm::Constant *CV =
2403 llvm::ConstantDataVector::get(Context&: getLLVMContext(), Elts: Indices);
2404 return LValue::MakeExtVectorElt(Addr: MaybeConvertMatrixAddress(Addr: MatAddr, CGF&: *this),
2405 Elts: CV, type: ResultType, BaseInfo: Base.getBaseInfo(),
2406 TBAAInfo: TBAAAccessInfo());
2407 }
2408 assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
2409
2410 llvm::Constant *BaseElts = Base.getExtVectorElts();
2411 SmallVector<llvm::Constant *, 4> CElts;
2412
2413 for (unsigned Index : Indices)
2414 CElts.push_back(Elt: BaseElts->getAggregateElement(Elt: Index));
2415 llvm::Constant *CV = llvm::ConstantVector::get(V: CElts);
2416
2417 return LValue::MakeExtVectorElt(
2418 Addr: MaybeConvertMatrixAddress(Addr: Base.getExtVectorAddress(), CGF&: *this), Elts: CV,
2419 type: ResultType, BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
2420}
2421
2422// Emit a store of a matrix LValue. This may require casting the original
2423// pointer to memory address (ArrayType) to a pointer to the value type
2424// (VectorType).
2425static void EmitStoreOfMatrixScalar(llvm::Value *value, LValue lvalue,
2426 bool isInit, CodeGenFunction &CGF) {
2427 Address Addr = MaybeConvertMatrixAddress(Addr: lvalue.getAddress(), CGF,
2428 IsVector: value->getType()->isVectorTy());
2429 CGF.EmitStoreOfScalar(Value: value, Addr, Volatile: lvalue.isVolatile(), Ty: lvalue.getType(),
2430 BaseInfo: lvalue.getBaseInfo(), TBAAInfo: lvalue.getTBAAInfo(), isInit,
2431 isNontemporal: lvalue.isNontemporal());
2432}
2433
2434void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr,
2435 bool Volatile, QualType Ty,
2436 LValueBaseInfo BaseInfo,
2437 TBAAAccessInfo TBAAInfo,
2438 bool isInit, bool isNontemporal) {
2439 if (auto *GV = dyn_cast<llvm::GlobalValue>(Val: Addr.getBasePointer()))
2440 if (GV->isThreadLocal())
2441 Addr = Addr.withPointer(NewPointer: Builder.CreateThreadLocalAddress(Ptr: GV),
2442 IsKnownNonNull: NotKnownNonNull);
2443
2444 // Handles vectors of sizes that are likely to be expanded to a larger size
2445 // to optimize performance.
2446 llvm::Type *SrcTy = Value->getType();
2447 if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
2448 if (auto *VecTy = dyn_cast<llvm::FixedVectorType>(Val: SrcTy)) {
2449 auto *NewVecTy =
2450 CGM.getABIInfo().getOptimalVectorMemoryType(T: VecTy, Opt: getLangOpts());
2451 if (!ClangVecTy->isPackedVectorBoolType(ctx: getContext()) &&
2452 VecTy != NewVecTy) {
2453 SmallVector<int, 16> Mask(NewVecTy->getNumElements(),
2454 VecTy->getNumElements());
2455 std::iota(first: Mask.begin(), last: Mask.begin() + VecTy->getNumElements(), value: 0);
2456 // Use undef instead of poison for the padding lanes, to make sure no
2457 // padding bits are poisoned, which may break coercion.
2458 Value = Builder.CreateShuffleVector(V1: Value, V2: llvm::UndefValue::get(T: VecTy),
2459 Mask, Name: "extractVec");
2460 SrcTy = NewVecTy;
2461 }
2462 if (Addr.getElementType() != SrcTy)
2463 Addr = Addr.withElementType(ElemTy: SrcTy);
2464 }
2465 }
2466
2467 Value = EmitToMemory(Value, Ty);
2468
2469 LValue AtomicLValue =
2470 LValue::MakeAddr(Addr, type: Ty, Context&: getContext(), BaseInfo, TBAAInfo);
2471 if (Ty->isAtomicType() ||
2472 (!isInit && LValueIsSuitableForInlineAtomic(Src: AtomicLValue))) {
2473 EmitAtomicStore(rvalue: RValue::get(V: Value), lvalue: AtomicLValue, isInit);
2474 return;
2475 }
2476
2477 llvm::StoreInst *Store = Builder.CreateStore(Val: Value, Addr, IsVolatile: Volatile);
2478 addInstToCurrentSourceAtom(KeyInstruction: Store, Backup: Value);
2479
2480 if (isNontemporal) {
2481 llvm::MDNode *Node =
2482 llvm::MDNode::get(Context&: Store->getContext(),
2483 MDs: llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: 1)));
2484 Store->setMetadata(KindID: llvm::LLVMContext::MD_nontemporal, Node);
2485 }
2486
2487 CGM.DecorateInstructionWithTBAA(Inst: Store, TBAAInfo);
2488}
2489
2490void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
2491 bool isInit) {
2492 if (lvalue.getType()->isConstantMatrixType()) {
2493 EmitStoreOfMatrixScalar(value, lvalue, isInit, CGF&: *this);
2494 return;
2495 }
2496
2497 EmitStoreOfScalar(Value: value, Addr: lvalue.getAddress(), Volatile: lvalue.isVolatile(),
2498 Ty: lvalue.getType(), BaseInfo: lvalue.getBaseInfo(),
2499 TBAAInfo: lvalue.getTBAAInfo(), isInit, isNontemporal: lvalue.isNontemporal());
2500}
2501
2502// Emit a load of a LValue of matrix type. This may require casting the pointer
2503// to memory address (ArrayType) to a pointer to the value type (VectorType).
2504static RValue EmitLoadOfMatrixLValue(LValue LV, SourceLocation Loc,
2505 CodeGenFunction &CGF) {
2506 assert(LV.getType()->isConstantMatrixType());
2507 RawAddress DestAddr = LV.getAddress();
2508
2509 // HLSL constant buffers may pad matrix layouts, so copy elements into a
2510 // non-padded local alloca before loading.
2511 if (CGF.getLangOpts().HLSL &&
2512 LV.getType().getAddressSpace() == LangAS::hlsl_constant)
2513 DestAddr = CGF.CGM.getHLSLRuntime().createBufferMatrixTempAddress(LV, CGF);
2514
2515 Address Addr = MaybeConvertMatrixAddress(Addr: DestAddr, CGF);
2516 LV.setAddress(Addr);
2517 return RValue::get(V: CGF.EmitLoadOfScalar(lvalue: LV, Loc));
2518}
2519
2520RValue CodeGenFunction::EmitLoadOfAnyValue(LValue LV, AggValueSlot Slot,
2521 SourceLocation Loc) {
2522 QualType Ty = LV.getType();
2523 switch (getEvaluationKind(T: Ty)) {
2524 case TEK_Scalar:
2525 return EmitLoadOfLValue(V: LV, Loc);
2526 case TEK_Complex:
2527 return RValue::getComplex(C: EmitLoadOfComplex(src: LV, loc: Loc));
2528 case TEK_Aggregate:
2529 EmitAggFinalDestCopy(Type: Ty, Dest: Slot, Src: LV, SrcKind: EVK_NonRValue);
2530 return Slot.asRValue();
2531 }
2532 llvm_unreachable("bad evaluation kind");
2533}
2534
2535/// EmitLoadOfLValue - Given an expression that represents a value lvalue, this
2536/// method emits the address of the lvalue, then loads the result as an rvalue,
2537/// returning the rvalue.
2538RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) {
2539 // Load from __ptrauth.
2540 if (PointerAuthQualifier PtrAuth = LV.getQuals().getPointerAuth()) {
2541 LV.getQuals().removePointerAuth();
2542 llvm::Value *Value = EmitLoadOfLValue(LV, Loc).getScalarVal();
2543 return RValue::get(V: EmitPointerAuthUnqualify(Qualifier: PtrAuth, Pointer: Value, PointerType: LV.getType(),
2544 StorageAddress: LV.getAddress(),
2545 /*known nonnull*/ IsKnownNonNull: false));
2546 }
2547
2548 if (LV.isObjCWeak()) {
2549 // load of a __weak object.
2550 Address AddrWeakObj = LV.getAddress();
2551 return RValue::get(V: CGM.getObjCRuntime().EmitObjCWeakRead(CGF&: *this,
2552 AddrWeakObj));
2553 }
2554 if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) {
2555 // In MRC mode, we do a load+autorelease.
2556 if (!getLangOpts().ObjCAutoRefCount) {
2557 return RValue::get(V: EmitARCLoadWeak(addr: LV.getAddress()));
2558 }
2559
2560 // In ARC mode, we load retained and then consume the value.
2561 llvm::Value *Object = EmitARCLoadWeakRetained(addr: LV.getAddress());
2562 Object = EmitObjCConsumeObject(T: LV.getType(), Ptr: Object);
2563 return RValue::get(V: Object);
2564 }
2565
2566 if (LV.isSimple()) {
2567 assert(!LV.getType()->isFunctionType());
2568
2569 if (LV.getType()->isConstantMatrixType())
2570 return EmitLoadOfMatrixLValue(LV, Loc, CGF&: *this);
2571
2572 // Everything needs a load.
2573 return RValue::get(V: EmitLoadOfScalar(lvalue: LV, Loc));
2574 }
2575
2576 if (LV.isVectorElt()) {
2577 llvm::LoadInst *Load = Builder.CreateLoad(Addr: LV.getVectorAddress(),
2578 IsVolatile: LV.isVolatileQualified());
2579 llvm::Value *Elt =
2580 Builder.CreateExtractElement(Vec: Load, Idx: LV.getVectorIdx(), Name: "vecext");
2581 return RValue::get(V: EmitFromMemory(Value: Elt, Ty: LV.getType()));
2582 }
2583
2584 // If this is a reference to a subset of the elements of a vector, either
2585 // shuffle the input or extract/insert them as appropriate.
2586 if (LV.isExtVectorElt()) {
2587 return EmitLoadOfExtVectorElementLValue(V: LV);
2588 }
2589
2590 // Global Register variables always invoke intrinsics
2591 if (LV.isGlobalReg())
2592 return EmitLoadOfGlobalRegLValue(LV);
2593
2594 if (LV.isMatrixElt()) {
2595 llvm::Value *Idx = LV.getMatrixIdx();
2596 QualType EltTy = LV.getType();
2597 if (const auto *MatTy = EltTy->getAs<ConstantMatrixType>()) {
2598 EltTy = MatTy->getElementType();
2599 if (CGM.getCodeGenOpts().isOptimizedBuild()) {
2600 llvm::MatrixBuilder MB(Builder);
2601 MB.CreateIndexAssumption(Idx, NumElements: MatTy->getNumElementsFlattened());
2602 }
2603 }
2604 llvm::LoadInst *Load =
2605 Builder.CreateLoad(Addr: LV.getMatrixAddress(), IsVolatile: LV.isVolatileQualified());
2606 llvm::Value *Elt = Builder.CreateExtractElement(Vec: Load, Idx, Name: "matrixext");
2607 return RValue::get(V: EmitFromMemory(Value: Elt, Ty: EltTy));
2608 }
2609 if (LV.isMatrixRow()) {
2610 QualType MatTy = LV.getType();
2611 const ConstantMatrixType *MT = MatTy->castAs<ConstantMatrixType>();
2612
2613 unsigned NumRows = MT->getNumRows();
2614 unsigned NumCols = MT->getNumColumns();
2615 unsigned NumLanes = NumCols;
2616 llvm::Value *MatrixVec = EmitLoadOfScalar(lvalue: LV, Loc);
2617 llvm::Value *Row = LV.getMatrixRowIdx();
2618 llvm::Type *ElemTy = ConvertType(T: MT->getElementType());
2619 llvm::Constant *ColConstsIndices = nullptr;
2620 llvm::MatrixBuilder MB(Builder);
2621
2622 if (LV.isMatrixRowSwizzle()) {
2623 ColConstsIndices = LV.getMatrixRowElts();
2624 NumLanes = llvm::cast<llvm::FixedVectorType>(Val: ColConstsIndices->getType())
2625 ->getNumElements();
2626 }
2627
2628 llvm::Type *RowTy = llvm::FixedVectorType::get(ElementType: ElemTy, NumElts: NumLanes);
2629 llvm::Value *Result = llvm::PoisonValue::get(T: RowTy); // <NumLanes x T>
2630
2631 for (unsigned Col = 0; Col < NumLanes; ++Col) {
2632 llvm::Value *ColIdx;
2633 if (ColConstsIndices)
2634 ColIdx = ColConstsIndices->getAggregateElement(Elt: Col);
2635 else
2636 ColIdx = llvm::ConstantInt::get(Ty: Row->getType(), V: Col);
2637 bool IsMatrixRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: MatTy);
2638 llvm::Value *EltIndex =
2639 MB.CreateIndex(RowIdx: Row, ColumnIdx: ColIdx, NumRows, NumCols, IsMatrixRowMajor);
2640 llvm::Value *Elt = Builder.CreateExtractElement(Vec: MatrixVec, Idx: EltIndex);
2641 llvm::Value *Lane = llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: Col);
2642 Result = Builder.CreateInsertElement(Vec: Result, NewElt: Elt, Idx: Lane);
2643 }
2644
2645 return RValue::get(V: Result);
2646 }
2647
2648 assert(LV.isBitField() && "Unknown LValue type!");
2649 return EmitLoadOfBitfieldLValue(LV, Loc);
2650}
2651
2652RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV,
2653 SourceLocation Loc) {
2654 const CGBitFieldInfo &Info = LV.getBitFieldInfo();
2655
2656 // Get the output type.
2657 llvm::Type *ResLTy = ConvertType(T: LV.getType());
2658
2659 Address Ptr = LV.getBitFieldAddress();
2660 llvm::Value *Val =
2661 Builder.CreateLoad(Addr: Ptr, IsVolatile: LV.isVolatileQualified(), Name: "bf.load");
2662
2663 bool UseVolatile = LV.isVolatileQualified() &&
2664 Info.VolatileStorageSize != 0 &&
2665 CodeGenUtils::isAAPCS(TargetInfo: CGM.getTarget());
2666 const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
2667 const unsigned StorageSize =
2668 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
2669 if (Info.IsSigned) {
2670 assert(static_cast<unsigned>(Offset + Info.Size) <= StorageSize);
2671 unsigned HighBits = StorageSize - Offset - Info.Size;
2672 if (HighBits)
2673 Val = Builder.CreateShl(LHS: Val, RHS: HighBits, Name: "bf.shl");
2674 if (Offset + HighBits)
2675 Val = Builder.CreateAShr(LHS: Val, RHS: Offset + HighBits, Name: "bf.ashr");
2676 } else {
2677 if (Offset)
2678 Val = Builder.CreateLShr(LHS: Val, RHS: Offset, Name: "bf.lshr");
2679 if (static_cast<unsigned>(Offset) + Info.Size < StorageSize)
2680 Val = Builder.CreateAnd(
2681 LHS: Val, RHS: llvm::APInt::getLowBitsSet(numBits: StorageSize, loBitsSet: Info.Size), Name: "bf.clear");
2682 }
2683 Val = Builder.CreateIntCast(V: Val, DestTy: ResLTy, isSigned: Info.IsSigned, Name: "bf.cast");
2684 EmitScalarRangeCheck(Value: Val, Ty: LV.getType(), Loc);
2685 return RValue::get(V: Val);
2686}
2687
2688// If this is a reference to a subset of the elements of a vector, create an
2689// appropriate shufflevector.
2690RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {
2691 llvm::Value *Vec = Builder.CreateLoad(Addr: LV.getExtVectorAddress(),
2692 IsVolatile: LV.isVolatileQualified());
2693
2694 // HLSL allows treating scalars as one-element vectors. Converting the scalar
2695 // IR value to a vector here allows the rest of codegen to behave as normal.
2696 if (getLangOpts().HLSL && !Vec->getType()->isVectorTy()) {
2697 llvm::Type *DstTy = llvm::FixedVectorType::get(ElementType: Vec->getType(), NumElts: 1);
2698 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: CGM.Int64Ty);
2699 Vec = Builder.CreateInsertElement(VecTy: DstTy, NewElt: Vec, Idx: Zero, Name: "cast.splat");
2700 }
2701
2702 const llvm::Constant *Elts = LV.getExtVectorElts();
2703
2704 // If the result of the expression is a non-vector type, we must be extracting
2705 // a single element. Just codegen as an extractelement.
2706 const VectorType *ExprVT = LV.getType()->getAs<VectorType>();
2707 if (!ExprVT) {
2708 unsigned InIdx = getAccessedFieldNo(Idx: 0, Elts);
2709 llvm::Value *Elt = llvm::ConstantInt::get(Ty: SizeTy, V: InIdx);
2710
2711 llvm::Value *Element = Builder.CreateExtractElement(Vec, Idx: Elt);
2712
2713 llvm::Type *LVTy = ConvertType(T: LV.getType());
2714 if (Element->getType()->getPrimitiveSizeInBits() >
2715 LVTy->getPrimitiveSizeInBits()) {
2716 if (LV.getType()->hasBooleanRepresentation() &&
2717 CGM.getCodeGenOpts().isConvertingBoolWithCmp0())
2718 Element = Builder.CreateICmpNE(
2719 LHS: Element, RHS: llvm::Constant::getNullValue(Ty: Element->getType()));
2720 else
2721 Element = Builder.CreateTrunc(V: Element, DestTy: LVTy);
2722 }
2723
2724 return RValue::get(V: Element);
2725 }
2726
2727 // Always use shuffle vector to try to retain the original program structure
2728 unsigned NumResultElts = ExprVT->getNumElements();
2729
2730 SmallVector<int, 4> Mask;
2731 for (unsigned i = 0; i != NumResultElts; ++i)
2732 Mask.push_back(Elt: getAccessedFieldNo(Idx: i, Elts));
2733
2734 Vec = Builder.CreateShuffleVector(V: Vec, Mask);
2735
2736 if (LV.getType()->isExtVectorBoolType()) {
2737 if (CGM.getCodeGenOpts().isConvertingBoolWithCmp0())
2738 Vec = Builder.CreateICmpNE(LHS: Vec,
2739 RHS: llvm::Constant::getNullValue(Ty: Vec->getType()));
2740 else
2741 Vec = Builder.CreateTrunc(V: Vec, DestTy: ConvertType(T: LV.getType()), Name: "truncv");
2742 }
2743
2744 return RValue::get(V: Vec);
2745}
2746
2747/// Generates lvalue for partial ext_vector access.
2748Address CodeGenFunction::EmitExtVectorElementLValue(LValue LV) {
2749 Address VectorAddress = LV.getExtVectorAddress();
2750 QualType EQT = LV.getType()->castAs<VectorType>()->getElementType();
2751 llvm::Type *VectorElementTy = CGM.getTypes().ConvertType(T: EQT);
2752
2753 Address CastToPointerElement = VectorAddress.withElementType(ElemTy: VectorElementTy);
2754
2755 const llvm::Constant *Elts = LV.getExtVectorElts();
2756 unsigned ix = getAccessedFieldNo(Idx: 0, Elts);
2757
2758 Address VectorBasePtrPlusIx =
2759 Builder.CreateConstInBoundsGEP(Addr: CastToPointerElement, Index: ix,
2760 Name: "vector.elt");
2761
2762 return VectorBasePtrPlusIx;
2763}
2764
2765/// Load of global named registers are always calls to intrinsics.
2766RValue CodeGenFunction::EmitLoadOfGlobalRegLValue(LValue LV) {
2767 assert((LV.getType()->isIntegerType() || LV.getType()->isPointerType()) &&
2768 "Bad type for register variable");
2769 llvm::MDNode *RegName = cast<llvm::MDNode>(
2770 Val: cast<llvm::MetadataAsValue>(Val: LV.getGlobalReg())->getMetadata());
2771
2772 // We accept integer and pointer types only
2773 llvm::Type *OrigTy = CGM.getTypes().ConvertType(T: LV.getType());
2774 llvm::Type *Ty = OrigTy;
2775 if (OrigTy->isPointerTy())
2776 Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
2777 llvm::Type *Types[] = { Ty };
2778
2779 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::read_register, Tys: Types);
2780 llvm::Value *Call = Builder.CreateCall(
2781 Callee: F, Args: llvm::MetadataAsValue::get(Context&: Ty->getContext(), MD: RegName));
2782 if (OrigTy->isPointerTy())
2783 Call = Builder.CreateIntToPtr(V: Call, DestTy: OrigTy);
2784 return RValue::get(V: Call);
2785}
2786
2787/// EmitStoreThroughLValue - Store the specified rvalue into the specified
2788/// lvalue, where both are guaranteed to the have the same type, and that type
2789/// is 'Ty'.
2790void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
2791 bool isInit) {
2792 if (!Dst.isSimple()) {
2793 if (Dst.isVectorElt()) {
2794 if (getLangOpts().HLSL) {
2795 // HLSL allows direct access to vector elements, so storing to
2796 // individual elements of a vector through VectorElt is handled as
2797 // separate store instructions.
2798 Address DstAddr = Dst.getVectorAddress();
2799 llvm::Type *DestAddrTy = DstAddr.getElementType();
2800 llvm::Type *ElemTy = DestAddrTy->getScalarType();
2801 CharUnits ElemAlign = CharUnits::fromQuantity(
2802 Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty: ElemTy));
2803
2804 assert(ElemTy->getScalarSizeInBits() >= 8 &&
2805 "vector element type must be at least byte-sized");
2806
2807 llvm::Value *Val = Src.getScalarVal();
2808 if (Val->getType()->getPrimitiveSizeInBits() <
2809 ElemTy->getScalarSizeInBits())
2810 Val = Builder.CreateZExt(V: Val, DestTy: ElemTy->getScalarType());
2811
2812 llvm::Value *Idx = Dst.getVectorIdx();
2813 llvm::Value *Zero = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
2814 Address DstElemAddr =
2815 Builder.CreateGEP(Addr: DstAddr, IdxList: {Zero, Idx}, ElementType: DestAddrTy, Align: ElemAlign);
2816 Builder.CreateStore(Val, Addr: DstElemAddr, IsVolatile: Dst.isVolatileQualified());
2817 return;
2818 }
2819
2820 // Read/modify/write the vector, inserting the new element.
2821 llvm::Value *Vec = Builder.CreateLoad(Addr: Dst.getVectorAddress(),
2822 IsVolatile: Dst.isVolatileQualified());
2823 llvm::Type *VecTy = Vec->getType();
2824 llvm::Value *SrcVal = Src.getScalarVal();
2825
2826 if (VecTy->isVectorTy() && SrcVal->getType()->getPrimitiveSizeInBits() <
2827 VecTy->getScalarSizeInBits())
2828 SrcVal = Builder.CreateZExt(V: SrcVal, DestTy: VecTy->getScalarType());
2829
2830 auto *IRStoreTy = dyn_cast<llvm::IntegerType>(Val: Vec->getType());
2831 if (IRStoreTy) {
2832 auto *IRVecTy = llvm::FixedVectorType::get(
2833 ElementType: Builder.getInt1Ty(), NumElts: IRStoreTy->getPrimitiveSizeInBits());
2834 Vec = Builder.CreateBitCast(V: Vec, DestTy: IRVecTy);
2835 // iN --> <N x i1>.
2836 }
2837
2838 // Allow inserting `<1 x T>` into an `<N x T>`. It can happen with scalar
2839 // types which are mapped to vector LLVM IR types (e.g. for implementing
2840 // an ABI).
2841 if (auto *EltTy = dyn_cast<llvm::FixedVectorType>(Val: SrcVal->getType());
2842 EltTy && EltTy->getNumElements() == 1)
2843 SrcVal = Builder.CreateBitCast(V: SrcVal, DestTy: EltTy->getElementType());
2844
2845 Vec = Builder.CreateInsertElement(Vec, NewElt: SrcVal, Idx: Dst.getVectorIdx(),
2846 Name: "vecins");
2847 if (IRStoreTy) {
2848 // <N x i1> --> <iN>.
2849 Vec = Builder.CreateBitCast(V: Vec, DestTy: IRStoreTy);
2850 }
2851
2852 auto *I = Builder.CreateStore(Val: Vec, Addr: Dst.getVectorAddress(),
2853 IsVolatile: Dst.isVolatileQualified());
2854 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Vec);
2855 return;
2856 }
2857
2858 // If this is an update of extended vector elements, insert them as
2859 // appropriate.
2860 if (Dst.isExtVectorElt())
2861 return EmitStoreThroughExtVectorComponentLValue(Src, Dst);
2862
2863 if (Dst.isGlobalReg())
2864 return EmitStoreThroughGlobalRegLValue(Src, Dst);
2865
2866 if (Dst.isMatrixElt()) {
2867 if (getLangOpts().HLSL) {
2868 // HLSL allows direct access to matrix elements, so storing to
2869 // individual elements of a matrix through MatrixElt is handled as
2870 // separate store instructions.
2871 Address DstAddr = Dst.getMatrixAddress();
2872 llvm::Type *DestAddrTy = DstAddr.getElementType();
2873 llvm::Type *ElemTy = DestAddrTy->getScalarType();
2874 CharUnits ElemAlign = CharUnits::fromQuantity(
2875 Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty: ElemTy));
2876
2877 assert(ElemTy->getScalarSizeInBits() >= 8 &&
2878 "matrix element type must be at least byte-sized");
2879
2880 llvm::Value *Val = Src.getScalarVal();
2881 if (Val->getType()->getPrimitiveSizeInBits() <
2882 ElemTy->getScalarSizeInBits())
2883 Val = Builder.CreateZExt(V: Val, DestTy: ElemTy->getScalarType());
2884
2885 llvm::Value *Idx = Dst.getMatrixIdx();
2886 llvm::Value *Zero = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
2887 Address DstElemAddr =
2888 Builder.CreateGEP(Addr: DstAddr, IdxList: {Zero, Idx}, ElementType: DestAddrTy, Align: ElemAlign);
2889 Builder.CreateStore(Val, Addr: DstElemAddr, IsVolatile: Dst.isVolatileQualified());
2890 return;
2891 }
2892
2893 llvm::Value *Idx = Dst.getMatrixIdx();
2894 if (CGM.getCodeGenOpts().isOptimizedBuild()) {
2895 const auto *const MatTy = Dst.getType()->castAs<ConstantMatrixType>();
2896 llvm::MatrixBuilder MB(Builder);
2897 MB.CreateIndexAssumption(Idx, NumElements: MatTy->getNumElementsFlattened());
2898 }
2899 llvm::Instruction *Load = Builder.CreateLoad(Addr: Dst.getMatrixAddress());
2900 llvm::Value *InsertVal = Src.getScalarVal();
2901 llvm::Value *Vec =
2902 Builder.CreateInsertElement(Vec: Load, NewElt: InsertVal, Idx, Name: "matins");
2903 auto *I = Builder.CreateStore(Val: Vec, Addr: Dst.getMatrixAddress(),
2904 IsVolatile: Dst.isVolatileQualified());
2905 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Vec);
2906 return;
2907 }
2908 if (Dst.isMatrixRow()) {
2909 // NOTE: Since there are no other languages that implement matrix single
2910 // subscripting, the logic here is specific to HLSL which allows
2911 // per-element stores to rows of matrices.
2912 assert(getLangOpts().HLSL &&
2913 "Store through matrix row LValues is only implemented for HLSL!");
2914 QualType MatTy = Dst.getType();
2915 const ConstantMatrixType *MT = MatTy->castAs<ConstantMatrixType>();
2916
2917 unsigned NumRows = MT->getNumRows();
2918 unsigned NumCols = MT->getNumColumns();
2919 unsigned NumLanes = NumCols;
2920
2921 Address DstAddr = Dst.getMatrixAddress();
2922 llvm::Type *DestAddrTy = DstAddr.getElementType();
2923 llvm::Type *ElemTy = DestAddrTy->getScalarType();
2924 CharUnits ElemAlign =
2925 CharUnits::fromQuantity(Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty: ElemTy));
2926
2927 assert(ElemTy->getScalarSizeInBits() >= 8 &&
2928 "matrix element type must be at least byte-sized");
2929
2930 llvm::Value *RowVal = Src.getScalarVal();
2931 if (RowVal->getType()->getScalarType()->getPrimitiveSizeInBits() <
2932 ElemTy->getScalarSizeInBits()) {
2933 auto *RowValVecTy = cast<llvm::FixedVectorType>(Val: RowVal->getType());
2934 llvm::Type *StorageElmTy = llvm::FixedVectorType::get(
2935 ElementType: ElemTy->getScalarType(), NumElts: RowValVecTy->getNumElements());
2936 RowVal = Builder.CreateZExt(V: RowVal, DestTy: StorageElmTy);
2937 }
2938
2939 llvm::MatrixBuilder MB(Builder);
2940
2941 llvm::Constant *ColConstsIndices = nullptr;
2942 if (Dst.isMatrixRowSwizzle()) {
2943 ColConstsIndices = Dst.getMatrixRowElts();
2944 NumLanes =
2945 llvm::cast<llvm::FixedVectorType>(Val: ColConstsIndices->getType())
2946 ->getNumElements();
2947 }
2948
2949 llvm::Value *Row = Dst.getMatrixRowIdx();
2950 for (unsigned Col = 0; Col < NumLanes; ++Col) {
2951 llvm::Value *ColIdx;
2952 if (ColConstsIndices)
2953 ColIdx = ColConstsIndices->getAggregateElement(Elt: Col);
2954 else
2955 ColIdx = llvm::ConstantInt::get(Ty: Row->getType(), V: Col);
2956 bool IsMatrixRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: Dst.getType());
2957 llvm::Value *EltIndex =
2958 MB.CreateIndex(RowIdx: Row, ColumnIdx: ColIdx, NumRows, NumCols, IsMatrixRowMajor);
2959 llvm::Value *Lane = llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: Col);
2960 llvm::Value *Zero = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
2961 llvm::Value *NewElt = Builder.CreateExtractElement(Vec: RowVal, Idx: Lane);
2962 Address DstElemAddr =
2963 Builder.CreateGEP(Addr: DstAddr, IdxList: {Zero, EltIndex}, ElementType: DestAddrTy, Align: ElemAlign);
2964 Builder.CreateStore(Val: NewElt, Addr: DstElemAddr, IsVolatile: Dst.isVolatileQualified());
2965 }
2966
2967 return;
2968 }
2969
2970 assert(Dst.isBitField() && "Unknown LValue type");
2971 return EmitStoreThroughBitfieldLValue(Src, Dst);
2972 }
2973
2974 // Handle __ptrauth qualification by re-signing the value.
2975 if (PointerAuthQualifier PointerAuth = Dst.getQuals().getPointerAuth()) {
2976 Src = RValue::get(V: EmitPointerAuthQualify(Qualifier: PointerAuth, Pointer: Src.getScalarVal(),
2977 ValueType: Dst.getType(), StorageAddress: Dst.getAddress(),
2978 /*known nonnull*/ IsKnownNonNull: false));
2979 }
2980
2981 // There's special magic for assigning into an ARC-qualified l-value.
2982 if (Qualifiers::ObjCLifetime Lifetime = Dst.getQuals().getObjCLifetime()) {
2983 switch (Lifetime) {
2984 case Qualifiers::OCL_None:
2985 llvm_unreachable("present but none");
2986
2987 case Qualifiers::OCL_ExplicitNone:
2988 // nothing special
2989 break;
2990
2991 case Qualifiers::OCL_Strong:
2992 if (isInit) {
2993 Src = RValue::get(V: EmitARCRetain(type: Dst.getType(), value: Src.getScalarVal()));
2994 break;
2995 }
2996 EmitARCStoreStrong(lvalue: Dst, value: Src.getScalarVal(), /*ignore*/ resultIgnored: true);
2997 return;
2998
2999 case Qualifiers::OCL_Weak:
3000 if (isInit)
3001 // Initialize and then skip the primitive store.
3002 EmitARCInitWeak(addr: Dst.getAddress(), value: Src.getScalarVal());
3003 else
3004 EmitARCStoreWeak(addr: Dst.getAddress(), value: Src.getScalarVal(),
3005 /*ignore*/ ignored: true);
3006 return;
3007
3008 case Qualifiers::OCL_Autoreleasing:
3009 Src = RValue::get(V: EmitObjCExtendObjectLifetime(T: Dst.getType(),
3010 Ptr: Src.getScalarVal()));
3011 // fall into the normal path
3012 break;
3013 }
3014 }
3015
3016 if (Dst.isObjCWeak() && !Dst.isNonGC()) {
3017 // load of a __weak object.
3018 Address LvalueDst = Dst.getAddress();
3019 llvm::Value *src = Src.getScalarVal();
3020 CGM.getObjCRuntime().EmitObjCWeakAssign(CGF&: *this, src, dest: LvalueDst);
3021 return;
3022 }
3023
3024 if (Dst.isObjCStrong() && !Dst.isNonGC()) {
3025 // load of a __strong object.
3026 Address LvalueDst = Dst.getAddress();
3027 llvm::Value *src = Src.getScalarVal();
3028 if (Dst.isObjCIvar()) {
3029 assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");
3030 llvm::Type *ResultType = IntPtrTy;
3031 Address dst = EmitPointerWithAlignment(E: Dst.getBaseIvarExp());
3032 llvm::Value *RHS = dst.emitRawPointer(CGF&: *this);
3033 RHS = Builder.CreatePtrToInt(V: RHS, DestTy: ResultType, Name: "sub.ptr.rhs.cast");
3034 llvm::Value *LHS = Builder.CreatePtrToInt(V: LvalueDst.emitRawPointer(CGF&: *this),
3035 DestTy: ResultType, Name: "sub.ptr.lhs.cast");
3036 llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, Name: "ivar.offset");
3037 CGM.getObjCRuntime().EmitObjCIvarAssign(CGF&: *this, src, dest: dst, ivarOffset: BytesBetween);
3038 } else if (Dst.isGlobalObjCRef()) {
3039 CGM.getObjCRuntime().EmitObjCGlobalAssign(CGF&: *this, src, dest: LvalueDst,
3040 threadlocal: Dst.isThreadLocalRef());
3041 }
3042 else
3043 CGM.getObjCRuntime().EmitObjCStrongCastAssign(CGF&: *this, src, dest: LvalueDst);
3044 return;
3045 }
3046
3047 assert(Src.isScalar() && "Can't emit an agg store with this method");
3048 EmitStoreOfScalar(value: Src.getScalarVal(), lvalue: Dst, isInit);
3049}
3050
3051void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
3052 llvm::Value **Result) {
3053 const CGBitFieldInfo &Info = Dst.getBitFieldInfo();
3054 llvm::Type *ResLTy = convertTypeForLoadStore(ASTTy: Dst.getType());
3055 Address Ptr = Dst.getBitFieldAddress();
3056
3057 // Get the source value, truncated to the width of the bit-field.
3058 llvm::Value *SrcVal = Src.getScalarVal();
3059
3060 // Cast the source to the storage type and shift it into place.
3061 SrcVal = Builder.CreateIntCast(V: SrcVal, DestTy: Ptr.getElementType(),
3062 /*isSigned=*/false);
3063 llvm::Value *MaskedVal = SrcVal;
3064
3065 const bool UseVolatile =
3066 CGM.getCodeGenOpts().AAPCSBitfieldWidth && Dst.isVolatileQualified() &&
3067 Info.VolatileStorageSize != 0 && CodeGenUtils::isAAPCS(TargetInfo: CGM.getTarget());
3068 const unsigned StorageSize =
3069 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
3070 const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
3071 // See if there are other bits in the bitfield's storage we'll need to load
3072 // and mask together with source before storing.
3073 if (StorageSize != Info.Size) {
3074 assert(StorageSize > Info.Size && "Invalid bitfield size.");
3075 llvm::Value *Val =
3076 Builder.CreateLoad(Addr: Ptr, IsVolatile: Dst.isVolatileQualified(), Name: "bf.load");
3077
3078 // Mask the source value as needed.
3079 if (!Dst.getType()->hasBooleanRepresentation())
3080 SrcVal = Builder.CreateAnd(
3081 LHS: SrcVal, RHS: llvm::APInt::getLowBitsSet(numBits: StorageSize, loBitsSet: Info.Size),
3082 Name: "bf.value");
3083 MaskedVal = SrcVal;
3084 if (Offset)
3085 SrcVal = Builder.CreateShl(LHS: SrcVal, RHS: Offset, Name: "bf.shl");
3086
3087 // Mask out the original value.
3088 Val = Builder.CreateAnd(
3089 LHS: Val, RHS: ~llvm::APInt::getBitsSet(numBits: StorageSize, loBit: Offset, hiBit: Offset + Info.Size),
3090 Name: "bf.clear");
3091
3092 // Or together the unchanged values and the source value.
3093 SrcVal = Builder.CreateOr(LHS: Val, RHS: SrcVal, Name: "bf.set");
3094 } else {
3095 assert(Offset == 0);
3096 // According to the AACPS:
3097 // When a volatile bit-field is written, and its container does not overlap
3098 // with any non-bit-field member, its container must be read exactly once
3099 // and written exactly once using the access width appropriate to the type
3100 // of the container. The two accesses are not atomic.
3101 if (Dst.isVolatileQualified() && CodeGenUtils::isAAPCS(TargetInfo: CGM.getTarget()) &&
3102 CGM.getCodeGenOpts().ForceAAPCSBitfieldLoad)
3103 Builder.CreateLoad(Addr: Ptr, IsVolatile: true, Name: "bf.load");
3104 }
3105
3106 // Write the new value back out.
3107 auto *I = Builder.CreateStore(Val: SrcVal, Addr: Ptr, IsVolatile: Dst.isVolatileQualified());
3108 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: SrcVal);
3109
3110 // Return the new value of the bit-field, if requested.
3111 if (Result) {
3112 llvm::Value *ResultVal = MaskedVal;
3113
3114 // Sign extend the value if needed.
3115 if (Info.IsSigned) {
3116 assert(Info.Size <= StorageSize);
3117 unsigned HighBits = StorageSize - Info.Size;
3118 if (HighBits) {
3119 ResultVal = Builder.CreateShl(LHS: ResultVal, RHS: HighBits, Name: "bf.result.shl");
3120 ResultVal = Builder.CreateAShr(LHS: ResultVal, RHS: HighBits, Name: "bf.result.ashr");
3121 }
3122 }
3123
3124 ResultVal = Builder.CreateIntCast(V: ResultVal, DestTy: ResLTy, isSigned: Info.IsSigned,
3125 Name: "bf.result.cast");
3126 *Result = EmitFromMemory(Value: ResultVal, Ty: Dst.getType());
3127 }
3128}
3129
3130void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
3131 LValue Dst) {
3132 llvm::Value *SrcVal = Src.getScalarVal();
3133 Address DstAddr = Dst.getExtVectorAddress();
3134 const llvm::Constant *Elts = Dst.getExtVectorElts();
3135 if (DstAddr.getElementType()->getScalarSizeInBits() >
3136 SrcVal->getType()->getScalarSizeInBits())
3137 SrcVal = Builder.CreateZExt(
3138 V: SrcVal, DestTy: convertTypeForLoadStore(ASTTy: Dst.getType(), LLVMTy: SrcVal->getType()));
3139
3140 if (getLangOpts().HLSL) {
3141 llvm::Type *DestAddrTy = DstAddr.getElementType();
3142 // HLSL allows storing to scalar values through ExtVector component LValues.
3143 // To support this we need to handle the case where the destination address
3144 // is a scalar.
3145 if (!DestAddrTy->isVectorTy()) {
3146 assert(!Dst.getType()->isVectorType() &&
3147 "this should only occur for non-vector l-values");
3148 Builder.CreateStore(Val: SrcVal, Addr: DstAddr, IsVolatile: Dst.isVolatileQualified());
3149 return;
3150 }
3151
3152 // HLSL allows direct access to vector elements, so storing to individual
3153 // elements of a vector through ExtVector is handled as separate store
3154 // instructions.
3155 // If we are updating multiple elements, Dst and Src are vectors; for
3156 // a single element update they are scalars.
3157 const VectorType *VTy = Dst.getType()->getAs<VectorType>();
3158 unsigned NumSrcElts = VTy ? VTy->getNumElements() : 1;
3159 CharUnits ElemAlign = CharUnits::fromQuantity(
3160 Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty: DestAddrTy->getScalarType()));
3161 llvm::Value *Zero = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
3162
3163 for (unsigned I = 0; I != NumSrcElts; ++I) {
3164 llvm::Value *Val = VTy ? Builder.CreateExtractElement(
3165 Vec: SrcVal, Idx: llvm::ConstantInt::get(Ty: Int32Ty, V: I))
3166 : SrcVal;
3167 unsigned FieldNo = getAccessedFieldNo(Idx: I, Elts);
3168 Address DstElemAddr = Address::invalid();
3169 if (FieldNo == 0)
3170 DstElemAddr = DstAddr.withAlignment(NewAlignment: ElemAlign);
3171 else
3172 DstElemAddr = Builder.CreateGEP(
3173 Addr: DstAddr, IdxList: {Zero, llvm::ConstantInt::get(Ty: Int32Ty, V: FieldNo)},
3174 ElementType: DestAddrTy, Align: ElemAlign);
3175 Builder.CreateStore(Val, Addr: DstElemAddr, IsVolatile: Dst.isVolatileQualified());
3176 }
3177 return;
3178 }
3179
3180 // This access turns into a read/modify/write of the vector. Load the input
3181 // value now.
3182 llvm::Value *Vec = Builder.CreateLoad(Addr: DstAddr, IsVolatile: Dst.isVolatileQualified());
3183 llvm::Type *VecTy = Vec->getType();
3184
3185 if (const VectorType *VTy = Dst.getType()->getAs<VectorType>()) {
3186 unsigned NumSrcElts = VTy->getNumElements();
3187 unsigned NumDstElts = cast<llvm::FixedVectorType>(Val: VecTy)->getNumElements();
3188 if (NumDstElts == NumSrcElts) {
3189 // Use shuffle vector is the src and destination are the same number of
3190 // elements and restore the vector mask since it is on the side it will be
3191 // stored.
3192 SmallVector<int, 4> Mask(NumDstElts);
3193 for (unsigned i = 0; i != NumSrcElts; ++i)
3194 Mask[getAccessedFieldNo(Idx: i, Elts)] = i;
3195
3196 Vec = Builder.CreateShuffleVector(V: SrcVal, Mask);
3197 } else if (NumDstElts > NumSrcElts) {
3198 // Extended the source vector to the same length and then shuffle it
3199 // into the destination.
3200 // FIXME: since we're shuffling with undef, can we just use the indices
3201 // into that? This could be simpler.
3202 SmallVector<int, 4> ExtMask;
3203 for (unsigned i = 0; i != NumSrcElts; ++i)
3204 ExtMask.push_back(Elt: i);
3205 ExtMask.resize(N: NumDstElts, NV: -1);
3206 llvm::Value *ExtSrcVal = Builder.CreateShuffleVector(V: SrcVal, Mask: ExtMask);
3207 // build identity
3208 SmallVector<int, 4> Mask;
3209 for (unsigned i = 0; i != NumDstElts; ++i)
3210 Mask.push_back(Elt: i);
3211
3212 // When the vector size is odd and .odd or .hi is used, the last element
3213 // of the Elts constant array will be one past the size of the vector.
3214 // Ignore the last element here, if it is greater than the mask size.
3215 if (getAccessedFieldNo(Idx: NumSrcElts - 1, Elts) == Mask.size())
3216 NumSrcElts--;
3217
3218 // modify when what gets shuffled in
3219 for (unsigned i = 0; i != NumSrcElts; ++i)
3220 Mask[getAccessedFieldNo(Idx: i, Elts)] = i + NumDstElts;
3221 Vec = Builder.CreateShuffleVector(V1: Vec, V2: ExtSrcVal, Mask);
3222 } else {
3223 // We should never shorten the vector
3224 llvm_unreachable("unexpected shorten vector length");
3225 }
3226 } else {
3227 // If the Src is a scalar (not a vector), and the target is a vector it must
3228 // be updating one element.
3229 unsigned InIdx = getAccessedFieldNo(Idx: 0, Elts);
3230 llvm::Value *Elt = llvm::ConstantInt::get(Ty: SizeTy, V: InIdx);
3231
3232 Vec = Builder.CreateInsertElement(Vec, NewElt: SrcVal, Idx: Elt);
3233 }
3234
3235 Builder.CreateStore(Val: Vec, Addr: Dst.getExtVectorAddress(),
3236 IsVolatile: Dst.isVolatileQualified());
3237}
3238
3239/// Store of global named registers are always calls to intrinsics.
3240void CodeGenFunction::EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst) {
3241 assert((Dst.getType()->isIntegerType() || Dst.getType()->isPointerType()) &&
3242 "Bad type for register variable");
3243 llvm::MDNode *RegName = cast<llvm::MDNode>(
3244 Val: cast<llvm::MetadataAsValue>(Val: Dst.getGlobalReg())->getMetadata());
3245 assert(RegName && "Register LValue is not metadata");
3246
3247 // We accept integer and pointer types only
3248 llvm::Type *OrigTy = CGM.getTypes().ConvertType(T: Dst.getType());
3249 llvm::Type *Ty = OrigTy;
3250 if (OrigTy->isPointerTy())
3251 Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
3252 llvm::Type *Types[] = { Ty };
3253
3254 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::write_register, Tys: Types);
3255 llvm::Value *Value = Src.getScalarVal();
3256 if (OrigTy->isPointerTy())
3257 Value = Builder.CreatePtrToInt(V: Value, DestTy: Ty);
3258 Builder.CreateCall(
3259 Callee: F, Args: {llvm::MetadataAsValue::get(Context&: Ty->getContext(), MD: RegName), Value});
3260}
3261
3262// setObjCGCLValueClass - sets class of the lvalue for the purpose of
3263// generating write-barries API. It is currently a global, ivar,
3264// or neither.
3265static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
3266 LValue &LV,
3267 bool IsMemberAccess=false) {
3268 if (Ctx.getLangOpts().getGC() == LangOptions::NonGC)
3269 return;
3270
3271 if (isa<ObjCIvarRefExpr>(Val: E)) {
3272 QualType ExpTy = E->getType();
3273 if (IsMemberAccess && ExpTy->isPointerType()) {
3274 // If ivar is a structure pointer, assigning to field of
3275 // this struct follows gcc's behavior and makes it a non-ivar
3276 // writer-barrier conservatively.
3277 ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
3278 if (ExpTy->isRecordType()) {
3279 LV.setObjCIvar(false);
3280 return;
3281 }
3282 }
3283 LV.setObjCIvar(true);
3284 auto *Exp = cast<ObjCIvarRefExpr>(Val: const_cast<Expr *>(E));
3285 LV.setBaseIvarExp(Exp->getBase());
3286 LV.setObjCArray(E->getType()->isArrayType());
3287 return;
3288 }
3289
3290 if (const auto *Exp = dyn_cast<DeclRefExpr>(Val: E)) {
3291 if (const auto *VD = dyn_cast<VarDecl>(Val: Exp->getDecl())) {
3292 if (VD->hasGlobalStorage()) {
3293 LV.setGlobalObjCRef(true);
3294 LV.setThreadLocalRef(VD->getTLSKind() != VarDecl::TLS_None);
3295 }
3296 }
3297 LV.setObjCArray(E->getType()->isArrayType());
3298 return;
3299 }
3300
3301 if (const auto *Exp = dyn_cast<UnaryOperator>(Val: E)) {
3302 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3303 return;
3304 }
3305
3306 if (const auto *Exp = dyn_cast<ParenExpr>(Val: E)) {
3307 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3308 if (LV.isObjCIvar()) {
3309 // If cast is to a structure pointer, follow gcc's behavior and make it
3310 // a non-ivar write-barrier.
3311 QualType ExpTy = E->getType();
3312 if (ExpTy->isPointerType())
3313 ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
3314 if (ExpTy->isRecordType())
3315 LV.setObjCIvar(false);
3316 }
3317 return;
3318 }
3319
3320 if (const auto *Exp = dyn_cast<GenericSelectionExpr>(Val: E)) {
3321 setObjCGCLValueClass(Ctx, E: Exp->getResultExpr(), LV);
3322 return;
3323 }
3324
3325 if (const auto *Exp = dyn_cast<ImplicitCastExpr>(Val: E)) {
3326 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3327 return;
3328 }
3329
3330 if (const auto *Exp = dyn_cast<CStyleCastExpr>(Val: E)) {
3331 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3332 return;
3333 }
3334
3335 if (const auto *Exp = dyn_cast<ObjCBridgedCastExpr>(Val: E)) {
3336 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3337 return;
3338 }
3339
3340 if (const auto *Exp = dyn_cast<ArraySubscriptExpr>(Val: E)) {
3341 setObjCGCLValueClass(Ctx, E: Exp->getBase(), LV);
3342 if (LV.isObjCIvar() && !LV.isObjCArray())
3343 // Using array syntax to assigning to what an ivar points to is not
3344 // same as assigning to the ivar itself. {id *Names;} Names[i] = 0;
3345 LV.setObjCIvar(false);
3346 else if (LV.isGlobalObjCRef() && !LV.isObjCArray())
3347 // Using array syntax to assigning to what global points to is not
3348 // same as assigning to the global itself. {id *G;} G[i] = 0;
3349 LV.setGlobalObjCRef(false);
3350 return;
3351 }
3352
3353 if (const auto *Exp = dyn_cast<MemberExpr>(Val: E)) {
3354 setObjCGCLValueClass(Ctx, E: Exp->getBase(), LV, IsMemberAccess: true);
3355 // We don't know if member is an 'ivar', but this flag is looked at
3356 // only in the context of LV.isObjCIvar().
3357 LV.setObjCArray(E->getType()->isArrayType());
3358 return;
3359 }
3360}
3361
3362static LValue EmitThreadPrivateVarDeclLValue(
3363 CodeGenFunction &CGF, const VarDecl *VD, QualType T, Address Addr,
3364 llvm::Type *RealVarTy, SourceLocation Loc) {
3365 if (CGF.CGM.getLangOpts().OpenMPIRBuilder)
3366 Addr = CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(
3367 CGF, VD, VDAddr: Addr, Loc);
3368 else
3369 Addr =
3370 CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, VDAddr: Addr, Loc);
3371
3372 Addr = Addr.withElementType(ElemTy: RealVarTy);
3373 return CGF.MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3374}
3375
3376static Address emitDeclTargetVarDeclLValue(CodeGenFunction &CGF,
3377 const VarDecl *VD, QualType T) {
3378 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
3379 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
3380 // Always return an invalid address for MT_Local, and also for
3381 // MT_To/MT_Enter when unified memory is not enabled. These use direct
3382 // access (global exists in device image). Otherwise, return a valid
3383 // address.
3384 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Local ||
3385 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
3386 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
3387 !CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory()))
3388 return Address::invalid();
3389 assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
3390 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
3391 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
3392 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) &&
3393 "Expected link clause OR to clause with unified memory enabled.");
3394 QualType PtrTy = CGF.getContext().getPointerType(T: VD->getType());
3395 Address Addr = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
3396 return CGF.EmitLoadOfPointer(Ptr: Addr, PtrTy: PtrTy->castAs<PointerType>());
3397}
3398
3399Address
3400CodeGenFunction::EmitLoadOfReference(LValue RefLVal,
3401 LValueBaseInfo *PointeeBaseInfo,
3402 TBAAAccessInfo *PointeeTBAAInfo) {
3403 llvm::LoadInst *Load =
3404 Builder.CreateLoad(Addr: RefLVal.getAddress(), IsVolatile: RefLVal.isVolatile());
3405 CGM.DecorateInstructionWithTBAA(Inst: Load, TBAAInfo: RefLVal.getTBAAInfo());
3406 QualType PTy = RefLVal.getType()->getPointeeType();
3407 CharUnits Align = CGM.getNaturalTypeAlignment(
3408 T: PTy, BaseInfo: PointeeBaseInfo, TBAAInfo: PointeeTBAAInfo, /*ForPointeeType=*/forPointeeType: true);
3409 if (!PTy->isIncompleteType()) {
3410 llvm::LLVMContext &Ctx = getLLVMContext();
3411 llvm::MDBuilder MDB(Ctx);
3412 // Emit !nonnull metadata
3413 if (CGM.getTypes().getTargetAddressSpace(T: PTy) == 0 &&
3414 !CGM.getCodeGenOpts().NullPointerIsValid)
3415 Load->setMetadata(KindID: llvm::LLVMContext::MD_nonnull,
3416 Node: llvm::MDNode::get(Context&: Ctx, MDs: {}));
3417 // Emit !align metadata
3418 if (PTy->isObjectType()) {
3419 auto AlignVal = Align.getQuantity();
3420 if (AlignVal > 1) {
3421 Load->setMetadata(
3422 KindID: llvm::LLVMContext::MD_align,
3423 Node: llvm::MDNode::get(Context&: Ctx, MDs: MDB.createConstant(C: llvm::ConstantInt::get(
3424 Ty: Builder.getInt64Ty(), V: AlignVal))));
3425 }
3426 }
3427 }
3428 return makeNaturalAddressForPointer(Ptr: Load, T: PTy, Alignment: Align,
3429 /*ForPointeeType=*/true, BaseInfo: PointeeBaseInfo,
3430 TBAAInfo: PointeeTBAAInfo);
3431}
3432
3433LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) {
3434 LValueBaseInfo PointeeBaseInfo;
3435 TBAAAccessInfo PointeeTBAAInfo;
3436 Address PointeeAddr = EmitLoadOfReference(RefLVal, PointeeBaseInfo: &PointeeBaseInfo,
3437 PointeeTBAAInfo: &PointeeTBAAInfo);
3438 return MakeAddrLValue(Addr: PointeeAddr, T: RefLVal.getType()->getPointeeType(),
3439 BaseInfo: PointeeBaseInfo, TBAAInfo: PointeeTBAAInfo);
3440}
3441
3442Address CodeGenFunction::EmitLoadOfPointer(Address Ptr,
3443 const PointerType *PtrTy,
3444 LValueBaseInfo *BaseInfo,
3445 TBAAAccessInfo *TBAAInfo) {
3446 llvm::Value *Addr = Builder.CreateLoad(Addr: Ptr);
3447 return makeNaturalAddressForPointer(Ptr: Addr, T: PtrTy->getPointeeType(),
3448 Alignment: CharUnits(), /*ForPointeeType=*/true,
3449 BaseInfo, TBAAInfo);
3450}
3451
3452LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr,
3453 const PointerType *PtrTy) {
3454 LValueBaseInfo BaseInfo;
3455 TBAAAccessInfo TBAAInfo;
3456 Address Addr = EmitLoadOfPointer(Ptr: PtrAddr, PtrTy, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
3457 return MakeAddrLValue(Addr, T: PtrTy->getPointeeType(), BaseInfo, TBAAInfo);
3458}
3459
3460static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
3461 const Expr *E, const VarDecl *VD) {
3462 QualType T = E->getType();
3463
3464 // If it's thread_local, emit a call to its wrapper function instead.
3465 if (VD->getTLSKind() == VarDecl::TLS_Dynamic &&
3466 CGF.CGM.getCXXABI().usesThreadWrapperFunction(VD))
3467 return CGF.CGM.getCXXABI().EmitThreadLocalVarDeclLValue(CGF, VD, LValType: T);
3468 // Check if the variable is marked as declare target with link clause in
3469 // device codegen.
3470 if (CGF.getLangOpts().OpenMPIsTargetDevice) {
3471 Address Addr = emitDeclTargetVarDeclLValue(CGF, VD, T);
3472 if (Addr.isValid())
3473 return CGF.MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3474 }
3475
3476 // Global HLSL resource arrays initialized on access; create a temporary with
3477 // the initialized global resource array.
3478 if (CGF.getLangOpts().HLSL && VD->getType()->isHLSLResourceRecordArray()) {
3479 std::optional<LValue> LV =
3480 CGF.CGM.getHLSLRuntime().emitGlobalResourceArrayAsLValue(CGF, ArrayDecl: VD);
3481 if (LV.has_value())
3482 return LV.value();
3483 }
3484
3485 llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(D: VD);
3486
3487 if (VD->getTLSKind() != VarDecl::TLS_None)
3488 V = CGF.Builder.CreateThreadLocalAddress(Ptr: V);
3489
3490 llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(T: VD->getType());
3491 CharUnits Alignment = CGF.getContext().getDeclAlign(D: VD);
3492 Address Addr(V, RealVarTy, Alignment);
3493 // Emit reference to the private copy of the variable if it is an OpenMP
3494 // threadprivate variable.
3495 if (CGF.getLangOpts().OpenMP && !CGF.getLangOpts().OpenMPSimd &&
3496 VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
3497 return EmitThreadPrivateVarDeclLValue(CGF, VD, T, Addr, RealVarTy,
3498 Loc: E->getExprLoc());
3499 }
3500 LValue LV = VD->getType()->isReferenceType() ?
3501 CGF.EmitLoadOfReferenceLValue(RefAddr: Addr, RefTy: VD->getType(),
3502 Source: AlignmentSource::Decl) :
3503 CGF.MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3504 setObjCGCLValueClass(Ctx: CGF.getContext(), E, LV);
3505 return LV;
3506}
3507
3508llvm::Constant *CodeGenModule::getRawFunctionPointer(GlobalDecl GD,
3509 llvm::Type *Ty) {
3510 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
3511 if (FD->hasAttr<WeakRefAttr>()) {
3512 ConstantAddress aliasee = GetWeakRefReference(VD: FD);
3513 return aliasee.getPointer();
3514 }
3515
3516 llvm::Constant *V = GetAddrOfFunction(GD, Ty);
3517 return V;
3518}
3519
3520static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF, const Expr *E,
3521 GlobalDecl GD) {
3522 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
3523 llvm::Constant *V = CGF.CGM.getFunctionPointer(GD);
3524 QualType ETy = E->getType();
3525 if (ETy->isCFIUncheckedCalleeFunctionType()) {
3526 if (auto *GV = dyn_cast<llvm::GlobalValue>(Val: V))
3527 V = llvm::NoCFIValue::get(GV);
3528 }
3529 CharUnits Alignment = CGF.getContext().getDeclAlign(D: FD);
3530 return CGF.MakeAddrLValue(V, T: ETy, Alignment, Source: AlignmentSource::Decl);
3531}
3532
3533static LValue EmitCapturedFieldLValue(CodeGenFunction &CGF, const FieldDecl *FD,
3534 llvm::Value *ThisValue) {
3535
3536 return CGF.EmitLValueForLambdaField(Field: FD, ThisValue);
3537}
3538
3539/// Named Registers are named metadata pointing to the register name
3540/// which will be read from/written to as an argument to the intrinsic
3541/// @llvm.read/write_register.
3542/// So far, only the name is being passed down, but other options such as
3543/// register type, allocation type or even optimization options could be
3544/// passed down via the metadata node.
3545static LValue EmitGlobalNamedRegister(const VarDecl *VD, CodeGenModule &CGM) {
3546 SmallString<64> Name("llvm.named.register.");
3547 AsmLabelAttr *Asm = VD->getAttr<AsmLabelAttr>();
3548 assert(Asm->getLabel().size() < 64-Name.size() &&
3549 "Register name too big");
3550 Name.append(RHS: Asm->getLabel());
3551 llvm::NamedMDNode *M =
3552 CGM.getModule().getOrInsertNamedMetadata(Name);
3553 if (M->getNumOperands() == 0) {
3554 llvm::MDString *Str = llvm::MDString::get(Context&: CGM.getLLVMContext(),
3555 Str: Asm->getLabel());
3556 llvm::Metadata *Ops[] = {Str};
3557 M->addOperand(M: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Ops));
3558 }
3559
3560 CharUnits Alignment = CGM.getContext().getDeclAlign(D: VD);
3561
3562 llvm::Value *Ptr =
3563 llvm::MetadataAsValue::get(Context&: CGM.getLLVMContext(), MD: M->getOperand(i: 0));
3564 return LValue::MakeGlobalReg(V: Ptr, alignment: Alignment, type: VD->getType());
3565}
3566
3567/// Determine whether we can emit a reference to \p VD from the current
3568/// context, despite not necessarily having seen an odr-use of the variable in
3569/// this context.
3570static bool canEmitSpuriousReferenceToVariable(CodeGenFunction &CGF,
3571 const DeclRefExpr *E,
3572 const VarDecl *VD) {
3573 // For a variable declared in an enclosing scope, do not emit a spurious
3574 // reference even if we have a capture, as that will emit an unwarranted
3575 // reference to our capture state, and will likely generate worse code than
3576 // emitting a local copy.
3577 if (E->refersToEnclosingVariableOrCapture())
3578 return false;
3579
3580 // For a local declaration declared in this function, we can always reference
3581 // it even if we don't have an odr-use.
3582 if (VD->hasLocalStorage()) {
3583 return VD->getDeclContext() ==
3584 dyn_cast_or_null<DeclContext>(Val: CGF.CurCodeDecl);
3585 }
3586
3587 // For a global declaration, we can emit a reference to it if we know
3588 // for sure that we are able to emit a definition of it.
3589 VD = VD->getDefinition(C&: CGF.getContext());
3590 if (!VD)
3591 return false;
3592
3593 // Don't emit a spurious reference if it might be to a variable that only
3594 // exists on a different device / target.
3595 // FIXME: This is unnecessarily broad. Check whether this would actually be a
3596 // cross-target reference.
3597 if (CGF.getLangOpts().OpenMP || CGF.getLangOpts().CUDA ||
3598 CGF.getLangOpts().OpenCL) {
3599 return false;
3600 }
3601
3602 // We can emit a spurious reference only if the linkage implies that we'll
3603 // be emitting a non-interposable symbol that will be retained until link
3604 // time.
3605 switch (CGF.CGM.getLLVMLinkageVarDefinition(VD)) {
3606 case llvm::GlobalValue::ExternalLinkage:
3607 case llvm::GlobalValue::LinkOnceODRLinkage:
3608 case llvm::GlobalValue::WeakODRLinkage:
3609 case llvm::GlobalValue::InternalLinkage:
3610 case llvm::GlobalValue::PrivateLinkage:
3611 return true;
3612 default:
3613 return false;
3614 }
3615}
3616
3617LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
3618 const NamedDecl *ND = E->getDecl();
3619 QualType T = E->getType();
3620
3621 assert(E->isNonOdrUse() != NOUR_Unevaluated &&
3622 "should not emit an unevaluated operand");
3623
3624 if (const auto *VD = dyn_cast<VarDecl>(Val: ND)) {
3625 // Global Named registers access via intrinsics only
3626 if (VD->getStorageClass() == SC_Register &&
3627 VD->hasAttr<AsmLabelAttr>() && !VD->isLocalVarDecl())
3628 return EmitGlobalNamedRegister(VD, CGM);
3629
3630 // If this DeclRefExpr does not constitute an odr-use of the variable,
3631 // we're not permitted to emit a reference to it in general, and it might
3632 // not be captured if capture would be necessary for a use. Emit the
3633 // constant value directly instead.
3634 if (E->isNonOdrUse() == NOUR_Constant &&
3635 (VD->getType()->isReferenceType() ||
3636 !canEmitSpuriousReferenceToVariable(CGF&: *this, E, VD))) {
3637 VD->getAnyInitializer(D&: VD);
3638 llvm::Constant *Val = ConstantEmitter(*this).emitAbstract(
3639 loc: E->getLocation(), value: *VD->evaluateValue(), T: VD->getType());
3640 assert(Val && "failed to emit constant expression");
3641
3642 Address Addr = Address::invalid();
3643 if (!VD->getType()->isReferenceType()) {
3644 // Spill the constant value to a global.
3645 Addr = CGM.createUnnamedGlobalFrom(D: *VD, Constant: Val,
3646 Align: getContext().getDeclAlign(D: VD));
3647 llvm::Type *VarTy = getTypes().ConvertTypeForMem(T: VD->getType());
3648 auto *PTy = llvm::PointerType::get(
3649 C&: getLLVMContext(), AddressSpace: getTypes().getTargetAddressSpace(T: VD->getType()));
3650 Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty: PTy, ElementTy: VarTy);
3651 } else {
3652 // Should we be using the alignment of the constant pointer we emitted?
3653 CharUnits Alignment =
3654 CGM.getNaturalTypeAlignment(T: E->getType(),
3655 /* BaseInfo= */ nullptr,
3656 /* TBAAInfo= */ nullptr,
3657 /* forPointeeType= */ true);
3658 Addr = makeNaturalAddressForPointer(Ptr: Val, T, Alignment);
3659 }
3660 return MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3661 }
3662
3663 // FIXME: Handle other kinds of non-odr-use DeclRefExprs.
3664
3665 // Check for captured variables.
3666 if (E->refersToEnclosingVariableOrCapture()) {
3667 VD = VD->getCanonicalDecl();
3668 if (auto *FD = LambdaCaptureFields.lookup(Val: VD))
3669 return EmitCapturedFieldLValue(CGF&: *this, FD, ThisValue: CXXABIThisValue);
3670 if (CapturedStmtInfo) {
3671 auto I = LocalDeclMap.find(Val: VD);
3672 if (I != LocalDeclMap.end()) {
3673 LValue CapLVal;
3674 if (VD->getType()->isReferenceType())
3675 CapLVal = EmitLoadOfReferenceLValue(RefAddr: I->second, RefTy: VD->getType(),
3676 Source: AlignmentSource::Decl);
3677 else
3678 CapLVal = MakeAddrLValue(Addr: I->second, T);
3679 // Mark lvalue as nontemporal if the variable is marked as nontemporal
3680 // in simd context.
3681 if (getLangOpts().OpenMP &&
3682 CGM.getOpenMPRuntime().isNontemporalDecl(VD))
3683 CapLVal.setNontemporal(/*Value=*/true);
3684 return CapLVal;
3685 }
3686 LValue CapLVal =
3687 EmitCapturedFieldLValue(CGF&: *this, FD: CapturedStmtInfo->lookup(VD),
3688 ThisValue: CapturedStmtInfo->getContextValue());
3689 Address LValueAddress = CapLVal.getAddress();
3690 CapLVal = MakeAddrLValue(Addr: Address(LValueAddress.emitRawPointer(CGF&: *this),
3691 LValueAddress.getElementType(),
3692 getContext().getDeclAlign(D: VD)),
3693 T: CapLVal.getType(),
3694 BaseInfo: LValueBaseInfo(AlignmentSource::Decl),
3695 TBAAInfo: CapLVal.getTBAAInfo());
3696 // Mark lvalue as nontemporal if the variable is marked as nontemporal
3697 // in simd context.
3698 if (getLangOpts().OpenMP &&
3699 CGM.getOpenMPRuntime().isNontemporalDecl(VD))
3700 CapLVal.setNontemporal(/*Value=*/true);
3701 return CapLVal;
3702 }
3703
3704 assert(isa<BlockDecl>(CurCodeDecl));
3705 Address addr = GetAddrOfBlockDecl(var: VD);
3706 return MakeAddrLValue(Addr: addr, T, Source: AlignmentSource::Decl);
3707 }
3708 }
3709
3710 // FIXME: We should be able to assert this for FunctionDecls as well!
3711 // FIXME: We should be able to assert this for all DeclRefExprs, not just
3712 // those with a valid source location.
3713 assert((ND->isUsed(false) || !isa<VarDecl>(ND) || E->isNonOdrUse() ||
3714 !E->getLocation().isValid()) &&
3715 "Should not use decl without marking it used!");
3716
3717 if (ND->hasAttr<WeakRefAttr>()) {
3718 const auto *VD = cast<ValueDecl>(Val: ND);
3719 ConstantAddress Aliasee = CGM.GetWeakRefReference(VD);
3720 return MakeAddrLValue(Addr: Aliasee, T, Source: AlignmentSource::Decl);
3721 }
3722
3723 if (const auto *VD = dyn_cast<VarDecl>(Val: ND)) {
3724 // Check if this is a global variable.
3725 if (VD->hasLinkage() || VD->isStaticDataMember())
3726 return EmitGlobalVarDeclLValue(CGF&: *this, E, VD);
3727
3728 Address addr = Address::invalid();
3729
3730 // The variable should generally be present in the local decl map.
3731 auto iter = LocalDeclMap.find(Val: VD);
3732 if (iter != LocalDeclMap.end()) {
3733 addr = iter->second;
3734
3735 // Otherwise, it might be static local we haven't emitted yet for
3736 // some reason; most likely, because it's in an outer function.
3737 } else if (VD->isStaticLocal()) {
3738 llvm::Constant *var = CGM.getOrCreateStaticVarDecl(
3739 D: *VD, Linkage: CGM.getLLVMLinkageVarDefinition(VD));
3740 addr = Address(
3741 var, ConvertTypeForMem(T: VD->getType()), getContext().getDeclAlign(D: VD));
3742
3743 // No other cases for now.
3744 } else {
3745 llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?");
3746 }
3747
3748 // Handle threadlocal function locals.
3749 if (VD->getTLSKind() != VarDecl::TLS_None)
3750 addr = addr.withPointer(
3751 NewPointer: Builder.CreateThreadLocalAddress(Ptr: addr.getBasePointer()),
3752 IsKnownNonNull: NotKnownNonNull);
3753
3754 // Check for OpenMP threadprivate variables.
3755 if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd &&
3756 VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
3757 return EmitThreadPrivateVarDeclLValue(
3758 CGF&: *this, VD, T, Addr: addr, RealVarTy: getTypes().ConvertTypeForMem(T: VD->getType()),
3759 Loc: E->getExprLoc());
3760 }
3761
3762 // Drill into block byref variables.
3763 bool isBlockByref = VD->isEscapingByref();
3764 if (isBlockByref) {
3765 addr = emitBlockByrefAddress(baseAddr: addr, V: VD);
3766 }
3767
3768 // Drill into reference types.
3769 LValue LV = VD->getType()->isReferenceType() ?
3770 EmitLoadOfReferenceLValue(RefAddr: addr, RefTy: VD->getType(), Source: AlignmentSource::Decl) :
3771 MakeAddrLValue(Addr: addr, T, Source: AlignmentSource::Decl);
3772
3773 bool isLocalStorage = VD->hasLocalStorage();
3774
3775 bool NonGCable = isLocalStorage &&
3776 !VD->getType()->isReferenceType() &&
3777 !isBlockByref;
3778 if (NonGCable) {
3779 LV.getQuals().removeObjCGCAttr();
3780 LV.setNonGC(true);
3781 }
3782
3783 bool isImpreciseLifetime =
3784 (isLocalStorage && !VD->hasAttr<ObjCPreciseLifetimeAttr>());
3785 if (isImpreciseLifetime)
3786 LV.setARCPreciseLifetime(ARCImpreciseLifetime);
3787 setObjCGCLValueClass(Ctx: getContext(), E, LV);
3788 return LV;
3789 }
3790
3791 if (const auto *FD = dyn_cast<FunctionDecl>(Val: ND))
3792 return EmitFunctionDeclLValue(CGF&: *this, E, GD: FD);
3793
3794 // FIXME: While we're emitting a binding from an enclosing scope, all other
3795 // DeclRefExprs we see should be implicitly treated as if they also refer to
3796 // an enclosing scope.
3797 if (const auto *BD = dyn_cast<BindingDecl>(Val: ND)) {
3798 if (E->refersToEnclosingVariableOrCapture()) {
3799 auto *FD = LambdaCaptureFields.lookup(Val: BD);
3800 return EmitCapturedFieldLValue(CGF&: *this, FD, ThisValue: CXXABIThisValue);
3801 }
3802 // Suppress debug location updates when visiting the binding, since the
3803 // binding may emit instructions that would otherwise be associated with the
3804 // binding itself, rather than the expression referencing the binding. (this
3805 // leads to jumpy debug stepping behavior where the location/debugger jump
3806 // back to the binding declaration, then back to the expression referencing
3807 // the binding)
3808 DisableDebugLocationUpdates D(*this);
3809 return EmitLValue(E: BD->getBinding(), IsKnownNonNull: NotKnownNonNull);
3810 }
3811
3812 // We can form DeclRefExprs naming GUID declarations when reconstituting
3813 // non-type template parameters into expressions.
3814 if (const auto *GD = dyn_cast<MSGuidDecl>(Val: ND))
3815 return MakeAddrLValue(Addr: CGM.GetAddrOfMSGuidDecl(GD), T,
3816 Source: AlignmentSource::Decl);
3817
3818 if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(Val: ND)) {
3819 ConstantAddress ATPO = CGM.GetAddrOfTemplateParamObject(TPO);
3820 auto AS = getLangASFromTargetAS(TargetAS: ATPO.getAddressSpace());
3821
3822 if (AS != T.getAddressSpace()) {
3823 auto TargetAS = getContext().getTargetAddressSpace(AS: T.getAddressSpace());
3824 llvm::Type *PtrTy =
3825 llvm::PointerType::get(C&: CGM.getLLVMContext(), AddressSpace: TargetAS);
3826 llvm::Constant *ASC = CGM.performAddrSpaceCast(Src: ATPO.getPointer(), DestTy: PtrTy);
3827 ATPO = ConstantAddress(ASC, ATPO.getElementType(), ATPO.getAlignment());
3828 }
3829
3830 return MakeAddrLValue(Addr: ATPO, T, Source: AlignmentSource::Decl);
3831 }
3832
3833 llvm_unreachable("Unhandled DeclRefExpr");
3834}
3835
3836LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
3837 // __extension__ doesn't affect lvalue-ness.
3838 if (E->getOpcode() == UO_Extension)
3839 return EmitLValue(E: E->getSubExpr());
3840
3841 QualType ExprTy = getContext().getCanonicalType(T: E->getSubExpr()->getType());
3842 switch (E->getOpcode()) {
3843 default: llvm_unreachable("Unknown unary operator lvalue!");
3844 case UO_Deref: {
3845 QualType T = E->getSubExpr()->getType()->getPointeeType();
3846 assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
3847
3848 LValueBaseInfo BaseInfo;
3849 TBAAAccessInfo TBAAInfo;
3850 Address Addr = EmitPointerWithAlignment(E: E->getSubExpr(), BaseInfo: &BaseInfo,
3851 TBAAInfo: &TBAAInfo);
3852 LValue LV = MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
3853 LV.getQuals().setAddressSpace(ExprTy.getAddressSpace());
3854
3855 // We should not generate __weak write barrier on indirect reference
3856 // of a pointer to object; as in void foo (__weak id *param); *param = 0;
3857 // But, we continue to generate __strong write barrier on indirect write
3858 // into a pointer to object.
3859 if (getLangOpts().ObjC &&
3860 getLangOpts().getGC() != LangOptions::NonGC &&
3861 LV.isObjCWeak())
3862 LV.setNonGC(!E->isOBJCGCCandidate(Ctx&: getContext()));
3863 return LV;
3864 }
3865 case UO_Real:
3866 case UO_Imag: {
3867 LValue LV = EmitLValue(E: E->getSubExpr());
3868 assert(LV.isSimple() && "real/imag on non-ordinary l-value");
3869
3870 // __real is valid on scalars. This is a faster way of testing that.
3871 // __imag can only produce an rvalue on scalars.
3872 if (E->getOpcode() == UO_Real &&
3873 !LV.getAddress().getElementType()->isStructTy()) {
3874 assert(E->getSubExpr()->getType()->isArithmeticType());
3875 return LV;
3876 }
3877
3878 QualType T = ExprTy->castAs<ComplexType>()->getElementType();
3879
3880 Address Component =
3881 (E->getOpcode() == UO_Real
3882 ? emitAddrOfRealComponent(complex: LV.getAddress(), complexType: LV.getType())
3883 : emitAddrOfImagComponent(complex: LV.getAddress(), complexType: LV.getType()));
3884 LValue ElemLV = MakeAddrLValue(Addr: Component, T, BaseInfo: LV.getBaseInfo(),
3885 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: T));
3886 ElemLV.getQuals().addQualifiers(Q: LV.getQuals());
3887 return ElemLV;
3888 }
3889 case UO_PreInc:
3890 case UO_PreDec: {
3891 LValue LV = EmitLValue(E: E->getSubExpr());
3892 bool isInc = E->getOpcode() == UO_PreInc;
3893
3894 if (E->getType()->isAnyComplexType())
3895 EmitComplexPrePostIncDec(E, LV, isInc, isPre: true/*isPre*/);
3896 else
3897 EmitScalarPrePostIncDec(E, LV, isInc, isPre: true/*isPre*/);
3898 return LV;
3899 }
3900 }
3901}
3902
3903LValue CodeGenFunction::EmitStringLiteralLValue(const StringLiteral *E) {
3904 return MakeAddrLValue(Addr: CGM.GetAddrOfConstantStringFromLiteral(S: E),
3905 T: E->getType(), Source: AlignmentSource::Decl);
3906}
3907
3908LValue CodeGenFunction::EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E) {
3909 return MakeAddrLValue(Addr: CGM.GetAddrOfConstantStringFromObjCEncode(E),
3910 T: E->getType(), Source: AlignmentSource::Decl);
3911}
3912
3913LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
3914 auto SL = E->getFunctionName();
3915 assert(SL != nullptr && "No StringLiteral name in PredefinedExpr");
3916 StringRef FnName = CurFn->getName();
3917 FnName.consume_front(Prefix: "\01");
3918 StringRef NameItems[] = {
3919 PredefinedExpr::getIdentKindName(IK: E->getIdentKind()), FnName};
3920 std::string GVName = llvm::join(Begin: NameItems, End: NameItems + 2, Separator: ".");
3921 if (auto *BD = dyn_cast_or_null<BlockDecl>(Val: CurCodeDecl)) {
3922 std::string Name = std::string(SL->getString());
3923 if (!Name.empty()) {
3924 unsigned Discriminator =
3925 CGM.getCXXABI().getMangleContext().getBlockId(BD, Local: true);
3926 if (Discriminator)
3927 Name += "_" + Twine(Discriminator + 1).str();
3928 auto C = CGM.GetAddrOfConstantCString(Str: Name, GlobalName: GVName);
3929 return MakeAddrLValue(Addr: C, T: E->getType(), Source: AlignmentSource::Decl);
3930 } else {
3931 auto C = CGM.GetAddrOfConstantCString(Str: std::string(FnName), GlobalName: GVName);
3932 return MakeAddrLValue(Addr: C, T: E->getType(), Source: AlignmentSource::Decl);
3933 }
3934 }
3935 auto C = CGM.GetAddrOfConstantStringFromLiteral(S: SL, Name: GVName);
3936 return MakeAddrLValue(Addr: C, T: E->getType(), Source: AlignmentSource::Decl);
3937}
3938
3939/// Emit a type description suitable for use by a runtime sanitizer library. The
3940/// format of a type descriptor is
3941///
3942/// \code
3943/// { i16 TypeKind, i16 TypeInfo }
3944/// \endcode
3945///
3946/// followed by an array of i8 containing the type name with extra information
3947/// for BitInt. TypeKind is TK_Integer(0) for an integer, TK_Float(1) for a
3948/// floating point value, TK_BitInt(2) for BitInt and TK_Unknown(0xFFFF) for
3949/// anything else.
3950llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) {
3951 // Only emit each type's descriptor once.
3952 if (llvm::Constant *C = CGM.getTypeDescriptorFromMap(Ty: T))
3953 return C;
3954
3955 uint16_t TypeKind = TK_Unknown;
3956 uint16_t TypeInfo = 0;
3957 bool IsBitInt = false;
3958
3959 if (T->isIntegerType()) {
3960 TypeKind = TK_Integer;
3961 TypeInfo = (llvm::Log2_32(Value: getContext().getTypeSize(T)) << 1) |
3962 (T->isSignedIntegerType() ? 1 : 0);
3963 // Follow suggestion from discussion of issue 64100.
3964 // So we can write the exact amount of bits in TypeName after '\0'
3965 // making it <diagnostic-like type name>.'\0'.<32-bit width>.
3966 if (T->isSignedIntegerType() && T->getAs<BitIntType>()) {
3967 // Do a sanity checks as we are using 32-bit type to store bit length.
3968 assert(getContext().getTypeSize(T) > 0 &&
3969 " non positive amount of bits in __BitInt type");
3970 assert(getContext().getTypeSize(T) <= 0xFFFFFFFF &&
3971 " too many bits in __BitInt type");
3972
3973 // Redefine TypeKind with the actual __BitInt type if we have signed
3974 // BitInt.
3975 TypeKind = TK_BitInt;
3976 IsBitInt = true;
3977 }
3978 } else if (T->isFloatingType()) {
3979 TypeKind = TK_Float;
3980 TypeInfo = getContext().getTypeSize(T);
3981 }
3982
3983 // Format the type name as if for a diagnostic, including quotes and
3984 // optionally an 'aka'.
3985 SmallString<32> Buffer;
3986 CGM.getDiags().ConvertArgToString(Kind: DiagnosticsEngine::ak_qualtype,
3987 Val: (intptr_t)T.getAsOpaquePtr(), Modifier: StringRef(),
3988 Argument: StringRef(), PrevArgs: {}, Output&: Buffer, QualTypeVals: {});
3989
3990 if (IsBitInt) {
3991 // The Structure is: 0 to end the string, 32 bit unsigned integer in target
3992 // endianness, zero.
3993 char S[6] = {'\0', '\0', '\0', '\0', '\0', '\0'};
3994 const auto *EIT = T->castAs<BitIntType>();
3995 uint32_t Bits = EIT->getNumBits();
3996 llvm::support::endian::write32(P: S + 1, V: Bits,
3997 E: getTarget().isBigEndian()
3998 ? llvm::endianness::big
3999 : llvm::endianness::little);
4000 StringRef Str = StringRef(S, sizeof(S) / sizeof(decltype(S[0])));
4001 Buffer.append(RHS: Str);
4002 }
4003
4004 llvm::Constant *Components[] = {
4005 Builder.getInt16(C: TypeKind), Builder.getInt16(C: TypeInfo),
4006 llvm::ConstantDataArray::getString(Context&: getLLVMContext(), Initializer: Buffer)
4007 };
4008 llvm::Constant *Descriptor = llvm::ConstantStruct::getAnon(V: Components);
4009
4010 auto *GV = new llvm::GlobalVariable(
4011 CGM.getModule(), Descriptor->getType(),
4012 /*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage, Descriptor);
4013 GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4014 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV);
4015
4016 // Remember the descriptor for this type.
4017 CGM.setTypeDescriptorInMap(Ty: T, C: GV);
4018
4019 return GV;
4020}
4021
4022llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) {
4023 llvm::Type *TargetTy = IntPtrTy;
4024
4025 if (V->getType() == TargetTy)
4026 return V;
4027
4028 // Floating-point types which fit into intptr_t are bitcast to integers
4029 // and then passed directly (after zero-extension, if necessary).
4030 if (V->getType()->isFloatingPointTy()) {
4031 unsigned Bits = V->getType()->getPrimitiveSizeInBits().getFixedValue();
4032 if (Bits <= TargetTy->getIntegerBitWidth())
4033 V = Builder.CreateBitCast(V, DestTy: llvm::Type::getIntNTy(C&: getLLVMContext(),
4034 N: Bits));
4035 }
4036
4037 // Integers which fit in intptr_t are zero-extended and passed directly.
4038 if (V->getType()->isIntegerTy() &&
4039 V->getType()->getIntegerBitWidth() <= TargetTy->getIntegerBitWidth())
4040 return Builder.CreateZExt(V, DestTy: TargetTy);
4041
4042 // Pointers are passed directly, everything else is passed by address.
4043 if (!V->getType()->isPointerTy()) {
4044 RawAddress Ptr = CreateDefaultAlignTempAlloca(Ty: V->getType());
4045 Builder.CreateStore(Val: V, Addr: Ptr);
4046 V = Ptr.getPointer();
4047 }
4048 return Builder.CreatePtrToInt(V, DestTy: TargetTy);
4049}
4050
4051/// Emit a representation of a SourceLocation for passing to a handler
4052/// in a sanitizer runtime library. The format for this data is:
4053/// \code
4054/// struct SourceLocation {
4055/// const char *Filename;
4056/// int32_t Line, Column;
4057/// };
4058/// \endcode
4059/// For an invalid SourceLocation, the Filename pointer is null.
4060llvm::Constant *CodeGenFunction::EmitCheckSourceLocation(SourceLocation Loc) {
4061 llvm::Constant *Filename;
4062 int Line, Column;
4063
4064 PresumedLoc PLoc = getContext().getSourceManager().getPresumedLoc(Loc);
4065 if (PLoc.isValid()) {
4066 StringRef FilenameString = PLoc.getFilename();
4067
4068 int PathComponentsToStrip =
4069 CGM.getCodeGenOpts().EmitCheckPathComponentsToStrip;
4070 if (PathComponentsToStrip < 0) {
4071 assert(PathComponentsToStrip != INT_MIN);
4072 int PathComponentsToKeep = -PathComponentsToStrip;
4073 auto I = llvm::sys::path::rbegin(path: FilenameString);
4074 auto E = llvm::sys::path::rend(path: FilenameString);
4075 while (I != E && --PathComponentsToKeep)
4076 ++I;
4077
4078 FilenameString = FilenameString.substr(Start: I - E);
4079 } else if (PathComponentsToStrip > 0) {
4080 auto I = llvm::sys::path::begin(path: FilenameString);
4081 auto E = llvm::sys::path::end(path: FilenameString);
4082 while (I != E && PathComponentsToStrip--)
4083 ++I;
4084
4085 if (I != E)
4086 FilenameString =
4087 FilenameString.substr(Start: I - llvm::sys::path::begin(path: FilenameString));
4088 else
4089 FilenameString = llvm::sys::path::filename(path: FilenameString);
4090 }
4091
4092 auto FilenameGV =
4093 CGM.GetAddrOfConstantCString(Str: std::string(FilenameString), GlobalName: ".src");
4094 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(
4095 GV: cast<llvm::GlobalVariable>(
4096 Val: FilenameGV.getPointer()->stripPointerCasts()));
4097 Filename = FilenameGV.getPointer();
4098 Line = PLoc.getLine();
4099 Column = PLoc.getColumn();
4100 } else {
4101 Filename = llvm::Constant::getNullValue(Ty: Int8PtrTy);
4102 Line = Column = 0;
4103 }
4104
4105 llvm::Constant *Data[] = {Filename, Builder.getInt32(C: Line),
4106 Builder.getInt32(C: Column)};
4107
4108 return llvm::ConstantStruct::getAnon(V: Data);
4109}
4110
4111namespace {
4112/// Specify under what conditions this check can be recovered
4113enum class CheckRecoverableKind {
4114 /// Always terminate program execution if this check fails.
4115 Unrecoverable,
4116 /// Check supports recovering, runtime has both fatal (noreturn) and
4117 /// non-fatal handlers for this check.
4118 Recoverable,
4119 /// Runtime conditionally aborts, always need to support recovery.
4120 AlwaysRecoverable
4121};
4122}
4123
4124static CheckRecoverableKind
4125getRecoverableKind(SanitizerKind::SanitizerOrdinal Ordinal) {
4126 if (Ordinal == SanitizerKind::SO_Vptr)
4127 return CheckRecoverableKind::AlwaysRecoverable;
4128 else if (Ordinal == SanitizerKind::SO_Return ||
4129 Ordinal == SanitizerKind::SO_Unreachable)
4130 return CheckRecoverableKind::Unrecoverable;
4131 else
4132 return CheckRecoverableKind::Recoverable;
4133}
4134
4135namespace {
4136struct SanitizerHandlerInfo {
4137 char const *const Name;
4138 unsigned Version;
4139};
4140}
4141
4142const SanitizerHandlerInfo SanitizerHandlers[] = {
4143#define SANITIZER_CHECK(Enum, Name, Version, Msg) {#Name, Version},
4144 LIST_SANITIZER_CHECKS
4145#undef SANITIZER_CHECK
4146};
4147
4148static void emitCheckHandlerCall(CodeGenFunction &CGF,
4149 llvm::FunctionType *FnType,
4150 ArrayRef<llvm::Value *> FnArgs,
4151 SanitizerHandler CheckHandler,
4152 CheckRecoverableKind RecoverKind, bool IsFatal,
4153 llvm::BasicBlock *ContBB, bool NoMerge) {
4154 assert(IsFatal || RecoverKind != CheckRecoverableKind::Unrecoverable);
4155 std::optional<ApplyDebugLocation> DL;
4156 if (!CGF.Builder.getCurrentDebugLocation()) {
4157 // Ensure that the call has at least an artificial debug location.
4158 DL.emplace(args&: CGF, args: SourceLocation());
4159 }
4160 bool NeedsAbortSuffix =
4161 IsFatal && RecoverKind != CheckRecoverableKind::Unrecoverable;
4162 bool MinimalRuntime = CGF.CGM.getCodeGenOpts().SanitizeMinimalRuntime;
4163 bool HandlerPreserveAllRegs =
4164 CGF.CGM.getCodeGenOpts().SanitizeHandlerPreserveAllRegs;
4165 const SanitizerHandlerInfo &CheckInfo = SanitizerHandlers[CheckHandler];
4166 const StringRef CheckName = CheckInfo.Name;
4167 std::string FnName = "__ubsan_handle_" + CheckName.str();
4168 if (CheckInfo.Version && !MinimalRuntime)
4169 FnName += "_v" + llvm::utostr(X: CheckInfo.Version);
4170 if (MinimalRuntime)
4171 FnName += "_minimal";
4172 if (NeedsAbortSuffix)
4173 FnName += "_abort";
4174 if (HandlerPreserveAllRegs && !NeedsAbortSuffix)
4175 FnName += "_preserve";
4176 bool MayReturn =
4177 !IsFatal || RecoverKind == CheckRecoverableKind::AlwaysRecoverable;
4178
4179 llvm::AttrBuilder B(CGF.getLLVMContext());
4180 if (!MayReturn) {
4181 B.addAttribute(Val: llvm::Attribute::NoReturn)
4182 .addAttribute(Val: llvm::Attribute::NoUnwind);
4183 }
4184 B.addUWTableAttr(Kind: llvm::UWTableKind::Default);
4185
4186 llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(
4187 Ty: FnType, Name: FnName,
4188 ExtraAttrs: llvm::AttributeList::get(C&: CGF.getLLVMContext(),
4189 Index: llvm::AttributeList::FunctionIndex, B),
4190 /*Local=*/true);
4191 llvm::CallInst *HandlerCall = CGF.EmitNounwindRuntimeCall(callee: Fn, args: FnArgs);
4192 NoMerge = NoMerge || !CGF.CGM.getCodeGenOpts().isOptimizedBuild() ||
4193 (CGF.CurCodeDecl && CGF.CurCodeDecl->hasAttr<OptimizeNoneAttr>());
4194 if (NoMerge)
4195 HandlerCall->addFnAttr(Kind: llvm::Attribute::NoMerge);
4196 if (HandlerPreserveAllRegs && !NeedsAbortSuffix) {
4197 // N.B. there is also a clang::CallingConv which is not what we want here.
4198 HandlerCall->setCallingConv(llvm::CallingConv::PreserveAll);
4199 }
4200 if (!MayReturn) {
4201 HandlerCall->setDoesNotReturn();
4202 CGF.Builder.CreateUnreachable();
4203 } else {
4204 CGF.Builder.CreateBr(Dest: ContBB);
4205 }
4206}
4207
4208void CodeGenFunction::EmitCheck(
4209 ArrayRef<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>> Checked,
4210 SanitizerHandler CheckHandler, ArrayRef<llvm::Constant *> StaticArgs,
4211 ArrayRef<llvm::Value *> DynamicArgs, const TrapReason *TR) {
4212 assert(IsSanitizerScope);
4213 assert(Checked.size() > 0);
4214 assert(CheckHandler >= 0 &&
4215 size_t(CheckHandler) < std::size(SanitizerHandlers));
4216 const StringRef CheckName = SanitizerHandlers[CheckHandler].Name;
4217
4218 llvm::Value *FatalCond = nullptr;
4219 llvm::Value *RecoverableCond = nullptr;
4220 llvm::Value *TrapCond = nullptr;
4221 bool NoMerge = false;
4222 // Expand checks into:
4223 // (Check1 || !allow_ubsan_check) && (Check2 || !allow_ubsan_check) ...
4224 // We need separate allow_ubsan_check intrinsics because they have separately
4225 // specified cutoffs.
4226 // This expression looks expensive but will be simplified after
4227 // LowerAllowCheckPass.
4228 for (auto &[Check, Ord] : Checked) {
4229 llvm::Value *GuardedCheck = Check;
4230 if (ClSanitizeGuardChecks ||
4231 (CGM.getCodeGenOpts().SanitizeSkipHotCutoffs[Ord] > 0)) {
4232 llvm::Value *Allow = Builder.CreateCall(
4233 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::allow_ubsan_check),
4234 Args: llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: Ord));
4235 GuardedCheck = Builder.CreateOr(LHS: Check, RHS: Builder.CreateNot(V: Allow));
4236 }
4237
4238 // -fsanitize-trap= overrides -fsanitize-recover=.
4239 llvm::Value *&Cond = CGM.getCodeGenOpts().SanitizeTrap.has(O: Ord) ? TrapCond
4240 : CGM.getCodeGenOpts().SanitizeRecover.has(O: Ord)
4241 ? RecoverableCond
4242 : FatalCond;
4243 Cond = Cond ? Builder.CreateAnd(LHS: Cond, RHS: GuardedCheck) : GuardedCheck;
4244
4245 if (!CGM.getCodeGenOpts().SanitizeMergeHandlers.has(O: Ord))
4246 NoMerge = true;
4247 }
4248
4249 if (TrapCond)
4250 EmitTrapCheck(Checked: TrapCond, CheckHandlerID: CheckHandler, NoMerge, TR);
4251 if (!FatalCond && !RecoverableCond)
4252 return;
4253
4254 llvm::Value *JointCond;
4255 if (FatalCond && RecoverableCond)
4256 JointCond = Builder.CreateAnd(LHS: FatalCond, RHS: RecoverableCond);
4257 else
4258 JointCond = FatalCond ? FatalCond : RecoverableCond;
4259 assert(JointCond);
4260
4261 CheckRecoverableKind RecoverKind = getRecoverableKind(Ordinal: Checked[0].second);
4262 assert(SanOpts.has(Checked[0].second));
4263#ifndef NDEBUG
4264 for (int i = 1, n = Checked.size(); i < n; ++i) {
4265 assert(RecoverKind == getRecoverableKind(Checked[i].second) &&
4266 "All recoverable kinds in a single check must be same!");
4267 assert(SanOpts.has(Checked[i].second));
4268 }
4269#endif
4270
4271 llvm::BasicBlock *Cont = createBasicBlock(name: "cont");
4272 llvm::BasicBlock *Handlers = createBasicBlock(name: "handler." + CheckName);
4273 llvm::Instruction *Branch = Builder.CreateCondBr(Cond: JointCond, True: Cont, False: Handlers);
4274 // Give hint that we very much don't expect to execute the handler
4275 llvm::MDBuilder MDHelper(getLLVMContext());
4276 llvm::MDNode *Node = MDHelper.createLikelyBranchWeights();
4277 Branch->setMetadata(KindID: llvm::LLVMContext::MD_prof, Node);
4278 EmitBlock(BB: Handlers);
4279
4280 // Clear arguments for the MinimalRuntime handler.
4281 if (CGM.getCodeGenOpts().SanitizeMinimalRuntime) {
4282 StaticArgs = {};
4283 DynamicArgs = {};
4284 }
4285
4286 // Handler functions take an i8* pointing to the (handler-specific) static
4287 // information block, followed by a sequence of intptr_t arguments
4288 // representing operand values.
4289 SmallVector<llvm::Value *, 4> Args;
4290 SmallVector<llvm::Type *, 4> ArgTypes;
4291
4292 Args.reserve(N: DynamicArgs.size() + 1);
4293 ArgTypes.reserve(N: DynamicArgs.size() + 1);
4294
4295 // Emit handler arguments and create handler function type.
4296 if (!StaticArgs.empty()) {
4297 llvm::Constant *Info = llvm::ConstantStruct::getAnon(V: StaticArgs);
4298 auto *InfoPtr = new llvm::GlobalVariable(
4299 CGM.getModule(), Info->getType(),
4300 // Non-constant global is used in a handler to deduplicate reports.
4301 // TODO: change deduplication logic and make it constant.
4302 /*isConstant=*/false, llvm::GlobalVariable::PrivateLinkage, Info, "",
4303 nullptr, llvm::GlobalVariable::NotThreadLocal,
4304 CGM.getDataLayout().getDefaultGlobalsAddressSpace());
4305 InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4306 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV: InfoPtr);
4307 Args.push_back(Elt: Builder.CreateAddrSpaceCast(V: InfoPtr, DestTy: CGM.VoidPtrTy));
4308 ArgTypes.push_back(Elt: CGM.VoidPtrTy);
4309 }
4310
4311 for (llvm::Value *DynamicArg : DynamicArgs) {
4312 Args.push_back(Elt: EmitCheckValue(V: DynamicArg));
4313 ArgTypes.push_back(Elt: IntPtrTy);
4314 }
4315
4316 llvm::FunctionType *FnType =
4317 llvm::FunctionType::get(Result: CGM.VoidTy, Params: ArgTypes, isVarArg: false);
4318
4319 if (!FatalCond || !RecoverableCond) {
4320 // Simple case: we need to generate a single handler call, either
4321 // fatal, or non-fatal.
4322 emitCheckHandlerCall(CGF&: *this, FnType, FnArgs: Args, CheckHandler, RecoverKind,
4323 IsFatal: (FatalCond != nullptr), ContBB: Cont, NoMerge);
4324 } else {
4325 // Emit two handler calls: first one for set of unrecoverable checks,
4326 // another one for recoverable.
4327 llvm::BasicBlock *NonFatalHandlerBB =
4328 createBasicBlock(name: "non_fatal." + CheckName);
4329 llvm::BasicBlock *FatalHandlerBB = createBasicBlock(name: "fatal." + CheckName);
4330 Builder.CreateCondBr(Cond: FatalCond, True: NonFatalHandlerBB, False: FatalHandlerBB);
4331 EmitBlock(BB: FatalHandlerBB);
4332 emitCheckHandlerCall(CGF&: *this, FnType, FnArgs: Args, CheckHandler, RecoverKind, IsFatal: true,
4333 ContBB: NonFatalHandlerBB, NoMerge);
4334 EmitBlock(BB: NonFatalHandlerBB);
4335 emitCheckHandlerCall(CGF&: *this, FnType, FnArgs: Args, CheckHandler, RecoverKind, IsFatal: false,
4336 ContBB: Cont, NoMerge);
4337 }
4338
4339 EmitBlock(BB: Cont);
4340}
4341
4342void CodeGenFunction::EmitCfiSlowPathCheck(
4343 SanitizerKind::SanitizerOrdinal Ordinal, llvm::Value *Cond,
4344 llvm::ConstantInt *TypeId, llvm::Value *Ptr,
4345 ArrayRef<llvm::Constant *> StaticArgs) {
4346 llvm::BasicBlock *Cont = createBasicBlock(name: "cfi.cont");
4347
4348 llvm::BasicBlock *CheckBB = createBasicBlock(name: "cfi.slowpath");
4349 llvm::CondBrInst *BI = Builder.CreateCondBr(Cond, True: Cont, False: CheckBB);
4350
4351 llvm::MDBuilder MDHelper(getLLVMContext());
4352 llvm::MDNode *Node = MDHelper.createLikelyBranchWeights();
4353 BI->setMetadata(KindID: llvm::LLVMContext::MD_prof, Node);
4354
4355 EmitBlock(BB: CheckBB);
4356
4357 bool WithDiag = !CGM.getCodeGenOpts().SanitizeTrap.has(O: Ordinal);
4358
4359 llvm::CallInst *CheckCall;
4360 llvm::FunctionCallee SlowPathFn;
4361 if (WithDiag) {
4362 llvm::Constant *Info = llvm::ConstantStruct::getAnon(V: StaticArgs);
4363 auto *InfoPtr =
4364 new llvm::GlobalVariable(CGM.getModule(), Info->getType(), false,
4365 llvm::GlobalVariable::PrivateLinkage, Info);
4366 InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4367 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV: InfoPtr);
4368
4369 SlowPathFn = CGM.getModule().getOrInsertFunction(
4370 Name: "__cfi_slowpath_diag",
4371 T: llvm::FunctionType::get(Result: VoidTy, Params: {Int64Ty, Int8PtrTy, Int8PtrTy},
4372 isVarArg: false));
4373 CheckCall = Builder.CreateCall(Callee: SlowPathFn, Args: {TypeId, Ptr, InfoPtr});
4374 } else {
4375 SlowPathFn = CGM.getModule().getOrInsertFunction(
4376 Name: "__cfi_slowpath",
4377 T: llvm::FunctionType::get(Result: VoidTy, Params: {Int64Ty, Int8PtrTy}, isVarArg: false));
4378 CheckCall = Builder.CreateCall(Callee: SlowPathFn, Args: {TypeId, Ptr});
4379 }
4380
4381 CGM.setDSOLocal(
4382 cast<llvm::GlobalValue>(Val: SlowPathFn.getCallee()->stripPointerCasts()));
4383 CheckCall->setDoesNotThrow();
4384
4385 EmitBlock(BB: Cont);
4386}
4387
4388// Emit a stub for __cfi_check function so that the linker knows about this
4389// symbol in LTO mode.
4390void CodeGenFunction::EmitCfiCheckStub() {
4391 llvm::Module *M = &CGM.getModule();
4392 ASTContext &C = getContext();
4393 QualType QInt64Ty = C.getIntTypeForBitwidth(DestWidth: 64, Signed: false);
4394
4395 auto *ArgCallsiteTypeId =
4396 ImplicitParamDecl::Create(C, T: QInt64Ty, ParamKind: ImplicitParamKind::Other);
4397 auto *ArgAddr =
4398 ImplicitParamDecl::Create(C, T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
4399 auto *ArgCFICheckFailData =
4400 ImplicitParamDecl::Create(C, T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
4401 FunctionArgList FnArgs{ArgCallsiteTypeId, ArgAddr, ArgCFICheckFailData};
4402 const CGFunctionInfo &FI =
4403 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: FnArgs);
4404
4405 llvm::Function *F = llvm::Function::Create(
4406 Ty: llvm::FunctionType::get(Result: VoidTy, Params: {Int64Ty, VoidPtrTy, VoidPtrTy}, isVarArg: false),
4407 Linkage: llvm::GlobalValue::WeakAnyLinkage, N: "__cfi_check", M);
4408 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F, /*IsThunk=*/false);
4409 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F);
4410 F->setAlignment(llvm::Align(4096));
4411 CGM.setDSOLocal(F);
4412
4413 llvm::LLVMContext &Ctx = M->getContext();
4414 llvm::BasicBlock *BB = llvm::BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: F);
4415 // CrossDSOCFI pass is not executed if there is no executable code.
4416 SmallVector<llvm::Value*> Args{F->getArg(i: 2), F->getArg(i: 1)};
4417 llvm::CallInst::Create(Func: M->getFunction(Name: "__cfi_check_fail"), Args, NameStr: "", InsertBefore: BB);
4418 llvm::ReturnInst::Create(C&: Ctx, retVal: nullptr, InsertBefore: BB);
4419}
4420
4421// This function is basically a switch over the CFI failure kind, which is
4422// extracted from CFICheckFailData (1st function argument). Each case is either
4423// llvm.trap or a call to one of the two runtime handlers, based on
4424// -fsanitize-trap and -fsanitize-recover settings. Default case (invalid
4425// failure kind) traps, but this should really never happen. CFICheckFailData
4426// can be nullptr if the calling module has -fsanitize-trap behavior for this
4427// check kind; in this case __cfi_check_fail traps as well.
4428void CodeGenFunction::EmitCfiCheckFail() {
4429 auto CheckHandler = SanitizerHandler::CFICheckFail;
4430 // TODO: the SanitizerKind is not yet determined for this check (and might
4431 // not even be available, if Data == nullptr). However, we still want to
4432 // annotate the instrumentation. We approximate this by using all the CFI
4433 // kinds.
4434 SanitizerDebugLocation SanScope(
4435 this,
4436 {SanitizerKind::SO_CFIVCall, SanitizerKind::SO_CFINVCall,
4437 SanitizerKind::SO_CFIDerivedCast, SanitizerKind::SO_CFIUnrelatedCast,
4438 SanitizerKind::SO_CFIICall},
4439 CheckHandler);
4440 auto *ArgData = ImplicitParamDecl::Create(
4441 C&: getContext(), T: getContext().VoidPtrTy, ParamKind: ImplicitParamKind::Other);
4442 auto *ArgAddr = ImplicitParamDecl::Create(
4443 C&: getContext(), T: getContext().VoidPtrTy, ParamKind: ImplicitParamKind::Other);
4444
4445 FunctionArgList Args{ArgData, ArgAddr};
4446 const CGFunctionInfo &FI =
4447 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: getContext().VoidTy, args: Args);
4448
4449 llvm::Function *F = llvm::Function::Create(
4450 Ty: llvm::FunctionType::get(Result: VoidTy, Params: {VoidPtrTy, VoidPtrTy}, isVarArg: false),
4451 Linkage: llvm::GlobalValue::WeakODRLinkage, N: "__cfi_check_fail", M: &CGM.getModule());
4452
4453 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F, /*IsThunk=*/false);
4454 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F);
4455 F->setVisibility(llvm::GlobalValue::HiddenVisibility);
4456
4457 StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidTy, Fn: F, FnInfo: FI, Args,
4458 Loc: SourceLocation());
4459
4460 ApplyDebugLocation ADL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
4461
4462 // This function is not affected by NoSanitizeList. This function does
4463 // not have a source location, but "src:*" would still apply. Revert any
4464 // changes to SanOpts made in StartFunction.
4465 SanOpts = CGM.getLangOpts().Sanitize;
4466
4467 llvm::Value *Data =
4468 EmitLoadOfScalar(Addr: GetAddrOfLocalVar(VD: ArgData), /*Volatile=*/false,
4469 Ty: CGM.getContext().VoidPtrTy, Loc: ArgData->getLocation());
4470 llvm::Value *Addr =
4471 EmitLoadOfScalar(Addr: GetAddrOfLocalVar(VD: ArgAddr), /*Volatile=*/false,
4472 Ty: CGM.getContext().VoidPtrTy, Loc: ArgAddr->getLocation());
4473
4474 // Data == nullptr means the calling module has trap behaviour for this check.
4475 llvm::Value *DataIsNotNullPtr =
4476 Builder.CreateICmpNE(LHS: Data, RHS: llvm::ConstantPointerNull::get(T: Int8PtrTy));
4477 // TODO: since there is no data, we don't know the CheckKind, and therefore
4478 // cannot inspect CGM.getCodeGenOpts().SanitizeMergeHandlers. We default to
4479 // NoMerge = false. Users can disable merging by disabling optimization.
4480 EmitTrapCheck(Checked: DataIsNotNullPtr, CheckHandlerID: SanitizerHandler::CFICheckFail,
4481 /*NoMerge=*/false);
4482
4483 llvm::StructType *SourceLocationTy =
4484 llvm::StructType::get(elt1: VoidPtrTy, elts: Int32Ty, elts: Int32Ty);
4485 llvm::StructType *CfiCheckFailDataTy =
4486 llvm::StructType::get(elt1: Int8Ty, elts: SourceLocationTy, elts: VoidPtrTy);
4487
4488 llvm::Value *V = Builder.CreateConstGEP2_32(
4489 Ty: CfiCheckFailDataTy, Ptr: Builder.CreatePointerCast(V: Data, DestTy: DefaultPtrTy), Idx0: 0, Idx1: 0);
4490
4491 Address CheckKindAddr(V, Int8Ty, getIntAlign());
4492 llvm::Value *CheckKind = Builder.CreateLoad(Addr: CheckKindAddr);
4493
4494 llvm::Value *AllVtables = llvm::MetadataAsValue::get(
4495 Context&: CGM.getLLVMContext(),
4496 MD: llvm::MDString::get(Context&: CGM.getLLVMContext(), Str: "all-vtables"));
4497 llvm::Value *ValidVtable = Builder.CreateZExt(
4498 V: Builder.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::type_test),
4499 Args: {Addr, AllVtables}),
4500 DestTy: IntPtrTy);
4501
4502 const std::pair<int, SanitizerKind::SanitizerOrdinal> CheckKinds[] = {
4503 {CFITCK_VCall, SanitizerKind::SO_CFIVCall},
4504 {CFITCK_NVCall, SanitizerKind::SO_CFINVCall},
4505 {CFITCK_DerivedCast, SanitizerKind::SO_CFIDerivedCast},
4506 {CFITCK_UnrelatedCast, SanitizerKind::SO_CFIUnrelatedCast},
4507 {CFITCK_ICall, SanitizerKind::SO_CFIICall}};
4508
4509 for (auto CheckKindOrdinalPair : CheckKinds) {
4510 int Kind = CheckKindOrdinalPair.first;
4511 SanitizerKind::SanitizerOrdinal Ordinal = CheckKindOrdinalPair.second;
4512
4513 // TODO: we could apply SanitizerAnnotateDebugInfo(Ordinal) instead of
4514 // relying on the SanitizerScope with all CFI ordinals
4515
4516 llvm::Value *Cond =
4517 Builder.CreateICmpNE(LHS: CheckKind, RHS: llvm::ConstantInt::get(Ty: Int8Ty, V: Kind));
4518 if (CGM.getLangOpts().Sanitize.has(O: Ordinal))
4519 EmitCheck(Checked: std::make_pair(x&: Cond, y&: Ordinal), CheckHandler: SanitizerHandler::CFICheckFail,
4520 StaticArgs: {}, DynamicArgs: {Data, Addr, ValidVtable});
4521 else
4522 // TODO: we can't rely on CGM.getCodeGenOpts().SanitizeMergeHandlers.
4523 // Although the compiler allows SanitizeMergeHandlers to be set
4524 // independently of CGM.getLangOpts().Sanitize, Driver/SanitizerArgs.cpp
4525 // requires that SanitizeMergeHandlers is a subset of Sanitize.
4526 EmitTrapCheck(Checked: Cond, CheckHandlerID: CheckHandler, /*NoMerge=*/false);
4527 }
4528
4529 FinishFunction();
4530 // The only reference to this function will be created during LTO link.
4531 // Make sure it survives until then.
4532 CGM.addUsedGlobal(GV: F);
4533}
4534
4535void CodeGenFunction::EmitUnreachable(SourceLocation Loc) {
4536 if (SanOpts.has(K: SanitizerKind::Unreachable)) {
4537 auto CheckOrdinal = SanitizerKind::SO_Unreachable;
4538 auto CheckHandler = SanitizerHandler::BuiltinUnreachable;
4539 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
4540 EmitCheck(Checked: std::make_pair(x: static_cast<llvm::Value *>(Builder.getFalse()),
4541 y&: CheckOrdinal),
4542 CheckHandler, StaticArgs: EmitCheckSourceLocation(Loc), DynamicArgs: {});
4543 }
4544 Builder.CreateUnreachable();
4545}
4546
4547void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked,
4548 SanitizerHandler CheckHandlerID,
4549 bool NoMerge, const TrapReason *TR) {
4550 llvm::BasicBlock *Cont = createBasicBlock(name: "cont");
4551
4552 // If we're optimizing, collapse all calls to trap down to just one per
4553 // check-type per function to save on code size.
4554 if ((int)TrapBBs.size() <= CheckHandlerID)
4555 TrapBBs.resize(N: CheckHandlerID + 1);
4556
4557 llvm::BasicBlock *&TrapBB = TrapBBs[CheckHandlerID];
4558
4559 llvm::DILocation *TrapLocation = Builder.getCurrentDebugLocation();
4560 llvm::StringRef TrapMessage;
4561 llvm::StringRef TrapCategory;
4562 auto DebugTrapReasonKind = CGM.getCodeGenOpts().getSanitizeDebugTrapReasons();
4563 if (TR && !TR->isEmpty() &&
4564 DebugTrapReasonKind ==
4565 CodeGenOptions::SanitizeDebugTrapReasonKind::Detailed) {
4566 TrapMessage = TR->getMessage();
4567 TrapCategory = TR->getCategory();
4568 } else {
4569 TrapMessage = GetUBSanTrapForHandler(ID: CheckHandlerID);
4570 TrapCategory = "Undefined Behavior Sanitizer";
4571 }
4572
4573 if (getDebugInfo() && !TrapMessage.empty() &&
4574 DebugTrapReasonKind !=
4575 CodeGenOptions::SanitizeDebugTrapReasonKind::None &&
4576 TrapLocation) {
4577 TrapLocation = getDebugInfo()->CreateTrapFailureMessageFor(
4578 TrapLocation, Category: TrapCategory, FailureMsg: TrapMessage);
4579 }
4580
4581 NoMerge = NoMerge || !CGM.getCodeGenOpts().isOptimizedBuild() ||
4582 (CurCodeDecl && CurCodeDecl->hasAttr<OptimizeNoneAttr>());
4583
4584 llvm::MDBuilder MDHelper(getLLVMContext());
4585 if (TrapBB && !NoMerge) {
4586 auto Call = TrapBB->begin();
4587 assert(isa<llvm::CallInst>(Call) && "Expected call in trap BB");
4588
4589 Call->applyMergedLocation(LocA: Call->getDebugLoc(), LocB: TrapLocation);
4590
4591 Builder.CreateCondBr(Cond: Checked, True: Cont, False: TrapBB,
4592 BranchWeights: MDHelper.createLikelyBranchWeights());
4593 } else {
4594 TrapBB = createBasicBlock(name: "trap");
4595 Builder.CreateCondBr(Cond: Checked, True: Cont, False: TrapBB,
4596 BranchWeights: MDHelper.createLikelyBranchWeights());
4597 EmitBlock(BB: TrapBB);
4598
4599 ApplyDebugLocation applyTrapDI(*this, TrapLocation);
4600
4601 llvm::CallInst *TrapCall;
4602 if (CGM.getCodeGenOpts().SanitizeTrapLoop)
4603 TrapCall =
4604 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::looptrap));
4605 else
4606 TrapCall = Builder.CreateCall(
4607 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::ubsantrap),
4608 Args: llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: CheckHandlerID));
4609
4610 if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
4611 auto A = llvm::Attribute::get(Context&: getLLVMContext(), Kind: "trap-func-name",
4612 Val: CGM.getCodeGenOpts().TrapFuncName);
4613 TrapCall->addFnAttr(Attr: A);
4614 }
4615 if (NoMerge)
4616 TrapCall->addFnAttr(Kind: llvm::Attribute::NoMerge);
4617 TrapCall->setDoesNotReturn();
4618 TrapCall->setDoesNotThrow();
4619 Builder.CreateUnreachable();
4620 }
4621
4622 EmitBlock(BB: Cont);
4623}
4624
4625llvm::CallInst *CodeGenFunction::EmitTrapCall(llvm::Intrinsic::ID IntrID,
4626 bool EnsureInsertPoint) {
4627 llvm::Function *TrapIntrinsic = CGM.getIntrinsic(IID: IntrID);
4628 llvm::CallInst *TrapCall = Builder.CreateCall(Callee: TrapIntrinsic);
4629
4630 if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
4631 auto A = llvm::Attribute::get(Context&: getLLVMContext(), Kind: "trap-func-name",
4632 Val: CGM.getCodeGenOpts().TrapFuncName);
4633 TrapCall->addFnAttr(Attr: A);
4634 }
4635
4636 if (InNoMergeAttributedStmt)
4637 TrapCall->addFnAttr(Kind: llvm::Attribute::NoMerge);
4638 if (TrapIntrinsic->doesNotThrow())
4639 TrapCall->setDoesNotThrow();
4640 if (TrapIntrinsic->doesNotReturn()) {
4641 TrapCall->setDoesNotReturn();
4642 Builder.CreateUnreachable();
4643 if (EnsureInsertPoint)
4644 EmitBlock(BB: createBasicBlock());
4645 else
4646 Builder.ClearInsertionPoint();
4647 }
4648 return TrapCall;
4649}
4650
4651void CodeGenFunction::EmitTrapCallAndMakeUnreachable() {
4652 llvm::CallInst *TrapCall =
4653 EmitTrapCall(IntrID: llvm::Intrinsic::trap, /*EnsureInsertPoint=*/false);
4654 TrapCall->setDoesNotReturn();
4655 TrapCall->setDoesNotThrow();
4656 if (HaveInsertPoint()) {
4657 Builder.CreateUnreachable();
4658 Builder.ClearInsertionPoint();
4659 }
4660}
4661
4662Address CodeGenFunction::EmitArrayToPointerDecay(const Expr *E,
4663 LValueBaseInfo *BaseInfo,
4664 TBAAAccessInfo *TBAAInfo) {
4665 assert(E->getType()->isArrayType() &&
4666 "Array to pointer decay must have array source type!");
4667
4668 // Expressions of array type can't be bitfields or vector elements.
4669 LValue LV = EmitLValue(E);
4670 Address Addr = LV.getAddress();
4671
4672 // If the array type was an incomplete type, we need to make sure
4673 // the decay ends up being the right type.
4674 llvm::Type *NewTy = ConvertType(T: E->getType());
4675 Addr = Addr.withElementType(ElemTy: NewTy);
4676
4677 // Note that VLA pointers are always decayed, so we don't need to do
4678 // anything here.
4679 if (!E->getType()->isVariableArrayType()) {
4680 assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
4681 "Expected pointer to array");
4682
4683 if (getLangOpts().EmitLogicalPointer) {
4684 // Array-to-pointer decay for an SGEP is a no-op as we don't do any
4685 // logical indexing. See #179951 for some additional context.
4686 auto *SGEP =
4687 Builder.CreateStructuredGEP(BaseType: NewTy, PtrBase: Addr.emitRawPointer(CGF&: *this), Indices: {});
4688 Addr = Address(SGEP, NewTy, Addr.getAlignment(), Addr.isKnownNonNull());
4689 } else {
4690 Addr = Builder.CreateConstArrayGEP(Addr, Index: 0, Name: "arraydecay");
4691 }
4692 }
4693
4694 // The result of this decay conversion points to an array element within the
4695 // base lvalue. However, since TBAA currently does not support representing
4696 // accesses to elements of member arrays, we conservatively represent accesses
4697 // to the pointee object as if it had no any base lvalue specified.
4698 // TODO: Support TBAA for member arrays.
4699 QualType EltType = E->getType()->castAsArrayTypeUnsafe()->getElementType();
4700 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
4701 if (TBAAInfo) *TBAAInfo = CGM.getTBAAAccessInfo(AccessType: EltType);
4702
4703 return Addr.withElementType(ElemTy: ConvertTypeForMem(T: EltType));
4704}
4705
4706/// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an
4707/// array to pointer, return the array subexpression.
4708static const Expr *isSimpleArrayDecayOperand(const Expr *E) {
4709 // If this isn't just an array->pointer decay, bail out.
4710 const auto *CE = dyn_cast<CastExpr>(Val: E);
4711 if (!CE || CE->getCastKind() != CK_ArrayToPointerDecay)
4712 return nullptr;
4713
4714 // If this is a decay from variable width array, bail out.
4715 const Expr *SubExpr = CE->getSubExpr();
4716 if (SubExpr->getType()->isVariableArrayType())
4717 return nullptr;
4718
4719 return SubExpr;
4720}
4721
4722static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF,
4723 llvm::Type *elemType,
4724 llvm::Value *ptr,
4725 ArrayRef<llvm::Value*> indices,
4726 bool inbounds,
4727 bool signedIndices,
4728 SourceLocation loc,
4729 const llvm::Twine &name = "arrayidx") {
4730 if (inbounds && CGF.getLangOpts().EmitLogicalPointer)
4731 return CGF.Builder.CreateStructuredGEP(BaseType: elemType, PtrBase: ptr, Indices: indices);
4732
4733 if (inbounds) {
4734 return CGF.EmitCheckedInBoundsGEP(ElemTy: elemType, Ptr: ptr, IdxList: indices, SignedIndices: signedIndices,
4735 IsSubtraction: CodeGenFunction::NotSubtraction, Loc: loc,
4736 Name: name);
4737 } else {
4738 return CGF.Builder.CreateGEP(Ty: elemType, Ptr: ptr, IdxList: indices, Name: name);
4739 }
4740}
4741
4742static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
4743 ArrayRef<llvm::Value *> indices,
4744 llvm::Type *arrayType,
4745 llvm::Type *elementType, bool inbounds,
4746 bool signedIndices, SourceLocation loc,
4747 CharUnits align,
4748 const llvm::Twine &name = "arrayidx") {
4749 if (inbounds && CGF.getLangOpts().EmitLogicalPointer)
4750 return RawAddress(CGF.Builder.CreateStructuredGEP(BaseType: arrayType,
4751 PtrBase: addr.emitRawPointer(CGF),
4752 Indices: indices.drop_front()),
4753 elementType, align);
4754
4755 if (inbounds) {
4756 return CGF.EmitCheckedInBoundsGEP(Addr: addr, IdxList: indices, elementType, SignedIndices: signedIndices,
4757 IsSubtraction: CodeGenFunction::NotSubtraction, Loc: loc,
4758 Align: align, Name: name);
4759 } else {
4760 return CGF.Builder.CreateGEP(Addr: addr, IdxList: indices, ElementType: elementType, Align: align, Name: name);
4761 }
4762}
4763
4764static QualType getFixedSizeElementType(const ASTContext &ctx,
4765 const VariableArrayType *vla) {
4766 QualType eltType;
4767 do {
4768 eltType = vla->getElementType();
4769 } while ((vla = ctx.getAsVariableArrayType(T: eltType)));
4770 return eltType;
4771}
4772
4773static bool hasBPFPreserveStaticOffset(const RecordDecl *D) {
4774 return D && D->hasAttr<BPFPreserveStaticOffsetAttr>();
4775}
4776
4777static bool hasBPFPreserveStaticOffset(const Expr *E) {
4778 if (!E)
4779 return false;
4780 QualType PointeeType = E->getType()->getPointeeType();
4781 if (PointeeType.isNull())
4782 return false;
4783 if (const auto *BaseDecl = PointeeType->getAsRecordDecl())
4784 return hasBPFPreserveStaticOffset(D: BaseDecl);
4785 return false;
4786}
4787
4788// Wraps Addr with a call to llvm.preserve.static.offset intrinsic.
4789static Address wrapWithBPFPreserveStaticOffset(CodeGenFunction &CGF,
4790 Address &Addr) {
4791 if (!CGF.getTarget().getTriple().isBPF())
4792 return Addr;
4793
4794 llvm::Function *Fn =
4795 CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::preserve_static_offset);
4796 llvm::CallInst *Call = CGF.Builder.CreateCall(Callee: Fn, Args: {Addr.emitRawPointer(CGF)});
4797 return Address(Call, Addr.getElementType(), Addr.getAlignment());
4798}
4799
4800/// Given an array base, check whether its member access belongs to a record
4801/// with preserve_access_index attribute or not.
4802static bool IsPreserveAIArrayBase(CodeGenFunction &CGF, const Expr *ArrayBase) {
4803 if (!ArrayBase || !CGF.getDebugInfo())
4804 return false;
4805
4806 // Only support base as either a MemberExpr or DeclRefExpr.
4807 // DeclRefExpr to cover cases like:
4808 // struct s { int a; int b[10]; };
4809 // struct s *p;
4810 // p[1].a
4811 // p[1] will generate a DeclRefExpr and p[1].a is a MemberExpr.
4812 // p->b[5] is a MemberExpr example.
4813 const Expr *E = ArrayBase->IgnoreImpCasts();
4814 if (const auto *ME = dyn_cast<MemberExpr>(Val: E))
4815 return ME->getMemberDecl()->hasAttr<BPFPreserveAccessIndexAttr>();
4816
4817 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
4818 const auto *VarDef = dyn_cast<VarDecl>(Val: DRE->getDecl());
4819 if (!VarDef)
4820 return false;
4821
4822 const auto *PtrT = VarDef->getType()->getAs<PointerType>();
4823 if (!PtrT)
4824 return false;
4825
4826 const auto *PointeeT = PtrT->getPointeeType()
4827 ->getUnqualifiedDesugaredType();
4828 if (const auto *RecT = dyn_cast<RecordType>(Val: PointeeT))
4829 return RecT->getDecl()
4830 ->getMostRecentDecl()
4831 ->hasAttr<BPFPreserveAccessIndexAttr>();
4832 return false;
4833 }
4834
4835 return false;
4836}
4837
4838static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
4839 ArrayRef<llvm::Value *> indices,
4840 QualType eltType, bool inbounds,
4841 bool signedIndices, SourceLocation loc,
4842 QualType *arrayType = nullptr,
4843 const Expr *Base = nullptr,
4844 const llvm::Twine &name = "arrayidx") {
4845 // All the indices except that last must be zero.
4846#ifndef NDEBUG
4847 for (auto *idx : indices.drop_back())
4848 assert(isa<llvm::ConstantInt>(idx) &&
4849 cast<llvm::ConstantInt>(idx)->isZero());
4850#endif
4851
4852 // Determine the element size of the statically-sized base. This is
4853 // the thing that the indices are expressed in terms of.
4854 if (auto vla = CGF.getContext().getAsVariableArrayType(T: eltType)) {
4855 eltType = getFixedSizeElementType(ctx: CGF.getContext(), vla);
4856 }
4857
4858 // We can use that to compute the best alignment of the element.
4859 CharUnits eltSize = CGF.getContext().getTypeSizeInChars(T: eltType);
4860 CharUnits eltAlign =
4861 getArrayElementAlign(arrayAlign: addr.getAlignment(), idx: indices.back(), eltSize);
4862
4863 if (hasBPFPreserveStaticOffset(E: Base))
4864 addr = wrapWithBPFPreserveStaticOffset(CGF, Addr&: addr);
4865
4866 llvm::Value *eltPtr;
4867 auto LastIndex = dyn_cast<llvm::ConstantInt>(Val: indices.back());
4868 if (!LastIndex ||
4869 (!CGF.IsInPreservedAIRegion && !IsPreserveAIArrayBase(CGF, ArrayBase: Base))) {
4870 addr = emitArraySubscriptGEP(CGF, addr, indices,
4871 arrayType: arrayType ? CGF.ConvertTypeForMem(T: *arrayType)
4872 : nullptr,
4873 elementType: CGF.ConvertTypeForMem(T: eltType), inbounds,
4874 signedIndices, loc, align: eltAlign, name);
4875 return addr;
4876 } else {
4877 // Remember the original array subscript for bpf target
4878 unsigned idx = LastIndex->getZExtValue();
4879 llvm::DIType *DbgInfo = nullptr;
4880 if (arrayType)
4881 DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(Ty: *arrayType, Loc: loc);
4882 eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(
4883 ElTy: addr.getElementType(), Base: addr.emitRawPointer(CGF), Dimension: indices.size() - 1,
4884 LastIndex: idx, DbgInfo);
4885 }
4886
4887 return Address(eltPtr, CGF.ConvertTypeForMem(T: eltType), eltAlign);
4888}
4889
4890namespace {
4891
4892/// StructFieldAccess is a simple visitor class to grab the first l-value to
4893/// r-value cast Expr.
4894struct StructFieldAccess
4895 : public ConstStmtVisitor<StructFieldAccess, const Expr *> {
4896 const Expr *VisitCastExpr(const CastExpr *E) {
4897 if (E->getCastKind() == CK_LValueToRValue)
4898 return E;
4899 return Visit(S: E->getSubExpr());
4900 }
4901 const Expr *VisitParenExpr(const ParenExpr *E) {
4902 return Visit(S: E->getSubExpr());
4903 }
4904};
4905
4906} // end anonymous namespace
4907
4908/// The offset of a field from the beginning of the record.
4909static bool getFieldOffsetInBits(CodeGenFunction &CGF, const RecordDecl *RD,
4910 const FieldDecl *Field, int64_t &Offset) {
4911 ASTContext &Ctx = CGF.getContext();
4912 const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(D: RD);
4913 unsigned FieldNo = 0;
4914
4915 for (const FieldDecl *FD : RD->fields()) {
4916 if (FD == Field) {
4917 Offset += Layout.getFieldOffset(FieldNo);
4918 return true;
4919 }
4920
4921 QualType Ty = FD->getType();
4922 if (Ty->isRecordType())
4923 if (getFieldOffsetInBits(CGF, RD: Ty->getAsRecordDecl(), Field, Offset)) {
4924 Offset += Layout.getFieldOffset(FieldNo);
4925 return true;
4926 }
4927
4928 if (!RD->isUnion())
4929 ++FieldNo;
4930 }
4931
4932 return false;
4933}
4934
4935/// Returns the relative offset difference between \p FD1 and \p FD2.
4936/// \code
4937/// offsetof(struct foo, FD1) - offsetof(struct foo, FD2)
4938/// \endcode
4939/// Both fields must be within the same struct.
4940static std::optional<int64_t> getOffsetDifferenceInBits(CodeGenFunction &CGF,
4941 const FieldDecl *FD1,
4942 const FieldDecl *FD2) {
4943 const RecordDecl *FD1OuterRec =
4944 FD1->getParent()->getOuterLexicalRecordContext();
4945 const RecordDecl *FD2OuterRec =
4946 FD2->getParent()->getOuterLexicalRecordContext();
4947
4948 if (FD1OuterRec != FD2OuterRec)
4949 // Fields must be within the same RecordDecl.
4950 return std::optional<int64_t>();
4951
4952 int64_t FD1Offset = 0;
4953 if (!getFieldOffsetInBits(CGF, RD: FD1OuterRec, Field: FD1, Offset&: FD1Offset))
4954 return std::optional<int64_t>();
4955
4956 int64_t FD2Offset = 0;
4957 if (!getFieldOffsetInBits(CGF, RD: FD2OuterRec, Field: FD2, Offset&: FD2Offset))
4958 return std::optional<int64_t>();
4959
4960 return std::make_optional<int64_t>(t: FD1Offset - FD2Offset);
4961}
4962
4963/// Convert a '__sized_by' byte-count bound to an element-count bound so it can
4964/// be compared against an element index. \p BoundsVal is the loaded byte count
4965/// and \p PointeeTy is the pointer's pointee type. Returns \p BoundsVal
4966/// unchanged when no scaling is needed, otherwise returns a new `llvm::Value*`
4967/// that is element count.
4968static llvm::Value *convertSizedByBoundToElementCount(CodeGenFunction &CGF,
4969 llvm::Value *BoundsVal,
4970 QualType PointeeTy,
4971 bool CountSigned) {
4972 assert(BoundsVal->getType()->isIntegerTy());
4973 assert(!PointeeTy.isNull() && "pointee type is never null");
4974 assert(!PointeeTy->isFunctionType() &&
4975 "Sema guarantees a '__sized_by' pointee is a non-function type");
4976
4977 if (PointeeTy->isIncompleteType()) {
4978 // Sema enforces that only 'void' can be subscripted here (GNU extension,
4979 // 1-byte stride), so the byte count already equals the element count.
4980 assert(PointeeTy->isVoidType() && "expected a 'void' incomplete pointee");
4981 return BoundsVal;
4982 }
4983
4984 CharUnits ElemSize = CGF.getContext().getTypeSizeInChars(T: PointeeTy);
4985 if (ElemSize <= CharUnits::One())
4986 return BoundsVal;
4987
4988 int64_t ElemSizeQ = ElemSize.getQuantity();
4989 unsigned CountWidth = BoundsVal->getType()->getIntegerBitWidth();
4990
4991 // The divisor must be representable in the count field's type.
4992 bool ElemSizeFits =
4993 CountSigned ? llvm::isIntN(N: CountWidth, x: ElemSizeQ)
4994 : llvm::isUIntN(N: CountWidth, x: static_cast<uint64_t>(ElemSizeQ));
4995 if (!ElemSizeFits)
4996 // No whole element fits in any representable byte count. Use a bound of 0
4997 // to always trap.
4998 // FIXME: Sema should just reject this (#223525).
4999 return llvm::ConstantInt::get(Ty: BoundsVal->getType(), V: 0);
5000
5001 llvm::Value *ElemSizeV =
5002 llvm::ConstantInt::get(Ty: BoundsVal->getType(), V: ElemSizeQ);
5003 // Use signed division for a signed count field so a negative byte count stays
5004 // non-positive and is still rejected by the negative-bounds guard in
5005 // EmitBoundsCheckImpl (unsigned division would turn it into a large positive
5006 // count).
5007 return CountSigned ? CGF.Builder.CreateSDiv(LHS: BoundsVal, RHS: ElemSizeV)
5008 : CGF.Builder.CreateUDiv(LHS: BoundsVal, RHS: ElemSizeV);
5009}
5010
5011/// EmitCountedByBoundsChecking - If the array being accessed has a "counted_by"
5012/// attribute, generate bounds checking code. The "count" field is at the top
5013/// level of the struct or in an anonymous struct, that's also at the top level.
5014/// Future expansions may allow the "count" to reside at any place in the
5015/// struct, but the value of "counted_by" will be a "simple" path to the count,
5016/// i.e. "a.b.count", so we shouldn't need the full force of EmitLValue or
5017/// similar to emit the correct GEP.
5018void CodeGenFunction::EmitCountedByBoundsChecking(
5019 const Expr *ArrayExpr, QualType ArrayType, Address ArrayInst,
5020 QualType IndexType, llvm::Value *IndexVal, bool Accessed,
5021 bool FlexibleArray) {
5022 const auto *ME = dyn_cast<MemberExpr>(Val: ArrayExpr->IgnoreImpCasts());
5023 if (!ME || !ME->getMemberDecl()->getType()->isCountAttributedType())
5024 return;
5025
5026 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
5027 getLangOpts().getStrictFlexArraysLevel();
5028 if (FlexibleArray &&
5029 !ME->isFlexibleArrayMemberLike(Context: getContext(), StrictFlexArraysLevel))
5030 return;
5031
5032 const FieldDecl *FD = cast<FieldDecl>(Val: ME->getMemberDecl());
5033 const FieldDecl *CountFD = FD->findCountedByField();
5034 if (!CountFD)
5035 return;
5036
5037 if (std::optional<int64_t> Diff =
5038 getOffsetDifferenceInBits(CGF&: *this, FD1: CountFD, FD2: FD)) {
5039 if (!ArrayInst.isValid()) {
5040 // An invalid Address indicates we're checking a pointer array access.
5041 // Emit the checked L-Value here.
5042 LValue LV = EmitCheckedLValue(E: ArrayExpr, TCK: TCK_MemberAccess);
5043 ArrayInst = LV.getAddress();
5044 }
5045
5046 // FIXME: The 'static_cast' is necessary, otherwise the result turns into a
5047 // uint64_t, which messes things up if we have a negative offset difference.
5048 Diff = *Diff / static_cast<int64_t>(CGM.getContext().getCharWidth());
5049
5050 // Create a GEP with the byte offset between the counted object and the
5051 // count and use that to load the count value.
5052 Address CountAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
5053 Addr: ArrayInst, Ty: Int8PtrTy, ElementTy: Int8Ty);
5054
5055 llvm::Type *BoundsType = ConvertType(T: CountFD->getType());
5056 llvm::Value *BoundsVal =
5057 Builder.CreateInBoundsGEP(Ty: Int8Ty, Ptr: CountAddr.emitRawPointer(CGF&: *this),
5058 IdxList: Builder.getInt32(C: *Diff), Name: ".counted_by.gep");
5059 BoundsVal = Builder.CreateAlignedLoad(Ty: BoundsType, Addr: BoundsVal, Align: getIntAlign(),
5060 Name: ".counted_by.load");
5061
5062 const auto *CountAttributedTy = FD->getType()->getAs<CountAttributedType>();
5063 assert(CountAttributedTy && "expected FD to have a CountAttributedType");
5064
5065 // For the '_or_null' variants a null pointer describes no accessible
5066 // memory, so treat the bound as 0 when the pointer is null; any access then
5067 // traps.
5068 if (CountAttributedTy->isOrNull()) {
5069 // Load the pointer from its address rather than re-emitting the
5070 // member expression, which would re-evaluate a side-effecting base.
5071 llvm::Value *Ptr = Builder.CreateLoad(Addr: ArrayInst);
5072 llvm::Value *IsNull = Builder.CreateIsNull(Arg: Ptr);
5073 BoundsVal = Builder.CreateSelect(
5074 C: IsNull, True: llvm::ConstantInt::get(Ty: BoundsType, V: 0), False: BoundsVal);
5075 }
5076
5077 // For '__sized_by' the loaded bound is a byte count. Convert it to an
5078 // element count by dividing by the element size so the check can compare
5079 // the (element) index directly. '__counted_by' already counts elements so
5080 // needs no special handling.
5081 if (CountAttributedTy->isCountInBytes())
5082 BoundsVal = convertSizedByBoundToElementCount(
5083 CGF&: *this, BoundsVal, PointeeTy: ArrayType->getPointeeType(),
5084 CountSigned: CountFD->getType()->isSignedIntegerOrEnumerationType());
5085
5086 // Now emit the bounds checking.
5087 EmitBoundsCheckImpl(ArrayExpr, ArrayBaseType: ArrayType, IndexVal, IndexType, BoundsVal,
5088 BoundsType: CountFD->getType(), Accessed);
5089 }
5090}
5091
5092LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
5093 bool Accessed) {
5094 // The index must always be an integer, which is not an aggregate. Emit it
5095 // in lexical order (this complexity is, sadly, required by C++17).
5096 llvm::Value *IdxPre =
5097 (E->getLHS() == E->getIdx()) ? EmitScalarExpr(E: E->getIdx()) : nullptr;
5098 bool SignedIndices = false;
5099 auto EmitIdxAfterBase = [&, IdxPre](bool Promote) -> llvm::Value * {
5100 auto *Idx = IdxPre;
5101 if (E->getLHS() != E->getIdx()) {
5102 assert(E->getRHS() == E->getIdx() && "index was neither LHS nor RHS");
5103 Idx = EmitScalarExpr(E: E->getIdx());
5104 }
5105
5106 QualType IdxTy = E->getIdx()->getType();
5107 bool IdxSigned = IdxTy->isSignedIntegerOrEnumerationType();
5108 SignedIndices |= IdxSigned;
5109
5110 if (SanOpts.has(K: SanitizerKind::ArrayBounds))
5111 EmitBoundsCheck(ArrayExpr: E, ArrayExprBase: E->getBase(), IndexVal: Idx, IndexType: IdxTy, Accessed);
5112
5113 // Extend or truncate the index type to 32 or 64-bits.
5114 if (Promote && Idx->getType() != IntPtrTy)
5115 Idx = Builder.CreateIntCast(V: Idx, DestTy: IntPtrTy, isSigned: IdxSigned, Name: "idxprom");
5116
5117 return Idx;
5118 };
5119 IdxPre = nullptr;
5120
5121 // If the base is a vector type, then we are forming a vector element lvalue
5122 // with this subscript.
5123 if (E->getBase()->getType()->isSubscriptableVectorType() &&
5124 !isa<ExtVectorElementExpr>(Val: E->getBase())) {
5125 // Emit the vector as an lvalue to get its address.
5126 LValue LHS = EmitLValue(E: E->getBase());
5127 auto *Idx = EmitIdxAfterBase(/*Promote*/false);
5128 assert(LHS.isSimple() && "Can only subscript lvalue vectors here!");
5129 return LValue::MakeVectorElt(vecAddress: LHS.getAddress(), Idx, type: E->getBase()->getType(),
5130 BaseInfo: LHS.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5131 }
5132
5133 // The HLSL runtime handles subscript expressions on global resource arrays
5134 // and objects with HLSL buffer layouts.
5135 if (getLangOpts().HLSL) {
5136 std::optional<LValue> LV;
5137 if (E->getType()->isHLSLResourceRecord() ||
5138 E->getType()->isHLSLResourceRecordArray()) {
5139 LV = CGM.getHLSLRuntime().emitResourceArraySubscriptExpr(E, CGF&: *this);
5140 } else if (E->getType().getAddressSpace() == LangAS::hlsl_constant) {
5141 LV = CGM.getHLSLRuntime().emitBufferArraySubscriptExpr(E, CGF&: *this,
5142 EmitIdxAfterBase);
5143 }
5144 if (LV.has_value())
5145 return *LV;
5146 }
5147
5148 // All the other cases basically behave like simple offsetting.
5149
5150 // Handle the extvector case we ignored above.
5151 if (isa<ExtVectorElementExpr>(Val: E->getBase())) {
5152 LValue LV = EmitLValue(E: E->getBase());
5153 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5154 Address Addr = EmitExtVectorElementLValue(LV);
5155
5156 QualType EltType = LV.getType()->castAs<VectorType>()->getElementType();
5157 Addr = emitArraySubscriptGEP(CGF&: *this, addr: Addr, indices: Idx, eltType: EltType, /*inbounds*/ true,
5158 signedIndices: SignedIndices, loc: E->getExprLoc());
5159 return MakeAddrLValue(Addr, T: EltType, BaseInfo: LV.getBaseInfo(),
5160 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: EltType));
5161 }
5162
5163 LValueBaseInfo EltBaseInfo;
5164 TBAAAccessInfo EltTBAAInfo;
5165 Address Addr = Address::invalid();
5166 if (const VariableArrayType *vla =
5167 getContext().getAsVariableArrayType(T: E->getType())) {
5168 // The base must be a pointer, which is not an aggregate. Emit
5169 // it. It needs to be emitted first in case it's what captures
5170 // the VLA bounds.
5171 Addr = EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
5172 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5173
5174 // The element count here is the total number of non-VLA elements.
5175 llvm::Value *numElements = getVLASize(vla).NumElts;
5176
5177 // Effectively, the multiply by the VLA size is part of the GEP.
5178 // GEP indexes are signed, and scaling an index isn't permitted to
5179 // signed-overflow, so we use the same semantics for our explicit
5180 // multiply. We suppress this if overflow is not undefined behavior.
5181 if (getLangOpts().PointerOverflowDefined) {
5182 Idx = Builder.CreateMul(LHS: Idx, RHS: numElements);
5183 } else {
5184 Idx = Builder.CreateNSWMul(LHS: Idx, RHS: numElements);
5185 }
5186
5187 Addr = emitArraySubscriptGEP(CGF&: *this, addr: Addr, indices: Idx, eltType: vla->getElementType(),
5188 inbounds: !getLangOpts().PointerOverflowDefined,
5189 signedIndices: SignedIndices, loc: E->getExprLoc());
5190
5191 } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){
5192 // Indexing over an interface, as in "NSString *P; P[4];"
5193
5194 // Emit the base pointer.
5195 Addr = EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
5196 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5197
5198 CharUnits InterfaceSize = getContext().getTypeSizeInChars(T: OIT);
5199 llvm::Value *InterfaceSizeVal =
5200 llvm::ConstantInt::get(Ty: Idx->getType(), V: InterfaceSize.getQuantity());
5201
5202 llvm::Value *ScaledIdx = Builder.CreateMul(LHS: Idx, RHS: InterfaceSizeVal);
5203
5204 // We don't necessarily build correct LLVM struct types for ObjC
5205 // interfaces, so we can't rely on GEP to do this scaling
5206 // correctly, so we need to cast to i8*. FIXME: is this actually
5207 // true? A lot of other things in the fragile ABI would break...
5208 llvm::Type *OrigBaseElemTy = Addr.getElementType();
5209
5210 // Do the GEP.
5211 CharUnits EltAlign =
5212 getArrayElementAlign(arrayAlign: Addr.getAlignment(), idx: Idx, eltSize: InterfaceSize);
5213 llvm::Value *EltPtr =
5214 emitArraySubscriptGEP(CGF&: *this, elemType: Int8Ty, ptr: Addr.emitRawPointer(CGF&: *this),
5215 indices: ScaledIdx, inbounds: false, signedIndices: SignedIndices, loc: E->getExprLoc());
5216 Addr = Address(EltPtr, OrigBaseElemTy, EltAlign);
5217 } else if (const Expr *Array = isSimpleArrayDecayOperand(E: E->getBase())) {
5218 // If this is A[i] where A is an array, the frontend will have decayed the
5219 // base to be a ArrayToPointerDecay implicit cast. While correct, it is
5220 // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
5221 // "gep x, i" here. Emit one "gep A, 0, i".
5222 assert(Array->getType()->isArrayType() &&
5223 "Array to pointer decay must have array source type!");
5224 LValue ArrayLV;
5225 // For simple multidimensional array indexing, set the 'accessed' flag for
5226 // better bounds-checking of the base expression.
5227 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Array))
5228 ArrayLV = EmitArraySubscriptExpr(E: ASE, /*Accessed*/ true);
5229 else
5230 ArrayLV = EmitLValue(E: Array);
5231 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5232
5233 if (SanOpts.has(K: SanitizerKind::ArrayBounds))
5234 EmitCountedByBoundsChecking(ArrayExpr: Array, ArrayType: Array->getType(), ArrayInst: ArrayLV.getAddress(),
5235 IndexType: E->getIdx()->getType(), IndexVal: Idx, Accessed,
5236 /*FlexibleArray=*/true);
5237
5238 // Propagate the alignment from the array itself to the result.
5239 QualType arrayType = Array->getType();
5240 Addr = emitArraySubscriptGEP(
5241 CGF&: *this, addr: ArrayLV.getAddress(), indices: {CGM.getSize(numChars: CharUnits::Zero()), Idx},
5242 eltType: E->getType(), inbounds: !getLangOpts().PointerOverflowDefined, signedIndices: SignedIndices,
5243 loc: E->getExprLoc(), arrayType: &arrayType, Base: E->getBase());
5244 EltBaseInfo = ArrayLV.getBaseInfo();
5245 if (!CGM.getCodeGenOpts().NewStructPathTBAA) {
5246 // Since CodeGenTBAA::getTypeInfoHelper only handles array types for
5247 // new struct path TBAA, we must a use a plain access.
5248 EltTBAAInfo = CGM.getTBAAInfoForSubobject(Base: ArrayLV, AccessType: E->getType());
5249 } else if (ArrayLV.getTBAAInfo().isMayAlias()) {
5250 EltTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
5251 } else if (ArrayLV.getTBAAInfo().isIncomplete()) {
5252 // The array element is complete, even if the array is not.
5253 EltTBAAInfo = CGM.getTBAAAccessInfo(AccessType: E->getType());
5254 } else {
5255 // The TBAA access info from the array (base) lvalue is ordinary. We will
5256 // adapt it to create access info for the element.
5257 EltTBAAInfo = ArrayLV.getTBAAInfo();
5258
5259 // We retain the TBAA struct path (BaseType and Offset members) from the
5260 // array. In the TBAA representation, we map any array access to the
5261 // element at index 0, as the index is generally a runtime value. This
5262 // element has the same offset in the base type as the array itself.
5263 // If the array lvalue had no base type, there is no point trying to
5264 // generate one, since an array itself is not a valid base type.
5265
5266 // We also retain the access type from the base lvalue, but the access
5267 // size must be updated to the size of an individual element.
5268 EltTBAAInfo.Size =
5269 getContext().getTypeSizeInChars(T: E->getType()).getQuantity();
5270 }
5271 } else {
5272 // The base must be a pointer; emit it with an estimate of its alignment.
5273 Address BaseAddr =
5274 EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
5275 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5276 QualType ptrType = E->getBase()->getType();
5277 Addr = emitArraySubscriptGEP(CGF&: *this, addr: BaseAddr, indices: Idx, eltType: E->getType(),
5278 inbounds: !getLangOpts().PointerOverflowDefined,
5279 signedIndices: SignedIndices, loc: E->getExprLoc(), arrayType: &ptrType,
5280 Base: E->getBase());
5281
5282 if (SanOpts.has(K: SanitizerKind::ArrayBounds)) {
5283 StructFieldAccess Visitor;
5284 const Expr *Base = Visitor.Visit(S: E->getBase());
5285
5286 if (const auto *CE = dyn_cast_if_present<CastExpr>(Val: Base);
5287 CE && CE->getCastKind() == CK_LValueToRValue)
5288 EmitCountedByBoundsChecking(ArrayExpr: CE, ArrayType: ptrType, ArrayInst: Address::invalid(),
5289 IndexType: E->getIdx()->getType(), IndexVal: Idx, Accessed,
5290 /*FlexibleArray=*/false);
5291 }
5292 }
5293
5294 LValue LV = MakeAddrLValue(Addr, T: E->getType(), BaseInfo: EltBaseInfo, TBAAInfo: EltTBAAInfo);
5295
5296 if (getLangOpts().ObjC &&
5297 getLangOpts().getGC() != LangOptions::NonGC) {
5298 LV.setNonGC(!E->isOBJCGCCandidate(Ctx&: getContext()));
5299 setObjCGCLValueClass(Ctx: getContext(), E, LV);
5300 }
5301 return LV;
5302}
5303
5304llvm::Value *CodeGenFunction::EmitMatrixIndexExpr(const Expr *E) {
5305 llvm::Value *Idx = EmitScalarExpr(E);
5306 if (Idx->getType() == IntPtrTy)
5307 return Idx;
5308 bool IsSigned = E->getType()->isSignedIntegerOrEnumerationType();
5309 return Builder.CreateIntCast(V: Idx, DestTy: IntPtrTy, isSigned: IsSigned);
5310}
5311
5312LValue CodeGenFunction::EmitMatrixSingleSubscriptExpr(
5313 const MatrixSingleSubscriptExpr *E) {
5314 LValue Base = EmitLValue(E: E->getBase());
5315 llvm::Value *RowIdx = EmitMatrixIndexExpr(E: E->getRowIdx());
5316
5317 RawAddress MatAddr = Base.getAddress();
5318 if (getLangOpts().HLSL &&
5319 E->getBase()->getType().getAddressSpace() == LangAS::hlsl_constant)
5320 MatAddr = CGM.getHLSLRuntime().createBufferMatrixTempAddress(LV: Base, CGF&: *this);
5321
5322 return LValue::MakeMatrixRow(Addr: MaybeConvertMatrixAddress(Addr: MatAddr, CGF&: *this),
5323 RowIdx, MatrixTy: E->getBase()->getType(),
5324 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5325}
5326
5327LValue CodeGenFunction::EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E) {
5328 assert(
5329 !E->isIncomplete() &&
5330 "incomplete matrix subscript expressions should be rejected during Sema");
5331 LValue Base = EmitLValue(E: E->getBase());
5332
5333 // Extend or truncate the index type to 32 or 64-bits if needed.
5334 llvm::Value *RowIdx = EmitMatrixIndexExpr(E: E->getRowIdx());
5335 llvm::Value *ColIdx = EmitMatrixIndexExpr(E: E->getColumnIdx());
5336 llvm::MatrixBuilder MB(Builder);
5337 const auto *MatrixTy = E->getBase()->getType()->castAs<ConstantMatrixType>();
5338 unsigned NumCols = MatrixTy->getNumColumns();
5339 unsigned NumRows = MatrixTy->getNumRows();
5340 bool IsMatrixRowMajor =
5341 isMatrixRowMajor(LangOpts: getLangOpts(), T: E->getBase()->getType());
5342 llvm::Value *FinalIdx =
5343 MB.CreateIndex(RowIdx, ColumnIdx: ColIdx, NumRows, NumCols, IsMatrixRowMajor);
5344
5345 return LValue::MakeMatrixElt(
5346 matAddress: MaybeConvertMatrixAddress(Addr: Base.getAddress(), CGF&: *this), Idx: FinalIdx,
5347 type: E->getBase()->getType(), BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5348}
5349
5350static Address emitOMPArraySectionBase(CodeGenFunction &CGF, const Expr *Base,
5351 LValueBaseInfo &BaseInfo,
5352 TBAAAccessInfo &TBAAInfo,
5353 QualType BaseTy, QualType ElTy,
5354 bool IsLowerBound) {
5355 LValue BaseLVal;
5356 if (auto *ASE = dyn_cast<ArraySectionExpr>(Val: Base->IgnoreParenImpCasts())) {
5357 BaseLVal = CGF.EmitArraySectionExpr(E: ASE, IsLowerBound);
5358 if (BaseTy->isArrayType()) {
5359 Address Addr = BaseLVal.getAddress();
5360 BaseInfo = BaseLVal.getBaseInfo();
5361
5362 // If the array type was an incomplete type, we need to make sure
5363 // the decay ends up being the right type.
5364 llvm::Type *NewTy = CGF.ConvertType(T: BaseTy);
5365 Addr = Addr.withElementType(ElemTy: NewTy);
5366
5367 // Note that VLA pointers are always decayed, so we don't need to do
5368 // anything here.
5369 if (!BaseTy->isVariableArrayType()) {
5370 assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
5371 "Expected pointer to array");
5372 Addr = CGF.Builder.CreateConstArrayGEP(Addr, Index: 0, Name: "arraydecay");
5373 }
5374
5375 return Addr.withElementType(ElemTy: CGF.ConvertTypeForMem(T: ElTy));
5376 }
5377 LValueBaseInfo TypeBaseInfo;
5378 TBAAAccessInfo TypeTBAAInfo;
5379 CharUnits Align =
5380 CGF.CGM.getNaturalTypeAlignment(T: ElTy, BaseInfo: &TypeBaseInfo, TBAAInfo: &TypeTBAAInfo);
5381 BaseInfo.mergeForCast(Info: TypeBaseInfo);
5382 TBAAInfo = CGF.CGM.mergeTBAAInfoForCast(SourceInfo: TBAAInfo, TargetInfo: TypeTBAAInfo);
5383 return Address(CGF.Builder.CreateLoad(Addr: BaseLVal.getAddress()),
5384 CGF.ConvertTypeForMem(T: ElTy), Align);
5385 }
5386 return CGF.EmitPointerWithAlignment(E: Base, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
5387}
5388
5389LValue CodeGenFunction::EmitArraySectionExpr(const ArraySectionExpr *E,
5390 bool IsLowerBound) {
5391
5392 assert(!E->isOpenACCArraySection() &&
5393 "OpenACC Array section codegen not implemented");
5394
5395 QualType BaseTy = ArraySectionExpr::getBaseOriginalType(Base: E->getBase());
5396 QualType ResultExprTy;
5397 if (auto *AT = getContext().getAsArrayType(T: BaseTy))
5398 ResultExprTy = AT->getElementType();
5399 else
5400 ResultExprTy = BaseTy->getPointeeType();
5401 llvm::Value *Idx = nullptr;
5402 if (IsLowerBound || E->getColonLocFirst().isInvalid()) {
5403 // Requesting lower bound or upper bound, but without provided length and
5404 // without ':' symbol for the default length -> length = 1.
5405 // Idx = LowerBound ?: 0;
5406 if (auto *LowerBound = E->getLowerBound()) {
5407 Idx = Builder.CreateIntCast(
5408 V: EmitScalarExpr(E: LowerBound), DestTy: IntPtrTy,
5409 isSigned: LowerBound->getType()->hasSignedIntegerRepresentation());
5410 } else
5411 Idx = llvm::ConstantInt::getNullValue(Ty: IntPtrTy);
5412 } else {
5413 // Try to emit length or lower bound as constant. If this is possible, 1
5414 // is subtracted from constant length or lower bound. Otherwise, emit LLVM
5415 // IR (LB + Len) - 1.
5416 auto &C = CGM.getContext();
5417 auto *Length = E->getLength();
5418 llvm::APSInt ConstLength;
5419 if (Length) {
5420 // Idx = LowerBound + Length - 1;
5421 if (std::optional<llvm::APSInt> CL = Length->getIntegerConstantExpr(Ctx: C)) {
5422 ConstLength = CL->zextOrTrunc(width: PointerWidthInBits);
5423 Length = nullptr;
5424 }
5425 auto *LowerBound = E->getLowerBound();
5426 llvm::APSInt ConstLowerBound(PointerWidthInBits, /*isUnsigned=*/false);
5427 if (LowerBound) {
5428 if (std::optional<llvm::APSInt> LB =
5429 LowerBound->getIntegerConstantExpr(Ctx: C)) {
5430 ConstLowerBound = LB->zextOrTrunc(width: PointerWidthInBits);
5431 LowerBound = nullptr;
5432 }
5433 }
5434 if (!Length)
5435 --ConstLength;
5436 else if (!LowerBound)
5437 --ConstLowerBound;
5438
5439 if (Length || LowerBound) {
5440 auto *LowerBoundVal =
5441 LowerBound
5442 ? Builder.CreateIntCast(
5443 V: EmitScalarExpr(E: LowerBound), DestTy: IntPtrTy,
5444 isSigned: LowerBound->getType()->hasSignedIntegerRepresentation())
5445 : llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLowerBound);
5446 auto *LengthVal =
5447 Length
5448 ? Builder.CreateIntCast(
5449 V: EmitScalarExpr(E: Length), DestTy: IntPtrTy,
5450 isSigned: Length->getType()->hasSignedIntegerRepresentation())
5451 : llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLength);
5452 Idx = Builder.CreateAdd(LHS: LowerBoundVal, RHS: LengthVal, Name: "lb_add_len",
5453 /*HasNUW=*/false,
5454 HasNSW: !getLangOpts().PointerOverflowDefined);
5455 if (Length && LowerBound) {
5456 Idx = Builder.CreateSub(
5457 LHS: Idx, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, /*V=*/1), Name: "idx_sub_1",
5458 /*HasNUW=*/false, HasNSW: !getLangOpts().PointerOverflowDefined);
5459 }
5460 } else
5461 Idx = llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLength + ConstLowerBound);
5462 } else {
5463 // Idx = ArraySize - 1;
5464 QualType ArrayTy = BaseTy->isPointerType()
5465 ? E->getBase()->IgnoreParenImpCasts()->getType()
5466 : BaseTy;
5467 if (auto *VAT = C.getAsVariableArrayType(T: ArrayTy)) {
5468 Length = VAT->getSizeExpr();
5469 if (std::optional<llvm::APSInt> L = Length->getIntegerConstantExpr(Ctx: C)) {
5470 ConstLength = *L;
5471 Length = nullptr;
5472 }
5473 } else {
5474 auto *CAT = C.getAsConstantArrayType(T: ArrayTy);
5475 assert(CAT && "unexpected type for array initializer");
5476 ConstLength = CAT->getSize();
5477 }
5478 if (Length) {
5479 auto *LengthVal = Builder.CreateIntCast(
5480 V: EmitScalarExpr(E: Length), DestTy: IntPtrTy,
5481 isSigned: Length->getType()->hasSignedIntegerRepresentation());
5482 Idx = Builder.CreateSub(
5483 LHS: LengthVal, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, /*V=*/1), Name: "len_sub_1",
5484 /*HasNUW=*/false, HasNSW: !getLangOpts().PointerOverflowDefined);
5485 } else {
5486 ConstLength = ConstLength.zextOrTrunc(width: PointerWidthInBits);
5487 --ConstLength;
5488 Idx = llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLength);
5489 }
5490 }
5491 }
5492 assert(Idx);
5493
5494 Address EltPtr = Address::invalid();
5495 LValueBaseInfo BaseInfo;
5496 TBAAAccessInfo TBAAInfo;
5497 if (auto *VLA = getContext().getAsVariableArrayType(T: ResultExprTy)) {
5498 // The base must be a pointer, which is not an aggregate. Emit
5499 // it. It needs to be emitted first in case it's what captures
5500 // the VLA bounds.
5501 Address Base =
5502 emitOMPArraySectionBase(CGF&: *this, Base: E->getBase(), BaseInfo, TBAAInfo,
5503 BaseTy, ElTy: VLA->getElementType(), IsLowerBound);
5504 // The element count here is the total number of non-VLA elements.
5505 llvm::Value *NumElements = getVLASize(vla: VLA).NumElts;
5506
5507 // Effectively, the multiply by the VLA size is part of the GEP.
5508 // GEP indexes are signed, and scaling an index isn't permitted to
5509 // signed-overflow, so we use the same semantics for our explicit
5510 // multiply. We suppress this if overflow is not undefined behavior.
5511 if (getLangOpts().PointerOverflowDefined)
5512 Idx = Builder.CreateMul(LHS: Idx, RHS: NumElements);
5513 else
5514 Idx = Builder.CreateNSWMul(LHS: Idx, RHS: NumElements);
5515 EltPtr = emitArraySubscriptGEP(CGF&: *this, addr: Base, indices: Idx, eltType: VLA->getElementType(),
5516 inbounds: !getLangOpts().PointerOverflowDefined,
5517 /*signedIndices=*/false, loc: E->getExprLoc());
5518 } else if (const Expr *Array = isSimpleArrayDecayOperand(E: E->getBase())) {
5519 // If this is A[i] where A is an array, the frontend will have decayed the
5520 // base to be a ArrayToPointerDecay implicit cast. While correct, it is
5521 // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
5522 // "gep x, i" here. Emit one "gep A, 0, i".
5523 assert(Array->getType()->isArrayType() &&
5524 "Array to pointer decay must have array source type!");
5525 LValue ArrayLV;
5526 // For simple multidimensional array indexing, set the 'accessed' flag for
5527 // better bounds-checking of the base expression.
5528 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Array))
5529 ArrayLV = EmitArraySubscriptExpr(E: ASE, /*Accessed*/ true);
5530 else
5531 ArrayLV = EmitLValue(E: Array);
5532
5533 // Propagate the alignment from the array itself to the result.
5534 EltPtr = emitArraySubscriptGEP(
5535 CGF&: *this, addr: ArrayLV.getAddress(), indices: {CGM.getSize(numChars: CharUnits::Zero()), Idx},
5536 eltType: ResultExprTy, inbounds: !getLangOpts().PointerOverflowDefined,
5537 /*signedIndices=*/false, loc: E->getExprLoc());
5538 BaseInfo = ArrayLV.getBaseInfo();
5539 TBAAInfo = CGM.getTBAAInfoForSubobject(Base: ArrayLV, AccessType: ResultExprTy);
5540 } else {
5541 Address Base =
5542 emitOMPArraySectionBase(CGF&: *this, Base: E->getBase(), BaseInfo, TBAAInfo, BaseTy,
5543 ElTy: ResultExprTy, IsLowerBound);
5544 EltPtr = emitArraySubscriptGEP(CGF&: *this, addr: Base, indices: Idx, eltType: ResultExprTy,
5545 inbounds: !getLangOpts().PointerOverflowDefined,
5546 /*signedIndices=*/false, loc: E->getExprLoc());
5547 }
5548
5549 return MakeAddrLValue(Addr: EltPtr, T: ResultExprTy, BaseInfo, TBAAInfo);
5550}
5551
5552LValue CodeGenFunction::
5553EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
5554 // Emit the base vector as an l-value.
5555 LValue Base;
5556
5557 // ExtVectorElementExpr's base can either be a vector or pointer to vector.
5558 if (E->isArrow()) {
5559 // If it is a pointer to a vector, emit the address and form an lvalue with
5560 // it.
5561 LValueBaseInfo BaseInfo;
5562 TBAAAccessInfo TBAAInfo;
5563 Address Ptr = EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
5564 const auto *PT = E->getBase()->getType()->castAs<PointerType>();
5565 Base = MakeAddrLValue(Addr: Ptr, T: PT->getPointeeType(), BaseInfo, TBAAInfo);
5566 Base.getQuals().removeObjCGCAttr();
5567 } else if (E->getBase()->isGLValue()) {
5568 // Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
5569 // emit the base as an lvalue.
5570 assert(E->getBase()->getType()->isVectorType());
5571 Base = EmitLValue(E: E->getBase());
5572 } else {
5573 // Otherwise, the base is a normal rvalue (as in (V+V).x), emit it as such.
5574 assert(E->getBase()->getType()->isVectorType() &&
5575 "Result must be a vector");
5576 llvm::Value *Vec = EmitScalarExpr(E: E->getBase());
5577
5578 // Store the vector to memory (because LValue wants an address).
5579 Address VecMem = CreateMemTemp(Ty: E->getBase()->getType());
5580 // need to zero extend an hlsl boolean vector to store it back to memory
5581 QualType Ty = E->getBase()->getType();
5582 llvm::Type *LTy = convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Vec->getType());
5583 if (LTy->getScalarSizeInBits() > Vec->getType()->getScalarSizeInBits())
5584 Vec = Builder.CreateZExt(V: Vec, DestTy: LTy);
5585 Builder.CreateStore(Val: Vec, Addr: VecMem);
5586 Base = MakeAddrLValue(Addr: VecMem, T: Ty, Source: AlignmentSource::Decl);
5587 }
5588
5589 QualType type =
5590 E->getType().withCVRQualifiers(CVR: Base.getQuals().getCVRQualifiers());
5591
5592 // Encode the element access list into a vector of unsigned indices.
5593 SmallVector<uint32_t, 4> Indices;
5594 E->getEncodedElementAccess(Elts&: Indices);
5595
5596 if (Base.isSimple()) {
5597 llvm::Constant *CV =
5598 llvm::ConstantDataVector::get(Context&: getLLVMContext(), Elts: Indices);
5599 return LValue::MakeExtVectorElt(Addr: Base.getAddress(), Elts: CV, type,
5600 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5601 }
5602
5603 if (Base.isMatrixRow()) {
5604 if (auto *RowIdx =
5605 llvm::dyn_cast<llvm::ConstantInt>(Val: Base.getMatrixRowIdx())) {
5606 llvm::SmallVector<llvm::Constant *> MatIndices;
5607 QualType MatTy = Base.getType();
5608 const ConstantMatrixType *MT = MatTy->castAs<ConstantMatrixType>();
5609 unsigned NumCols = Indices.size();
5610 unsigned NumRows = MT->getNumRows();
5611 unsigned Row = RowIdx->getZExtValue();
5612 QualType VecQT = E->getBase()->getType();
5613 if (NumCols != MT->getNumColumns()) {
5614 const auto *EVT = VecQT->getAs<ExtVectorType>();
5615 QualType ElemQT = EVT->getElementType();
5616 VecQT = getContext().getExtVectorType(VectorType: ElemQT, NumElts: NumCols);
5617 }
5618 for (unsigned C = 0; C < NumCols; ++C) {
5619 unsigned Col = Indices[C];
5620 unsigned Linear = Col * NumRows + Row;
5621 MatIndices.push_back(Elt: llvm::ConstantInt::get(Ty: Int32Ty, V: Linear));
5622 }
5623
5624 llvm::Constant *ConstIdxs = llvm::ConstantVector::get(V: MatIndices);
5625 return LValue::MakeExtVectorElt(Addr: Base.getMatrixAddress(), Elts: ConstIdxs, type: VecQT,
5626 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5627 }
5628 llvm::Constant *Cols =
5629 llvm::ConstantDataVector::get(Context&: getLLVMContext(), Elts: Indices);
5630 // Note: intentionally not using E.getType() so we can reuse isMatrixRow()
5631 // implementations in EmitLoadOfLValue & EmitStoreThroughLValue and don't
5632 // need the LValue to have its own number of rows and columns when the
5633 // type is a vector.
5634 return LValue::MakeMatrixRowSwizzle(
5635 MatAddr: Base.getMatrixAddress(), RowIdx: Base.getMatrixRowIdx(), Cols, MatrixTy: Base.getType(),
5636 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5637 }
5638
5639 assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
5640
5641 llvm::Constant *BaseElts = Base.getExtVectorElts();
5642 SmallVector<llvm::Constant *, 4> CElts;
5643
5644 for (unsigned Index : Indices)
5645 CElts.push_back(Elt: BaseElts->getAggregateElement(Elt: Index));
5646 llvm::Constant *CV = llvm::ConstantVector::get(V: CElts);
5647 return LValue::MakeExtVectorElt(Addr: Base.getExtVectorAddress(), Elts: CV, type,
5648 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5649}
5650
5651bool CodeGenFunction::isUnderlyingBasePointerConstantNull(const Expr *E) {
5652 const Expr *UnderlyingBaseExpr = E->IgnoreParens();
5653 while (auto *BaseMemberExpr = dyn_cast<MemberExpr>(Val: UnderlyingBaseExpr))
5654 UnderlyingBaseExpr = BaseMemberExpr->getBase()->IgnoreParens();
5655 return getContext().isSentinelNullExpr(E: UnderlyingBaseExpr);
5656}
5657
5658LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
5659 if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(CGF&: *this, ME: E)) {
5660 EmitIgnoredExpr(E: E->getBase());
5661 return EmitDeclRefLValue(E: DRE);
5662 }
5663
5664 if (getLangOpts().HLSL) {
5665 QualType QT = E->getType();
5666 if (QT.getAddressSpace() == LangAS::hlsl_constant)
5667 return CGM.getHLSLRuntime().emitBufferMemberExpr(CGF&: *this, E);
5668
5669 if (QT->isHLSLResourceRecord() || QT->isHLSLResourceRecordArray()) {
5670 std::optional<LValue> LV;
5671 LV = CGM.getHLSLRuntime().emitResourceMemberExpr(CGF&: *this, E);
5672 if (LV.has_value())
5673 return *LV;
5674 }
5675 }
5676
5677 Expr *BaseExpr = E->getBase();
5678 // Check whether the underlying base pointer is a constant null.
5679 // If so, we do not set inbounds flag for GEP to avoid breaking some
5680 // old-style offsetof idioms.
5681 bool IsInBounds = !getLangOpts().PointerOverflowDefined &&
5682 !isUnderlyingBasePointerConstantNull(E: BaseExpr);
5683 // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar.
5684 LValue BaseLV;
5685 if (E->isArrow()) {
5686 LValueBaseInfo BaseInfo;
5687 TBAAAccessInfo TBAAInfo;
5688 Address Addr = EmitPointerWithAlignment(E: BaseExpr, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
5689 QualType PtrTy = BaseExpr->getType()->getPointeeType();
5690 SanitizerSet SkippedChecks;
5691 bool IsBaseCXXThis = IsWrappedCXXThis(Obj: BaseExpr);
5692 if (IsBaseCXXThis)
5693 SkippedChecks.set(K: SanitizerKind::Alignment, Value: true);
5694 if (IsBaseCXXThis || isa<DeclRefExpr>(Val: BaseExpr))
5695 SkippedChecks.set(K: SanitizerKind::Null, Value: true);
5696 EmitTypeCheck(TCK: TCK_MemberAccess, Loc: E->getExprLoc(), Addr, Type: PtrTy,
5697 /*Alignment=*/CharUnits::Zero(), SkippedChecks);
5698 BaseLV = MakeAddrLValue(Addr, T: PtrTy, BaseInfo, TBAAInfo);
5699 } else
5700 BaseLV = EmitCheckedLValue(E: BaseExpr, TCK: TCK_MemberAccess);
5701
5702 NamedDecl *ND = E->getMemberDecl();
5703 if (auto *Field = dyn_cast<FieldDecl>(Val: ND)) {
5704 LValue LV = EmitLValueForField(Base: BaseLV, Field, IsInBounds);
5705 setObjCGCLValueClass(Ctx: getContext(), E, LV);
5706 if (getLangOpts().OpenMP) {
5707 // If the member was explicitly marked as nontemporal, mark it as
5708 // nontemporal. If the base lvalue is marked as nontemporal, mark access
5709 // to children as nontemporal too.
5710 if ((IsWrappedCXXThis(Obj: BaseExpr) &&
5711 CGM.getOpenMPRuntime().isNontemporalDecl(VD: Field)) ||
5712 BaseLV.isNontemporal())
5713 LV.setNontemporal(/*Value=*/true);
5714 }
5715 return LV;
5716 }
5717
5718 if (const auto *FD = dyn_cast<FunctionDecl>(Val: ND))
5719 return EmitFunctionDeclLValue(CGF&: *this, E, GD: FD);
5720
5721 llvm_unreachable("Unhandled member declaration!");
5722}
5723
5724/// Given that we are currently emitting a lambda, emit an l-value for
5725/// one of its members.
5726///
5727LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field,
5728 llvm::Value *ThisValue) {
5729 bool HasExplicitObjectParameter = false;
5730 const auto *MD = dyn_cast_if_present<CXXMethodDecl>(Val: CurCodeDecl);
5731 if (MD) {
5732 HasExplicitObjectParameter = MD->isExplicitObjectMemberFunction();
5733 assert(MD->getParent()->isLambda());
5734 assert(MD->getParent() == Field->getParent());
5735 }
5736 LValue LambdaLV;
5737 if (HasExplicitObjectParameter) {
5738 const VarDecl *D = cast<CXXMethodDecl>(Val: CurCodeDecl)->getParamDecl(i: 0);
5739 auto It = LocalDeclMap.find(Val: D);
5740 assert(It != LocalDeclMap.end() && "explicit parameter not loaded?");
5741 Address AddrOfExplicitObject = It->getSecond();
5742 if (D->getType()->isReferenceType())
5743 LambdaLV = EmitLoadOfReferenceLValue(RefAddr: AddrOfExplicitObject, RefTy: D->getType(),
5744 Source: AlignmentSource::Decl);
5745 else
5746 LambdaLV = MakeAddrLValue(Addr: AddrOfExplicitObject,
5747 T: D->getType().getNonReferenceType());
5748
5749 // Make sure we have an lvalue to the lambda itself and not a derived class.
5750 auto *ThisTy = D->getType().getNonReferenceType()->getAsCXXRecordDecl();
5751 auto *LambdaTy = cast<CXXRecordDecl>(Val: Field->getParent());
5752 if (ThisTy != LambdaTy) {
5753 const CXXCastPath &BasePathArray = getContext().LambdaCastPaths.at(Val: MD);
5754 Address Base = GetAddressOfBaseClass(
5755 Value: LambdaLV.getAddress(), Derived: ThisTy, PathBegin: BasePathArray.begin(),
5756 PathEnd: BasePathArray.end(), /*NullCheckValue=*/false, Loc: SourceLocation());
5757 CanQualType T = getContext().getCanonicalTagType(TD: LambdaTy);
5758 LambdaLV = MakeAddrLValue(Addr: Base, T);
5759 }
5760 } else {
5761 CanQualType LambdaTagType =
5762 getContext().getCanonicalTagType(TD: Field->getParent());
5763 LambdaLV = MakeNaturalAlignAddrLValue(V: ThisValue, T: LambdaTagType);
5764 }
5765 return EmitLValueForField(Base: LambdaLV, Field);
5766}
5767
5768LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field) {
5769 return EmitLValueForLambdaField(Field, ThisValue: CXXABIThisValue);
5770}
5771
5772/// Get the field index in the debug info. The debug info structure/union
5773/// will ignore the unnamed bitfields.
5774unsigned CodeGenFunction::getDebugInfoFIndex(const RecordDecl *Rec,
5775 unsigned FieldIndex) {
5776 unsigned I = 0, Skipped = 0;
5777
5778 for (auto *F : Rec->getDefinition()->fields()) {
5779 if (I == FieldIndex)
5780 break;
5781 if (F->isUnnamedBitField())
5782 Skipped++;
5783 I++;
5784 }
5785
5786 return FieldIndex - Skipped;
5787}
5788
5789/// Get the address of a zero-sized field within a record. The resulting
5790/// address doesn't necessarily have the right type.
5791static Address emitAddrOfZeroSizeField(CodeGenFunction &CGF, Address Base,
5792 const FieldDecl *Field,
5793 bool IsInBounds) {
5794 CharUnits Offset = CGF.getContext().toCharUnitsFromBits(
5795 BitSize: CGF.getContext().getFieldOffset(FD: Field));
5796 if (Offset.isZero())
5797 return Base;
5798 Base = Base.withElementType(ElemTy: CGF.Int8Ty);
5799 if (!IsInBounds)
5800 return CGF.Builder.CreateConstByteGEP(Addr: Base, Offset);
5801 return CGF.Builder.CreateConstInBoundsByteGEP(Addr: Base, Offset);
5802}
5803
5804/// Drill down to the storage of a field without walking into reference types,
5805/// and without respect for pointer field protection.
5806///
5807/// The resulting address doesn't necessarily have the right type.
5808static Address emitRawAddrOfFieldStorage(CodeGenFunction &CGF, Address base,
5809 const FieldDecl *field,
5810 bool IsInBounds) {
5811 if (isEmptyFieldForLayout(Context: CGF.getContext(), FD: field))
5812 return emitAddrOfZeroSizeField(CGF, Base: base, Field: field, IsInBounds);
5813
5814 const RecordDecl *rec = field->getParent();
5815
5816 unsigned idx =
5817 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(FD: field);
5818 llvm::Type *StructType =
5819 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMType();
5820
5821 if (CGF.getLangOpts().EmitLogicalPointer)
5822 return RawAddress(
5823 CGF.Builder.CreateStructuredGEP(BaseType: StructType, PtrBase: base.emitRawPointer(CGF),
5824 Indices: {CGF.Builder.getSize(N: idx)}),
5825 base.getElementType(), base.getAlignment());
5826
5827 if (!IsInBounds)
5828 return CGF.Builder.CreateConstGEP2_32(Addr: base, Idx0: 0, Idx1: idx, Name: field->getName());
5829
5830 return CGF.Builder.CreateStructGEP(Addr: base, Index: idx, Name: field->getName());
5831}
5832
5833/// Drill down to the storage of a field without walking into reference types,
5834/// wrapping the address in an llvm.protected.field.ptr intrinsic for the
5835/// pointer field protection feature if necessary.
5836///
5837/// The resulting address doesn't necessarily have the right type.
5838static Address emitAddrOfFieldStorage(CodeGenFunction &CGF, Address base,
5839 const FieldDecl *field, bool IsInBounds) {
5840 Address Addr = emitRawAddrOfFieldStorage(CGF, base, field, IsInBounds);
5841
5842 if (!CGF.getContext().isPFPField(Field: field))
5843 return Addr;
5844
5845 return CGF.EmitAddressOfPFPField(RecordPtr: base, FieldPtr: Addr, Field: field);
5846}
5847
5848static Address emitPreserveStructAccess(CodeGenFunction &CGF, LValue base,
5849 Address addr, const FieldDecl *field) {
5850 const RecordDecl *rec = field->getParent();
5851 llvm::DIType *DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(
5852 Ty: base.getType(), Loc: rec->getLocation());
5853
5854 unsigned idx =
5855 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(FD: field);
5856
5857 return CGF.Builder.CreatePreserveStructAccessIndex(
5858 Addr: addr, Index: idx, FieldIndex: CGF.getDebugInfoFIndex(Rec: rec, FieldIndex: field->getFieldIndex()), DbgInfo);
5859}
5860
5861static bool hasAnyVptr(const QualType Type, const ASTContext &Context) {
5862 const auto *RD = Type.getTypePtr()->getAsCXXRecordDecl();
5863 if (!RD)
5864 return false;
5865
5866 if (RD->isDynamicClass())
5867 return true;
5868
5869 for (const auto &Base : RD->bases())
5870 if (hasAnyVptr(Type: Base.getType(), Context))
5871 return true;
5872
5873 for (const FieldDecl *Field : RD->fields())
5874 if (hasAnyVptr(Type: Field->getType(), Context))
5875 return true;
5876
5877 return false;
5878}
5879
5880LValue CodeGenFunction::EmitLValueForField(LValue base, const FieldDecl *field,
5881 bool IsInBounds) {
5882 LValueBaseInfo BaseInfo = base.getBaseInfo();
5883
5884 if (field->isBitField()) {
5885 const CGRecordLayout &RL =
5886 CGM.getTypes().getCGRecordLayout(field->getParent());
5887 const CGBitFieldInfo &Info = RL.getBitFieldInfo(FD: field);
5888 const bool UseVolatile = CodeGenUtils::isAAPCS(TargetInfo: CGM.getTarget()) &&
5889 CGM.getCodeGenOpts().AAPCSBitfieldWidth &&
5890 Info.VolatileStorageSize != 0 &&
5891 field->getType()
5892 .withCVRQualifiers(CVR: base.getVRQualifiers())
5893 .isVolatileQualified();
5894 Address Addr = base.getAddress();
5895 unsigned Idx = RL.getLLVMFieldNo(FD: field);
5896 const RecordDecl *rec = field->getParent();
5897 if (hasBPFPreserveStaticOffset(D: rec))
5898 Addr = wrapWithBPFPreserveStaticOffset(CGF&: *this, Addr);
5899 if (!UseVolatile) {
5900 if (!IsInPreservedAIRegion &&
5901 (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
5902 if (Idx != 0) {
5903 // For structs, we GEP to the field that the record layout suggests.
5904 if (!IsInBounds)
5905 Addr = Builder.CreateConstGEP2_32(Addr, Idx0: 0, Idx1: Idx, Name: field->getName());
5906 else
5907 Addr = Builder.CreateStructGEP(Addr, Index: Idx, Name: field->getName());
5908 }
5909 } else {
5910 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType(
5911 Ty: getContext().getCanonicalTagType(TD: rec), L: rec->getLocation());
5912 Addr = Builder.CreatePreserveStructAccessIndex(
5913 Addr, Index: Idx, FieldIndex: getDebugInfoFIndex(Rec: rec, FieldIndex: field->getFieldIndex()),
5914 DbgInfo);
5915 }
5916 }
5917 const unsigned SS =
5918 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
5919 // Get the access type.
5920 llvm::Type *FieldIntTy = llvm::Type::getIntNTy(C&: getLLVMContext(), N: SS);
5921 Addr = Addr.withElementType(ElemTy: FieldIntTy);
5922 if (UseVolatile) {
5923 const unsigned VolatileOffset = Info.VolatileStorageOffset.getQuantity();
5924 if (VolatileOffset)
5925 Addr = Builder.CreateConstInBoundsGEP(Addr, Index: VolatileOffset);
5926 }
5927
5928 QualType fieldType =
5929 field->getType().withCVRQualifiers(CVR: base.getVRQualifiers());
5930 // TODO: Support TBAA for bit fields.
5931 LValueBaseInfo FieldBaseInfo(BaseInfo.getAlignmentSource());
5932 return LValue::MakeBitfield(Addr, Info, type: fieldType, BaseInfo: FieldBaseInfo,
5933 TBAAInfo: TBAAAccessInfo());
5934 }
5935
5936 // Fields of may-alias structures are may-alias themselves.
5937 // FIXME: this should get propagated down through anonymous structs
5938 // and unions.
5939 QualType FieldType = field->getType();
5940 const RecordDecl *rec = field->getParent();
5941 AlignmentSource BaseAlignSource = BaseInfo.getAlignmentSource();
5942 LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(Source: BaseAlignSource));
5943 TBAAAccessInfo FieldTBAAInfo;
5944 if (base.getTBAAInfo().isMayAlias() ||
5945 rec->hasAttr<MayAliasAttr>() || FieldType->isVectorType()) {
5946 FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
5947 } else if (rec->isUnion()) {
5948 // TODO: Support TBAA for unions.
5949 FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
5950 } else {
5951 // If no base type been assigned for the base access, then try to generate
5952 // one for this base lvalue.
5953 FieldTBAAInfo = base.getTBAAInfo();
5954 if (!FieldTBAAInfo.BaseType) {
5955 FieldTBAAInfo.BaseType = CGM.getTBAABaseTypeInfo(QTy: base.getType());
5956 assert(!FieldTBAAInfo.Offset &&
5957 "Nonzero offset for an access with no base type!");
5958 }
5959
5960 // Adjust offset to be relative to the base type.
5961 const ASTRecordLayout &Layout =
5962 getContext().getASTRecordLayout(D: field->getParent());
5963 unsigned CharWidth = getContext().getCharWidth();
5964 if (FieldTBAAInfo.BaseType)
5965 FieldTBAAInfo.Offset +=
5966 Layout.getFieldOffset(FieldNo: field->getFieldIndex()) / CharWidth;
5967
5968 // Update the final access type and size.
5969 FieldTBAAInfo.AccessType = CGM.getTBAATypeInfo(QTy: FieldType);
5970 FieldTBAAInfo.Size =
5971 getContext().getTypeSizeInChars(T: FieldType).getQuantity();
5972 }
5973
5974 Address addr = base.getAddress();
5975 if (hasBPFPreserveStaticOffset(D: rec))
5976 addr = wrapWithBPFPreserveStaticOffset(CGF&: *this, Addr&: addr);
5977 if (auto *ClassDef = dyn_cast<CXXRecordDecl>(Val: rec)) {
5978 if (CGM.getCodeGenOpts().StrictVTablePointers &&
5979 ClassDef->isDynamicClass()) {
5980 // Getting to any field of dynamic object requires stripping dynamic
5981 // information provided by invariant.group. This is because accessing
5982 // fields may leak the real address of dynamic object, which could result
5983 // in miscompilation when leaked pointer would be compared.
5984 auto *stripped =
5985 Builder.CreateStripInvariantGroup(Ptr: addr.emitRawPointer(CGF&: *this));
5986 addr = Address(stripped, addr.getElementType(), addr.getAlignment());
5987 }
5988 }
5989
5990 unsigned RecordCVR = base.getVRQualifiers();
5991 if (rec->isUnion()) {
5992 // For unions, there is no pointer adjustment.
5993 if (CGM.getCodeGenOpts().StrictVTablePointers &&
5994 hasAnyVptr(Type: FieldType, Context: getContext()))
5995 // Because unions can easily skip invariant.barriers, we need to add
5996 // a barrier every time CXXRecord field with vptr is referenced.
5997 addr = Builder.CreateLaunderInvariantGroup(Addr: addr);
5998
5999 if (IsInPreservedAIRegion ||
6000 (getDebugInfo() && rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
6001 // Remember the original union field index
6002 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(Ty: base.getType(),
6003 Loc: rec->getLocation());
6004 addr =
6005 Address(Builder.CreatePreserveUnionAccessIndex(
6006 Base: addr.emitRawPointer(CGF&: *this),
6007 FieldIndex: getDebugInfoFIndex(Rec: rec, FieldIndex: field->getFieldIndex()), DbgInfo),
6008 addr.getElementType(), addr.getAlignment());
6009 }
6010
6011 if (FieldType->isReferenceType())
6012 addr = addr.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: FieldType));
6013 } else {
6014 if (!IsInPreservedAIRegion &&
6015 (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>()))
6016 // For structs, we GEP to the field that the record layout suggests.
6017 addr = emitAddrOfFieldStorage(CGF&: *this, base: addr, field, IsInBounds);
6018 else
6019 // Remember the original struct field index
6020 addr = emitPreserveStructAccess(CGF&: *this, base, addr, field);
6021 }
6022
6023 // If this is a reference field, load the reference right now.
6024 if (FieldType->isReferenceType()) {
6025 LValue RefLVal =
6026 MakeAddrLValue(Addr: addr, T: FieldType, BaseInfo: FieldBaseInfo, TBAAInfo: FieldTBAAInfo);
6027 if (RecordCVR & Qualifiers::Volatile)
6028 RefLVal.getQuals().addVolatile();
6029 addr = EmitLoadOfReference(RefLVal, PointeeBaseInfo: &FieldBaseInfo, PointeeTBAAInfo: &FieldTBAAInfo);
6030
6031 // Qualifiers on the struct don't apply to the referencee.
6032 RecordCVR = 0;
6033 FieldType = FieldType->getPointeeType();
6034 }
6035
6036 // Make sure that the address is pointing to the right type. This is critical
6037 // for both unions and structs.
6038 addr = addr.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: FieldType));
6039
6040 if (field->hasAttr<AnnotateAttr>())
6041 addr = EmitFieldAnnotations(D: field, V: addr);
6042
6043 LValue LV = MakeAddrLValue(Addr: addr, T: FieldType, BaseInfo: FieldBaseInfo, TBAAInfo: FieldTBAAInfo);
6044 LV.getQuals().addCVRQualifiers(mask: RecordCVR);
6045
6046 // __weak attribute on a field is ignored.
6047 if (LV.getQuals().getObjCGCAttr() == Qualifiers::Weak)
6048 LV.getQuals().removeObjCGCAttr();
6049
6050 return LV;
6051}
6052
6053LValue
6054CodeGenFunction::EmitLValueForFieldInitialization(LValue Base,
6055 const FieldDecl *Field) {
6056 QualType FieldType = Field->getType();
6057
6058 if (!FieldType->isReferenceType())
6059 return EmitLValueForField(base: Base, field: Field);
6060
6061 Address V = emitAddrOfFieldStorage(
6062 CGF&: *this, base: Base.getAddress(), field: Field,
6063 /*IsInBounds=*/!getLangOpts().PointerOverflowDefined);
6064
6065 // Make sure that the address is pointing to the right type.
6066 llvm::Type *llvmType = ConvertTypeForMem(T: FieldType);
6067 V = V.withElementType(ElemTy: llvmType);
6068
6069 // TODO: Generate TBAA information that describes this access as a structure
6070 // member access and not just an access to an object of the field's type. This
6071 // should be similar to what we do in EmitLValueForField().
6072 LValueBaseInfo BaseInfo = Base.getBaseInfo();
6073 AlignmentSource FieldAlignSource = BaseInfo.getAlignmentSource();
6074 LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(Source: FieldAlignSource));
6075 return MakeAddrLValue(Addr: V, T: FieldType, BaseInfo: FieldBaseInfo,
6076 TBAAInfo: CGM.getTBAAInfoForSubobject(Base, AccessType: FieldType));
6077}
6078
6079LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr *E){
6080 if (E->isFileScope()) {
6081 ConstantAddress GlobalPtr = CGM.GetAddrOfConstantCompoundLiteral(E);
6082 return MakeAddrLValue(Addr: GlobalPtr, T: E->getType(), Source: AlignmentSource::Decl);
6083 }
6084 if (E->getType()->isVariablyModifiedType())
6085 // make sure to emit the VLA size.
6086 EmitVariablyModifiedType(Ty: E->getType());
6087
6088 Address DeclPtr = CreateMemTempWithoutCast(Ty: E->getType(), Name: ".compoundliteral");
6089 const Expr *InitExpr = E->getInitializer();
6090 LValue Result = MakeAddrLValue(Addr: DeclPtr, T: E->getType(), Source: AlignmentSource::Decl);
6091
6092 if (!getLangOpts().CPlusPlus) {
6093 if (HaveInsertPoint() && !hasLabelBeenSeenInCurrentScope() &&
6094 EmitLifetimeStart(Addr: DeclPtr.getBasePointer()))
6095 pushCleanupAfterFullExpr<CallLifetimeEnd>(Kind: NormalEHLifetimeMarker,
6096 A: DeclPtr);
6097 }
6098
6099 EmitAnyExprToMem(E: InitExpr, Location: DeclPtr, Quals: E->getType().getQualifiers(),
6100 /*Init*/ IsInit: true);
6101
6102 // Block-scope compound literals are destroyed at the end of the enclosing
6103 // scope in C.
6104 if (!getLangOpts().CPlusPlus)
6105 if (QualType::DestructionKind DtorKind = E->getType().isDestructedType())
6106 pushLifetimeExtendedDestroy(kind: getCleanupKind(kind: DtorKind), addr: DeclPtr,
6107 type: E->getType(), destroyer: getDestroyer(destructionKind: DtorKind),
6108 useEHCleanupForArray: DtorKind & EHCleanup);
6109
6110 return Result;
6111}
6112
6113LValue CodeGenFunction::EmitInitListLValue(const InitListExpr *E) {
6114 if (!E->isGLValue())
6115 // Initializing an aggregate temporary in C++11: T{...}.
6116 return EmitAggExprToLValue(E);
6117
6118 // An lvalue initializer list must be initializing a reference.
6119 assert(E->isTransparent() && "non-transparent glvalue init list");
6120 return EmitLValue(E: E->getInit(Init: 0));
6121}
6122
6123/// Emit the operand of a glvalue conditional operator. This is either a glvalue
6124/// or a (possibly-parenthesized) throw-expression. If this is a throw, no
6125/// LValue is returned and the current block has been terminated.
6126static std::optional<LValue> EmitLValueOrThrowExpression(CodeGenFunction &CGF,
6127 const Expr *Operand) {
6128 if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Val: Operand->IgnoreParens())) {
6129 CGF.EmitCXXThrowExpr(E: ThrowExpr, /*KeepInsertionPoint*/false);
6130 return std::nullopt;
6131 }
6132
6133 return CGF.EmitLValue(E: Operand);
6134}
6135
6136namespace {
6137// Handle the case where the condition is a constant evaluatable simple integer,
6138// which means we don't have to separately handle the true/false blocks.
6139std::optional<LValue> HandleConditionalOperatorLValueSimpleCase(
6140 CodeGenFunction &CGF, const AbstractConditionalOperator *E) {
6141 const Expr *condExpr = E->getCond();
6142 bool CondExprBool;
6143 if (CGF.ConstantFoldsToSimpleInteger(Cond: condExpr, Result&: CondExprBool)) {
6144 const Expr *Live = E->getTrueExpr(), *Dead = E->getFalseExpr();
6145 if (!CondExprBool)
6146 std::swap(a&: Live, b&: Dead);
6147
6148 if (!CGF.ContainsLabel(S: Dead)) {
6149 // If the true case is live, we need to track its region.
6150 CGF.incrementProfileCounter(ExecSkip: CondExprBool ? CGF.UseExecPath
6151 : CGF.UseSkipPath,
6152 S: E, /*UseBoth=*/true);
6153 CGF.markStmtMaybeUsed(S: Dead);
6154 // If a throw expression we emit it and return an undefined lvalue
6155 // because it can't be used.
6156 if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Val: Live->IgnoreParens())) {
6157 CGF.EmitCXXThrowExpr(E: ThrowExpr);
6158 llvm::Type *ElemTy = CGF.ConvertType(T: Dead->getType());
6159 llvm::Type *Ty = CGF.DefaultPtrTy;
6160 return CGF.MakeAddrLValue(
6161 Addr: Address(llvm::UndefValue::get(T: Ty), ElemTy, CharUnits::One()),
6162 T: Dead->getType());
6163 }
6164 return CGF.EmitLValue(E: Live);
6165 }
6166 }
6167 return std::nullopt;
6168}
6169struct ConditionalInfo {
6170 llvm::BasicBlock *lhsBlock, *rhsBlock;
6171 std::optional<LValue> LHS, RHS;
6172};
6173
6174// Create and generate the 3 blocks for a conditional operator.
6175// Leaves the 'current block' in the continuation basic block.
6176template<typename FuncTy>
6177ConditionalInfo EmitConditionalBlocks(CodeGenFunction &CGF,
6178 const AbstractConditionalOperator *E,
6179 const FuncTy &BranchGenFunc) {
6180 ConditionalInfo Info{.lhsBlock: CGF.createBasicBlock(name: "cond.true"),
6181 .rhsBlock: CGF.createBasicBlock(name: "cond.false"), .LHS: std::nullopt,
6182 .RHS: std::nullopt};
6183 llvm::BasicBlock *endBlock = CGF.createBasicBlock(name: "cond.end");
6184
6185 CodeGenFunction::ConditionalEvaluation eval(CGF);
6186 CGF.EmitBranchOnBoolExpr(Cond: E->getCond(), TrueBlock: Info.lhsBlock, FalseBlock: Info.rhsBlock,
6187 TrueCount: CGF.getProfileCount(S: E));
6188
6189 // Any temporaries created here are conditional.
6190 CGF.EmitBlock(BB: Info.lhsBlock);
6191 CGF.incrementProfileCounter(ExecSkip: CGF.UseExecPath, S: E);
6192 eval.begin(CGF);
6193 Info.LHS = BranchGenFunc(CGF, E->getTrueExpr());
6194 eval.end(CGF);
6195 Info.lhsBlock = CGF.Builder.GetInsertBlock();
6196
6197 if (Info.LHS)
6198 CGF.Builder.CreateBr(Dest: endBlock);
6199
6200 // Any temporaries created here are conditional.
6201 CGF.EmitBlock(BB: Info.rhsBlock);
6202 CGF.incrementProfileCounter(ExecSkip: CGF.UseSkipPath, S: E);
6203 eval.begin(CGF);
6204 Info.RHS = BranchGenFunc(CGF, E->getFalseExpr());
6205 eval.end(CGF);
6206 Info.rhsBlock = CGF.Builder.GetInsertBlock();
6207 CGF.EmitBlock(BB: endBlock);
6208
6209 return Info;
6210}
6211} // namespace
6212
6213void CodeGenFunction::EmitIgnoredConditionalOperator(
6214 const AbstractConditionalOperator *E) {
6215 if (!E->isGLValue()) {
6216 // ?: here should be an aggregate.
6217 assert(hasAggregateEvaluationKind(E->getType()) &&
6218 "Unexpected conditional operator!");
6219 return (void)EmitAggExprToLValue(E);
6220 }
6221
6222 OpaqueValueMapping binding(*this, E);
6223 if (HandleConditionalOperatorLValueSimpleCase(CGF&: *this, E))
6224 return;
6225
6226 EmitConditionalBlocks(CGF&: *this, E, BranchGenFunc: [](CodeGenFunction &CGF, const Expr *E) {
6227 CGF.EmitIgnoredExpr(E);
6228 return LValue{};
6229 });
6230}
6231LValue CodeGenFunction::EmitConditionalOperatorLValue(
6232 const AbstractConditionalOperator *expr) {
6233 if (!expr->isGLValue()) {
6234 // ?: here should be an aggregate.
6235 assert(hasAggregateEvaluationKind(expr->getType()) &&
6236 "Unexpected conditional operator!");
6237 return EmitAggExprToLValue(E: expr);
6238 }
6239
6240 OpaqueValueMapping binding(*this, expr);
6241 if (std::optional<LValue> Res =
6242 HandleConditionalOperatorLValueSimpleCase(CGF&: *this, E: expr))
6243 return *Res;
6244
6245 ConditionalInfo Info = EmitConditionalBlocks(
6246 CGF&: *this, E: expr, BranchGenFunc: [](CodeGenFunction &CGF, const Expr *E) {
6247 return EmitLValueOrThrowExpression(CGF, Operand: E);
6248 });
6249
6250 if ((Info.LHS && !Info.LHS->isSimple()) ||
6251 (Info.RHS && !Info.RHS->isSimple()))
6252 return EmitUnsupportedLValue(E: expr, Name: "conditional operator");
6253
6254 if (Info.LHS && Info.RHS) {
6255 Address lhsAddr = Info.LHS->getAddress();
6256 Address rhsAddr = Info.RHS->getAddress();
6257 Address result = mergeAddressesInConditionalExpr(
6258 LHS: lhsAddr, RHS: rhsAddr, LHSBlock: Info.lhsBlock, RHSBlock: Info.rhsBlock,
6259 MergeBlock: Builder.GetInsertBlock(), MergedType: expr->getType());
6260 AlignmentSource alignSource =
6261 std::max(a: Info.LHS->getBaseInfo().getAlignmentSource(),
6262 b: Info.RHS->getBaseInfo().getAlignmentSource());
6263 TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForConditionalOperator(
6264 InfoA: Info.LHS->getTBAAInfo(), InfoB: Info.RHS->getTBAAInfo());
6265 return MakeAddrLValue(Addr: result, T: expr->getType(), BaseInfo: LValueBaseInfo(alignSource),
6266 TBAAInfo);
6267 } else {
6268 assert((Info.LHS || Info.RHS) &&
6269 "both operands of glvalue conditional are throw-expressions?");
6270 return Info.LHS ? *Info.LHS : *Info.RHS;
6271 }
6272}
6273
6274/// EmitCastLValue - Casts are never lvalues unless that cast is to a reference
6275/// type. If the cast is to a reference, we can have the usual lvalue result,
6276/// otherwise if a cast is needed by the code generator in an lvalue context,
6277/// then it must mean that we need the address of an aggregate in order to
6278/// access one of its members. This can happen for all the reasons that casts
6279/// are permitted with aggregate result, including noop aggregate casts, and
6280/// cast from scalar to union.
6281LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
6282 llvm::scope_exit RestoreCurCast([this, Prev = CurCast] { CurCast = Prev; });
6283 CurCast = E;
6284 switch (E->getCastKind()) {
6285 case CK_ToVoid:
6286 case CK_BitCast:
6287 case CK_LValueToRValueBitCast:
6288 case CK_ArrayToPointerDecay:
6289 case CK_FunctionToPointerDecay:
6290 case CK_NullToMemberPointer:
6291 case CK_NullToPointer:
6292 case CK_IntegralToPointer:
6293 case CK_PointerToIntegral:
6294 case CK_PointerToBoolean:
6295 case CK_IntegralCast:
6296 case CK_BooleanToSignedIntegral:
6297 case CK_IntegralToBoolean:
6298 case CK_IntegralToFloating:
6299 case CK_FloatingToIntegral:
6300 case CK_FloatingToBoolean:
6301 case CK_FloatingCast:
6302 case CK_FloatingRealToComplex:
6303 case CK_FloatingComplexToReal:
6304 case CK_FloatingComplexToBoolean:
6305 case CK_FloatingComplexCast:
6306 case CK_FloatingComplexToIntegralComplex:
6307 case CK_IntegralRealToComplex:
6308 case CK_IntegralComplexToReal:
6309 case CK_IntegralComplexToBoolean:
6310 case CK_IntegralComplexCast:
6311 case CK_IntegralComplexToFloatingComplex:
6312 case CK_DerivedToBaseMemberPointer:
6313 case CK_BaseToDerivedMemberPointer:
6314 case CK_MemberPointerToBoolean:
6315 case CK_ReinterpretMemberPointer:
6316 case CK_AnyPointerToBlockPointerCast:
6317 case CK_ARCProduceObject:
6318 case CK_ARCConsumeObject:
6319 case CK_ARCReclaimReturnedObject:
6320 case CK_ARCExtendBlockObject:
6321 case CK_CopyAndAutoreleaseBlockObject:
6322 case CK_IntToOCLSampler:
6323 case CK_FloatingToFixedPoint:
6324 case CK_FixedPointToFloating:
6325 case CK_FixedPointCast:
6326 case CK_FixedPointToBoolean:
6327 case CK_FixedPointToIntegral:
6328 case CK_IntegralToFixedPoint:
6329 case CK_MatrixCast:
6330 case CK_HLSLVectorTruncation:
6331 case CK_HLSLMatrixTruncation:
6332 case CK_HLSLArrayRValue:
6333 case CK_HLSLElementwiseCast:
6334 case CK_HLSLAggregateSplatCast:
6335 return EmitUnsupportedLValue(E, Name: "unexpected cast lvalue");
6336
6337 case CK_Dependent:
6338 llvm_unreachable("dependent cast kind in IR gen!");
6339
6340 case CK_BuiltinFnToFnPtr:
6341 llvm_unreachable("builtin functions are handled elsewhere");
6342
6343 // These are never l-values; just use the aggregate emission code.
6344 case CK_NonAtomicToAtomic:
6345 case CK_AtomicToNonAtomic:
6346 return EmitAggExprToLValue(E);
6347
6348 case CK_Dynamic: {
6349 LValue LV = EmitLValue(E: E->getSubExpr());
6350 Address V = LV.getAddress();
6351 const auto *DCE = cast<CXXDynamicCastExpr>(Val: E);
6352 return MakeNaturalAlignRawAddrLValue(V: EmitDynamicCast(V, DCE), T: E->getType());
6353 }
6354
6355 case CK_ConstructorConversion:
6356 case CK_UserDefinedConversion:
6357 case CK_CPointerToObjCPointerCast:
6358 case CK_BlockPointerToObjCPointerCast:
6359 case CK_LValueToRValue:
6360 return EmitLValue(E: E->getSubExpr());
6361
6362 case CK_NoOp: {
6363 // CK_NoOp can model a qualification conversion, which can remove an array
6364 // bound and change the IR type.
6365 // FIXME: Once pointee types are removed from IR, remove this.
6366 LValue LV = EmitLValue(E: E->getSubExpr());
6367 // Propagate the volatile qualifer to LValue, if exist in E.
6368 if (E->changesVolatileQualification())
6369 LV.getQuals() = E->getType().getQualifiers();
6370 if (LV.isSimple()) {
6371 Address V = LV.getAddress();
6372 if (V.isValid()) {
6373 llvm::Type *T = ConvertTypeForMem(T: E->getType());
6374 if (V.getElementType() != T)
6375 LV.setAddress(V.withElementType(ElemTy: T));
6376 }
6377 }
6378 return LV;
6379 }
6380
6381 case CK_UncheckedDerivedToBase:
6382 case CK_DerivedToBase: {
6383 auto *DerivedClassDecl = E->getSubExpr()->getType()->castAsCXXRecordDecl();
6384 LValue LV = EmitLValue(E: E->getSubExpr());
6385 Address This = LV.getAddress();
6386
6387 // Perform the derived-to-base conversion
6388 Address Base = GetAddressOfBaseClass(
6389 Value: This, Derived: DerivedClassDecl, PathBegin: E->path_begin(), PathEnd: E->path_end(),
6390 /*NullCheckValue=*/false, Loc: E->getExprLoc());
6391
6392 // TODO: Support accesses to members of base classes in TBAA. For now, we
6393 // conservatively pretend that the complete object is of the base class
6394 // type.
6395 return MakeAddrLValue(Addr: Base, T: E->getType(), BaseInfo: LV.getBaseInfo(),
6396 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
6397 }
6398 case CK_ToUnion:
6399 return EmitAggExprToLValue(E);
6400 case CK_BaseToDerived: {
6401 auto *DerivedClassDecl = E->getType()->castAsCXXRecordDecl();
6402 LValue LV = EmitLValue(E: E->getSubExpr());
6403
6404 // Perform the base-to-derived conversion
6405 Address Derived = GetAddressOfDerivedClass(
6406 Value: LV.getAddress(), Derived: DerivedClassDecl, PathBegin: E->path_begin(), PathEnd: E->path_end(),
6407 /*NullCheckValue=*/false);
6408
6409 // C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is
6410 // performed and the object is not of the derived type.
6411 if (sanitizePerformTypeCheck())
6412 EmitTypeCheck(TCK: TCK_DowncastReference, Loc: E->getExprLoc(), Addr: Derived,
6413 Type: E->getType());
6414
6415 if (SanOpts.has(K: SanitizerKind::CFIDerivedCast))
6416 EmitVTablePtrCheckForCast(T: E->getType(), Derived,
6417 /*MayBeNull=*/false, TCK: CFITCK_DerivedCast,
6418 Loc: E->getBeginLoc());
6419
6420 return MakeAddrLValue(Addr: Derived, T: E->getType(), BaseInfo: LV.getBaseInfo(),
6421 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
6422 }
6423 case CK_LValueBitCast: {
6424 // This must be a reinterpret_cast (or c-style equivalent).
6425 const auto *CE = cast<ExplicitCastExpr>(Val: E);
6426
6427 CGM.EmitExplicitCastExprType(E: CE, CGF: this);
6428 LValue LV = EmitLValue(E: E->getSubExpr());
6429 Address V = LV.getAddress().withElementType(
6430 ElemTy: ConvertTypeForMem(T: CE->getTypeAsWritten()->getPointeeType()));
6431
6432 if (SanOpts.has(K: SanitizerKind::CFIUnrelatedCast))
6433 EmitVTablePtrCheckForCast(T: E->getType(), Derived: V,
6434 /*MayBeNull=*/false, TCK: CFITCK_UnrelatedCast,
6435 Loc: E->getBeginLoc());
6436
6437 return MakeAddrLValue(Addr: V, T: E->getType(), BaseInfo: LV.getBaseInfo(),
6438 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
6439 }
6440 case CK_AddressSpaceConversion: {
6441 LValue LV = EmitLValue(E: E->getSubExpr());
6442 QualType DestTy = getContext().getPointerType(T: E->getType());
6443 llvm::Value *V =
6444 performAddrSpaceCast(Src: LV.getPointer(CGF&: *this), DestTy: ConvertType(T: DestTy));
6445 return MakeAddrLValue(Addr: Address(V, ConvertTypeForMem(T: E->getType()),
6446 LV.getAddress().getAlignment()),
6447 T: E->getType(), BaseInfo: LV.getBaseInfo(), TBAAInfo: LV.getTBAAInfo());
6448 }
6449 case CK_ObjCObjectLValueCast: {
6450 LValue LV = EmitLValue(E: E->getSubExpr());
6451 Address V = LV.getAddress().withElementType(ElemTy: ConvertType(T: E->getType()));
6452 return MakeAddrLValue(Addr: V, T: E->getType(), BaseInfo: LV.getBaseInfo(),
6453 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
6454 }
6455 case CK_ZeroToOCLOpaqueType:
6456 llvm_unreachable("NULL to OpenCL opaque type lvalue cast is not valid");
6457
6458 case CK_VectorSplat: {
6459 // LValue results of vector splats are only supported in HLSL.
6460 if (!getLangOpts().HLSL)
6461 return EmitUnsupportedLValue(E, Name: "unexpected cast lvalue");
6462 return EmitLValue(E: E->getSubExpr());
6463 }
6464 }
6465
6466 llvm_unreachable("Unhandled lvalue cast kind?");
6467}
6468
6469LValue CodeGenFunction::EmitOpaqueValueLValue(const OpaqueValueExpr *e) {
6470 assert(OpaqueValueMappingData::shouldBindAsLValue(e));
6471 return getOrCreateOpaqueLValueMapping(e);
6472}
6473
6474std::pair<LValue, LValue>
6475CodeGenFunction::EmitHLSLOutArgLValues(const HLSLOutArgExpr *E, QualType Ty) {
6476 // Emitting the casted temporary through an opaque value.
6477 LValue BaseLV = EmitLValue(E: E->getArgLValue());
6478 OpaqueValueMappingData::bind(CGF&: *this, ov: E->getOpaqueArgLValue(), lv: BaseLV);
6479
6480 QualType ExprTy = E->getType();
6481 Address OutTemp = CreateIRTempWithoutCast(Ty: ExprTy);
6482 LValue TempLV = MakeAddrLValue(Addr: OutTemp, T: ExprTy);
6483
6484 // Start the lifetime before the copy-in so that the temporary is live when
6485 // the initial value is written. This ensures the store is within the
6486 // lifetime and is not killed by a store undef inserted at lifetime.start.
6487 EmitLifetimeStart(Addr: OutTemp.getBasePointer());
6488
6489 if (E->isInOut())
6490 EmitInitializationToLValue(E: E->getCastedTemporary()->getSourceExpr(),
6491 LV: TempLV);
6492
6493 OpaqueValueMappingData::bind(CGF&: *this, ov: E->getCastedTemporary(), lv: TempLV);
6494 return std::make_pair(x&: BaseLV, y&: TempLV);
6495}
6496
6497LValue CodeGenFunction::EmitHLSLOutArgExpr(const HLSLOutArgExpr *E,
6498 CallArgList &Args, QualType Ty) {
6499
6500 auto [BaseLV, TempLV] = EmitHLSLOutArgLValues(E, Ty);
6501
6502 llvm::Value *Addr = TempLV.getAddress().getBasePointer();
6503 llvm::Type *ElTy = ConvertTypeForMem(T: TempLV.getType());
6504
6505 Address TmpAddr(Addr, ElTy, TempLV.getAlignment());
6506 Args.addWriteback(srcLV: BaseLV, temporary: TmpAddr, toUse: nullptr, writebackExpr: E->getWritebackCast());
6507 Args.add(rvalue: RValue::get(Addr: TmpAddr, CGF&: *this), type: Ty);
6508 return TempLV;
6509}
6510
6511LValue
6512CodeGenFunction::getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e) {
6513 assert(OpaqueValueMapping::shouldBindAsLValue(e));
6514
6515 llvm::DenseMap<const OpaqueValueExpr*,LValue>::iterator
6516 it = OpaqueLValues.find(Val: e);
6517
6518 if (it != OpaqueLValues.end())
6519 return it->second;
6520
6521 assert(e->isUnique() && "LValue for a nonunique OVE hasn't been emitted");
6522 return EmitLValue(E: e->getSourceExpr());
6523}
6524
6525RValue
6526CodeGenFunction::getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e) {
6527 assert(!OpaqueValueMapping::shouldBindAsLValue(e));
6528
6529 llvm::DenseMap<const OpaqueValueExpr*,RValue>::iterator
6530 it = OpaqueRValues.find(Val: e);
6531
6532 if (it != OpaqueRValues.end())
6533 return it->second;
6534
6535 assert(e->isUnique() && "RValue for a nonunique OVE hasn't been emitted");
6536 return EmitAnyExpr(E: e->getSourceExpr());
6537}
6538
6539bool CodeGenFunction::isOpaqueValueEmitted(const OpaqueValueExpr *E) {
6540 if (OpaqueValueMapping::shouldBindAsLValue(expr: E))
6541 return OpaqueLValues.contains(Val: E);
6542 return OpaqueRValues.contains(Val: E);
6543}
6544
6545RValue CodeGenFunction::EmitRValueForField(LValue LV,
6546 const FieldDecl *FD,
6547 SourceLocation Loc) {
6548 QualType FT = FD->getType();
6549 LValue FieldLV = EmitLValueForField(base: LV, field: FD);
6550 switch (getEvaluationKind(T: FT)) {
6551 case TEK_Complex:
6552 return RValue::getComplex(C: EmitLoadOfComplex(src: FieldLV, loc: Loc));
6553 case TEK_Aggregate:
6554 return FieldLV.asAggregateRValue();
6555 case TEK_Scalar:
6556 // This routine is used to load fields one-by-one to perform a copy, so
6557 // don't load reference fields.
6558 if (FD->getType()->isReferenceType())
6559 return RValue::get(V: FieldLV.getPointer(CGF&: *this));
6560 // Call EmitLoadOfScalar except when the lvalue is a bitfield to emit a
6561 // primitive load.
6562 if (FieldLV.isBitField())
6563 return EmitLoadOfLValue(LV: FieldLV, Loc);
6564 return RValue::get(V: EmitLoadOfScalar(lvalue: FieldLV, Loc));
6565 }
6566 llvm_unreachable("bad evaluation kind");
6567}
6568
6569//===--------------------------------------------------------------------===//
6570// Expression Emission
6571//===--------------------------------------------------------------------===//
6572
6573RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
6574 ReturnValueSlot ReturnValue,
6575 llvm::CallBase **CallOrInvoke) {
6576 llvm::CallBase *CallOrInvokeStorage;
6577 if (!CallOrInvoke) {
6578 CallOrInvoke = &CallOrInvokeStorage;
6579 }
6580
6581 llvm::scope_exit AddCoroElideSafeOnExit([&] {
6582 if (E->isCoroElideSafe()) {
6583 auto *I = *CallOrInvoke;
6584 if (I)
6585 I->addFnAttr(Kind: llvm::Attribute::CoroElideSafe);
6586 }
6587 });
6588
6589 // Builtins never have block type.
6590 if (E->getCallee()->getType()->isBlockPointerType())
6591 return EmitBlockCallExpr(E, ReturnValue, CallOrInvoke);
6592
6593 if (const auto *CE = dyn_cast<CXXMemberCallExpr>(Val: E))
6594 return EmitCXXMemberCallExpr(E: CE, ReturnValue, CallOrInvoke);
6595
6596 if (const auto *CE = dyn_cast<CUDAKernelCallExpr>(Val: E))
6597 return EmitCUDAKernelCallExpr(E: CE, ReturnValue, CallOrInvoke);
6598
6599 // A CXXOperatorCallExpr is created even for explicit object methods, but
6600 // these should be treated like static function call.
6601 if (const auto *CE = dyn_cast<CXXOperatorCallExpr>(Val: E))
6602 if (const auto *MD =
6603 dyn_cast_if_present<CXXMethodDecl>(Val: CE->getCalleeDecl());
6604 MD && MD->isImplicitObjectMemberFunction())
6605 return EmitCXXOperatorMemberCallExpr(E: CE, MD, ReturnValue, CallOrInvoke);
6606
6607 CGCallee callee = EmitCallee(E: E->getCallee());
6608
6609 if (callee.isBuiltin()) {
6610 return EmitBuiltinExpr(GD: callee.getBuiltinDecl(), BuiltinID: callee.getBuiltinID(),
6611 E, ReturnValue);
6612 }
6613
6614 if (callee.isPseudoDestructor()) {
6615 return EmitCXXPseudoDestructorExpr(E: callee.getPseudoDestructorExpr());
6616 }
6617
6618 return EmitCall(FnType: E->getCallee()->getType(), Callee: callee, E, ReturnValue,
6619 /*Chain=*/nullptr, CallOrInvoke);
6620}
6621
6622/// Emit a CallExpr without considering whether it might be a subclass.
6623RValue CodeGenFunction::EmitSimpleCallExpr(const CallExpr *E,
6624 ReturnValueSlot ReturnValue,
6625 llvm::CallBase **CallOrInvoke) {
6626 CGCallee Callee = EmitCallee(E: E->getCallee());
6627 return EmitCall(FnType: E->getCallee()->getType(), Callee, E, ReturnValue,
6628 /*Chain=*/nullptr, CallOrInvoke);
6629}
6630
6631// Detect the unusual situation where an inline version is shadowed by a
6632// non-inline version. In that case we should pick the external one
6633// everywhere. That's GCC behavior too.
6634static bool OnlyHasInlineBuiltinDeclaration(const FunctionDecl *FD) {
6635 for (const FunctionDecl *PD = FD; PD; PD = PD->getPreviousDecl())
6636 if (!PD->isInlineBuiltinDeclaration())
6637 return false;
6638 return true;
6639}
6640
6641static CGCallee EmitDirectCallee(CodeGenFunction &CGF, GlobalDecl GD) {
6642 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
6643
6644 if (auto builtinID = FD->getBuiltinID()) {
6645 std::string NoBuiltinFD = ("no-builtin-" + FD->getName()).str();
6646 std::string NoBuiltins = "no-builtins";
6647
6648 StringRef Ident = CGF.CGM.getMangledName(GD);
6649 std::string FDInlineName = (Ident + ".inline").str();
6650
6651 bool IsPredefinedLibFunction =
6652 CGF.getContext().BuiltinInfo.isPredefinedLibFunction(ID: builtinID);
6653 bool HasAttributeNoBuiltin =
6654 CGF.CurFn->getAttributes().hasFnAttr(Kind: NoBuiltinFD) ||
6655 CGF.CurFn->getAttributes().hasFnAttr(Kind: NoBuiltins);
6656
6657 // When directing calling an inline builtin, call it through it's mangled
6658 // name to make it clear it's not the actual builtin.
6659 if (CGF.CurFn->getName() != FDInlineName &&
6660 OnlyHasInlineBuiltinDeclaration(FD)) {
6661 llvm::Constant *CalleePtr = CGF.CGM.getRawFunctionPointer(GD);
6662 llvm::Function *Fn = llvm::cast<llvm::Function>(Val: CalleePtr);
6663 llvm::Module *M = Fn->getParent();
6664 llvm::Function *Clone = M->getFunction(Name: FDInlineName);
6665 if (!Clone) {
6666 Clone = llvm::Function::Create(Ty: Fn->getFunctionType(),
6667 Linkage: llvm::GlobalValue::InternalLinkage,
6668 AddrSpace: Fn->getAddressSpace(), N: FDInlineName, M);
6669 Clone->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
6670 }
6671 return CGCallee::forDirect(functionPtr: Clone, abstractInfo: GD);
6672 }
6673
6674 // Replaceable builtins provide their own implementation of a builtin. If we
6675 // are in an inline builtin implementation, avoid trivial infinite
6676 // recursion. Honor __attribute__((no_builtin("foo"))) or
6677 // __attribute__((no_builtin)) on the current function unless foo is
6678 // not a predefined library function which means we must generate the
6679 // builtin no matter what.
6680 else if (!IsPredefinedLibFunction || !HasAttributeNoBuiltin)
6681 return CGCallee::forBuiltin(builtinID, builtinDecl: FD);
6682 }
6683
6684 llvm::Constant *CalleePtr = CGF.CGM.getRawFunctionPointer(GD);
6685 if (CGF.CGM.getLangOpts().CUDA && !CGF.CGM.getLangOpts().CUDAIsDevice &&
6686 FD->hasAttr<CUDAGlobalAttr>())
6687 CalleePtr = CGF.CGM.getCUDARuntime().getKernelStub(
6688 Handle: cast<llvm::GlobalValue>(Val: CalleePtr->stripPointerCasts()));
6689
6690 return CGCallee::forDirect(functionPtr: CalleePtr, abstractInfo: GD);
6691}
6692
6693static GlobalDecl getGlobalDeclForDirectCall(const FunctionDecl *FD) {
6694 if (DeviceKernelAttr::isOpenCLSpelling(A: FD->getAttr<DeviceKernelAttr>()))
6695 return GlobalDecl(FD, KernelReferenceKind::Stub);
6696 return GlobalDecl(FD);
6697}
6698
6699CGCallee CodeGenFunction::EmitCallee(const Expr *E) {
6700 E = E->IgnoreParens();
6701
6702 // A WebAssembly funcref is an opaque reference type and llvm only accepts
6703 // function pointers as the call target. To make an indirect call through a
6704 // reference type, first use the llvm.wasm.funcref.to_ptr intrinsic to make a
6705 // fake function pointer to it. The backend lowers the resulting indirect call
6706 // to a table.set into a single element dummy table + call_indirect 0.
6707 auto ConvertFuncrefToPtr = [&](llvm::Value *CalleePtr) -> llvm::Value * {
6708 if (auto *TET = dyn_cast<llvm::TargetExtType>(Val: CalleePtr->getType());
6709 TET && TET->getName() == "wasm.funcref") {
6710 llvm::Function *ToPtr =
6711 CGM.getIntrinsic(IID: llvm::Intrinsic::wasm_funcref_to_ptr);
6712 return Builder.CreateCall(Callee: ToPtr, Args: {CalleePtr});
6713 }
6714 return CalleePtr;
6715 };
6716
6717 // Look through function-to-pointer decay.
6718 if (auto ICE = dyn_cast<ImplicitCastExpr>(Val: E)) {
6719 if (ICE->getCastKind() == CK_FunctionToPointerDecay ||
6720 ICE->getCastKind() == CK_BuiltinFnToFnPtr) {
6721 return EmitCallee(E: ICE->getSubExpr());
6722 }
6723
6724 // Try to remember the original __ptrauth qualifier for loads of
6725 // function pointers.
6726 if (ICE->getCastKind() == CK_LValueToRValue) {
6727 const Expr *SubExpr = ICE->getSubExpr();
6728 if (const auto *PtrType = SubExpr->getType()->getAs<PointerType>()) {
6729 std::pair<llvm::Value *, CGPointerAuthInfo> Result =
6730 EmitOrigPointerRValue(E);
6731
6732 QualType FunctionType = PtrType->getPointeeType();
6733 assert(FunctionType->isFunctionType());
6734
6735 GlobalDecl GD;
6736 if (const auto *VD =
6737 dyn_cast_or_null<VarDecl>(Val: E->getReferencedDeclOfCallee())) {
6738 GD = GlobalDecl(VD);
6739 }
6740 CGCalleeInfo CalleeInfo(FunctionType->castAs<clang::FunctionType>(),
6741 GD);
6742 CGCallee Callee(CalleeInfo, ConvertFuncrefToPtr(Result.first),
6743 Result.second);
6744 return Callee;
6745 }
6746 }
6747
6748 // Resolve direct calls.
6749 } else if (auto DRE = dyn_cast<DeclRefExpr>(Val: E)) {
6750 if (auto FD = dyn_cast<FunctionDecl>(Val: DRE->getDecl())) {
6751 return EmitDirectCallee(CGF&: *this, GD: getGlobalDeclForDirectCall(FD));
6752 }
6753 } else if (auto ME = dyn_cast<MemberExpr>(Val: E)) {
6754 if (auto FD = dyn_cast<FunctionDecl>(Val: ME->getMemberDecl())) {
6755 EmitIgnoredExpr(E: ME->getBase());
6756 return EmitDirectCallee(CGF&: *this, GD: FD);
6757 }
6758
6759 // Look through template substitutions.
6760 } else if (auto NTTP = dyn_cast<SubstNonTypeTemplateParmExpr>(Val: E)) {
6761 return EmitCallee(E: NTTP->getReplacement());
6762
6763 // Treat pseudo-destructor calls differently.
6764 } else if (auto PDE = dyn_cast<CXXPseudoDestructorExpr>(Val: E)) {
6765 return CGCallee::forPseudoDestructor(E: PDE);
6766 }
6767
6768 // Otherwise, we have an indirect reference.
6769 llvm::Value *calleePtr;
6770 QualType functionType;
6771 if (auto ptrType = E->getType()->getAs<PointerType>()) {
6772 calleePtr = EmitScalarExpr(E);
6773 functionType = ptrType->getPointeeType();
6774 } else {
6775 functionType = E->getType();
6776 calleePtr = EmitLValue(E, IsKnownNonNull: KnownNonNull).getPointer(CGF&: *this);
6777 }
6778 assert(functionType->isFunctionType());
6779
6780 GlobalDecl GD;
6781 if (const auto *VD =
6782 dyn_cast_or_null<VarDecl>(Val: E->getReferencedDeclOfCallee()))
6783 GD = GlobalDecl(VD);
6784
6785 CGCalleeInfo calleeInfo(functionType->castAs<clang::FunctionType>(), GD);
6786 CGPointerAuthInfo pointerAuth = CGM.getFunctionPointerAuthInfo(T: functionType);
6787 CGCallee callee(calleeInfo, ConvertFuncrefToPtr(calleePtr), pointerAuth);
6788 return callee;
6789}
6790
6791LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
6792 // Comma expressions just emit their LHS then their RHS as an l-value.
6793 if (E->getOpcode() == BO_Comma) {
6794 EmitIgnoredExpr(E: E->getLHS());
6795 EnsureInsertPoint();
6796 return EmitLValue(E: E->getRHS());
6797 }
6798
6799 if (E->getOpcode() == BO_PtrMemD ||
6800 E->getOpcode() == BO_PtrMemI)
6801 return EmitPointerToDataMemberBinaryExpr(E);
6802
6803 assert(E->getOpcode() == BO_Assign && "unexpected binary l-value");
6804
6805 // Create a Key Instructions source location atom group that covers both
6806 // LHS and RHS expressions. Nested RHS expressions may get subsequently
6807 // separately grouped (1 below):
6808 //
6809 // 1. `a = b = c` -> Two atoms.
6810 // 2. `x = new(1)` -> One atom (for both addr store and value store).
6811 // 3. Complex and agg assignment -> One atom.
6812 ApplyAtomGroup Grp(getDebugInfo());
6813
6814 // Note that in all of these cases, __block variables need the RHS
6815 // evaluated first just in case the variable gets moved by the RHS.
6816
6817 switch (getEvaluationKind(T: E->getType())) {
6818 case TEK_Scalar: {
6819 if (PointerAuthQualifier PtrAuth =
6820 E->getLHS()->getType().getPointerAuth()) {
6821 LValue LV = EmitCheckedLValue(E: E->getLHS(), TCK: TCK_Store);
6822 LValue CopiedLV = LV;
6823 CopiedLV.getQuals().removePointerAuth();
6824 llvm::Value *RV =
6825 EmitPointerAuthQualify(Qualifier: PtrAuth, PointerExpr: E->getRHS(), StorageAddress: CopiedLV.getAddress());
6826 EmitNullabilityCheck(LHS: CopiedLV, RHS: RV, Loc: E->getExprLoc());
6827 EmitStoreThroughLValue(Src: RValue::get(V: RV), Dst: CopiedLV);
6828 return LV;
6829 }
6830
6831 switch (E->getLHS()->getType().getObjCLifetime()) {
6832 case Qualifiers::OCL_Strong:
6833 return EmitARCStoreStrong(e: E, /*ignored*/ false).first;
6834
6835 case Qualifiers::OCL_Autoreleasing:
6836 return EmitARCStoreAutoreleasing(e: E).first;
6837
6838 // No reason to do any of these differently.
6839 case Qualifiers::OCL_None:
6840 case Qualifiers::OCL_ExplicitNone:
6841 case Qualifiers::OCL_Weak:
6842 break;
6843 }
6844
6845 // TODO: Can we de-duplicate this code with the corresponding code in
6846 // CGExprScalar, similar to the way EmitCompoundAssignmentLValue works?
6847 RValue RV;
6848 llvm::Value *Previous = nullptr;
6849 QualType SrcType = E->getRHS()->getType();
6850 // Check if LHS is a bitfield, if RHS contains an implicit cast expression
6851 // we want to extract that value and potentially (if the bitfield sanitizer
6852 // is enabled) use it to check for an implicit conversion.
6853 if (E->getLHS()->refersToBitField()) {
6854 llvm::Value *RHS =
6855 EmitWithOriginalRHSBitfieldAssignment(E, Previous: &Previous, SrcType: &SrcType);
6856 RV = RValue::get(V: RHS);
6857 } else
6858 RV = EmitAnyExpr(E: E->getRHS());
6859
6860 LValue LV = EmitCheckedLValue(E: E->getLHS(), TCK: TCK_Store);
6861
6862 if (RV.isScalar())
6863 EmitNullabilityCheck(LHS: LV, RHS: RV.getScalarVal(), Loc: E->getExprLoc());
6864
6865 if (LV.isBitField()) {
6866 llvm::Value *Result = nullptr;
6867 // If bitfield sanitizers are enabled we want to use the result
6868 // to check whether a truncation or sign change has occurred.
6869 if (SanOpts.has(K: SanitizerKind::ImplicitBitfieldConversion))
6870 EmitStoreThroughBitfieldLValue(Src: RV, Dst: LV, Result: &Result);
6871 else
6872 EmitStoreThroughBitfieldLValue(Src: RV, Dst: LV);
6873
6874 // If the expression contained an implicit conversion, make sure
6875 // to use the value before the scalar conversion.
6876 llvm::Value *Src = Previous ? Previous : RV.getScalarVal();
6877 QualType DstType = E->getLHS()->getType();
6878 EmitBitfieldConversionCheck(Src, SrcType, Dst: Result, DstType,
6879 Info: LV.getBitFieldInfo(), Loc: E->getExprLoc());
6880 } else
6881 EmitStoreThroughLValue(Src: RV, Dst: LV);
6882
6883 if (getLangOpts().OpenMP)
6884 CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF&: *this,
6885 LHS: E->getLHS());
6886 return LV;
6887 }
6888
6889 case TEK_Complex:
6890 return EmitComplexAssignmentLValue(E);
6891
6892 case TEK_Aggregate:
6893 // If the lang opt is HLSL and the LHS is a constant array
6894 // then we are performing a copy assignment and call a special
6895 // function because EmitAggExprToLValue emits to a temporary LValue
6896 if (getLangOpts().HLSL && E->getLHS()->getType()->isConstantArrayType())
6897 return EmitHLSLArrayAssignLValue(E);
6898
6899 return EmitAggExprToLValue(E);
6900 }
6901 llvm_unreachable("bad evaluation kind");
6902}
6903
6904// This function implements trivial copy assignment for HLSL's
6905// assignable constant arrays.
6906LValue CodeGenFunction::EmitHLSLArrayAssignLValue(const BinaryOperator *E) {
6907 // Don't emit an LValue for the RHS because it might not be an LValue
6908 LValue LHS = EmitLValue(E: E->getLHS());
6909
6910 // If the RHS is a global resource array, copy all individual resources
6911 // into LHS.
6912 if (E->getRHS()->getType()->isHLSLResourceRecordArray()) {
6913 AggValueSlot Slot = AggValueSlot::forAddr(
6914 addr: LHS.getAddress(), quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed_t(true),
6915 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsAliased_t(false),
6916 mayOverlap: AggValueSlot::DoesNotOverlap);
6917 if (CGM.getHLSLRuntime().emitGlobalResourceArray(CGF&: *this, E: E->getRHS(), DestSlot&: Slot))
6918 return LHS;
6919 }
6920
6921 // In C the RHS of an assignment operator is an RValue.
6922 // EmitAggregateAssign takes an LValue for the RHS. Instead we can call
6923 // EmitInitializationToLValue to emit an RValue into an LValue.
6924 EmitInitializationToLValue(E: E->getRHS(), LV: LHS);
6925 return LHS;
6926}
6927
6928LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E,
6929 llvm::CallBase **CallOrInvoke) {
6930 RValue RV = EmitCallExpr(E, ReturnValue: ReturnValueSlot(), CallOrInvoke);
6931
6932 if (!RV.isScalar())
6933 return MakeAddrLValue(Addr: RV.getAggregateAddress(), T: E->getType(),
6934 Source: AlignmentSource::Decl);
6935
6936 assert(E->getCallReturnType(getContext())->isReferenceType() &&
6937 "Can't have a scalar return unless the return type is a "
6938 "reference type!");
6939
6940 return MakeNaturalAlignPointeeAddrLValue(V: RV.getScalarVal(), T: E->getType());
6941}
6942
6943LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {
6944 // FIXME: This shouldn't require another copy.
6945 return EmitAggExprToLValue(E);
6946}
6947
6948LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
6949 assert(E->getType()->getAsCXXRecordDecl()->hasTrivialDestructor()
6950 && "binding l-value to type which needs a temporary");
6951 AggValueSlot Slot = CreateAggTemp(T: E->getType());
6952 EmitCXXConstructExpr(E, Dest: Slot);
6953 return MakeAddrLValue(Addr: Slot.getAddress(), T: E->getType(), Source: AlignmentSource::Decl);
6954}
6955
6956LValue
6957CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
6958 return MakeNaturalAlignRawAddrLValue(V: EmitCXXTypeidExpr(E), T: E->getType());
6959}
6960
6961Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) {
6962 return CGM.GetAddrOfMSGuidDecl(GD: E->getGuidDecl())
6963 .withElementType(ElemTy: ConvertType(T: E->getType()));
6964}
6965
6966LValue CodeGenFunction::EmitCXXUuidofLValue(const CXXUuidofExpr *E) {
6967 return MakeAddrLValue(Addr: EmitCXXUuidofExpr(E), T: E->getType(),
6968 Source: AlignmentSource::Decl);
6969}
6970
6971LValue
6972CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) {
6973 AggValueSlot Slot = CreateAggTemp(T: E->getType(), Name: "temp.lvalue");
6974 Slot.setExternallyDestructed();
6975 EmitAggExpr(E: E->getSubExpr(), AS: Slot);
6976 EmitCXXTemporary(Temporary: E->getTemporary(), TempType: E->getType(), Ptr: Slot.getAddress());
6977 return MakeAddrLValue(Addr: Slot.getAddress(), T: E->getType(), Source: AlignmentSource::Decl);
6978}
6979
6980LValue CodeGenFunction::EmitObjCMessageExprLValue(const ObjCMessageExpr *E) {
6981 RValue RV = EmitObjCMessageExpr(E);
6982
6983 if (!RV.isScalar())
6984 return MakeAddrLValue(Addr: RV.getAggregateAddress(), T: E->getType(),
6985 Source: AlignmentSource::Decl);
6986
6987 assert(E->getMethodDecl()->getReturnType()->isReferenceType() &&
6988 "Can't have a scalar return unless the return type is a "
6989 "reference type!");
6990
6991 return MakeNaturalAlignPointeeAddrLValue(V: RV.getScalarVal(), T: E->getType());
6992}
6993
6994LValue CodeGenFunction::EmitObjCSelectorLValue(const ObjCSelectorExpr *E) {
6995 Address V =
6996 CGM.getObjCRuntime().GetAddrOfSelector(CGF&: *this, Sel: E->getSelector());
6997 return MakeAddrLValue(Addr: V, T: E->getType(), Source: AlignmentSource::Decl);
6998}
6999
7000llvm::Value *CodeGenFunction::EmitIvarOffset(const ObjCInterfaceDecl *Interface,
7001 const ObjCIvarDecl *Ivar) {
7002 return CGM.getObjCRuntime().EmitIvarOffset(CGF&: *this, Interface, Ivar);
7003}
7004
7005llvm::Value *
7006CodeGenFunction::EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,
7007 const ObjCIvarDecl *Ivar) {
7008 llvm::Value *OffsetValue = EmitIvarOffset(Interface, Ivar);
7009 QualType PointerDiffType = getContext().getPointerDiffType();
7010 return Builder.CreateZExtOrTrunc(V: OffsetValue,
7011 DestTy: getTypes().ConvertType(T: PointerDiffType));
7012}
7013
7014LValue CodeGenFunction::EmitLValueForIvar(QualType ObjectTy,
7015 llvm::Value *BaseValue,
7016 const ObjCIvarDecl *Ivar,
7017 unsigned CVRQualifiers) {
7018 return CGM.getObjCRuntime().EmitObjCValueForIvar(CGF&: *this, ObjectTy, BaseValue,
7019 Ivar, CVRQualifiers);
7020}
7021
7022LValue CodeGenFunction::EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E) {
7023 // FIXME: A lot of the code below could be shared with EmitMemberExpr.
7024 llvm::Value *BaseValue = nullptr;
7025 const Expr *BaseExpr = E->getBase();
7026 Qualifiers BaseQuals;
7027 QualType ObjectTy;
7028 if (E->isArrow()) {
7029 BaseValue = EmitScalarExpr(E: BaseExpr);
7030 ObjectTy = BaseExpr->getType()->getPointeeType();
7031 BaseQuals = ObjectTy.getQualifiers();
7032 } else {
7033 LValue BaseLV = EmitLValue(E: BaseExpr);
7034 BaseValue = BaseLV.getPointer(CGF&: *this);
7035 ObjectTy = BaseExpr->getType();
7036 BaseQuals = ObjectTy.getQualifiers();
7037 }
7038
7039 LValue LV =
7040 EmitLValueForIvar(ObjectTy, BaseValue, Ivar: E->getDecl(),
7041 CVRQualifiers: BaseQuals.getCVRQualifiers());
7042 setObjCGCLValueClass(Ctx: getContext(), E, LV);
7043 return LV;
7044}
7045
7046LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {
7047 // Can only get l-value for message expression returning aggregate type
7048 RValue RV = EmitAnyExprToTemp(E);
7049 return MakeAddrLValue(Addr: RV.getAggregateAddress(), T: E->getType(),
7050 Source: AlignmentSource::Decl);
7051}
7052
7053RValue CodeGenFunction::EmitCall(QualType CalleeType,
7054 const CGCallee &OrigCallee, const CallExpr *E,
7055 ReturnValueSlot ReturnValue,
7056 llvm::Value *Chain,
7057 llvm::CallBase **CallOrInvoke,
7058 CGFunctionInfo const **ResolvedFnInfo) {
7059 // Get the actual function type. The callee type will always be a pointer to
7060 // function type or a block pointer type.
7061 assert(CalleeType->isFunctionPointerType() &&
7062 "Call must have function pointer type!");
7063
7064 const Decl *TargetDecl =
7065 OrigCallee.getAbstractInfo().getCalleeDecl().getDecl();
7066
7067 assert((!isa_and_present<FunctionDecl>(TargetDecl) ||
7068 !cast<FunctionDecl>(TargetDecl)->isImmediateFunction()) &&
7069 "trying to emit a call to an immediate function");
7070
7071 CalleeType = getContext().getCanonicalType(T: CalleeType);
7072
7073 auto PointeeType = cast<PointerType>(Val&: CalleeType)->getPointeeType();
7074
7075 CGCallee Callee = OrigCallee;
7076
7077 bool CFIUnchecked = CalleeType->hasPointeeToCFIUncheckedCalleeFunctionType();
7078
7079 if (SanOpts.has(K: SanitizerKind::Function) &&
7080 (!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl)) &&
7081 !isa<FunctionNoProtoType>(Val: PointeeType) && !CFIUnchecked) {
7082 if (llvm::Constant *PrefixSig =
7083 CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM)) {
7084 auto CheckOrdinal = SanitizerKind::SO_Function;
7085 auto CheckHandler = SanitizerHandler::FunctionTypeMismatch;
7086 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
7087 auto *TypeHash = getUBSanFunctionTypeHash(T: PointeeType);
7088
7089 llvm::Type *PrefixSigType = PrefixSig->getType();
7090 llvm::StructType *PrefixStructTy = llvm::StructType::get(
7091 Context&: CGM.getLLVMContext(), Elements: {PrefixSigType, Int32Ty}, /*isPacked=*/true);
7092
7093 llvm::Value *CalleePtr = Callee.getFunctionPointer();
7094 if (CGM.getCodeGenOpts().PointerAuth.FunctionPointers) {
7095 // Use raw pointer since we are using the callee pointer as data here.
7096 Address Addr =
7097 Address(CalleePtr, CalleePtr->getType(),
7098 CharUnits::fromQuantity(
7099 Quantity: CalleePtr->getPointerAlignment(DL: CGM.getDataLayout())),
7100 Callee.getPointerAuthInfo(), nullptr);
7101 CalleePtr = Addr.emitRawPointer(CGF&: *this);
7102 }
7103
7104 // On 32-bit Arm, the low bit of a function pointer indicates whether
7105 // it's using the Arm or Thumb instruction set. The actual first
7106 // instruction lives at the same address either way, so we must clear
7107 // that low bit before using the function address to find the prefix
7108 // structure.
7109 //
7110 // This applies to both Arm and Thumb target triples, because
7111 // either one could be used in an interworking context where it
7112 // might be passed function pointers of both types.
7113 llvm::Value *AlignedCalleePtr;
7114 if (CGM.getTriple().isARM() || CGM.getTriple().isThumb()) {
7115 AlignedCalleePtr = Builder.CreateIntrinsic(
7116 RetTy: CalleePtr->getType(), ID: llvm::Intrinsic::ptrmask,
7117 Args: {CalleePtr, llvm::ConstantInt::getSigned(Ty: IntPtrTy, V: ~1)});
7118 } else {
7119 AlignedCalleePtr = CalleePtr;
7120 }
7121
7122 llvm::Value *CalleePrefixStruct = AlignedCalleePtr;
7123 llvm::Value *CalleeSigPtr =
7124 Builder.CreateConstGEP2_32(Ty: PrefixStructTy, Ptr: CalleePrefixStruct, Idx0: -1, Idx1: 0);
7125 llvm::Value *CalleeSig =
7126 Builder.CreateAlignedLoad(Ty: PrefixSigType, Addr: CalleeSigPtr, Align: getIntAlign());
7127 llvm::Value *CalleeSigMatch = Builder.CreateICmpEQ(LHS: CalleeSig, RHS: PrefixSig);
7128
7129 llvm::BasicBlock *Cont = createBasicBlock(name: "cont");
7130 llvm::BasicBlock *TypeCheck = createBasicBlock(name: "typecheck");
7131 Builder.CreateCondBr(Cond: CalleeSigMatch, True: TypeCheck, False: Cont);
7132
7133 EmitBlock(BB: TypeCheck);
7134 llvm::Value *CalleeTypeHash = Builder.CreateAlignedLoad(
7135 Ty: Int32Ty,
7136 Addr: Builder.CreateConstGEP2_32(Ty: PrefixStructTy, Ptr: CalleePrefixStruct, Idx0: -1, Idx1: 1),
7137 Align: getPointerAlign());
7138 llvm::Value *CalleeTypeHashMatch =
7139 Builder.CreateICmpEQ(LHS: CalleeTypeHash, RHS: TypeHash);
7140 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc: E->getBeginLoc()),
7141 EmitCheckTypeDescriptor(T: CalleeType)};
7142 EmitCheck(Checked: std::make_pair(x&: CalleeTypeHashMatch, y&: CheckOrdinal), CheckHandler,
7143 StaticArgs: StaticData, DynamicArgs: {CalleePtr});
7144
7145 Builder.CreateBr(Dest: Cont);
7146 EmitBlock(BB: Cont);
7147 }
7148 }
7149
7150 const auto *FnType = cast<FunctionType>(Val&: PointeeType);
7151
7152 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
7153 FD && DeviceKernelAttr::isOpenCLSpelling(A: FD->getAttr<DeviceKernelAttr>()))
7154 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FnType);
7155
7156 // If we are checking indirect calls and this call is indirect, check that the
7157 // function pointer is a member of the bit set for the function type.
7158 if (SanOpts.has(K: SanitizerKind::CFIICall) &&
7159 (!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl)) && !CFIUnchecked) {
7160 auto CheckOrdinal = SanitizerKind::SO_CFIICall;
7161 auto CheckHandler = SanitizerHandler::CFICheckFail;
7162 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
7163 EmitSanitizerStatReport(SSK: llvm::SanStat_CFI_ICall);
7164
7165 llvm::Metadata *MD =
7166 CGM.CreateMetadataIdentifierForFnType(T: QualType(FnType, 0));
7167
7168 llvm::Value *TypeId = llvm::MetadataAsValue::get(Context&: getLLVMContext(), MD);
7169
7170 llvm::Value *CalleePtr = Callee.getFunctionPointer();
7171 llvm::Value *TypeTest = Builder.CreateCall(
7172 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::type_test), Args: {CalleePtr, TypeId});
7173
7174 auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
7175 llvm::Constant *StaticData[] = {
7176 llvm::ConstantInt::get(Ty: Int8Ty, V: CFITCK_ICall),
7177 EmitCheckSourceLocation(Loc: E->getBeginLoc()),
7178 EmitCheckTypeDescriptor(T: QualType(FnType, 0)),
7179 };
7180 if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) {
7181 EmitCfiSlowPathCheck(Ordinal: CheckOrdinal, Cond: TypeTest, TypeId: CrossDsoTypeId, Ptr: CalleePtr,
7182 StaticArgs: StaticData);
7183 } else {
7184 EmitCheck(Checked: std::make_pair(x&: TypeTest, y&: CheckOrdinal), CheckHandler,
7185 StaticArgs: StaticData, DynamicArgs: {CalleePtr, llvm::UndefValue::get(T: IntPtrTy)});
7186 }
7187 }
7188
7189 CallArgList Args;
7190 if (Chain)
7191 Args.add(rvalue: RValue::get(V: Chain), type: CGM.getContext().VoidPtrTy);
7192
7193 // C++17 requires that we evaluate arguments to a call using assignment syntax
7194 // right-to-left, and that we evaluate arguments to certain other operators
7195 // left-to-right. Note that we allow this to override the order dictated by
7196 // the calling convention on the MS ABI, which means that parameter
7197 // destruction order is not necessarily reverse construction order.
7198 // FIXME: Revisit this based on C++ committee response to unimplementability.
7199 EvaluationOrder Order = EvaluationOrder::Default;
7200 bool StaticOperator = false;
7201 if (auto *OCE = dyn_cast<CXXOperatorCallExpr>(Val: E)) {
7202 if (OCE->isAssignmentOp())
7203 Order = EvaluationOrder::ForceRightToLeft;
7204 else {
7205 switch (OCE->getOperator()) {
7206 case OO_LessLess:
7207 case OO_GreaterGreater:
7208 case OO_AmpAmp:
7209 case OO_PipePipe:
7210 case OO_Comma:
7211 case OO_ArrowStar:
7212 Order = EvaluationOrder::ForceLeftToRight;
7213 break;
7214 default:
7215 break;
7216 }
7217 }
7218
7219 if (const auto *MD =
7220 dyn_cast_if_present<CXXMethodDecl>(Val: OCE->getCalleeDecl());
7221 MD && MD->isStatic())
7222 StaticOperator = true;
7223 }
7224
7225 auto Arguments = E->arguments();
7226 if (StaticOperator) {
7227 // If we're calling a static operator, we need to emit the object argument
7228 // and ignore it.
7229 EmitIgnoredExpr(E: E->getArg(Arg: 0));
7230 Arguments = drop_begin(RangeOrContainer&: Arguments, N: 1);
7231 }
7232 EmitCallArgs(Args, Prototype: dyn_cast<FunctionProtoType>(Val: FnType), ArgRange: Arguments,
7233 AC: E->getDirectCallee(), /*ParamsToSkip=*/0, Order);
7234
7235 const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(
7236 Args, Ty: FnType, /*ChainCall=*/Chain, ABIInfoFD: getCurrentFunctionDecl());
7237
7238 if (ResolvedFnInfo)
7239 *ResolvedFnInfo = &FnInfo;
7240
7241 // HIP function pointer contains kernel handle when it is used in triple
7242 // chevron. The kernel stub needs to be loaded from kernel handle and used
7243 // as callee.
7244 if (CGM.getLangOpts().HIP && !CGM.getLangOpts().CUDAIsDevice &&
7245 isa<CUDAKernelCallExpr>(Val: E) &&
7246 (!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl))) {
7247 llvm::Value *Handle = Callee.getFunctionPointer();
7248 auto *Stub = Builder.CreateLoad(
7249 Addr: Address(Handle, Handle->getType(), CGM.getPointerAlign()));
7250 Callee.setFunctionPointer(Stub);
7251 }
7252
7253 // Insert function pointer lookup if this is a target call
7254 //
7255 // This is used for the indirect function case, virtual function case is
7256 // handled in ItaniumCXXABI.cpp
7257 if (getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU() &&
7258 (!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl))) {
7259 const Expr *CalleeExpr = E->getCallee()->IgnoreParenImpCasts();
7260 const DeclRefExpr *DRE = nullptr;
7261 while (CalleeExpr) {
7262 if ((DRE = dyn_cast<DeclRefExpr>(Val: CalleeExpr)))
7263 break;
7264 if (const auto *ME = dyn_cast<MemberExpr>(Val: CalleeExpr))
7265 CalleeExpr = ME->getBase()->IgnoreParenImpCasts();
7266 else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: CalleeExpr))
7267 CalleeExpr = ASE->getBase()->IgnoreParenImpCasts();
7268 else
7269 break;
7270 }
7271
7272 const auto *VD = DRE ? dyn_cast<VarDecl>(Val: DRE->getDecl()) : nullptr;
7273 if (VD && VD->hasAttr<OMPTargetIndirectCallAttr>()) {
7274 auto *FuncPtrTy = llvm::PointerType::get(
7275 C&: CGM.getLLVMContext(), AddressSpace: CGM.getDataLayout().getProgramAddressSpace());
7276 llvm::Type *RtlFnArgs[] = {FuncPtrTy};
7277 llvm::FunctionCallee DeviceRtlFn = CGM.CreateRuntimeFunction(
7278 Ty: llvm::FunctionType::get(Result: FuncPtrTy, Params: RtlFnArgs, isVarArg: false),
7279 Name: "__llvm_omp_indirect_call_lookup");
7280 llvm::Value *Func = Callee.getFunctionPointer();
7281 llvm::Type *BackupTy = Func->getType();
7282 Func = Builder.CreatePointerBitCastOrAddrSpaceCast(V: Func, DestTy: FuncPtrTy);
7283 Func = EmitRuntimeCall(callee: DeviceRtlFn, args: {Func});
7284 Func = Builder.CreatePointerBitCastOrAddrSpaceCast(V: Func, DestTy: BackupTy);
7285 Callee.setFunctionPointer(Func);
7286 }
7287 }
7288
7289 llvm::CallBase *LocalCallOrInvoke = nullptr;
7290 RValue Call = EmitCall(CallInfo: FnInfo, Callee, ReturnValue, Args, CallOrInvoke: &LocalCallOrInvoke,
7291 IsMustTail: E == MustTailCall, Loc: E->getExprLoc());
7292
7293 if (auto *CalleeDecl = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
7294 if (CalleeDecl->hasAttr<RestrictAttr>() ||
7295 CalleeDecl->hasAttr<MallocSpanAttr>() ||
7296 CalleeDecl->hasAttr<AllocSizeAttr>()) {
7297 // Function has 'malloc' (aka. 'restrict') or 'alloc_size' attribute.
7298 if (SanOpts.has(K: SanitizerKind::AllocToken)) {
7299 // Set !alloc_token metadata.
7300 EmitAllocToken(CB: LocalCallOrInvoke, E);
7301 }
7302 }
7303 }
7304 if (CallOrInvoke)
7305 *CallOrInvoke = LocalCallOrInvoke;
7306
7307 return Call;
7308}
7309
7310LValue CodeGenFunction::
7311EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) {
7312 Address BaseAddr = Address::invalid();
7313 if (E->getOpcode() == BO_PtrMemI) {
7314 BaseAddr = EmitPointerWithAlignment(E: E->getLHS());
7315 } else {
7316 BaseAddr = EmitLValue(E: E->getLHS()).getAddress();
7317 }
7318
7319 llvm::Value *OffsetV = EmitScalarExpr(E: E->getRHS());
7320 const auto *MPT = E->getRHS()->getType()->castAs<MemberPointerType>();
7321
7322 LValueBaseInfo BaseInfo;
7323 TBAAAccessInfo TBAAInfo;
7324 bool IsInBounds = !getLangOpts().PointerOverflowDefined &&
7325 !isUnderlyingBasePointerConstantNull(E: E->getLHS());
7326 Address MemberAddr = EmitCXXMemberDataPointerAddress(
7327 E, base: BaseAddr, memberPtr: OffsetV, memberPtrType: MPT, IsInBounds, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
7328
7329 return MakeAddrLValue(Addr: MemberAddr, T: MPT->getPointeeType(), BaseInfo, TBAAInfo);
7330}
7331
7332/// Given the address of a temporary variable, produce an r-value of
7333/// its type.
7334RValue CodeGenFunction::convertTempToRValue(Address addr,
7335 QualType type,
7336 SourceLocation loc) {
7337 LValue lvalue = MakeAddrLValue(Addr: addr, T: type, Source: AlignmentSource::Decl);
7338 switch (getEvaluationKind(T: type)) {
7339 case TEK_Complex:
7340 return RValue::getComplex(C: EmitLoadOfComplex(src: lvalue, loc));
7341 case TEK_Aggregate:
7342 return lvalue.asAggregateRValue();
7343 case TEK_Scalar:
7344 return RValue::get(V: EmitLoadOfScalar(lvalue, Loc: loc));
7345 }
7346 llvm_unreachable("bad evaluation kind");
7347}
7348
7349void CodeGenFunction::SetFPAccuracy(llvm::Value *Val, float Accuracy) {
7350 assert(Val->getType()->isFPOrFPVectorTy());
7351 if (Accuracy == 0.0 || !isa<llvm::Instruction>(Val))
7352 return;
7353
7354 llvm::MDBuilder MDHelper(getLLVMContext());
7355 llvm::MDNode *Node = MDHelper.createFPMath(Accuracy);
7356
7357 cast<llvm::Instruction>(Val)->setMetadata(KindID: llvm::LLVMContext::MD_fpmath, Node);
7358}
7359
7360void CodeGenFunction::SetSqrtFPAccuracy(llvm::Value *Val) {
7361 llvm::Type *EltTy = Val->getType()->getScalarType();
7362 if (!EltTy->isFloatTy() && !EltTy->isHalfTy())
7363 return;
7364
7365 if ((getLangOpts().OpenCL &&
7366 !CGM.getCodeGenOpts().OpenCLCorrectlyRoundedDivSqrt) ||
7367 (getLangOpts().HIP && getLangOpts().CUDAIsDevice &&
7368 !CGM.getCodeGenOpts().HIPCorrectlyRoundedDivSqrt)) {
7369 // OpenCL v1.1 s7.4: minimum accuracy of single precision sqrt is 3 ulp.
7370 // OpenCL v3.0 s7.4: minimum accuracy of half precision sqrt is 1.5 ulp.
7371 //
7372 // OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt
7373 // build option allows an application to specify that single precision
7374 // floating-point divide (x/y and 1/x) and sqrt used in the program
7375 // source are correctly rounded.
7376 //
7377 // TODO: CUDA has a prec-sqrt flag
7378 SetFPAccuracy(Val, Accuracy: EltTy->isFloatTy() ? 3.0f : 1.5f);
7379 }
7380}
7381
7382void CodeGenFunction::SetDivFPAccuracy(llvm::Value *Val) {
7383 llvm::Type *EltTy = Val->getType()->getScalarType();
7384 if (!EltTy->isFloatTy() && !EltTy->isHalfTy())
7385 return;
7386
7387 if ((getLangOpts().OpenCL &&
7388 !CGM.getCodeGenOpts().OpenCLCorrectlyRoundedDivSqrt) ||
7389 (getLangOpts().HIP && getLangOpts().CUDAIsDevice &&
7390 !CGM.getCodeGenOpts().HIPCorrectlyRoundedDivSqrt)) {
7391 // OpenCL v1.1 s7.4: minimum accuracy of single precision / is 2.5 ulp.
7392 // OpenCL v3.0 s7.4: minimum accuracy of half precision / is 1 ulp.
7393 //
7394 // OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt
7395 // build option allows an application to specify that single precision
7396 // floating-point divide (x/y and 1/x) and sqrt used in the program
7397 // source are correctly rounded.
7398 //
7399 // TODO: CUDA has a prec-div flag
7400 SetFPAccuracy(Val, Accuracy: EltTy->isFloatTy() ? 2.5f : 1.f);
7401 }
7402}
7403
7404namespace {
7405 struct LValueOrRValue {
7406 LValue LV;
7407 RValue RV;
7408 };
7409}
7410
7411static LValueOrRValue emitPseudoObjectExpr(CodeGenFunction &CGF,
7412 const PseudoObjectExpr *E,
7413 bool forLValue,
7414 AggValueSlot slot) {
7415 SmallVector<CodeGenFunction::OpaqueValueMappingData, 4> opaques;
7416
7417 // Find the result expression, if any.
7418 const Expr *resultExpr = E->getResultExpr();
7419 LValueOrRValue result;
7420
7421 for (PseudoObjectExpr::const_semantics_iterator
7422 i = E->semantics_begin(), e = E->semantics_end(); i != e; ++i) {
7423 const Expr *semantic = *i;
7424
7425 // If this semantic expression is an opaque value, bind it
7426 // to the result of its source expression.
7427 if (const auto *ov = dyn_cast<OpaqueValueExpr>(Val: semantic)) {
7428 // Skip unique OVEs.
7429 if (ov->isUnique()) {
7430 assert(ov != resultExpr &&
7431 "A unique OVE cannot be used as the result expression");
7432 continue;
7433 }
7434
7435 // If this is the result expression, we may need to evaluate
7436 // directly into the slot.
7437 typedef CodeGenFunction::OpaqueValueMappingData OVMA;
7438 OVMA opaqueData;
7439 if (ov == resultExpr && ov->isPRValue() && !forLValue &&
7440 CodeGenFunction::hasAggregateEvaluationKind(T: ov->getType())) {
7441 CGF.EmitAggExpr(E: ov->getSourceExpr(), AS: slot);
7442 LValue LV = CGF.MakeAddrLValue(Addr: slot.getAddress(), T: ov->getType(),
7443 Source: AlignmentSource::Decl);
7444 opaqueData = OVMA::bind(CGF, ov, lv: LV);
7445 result.RV = slot.asRValue();
7446
7447 // Otherwise, emit as normal.
7448 } else {
7449 opaqueData = OVMA::bind(CGF, ov, e: ov->getSourceExpr());
7450
7451 // If this is the result, also evaluate the result now.
7452 if (ov == resultExpr) {
7453 if (forLValue)
7454 result.LV = CGF.EmitLValue(E: ov);
7455 else
7456 result.RV = CGF.EmitAnyExpr(E: ov, aggSlot: slot);
7457 }
7458 }
7459
7460 opaques.push_back(Elt: opaqueData);
7461
7462 // Otherwise, if the expression is the result, evaluate it
7463 // and remember the result.
7464 } else if (semantic == resultExpr) {
7465 if (forLValue)
7466 result.LV = CGF.EmitLValue(E: semantic);
7467 else
7468 result.RV = CGF.EmitAnyExpr(E: semantic, aggSlot: slot);
7469
7470 // Otherwise, evaluate the expression in an ignored context.
7471 } else {
7472 CGF.EmitIgnoredExpr(E: semantic);
7473 }
7474 }
7475
7476 // Unbind all the opaques now.
7477 for (CodeGenFunction::OpaqueValueMappingData &opaque : opaques)
7478 opaque.unbind(CGF);
7479
7480 return result;
7481}
7482
7483RValue CodeGenFunction::EmitPseudoObjectRValue(const PseudoObjectExpr *E,
7484 AggValueSlot slot) {
7485 return emitPseudoObjectExpr(CGF&: *this, E, forLValue: false, slot).RV;
7486}
7487
7488LValue CodeGenFunction::EmitPseudoObjectLValue(const PseudoObjectExpr *E) {
7489 return emitPseudoObjectExpr(CGF&: *this, E, forLValue: true, slot: AggValueSlot::ignored()).LV;
7490}
7491
7492void CodeGenFunction::FlattenAccessAndTypeLValue(
7493 LValue Val, SmallVectorImpl<LValue> &AccessList) {
7494
7495 llvm::SmallVector<
7496 std::tuple<LValue, QualType, llvm::SmallVector<llvm::Value *, 4>>, 16>
7497 WorkList;
7498 llvm::IntegerType *IdxTy = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: 32);
7499 WorkList.push_back(Elt: {Val, Val.getType(), {llvm::ConstantInt::get(Ty: IdxTy, V: 0)}});
7500
7501 while (!WorkList.empty()) {
7502 auto [LVal, T, IdxList] = WorkList.pop_back_val();
7503 T = T.getCanonicalType().getUnqualifiedType();
7504 if (const auto *CAT = dyn_cast<ConstantArrayType>(Val&: T)) {
7505 uint64_t Size = CAT->getZExtSize();
7506 for (int64_t I = Size - 1; I > -1; I--) {
7507 llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList;
7508 IdxListCopy.push_back(Elt: llvm::ConstantInt::get(Ty: IdxTy, V: I));
7509 WorkList.emplace_back(Args&: LVal, Args: CAT->getElementType(), Args&: IdxListCopy);
7510 }
7511 } else if (const auto *RT = dyn_cast<RecordType>(Val&: T)) {
7512 const RecordDecl *Record = RT->getDecl()->getDefinitionOrSelf();
7513 assert(!Record->isUnion() && "Union types not supported in flat cast.");
7514
7515 const CXXRecordDecl *CXXD = dyn_cast<CXXRecordDecl>(Val: Record);
7516
7517 llvm::SmallVector<
7518 std::tuple<LValue, QualType, llvm::SmallVector<llvm::Value *, 4>>, 16>
7519 ReverseList;
7520 if (CXXD && CXXD->isStandardLayout())
7521 Record = CXXD->getStandardLayoutBaseWithFields();
7522
7523 // deal with potential base classes
7524 if (CXXD && !CXXD->isStandardLayout()) {
7525 if (CXXD->getNumBases() > 0) {
7526 assert(CXXD->getNumBases() == 1 &&
7527 "HLSL doesn't support multiple inheritance.");
7528 auto Base = CXXD->bases_begin();
7529 llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList;
7530 IdxListCopy.push_back(Elt: llvm::ConstantInt::get(
7531 Ty: IdxTy, V: 0)); // base struct should be at index zero
7532 ReverseList.emplace_back(Args&: LVal, Args: Base->getType(), Args&: IdxListCopy);
7533 }
7534 }
7535
7536 const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(Record);
7537
7538 llvm::Type *LLVMT = ConvertTypeForMem(T);
7539 CharUnits Align = getContext().getTypeAlignInChars(T);
7540 LValue RLValue;
7541 bool createdGEP = false;
7542 for (auto *FD : Record->fields()) {
7543 if (FD->isBitField()) {
7544 if (FD->isUnnamedBitField())
7545 continue;
7546 if (!createdGEP) {
7547 createdGEP = true;
7548 Address GEP = Builder.CreateInBoundsGEP(Addr: LVal.getAddress(), IdxList,
7549 ElementType: LLVMT, Align, Name: "gep");
7550 RLValue = MakeAddrLValue(Addr: GEP, T);
7551 }
7552 LValue FieldLVal = EmitLValueForField(base: RLValue, field: FD, IsInBounds: true);
7553 ReverseList.push_back(Elt: {FieldLVal, FD->getType(), {}});
7554 } else {
7555 llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList;
7556 IdxListCopy.push_back(
7557 Elt: llvm::ConstantInt::get(Ty: IdxTy, V: Layout.getLLVMFieldNo(FD)));
7558 ReverseList.emplace_back(Args&: LVal, Args: FD->getType(), Args&: IdxListCopy);
7559 }
7560 }
7561
7562 std::reverse(first: ReverseList.begin(), last: ReverseList.end());
7563 llvm::append_range(C&: WorkList, R&: ReverseList);
7564 } else if (const auto *VT = dyn_cast<VectorType>(Val&: T)) {
7565 llvm::Type *LLVMT = ConvertTypeForMem(T);
7566 CharUnits Align = getContext().getTypeAlignInChars(T);
7567 Address GEP = Builder.CreateInBoundsGEP(Addr: LVal.getAddress(), IdxList, ElementType: LLVMT,
7568 Align, Name: "vector.gep");
7569 LValue Base = MakeAddrLValue(Addr: GEP, T);
7570 for (unsigned I = 0, E = VT->getNumElements(); I < E; I++) {
7571 llvm::Constant *Idx = llvm::ConstantInt::get(Ty: IdxTy, V: I);
7572 LValue LV =
7573 LValue::MakeVectorElt(vecAddress: Base.getAddress(), Idx, type: VT->getElementType(),
7574 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
7575 AccessList.emplace_back(Args&: LV);
7576 }
7577 } else if (const auto *MT = dyn_cast<ConstantMatrixType>(Val&: T)) {
7578 // Matrices are represented as flat arrays in memory, but has a vector
7579 // value type. So we use ConvertMatrixAddress to convert the address from
7580 // array to vector, and extract elements similar to the vector case above.
7581 // The matrix elements are iterated over in row-major order regardless of
7582 // the memory layout of the matrix.
7583 llvm::Type *LLVMT = ConvertTypeForMem(T);
7584 CharUnits Align = getContext().getTypeAlignInChars(T);
7585 Address GEP = Builder.CreateInBoundsGEP(Addr: LVal.getAddress(), IdxList, ElementType: LLVMT,
7586 Align, Name: "matrix.gep");
7587 LValue Base = MakeAddrLValue(Addr: GEP, T);
7588 Address MatAddr = MaybeConvertMatrixAddress(Addr: Base.getAddress(), CGF&: *this);
7589 unsigned NumRows = MT->getNumRows();
7590 unsigned NumCols = MT->getNumColumns();
7591 bool IsMatrixRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T);
7592 llvm::MatrixBuilder MB(Builder);
7593 for (unsigned Row = 0; Row < MT->getNumRows(); Row++) {
7594 for (unsigned Col = 0; Col < MT->getNumColumns(); Col++) {
7595 llvm::Value *RowIdx = llvm::ConstantInt::get(Ty: IdxTy, V: Row);
7596 llvm::Value *ColIdx = llvm::ConstantInt::get(Ty: IdxTy, V: Col);
7597 llvm::Value *Idx = MB.CreateIndex(RowIdx, ColumnIdx: ColIdx, NumRows, NumCols,
7598 IsMatrixRowMajor);
7599 LValue LV =
7600 LValue::MakeMatrixElt(matAddress: MatAddr, Idx, type: MT->getElementType(),
7601 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
7602 AccessList.emplace_back(Args&: LV);
7603 }
7604 }
7605 } else { // a scalar/builtin type
7606 if (!IdxList.empty()) {
7607 llvm::Type *LLVMT = ConvertTypeForMem(T);
7608 CharUnits Align = getContext().getTypeAlignInChars(T);
7609 Address GEP = Builder.CreateInBoundsGEP(Addr: LVal.getAddress(), IdxList,
7610 ElementType: LLVMT, Align, Name: "gep");
7611 AccessList.emplace_back(Args: MakeAddrLValue(Addr: GEP, T));
7612 } else // must be a bitfield we already created an lvalue for
7613 AccessList.emplace_back(Args&: LVal);
7614 }
7615 }
7616}
7617