1//===--- CGExpr.cpp - Emit LLVM Code from Expressions ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Expr nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ABIInfoImpl.h"
14#include "CGCUDARuntime.h"
15#include "CGCXXABI.h"
16#include "CGCall.h"
17#include "CGCleanup.h"
18#include "CGDebugInfo.h"
19#include "CGHLSLRuntime.h"
20#include "CGObjCRuntime.h"
21#include "CGOpenMPRuntime.h"
22#include "CGRecordLayout.h"
23#include "CodeGenFunction.h"
24#include "CodeGenModule.h"
25#include "CodeGenPGO.h"
26#include "ConstantEmitter.h"
27#include "TargetInfo.h"
28#include "clang/AST/ASTContext.h"
29#include "clang/AST/ASTLambda.h"
30#include "clang/AST/Attr.h"
31#include "clang/AST/DeclObjC.h"
32#include "clang/AST/Expr.h"
33#include "clang/AST/InferAlloc.h"
34#include "clang/AST/MatrixUtils.h"
35#include "clang/AST/NSAPI.h"
36#include "clang/AST/ParentMapContext.h"
37#include "clang/AST/StmtVisitor.h"
38#include "clang/Basic/Builtins.h"
39#include "clang/Basic/CodeGenOptions.h"
40#include "clang/Basic/Module.h"
41#include "clang/Basic/SourceManager.h"
42#include "clang/CodeGenUtils/ExprUtils.h"
43#include "clang/CodeGenUtils/RecordLayoutUtils.h"
44#include "clang/CodeGenUtils/TargetUtils.h"
45#include "llvm/ADT/STLExtras.h"
46#include "llvm/ADT/ScopeExit.h"
47#include "llvm/ADT/StringExtras.h"
48#include "llvm/IR/Constants.h"
49#include "llvm/IR/DataLayout.h"
50#include "llvm/IR/Intrinsics.h"
51#include "llvm/IR/IntrinsicsWebAssembly.h"
52#include "llvm/IR/LLVMContext.h"
53#include "llvm/IR/MDBuilder.h"
54#include "llvm/IR/MatrixBuilder.h"
55#include "llvm/Support/ConvertUTF.h"
56#include "llvm/Support/Endian.h"
57#include "llvm/Support/MathExtras.h"
58#include "llvm/Support/Path.h"
59#include "llvm/Support/xxhash.h"
60#include "llvm/Transforms/Utils/SanitizerStats.h"
61
62#include <numeric>
63#include <optional>
64#include <string>
65
66using namespace clang;
67using namespace CodeGen;
68
69namespace clang {
70// TODO: consider deprecating ClSanitizeGuardChecks; functionality is subsumed
71// by -fsanitize-skip-hot-cutoff
72llvm::cl::opt<bool> ClSanitizeGuardChecks(
73 "ubsan-guard-checks",
74 llvm::cl::desc("Guard UBSAN checks with `llvm.allow.ubsan.check()`."));
75
76} // namespace clang
77
78//===--------------------------------------------------------------------===//
79// Defines for metadata
80//===--------------------------------------------------------------------===//
81
82// Those values are crucial to be the SAME as in ubsan runtime library.
83enum VariableTypeDescriptorKind : uint16_t {
84 /// An integer type.
85 TK_Integer = 0x0000,
86 /// A floating-point type.
87 TK_Float = 0x0001,
88 /// An _BitInt(N) type.
89 TK_BitInt = 0x0002,
90 /// Any other type. The value representation is unspecified.
91 TK_Unknown = 0xffff
92};
93
94//===--------------------------------------------------------------------===//
95// Miscellaneous Helper Methods
96//===--------------------------------------------------------------------===//
97
98static llvm::StringRef GetUBSanTrapForHandler(SanitizerHandler ID) {
99 switch (ID) {
100#define SANITIZER_CHECK(Enum, Name, Version, Msg) \
101 case SanitizerHandler::Enum: \
102 return Msg;
103 LIST_SANITIZER_CHECKS
104#undef SANITIZER_CHECK
105 }
106 llvm_unreachable("unhandled switch case");
107}
108
109/// CreateTempAlloca - This creates a alloca and inserts it into the entry
110/// block.
111RawAddress
112CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits Align,
113 const Twine &Name,
114 llvm::Value *ArraySize) {
115 if (getLangOpts().EmitLogicalPointer) {
116 auto Alloca = Builder.CreateStructuredAlloca(BaseType: Ty, Name);
117 return RawAddress(Alloca, Ty, Align, KnownNonNull);
118 }
119
120 auto *Alloca = CreateTempAlloca(Ty, Name, ArraySize);
121 Alloca->setAlignment(Align.getAsAlign());
122 return RawAddress(Alloca, Ty, Align, KnownNonNull);
123}
124
125RawAddress CodeGenFunction::MaybeCastStackAddressSpace(RawAddress Alloca,
126 LangAS DestLangAS,
127 llvm::Value *ArraySize) {
128
129 llvm::Value *V = Alloca.getPointer();
130 // Alloca always returns a pointer in alloca address space, which may
131 // be different from the type defined by the language. For example,
132 // in C++ the auto variables are in the default address space. Therefore
133 // cast alloca to the default address space when necessary.
134
135 unsigned DestAddrSpace = getContext().getTargetAddressSpace(AS: DestLangAS);
136 if (DestAddrSpace != Alloca.getAddressSpace()) {
137 llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
138 // When ArraySize is nullptr, alloca is inserted at AllocaInsertPt,
139 // otherwise alloca is inserted at the current insertion point of the
140 // builder.
141 if (!ArraySize)
142 Builder.SetInsertPoint(getPostAllocaInsertPoint());
143 V = performAddrSpaceCast(Src: V, DestTy: Builder.getPtrTy(AddrSpace: DestAddrSpace));
144 }
145
146 return RawAddress(V, Alloca.getElementType(), Alloca.getAlignment(),
147 KnownNonNull);
148}
149
150RawAddress CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, LangAS DestLangAS,
151 CharUnits Align, const Twine &Name,
152 llvm::Value *ArraySize,
153 RawAddress *AllocaAddr) {
154 RawAddress Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize);
155 if (AllocaAddr)
156 *AllocaAddr = Alloca;
157 return MaybeCastStackAddressSpace(Alloca, DestLangAS, ArraySize);
158}
159
160/// CreateTempAlloca - This creates an alloca and inserts it into the entry
161/// block if \p ArraySize is nullptr, otherwise inserts it at the current
162/// insertion point of the builder.
163llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
164 const Twine &Name,
165 llvm::Value *ArraySize) {
166 llvm::AllocaInst *Alloca;
167 if (ArraySize)
168 Alloca = Builder.CreateAlloca(Ty, ArraySize, Name);
169 else
170 Alloca =
171 new llvm::AllocaInst(Ty, CGM.getDataLayout().getAllocaAddrSpace(),
172 ArraySize, Name, AllocaInsertPt->getIterator());
173 if (SanOpts.Mask & SanitizerKind::Address) {
174 Alloca->addAnnotationMetadata(Annotations: {"alloca_name_altered", Name.str()});
175 }
176 if (Allocas) {
177 Allocas->Add(I: Alloca);
178 }
179 return Alloca;
180}
181
182/// CreateDefaultAlignTempAlloca - This creates an alloca with the
183/// default alignment of the corresponding LLVM type, which is *not*
184/// guaranteed to be related in any way to the expected alignment of
185/// an AST type that might have been lowered to Ty.
186RawAddress CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty,
187 const Twine &Name) {
188 CharUnits Align =
189 CharUnits::fromQuantity(Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty));
190 return CreateTempAlloca(Ty, DestLangAS: LangAS::Default, Align, Name);
191}
192
193RawAddress CodeGenFunction::CreateIRTempWithoutCast(QualType Ty,
194 const Twine &Name) {
195 CharUnits Align = getContext().getTypeAlignInChars(T: Ty);
196 return CreateTempAllocaWithoutCast(Ty: ConvertType(T: Ty), Align, Name, ArraySize: nullptr);
197}
198
199RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name,
200 RawAddress *Alloca) {
201 // FIXME: Should we prefer the preferred type alignment here?
202 return CreateMemTemp(T: Ty, Align: getContext().getTypeAlignInChars(T: Ty), Name, Alloca);
203}
204
205RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align,
206 const Twine &Name,
207 RawAddress *Alloca) {
208 RawAddress Result =
209 CreateTempAlloca(Ty: ConvertTypeForMem(T: Ty), DestLangAS: Ty.getAddressSpace(), Align, Name,
210 /*ArraySize=*/nullptr, AllocaAddr: Alloca);
211
212 if (Ty->isConstantMatrixType()) {
213 auto *ArrayTy = cast<llvm::ArrayType>(Val: Result.getElementType());
214 auto *ArrayElementTy = ArrayTy->getElementType();
215 auto ArrayElements = ArrayTy->getNumElements();
216 if (getContext().getLangOpts().HLSL) {
217 auto *VectorTy = cast<llvm::FixedVectorType>(Val: ArrayElementTy);
218 ArrayElementTy = VectorTy->getElementType();
219 ArrayElements *= VectorTy->getNumElements();
220 }
221 auto *VectorTy = llvm::FixedVectorType::get(ElementType: ArrayElementTy, NumElts: ArrayElements);
222
223 Result = Address(Result.getPointer(), VectorTy, Result.getAlignment(),
224 KnownNonNull);
225 }
226 return Result;
227}
228
229RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
230 CharUnits Align,
231 const Twine &Name) {
232 return CreateTempAllocaWithoutCast(Ty: ConvertTypeForMem(T: Ty), Align, Name);
233}
234
235RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
236 const Twine &Name) {
237 return CreateMemTempWithoutCast(Ty, Align: getContext().getTypeAlignInChars(T: Ty),
238 Name);
239}
240
241/// EvaluateExprAsBool - Perform the usual unary conversions on the specified
242/// expression and compare the result against zero, returning an Int1Ty value.
243llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
244 PGO->setCurrentStmt(E);
245 if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {
246 llvm::Value *MemPtr = EmitScalarExpr(E);
247 return CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF&: *this, MemPtr, MPT);
248 }
249
250 QualType BoolTy = getContext().BoolTy;
251 SourceLocation Loc = E->getExprLoc();
252 CGFPOptionsRAII FPOptsRAII(*this, E);
253 if (!E->getType()->isAnyComplexType())
254 return EmitScalarConversion(Src: EmitScalarExpr(E), SrcTy: E->getType(), DstTy: BoolTy, Loc);
255
256 return EmitComplexToScalarConversion(Src: EmitComplexExpr(E), SrcTy: E->getType(), DstTy: BoolTy,
257 Loc);
258}
259
260/// EmitIgnoredExpr - Emit code to compute the specified expression,
261/// ignoring the result.
262void CodeGenFunction::EmitIgnoredExpr(const Expr *E) {
263 if (E->isPRValue())
264 return (void)EmitAnyExpr(E, aggSlot: AggValueSlot::ignored(), ignoreResult: true);
265
266 // if this is a bitfield-resulting conditional operator, we can special case
267 // emit this. The normal 'EmitLValue' version of this is particularly
268 // difficult to codegen for, since creating a single "LValue" for two
269 // different sized arguments here is not particularly doable.
270 if (const auto *CondOp = dyn_cast<AbstractConditionalOperator>(
271 Val: E->IgnoreParenNoopCasts(Ctx: getContext()))) {
272 if (CondOp->getObjectKind() == OK_BitField)
273 return EmitIgnoredConditionalOperator(E: CondOp);
274 }
275
276 // Just emit it as an l-value and drop the result.
277 EmitLValue(E);
278}
279
280/// EmitAnyExpr - Emit code to compute the specified expression which
281/// can have any type. The result is returned as an RValue struct.
282/// If this is an aggregate expression, AggSlot indicates where the
283/// result should be returned.
284RValue CodeGenFunction::EmitAnyExpr(const Expr *E,
285 AggValueSlot aggSlot,
286 bool ignoreResult) {
287 switch (getEvaluationKind(T: E->getType())) {
288 case TEK_Scalar:
289 return RValue::get(V: EmitScalarExpr(E, IgnoreResultAssign: ignoreResult));
290 case TEK_Complex:
291 return RValue::getComplex(C: EmitComplexExpr(E, IgnoreReal: ignoreResult, IgnoreImag: ignoreResult));
292 case TEK_Aggregate:
293 if (!ignoreResult && aggSlot.isIgnored())
294 aggSlot = CreateAggTemp(T: E->getType().getUnqualifiedType(), Name: "agg-temp");
295 EmitAggExpr(E, AS: aggSlot);
296 return aggSlot.asRValue();
297 }
298 llvm_unreachable("bad evaluation kind");
299}
300
301/// EmitAnyExprToTemp - Similar to EmitAnyExpr(), however, the result will
302/// always be accessible even if no aggregate location is provided.
303RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E) {
304 AggValueSlot AggSlot = AggValueSlot::ignored();
305
306 if (hasAggregateEvaluationKind(T: E->getType()))
307 AggSlot = CreateAggTemp(T: E->getType(), Name: "agg.tmp");
308 return EmitAnyExpr(E, aggSlot: AggSlot);
309}
310
311/// EmitAnyExprToMem - Evaluate an expression into a given memory
312/// location.
313void CodeGenFunction::EmitAnyExprToMem(const Expr *E,
314 Address Location,
315 Qualifiers Quals,
316 bool IsInit) {
317 // FIXME: This function should take an LValue as an argument.
318 switch (getEvaluationKind(T: E->getType())) {
319 case TEK_Complex:
320 EmitComplexExprIntoLValue(E, dest: MakeAddrLValue(Addr: Location, T: E->getType()),
321 /*isInit*/ false);
322 return;
323
324 case TEK_Aggregate: {
325 EmitAggExpr(E, AS: AggValueSlot::forAddr(addr: Location, quals: Quals,
326 isDestructed: AggValueSlot::IsDestructed_t(IsInit),
327 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
328 isAliased: AggValueSlot::IsAliased_t(!IsInit),
329 mayOverlap: AggValueSlot::MayOverlap));
330 return;
331 }
332
333 case TEK_Scalar: {
334 RValue RV = RValue::get(V: EmitScalarExpr(E, /*Ignore*/ IgnoreResultAssign: false));
335 LValue LV = MakeAddrLValue(Addr: Location, T: E->getType());
336 EmitStoreThroughLValue(Src: RV, Dst: LV);
337 return;
338 }
339 }
340 llvm_unreachable("bad evaluation kind");
341}
342
343void CodeGenFunction::EmitInitializationToLValue(
344 const Expr *E, LValue LV, AggValueSlot::IsZeroed_t IsZeroed) {
345 QualType Type = LV.getType();
346 switch (getEvaluationKind(T: Type)) {
347 case TEK_Complex:
348 EmitComplexExprIntoLValue(E, dest: LV, /*isInit*/ true);
349 return;
350 case TEK_Aggregate:
351 EmitAggExpr(E, AS: AggValueSlot::forLValue(LV, isDestructed: AggValueSlot::IsDestructed,
352 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
353 isAliased: AggValueSlot::IsNotAliased,
354 mayOverlap: AggValueSlot::MayOverlap, isZeroed: IsZeroed));
355 return;
356 case TEK_Scalar:
357 if (LV.isSimple())
358 EmitScalarInit(init: E, /*D=*/nullptr, lvalue: LV, /*Captured=*/capturedByInit: false);
359 else
360 EmitStoreThroughLValue(Src: RValue::get(V: EmitScalarExpr(E)), Dst: LV);
361 return;
362 }
363 llvm_unreachable("bad evaluation kind");
364}
365
366static void
367pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M,
368 const Expr *E, Address ReferenceTemporary) {
369 // Objective-C++ ARC:
370 // If we are binding a reference to a temporary that has ownership, we
371 // need to perform retain/release operations on the temporary.
372 //
373 // FIXME: This should be looking at E, not M.
374 if (auto Lifetime = M->getType().getObjCLifetime()) {
375 switch (Lifetime) {
376 case Qualifiers::OCL_None:
377 case Qualifiers::OCL_ExplicitNone:
378 // Carry on to normal cleanup handling.
379 break;
380
381 case Qualifiers::OCL_Autoreleasing:
382 // Nothing to do; cleaned up by an autorelease pool.
383 return;
384
385 case Qualifiers::OCL_Strong:
386 case Qualifiers::OCL_Weak:
387 switch (StorageDuration Duration = M->getStorageDuration()) {
388 case SD_Static:
389 // Note: we intentionally do not register a cleanup to release
390 // the object on program termination.
391 return;
392
393 case SD_Thread:
394 // FIXME: We should probably register a cleanup in this case.
395 return;
396
397 case SD_Automatic:
398 case SD_FullExpression:
399 CodeGenFunction::Destroyer *Destroy;
400 CleanupKind CleanupKind;
401 if (Lifetime == Qualifiers::OCL_Strong) {
402 const ValueDecl *VD = M->getExtendingDecl();
403 bool Precise = isa_and_nonnull<VarDecl>(Val: VD) &&
404 VD->hasAttr<ObjCPreciseLifetimeAttr>();
405 CleanupKind = CGF.getARCCleanupKind();
406 Destroy = Precise ? &CodeGenFunction::destroyARCStrongPrecise
407 : &CodeGenFunction::destroyARCStrongImprecise;
408 } else {
409 // __weak objects always get EH cleanups; otherwise, exceptions
410 // could cause really nasty crashes instead of mere leaks.
411 CleanupKind = NormalAndEHCleanup;
412 Destroy = &CodeGenFunction::destroyARCWeak;
413 }
414 if (Duration == SD_FullExpression)
415 CGF.pushDestroy(kind: CleanupKind, addr: ReferenceTemporary,
416 type: M->getType(), destroyer: *Destroy,
417 useEHCleanupForArray: CleanupKind & EHCleanup);
418 else
419 CGF.pushLifetimeExtendedDestroy(kind: CleanupKind, addr: ReferenceTemporary,
420 type: M->getType(),
421 destroyer: *Destroy, useEHCleanupForArray: CleanupKind & EHCleanup);
422 return;
423
424 case SD_Dynamic:
425 llvm_unreachable("temporary cannot have dynamic storage duration");
426 }
427 llvm_unreachable("unknown storage duration");
428 }
429 }
430
431 QualType::DestructionKind DK = E->getType().isDestructedType();
432 if (DK != QualType::DK_none) {
433 switch (M->getStorageDuration()) {
434 case SD_Static:
435 case SD_Thread: {
436 CXXDestructorDecl *ReferenceTemporaryDtor = nullptr;
437 if (const auto *ClassDecl =
438 E->getType()->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
439 ClassDecl && !ClassDecl->hasTrivialDestructor())
440 // Get the destructor for the reference temporary.
441 ReferenceTemporaryDtor = ClassDecl->getDestructor();
442
443 if (!ReferenceTemporaryDtor)
444 return;
445
446 // Like in `EmitDeclDestroy`, destructors that return `this` need a helper
447 // if the target does not tolerate the mismatch (e.g. WebAssembly).
448 bool CanRegisterDestructor =
449 !CGF.CGM.getCXXABI().HasThisReturn(
450 GD: GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete)) ||
451 CGF.CGM.getCXXABI().canCallMismatchedFunctionType();
452
453 llvm::FunctionCallee CleanupFn;
454 llvm::Constant *CleanupArg;
455 if (E->getType()->isArrayType() || !CanRegisterDestructor) {
456 CleanupFn = CodeGenFunction(CGF.CGM).generateDestroyHelper(
457 addr: ReferenceTemporary, type: E->getType(), destroyer: CodeGenFunction::destroyCXXObject,
458 useEHCleanupForArray: CGF.getLangOpts().Exceptions,
459 VD: dyn_cast_or_null<VarDecl>(Val: M->getExtendingDecl()));
460 CleanupArg = llvm::Constant::getNullValue(Ty: CGF.Int8PtrTy);
461 } else {
462 CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor(
463 GD: GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete));
464 CleanupArg =
465 cast<llvm::Constant>(Val: ReferenceTemporary.emitRawPointer(CGF));
466 }
467 CGF.CGM.getCXXABI().registerGlobalDtor(
468 CGF, D: *cast<VarDecl>(Val: M->getExtendingDecl()), Dtor: CleanupFn, Addr: CleanupArg);
469 } break;
470 case SD_FullExpression:
471 CGF.pushDestroy(dtorKind: DK, addr: ReferenceTemporary, type: E->getType());
472 break;
473 case SD_Automatic:
474 CGF.pushLifetimeExtendedDestroy(dtorKind: DK, addr: ReferenceTemporary, type: E->getType());
475 break;
476 case SD_Dynamic:
477 llvm_unreachable("temporary cannot have dynamic storage duration");
478 }
479 }
480}
481
482static RawAddress createReferenceTemporary(CodeGenFunction &CGF,
483 const MaterializeTemporaryExpr *M,
484 const Expr *Inner,
485 RawAddress *Alloca = nullptr) {
486 switch (M->getStorageDuration()) {
487 case SD_FullExpression:
488 case SD_Automatic: {
489 // If we have a constant temporary array or record try to promote it into a
490 // constant global under the same rules a normal constant would've been
491 // promoted. This is easier on the optimizer and generally emits fewer
492 // instructions.
493 QualType Ty = Inner->getType();
494 if (CGF.CGM.getCodeGenOpts().MergeAllConstants &&
495 (Ty->isArrayType() || Ty->isRecordType()) &&
496 Ty.isConstantStorage(Ctx: CGF.getContext(), ExcludeCtor: true, ExcludeDtor: false))
497 if (auto Init = ConstantEmitter(CGF).tryEmitAbstract(E: Inner, T: Ty)) {
498 auto AS = CGF.CGM.GetGlobalConstantAddressSpace();
499 auto *GV = new llvm::GlobalVariable(
500 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
501 llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp", nullptr,
502 llvm::GlobalValue::NotThreadLocal,
503 CGF.getContext().getTargetAddressSpace(AS));
504 CharUnits alignment = CGF.getContext().getTypeAlignInChars(T: Ty);
505 GV->setAlignment(alignment.getAsAlign());
506 llvm::Constant *C = GV;
507 if (AS != Ty.getAddressSpace())
508 C = CGF.CGM.performAddrSpaceCast(
509 Src: GV, DestTy: llvm::PointerType::get(C&: CGF.getLLVMContext(),
510 AddressSpace: CGF.getContext().getTargetAddressSpace(
511 AS: Ty.getAddressSpace())));
512 // FIXME: Should we put the new global into a COMDAT?
513 return RawAddress(C, GV->getValueType(), alignment);
514 }
515 return CGF.CreateMemTemp(Ty, Name: "ref.tmp", Alloca);
516 }
517 case SD_Thread:
518 case SD_Static:
519 return CGF.CGM.GetAddrOfGlobalTemporary(E: M, Inner);
520
521 case SD_Dynamic:
522 llvm_unreachable("temporary can't have dynamic storage duration");
523 }
524 llvm_unreachable("unknown storage duration");
525}
526
527LValue CodeGenFunction::
528EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
529 const Expr *E = M->getSubExpr();
530
531 assert((!M->getExtendingDecl() || !isa<VarDecl>(M->getExtendingDecl()) ||
532 !cast<VarDecl>(M->getExtendingDecl())->isARCPseudoStrong()) &&
533 "Reference should never be pseudo-strong!");
534
535 // FIXME: ideally this would use EmitAnyExprToMem, however, we cannot do so
536 // as that will cause the lifetime adjustment to be lost for ARC
537 auto ownership = M->getType().getObjCLifetime();
538 if (ownership != Qualifiers::OCL_None &&
539 ownership != Qualifiers::OCL_ExplicitNone) {
540 RawAddress Object = createReferenceTemporary(CGF&: *this, M, Inner: E);
541 if (auto *Var = dyn_cast<llvm::GlobalVariable>(Val: Object.getPointer())) {
542 llvm::Type *Ty = ConvertTypeForMem(T: E->getType());
543 Object = Object.withElementType(ElemTy: Ty);
544
545 // createReferenceTemporary will promote the temporary to a global with a
546 // constant initializer if it can. It can only do this to a value of
547 // ARC-manageable type if the value is global and therefore "immune" to
548 // ref-counting operations. Therefore we have no need to emit either a
549 // dynamic initialization or a cleanup and we can just return the address
550 // of the temporary.
551 if (Var->hasInitializer())
552 return MakeAddrLValue(Addr: Object, T: M->getType(), Source: AlignmentSource::Decl);
553
554 Var->setInitializer(CGM.EmitNullConstant(T: E->getType()));
555 }
556 LValue RefTempDst = MakeAddrLValue(Addr: Object, T: M->getType(),
557 Source: AlignmentSource::Decl);
558
559 switch (getEvaluationKind(T: E->getType())) {
560 default: llvm_unreachable("expected scalar or aggregate expression");
561 case TEK_Scalar:
562 EmitScalarInit(init: E, D: M->getExtendingDecl(), lvalue: RefTempDst, capturedByInit: false);
563 break;
564 case TEK_Aggregate: {
565 EmitAggExpr(E, AS: AggValueSlot::forAddr(addr: Object,
566 quals: E->getType().getQualifiers(),
567 isDestructed: AggValueSlot::IsDestructed,
568 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
569 isAliased: AggValueSlot::IsNotAliased,
570 mayOverlap: AggValueSlot::DoesNotOverlap));
571 break;
572 }
573 }
574
575 pushTemporaryCleanup(CGF&: *this, M, E, ReferenceTemporary: Object);
576 return RefTempDst;
577 }
578
579 SmallVector<const Expr *, 2> CommaLHSs;
580 SmallVector<SubobjectAdjustment, 2> Adjustments;
581 E = E->skipRValueSubobjectAdjustments(CommaLHS&: CommaLHSs, Adjustments);
582
583 for (const auto &Ignored : CommaLHSs)
584 EmitIgnoredExpr(E: Ignored);
585
586 if (const auto *opaque = dyn_cast<OpaqueValueExpr>(Val: E)) {
587 if (opaque->getType()->isRecordType()) {
588 assert(Adjustments.empty());
589 return EmitOpaqueValueLValue(e: opaque);
590 }
591 }
592
593 // Create and initialize the reference temporary.
594 RawAddress Alloca = Address::invalid();
595 RawAddress Object = createReferenceTemporary(CGF&: *this, M, Inner: E, Alloca: &Alloca);
596 if (auto *Var = dyn_cast<llvm::GlobalVariable>(
597 Val: Object.getPointer()->stripPointerCasts())) {
598 llvm::Type *TemporaryType = ConvertTypeForMem(T: E->getType());
599 Object = Object.withElementType(ElemTy: TemporaryType);
600 // If the temporary is a global and has a constant initializer or is a
601 // constant temporary that we promoted to a global, we may have already
602 // initialized it.
603 if (!Var->hasInitializer()) {
604 Var->setInitializer(CGM.EmitNullConstant(T: E->getType()));
605 QualType RefType = M->getType().withoutLocalFastQualifiers();
606 if (RefType.getPointerAuth()) {
607 // Use the qualifier of the reference temporary to sign the pointer.
608 LValue LV = MakeRawAddrLValue(V: Object.getPointer(), T: RefType,
609 Alignment: Object.getAlignment());
610 EmitScalarInit(init: E, D: M->getExtendingDecl(), lvalue: LV, capturedByInit: false);
611 } else {
612 EmitAnyExprToMem(E, Location: Object, Quals: Qualifiers(), /*IsInit*/ true);
613 }
614 }
615 } else {
616 switch (M->getStorageDuration()) {
617 case SD_Automatic:
618 if (EmitLifetimeStart(Addr: Alloca.getPointer())) {
619 pushCleanupAfterFullExpr<CallLifetimeEnd>(Kind: NormalEHLifetimeMarker,
620 A: Alloca);
621 }
622 break;
623
624 case SD_FullExpression: {
625 if (!ShouldEmitLifetimeMarkers)
626 break;
627
628 // Avoid creating a conditional cleanup just to hold an llvm.lifetime.end
629 // marker. Instead, start the lifetime of a conditional temporary earlier
630 // so that it's unconditional. Don't do this with sanitizers which need
631 // more precise lifetime marks. However when inside an "await.suspend"
632 // block, we should always avoid conditional cleanup because it creates
633 // boolean marker that lives across await_suspend, which can destroy coro
634 // frame.
635 ConditionalEvaluation *OldConditional = nullptr;
636 CGBuilderTy::InsertPoint OldIP;
637 if (isInConditionalBranch() && !E->getType().isDestructedType() &&
638 ((!SanOpts.has(K: SanitizerKind::HWAddress) &&
639 !SanOpts.has(K: SanitizerKind::Memory) &&
640 !SanOpts.has(K: SanitizerKind::MemtagStack) &&
641 !CGM.getCodeGenOpts().SanitizeAddressUseAfterScope) ||
642 inSuspendBlock())) {
643 OldConditional = OutermostConditional;
644 OutermostConditional = nullptr;
645
646 OldIP = Builder.saveIP();
647 llvm::BasicBlock *Block = OldConditional->getStartingBlock();
648 Builder.restoreIP(IP: Block->back().getIterator());
649 }
650
651 if (EmitLifetimeStart(Addr: Alloca.getPointer())) {
652 pushFullExprCleanup<CallLifetimeEnd>(kind: NormalEHLifetimeMarker, A: Alloca);
653 }
654
655 if (OldConditional) {
656 OutermostConditional = OldConditional;
657 Builder.restoreIP(IP: OldIP);
658 }
659 break;
660 }
661
662 default:
663 break;
664 }
665 EmitAnyExprToMem(E, Location: Object, Quals: Qualifiers(), /*IsInit*/true);
666 }
667 pushTemporaryCleanup(CGF&: *this, M, E, ReferenceTemporary: Object);
668
669 // Perform derived-to-base casts and/or field accesses, to get from the
670 // temporary object we created (and, potentially, for which we extended
671 // the lifetime) to the subobject we're binding the reference to.
672 for (SubobjectAdjustment &Adjustment : llvm::reverse(C&: Adjustments)) {
673 switch (Adjustment.Kind) {
674 case SubobjectAdjustment::DerivedToBaseAdjustment:
675 Object =
676 GetAddressOfBaseClass(Value: Object, Derived: Adjustment.DerivedToBase.DerivedClass,
677 PathBegin: Adjustment.DerivedToBase.BasePath->path_begin(),
678 PathEnd: Adjustment.DerivedToBase.BasePath->path_end(),
679 /*NullCheckValue=*/ false, Loc: E->getExprLoc());
680 break;
681
682 case SubobjectAdjustment::FieldAdjustment: {
683 LValue LV = MakeAddrLValue(Addr: Object, T: E->getType(), Source: AlignmentSource::Decl);
684 LV = EmitLValueForField(Base: LV, Field: Adjustment.Field);
685 assert(LV.isSimple() &&
686 "materialized temporary field is not a simple lvalue");
687 Object = LV.getAddress();
688 break;
689 }
690
691 case SubobjectAdjustment::MemberPointerAdjustment: {
692 llvm::Value *Ptr = EmitScalarExpr(E: Adjustment.Ptr.RHS);
693 Object = EmitCXXMemberDataPointerAddress(
694 E, base: Object, memberPtr: Ptr, memberPtrType: Adjustment.Ptr.MPT, /*IsInBounds=*/true);
695 break;
696 }
697 }
698 }
699
700 return MakeAddrLValue(Addr: Object, T: M->getType(), Source: AlignmentSource::Decl);
701}
702
703RValue
704CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E) {
705 // Emit the expression as an lvalue.
706 LValue LV = EmitLValue(E);
707 assert(LV.isSimple());
708 llvm::Value *Value = LV.getPointer(CGF&: *this);
709
710 if (sanitizePerformTypeCheck() && !E->getType()->isFunctionType()) {
711 // C++11 [dcl.ref]p5 (as amended by core issue 453):
712 // If a glvalue to which a reference is directly bound designates neither
713 // an existing object or function of an appropriate type nor a region of
714 // storage of suitable size and alignment to contain an object of the
715 // reference's type, the behavior is undefined.
716 QualType Ty = E->getType();
717 EmitTypeCheck(TCK: TCK_ReferenceBinding, Loc: E->getExprLoc(), V: Value, Type: Ty);
718 }
719
720 return RValue::get(V: Value);
721}
722
723
724/// getAccessedFieldNo - Given an encoded value and a result number, return the
725/// input field number being accessed.
726unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx,
727 const llvm::Constant *Elts) {
728 return cast<llvm::ConstantInt>(Val: Elts->getAggregateElement(Elt: Idx))
729 ->getZExtValue();
730}
731
732static llvm::Value *emitHashMix(CGBuilderTy &Builder, llvm::Value *Acc,
733 llvm::Value *Ptr) {
734 llvm::Value *A0 =
735 Builder.CreateMul(LHS: Ptr, RHS: Builder.getInt64(C: 0xbf58476d1ce4e5b9u));
736 llvm::Value *A1 =
737 Builder.CreateXor(LHS: A0, RHS: Builder.CreateLShr(LHS: A0, RHS: Builder.getInt64(C: 31)));
738 return Builder.CreateXor(LHS: Acc, RHS: A1);
739}
740
741bool CodeGenFunction::isNullPointerAllowed(TypeCheckKind TCK) {
742 return TCK == TCK_DowncastPointer || TCK == TCK_Upcast ||
743 TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation;
744}
745
746bool CodeGenFunction::isVptrCheckRequired(TypeCheckKind TCK, QualType Ty) {
747 CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
748 return (RD && RD->hasDefinition() && RD->isDynamicClass()) &&
749 (TCK == TCK_MemberAccess || TCK == TCK_MemberCall ||
750 TCK == TCK_DowncastPointer || TCK == TCK_DowncastReference ||
751 TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation);
752}
753
754bool CodeGenFunction::sanitizePerformTypeCheck() const {
755 return SanOpts.has(K: SanitizerKind::Null) ||
756 SanOpts.has(K: SanitizerKind::Alignment) ||
757 SanOpts.has(K: SanitizerKind::ObjectSize) ||
758 SanOpts.has(K: SanitizerKind::Vptr);
759}
760
761void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
762 llvm::Value *Ptr, QualType Ty,
763 CharUnits Alignment,
764 SanitizerSet SkippedChecks,
765 llvm::Value *ArraySize) {
766 if (!sanitizePerformTypeCheck())
767 return;
768
769 // Don't check pointers outside the default address space. The null check
770 // isn't correct, the object-size check isn't supported by LLVM, and we can't
771 // communicate the addresses to the runtime handler for the vptr check.
772 if (Ptr->getType()->getPointerAddressSpace())
773 return;
774
775 // Don't check pointers to volatile data. The behavior here is implementation-
776 // defined.
777 if (Ty.isVolatileQualified())
778 return;
779
780 // Quickly determine whether we have a pointer to an alloca. It's possible
781 // to skip null checks, and some alignment checks, for these pointers. This
782 // can reduce compile-time significantly.
783 auto PtrToAlloca = dyn_cast<llvm::AllocaInst>(Val: Ptr->stripPointerCasts());
784
785 llvm::Value *IsNonNull = nullptr;
786 bool IsGuaranteedNonNull =
787 SkippedChecks.has(K: SanitizerKind::Null) || PtrToAlloca;
788
789 llvm::BasicBlock *Done = nullptr;
790 bool DoneViaNullSanitize = false;
791
792 {
793 auto CheckHandler = SanitizerHandler::TypeMismatch;
794 SanitizerDebugLocation SanScope(this,
795 {SanitizerKind::SO_Null,
796 SanitizerKind::SO_ObjectSize,
797 SanitizerKind::SO_Alignment},
798 CheckHandler);
799
800 SmallVector<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>, 3>
801 Checks;
802
803 llvm::Value *True = llvm::ConstantInt::getTrue(Context&: getLLVMContext());
804 bool AllowNullPointers = isNullPointerAllowed(TCK);
805 if ((SanOpts.has(K: SanitizerKind::Null) || AllowNullPointers) &&
806 !IsGuaranteedNonNull) {
807 // The glvalue must not be an empty glvalue.
808 IsNonNull = Builder.CreateIsNotNull(Arg: Ptr);
809
810 // The IR builder can constant-fold the null check if the pointer points
811 // to a constant.
812 IsGuaranteedNonNull = IsNonNull == True;
813
814 // Skip the null check if the pointer is known to be non-null.
815 if (!IsGuaranteedNonNull) {
816 if (AllowNullPointers) {
817 // When performing pointer casts, it's OK if the value is null.
818 // Skip the remaining checks in that case.
819 Done = createBasicBlock(name: "null");
820 DoneViaNullSanitize = true;
821 llvm::BasicBlock *Rest = createBasicBlock(name: "not.null");
822 Builder.CreateCondBr(Cond: IsNonNull, True: Rest, False: Done);
823 EmitBlock(BB: Rest);
824 } else {
825 Checks.push_back(Elt: std::make_pair(x&: IsNonNull, y: SanitizerKind::SO_Null));
826 }
827 }
828 }
829
830 if (SanOpts.has(K: SanitizerKind::ObjectSize) &&
831 !SkippedChecks.has(K: SanitizerKind::ObjectSize) &&
832 !Ty->isIncompleteType()) {
833 uint64_t TySize = CGM.getMinimumObjectSize(Ty).getQuantity();
834 llvm::Value *Size = llvm::ConstantInt::get(Ty: IntPtrTy, V: TySize);
835 if (ArraySize)
836 Size = Builder.CreateMul(LHS: Size, RHS: ArraySize);
837
838 // Degenerate case: new X[0] does not need an objectsize check.
839 llvm::Constant *ConstantSize = dyn_cast<llvm::Constant>(Val: Size);
840 if (!ConstantSize || !ConstantSize->isNullValue()) {
841 // The glvalue must refer to a large enough storage region.
842 // FIXME: If Address Sanitizer is enabled, insert dynamic
843 // instrumentation
844 // to check this.
845 // FIXME: Get object address space
846 llvm::Type *Tys[2] = {IntPtrTy, Int8PtrTy};
847 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::objectsize, Tys);
848 llvm::Value *Min = Builder.getFalse();
849 llvm::Value *NullIsUnknown = Builder.getFalse();
850 llvm::Value *Dynamic = Builder.getFalse();
851 llvm::Value *LargeEnough = Builder.CreateICmpUGE(
852 LHS: Builder.CreateCall(Callee: F, Args: {Ptr, Min, NullIsUnknown, Dynamic}), RHS: Size);
853 Checks.push_back(
854 Elt: std::make_pair(x&: LargeEnough, y: SanitizerKind::SO_ObjectSize));
855 }
856 }
857
858 llvm::MaybeAlign AlignVal;
859 llvm::Value *PtrAsInt = nullptr;
860
861 if (SanOpts.has(K: SanitizerKind::Alignment) &&
862 !SkippedChecks.has(K: SanitizerKind::Alignment)) {
863 AlignVal = Alignment.getAsMaybeAlign();
864 if (!Ty->isIncompleteType() && !AlignVal)
865 AlignVal = CGM.getNaturalTypeAlignment(T: Ty, BaseInfo: nullptr, TBAAInfo: nullptr,
866 /*ForPointeeType=*/forPointeeType: true)
867 .getAsMaybeAlign();
868
869 // The glvalue must be suitably aligned.
870 if (AlignVal && *AlignVal > llvm::Align(1) &&
871 (!PtrToAlloca || PtrToAlloca->getAlign() < *AlignVal)) {
872 PtrAsInt = Builder.CreatePtrToInt(V: Ptr, DestTy: IntPtrTy);
873 llvm::Value *Align = Builder.CreateAnd(
874 LHS: PtrAsInt, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, V: AlignVal->value() - 1));
875 llvm::Value *Aligned =
876 Builder.CreateICmpEQ(LHS: Align, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, V: 0));
877 if (Aligned != True)
878 Checks.push_back(
879 Elt: std::make_pair(x&: Aligned, y: SanitizerKind::SO_Alignment));
880 }
881 }
882
883 if (Checks.size() > 0) {
884 llvm::Constant *StaticData[] = {
885 EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(T: Ty),
886 llvm::ConstantInt::get(Ty: Int8Ty, V: AlignVal ? llvm::Log2(A: *AlignVal) : 1),
887 llvm::ConstantInt::get(Ty: Int8Ty, V: TCK)};
888 EmitCheck(Checked: Checks, Check: CheckHandler, StaticArgs: StaticData, DynamicArgs: PtrAsInt ? PtrAsInt : Ptr);
889 }
890 }
891
892 // If possible, check that the vptr indicates that there is a subobject of
893 // type Ty at offset zero within this object.
894 //
895 // C++11 [basic.life]p5,6:
896 // [For storage which does not refer to an object within its lifetime]
897 // The program has undefined behavior if:
898 // -- the [pointer or glvalue] is used to access a non-static data member
899 // or call a non-static member function
900 if (SanOpts.has(K: SanitizerKind::Vptr) &&
901 !SkippedChecks.has(K: SanitizerKind::Vptr) && isVptrCheckRequired(TCK, Ty)) {
902 SanitizerDebugLocation SanScope(this, {SanitizerKind::SO_Vptr},
903 SanitizerHandler::DynamicTypeCacheMiss);
904
905 // Ensure that the pointer is non-null before loading it. If there is no
906 // compile-time guarantee, reuse the run-time null check or emit a new one.
907 if (!IsGuaranteedNonNull) {
908 if (!IsNonNull)
909 IsNonNull = Builder.CreateIsNotNull(Arg: Ptr);
910 if (!Done)
911 Done = createBasicBlock(name: "vptr.null");
912 llvm::BasicBlock *VptrNotNull = createBasicBlock(name: "vptr.not.null");
913 Builder.CreateCondBr(Cond: IsNonNull, True: VptrNotNull, False: Done);
914 EmitBlock(BB: VptrNotNull);
915 }
916
917 // Compute a deterministic hash of the mangled name of the type.
918 SmallString<64> MangledName;
919 llvm::raw_svector_ostream Out(MangledName);
920 CGM.getCXXABI().getMangleContext().mangleCXXRTTI(T: Ty.getUnqualifiedType(),
921 Out);
922
923 // Contained in NoSanitizeList based on the mangled type.
924 if (!CGM.getContext().getNoSanitizeList().containsType(Mask: SanitizerKind::Vptr,
925 MangledTypeName: Out.str())) {
926 // Load the vptr, and mix it with TypeHash.
927 llvm::Value *TypeHash =
928 llvm::ConstantInt::get(Ty: Int64Ty, V: xxh3_64bits(data: Out.str()));
929
930 llvm::Type *VPtrTy = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: 0);
931 Address VPtrAddr(Ptr, IntPtrTy, getPointerAlign());
932 llvm::Value *VPtrVal = GetVTablePtr(This: VPtrAddr, VTableTy: VPtrTy,
933 VTableClass: Ty->getAsCXXRecordDecl(),
934 AuthMode: VTableAuthMode::UnsafeUbsanStrip);
935 VPtrVal = Builder.CreateBitOrPointerCast(V: VPtrVal, DestTy: IntPtrTy);
936
937 llvm::Value *Hash =
938 emitHashMix(Builder, Acc: TypeHash, Ptr: Builder.CreateZExt(V: VPtrVal, DestTy: Int64Ty));
939 Hash = Builder.CreateTrunc(V: Hash, DestTy: IntPtrTy);
940
941 // Look the hash up in our cache.
942 const int CacheSize = 128;
943 llvm::Type *HashTable = llvm::ArrayType::get(ElementType: IntPtrTy, NumElements: CacheSize);
944 llvm::Value *Cache = CGM.CreateRuntimeVariable(Ty: HashTable,
945 Name: "__ubsan_vptr_type_cache");
946 llvm::Value *Slot = Builder.CreateAnd(LHS: Hash,
947 RHS: llvm::ConstantInt::get(Ty: IntPtrTy,
948 V: CacheSize-1));
949 llvm::Value *Indices[] = { Builder.getInt32(C: 0), Slot };
950 llvm::Value *CacheVal = Builder.CreateAlignedLoad(
951 Ty: IntPtrTy, Addr: Builder.CreateInBoundsGEP(Ty: HashTable, Ptr: Cache, IdxList: Indices),
952 Align: getPointerAlign());
953
954 // If the hash isn't in the cache, call a runtime handler to perform the
955 // hard work of checking whether the vptr is for an object of the right
956 // type. This will either fill in the cache and return, or produce a
957 // diagnostic.
958 llvm::Value *EqualHash = Builder.CreateICmpEQ(LHS: CacheVal, RHS: Hash);
959 llvm::Constant *StaticData[] = {
960 EmitCheckSourceLocation(Loc),
961 EmitCheckTypeDescriptor(T: Ty),
962 CGM.GetAddrOfRTTIDescriptor(Ty: Ty.getUnqualifiedType()),
963 llvm::ConstantInt::get(Ty: Int8Ty, V: TCK)
964 };
965 llvm::Value *DynamicData[] = { Ptr, Hash };
966 EmitCheck(Checked: std::make_pair(x&: EqualHash, y: SanitizerKind::SO_Vptr),
967 Check: SanitizerHandler::DynamicTypeCacheMiss, StaticArgs: StaticData,
968 DynamicArgs: DynamicData);
969 }
970 }
971
972 if (Done) {
973 SanitizerDebugLocation SanScope(
974 this,
975 {DoneViaNullSanitize ? SanitizerKind::SO_Null : SanitizerKind::SO_Vptr},
976 DoneViaNullSanitize ? SanitizerHandler::TypeMismatch
977 : SanitizerHandler::DynamicTypeCacheMiss);
978 Builder.CreateBr(Dest: Done);
979 EmitBlock(BB: Done);
980 }
981}
982
983llvm::Value *CodeGenFunction::LoadPassedObjectSize(const Expr *E,
984 QualType EltTy) {
985 ASTContext &C = getContext();
986 uint64_t EltSize = C.getTypeSizeInChars(T: EltTy).getQuantity();
987 if (!EltSize)
988 return nullptr;
989
990 auto *ArrayDeclRef = dyn_cast<DeclRefExpr>(Val: E->IgnoreParenImpCasts());
991 if (!ArrayDeclRef)
992 return nullptr;
993
994 auto *ParamDecl = dyn_cast<ParmVarDecl>(Val: ArrayDeclRef->getDecl());
995 if (!ParamDecl)
996 return nullptr;
997
998 auto *POSAttr = ParamDecl->getAttr<PassObjectSizeAttr>();
999 if (!POSAttr)
1000 return nullptr;
1001
1002 // Don't load the size if it's a lower bound.
1003 int POSType = POSAttr->getType();
1004 if (POSType != 0 && POSType != 1)
1005 return nullptr;
1006
1007 // Find the implicit size parameter.
1008 auto PassedSizeIt = SizeArguments.find(Val: ParamDecl);
1009 if (PassedSizeIt == SizeArguments.end())
1010 return nullptr;
1011
1012 const ImplicitParamDecl *PassedSizeDecl = PassedSizeIt->second;
1013 assert(LocalDeclMap.count(PassedSizeDecl) && "Passed size not loadable");
1014 Address AddrOfSize = LocalDeclMap.find(Val: PassedSizeDecl)->second;
1015 llvm::Value *SizeInBytes = EmitLoadOfScalar(Addr: AddrOfSize, /*Volatile=*/false,
1016 Ty: C.getSizeType(), Loc: E->getExprLoc());
1017 llvm::Value *SizeOfElement =
1018 llvm::ConstantInt::get(Ty: SizeInBytes->getType(), V: EltSize);
1019 return Builder.CreateUDiv(LHS: SizeInBytes, RHS: SizeOfElement);
1020}
1021
1022/// If Base is known to point to the start of an array, return the length of
1023/// that array. Return 0 if the length cannot be determined.
1024static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF,
1025 const Expr *Base,
1026 QualType &IndexedType,
1027 LangOptions::StrictFlexArraysLevelKind
1028 StrictFlexArraysLevel) {
1029 // For the vector indexing extension, the bound is the number of elements.
1030 if (const VectorType *VT = Base->getType()->getAs<VectorType>()) {
1031 IndexedType = Base->getType();
1032 return CGF.Builder.getInt32(C: VT->getNumElements());
1033 }
1034
1035 Base = Base->IgnoreParens();
1036
1037 if (const auto *CE = dyn_cast<CastExpr>(Val: Base)) {
1038 if (CE->getCastKind() == CK_ArrayToPointerDecay &&
1039 !CE->getSubExpr()->isFlexibleArrayMemberLike(Context: CGF.getContext(),
1040 StrictFlexArraysLevel)) {
1041 CodeGenFunction::SanitizerScope SanScope(&CGF);
1042
1043 IndexedType = CE->getSubExpr()->getType();
1044 const ArrayType *AT = IndexedType->castAsArrayTypeUnsafe();
1045 if (const auto *CAT = dyn_cast<ConstantArrayType>(Val: AT))
1046 return CGF.Builder.getInt(AI: CAT->getSize());
1047
1048 if (const auto *VAT = dyn_cast<VariableArrayType>(Val: AT))
1049 return CGF.getVLASize(vla: VAT).NumElts;
1050 // Ignore pass_object_size here. It's not applicable on decayed pointers.
1051 }
1052 }
1053
1054 CodeGenFunction::SanitizerScope SanScope(&CGF);
1055
1056 QualType EltTy{Base->getType()->getPointeeOrArrayElementType(), 0};
1057 if (llvm::Value *POS = CGF.LoadPassedObjectSize(E: Base, EltTy)) {
1058 IndexedType = Base->getType();
1059 return POS;
1060 }
1061
1062 return nullptr;
1063}
1064
1065/// Returns true if \p Field is reachable from \p RD either as a direct field or
1066/// through a chain of nested record fields (including anonymous
1067/// structs/unions). This mirrors the GEP path that getGEPIndicesToField builds,
1068/// and is used to identify the right anchor expression in Base.
1069static bool RecordContainsField(const RecordDecl *RD, const FieldDecl *Field) {
1070 for (const FieldDecl *FD : RD->fields()) {
1071 if (FD == Field)
1072 return true;
1073 QualType Ty = FD->getType();
1074 if (Ty->isRecordType())
1075 if (RecordContainsField(RD: Ty->getAsRecordDecl(), Field))
1076 return true;
1077 }
1078 return false;
1079}
1080
1081namespace {
1082
1083/// \p StructAccessBase returns the base \p Expr of a field access. It returns
1084/// either a \p DeclRefExpr, representing the base pointer to the struct, i.e.:
1085///
1086/// p in p-> a.b.c
1087///
1088/// or a \p MemberExpr, if the \p MemberExpr has the \p RecordDecl we're
1089/// looking for:
1090///
1091/// struct s {
1092/// struct s *ptr;
1093/// int count;
1094/// char array[] __attribute__((counted_by(count)));
1095/// };
1096///
1097/// If we have an expression like \p p->ptr->array[index], we want the
1098/// \p MemberExpr for \p p->ptr instead of \p p.
1099class StructAccessBase
1100 : public ConstStmtVisitor<StructAccessBase, const Expr *> {
1101 /// The count field we're navigating to. We stop at the innermost expression
1102 /// whose struct type transitively contains this field, so that
1103 /// getGEPIndicesToField can navigate from that struct down to it.
1104 const FieldDecl *CountDecl;
1105
1106 /// Returns true if E's record type (or pointee record type) transitively
1107 /// contains CountDecl. Handles both direct containment and nested structs,
1108 /// so we don't need a pre-computed RD from the caller.
1109 bool IsExpectedRecordDecl(const Expr *E) const {
1110 QualType Ty = E->getType();
1111 if (Ty->isPointerType())
1112 Ty = Ty->getPointeeType();
1113 const RecordDecl *RD = Ty->getAsRecordDecl();
1114 return RD && RecordContainsField(RD, Field: CountDecl);
1115 }
1116
1117public:
1118 StructAccessBase(const FieldDecl *CountDecl) : CountDecl(CountDecl) {}
1119
1120 //===--------------------------------------------------------------------===//
1121 // Visitor Methods
1122 //===--------------------------------------------------------------------===//
1123
1124 // NOTE: If we build C++ support for counted_by, then we'll have to handle
1125 // horrors like this:
1126 //
1127 // struct S {
1128 // int x, y;
1129 // int blah[] __attribute__((counted_by(x)));
1130 // } s;
1131 //
1132 // int foo(int index, int val) {
1133 // int (S::*IHatePMDs)[] = &S::blah;
1134 // (s.*IHatePMDs)[index] = val;
1135 // }
1136
1137 const Expr *Visit(const Expr *E) {
1138 return ConstStmtVisitor<StructAccessBase, const Expr *>::Visit(S: E);
1139 }
1140
1141 const Expr *VisitStmt(const Stmt *S) { return nullptr; }
1142
1143 // These are the types we expect to return (in order of most to least
1144 // likely):
1145 //
1146 // 1. DeclRefExpr - This is the expression for the base of the structure.
1147 // It's exactly what we want to build an access to the \p counted_by
1148 // field.
1149 // 2. MemberExpr - This is the expression that has the same \p RecordDecl
1150 // as the flexble array member's lexical enclosing \p RecordDecl. This
1151 // allows us to catch things like: "p->p->array"
1152 // 3. CompoundLiteralExpr - This is for people who create something
1153 // heretical like (struct foo has a flexible array member):
1154 //
1155 // (struct foo){ 1, 2 }.blah[idx];
1156 const Expr *VisitDeclRefExpr(const DeclRefExpr *E) {
1157 return IsExpectedRecordDecl(E) ? E : nullptr;
1158 }
1159 const Expr *VisitMemberExpr(const MemberExpr *E) {
1160 if (IsExpectedRecordDecl(E) && E->isArrow())
1161 return E;
1162 const Expr *Res = Visit(E: E->getBase());
1163 return !Res && IsExpectedRecordDecl(E) ? E : Res;
1164 }
1165 const Expr *VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) {
1166 return IsExpectedRecordDecl(E) ? E : nullptr;
1167 }
1168 const Expr *VisitCallExpr(const CallExpr *E) {
1169 return IsExpectedRecordDecl(E) ? E : nullptr;
1170 }
1171
1172 const Expr *VisitArraySubscriptExpr(const ArraySubscriptExpr *E) {
1173 if (IsExpectedRecordDecl(E))
1174 return E;
1175 return Visit(E: E->getBase());
1176 }
1177 const Expr *VisitCastExpr(const CastExpr *E) {
1178 if (E->getCastKind() == CK_LValueToRValue)
1179 return IsExpectedRecordDecl(E) ? E : nullptr;
1180 return Visit(E: E->getSubExpr());
1181 }
1182 const Expr *VisitParenExpr(const ParenExpr *E) {
1183 return Visit(E: E->getSubExpr());
1184 }
1185 const Expr *VisitUnaryAddrOf(const UnaryOperator *E) {
1186 return Visit(E: E->getSubExpr());
1187 }
1188 const Expr *VisitUnaryDeref(const UnaryOperator *E) {
1189 return Visit(E: E->getSubExpr());
1190 }
1191};
1192
1193} // end anonymous namespace
1194
1195using RecIndicesTy = SmallVector<llvm::Value *, 8>;
1196
1197static bool getGEPIndicesToField(CodeGenFunction &CGF, const RecordDecl *RD,
1198 const FieldDecl *Field,
1199 RecIndicesTy &Indices) {
1200 const CGRecordLayout &Layout = CGF.CGM.getTypes().getCGRecordLayout(RD);
1201 int64_t FieldNo = -1;
1202 for (const FieldDecl *FD : RD->fields()) {
1203 if (!Layout.containsFieldDecl(FD))
1204 // This could happen if the field has a struct type that's empty. I don't
1205 // know why either.
1206 continue;
1207
1208 FieldNo = Layout.getLLVMFieldNo(FD);
1209 if (FD == Field) {
1210 Indices.emplace_back(Args: CGF.Builder.getInt32(C: FieldNo));
1211 return true;
1212 }
1213
1214 QualType Ty = FD->getType();
1215 if (Ty->isRecordType()) {
1216 if (getGEPIndicesToField(CGF, RD: Ty->getAsRecordDecl(), Field, Indices)) {
1217 if (RD->isUnion())
1218 FieldNo = 0;
1219 Indices.emplace_back(Args: CGF.Builder.getInt32(C: FieldNo));
1220 return true;
1221 }
1222 }
1223 }
1224
1225 return false;
1226}
1227
1228llvm::Value *CodeGenFunction::GetCountedByFieldExprGEP(
1229 const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl) {
1230 // Walk Base to find the deepest sub-expression whose struct type transitively
1231 // contains CountDecl. This is our GEP anchor — getGEPIndicesToField then
1232 // builds the field indices from that struct down to CountDecl, handling any
1233 // intermediate nesting without requiring us to pre-compute a RecordDecl from
1234 // Base's type or from CountDecl's parent chain.
1235 const Expr *StructBase = StructAccessBase(CountDecl).Visit(E: Base);
1236 if (!StructBase || StructBase->HasSideEffects(Ctx: getContext()))
1237 return nullptr;
1238
1239 // Derive the record type from the anchor expression itself.
1240 QualType StructTy = StructBase->getType();
1241 if (StructTy->isPointerType())
1242 StructTy = StructTy->getPointeeType();
1243 const RecordDecl *RD = StructTy->getAsRecordDecl();
1244 if (!RD)
1245 return nullptr;
1246
1247 llvm::Value *Res = nullptr;
1248 if (StructBase->getType()->isPointerType()) {
1249 LValueBaseInfo BaseInfo;
1250 TBAAAccessInfo TBAAInfo;
1251 Address Addr = EmitPointerWithAlignment(Addr: StructBase, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
1252 Res = Addr.emitRawPointer(CGF&: *this);
1253 } else if (StructBase->isLValue()) {
1254 LValue LV = EmitLValue(E: StructBase);
1255 Address Addr = LV.getAddress();
1256 Res = Addr.emitRawPointer(CGF&: *this);
1257 } else {
1258 return nullptr;
1259 }
1260
1261 RecIndicesTy Indices;
1262 getGEPIndicesToField(CGF&: *this, RD, Field: CountDecl, Indices);
1263 if (Indices.empty())
1264 return nullptr;
1265
1266 Indices.push_back(Elt: Builder.getInt32(C: 0));
1267 CanQualType T = CGM.getContext().getCanonicalTagType(TD: RD);
1268 return Builder.CreateInBoundsGEP(Ty: ConvertType(T), Ptr: Res,
1269 IdxList: RecIndicesTy(llvm::reverse(C&: Indices)),
1270 Name: "counted_by.gep");
1271}
1272
1273/// This method is typically called in contexts where we can't generate
1274/// side-effects, like in __builtin_dynamic_object_size. When finding
1275/// expressions, only choose those that have either already been emitted or can
1276/// be loaded without side-effects.
1277///
1278/// - \p FAMDecl: the \p Decl for the flexible array member. It may not be
1279/// within the top-level struct.
1280/// - \p CountDecl: must be within the same non-anonymous struct as \p FAMDecl.
1281llvm::Value *CodeGenFunction::EmitLoadOfCountedByField(
1282 const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl) {
1283 if (llvm::Value *GEP = GetCountedByFieldExprGEP(Base, FAMDecl, CountDecl))
1284 return Builder.CreateAlignedLoad(Ty: ConvertType(T: CountDecl->getType()), Addr: GEP,
1285 Align: getIntAlign(), Name: "counted_by.load");
1286 return nullptr;
1287}
1288
1289void CodeGenFunction::EmitBoundsCheck(const Expr *ArrayExpr,
1290 const Expr *ArrayExprBase,
1291 llvm::Value *IndexVal, QualType IndexType,
1292 bool Accessed) {
1293 assert(SanOpts.has(SanitizerKind::ArrayBounds) &&
1294 "should not be called unless adding bounds checks");
1295 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
1296 getLangOpts().getStrictFlexArraysLevel();
1297 QualType ArrayExprBaseType;
1298 llvm::Value *BoundsVal = getArrayIndexingBound(
1299 CGF&: *this, Base: ArrayExprBase, IndexedType&: ArrayExprBaseType, StrictFlexArraysLevel);
1300
1301 EmitBoundsCheckImpl(ArrayExpr, ArrayBaseType: ArrayExprBaseType, IndexVal, IndexType,
1302 BoundsVal, BoundsType: getContext().getSizeType(), Accessed);
1303}
1304
1305void CodeGenFunction::EmitBoundsCheckImpl(const Expr *ArrayExpr,
1306 QualType ArrayBaseType,
1307 llvm::Value *IndexVal,
1308 QualType IndexType,
1309 llvm::Value *BoundsVal,
1310 QualType BoundsType, bool Accessed) {
1311 if (!BoundsVal)
1312 return;
1313
1314 auto CheckKind = SanitizerKind::SO_ArrayBounds;
1315 auto CheckHandler = SanitizerHandler::OutOfBounds;
1316 SanitizerDebugLocation SanScope(this, {CheckKind}, CheckHandler);
1317
1318 // All hail the C implicit type conversion rules!!!
1319 bool IndexSigned = IndexType->isSignedIntegerOrEnumerationType();
1320 bool BoundsSigned = BoundsType->isSignedIntegerOrEnumerationType();
1321
1322 const ASTContext &Ctx = getContext();
1323 llvm::Type *Ty = ConvertType(
1324 T: Ctx.getTypeSize(T: IndexType) >= Ctx.getTypeSize(T: BoundsType) ? IndexType
1325 : BoundsType);
1326
1327 llvm::Value *IndexInst = Builder.CreateIntCast(V: IndexVal, DestTy: Ty, isSigned: IndexSigned);
1328 llvm::Value *BoundsInst = Builder.CreateIntCast(V: BoundsVal, DestTy: Ty, isSigned: false);
1329
1330 llvm::Constant *StaticData[] = {
1331 EmitCheckSourceLocation(Loc: ArrayExpr->getExprLoc()),
1332 EmitCheckTypeDescriptor(T: ArrayBaseType),
1333 EmitCheckTypeDescriptor(T: IndexType),
1334 };
1335
1336 llvm::Value *Check = Accessed ? Builder.CreateICmpULT(LHS: IndexInst, RHS: BoundsInst)
1337 : Builder.CreateICmpULE(LHS: IndexInst, RHS: BoundsInst);
1338
1339 if (BoundsSigned) {
1340 // Don't allow a negative bounds.
1341 llvm::Value *Cmp = Builder.CreateICmpSGT(
1342 LHS: BoundsVal, RHS: llvm::ConstantInt::get(Ty: BoundsVal->getType(), V: 0));
1343 Check = Builder.CreateAnd(LHS: Cmp, RHS: Check);
1344 }
1345
1346 EmitCheck(Checked: std::make_pair(x&: Check, y&: CheckKind), Check: CheckHandler, StaticArgs: StaticData,
1347 DynamicArgs: IndexInst);
1348}
1349
1350llvm::MDNode *CodeGenFunction::buildAllocToken(QualType AllocType) {
1351 auto ATMD = infer_alloc::getAllocTokenMetadata(T: AllocType, Ctx: getContext());
1352 if (!ATMD)
1353 return nullptr;
1354
1355 llvm::MDBuilder MDB(getLLVMContext());
1356 auto *TypeNameMD = MDB.createString(Str: ATMD->TypeName);
1357 auto *ContainsPtrC = Builder.getInt1(V: ATMD->ContainsPointer);
1358 auto *ContainsPtrMD = MDB.createConstant(C: ContainsPtrC);
1359
1360 // Format: !{<type-name>, <contains-pointer>}
1361 return llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: {TypeNameMD, ContainsPtrMD});
1362}
1363
1364void CodeGenFunction::EmitAllocToken(llvm::CallBase *CB, QualType AllocType) {
1365 assert(SanOpts.has(SanitizerKind::AllocToken) &&
1366 "Only needed with -fsanitize=alloc-token");
1367 CB->setMetadata(KindID: llvm::LLVMContext::MD_alloc_token,
1368 Node: buildAllocToken(AllocType));
1369}
1370
1371llvm::MDNode *CodeGenFunction::buildAllocToken(const CallExpr *E) {
1372 QualType AllocType = infer_alloc::inferPossibleType(E, Ctx: getContext(), CastE: CurCast);
1373 if (!AllocType.isNull())
1374 return buildAllocToken(AllocType);
1375 return nullptr;
1376}
1377
1378void CodeGenFunction::EmitAllocToken(llvm::CallBase *CB, const CallExpr *E) {
1379 assert(SanOpts.has(SanitizerKind::AllocToken) &&
1380 "Only needed with -fsanitize=alloc-token");
1381 if (llvm::MDNode *MDN = buildAllocToken(E))
1382 CB->setMetadata(KindID: llvm::LLVMContext::MD_alloc_token, Node: MDN);
1383}
1384
1385CodeGenFunction::ComplexPairTy CodeGenFunction::
1386EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
1387 bool isInc, bool isPre) {
1388 ComplexPairTy InVal = EmitLoadOfComplex(src: LV, loc: E->getExprLoc());
1389
1390 llvm::Value *NextVal;
1391 if (isa<llvm::IntegerType>(Val: InVal.first->getType())) {
1392 uint64_t AmountVal = isInc ? 1 : -1;
1393 NextVal = llvm::ConstantInt::get(Ty: InVal.first->getType(), V: AmountVal, IsSigned: true);
1394
1395 // Add the inc/dec to the real part.
1396 NextVal = Builder.CreateAdd(LHS: InVal.first, RHS: NextVal, Name: isInc ? "inc" : "dec");
1397 } else {
1398 QualType ElemTy = E->getType()->castAs<ComplexType>()->getElementType();
1399 llvm::APFloat FVal(getContext().getFloatTypeSemantics(T: ElemTy), 1);
1400 if (!isInc)
1401 FVal.changeSign();
1402 NextVal = llvm::ConstantFP::get(Context&: getLLVMContext(), V: FVal);
1403
1404 // Add the inc/dec to the real part.
1405 NextVal = Builder.CreateFAdd(L: InVal.first, R: NextVal, Name: isInc ? "inc" : "dec");
1406 }
1407
1408 ComplexPairTy IncVal(NextVal, InVal.second);
1409
1410 // Store the updated result through the lvalue.
1411 EmitStoreOfComplex(V: IncVal, dest: LV, /*init*/ isInit: false);
1412 if (getLangOpts().OpenMP)
1413 CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF&: *this,
1414 LHS: E->getSubExpr());
1415
1416 // If this is a postinc, return the value read from memory, otherwise use the
1417 // updated value.
1418 return isPre ? IncVal : InVal;
1419}
1420
1421void CodeGenModule::EmitExplicitCastExprType(const ExplicitCastExpr *E,
1422 CodeGenFunction *CGF) {
1423 // Bind VLAs in the cast type.
1424 if (CGF && E->getType()->isVariablyModifiedType())
1425 CGF->EmitVariablyModifiedType(Ty: E->getType());
1426
1427 if (CGDebugInfo *DI = getModuleDebugInfo())
1428 DI->EmitExplicitCastType(Ty: E->getType());
1429}
1430
1431//===----------------------------------------------------------------------===//
1432// LValue Expression Emission
1433//===----------------------------------------------------------------------===//
1434
1435static CharUnits getArrayElementAlign(CharUnits arrayAlign, llvm::Value *idx,
1436 CharUnits eltSize) {
1437 // If we have a constant index, we can use the exact offset of the
1438 // element we're accessing.
1439 if (auto *constantIdx = dyn_cast<llvm::ConstantInt>(Val: idx)) {
1440 CharUnits offset = constantIdx->getZExtValue() * eltSize;
1441 return arrayAlign.alignmentAtOffset(offset);
1442 }
1443
1444 // Otherwise, use the worst-case alignment for any element.
1445 return arrayAlign.alignmentOfArrayElement(elementSize: eltSize);
1446}
1447
1448/// Emit pointer + index arithmetic.
1449static Address emitPointerArithmetic(CodeGenFunction &CGF,
1450 const BinaryOperator *BO,
1451 LValueBaseInfo *BaseInfo,
1452 TBAAAccessInfo *TBAAInfo,
1453 KnownNonNull_t IsKnownNonNull) {
1454 assert(BO->isAdditiveOp() && "Expect an addition or subtraction.");
1455 Expr *pointerOperand = BO->getLHS();
1456 Expr *indexOperand = BO->getRHS();
1457 bool isSubtraction = BO->getOpcode() == BO_Sub;
1458
1459 Address BaseAddr = Address::invalid();
1460 llvm::Value *index = nullptr;
1461 // In a subtraction, the LHS is always the pointer.
1462 // Note: do not change the evaluation order.
1463 if (!isSubtraction && !pointerOperand->getType()->isAnyPointerType()) {
1464 std::swap(a&: pointerOperand, b&: indexOperand);
1465 index = CGF.EmitScalarExpr(E: indexOperand);
1466 BaseAddr = CGF.EmitPointerWithAlignment(Addr: pointerOperand, BaseInfo, TBAAInfo,
1467 IsKnownNonNull: NotKnownNonNull);
1468 } else {
1469 BaseAddr = CGF.EmitPointerWithAlignment(Addr: pointerOperand, BaseInfo, TBAAInfo,
1470 IsKnownNonNull: NotKnownNonNull);
1471 index = CGF.EmitScalarExpr(E: indexOperand);
1472 }
1473
1474 llvm::Value *pointer = BaseAddr.getBasePointer();
1475 llvm::Value *Res = CGF.EmitPointerArithmetic(
1476 BO, pointerOperand, pointer, indexOperand, index, isSubtraction);
1477 QualType PointeeTy = BO->getType()->getPointeeType();
1478 CharUnits Align =
1479 getArrayElementAlign(arrayAlign: BaseAddr.getAlignment(), idx: index,
1480 eltSize: CGF.getContext().getTypeSizeInChars(T: PointeeTy));
1481 return Address(Res, CGF.ConvertTypeForMem(T: PointeeTy), Align,
1482 CGF.CGM.getPointerAuthInfoForPointeeType(type: PointeeTy),
1483 /*Offset=*/nullptr, IsKnownNonNull);
1484}
1485
1486static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo,
1487 TBAAAccessInfo *TBAAInfo,
1488 KnownNonNull_t IsKnownNonNull,
1489 CodeGenFunction &CGF) {
1490 // We allow this with ObjC object pointers because of fragile ABIs.
1491 assert(E->getType()->isPointerType() ||
1492 E->getType()->isObjCObjectPointerType());
1493 E = E->IgnoreParens();
1494
1495 // Casts:
1496 if (const CastExpr *CE = dyn_cast<CastExpr>(Val: E)) {
1497 if (const auto *ECE = dyn_cast<ExplicitCastExpr>(Val: CE))
1498 CGF.CGM.EmitExplicitCastExprType(E: ECE, CGF: &CGF);
1499
1500 switch (CE->getCastKind()) {
1501 // Non-converting casts (but not C's implicit conversion from void*).
1502 case CK_BitCast:
1503 case CK_NoOp:
1504 case CK_AddressSpaceConversion:
1505 if (auto PtrTy = CE->getSubExpr()->getType()->getAs<PointerType>()) {
1506 if (PtrTy->getPointeeType()->isVoidType())
1507 break;
1508
1509 LValueBaseInfo InnerBaseInfo;
1510 TBAAAccessInfo InnerTBAAInfo;
1511 Address Addr = CGF.EmitPointerWithAlignment(
1512 Addr: CE->getSubExpr(), BaseInfo: &InnerBaseInfo, TBAAInfo: &InnerTBAAInfo, IsKnownNonNull);
1513 if (BaseInfo) *BaseInfo = InnerBaseInfo;
1514 if (TBAAInfo) *TBAAInfo = InnerTBAAInfo;
1515
1516 if (isa<ExplicitCastExpr>(Val: CE)) {
1517 LValueBaseInfo TargetTypeBaseInfo;
1518 TBAAAccessInfo TargetTypeTBAAInfo;
1519 CharUnits Align = CGF.CGM.getNaturalPointeeTypeAlignment(
1520 T: E->getType(), BaseInfo: &TargetTypeBaseInfo, TBAAInfo: &TargetTypeTBAAInfo);
1521 if (TBAAInfo)
1522 *TBAAInfo =
1523 CGF.CGM.mergeTBAAInfoForCast(SourceInfo: *TBAAInfo, TargetInfo: TargetTypeTBAAInfo);
1524 // If the source l-value is opaque, honor the alignment of the
1525 // casted-to type.
1526 if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) {
1527 if (BaseInfo)
1528 BaseInfo->mergeForCast(Info: TargetTypeBaseInfo);
1529 Addr.setAlignment(Align);
1530 }
1531 }
1532
1533 if (CGF.SanOpts.has(K: SanitizerKind::CFIUnrelatedCast) &&
1534 CE->getCastKind() == CK_BitCast) {
1535 if (auto PT = E->getType()->getAs<PointerType>())
1536 CGF.EmitVTablePtrCheckForCast(T: PT->getPointeeType(), Derived: Addr,
1537 /*MayBeNull=*/true,
1538 TCK: CodeGenFunction::CFITCK_UnrelatedCast,
1539 Loc: CE->getBeginLoc());
1540 }
1541
1542 llvm::Type *ElemTy =
1543 CGF.ConvertTypeForMem(T: E->getType()->getPointeeType());
1544 Addr = Addr.withElementType(ElemTy);
1545 if (CE->getCastKind() == CK_AddressSpaceConversion)
1546 Addr = CGF.Builder.CreateAddrSpaceCast(
1547 Addr, Ty: CGF.ConvertType(T: E->getType()), ElementTy: ElemTy);
1548
1549 return CGF.authPointerToPointerCast(Ptr: Addr, SourceType: CE->getSubExpr()->getType(),
1550 DestType: CE->getType());
1551 }
1552 break;
1553
1554 // Array-to-pointer decay.
1555 case CK_ArrayToPointerDecay:
1556 return CGF.EmitArrayToPointerDecay(Array: CE->getSubExpr(), BaseInfo, TBAAInfo);
1557
1558 // Derived-to-base conversions.
1559 case CK_UncheckedDerivedToBase:
1560 case CK_DerivedToBase: {
1561 // TODO: Support accesses to members of base classes in TBAA. For now, we
1562 // conservatively pretend that the complete object is of the base class
1563 // type.
1564 if (TBAAInfo)
1565 *TBAAInfo = CGF.CGM.getTBAAAccessInfo(AccessType: E->getType());
1566 Address Addr = CGF.EmitPointerWithAlignment(
1567 Addr: CE->getSubExpr(), BaseInfo, TBAAInfo: nullptr,
1568 IsKnownNonNull: (KnownNonNull_t)(IsKnownNonNull ||
1569 CE->getCastKind() == CK_UncheckedDerivedToBase));
1570 auto Derived = CE->getSubExpr()->getType()->getPointeeCXXRecordDecl();
1571 return CGF.GetAddressOfBaseClass(
1572 Value: Addr, Derived, PathBegin: CE->path_begin(), PathEnd: CE->path_end(),
1573 NullCheckValue: CGF.ShouldNullCheckClassCastValue(Cast: CE), Loc: CE->getExprLoc());
1574 }
1575
1576 // TODO: Is there any reason to treat base-to-derived conversions
1577 // specially?
1578 default:
1579 break;
1580 }
1581 }
1582
1583 // Unary &.
1584 if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(Val: E)) {
1585 if (UO->getOpcode() == UO_AddrOf) {
1586 LValue LV = CGF.EmitLValue(E: UO->getSubExpr(), IsKnownNonNull);
1587 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1588 if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1589 return LV.getAddress();
1590 }
1591 }
1592
1593 // std::addressof and variants.
1594 if (auto *Call = dyn_cast<CallExpr>(Val: E)) {
1595 switch (Call->getBuiltinCallee()) {
1596 default:
1597 break;
1598 case Builtin::BIaddressof:
1599 case Builtin::BI__addressof:
1600 case Builtin::BI__builtin_addressof: {
1601 LValue LV = CGF.EmitLValue(E: Call->getArg(Arg: 0), IsKnownNonNull);
1602 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1603 if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1604 return LV.getAddress();
1605 }
1606 }
1607 }
1608
1609 // Pointer arithmetic: pointer +/- index.
1610 if (auto *BO = dyn_cast<BinaryOperator>(Val: E)) {
1611 if (BO->isAdditiveOp())
1612 return emitPointerArithmetic(CGF, BO, BaseInfo, TBAAInfo, IsKnownNonNull);
1613 }
1614
1615 // TODO: conditional operators, comma.
1616
1617 // Otherwise, use the alignment of the type.
1618 return CGF.makeNaturalAddressForPointer(
1619 Ptr: CGF.EmitScalarExpr(E), T: E->getType()->getPointeeType(), Alignment: CharUnits(),
1620 /*ForPointeeType=*/true, BaseInfo, TBAAInfo, IsKnownNonNull);
1621}
1622
1623/// EmitPointerWithAlignment - Given an expression of pointer type, try to
1624/// derive a more accurate bound on the alignment of the pointer.
1625Address CodeGenFunction::EmitPointerWithAlignment(
1626 const Expr *E, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo,
1627 KnownNonNull_t IsKnownNonNull) {
1628 Address Addr =
1629 ::EmitPointerWithAlignment(E, BaseInfo, TBAAInfo, IsKnownNonNull, CGF&: *this);
1630 if (IsKnownNonNull && !Addr.isKnownNonNull())
1631 Addr.setKnownNonNull();
1632 return Addr;
1633}
1634
1635llvm::Value *CodeGenFunction::EmitNonNullRValueCheck(RValue RV, QualType T) {
1636 llvm::Value *V = RV.getScalarVal();
1637 if (auto MPT = T->getAs<MemberPointerType>())
1638 return CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF&: *this, MemPtr: V, MPT);
1639 return Builder.CreateICmpNE(LHS: V, RHS: llvm::Constant::getNullValue(Ty: V->getType()));
1640}
1641
1642RValue CodeGenFunction::GetUndefRValue(QualType Ty) {
1643 if (Ty->isVoidType())
1644 return RValue::get(V: nullptr);
1645
1646 switch (getEvaluationKind(T: Ty)) {
1647 case TEK_Complex: {
1648 llvm::Type *EltTy =
1649 ConvertType(T: Ty->castAs<ComplexType>()->getElementType());
1650 llvm::Value *U = llvm::UndefValue::get(T: EltTy);
1651 return RValue::getComplex(C: std::make_pair(x&: U, y&: U));
1652 }
1653
1654 // If this is a use of an undefined aggregate type, the aggregate must have an
1655 // identifiable address. Just because the contents of the value are undefined
1656 // doesn't mean that the address can't be taken and compared.
1657 case TEK_Aggregate: {
1658 Address DestPtr = CreateMemTemp(Ty, Name: "undef.agg.tmp");
1659 return RValue::getAggregate(addr: DestPtr);
1660 }
1661
1662 case TEK_Scalar:
1663 return RValue::get(V: llvm::UndefValue::get(T: ConvertType(T: Ty)));
1664 }
1665 llvm_unreachable("bad evaluation kind");
1666}
1667
1668RValue CodeGenFunction::EmitUnsupportedRValue(const Expr *E,
1669 const char *Name) {
1670 ErrorUnsupported(S: E, Type: Name);
1671 return GetUndefRValue(Ty: E->getType());
1672}
1673
1674LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E,
1675 const char *Name) {
1676 ErrorUnsupported(S: E, Type: Name);
1677 llvm::Type *ElTy = ConvertType(T: E->getType());
1678 llvm::Type *Ty = DefaultPtrTy;
1679 return MakeAddrLValue(
1680 Addr: Address(llvm::UndefValue::get(T: Ty), ElTy, CharUnits::One()), T: E->getType());
1681}
1682
1683bool CodeGenFunction::IsWrappedCXXThis(const Expr *Obj) {
1684 const Expr *Base = Obj;
1685 while (!isa<CXXThisExpr>(Val: Base)) {
1686 // The result of a dynamic_cast can be null.
1687 if (isa<CXXDynamicCastExpr>(Val: Base))
1688 return false;
1689
1690 if (const auto *CE = dyn_cast<CastExpr>(Val: Base)) {
1691 Base = CE->getSubExpr();
1692 } else if (const auto *PE = dyn_cast<ParenExpr>(Val: Base)) {
1693 Base = PE->getSubExpr();
1694 } else if (const auto *UO = dyn_cast<UnaryOperator>(Val: Base)) {
1695 if (UO->getOpcode() == UO_Extension)
1696 Base = UO->getSubExpr();
1697 else
1698 return false;
1699 } else {
1700 return false;
1701 }
1702 }
1703 return true;
1704}
1705
1706LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {
1707 LValue LV;
1708 if (SanOpts.has(K: SanitizerKind::ArrayBounds) && isa<ArraySubscriptExpr>(Val: E))
1709 LV = EmitArraySubscriptExpr(E: cast<ArraySubscriptExpr>(Val: E), /*Accessed*/true);
1710 else
1711 LV = EmitLValue(E);
1712 if (!isa<DeclRefExpr>(Val: E) && !LV.isBitField() && LV.isSimple()) {
1713 SanitizerSet SkippedChecks;
1714 if (const auto *ME = dyn_cast<MemberExpr>(Val: E)) {
1715 bool IsBaseCXXThis = IsWrappedCXXThis(Obj: ME->getBase());
1716 if (IsBaseCXXThis)
1717 SkippedChecks.set(K: SanitizerKind::Alignment, Value: true);
1718 if (IsBaseCXXThis || isa<DeclRefExpr>(Val: ME->getBase()))
1719 SkippedChecks.set(K: SanitizerKind::Null, Value: true);
1720 }
1721 EmitTypeCheck(TCK, Loc: E->getExprLoc(), LV, Type: E->getType(), SkippedChecks);
1722 }
1723 return LV;
1724}
1725
1726/// EmitLValue - Emit code to compute a designator that specifies the location
1727/// of the expression.
1728///
1729/// This can return one of two things: a simple address or a bitfield reference.
1730/// In either case, the LLVM Value* in the LValue structure is guaranteed to be
1731/// an LLVM pointer type.
1732///
1733/// If this returns a bitfield reference, nothing about the pointee type of the
1734/// LLVM value is known: For example, it may not be a pointer to an integer.
1735///
1736/// If this returns a normal address, and if the lvalue's C type is fixed size,
1737/// this method guarantees that the returned pointer type will point to an LLVM
1738/// type of the same size of the lvalue's type. If the lvalue has a variable
1739/// length type, this is not possible.
1740///
1741LValue CodeGenFunction::EmitLValue(const Expr *E,
1742 KnownNonNull_t IsKnownNonNull) {
1743 // Running with sufficient stack space to avoid deeply nested expressions
1744 // cause a stack overflow.
1745 LValue LV;
1746 CGM.runWithSufficientStackSpace(
1747 Loc: E->getExprLoc(), Fn: [&] { LV = EmitLValueHelper(E, IsKnownNonNull); });
1748
1749 if (IsKnownNonNull && !LV.isKnownNonNull())
1750 LV.setKnownNonNull();
1751 return LV;
1752}
1753
1754LValue CodeGenFunction::EmitLValueHelper(const Expr *E,
1755 KnownNonNull_t IsKnownNonNull) {
1756 ApplyDebugLocation DL(*this, E);
1757 switch (E->getStmtClass()) {
1758 default: return EmitUnsupportedLValue(E, Name: "l-value expression");
1759
1760 case Expr::ObjCPropertyRefExprClass:
1761 llvm_unreachable("cannot emit a property reference directly");
1762
1763 case Expr::ObjCSelectorExprClass:
1764 return EmitObjCSelectorLValue(E: cast<ObjCSelectorExpr>(Val: E));
1765 case Expr::ObjCIsaExprClass:
1766 return EmitObjCIsaExpr(E: cast<ObjCIsaExpr>(Val: E));
1767 case Expr::BinaryOperatorClass:
1768 return EmitBinaryOperatorLValue(E: cast<BinaryOperator>(Val: E));
1769 case Expr::CompoundAssignOperatorClass: {
1770 QualType Ty = E->getType();
1771 if (const AtomicType *AT = Ty->getAs<AtomicType>())
1772 Ty = AT->getValueType();
1773 if (!Ty->isAnyComplexType())
1774 return EmitCompoundAssignmentLValue(E: cast<CompoundAssignOperator>(Val: E));
1775 return EmitComplexCompoundAssignmentLValue(E: cast<CompoundAssignOperator>(Val: E));
1776 }
1777 case Expr::CallExprClass:
1778 case Expr::CXXMemberCallExprClass:
1779 case Expr::CXXOperatorCallExprClass:
1780 case Expr::UserDefinedLiteralClass:
1781 return EmitCallExprLValue(E: cast<CallExpr>(Val: E));
1782 case Expr::CXXRewrittenBinaryOperatorClass:
1783 return EmitLValue(E: cast<CXXRewrittenBinaryOperator>(Val: E)->getSemanticForm(),
1784 IsKnownNonNull);
1785 case Expr::VAArgExprClass:
1786 return EmitVAArgExprLValue(E: cast<VAArgExpr>(Val: E));
1787 case Expr::DeclRefExprClass:
1788 return EmitDeclRefLValue(E: cast<DeclRefExpr>(Val: E));
1789 case Expr::ConstantExprClass: {
1790 const ConstantExpr *CE = cast<ConstantExpr>(Val: E);
1791 if (llvm::Value *Result = ConstantEmitter(*this).tryEmitConstantExpr(CE))
1792 return MakeNaturalAlignPointeeAddrLValue(V: Result, T: CE->getType());
1793 return EmitLValue(E: cast<ConstantExpr>(Val: E)->getSubExpr(), IsKnownNonNull);
1794 }
1795 case Expr::ParenExprClass:
1796 return EmitLValue(E: cast<ParenExpr>(Val: E)->getSubExpr(), IsKnownNonNull);
1797 case Expr::GenericSelectionExprClass:
1798 return EmitLValue(E: cast<GenericSelectionExpr>(Val: E)->getResultExpr(),
1799 IsKnownNonNull);
1800 case Expr::PredefinedExprClass:
1801 return EmitPredefinedLValue(E: cast<PredefinedExpr>(Val: E));
1802 case Expr::StringLiteralClass:
1803 return EmitStringLiteralLValue(E: cast<StringLiteral>(Val: E));
1804 case Expr::ObjCEncodeExprClass:
1805 return EmitObjCEncodeExprLValue(E: cast<ObjCEncodeExpr>(Val: E));
1806 case Expr::PseudoObjectExprClass:
1807 return EmitPseudoObjectLValue(e: cast<PseudoObjectExpr>(Val: E));
1808 case Expr::InitListExprClass:
1809 return EmitInitListLValue(E: cast<InitListExpr>(Val: E));
1810 case Expr::CXXTemporaryObjectExprClass:
1811 case Expr::CXXConstructExprClass:
1812 return EmitCXXConstructLValue(E: cast<CXXConstructExpr>(Val: E));
1813 case Expr::CXXBindTemporaryExprClass:
1814 return EmitCXXBindTemporaryLValue(E: cast<CXXBindTemporaryExpr>(Val: E));
1815 case Expr::CXXUuidofExprClass:
1816 return EmitCXXUuidofLValue(E: cast<CXXUuidofExpr>(Val: E));
1817 case Expr::LambdaExprClass:
1818 return EmitAggExprToLValue(E);
1819
1820 case Expr::ExprWithCleanupsClass: {
1821 const auto *cleanups = cast<ExprWithCleanups>(Val: E);
1822 RunCleanupsScope Scope(*this);
1823 LValue LV = EmitLValue(E: cleanups->getSubExpr(), IsKnownNonNull);
1824 if (LV.isSimple()) {
1825 // Defend against branches out of gnu statement expressions surrounded by
1826 // cleanups.
1827 Address Addr = LV.getAddress();
1828 llvm::Value *V = Addr.getBasePointer();
1829 Scope.ForceCleanup(ValuesToReload: {&V});
1830 Addr.replaceBasePointer(P: V);
1831 return LValue::MakeAddr(Addr, type: LV.getType(), Context&: getContext(),
1832 BaseInfo: LV.getBaseInfo(), TBAAInfo: LV.getTBAAInfo());
1833 }
1834 // FIXME: Is it possible to create an ExprWithCleanups that produces a
1835 // bitfield lvalue or some other non-simple lvalue?
1836 return LV;
1837 }
1838
1839 case Expr::CXXDefaultArgExprClass: {
1840 auto *DAE = cast<CXXDefaultArgExpr>(Val: E);
1841 CXXDefaultArgExprScope Scope(*this, DAE);
1842 return EmitLValue(E: DAE->getExpr(), IsKnownNonNull);
1843 }
1844 case Expr::CXXDefaultInitExprClass: {
1845 auto *DIE = cast<CXXDefaultInitExpr>(Val: E);
1846 CXXDefaultInitExprScope Scope(*this, DIE);
1847 return EmitLValue(E: DIE->getExpr(), IsKnownNonNull);
1848 }
1849 case Expr::CXXTypeidExprClass:
1850 return EmitCXXTypeidLValue(E: cast<CXXTypeidExpr>(Val: E));
1851
1852 case Expr::ObjCMessageExprClass:
1853 return EmitObjCMessageExprLValue(E: cast<ObjCMessageExpr>(Val: E));
1854 case Expr::ObjCIvarRefExprClass:
1855 return EmitObjCIvarRefLValue(E: cast<ObjCIvarRefExpr>(Val: E));
1856 case Expr::StmtExprClass:
1857 return EmitStmtExprLValue(E: cast<StmtExpr>(Val: E));
1858 case Expr::UnaryOperatorClass:
1859 return EmitUnaryOpLValue(E: cast<UnaryOperator>(Val: E));
1860 case Expr::ArraySubscriptExprClass:
1861 return EmitArraySubscriptExpr(E: cast<ArraySubscriptExpr>(Val: E));
1862 case Expr::MatrixSingleSubscriptExprClass:
1863 return EmitMatrixSingleSubscriptExpr(E: cast<MatrixSingleSubscriptExpr>(Val: E));
1864 case Expr::MatrixSubscriptExprClass:
1865 return EmitMatrixSubscriptExpr(E: cast<MatrixSubscriptExpr>(Val: E));
1866 case Expr::ArraySectionExprClass:
1867 return EmitArraySectionExpr(E: cast<ArraySectionExpr>(Val: E));
1868 case Expr::ExtVectorElementExprClass:
1869 return EmitExtVectorElementExpr(E: cast<ExtVectorElementExpr>(Val: E));
1870 case Expr::MatrixElementExprClass:
1871 return EmitMatrixElementExpr(E: cast<MatrixElementExpr>(Val: E));
1872 case Expr::CXXThisExprClass:
1873 return MakeAddrLValue(Addr: LoadCXXThisAddress(), T: E->getType());
1874 case Expr::MemberExprClass:
1875 return EmitMemberExpr(E: cast<MemberExpr>(Val: E));
1876 case Expr::CompoundLiteralExprClass:
1877 return EmitCompoundLiteralLValue(E: cast<CompoundLiteralExpr>(Val: E));
1878 case Expr::ConditionalOperatorClass:
1879 return EmitConditionalOperatorLValue(E: cast<ConditionalOperator>(Val: E));
1880 case Expr::BinaryConditionalOperatorClass:
1881 return EmitConditionalOperatorLValue(E: cast<BinaryConditionalOperator>(Val: E));
1882 case Expr::ChooseExprClass:
1883 return EmitLValue(E: cast<ChooseExpr>(Val: E)->getChosenSubExpr(), IsKnownNonNull);
1884 case Expr::OpaqueValueExprClass:
1885 return EmitOpaqueValueLValue(e: cast<OpaqueValueExpr>(Val: E));
1886 case Expr::SubstNonTypeTemplateParmExprClass:
1887 return EmitLValue(E: cast<SubstNonTypeTemplateParmExpr>(Val: E)->getReplacement(),
1888 IsKnownNonNull);
1889 case Expr::ImplicitCastExprClass:
1890 case Expr::CStyleCastExprClass:
1891 case Expr::CXXFunctionalCastExprClass:
1892 case Expr::CXXStaticCastExprClass:
1893 case Expr::CXXDynamicCastExprClass:
1894 case Expr::CXXReinterpretCastExprClass:
1895 case Expr::CXXConstCastExprClass:
1896 case Expr::CXXAddrspaceCastExprClass:
1897 case Expr::ObjCBridgedCastExprClass:
1898 return EmitCastLValue(E: cast<CastExpr>(Val: E));
1899
1900 case Expr::MaterializeTemporaryExprClass:
1901 return EmitMaterializeTemporaryExpr(M: cast<MaterializeTemporaryExpr>(Val: E));
1902
1903 case Expr::CoawaitExprClass:
1904 return EmitCoawaitLValue(E: cast<CoawaitExpr>(Val: E));
1905 case Expr::CoyieldExprClass:
1906 return EmitCoyieldLValue(E: cast<CoyieldExpr>(Val: E));
1907 case Expr::PackIndexingExprClass:
1908 return EmitLValue(E: cast<PackIndexingExpr>(Val: E)->getSelectedExpr());
1909 case Expr::HLSLOutArgExprClass:
1910 llvm_unreachable("cannot emit a HLSL out argument directly");
1911 }
1912}
1913
1914/// Given an object of the given canonical type, can we safely copy a
1915/// value out of it based on its initializer?
1916static bool isConstantEmittableObjectType(QualType type) {
1917 assert(type.isCanonical());
1918 assert(!type->isReferenceType());
1919
1920 // Must be const-qualified but non-volatile.
1921 Qualifiers qs = type.getLocalQualifiers();
1922 if (!qs.hasConst() || qs.hasVolatile()) return false;
1923
1924 // Otherwise, all object types satisfy this except C++ classes with
1925 // mutable subobjects or non-trivial copy/destroy behavior.
1926 if (const auto *RT = dyn_cast<RecordType>(Val&: type))
1927 if (const auto *RD = dyn_cast<CXXRecordDecl>(Val: RT->getDecl())) {
1928 RD = RD->getDefinitionOrSelf();
1929 if (RD->hasMutableFields() || !RD->isTrivial())
1930 return false;
1931 }
1932
1933 return true;
1934}
1935
1936/// Can we constant-emit a load of a reference to a variable of the
1937/// given type? This is different from predicates like
1938/// Decl::mightBeUsableInConstantExpressions because we do want it to apply
1939/// in situations that don't necessarily satisfy the language's rules
1940/// for this (e.g. C++'s ODR-use rules). For example, we want to able
1941/// to do this with const float variables even if those variables
1942/// aren't marked 'constexpr'.
1943enum ConstantEmissionKind {
1944 CEK_None,
1945 CEK_AsReferenceOnly,
1946 CEK_AsValueOrReference,
1947 CEK_AsValueOnly
1948};
1949static ConstantEmissionKind checkVarTypeForConstantEmission(QualType type) {
1950 type = type.getCanonicalType();
1951 if (const auto *ref = dyn_cast<ReferenceType>(Val&: type)) {
1952 if (isConstantEmittableObjectType(type: ref->getPointeeType()))
1953 return CEK_AsValueOrReference;
1954 return CEK_AsReferenceOnly;
1955 }
1956 if (isConstantEmittableObjectType(type))
1957 return CEK_AsValueOnly;
1958 return CEK_None;
1959}
1960
1961/// Try to emit a reference to the given value without producing it as
1962/// an l-value. This is just an optimization, but it avoids us needing
1963/// to emit global copies of variables if they're named without triggering
1964/// a formal use in a context where we can't emit a direct reference to them,
1965/// for instance if a block or lambda or a member of a local class uses a
1966/// const int variable or constexpr variable from an enclosing function.
1967CodeGenFunction::ConstantEmission
1968CodeGenFunction::tryEmitAsConstant(const DeclRefExpr *RefExpr) {
1969 const ValueDecl *Value = RefExpr->getDecl();
1970
1971 // The value needs to be an enum constant or a constant variable.
1972 ConstantEmissionKind CEK;
1973 if (isa<ParmVarDecl>(Val: Value)) {
1974 CEK = CEK_None;
1975 } else if (const auto *var = dyn_cast<VarDecl>(Val: Value)) {
1976 CEK = checkVarTypeForConstantEmission(type: var->getType());
1977 } else if (isa<EnumConstantDecl>(Val: Value)) {
1978 CEK = CEK_AsValueOnly;
1979 } else {
1980 CEK = CEK_None;
1981 }
1982 if (CEK == CEK_None) return ConstantEmission();
1983
1984 Expr::EvalResult result;
1985 bool resultIsReference;
1986 QualType resultType;
1987
1988 // It's best to evaluate all the way as an r-value if that's permitted.
1989 if (CEK != CEK_AsReferenceOnly &&
1990 RefExpr->EvaluateAsRValue(Result&: result, Ctx: getContext())) {
1991 resultIsReference = false;
1992 resultType = RefExpr->getType().getUnqualifiedType();
1993
1994 // Otherwise, try to evaluate as an l-value.
1995 } else if (CEK != CEK_AsValueOnly &&
1996 RefExpr->EvaluateAsLValue(Result&: result, Ctx: getContext())) {
1997 resultIsReference = true;
1998 resultType = Value->getType();
1999
2000 // Failure.
2001 } else {
2002 return ConstantEmission();
2003 }
2004
2005 // In any case, if the initializer has side-effects, abandon ship.
2006 if (result.HasSideEffects)
2007 return ConstantEmission();
2008
2009 // In CUDA/HIP device compilation, a lambda may capture a reference variable
2010 // referencing a global host variable by copy. In this case the lambda should
2011 // make a copy of the value of the global host variable. The DRE of the
2012 // captured reference variable cannot be emitted as load from the host
2013 // global variable as compile time constant, since the host variable is not
2014 // accessible on device. The DRE of the captured reference variable has to be
2015 // loaded from captures.
2016 if (CGM.getLangOpts().CUDAIsDevice && result.Val.isLValue() &&
2017 RefExpr->refersToEnclosingVariableOrCapture()) {
2018 auto *MD = dyn_cast_or_null<CXXMethodDecl>(Val: CurCodeDecl);
2019 if (isLambdaMethod(DC: MD) && MD->getOverloadedOperator() == OO_Call) {
2020 const APValue::LValueBase &base = result.Val.getLValueBase();
2021 if (const ValueDecl *D = base.dyn_cast<const ValueDecl *>()) {
2022 if (const VarDecl *VD = dyn_cast<const VarDecl>(Val: D)) {
2023 if (!VD->hasAttr<CUDADeviceAttr>()) {
2024 return ConstantEmission();
2025 }
2026 }
2027 }
2028 }
2029 }
2030
2031 // Emit as a constant.
2032 llvm::Constant *C = ConstantEmitter(*this).emitAbstract(
2033 loc: RefExpr->getLocation(), value: result.Val, T: resultType);
2034
2035 // Make sure we emit a debug reference to the global variable.
2036 // This should probably fire even for
2037 if (isa<VarDecl>(Val: Value)) {
2038 if (!getContext().DeclMustBeEmitted(D: cast<VarDecl>(Val: Value)))
2039 EmitDeclRefExprDbgValue(E: RefExpr, Init: result.Val);
2040 } else {
2041 assert(isa<EnumConstantDecl>(Value));
2042 EmitDeclRefExprDbgValue(E: RefExpr, Init: result.Val);
2043 }
2044
2045 // If we emitted a reference constant, we need to dereference that.
2046 if (resultIsReference)
2047 return ConstantEmission::forReference(C);
2048
2049 return ConstantEmission::forValue(C);
2050}
2051
2052static DeclRefExpr *tryToConvertMemberExprToDeclRefExpr(CodeGenFunction &CGF,
2053 const MemberExpr *ME) {
2054 if (auto *VD = dyn_cast<VarDecl>(Val: ME->getMemberDecl())) {
2055 // Try to emit static variable member expressions as DREs.
2056 return DeclRefExpr::Create(
2057 Context: CGF.getContext(), QualifierLoc: NestedNameSpecifierLoc(), TemplateKWLoc: SourceLocation(), D: VD,
2058 /*RefersToEnclosingVariableOrCapture=*/false, NameLoc: ME->getExprLoc(),
2059 T: ME->getType(), VK: ME->getValueKind(), FoundD: nullptr, TemplateArgs: nullptr, NOUR: ME->isNonOdrUse());
2060 }
2061 return nullptr;
2062}
2063
2064CodeGenFunction::ConstantEmission
2065CodeGenFunction::tryEmitAsConstant(const MemberExpr *ME) {
2066 if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(CGF&: *this, ME))
2067 return tryEmitAsConstant(RefExpr: DRE);
2068 return ConstantEmission();
2069}
2070
2071llvm::Value *CodeGenFunction::emitScalarConstant(
2072 const CodeGenFunction::ConstantEmission &Constant, Expr *E) {
2073 assert(Constant && "not a constant");
2074 if (Constant.isReference())
2075 return EmitLoadOfLValue(V: Constant.getReferenceLValue(CGF&: *this, RefExpr: E),
2076 Loc: E->getExprLoc())
2077 .getScalarVal();
2078 return Constant.getValue();
2079}
2080
2081llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue,
2082 SourceLocation Loc) {
2083 return EmitLoadOfScalar(Addr: lvalue.getAddress(), Volatile: lvalue.isVolatile(),
2084 Ty: lvalue.getType(), Loc, BaseInfo: lvalue.getBaseInfo(),
2085 TBAAInfo: lvalue.getTBAAInfo(), isNontemporal: lvalue.isNontemporal());
2086}
2087
2088// This method SHOULD NOT be extended to support additional types, like BitInt
2089// types, without an opt-in bool controlled by a CodeGenOptions setting (like
2090// -fstrict-bool) and a new UBSan check (like SanitizerKind::Bool) as breaking
2091// that assumption would lead to memory corruption. See link for examples of how
2092// having a bool that has a value different from 0 or 1 in memory can lead to
2093// memory corruption.
2094// https://discourse.llvm.org/t/defining-what-happens-when-a-bool-isn-t-0-or-1/86778
2095static bool getRangeForType(CodeGenFunction &CGF, QualType Ty, llvm::APInt &Min,
2096 llvm::APInt &End, bool StrictEnums, bool StrictBool,
2097 bool IsBool) {
2098 const auto *ED = Ty->getAsEnumDecl();
2099 bool IsRegularCPlusPlusEnum =
2100 CGF.getLangOpts().CPlusPlus && StrictEnums && ED && !ED->isFixed();
2101 if (!IsBool && !IsRegularCPlusPlusEnum)
2102 return false;
2103
2104 if (IsBool) {
2105 if (!StrictBool)
2106 return false;
2107 Min = llvm::APInt(CGF.getContext().getTypeSize(T: Ty), 0);
2108 End = llvm::APInt(CGF.getContext().getTypeSize(T: Ty), 2);
2109 } else {
2110 ED->getValueRange(Max&: End, Min);
2111 }
2112 return true;
2113}
2114
2115llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) {
2116 llvm::APInt Min, End;
2117 bool IsBool = Ty->hasBooleanRepresentation() && !Ty->isVectorType();
2118 bool StrictBoolEnabled = CGM.getCodeGenOpts().getLoadBoolFromMem() ==
2119 CodeGenOptions::BoolFromMem::Strict;
2120 if (!getRangeForType(CGF&: *this, Ty, Min, End,
2121 /*StrictEnums=*/CGM.getCodeGenOpts().StrictEnums,
2122 /*StrictBool=*/StrictBoolEnabled, /*IsBool=*/IsBool))
2123 return nullptr;
2124
2125 llvm::MDBuilder MDHelper(getLLVMContext());
2126 return MDHelper.createRange(Lo: Min, Hi: End);
2127}
2128
2129void CodeGenFunction::maybeAttachRangeForLoad(llvm::LoadInst *Load, QualType Ty,
2130 SourceLocation Loc) {
2131 if (EmitScalarRangeCheck(Value: Load, Ty, Loc)) {
2132 // In order to prevent the optimizer from throwing away the check, don't
2133 // attach range metadata to the load.
2134 } else if (CGM.getCodeGenOpts().isOptimizedBuild()) {
2135 if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty)) {
2136 Load->setMetadata(KindID: llvm::LLVMContext::MD_range, Node: RangeInfo);
2137 Load->setMetadata(KindID: llvm::LLVMContext::MD_noundef,
2138 Node: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: {}));
2139 }
2140 }
2141}
2142
2143bool CodeGenFunction::EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
2144 SourceLocation Loc) {
2145 bool HasBoolCheck = SanOpts.has(K: SanitizerKind::Bool);
2146 bool HasEnumCheck = SanOpts.has(K: SanitizerKind::Enum);
2147 if (!HasBoolCheck && !HasEnumCheck)
2148 return false;
2149
2150 bool IsBool = (Ty->hasBooleanRepresentation() && !Ty->isVectorType()) ||
2151 NSAPI(CGM.getContext()).isObjCBOOLType(T: Ty);
2152 bool NeedsBoolCheck = HasBoolCheck && IsBool;
2153 bool NeedsEnumCheck = HasEnumCheck && Ty->isEnumeralType();
2154 if (!NeedsBoolCheck && !NeedsEnumCheck)
2155 return false;
2156
2157 // Single-bit booleans don't need to be checked. Special-case this to avoid
2158 // a bit width mismatch when handling bitfield values. This is handled by
2159 // EmitFromMemory for the non-bitfield case.
2160 if (IsBool &&
2161 cast<llvm::IntegerType>(Val: Value->getType())->getBitWidth() == 1)
2162 return false;
2163
2164 if (NeedsEnumCheck &&
2165 getContext().isTypeIgnoredBySanitizer(Mask: SanitizerKind::Enum, Ty))
2166 return false;
2167
2168 llvm::APInt Min, End;
2169 if (!getRangeForType(CGF&: *this, Ty, Min, End, /*StrictEnums=*/true,
2170 /*StrictBool=*/true, IsBool))
2171 return true;
2172
2173 SanitizerKind::SanitizerOrdinal Kind =
2174 NeedsEnumCheck ? SanitizerKind::SO_Enum : SanitizerKind::SO_Bool;
2175
2176 auto &Ctx = getLLVMContext();
2177 auto CheckHandler = SanitizerHandler::LoadInvalidValue;
2178 SanitizerDebugLocation SanScope(this, {Kind}, CheckHandler);
2179 llvm::Value *Check;
2180 --End;
2181 if (!Min) {
2182 Check = Builder.CreateICmpULE(LHS: Value, RHS: llvm::ConstantInt::get(Context&: Ctx, V: End));
2183 } else {
2184 llvm::Value *Upper =
2185 Builder.CreateICmpSLE(LHS: Value, RHS: llvm::ConstantInt::get(Context&: Ctx, V: End));
2186 llvm::Value *Lower =
2187 Builder.CreateICmpSGE(LHS: Value, RHS: llvm::ConstantInt::get(Context&: Ctx, V: Min));
2188 Check = Builder.CreateAnd(LHS: Upper, RHS: Lower);
2189 }
2190 llvm::Constant *StaticArgs[] = {EmitCheckSourceLocation(Loc),
2191 EmitCheckTypeDescriptor(T: Ty)};
2192 EmitCheck(Checked: std::make_pair(x&: Check, y&: Kind), Check: CheckHandler, StaticArgs, DynamicArgs: Value);
2193 return true;
2194}
2195
2196llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
2197 QualType Ty,
2198 SourceLocation Loc,
2199 LValueBaseInfo BaseInfo,
2200 TBAAAccessInfo TBAAInfo,
2201 bool isNontemporal) {
2202 if (auto *GV = dyn_cast<llvm::GlobalValue>(Val: Addr.getBasePointer()))
2203 if (GV->isThreadLocal())
2204 Addr = Addr.withPointer(NewPointer: Builder.CreateThreadLocalAddress(Ptr: GV),
2205 IsKnownNonNull: NotKnownNonNull);
2206
2207 if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
2208 // Boolean vectors use `iN` as storage type.
2209 if (ClangVecTy->isPackedVectorBoolType(ctx: getContext())) {
2210 llvm::Type *ValTy = ConvertType(T: Ty);
2211 unsigned ValNumElems =
2212 cast<llvm::FixedVectorType>(Val: ValTy)->getNumElements();
2213 // Load the `iP` storage object (P is the padded vector size).
2214 auto *RawIntV = Builder.CreateLoad(Addr, IsVolatile: Volatile, Name: "load_bits");
2215 const auto *RawIntTy = RawIntV->getType();
2216 assert(RawIntTy->isIntegerTy() && "compressed iN storage for bitvectors");
2217 // Bitcast iP --> <P x i1>.
2218 auto *PaddedVecTy = llvm::FixedVectorType::get(
2219 ElementType: Builder.getInt1Ty(), NumElts: RawIntTy->getPrimitiveSizeInBits());
2220 llvm::Value *V = Builder.CreateBitCast(V: RawIntV, DestTy: PaddedVecTy);
2221 // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
2222 V = emitBoolVecConversion(SrcVec: V, NumElementsDst: ValNumElems, Name: "extractvec");
2223
2224 return EmitFromMemory(Value: V, Ty);
2225 }
2226
2227 // Handles vectors of sizes that are likely to be expanded to a larger size
2228 // to optimize performance.
2229 auto *VTy = cast<llvm::FixedVectorType>(Val: Addr.getElementType());
2230 auto *NewVecTy =
2231 CGM.getABIInfo().getOptimalVectorMemoryType(T: VTy, Opt: getLangOpts());
2232
2233 if (VTy != NewVecTy) {
2234 Address Cast = Addr.withElementType(ElemTy: NewVecTy);
2235 llvm::Value *V = Builder.CreateLoad(Addr: Cast, IsVolatile: Volatile, Name: "loadVecN");
2236 unsigned OldNumElements = VTy->getNumElements();
2237 SmallVector<int, 16> Mask(OldNumElements);
2238 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
2239 V = Builder.CreateShuffleVector(V, Mask, Name: "extractVec");
2240 return EmitFromMemory(Value: V, Ty);
2241 }
2242 }
2243
2244 // Atomic operations have to be done on integral types.
2245 LValue AtomicLValue =
2246 LValue::MakeAddr(Addr, type: Ty, Context&: getContext(), BaseInfo, TBAAInfo);
2247 if (Ty->isAtomicType() || LValueIsSuitableForInlineAtomic(Src: AtomicLValue)) {
2248 return EmitAtomicLoad(LV: AtomicLValue, SL: Loc).getScalarVal();
2249 }
2250
2251 Addr =
2252 Addr.withElementType(ElemTy: convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Addr.getElementType()));
2253
2254 llvm::LoadInst *Load = Builder.CreateLoad(Addr, IsVolatile: Volatile);
2255 if (isNontemporal) {
2256 llvm::MDNode *Node = llvm::MDNode::get(
2257 Context&: Load->getContext(), MDs: llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: 1)));
2258 Load->setMetadata(KindID: llvm::LLVMContext::MD_nontemporal, Node);
2259 }
2260
2261 CGM.DecorateInstructionWithTBAA(Inst: Load, TBAAInfo);
2262
2263 maybeAttachRangeForLoad(Load, Ty, Loc);
2264
2265 return EmitFromMemory(Value: Load, Ty);
2266}
2267
2268/// Converts a scalar value from its primary IR type (as returned
2269/// by ConvertType) to its load/store type (as returned by
2270/// convertTypeForLoadStore).
2271llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) {
2272 if (auto *AtomicTy = Ty->getAs<AtomicType>())
2273 Ty = AtomicTy->getValueType();
2274
2275 if (Ty->isExtVectorBoolType() || Ty->isConstantMatrixBoolType()) {
2276 llvm::Type *StoreTy = convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Value->getType());
2277
2278 if (Value->getType() == StoreTy)
2279 return Value;
2280
2281 if (StoreTy->isVectorTy() && StoreTy->getScalarSizeInBits() >
2282 Value->getType()->getScalarSizeInBits())
2283 return Builder.CreateZExt(V: Value, DestTy: StoreTy);
2284
2285 // Expand to the memory bit width.
2286 unsigned MemNumElems = StoreTy->getPrimitiveSizeInBits();
2287 // <N x i1> --> <P x i1>.
2288 Value = emitBoolVecConversion(SrcVec: Value, NumElementsDst: MemNumElems, Name: "insertvec");
2289 // <P x i1> --> iP.
2290 Value = Builder.CreateBitCast(V: Value, DestTy: StoreTy);
2291 }
2292
2293 if (Ty->hasBooleanRepresentation() || Ty->isBitIntType()) {
2294 llvm::Type *StoreTy = convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Value->getType());
2295 bool Signed = Ty->isSignedIntegerOrEnumerationType();
2296 return Builder.CreateIntCast(V: Value, DestTy: StoreTy, isSigned: Signed, Name: "storedv");
2297 }
2298
2299 return Value;
2300}
2301
2302/// Converts a scalar value from its load/store type (as returned
2303/// by convertTypeForLoadStore) to its primary IR type (as returned
2304/// by ConvertType).
2305llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
2306 if (auto *AtomicTy = Ty->getAs<AtomicType>())
2307 Ty = AtomicTy->getValueType();
2308
2309 if (Ty->isPackedVectorBoolType(ctx: getContext())) {
2310 const auto *RawIntTy = Value->getType();
2311
2312 // Bitcast iP --> <P x i1>.
2313 auto *PaddedVecTy = llvm::FixedVectorType::get(
2314 ElementType: Builder.getInt1Ty(), NumElts: RawIntTy->getPrimitiveSizeInBits());
2315 auto *V = Builder.CreateBitCast(V: Value, DestTy: PaddedVecTy);
2316 // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
2317 llvm::Type *ValTy = ConvertType(T: Ty);
2318 unsigned ValNumElems = cast<llvm::FixedVectorType>(Val: ValTy)->getNumElements();
2319 return emitBoolVecConversion(SrcVec: V, NumElementsDst: ValNumElems, Name: "extractvec");
2320 }
2321
2322 llvm::Type *ResTy = ConvertType(T: Ty);
2323 bool HasBoolRep = Ty->hasBooleanRepresentation() ||
2324 Ty->isExtVectorBoolType() || Ty->isConstantMatrixBoolType();
2325 if (HasBoolRep && CGM.getCodeGenOpts().isConvertingBoolWithCmp0()) {
2326 return Builder.CreateICmpNE(
2327 LHS: Value, RHS: llvm::Constant::getNullValue(Ty: Value->getType()), Name: "loadedv");
2328 }
2329 if (HasBoolRep || Ty->isBitIntType())
2330 return Builder.CreateTrunc(V: Value, DestTy: ResTy, Name: "loadedv");
2331
2332 return Value;
2333}
2334
2335// Convert the pointer of \p Addr to a pointer to a vector (the value type of
2336// MatrixType), if it points to a array (the memory type of MatrixType).
2337static RawAddress MaybeConvertMatrixAddress(RawAddress Addr,
2338 CodeGenFunction &CGF,
2339 bool IsVector = true) {
2340 auto *ArrayTy = dyn_cast<llvm::ArrayType>(Val: Addr.getElementType());
2341 if (ArrayTy && IsVector) {
2342 auto ArrayElements = ArrayTy->getNumElements();
2343 auto *ArrayElementTy = ArrayTy->getElementType();
2344 if (CGF.getContext().getLangOpts().HLSL) {
2345 auto *VectorTy = cast<llvm::FixedVectorType>(Val: ArrayElementTy);
2346 ArrayElementTy = VectorTy->getElementType();
2347 ArrayElements *= VectorTy->getNumElements();
2348 }
2349 auto *VectorTy = llvm::FixedVectorType::get(ElementType: ArrayElementTy, NumElts: ArrayElements);
2350
2351 return Addr.withElementType(ElemTy: VectorTy);
2352 }
2353 auto *VectorTy = dyn_cast<llvm::VectorType>(Val: Addr.getElementType());
2354 if (VectorTy && !IsVector) {
2355 auto *ArrayTy = llvm::ArrayType::get(
2356 ElementType: VectorTy->getElementType(),
2357 NumElements: cast<llvm::FixedVectorType>(Val: VectorTy)->getNumElements());
2358
2359 return Addr.withElementType(ElemTy: ArrayTy);
2360 }
2361
2362 return Addr;
2363}
2364
2365// Emit a store of a matrix LValue. This may require casting the original
2366// pointer to memory address (ArrayType) to a pointer to the value type
2367// (VectorType).
2368static void EmitStoreOfMatrixScalar(llvm::Value *value, LValue lvalue,
2369 bool isInit, CodeGenFunction &CGF) {
2370 if (CGF.getLangOpts().HLSL &&
2371 isMatrixRowMajor(LangOpts: CGF.getLangOpts(), T: lvalue.getType())) {
2372 const auto *MatrixTy = lvalue.getType()->castAs<ConstantMatrixType>();
2373 llvm::MatrixBuilder MB(CGF.Builder);
2374 value = MB.CreateColumnMajorToRowMajorTransform(
2375 Matrix: value, Rows: MatrixTy->getNumRows(), Columns: MatrixTy->getNumColumns());
2376 }
2377 Address Addr = MaybeConvertMatrixAddress(Addr: lvalue.getAddress(), CGF,
2378 IsVector: value->getType()->isVectorTy());
2379 CGF.EmitStoreOfScalar(Value: value, Addr, Volatile: lvalue.isVolatile(), Ty: lvalue.getType(),
2380 BaseInfo: lvalue.getBaseInfo(), TBAAInfo: lvalue.getTBAAInfo(), isInit,
2381 isNontemporal: lvalue.isNontemporal());
2382}
2383
2384LValue CodeGenFunction::EmitMatrixElementExpr(const MatrixElementExpr *E) {
2385 LValue Base;
2386 if (E->getBase()->isGLValue())
2387 Base = EmitLValue(E: E->getBase());
2388 else {
2389 assert(E->getBase()->getType()->isConstantMatrixType() &&
2390 "Result must be a Constant Matrix");
2391 llvm::Value *Mat = EmitScalarExpr(E: E->getBase());
2392 Address MatMem = CreateMemTemp(Ty: E->getBase()->getType());
2393 QualType Ty = E->getBase()->getType();
2394 Base = MakeAddrLValue(Addr: MatMem, T: Ty, Source: AlignmentSource::Decl);
2395 EmitStoreOfMatrixScalar(value: Mat, lvalue: Base, /*isInit=*/true, CGF&: *this);
2396 }
2397 QualType ResultType =
2398 E->getType().withCVRQualifiers(CVR: Base.getQuals().getCVRQualifiers());
2399
2400 // Encode the element access list into a vector of unsigned indices.
2401 // getEncodedElementAccess returns row-major linearized indices.
2402 SmallVector<uint32_t, 4> Indices;
2403 E->getEncodedElementAccess(Elts&: Indices);
2404
2405 // getEncodedElementAccess returns row-major linearized indices
2406 // If the matrix memory layout is column-major, convert indices
2407 // to column-major indices.
2408 bool IsRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: E->getBase()->getType());
2409 if (!IsRowMajor) {
2410 const auto *MT = E->getBase()->getType()->castAs<ConstantMatrixType>();
2411 unsigned NumCols = MT->getNumColumns();
2412 for (uint32_t &Idx : Indices) {
2413 // Decompose row-major index: Row = Idx / NumCols, Col = Idx % NumCols
2414 unsigned Row = Idx / NumCols;
2415 unsigned Col = Idx % NumCols;
2416 // Re-linearize as column-major
2417 Idx = MT->getColumnMajorFlattenedIndex(Row, Column: Col);
2418 }
2419 }
2420
2421 if (Base.isSimple()) {
2422 RawAddress MatAddr = Base.getAddress();
2423 if (getLangOpts().HLSL &&
2424 E->getBase()->getType().getAddressSpace() == LangAS::hlsl_constant)
2425 MatAddr = CGM.getHLSLRuntime().createBufferMatrixTempAddress(LV: Base, CGF&: *this);
2426
2427 llvm::Constant *CV =
2428 llvm::ConstantDataVector::get(Context&: getLLVMContext(), Elts: Indices);
2429 return LValue::MakeExtVectorElt(Addr: MaybeConvertMatrixAddress(Addr: MatAddr, CGF&: *this),
2430 Elts: CV, type: ResultType, BaseInfo: Base.getBaseInfo(),
2431 TBAAInfo: TBAAAccessInfo());
2432 }
2433 assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
2434
2435 llvm::Constant *BaseElts = Base.getExtVectorElts();
2436 SmallVector<llvm::Constant *, 4> CElts;
2437
2438 for (unsigned Index : Indices)
2439 CElts.push_back(Elt: BaseElts->getAggregateElement(Elt: Index));
2440 llvm::Constant *CV = llvm::ConstantVector::get(V: CElts);
2441
2442 return LValue::MakeExtVectorElt(
2443 Addr: MaybeConvertMatrixAddress(Addr: Base.getExtVectorAddress(), CGF&: *this), Elts: CV,
2444 type: ResultType, BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
2445}
2446
2447void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr,
2448 bool Volatile, QualType Ty,
2449 LValueBaseInfo BaseInfo,
2450 TBAAAccessInfo TBAAInfo,
2451 bool isInit, bool isNontemporal) {
2452 if (auto *GV = dyn_cast<llvm::GlobalValue>(Val: Addr.getBasePointer()))
2453 if (GV->isThreadLocal())
2454 Addr = Addr.withPointer(NewPointer: Builder.CreateThreadLocalAddress(Ptr: GV),
2455 IsKnownNonNull: NotKnownNonNull);
2456
2457 // Handles vectors of sizes that are likely to be expanded to a larger size
2458 // to optimize performance.
2459 llvm::Type *SrcTy = Value->getType();
2460 if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
2461 if (auto *VecTy = dyn_cast<llvm::FixedVectorType>(Val: SrcTy)) {
2462 auto *NewVecTy =
2463 CGM.getABIInfo().getOptimalVectorMemoryType(T: VecTy, Opt: getLangOpts());
2464 if (!ClangVecTy->isPackedVectorBoolType(ctx: getContext()) &&
2465 VecTy != NewVecTy) {
2466 SmallVector<int, 16> Mask(NewVecTy->getNumElements(),
2467 VecTy->getNumElements());
2468 std::iota(first: Mask.begin(), last: Mask.begin() + VecTy->getNumElements(), value: 0);
2469 // Use undef instead of poison for the padding lanes, to make sure no
2470 // padding bits are poisoned, which may break coercion.
2471 Value = Builder.CreateShuffleVector(V1: Value, V2: llvm::UndefValue::get(T: VecTy),
2472 Mask, Name: "extractVec");
2473 SrcTy = NewVecTy;
2474 }
2475 if (Addr.getElementType() != SrcTy)
2476 Addr = Addr.withElementType(ElemTy: SrcTy);
2477 }
2478 }
2479
2480 Value = EmitToMemory(Value, Ty);
2481
2482 LValue AtomicLValue =
2483 LValue::MakeAddr(Addr, type: Ty, Context&: getContext(), BaseInfo, TBAAInfo);
2484 if (Ty->isAtomicType() ||
2485 (!isInit && LValueIsSuitableForInlineAtomic(Src: AtomicLValue))) {
2486 EmitAtomicStore(rvalue: RValue::get(V: Value), lvalue: AtomicLValue, isInit);
2487 return;
2488 }
2489
2490 llvm::StoreInst *Store = Builder.CreateStore(Val: Value, Addr, IsVolatile: Volatile);
2491 addInstToCurrentSourceAtom(KeyInstruction: Store, Backup: Value);
2492
2493 if (isNontemporal) {
2494 llvm::MDNode *Node =
2495 llvm::MDNode::get(Context&: Store->getContext(),
2496 MDs: llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: 1)));
2497 Store->setMetadata(KindID: llvm::LLVMContext::MD_nontemporal, Node);
2498 }
2499
2500 CGM.DecorateInstructionWithTBAA(Inst: Store, TBAAInfo);
2501}
2502
2503void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
2504 bool isInit) {
2505 if (lvalue.getType()->isConstantMatrixType()) {
2506 EmitStoreOfMatrixScalar(value, lvalue, isInit, CGF&: *this);
2507 return;
2508 }
2509
2510 EmitStoreOfScalar(Value: value, Addr: lvalue.getAddress(), Volatile: lvalue.isVolatile(),
2511 Ty: lvalue.getType(), BaseInfo: lvalue.getBaseInfo(),
2512 TBAAInfo: lvalue.getTBAAInfo(), isInit, isNontemporal: lvalue.isNontemporal());
2513}
2514
2515// Emit a load of a LValue of matrix type. This may require casting the pointer
2516// to memory address (ArrayType) to a pointer to the value type (VectorType).
2517static RValue EmitLoadOfMatrixLValue(LValue LV, SourceLocation Loc,
2518 CodeGenFunction &CGF) {
2519 assert(LV.getType()->isConstantMatrixType());
2520 RawAddress DestAddr = LV.getAddress();
2521
2522 // HLSL constant buffers may pad matrix layouts, so copy elements into a
2523 // non-padded local alloca before loading.
2524 if (CGF.getLangOpts().HLSL &&
2525 LV.getType().getAddressSpace() == LangAS::hlsl_constant)
2526 DestAddr = CGF.CGM.getHLSLRuntime().createBufferMatrixTempAddress(LV, CGF);
2527
2528 Address Addr = MaybeConvertMatrixAddress(Addr: DestAddr, CGF);
2529 LV.setAddress(Addr);
2530 llvm::Value *Value = CGF.EmitLoadOfScalar(lvalue: LV, Loc);
2531 if (CGF.getLangOpts().HLSL &&
2532 isMatrixRowMajor(LangOpts: CGF.getLangOpts(), T: LV.getType())) {
2533 const auto *MatrixTy = LV.getType()->castAs<ConstantMatrixType>();
2534 llvm::MatrixBuilder MB(CGF.Builder);
2535 Value = MB.CreateRowMajorToColumnMajorTransform(
2536 Matrix: Value, Rows: MatrixTy->getNumRows(), Columns: MatrixTy->getNumColumns());
2537 }
2538 return RValue::get(V: Value);
2539}
2540
2541RValue CodeGenFunction::EmitLoadOfAnyValue(LValue LV, AggValueSlot Slot,
2542 SourceLocation Loc) {
2543 QualType Ty = LV.getType();
2544 switch (getEvaluationKind(T: Ty)) {
2545 case TEK_Scalar:
2546 return EmitLoadOfLValue(V: LV, Loc);
2547 case TEK_Complex:
2548 return RValue::getComplex(C: EmitLoadOfComplex(src: LV, loc: Loc));
2549 case TEK_Aggregate:
2550 EmitAggFinalDestCopy(Type: Ty, Dest: Slot, Src: LV, SrcKind: EVK_NonRValue);
2551 return Slot.asRValue();
2552 }
2553 llvm_unreachable("bad evaluation kind");
2554}
2555
2556/// EmitLoadOfLValue - Given an expression that represents a value lvalue, this
2557/// method emits the address of the lvalue, then loads the result as an rvalue,
2558/// returning the rvalue.
2559RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) {
2560 // Load from __ptrauth.
2561 if (PointerAuthQualifier PtrAuth = LV.getQuals().getPointerAuth()) {
2562 LV.getQuals().removePointerAuth();
2563 llvm::Value *Value = EmitLoadOfLValue(LV, Loc).getScalarVal();
2564 return RValue::get(V: EmitPointerAuthUnqualify(Qualifier: PtrAuth, Pointer: Value, PointerType: LV.getType(),
2565 StorageAddress: LV.getAddress(),
2566 /*known nonnull*/ IsKnownNonNull: false));
2567 }
2568
2569 if (LV.isObjCWeak()) {
2570 // load of a __weak object.
2571 Address AddrWeakObj = LV.getAddress();
2572 return RValue::get(V: CGM.getObjCRuntime().EmitObjCWeakRead(CGF&: *this,
2573 AddrWeakObj));
2574 }
2575 if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) {
2576 // In MRC mode, we do a load+autorelease.
2577 if (!getLangOpts().ObjCAutoRefCount) {
2578 return RValue::get(V: EmitARCLoadWeak(addr: LV.getAddress()));
2579 }
2580
2581 // In ARC mode, we load retained and then consume the value.
2582 llvm::Value *Object = EmitARCLoadWeakRetained(addr: LV.getAddress());
2583 Object = EmitObjCConsumeObject(T: LV.getType(), Ptr: Object);
2584 return RValue::get(V: Object);
2585 }
2586
2587 if (LV.isSimple()) {
2588 assert(!LV.getType()->isFunctionType());
2589
2590 if (LV.getType()->isConstantMatrixType())
2591 return EmitLoadOfMatrixLValue(LV, Loc, CGF&: *this);
2592
2593 // Everything needs a load.
2594 return RValue::get(V: EmitLoadOfScalar(lvalue: LV, Loc));
2595 }
2596
2597 if (LV.isVectorElt()) {
2598 llvm::LoadInst *Load = Builder.CreateLoad(Addr: LV.getVectorAddress(),
2599 IsVolatile: LV.isVolatileQualified());
2600 llvm::Value *Elt =
2601 Builder.CreateExtractElement(Vec: Load, Idx: LV.getVectorIdx(), Name: "vecext");
2602 return RValue::get(V: EmitFromMemory(Value: Elt, Ty: LV.getType()));
2603 }
2604
2605 // If this is a reference to a subset of the elements of a vector, either
2606 // shuffle the input or extract/insert them as appropriate.
2607 if (LV.isExtVectorElt()) {
2608 return EmitLoadOfExtVectorElementLValue(V: LV);
2609 }
2610
2611 // Global Register variables always invoke intrinsics
2612 if (LV.isGlobalReg())
2613 return EmitLoadOfGlobalRegLValue(LV);
2614
2615 if (LV.isMatrixElt()) {
2616 llvm::Value *Idx = LV.getMatrixIdx();
2617 QualType EltTy = LV.getType();
2618 if (const auto *MatTy = EltTy->getAs<ConstantMatrixType>()) {
2619 EltTy = MatTy->getElementType();
2620 if (CGM.getCodeGenOpts().isOptimizedBuild()) {
2621 llvm::MatrixBuilder MB(Builder);
2622 MB.CreateIndexAssumption(Idx, NumElements: MatTy->getNumElementsFlattened());
2623 }
2624 }
2625 llvm::LoadInst *Load =
2626 Builder.CreateLoad(Addr: LV.getMatrixAddress(), IsVolatile: LV.isVolatileQualified());
2627 llvm::Value *Elt = Builder.CreateExtractElement(Vec: Load, Idx, Name: "matrixext");
2628 return RValue::get(V: EmitFromMemory(Value: Elt, Ty: EltTy));
2629 }
2630 if (LV.isMatrixRow()) {
2631 QualType MatTy = LV.getType();
2632 const ConstantMatrixType *MT = MatTy->castAs<ConstantMatrixType>();
2633
2634 unsigned NumRows = MT->getNumRows();
2635 unsigned NumCols = MT->getNumColumns();
2636 unsigned NumLanes = NumCols;
2637 llvm::Value *MatrixVec = EmitLoadOfScalar(lvalue: LV, Loc);
2638 llvm::Value *Row = LV.getMatrixRowIdx();
2639 llvm::Type *ElemTy = ConvertType(T: MT->getElementType());
2640 llvm::Constant *ColConstsIndices = nullptr;
2641 llvm::MatrixBuilder MB(Builder);
2642
2643 if (LV.isMatrixRowSwizzle()) {
2644 ColConstsIndices = LV.getMatrixRowElts();
2645 NumLanes = llvm::cast<llvm::FixedVectorType>(Val: ColConstsIndices->getType())
2646 ->getNumElements();
2647 }
2648
2649 llvm::Type *RowTy = llvm::FixedVectorType::get(ElementType: ElemTy, NumElts: NumLanes);
2650 llvm::Value *Result = llvm::PoisonValue::get(T: RowTy); // <NumLanes x T>
2651
2652 for (unsigned Col = 0; Col < NumLanes; ++Col) {
2653 llvm::Value *ColIdx;
2654 if (ColConstsIndices)
2655 ColIdx = ColConstsIndices->getAggregateElement(Elt: Col);
2656 else
2657 ColIdx = llvm::ConstantInt::get(Ty: Row->getType(), V: Col);
2658 bool IsMatrixRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: MatTy);
2659 llvm::Value *EltIndex =
2660 MB.CreateIndex(RowIdx: Row, ColumnIdx: ColIdx, NumRows, NumCols, IsMatrixRowMajor);
2661 llvm::Value *Elt = Builder.CreateExtractElement(Vec: MatrixVec, Idx: EltIndex);
2662 llvm::Value *Lane = llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: Col);
2663 Result = Builder.CreateInsertElement(Vec: Result, NewElt: Elt, Idx: Lane);
2664 }
2665
2666 return RValue::get(V: Result);
2667 }
2668
2669 assert(LV.isBitField() && "Unknown LValue type!");
2670 return EmitLoadOfBitfieldLValue(LV, Loc);
2671}
2672
2673RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV,
2674 SourceLocation Loc) {
2675 const CGBitFieldInfo &Info = LV.getBitFieldInfo();
2676
2677 // Get the output type.
2678 llvm::Type *ResLTy = ConvertType(T: LV.getType());
2679
2680 Address Ptr = LV.getBitFieldAddress();
2681 llvm::Value *Val =
2682 Builder.CreateLoad(Addr: Ptr, IsVolatile: LV.isVolatileQualified(), Name: "bf.load");
2683
2684 bool UseVolatile = LV.isVolatileQualified() &&
2685 Info.VolatileStorageSize != 0 &&
2686 CodeGenUtils::isAAPCS(TargetInfo: CGM.getTarget());
2687 const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
2688 const unsigned StorageSize =
2689 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
2690 if (Info.IsSigned) {
2691 assert(static_cast<unsigned>(Offset + Info.Size) <= StorageSize);
2692 unsigned HighBits = StorageSize - Offset - Info.Size;
2693 if (HighBits)
2694 Val = Builder.CreateShl(LHS: Val, RHS: HighBits, Name: "bf.shl");
2695 if (Offset + HighBits)
2696 Val = Builder.CreateAShr(LHS: Val, RHS: Offset + HighBits, Name: "bf.ashr");
2697 } else {
2698 if (Offset)
2699 Val = Builder.CreateLShr(LHS: Val, RHS: Offset, Name: "bf.lshr");
2700 if (static_cast<unsigned>(Offset) + Info.Size < StorageSize)
2701 Val = Builder.CreateAnd(
2702 LHS: Val, RHS: llvm::APInt::getLowBitsSet(numBits: StorageSize, loBitsSet: Info.Size), Name: "bf.clear");
2703 }
2704 Val = Builder.CreateIntCast(V: Val, DestTy: ResLTy, isSigned: Info.IsSigned, Name: "bf.cast");
2705 EmitScalarRangeCheck(Value: Val, Ty: LV.getType(), Loc);
2706 return RValue::get(V: Val);
2707}
2708
2709// If this is a reference to a subset of the elements of a vector, create an
2710// appropriate shufflevector.
2711RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {
2712 llvm::Value *Vec = Builder.CreateLoad(Addr: LV.getExtVectorAddress(),
2713 IsVolatile: LV.isVolatileQualified());
2714
2715 // HLSL allows treating scalars as one-element vectors. Converting the scalar
2716 // IR value to a vector here allows the rest of codegen to behave as normal.
2717 if (getLangOpts().HLSL && !Vec->getType()->isVectorTy()) {
2718 llvm::Type *DstTy = llvm::FixedVectorType::get(ElementType: Vec->getType(), NumElts: 1);
2719 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: CGM.Int64Ty);
2720 Vec = Builder.CreateInsertElement(VecTy: DstTy, NewElt: Vec, Idx: Zero, Name: "cast.splat");
2721 }
2722
2723 const llvm::Constant *Elts = LV.getExtVectorElts();
2724
2725 // If the result of the expression is a non-vector type, we must be extracting
2726 // a single element. Just codegen as an extractelement.
2727 const VectorType *ExprVT = LV.getType()->getAs<VectorType>();
2728 if (!ExprVT) {
2729 unsigned InIdx = getAccessedFieldNo(Idx: 0, Elts);
2730 llvm::Value *Elt = llvm::ConstantInt::get(Ty: SizeTy, V: InIdx);
2731
2732 llvm::Value *Element = Builder.CreateExtractElement(Vec, Idx: Elt);
2733
2734 llvm::Type *LVTy = ConvertType(T: LV.getType());
2735 if (Element->getType()->getPrimitiveSizeInBits() >
2736 LVTy->getPrimitiveSizeInBits()) {
2737 if (LV.getType()->hasBooleanRepresentation() &&
2738 CGM.getCodeGenOpts().isConvertingBoolWithCmp0())
2739 Element = Builder.CreateICmpNE(
2740 LHS: Element, RHS: llvm::Constant::getNullValue(Ty: Element->getType()));
2741 else
2742 Element = Builder.CreateTrunc(V: Element, DestTy: LVTy);
2743 }
2744
2745 return RValue::get(V: Element);
2746 }
2747
2748 // Always use shuffle vector to try to retain the original program structure
2749 unsigned NumResultElts = ExprVT->getNumElements();
2750
2751 SmallVector<int, 4> Mask;
2752 for (unsigned i = 0; i != NumResultElts; ++i)
2753 Mask.push_back(Elt: getAccessedFieldNo(Idx: i, Elts));
2754
2755 Vec = Builder.CreateShuffleVector(V: Vec, Mask);
2756
2757 if (LV.getType()->isExtVectorBoolType()) {
2758 if (CGM.getCodeGenOpts().isConvertingBoolWithCmp0())
2759 Vec = Builder.CreateICmpNE(LHS: Vec,
2760 RHS: llvm::Constant::getNullValue(Ty: Vec->getType()));
2761 else
2762 Vec = Builder.CreateTrunc(V: Vec, DestTy: ConvertType(T: LV.getType()), Name: "truncv");
2763 }
2764
2765 return RValue::get(V: Vec);
2766}
2767
2768/// Generates lvalue for partial ext_vector access.
2769Address CodeGenFunction::EmitExtVectorElementLValue(LValue LV) {
2770 Address VectorAddress = LV.getExtVectorAddress();
2771 QualType EQT = LV.getType()->castAs<VectorType>()->getElementType();
2772 llvm::Type *VectorElementTy = CGM.getTypes().ConvertType(T: EQT);
2773
2774 Address CastToPointerElement = VectorAddress.withElementType(ElemTy: VectorElementTy);
2775
2776 const llvm::Constant *Elts = LV.getExtVectorElts();
2777 unsigned ix = getAccessedFieldNo(Idx: 0, Elts);
2778
2779 Address VectorBasePtrPlusIx =
2780 Builder.CreateConstInBoundsGEP(Addr: CastToPointerElement, Index: ix,
2781 Name: "vector.elt");
2782
2783 return VectorBasePtrPlusIx;
2784}
2785
2786/// Load of global named registers are always calls to intrinsics.
2787RValue CodeGenFunction::EmitLoadOfGlobalRegLValue(LValue LV) {
2788 assert((LV.getType()->isIntegerType() || LV.getType()->isPointerType()) &&
2789 "Bad type for register variable");
2790 llvm::MDNode *RegName = cast<llvm::MDNode>(
2791 Val: cast<llvm::MetadataAsValue>(Val: LV.getGlobalReg())->getMetadata());
2792
2793 // We accept integer and pointer types only
2794 llvm::Type *OrigTy = CGM.getTypes().ConvertType(T: LV.getType());
2795 llvm::Type *Ty = OrigTy;
2796 if (OrigTy->isPointerTy())
2797 Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
2798 llvm::Type *Types[] = { Ty };
2799
2800 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::read_register, Tys: Types);
2801 llvm::Value *Call = Builder.CreateCall(
2802 Callee: F, Args: llvm::MetadataAsValue::get(Context&: Ty->getContext(), MD: RegName));
2803 if (OrigTy->isPointerTy())
2804 Call = Builder.CreateIntToPtr(V: Call, DestTy: OrigTy);
2805 return RValue::get(V: Call);
2806}
2807
2808/// EmitStoreThroughLValue - Store the specified rvalue into the specified
2809/// lvalue, where both are guaranteed to the have the same type, and that type
2810/// is 'Ty'.
2811void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
2812 bool isInit) {
2813 if (!Dst.isSimple()) {
2814 if (Dst.isVectorElt()) {
2815 if (getLangOpts().HLSL) {
2816 // HLSL allows direct access to vector elements, so storing to
2817 // individual elements of a vector through VectorElt is handled as
2818 // separate store instructions.
2819 Address DstAddr = Dst.getVectorAddress();
2820 llvm::Type *DestAddrTy = DstAddr.getElementType();
2821 llvm::Type *ElemTy = DestAddrTy->getScalarType();
2822 CharUnits ElemAlign = CharUnits::fromQuantity(
2823 Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty: ElemTy));
2824
2825 assert(ElemTy->getScalarSizeInBits() >= 8 &&
2826 "vector element type must be at least byte-sized");
2827
2828 llvm::Value *Val = Src.getScalarVal();
2829 if (Val->getType()->getPrimitiveSizeInBits() <
2830 ElemTy->getScalarSizeInBits())
2831 Val = Builder.CreateZExt(V: Val, DestTy: ElemTy->getScalarType());
2832
2833 llvm::Value *Idx = Dst.getVectorIdx();
2834 llvm::Value *Zero = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
2835 Address DstElemAddr =
2836 Builder.CreateGEP(Addr: DstAddr, IdxList: {Zero, Idx}, ElementType: DestAddrTy, Align: ElemAlign);
2837 Builder.CreateStore(Val, Addr: DstElemAddr, IsVolatile: Dst.isVolatileQualified());
2838 return;
2839 }
2840
2841 // Read/modify/write the vector, inserting the new element.
2842 llvm::Value *Vec = Builder.CreateLoad(Addr: Dst.getVectorAddress(),
2843 IsVolatile: Dst.isVolatileQualified());
2844 llvm::Type *VecTy = Vec->getType();
2845 llvm::Value *SrcVal = Src.getScalarVal();
2846
2847 if (VecTy->isVectorTy() && SrcVal->getType()->getPrimitiveSizeInBits() <
2848 VecTy->getScalarSizeInBits())
2849 SrcVal = Builder.CreateZExt(V: SrcVal, DestTy: VecTy->getScalarType());
2850
2851 auto *IRStoreTy = dyn_cast<llvm::IntegerType>(Val: Vec->getType());
2852 if (IRStoreTy) {
2853 auto *IRVecTy = llvm::FixedVectorType::get(
2854 ElementType: Builder.getInt1Ty(), NumElts: IRStoreTy->getPrimitiveSizeInBits());
2855 Vec = Builder.CreateBitCast(V: Vec, DestTy: IRVecTy);
2856 // iN --> <N x i1>.
2857 }
2858
2859 // Allow inserting `<1 x T>` into an `<N x T>`. It can happen with scalar
2860 // types which are mapped to vector LLVM IR types (e.g. for implementing
2861 // an ABI).
2862 if (auto *EltTy = dyn_cast<llvm::FixedVectorType>(Val: SrcVal->getType());
2863 EltTy && EltTy->getNumElements() == 1)
2864 SrcVal = Builder.CreateBitCast(V: SrcVal, DestTy: EltTy->getElementType());
2865
2866 Vec = Builder.CreateInsertElement(Vec, NewElt: SrcVal, Idx: Dst.getVectorIdx(),
2867 Name: "vecins");
2868 if (IRStoreTy) {
2869 // <N x i1> --> <iN>.
2870 Vec = Builder.CreateBitCast(V: Vec, DestTy: IRStoreTy);
2871 }
2872
2873 auto *I = Builder.CreateStore(Val: Vec, Addr: Dst.getVectorAddress(),
2874 IsVolatile: Dst.isVolatileQualified());
2875 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Vec);
2876 return;
2877 }
2878
2879 // If this is an update of extended vector elements, insert them as
2880 // appropriate.
2881 if (Dst.isExtVectorElt())
2882 return EmitStoreThroughExtVectorComponentLValue(Src, Dst);
2883
2884 if (Dst.isGlobalReg())
2885 return EmitStoreThroughGlobalRegLValue(Src, Dst);
2886
2887 if (Dst.isMatrixElt()) {
2888 if (getLangOpts().HLSL) {
2889 // HLSL allows direct access to matrix elements, so storing to
2890 // individual elements of a matrix through MatrixElt is handled as
2891 // separate store instructions.
2892 Address DstAddr = Dst.getMatrixAddress();
2893 llvm::Type *DestAddrTy = DstAddr.getElementType();
2894 llvm::Type *ElemTy = DestAddrTy->getScalarType();
2895 CharUnits ElemAlign = CharUnits::fromQuantity(
2896 Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty: ElemTy));
2897
2898 assert(ElemTy->getScalarSizeInBits() >= 8 &&
2899 "matrix element type must be at least byte-sized");
2900
2901 llvm::Value *Val = Src.getScalarVal();
2902 if (Val->getType()->getPrimitiveSizeInBits() <
2903 ElemTy->getScalarSizeInBits())
2904 Val = Builder.CreateZExt(V: Val, DestTy: ElemTy->getScalarType());
2905
2906 llvm::Value *Idx = Dst.getMatrixIdx();
2907 llvm::Value *Zero = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
2908 Address DstElemAddr =
2909 Builder.CreateGEP(Addr: DstAddr, IdxList: {Zero, Idx}, ElementType: DestAddrTy, Align: ElemAlign);
2910 Builder.CreateStore(Val, Addr: DstElemAddr, IsVolatile: Dst.isVolatileQualified());
2911 return;
2912 }
2913
2914 llvm::Value *Idx = Dst.getMatrixIdx();
2915 if (CGM.getCodeGenOpts().isOptimizedBuild()) {
2916 const auto *const MatTy = Dst.getType()->castAs<ConstantMatrixType>();
2917 llvm::MatrixBuilder MB(Builder);
2918 MB.CreateIndexAssumption(Idx, NumElements: MatTy->getNumElementsFlattened());
2919 }
2920 llvm::Instruction *Load = Builder.CreateLoad(Addr: Dst.getMatrixAddress());
2921 llvm::Value *InsertVal = Src.getScalarVal();
2922 llvm::Value *Vec =
2923 Builder.CreateInsertElement(Vec: Load, NewElt: InsertVal, Idx, Name: "matins");
2924 auto *I = Builder.CreateStore(Val: Vec, Addr: Dst.getMatrixAddress(),
2925 IsVolatile: Dst.isVolatileQualified());
2926 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Vec);
2927 return;
2928 }
2929 if (Dst.isMatrixRow()) {
2930 // NOTE: Since there are no other languages that implement matrix single
2931 // subscripting, the logic here is specific to HLSL which allows
2932 // per-element stores to rows of matrices.
2933 assert(getLangOpts().HLSL &&
2934 "Store through matrix row LValues is only implemented for HLSL!");
2935 QualType MatTy = Dst.getType();
2936 const ConstantMatrixType *MT = MatTy->castAs<ConstantMatrixType>();
2937
2938 unsigned NumRows = MT->getNumRows();
2939 unsigned NumCols = MT->getNumColumns();
2940 unsigned NumLanes = NumCols;
2941
2942 Address DstAddr = Dst.getMatrixAddress();
2943 llvm::Type *DestAddrTy = DstAddr.getElementType();
2944 llvm::Type *ElemTy = DestAddrTy->getScalarType();
2945 CharUnits ElemAlign =
2946 CharUnits::fromQuantity(Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty: ElemTy));
2947
2948 assert(ElemTy->getScalarSizeInBits() >= 8 &&
2949 "matrix element type must be at least byte-sized");
2950
2951 llvm::Value *RowVal = Src.getScalarVal();
2952 if (RowVal->getType()->getScalarType()->getPrimitiveSizeInBits() <
2953 ElemTy->getScalarSizeInBits()) {
2954 auto *RowValVecTy = cast<llvm::FixedVectorType>(Val: RowVal->getType());
2955 llvm::Type *StorageElmTy = llvm::FixedVectorType::get(
2956 ElementType: ElemTy->getScalarType(), NumElts: RowValVecTy->getNumElements());
2957 RowVal = Builder.CreateZExt(V: RowVal, DestTy: StorageElmTy);
2958 }
2959
2960 llvm::MatrixBuilder MB(Builder);
2961
2962 llvm::Constant *ColConstsIndices = nullptr;
2963 if (Dst.isMatrixRowSwizzle()) {
2964 ColConstsIndices = Dst.getMatrixRowElts();
2965 NumLanes =
2966 llvm::cast<llvm::FixedVectorType>(Val: ColConstsIndices->getType())
2967 ->getNumElements();
2968 }
2969
2970 llvm::Value *Row = Dst.getMatrixRowIdx();
2971 for (unsigned Col = 0; Col < NumLanes; ++Col) {
2972 llvm::Value *ColIdx;
2973 if (ColConstsIndices)
2974 ColIdx = ColConstsIndices->getAggregateElement(Elt: Col);
2975 else
2976 ColIdx = llvm::ConstantInt::get(Ty: Row->getType(), V: Col);
2977 bool IsMatrixRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T: Dst.getType());
2978 llvm::Value *EltIndex =
2979 MB.CreateIndex(RowIdx: Row, ColumnIdx: ColIdx, NumRows, NumCols, IsMatrixRowMajor);
2980 llvm::Value *Lane = llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: Col);
2981 llvm::Value *Zero = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
2982 llvm::Value *NewElt = Builder.CreateExtractElement(Vec: RowVal, Idx: Lane);
2983 Address DstElemAddr =
2984 Builder.CreateGEP(Addr: DstAddr, IdxList: {Zero, EltIndex}, ElementType: DestAddrTy, Align: ElemAlign);
2985 Builder.CreateStore(Val: NewElt, Addr: DstElemAddr, IsVolatile: Dst.isVolatileQualified());
2986 }
2987
2988 return;
2989 }
2990
2991 assert(Dst.isBitField() && "Unknown LValue type");
2992 return EmitStoreThroughBitfieldLValue(Src, Dst);
2993 }
2994
2995 // Handle __ptrauth qualification by re-signing the value.
2996 if (PointerAuthQualifier PointerAuth = Dst.getQuals().getPointerAuth()) {
2997 Src = RValue::get(V: EmitPointerAuthQualify(Qualifier: PointerAuth, Pointer: Src.getScalarVal(),
2998 ValueType: Dst.getType(), StorageAddress: Dst.getAddress(),
2999 /*known nonnull*/ IsKnownNonNull: false));
3000 }
3001
3002 // There's special magic for assigning into an ARC-qualified l-value.
3003 if (Qualifiers::ObjCLifetime Lifetime = Dst.getQuals().getObjCLifetime()) {
3004 switch (Lifetime) {
3005 case Qualifiers::OCL_None:
3006 llvm_unreachable("present but none");
3007
3008 case Qualifiers::OCL_ExplicitNone:
3009 // nothing special
3010 break;
3011
3012 case Qualifiers::OCL_Strong:
3013 if (isInit) {
3014 Src = RValue::get(V: EmitARCRetain(type: Dst.getType(), value: Src.getScalarVal()));
3015 break;
3016 }
3017 EmitARCStoreStrong(lvalue: Dst, value: Src.getScalarVal(), /*ignore*/ resultIgnored: true);
3018 return;
3019
3020 case Qualifiers::OCL_Weak:
3021 if (isInit)
3022 // Initialize and then skip the primitive store.
3023 EmitARCInitWeak(addr: Dst.getAddress(), value: Src.getScalarVal());
3024 else
3025 EmitARCStoreWeak(addr: Dst.getAddress(), value: Src.getScalarVal(),
3026 /*ignore*/ ignored: true);
3027 return;
3028
3029 case Qualifiers::OCL_Autoreleasing:
3030 Src = RValue::get(V: EmitObjCExtendObjectLifetime(T: Dst.getType(),
3031 Ptr: Src.getScalarVal()));
3032 // fall into the normal path
3033 break;
3034 }
3035 }
3036
3037 if (Dst.isObjCWeak() && !Dst.isNonGC()) {
3038 // load of a __weak object.
3039 Address LvalueDst = Dst.getAddress();
3040 llvm::Value *src = Src.getScalarVal();
3041 CGM.getObjCRuntime().EmitObjCWeakAssign(CGF&: *this, src, dest: LvalueDst);
3042 return;
3043 }
3044
3045 if (Dst.isObjCStrong() && !Dst.isNonGC()) {
3046 // load of a __strong object.
3047 Address LvalueDst = Dst.getAddress();
3048 llvm::Value *src = Src.getScalarVal();
3049 if (Dst.isObjCIvar()) {
3050 assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");
3051 llvm::Type *ResultType = IntPtrTy;
3052 Address dst = EmitPointerWithAlignment(E: Dst.getBaseIvarExp());
3053 llvm::Value *RHS = dst.emitRawPointer(CGF&: *this);
3054 RHS = Builder.CreatePtrToInt(V: RHS, DestTy: ResultType, Name: "sub.ptr.rhs.cast");
3055 llvm::Value *LHS = Builder.CreatePtrToInt(V: LvalueDst.emitRawPointer(CGF&: *this),
3056 DestTy: ResultType, Name: "sub.ptr.lhs.cast");
3057 llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, Name: "ivar.offset");
3058 CGM.getObjCRuntime().EmitObjCIvarAssign(CGF&: *this, src, dest: dst, ivarOffset: BytesBetween);
3059 } else if (Dst.isGlobalObjCRef()) {
3060 CGM.getObjCRuntime().EmitObjCGlobalAssign(CGF&: *this, src, dest: LvalueDst,
3061 threadlocal: Dst.isThreadLocalRef());
3062 }
3063 else
3064 CGM.getObjCRuntime().EmitObjCStrongCastAssign(CGF&: *this, src, dest: LvalueDst);
3065 return;
3066 }
3067
3068 assert(Src.isScalar() && "Can't emit an agg store with this method");
3069 EmitStoreOfScalar(value: Src.getScalarVal(), lvalue: Dst, isInit);
3070}
3071
3072void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
3073 llvm::Value **Result) {
3074 const CGBitFieldInfo &Info = Dst.getBitFieldInfo();
3075 llvm::Type *ResLTy = convertTypeForLoadStore(ASTTy: Dst.getType());
3076 Address Ptr = Dst.getBitFieldAddress();
3077
3078 // Get the source value, truncated to the width of the bit-field.
3079 llvm::Value *SrcVal = Src.getScalarVal();
3080
3081 // Cast the source to the storage type and shift it into place.
3082 SrcVal = Builder.CreateIntCast(V: SrcVal, DestTy: Ptr.getElementType(),
3083 /*isSigned=*/false);
3084 llvm::Value *MaskedVal = SrcVal;
3085
3086 const bool UseVolatile =
3087 CGM.getCodeGenOpts().AAPCSBitfieldWidth && Dst.isVolatileQualified() &&
3088 Info.VolatileStorageSize != 0 && CodeGenUtils::isAAPCS(TargetInfo: CGM.getTarget());
3089 const unsigned StorageSize =
3090 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
3091 const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
3092 // See if there are other bits in the bitfield's storage we'll need to load
3093 // and mask together with source before storing.
3094 if (StorageSize != Info.Size) {
3095 assert(StorageSize > Info.Size && "Invalid bitfield size.");
3096 llvm::Value *Val =
3097 Builder.CreateLoad(Addr: Ptr, IsVolatile: Dst.isVolatileQualified(), Name: "bf.load");
3098
3099 // Mask the source value as needed.
3100 if (!Dst.getType()->hasBooleanRepresentation())
3101 SrcVal = Builder.CreateAnd(
3102 LHS: SrcVal, RHS: llvm::APInt::getLowBitsSet(numBits: StorageSize, loBitsSet: Info.Size),
3103 Name: "bf.value");
3104 MaskedVal = SrcVal;
3105 if (Offset)
3106 SrcVal = Builder.CreateShl(LHS: SrcVal, RHS: Offset, Name: "bf.shl");
3107
3108 // Mask out the original value.
3109 Val = Builder.CreateAnd(
3110 LHS: Val, RHS: ~llvm::APInt::getBitsSet(numBits: StorageSize, loBit: Offset, hiBit: Offset + Info.Size),
3111 Name: "bf.clear");
3112
3113 // Or together the unchanged values and the source value.
3114 SrcVal = Builder.CreateOr(LHS: Val, RHS: SrcVal, Name: "bf.set");
3115 } else {
3116 assert(Offset == 0);
3117 // According to the AACPS:
3118 // When a volatile bit-field is written, and its container does not overlap
3119 // with any non-bit-field member, its container must be read exactly once
3120 // and written exactly once using the access width appropriate to the type
3121 // of the container. The two accesses are not atomic.
3122 if (Dst.isVolatileQualified() && CodeGenUtils::isAAPCS(TargetInfo: CGM.getTarget()) &&
3123 CGM.getCodeGenOpts().ForceAAPCSBitfieldLoad)
3124 Builder.CreateLoad(Addr: Ptr, IsVolatile: true, Name: "bf.load");
3125 }
3126
3127 // Write the new value back out.
3128 auto *I = Builder.CreateStore(Val: SrcVal, Addr: Ptr, IsVolatile: Dst.isVolatileQualified());
3129 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: SrcVal);
3130
3131 // Return the new value of the bit-field, if requested.
3132 if (Result) {
3133 llvm::Value *ResultVal = MaskedVal;
3134
3135 // Sign extend the value if needed.
3136 if (Info.IsSigned) {
3137 assert(Info.Size <= StorageSize);
3138 unsigned HighBits = StorageSize - Info.Size;
3139 if (HighBits) {
3140 ResultVal = Builder.CreateShl(LHS: ResultVal, RHS: HighBits, Name: "bf.result.shl");
3141 ResultVal = Builder.CreateAShr(LHS: ResultVal, RHS: HighBits, Name: "bf.result.ashr");
3142 }
3143 }
3144
3145 ResultVal = Builder.CreateIntCast(V: ResultVal, DestTy: ResLTy, isSigned: Info.IsSigned,
3146 Name: "bf.result.cast");
3147 *Result = EmitFromMemory(Value: ResultVal, Ty: Dst.getType());
3148 }
3149}
3150
3151void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
3152 LValue Dst) {
3153 llvm::Value *SrcVal = Src.getScalarVal();
3154 Address DstAddr = Dst.getExtVectorAddress();
3155 const llvm::Constant *Elts = Dst.getExtVectorElts();
3156 if (DstAddr.getElementType()->getScalarSizeInBits() >
3157 SrcVal->getType()->getScalarSizeInBits())
3158 SrcVal = Builder.CreateZExt(
3159 V: SrcVal, DestTy: convertTypeForLoadStore(ASTTy: Dst.getType(), LLVMTy: SrcVal->getType()));
3160
3161 if (getLangOpts().HLSL) {
3162 llvm::Type *DestAddrTy = DstAddr.getElementType();
3163 // HLSL allows storing to scalar values through ExtVector component LValues.
3164 // To support this we need to handle the case where the destination address
3165 // is a scalar.
3166 if (!DestAddrTy->isVectorTy()) {
3167 assert(!Dst.getType()->isVectorType() &&
3168 "this should only occur for non-vector l-values");
3169 Builder.CreateStore(Val: SrcVal, Addr: DstAddr, IsVolatile: Dst.isVolatileQualified());
3170 return;
3171 }
3172
3173 // HLSL allows direct access to vector elements, so storing to individual
3174 // elements of a vector through ExtVector is handled as separate store
3175 // instructions.
3176 // If we are updating multiple elements, Dst and Src are vectors; for
3177 // a single element update they are scalars.
3178 const VectorType *VTy = Dst.getType()->getAs<VectorType>();
3179 unsigned NumSrcElts = VTy ? VTy->getNumElements() : 1;
3180 CharUnits ElemAlign = CharUnits::fromQuantity(
3181 Quantity: CGM.getDataLayout().getPrefTypeAlign(Ty: DestAddrTy->getScalarType()));
3182 llvm::Value *Zero = llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
3183
3184 for (unsigned I = 0; I != NumSrcElts; ++I) {
3185 llvm::Value *Val = VTy ? Builder.CreateExtractElement(
3186 Vec: SrcVal, Idx: llvm::ConstantInt::get(Ty: Int32Ty, V: I))
3187 : SrcVal;
3188 unsigned FieldNo = getAccessedFieldNo(Idx: I, Elts);
3189 Address DstElemAddr = Address::invalid();
3190 if (FieldNo == 0)
3191 DstElemAddr = DstAddr.withAlignment(NewAlignment: ElemAlign);
3192 else
3193 DstElemAddr = Builder.CreateGEP(
3194 Addr: DstAddr, IdxList: {Zero, llvm::ConstantInt::get(Ty: Int32Ty, V: FieldNo)},
3195 ElementType: DestAddrTy, Align: ElemAlign);
3196 Builder.CreateStore(Val, Addr: DstElemAddr, IsVolatile: Dst.isVolatileQualified());
3197 }
3198 return;
3199 }
3200
3201 // This access turns into a read/modify/write of the vector. Load the input
3202 // value now.
3203 llvm::Value *Vec = Builder.CreateLoad(Addr: DstAddr, IsVolatile: Dst.isVolatileQualified());
3204 llvm::Type *VecTy = Vec->getType();
3205
3206 if (const VectorType *VTy = Dst.getType()->getAs<VectorType>()) {
3207 unsigned NumSrcElts = VTy->getNumElements();
3208 unsigned NumDstElts = cast<llvm::FixedVectorType>(Val: VecTy)->getNumElements();
3209 if (NumDstElts == NumSrcElts) {
3210 // Use shuffle vector is the src and destination are the same number of
3211 // elements and restore the vector mask since it is on the side it will be
3212 // stored.
3213 SmallVector<int, 4> Mask(NumDstElts);
3214 for (unsigned i = 0; i != NumSrcElts; ++i)
3215 Mask[getAccessedFieldNo(Idx: i, Elts)] = i;
3216
3217 Vec = Builder.CreateShuffleVector(V: SrcVal, Mask);
3218 } else if (NumDstElts > NumSrcElts) {
3219 // Extended the source vector to the same length and then shuffle it
3220 // into the destination.
3221 // FIXME: since we're shuffling with undef, can we just use the indices
3222 // into that? This could be simpler.
3223 SmallVector<int, 4> ExtMask;
3224 for (unsigned i = 0; i != NumSrcElts; ++i)
3225 ExtMask.push_back(Elt: i);
3226 ExtMask.resize(N: NumDstElts, NV: -1);
3227 llvm::Value *ExtSrcVal = Builder.CreateShuffleVector(V: SrcVal, Mask: ExtMask);
3228 // build identity
3229 SmallVector<int, 4> Mask;
3230 for (unsigned i = 0; i != NumDstElts; ++i)
3231 Mask.push_back(Elt: i);
3232
3233 // When the vector size is odd and .odd or .hi is used, the last element
3234 // of the Elts constant array will be one past the size of the vector.
3235 // Ignore the last element here, if it is greater than the mask size.
3236 if (getAccessedFieldNo(Idx: NumSrcElts - 1, Elts) == Mask.size())
3237 NumSrcElts--;
3238
3239 // modify when what gets shuffled in
3240 for (unsigned i = 0; i != NumSrcElts; ++i)
3241 Mask[getAccessedFieldNo(Idx: i, Elts)] = i + NumDstElts;
3242 Vec = Builder.CreateShuffleVector(V1: Vec, V2: ExtSrcVal, Mask);
3243 } else {
3244 // We should never shorten the vector
3245 llvm_unreachable("unexpected shorten vector length");
3246 }
3247 } else {
3248 // If the Src is a scalar (not a vector), and the target is a vector it must
3249 // be updating one element.
3250 unsigned InIdx = getAccessedFieldNo(Idx: 0, Elts);
3251 llvm::Value *Elt = llvm::ConstantInt::get(Ty: SizeTy, V: InIdx);
3252
3253 Vec = Builder.CreateInsertElement(Vec, NewElt: SrcVal, Idx: Elt);
3254 }
3255
3256 Builder.CreateStore(Val: Vec, Addr: Dst.getExtVectorAddress(),
3257 IsVolatile: Dst.isVolatileQualified());
3258}
3259
3260/// Store of global named registers are always calls to intrinsics.
3261void CodeGenFunction::EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst) {
3262 assert((Dst.getType()->isIntegerType() || Dst.getType()->isPointerType()) &&
3263 "Bad type for register variable");
3264 llvm::MDNode *RegName = cast<llvm::MDNode>(
3265 Val: cast<llvm::MetadataAsValue>(Val: Dst.getGlobalReg())->getMetadata());
3266 assert(RegName && "Register LValue is not metadata");
3267
3268 // We accept integer and pointer types only
3269 llvm::Type *OrigTy = CGM.getTypes().ConvertType(T: Dst.getType());
3270 llvm::Type *Ty = OrigTy;
3271 if (OrigTy->isPointerTy())
3272 Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
3273 llvm::Type *Types[] = { Ty };
3274
3275 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::write_register, Tys: Types);
3276 llvm::Value *Value = Src.getScalarVal();
3277 if (OrigTy->isPointerTy())
3278 Value = Builder.CreatePtrToInt(V: Value, DestTy: Ty);
3279 Builder.CreateCall(
3280 Callee: F, Args: {llvm::MetadataAsValue::get(Context&: Ty->getContext(), MD: RegName), Value});
3281}
3282
3283// setObjCGCLValueClass - sets class of the lvalue for the purpose of
3284// generating write-barries API. It is currently a global, ivar,
3285// or neither.
3286static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
3287 LValue &LV,
3288 bool IsMemberAccess=false) {
3289 if (Ctx.getLangOpts().getGC() == LangOptions::NonGC)
3290 return;
3291
3292 if (isa<ObjCIvarRefExpr>(Val: E)) {
3293 QualType ExpTy = E->getType();
3294 if (IsMemberAccess && ExpTy->isPointerType()) {
3295 // If ivar is a structure pointer, assigning to field of
3296 // this struct follows gcc's behavior and makes it a non-ivar
3297 // writer-barrier conservatively.
3298 ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
3299 if (ExpTy->isRecordType()) {
3300 LV.setObjCIvar(false);
3301 return;
3302 }
3303 }
3304 LV.setObjCIvar(true);
3305 auto *Exp = cast<ObjCIvarRefExpr>(Val: const_cast<Expr *>(E));
3306 LV.setBaseIvarExp(Exp->getBase());
3307 LV.setObjCArray(E->getType()->isArrayType());
3308 return;
3309 }
3310
3311 if (const auto *Exp = dyn_cast<DeclRefExpr>(Val: E)) {
3312 if (const auto *VD = dyn_cast<VarDecl>(Val: Exp->getDecl())) {
3313 if (VD->hasGlobalStorage()) {
3314 LV.setGlobalObjCRef(true);
3315 LV.setThreadLocalRef(VD->getTLSKind() != VarDecl::TLS_None);
3316 }
3317 }
3318 LV.setObjCArray(E->getType()->isArrayType());
3319 return;
3320 }
3321
3322 if (const auto *Exp = dyn_cast<UnaryOperator>(Val: E)) {
3323 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3324 return;
3325 }
3326
3327 if (const auto *Exp = dyn_cast<ParenExpr>(Val: E)) {
3328 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3329 if (LV.isObjCIvar()) {
3330 // If cast is to a structure pointer, follow gcc's behavior and make it
3331 // a non-ivar write-barrier.
3332 QualType ExpTy = E->getType();
3333 if (ExpTy->isPointerType())
3334 ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
3335 if (ExpTy->isRecordType())
3336 LV.setObjCIvar(false);
3337 }
3338 return;
3339 }
3340
3341 if (const auto *Exp = dyn_cast<GenericSelectionExpr>(Val: E)) {
3342 setObjCGCLValueClass(Ctx, E: Exp->getResultExpr(), LV);
3343 return;
3344 }
3345
3346 if (const auto *Exp = dyn_cast<ImplicitCastExpr>(Val: E)) {
3347 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3348 return;
3349 }
3350
3351 if (const auto *Exp = dyn_cast<CStyleCastExpr>(Val: E)) {
3352 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3353 return;
3354 }
3355
3356 if (const auto *Exp = dyn_cast<ObjCBridgedCastExpr>(Val: E)) {
3357 setObjCGCLValueClass(Ctx, E: Exp->getSubExpr(), LV, IsMemberAccess);
3358 return;
3359 }
3360
3361 if (const auto *Exp = dyn_cast<ArraySubscriptExpr>(Val: E)) {
3362 setObjCGCLValueClass(Ctx, E: Exp->getBase(), LV);
3363 if (LV.isObjCIvar() && !LV.isObjCArray())
3364 // Using array syntax to assigning to what an ivar points to is not
3365 // same as assigning to the ivar itself. {id *Names;} Names[i] = 0;
3366 LV.setObjCIvar(false);
3367 else if (LV.isGlobalObjCRef() && !LV.isObjCArray())
3368 // Using array syntax to assigning to what global points to is not
3369 // same as assigning to the global itself. {id *G;} G[i] = 0;
3370 LV.setGlobalObjCRef(false);
3371 return;
3372 }
3373
3374 if (const auto *Exp = dyn_cast<MemberExpr>(Val: E)) {
3375 setObjCGCLValueClass(Ctx, E: Exp->getBase(), LV, IsMemberAccess: true);
3376 // We don't know if member is an 'ivar', but this flag is looked at
3377 // only in the context of LV.isObjCIvar().
3378 LV.setObjCArray(E->getType()->isArrayType());
3379 return;
3380 }
3381}
3382
3383static LValue EmitThreadPrivateVarDeclLValue(
3384 CodeGenFunction &CGF, const VarDecl *VD, QualType T, Address Addr,
3385 llvm::Type *RealVarTy, SourceLocation Loc) {
3386 if (CGF.CGM.getLangOpts().OpenMPIRBuilder)
3387 Addr = CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(
3388 CGF, VD, VDAddr: Addr, Loc);
3389 else
3390 Addr =
3391 CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, VDAddr: Addr, Loc);
3392
3393 Addr = Addr.withElementType(ElemTy: RealVarTy);
3394 return CGF.MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3395}
3396
3397static Address emitDeclTargetVarDeclLValue(CodeGenFunction &CGF,
3398 const VarDecl *VD, QualType T) {
3399 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
3400 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
3401 // Always return an invalid address for MT_Local, and also for
3402 // MT_To/MT_Enter when unified memory is not enabled. These use direct
3403 // access (global exists in device image). Otherwise, return a valid
3404 // address.
3405 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Local ||
3406 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
3407 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
3408 !CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory()))
3409 return Address::invalid();
3410 assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
3411 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
3412 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
3413 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) &&
3414 "Expected link clause OR to clause with unified memory enabled.");
3415 QualType PtrTy = CGF.getContext().getPointerType(T: VD->getType());
3416 Address Addr = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
3417 return CGF.EmitLoadOfPointer(Ptr: Addr, PtrTy: PtrTy->castAs<PointerType>());
3418}
3419
3420Address
3421CodeGenFunction::EmitLoadOfReference(LValue RefLVal,
3422 LValueBaseInfo *PointeeBaseInfo,
3423 TBAAAccessInfo *PointeeTBAAInfo) {
3424 llvm::LoadInst *Load =
3425 Builder.CreateLoad(Addr: RefLVal.getAddress(), IsVolatile: RefLVal.isVolatile());
3426 CGM.DecorateInstructionWithTBAA(Inst: Load, TBAAInfo: RefLVal.getTBAAInfo());
3427 QualType PTy = RefLVal.getType()->getPointeeType();
3428 CharUnits Align = CGM.getNaturalTypeAlignment(
3429 T: PTy, BaseInfo: PointeeBaseInfo, TBAAInfo: PointeeTBAAInfo, /*ForPointeeType=*/forPointeeType: true);
3430 if (!PTy->isIncompleteType()) {
3431 llvm::LLVMContext &Ctx = getLLVMContext();
3432 llvm::MDBuilder MDB(Ctx);
3433 // Emit !nonnull metadata
3434 if (CGM.getTypes().getTargetAddressSpace(T: PTy) == 0 &&
3435 !CGM.getCodeGenOpts().NullPointerIsValid)
3436 Load->setMetadata(KindID: llvm::LLVMContext::MD_nonnull,
3437 Node: llvm::MDNode::get(Context&: Ctx, MDs: {}));
3438 // Emit !align metadata
3439 if (PTy->isObjectType()) {
3440 auto AlignVal = Align.getQuantity();
3441 if (AlignVal > 1) {
3442 Load->setMetadata(
3443 KindID: llvm::LLVMContext::MD_align,
3444 Node: llvm::MDNode::get(Context&: Ctx, MDs: MDB.createConstant(C: llvm::ConstantInt::get(
3445 Ty: Builder.getInt64Ty(), V: AlignVal))));
3446 }
3447 }
3448 }
3449 return makeNaturalAddressForPointer(Ptr: Load, T: PTy, Alignment: Align,
3450 /*ForPointeeType=*/true, BaseInfo: PointeeBaseInfo,
3451 TBAAInfo: PointeeTBAAInfo);
3452}
3453
3454LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) {
3455 LValueBaseInfo PointeeBaseInfo;
3456 TBAAAccessInfo PointeeTBAAInfo;
3457 Address PointeeAddr = EmitLoadOfReference(RefLVal, PointeeBaseInfo: &PointeeBaseInfo,
3458 PointeeTBAAInfo: &PointeeTBAAInfo);
3459 return MakeAddrLValue(Addr: PointeeAddr, T: RefLVal.getType()->getPointeeType(),
3460 BaseInfo: PointeeBaseInfo, TBAAInfo: PointeeTBAAInfo);
3461}
3462
3463Address CodeGenFunction::EmitLoadOfPointer(Address Ptr,
3464 const PointerType *PtrTy,
3465 LValueBaseInfo *BaseInfo,
3466 TBAAAccessInfo *TBAAInfo) {
3467 llvm::Value *Addr = Builder.CreateLoad(Addr: Ptr);
3468 return makeNaturalAddressForPointer(Ptr: Addr, T: PtrTy->getPointeeType(),
3469 Alignment: CharUnits(), /*ForPointeeType=*/true,
3470 BaseInfo, TBAAInfo);
3471}
3472
3473LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr,
3474 const PointerType *PtrTy) {
3475 LValueBaseInfo BaseInfo;
3476 TBAAAccessInfo TBAAInfo;
3477 Address Addr = EmitLoadOfPointer(Ptr: PtrAddr, PtrTy, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
3478 return MakeAddrLValue(Addr, T: PtrTy->getPointeeType(), BaseInfo, TBAAInfo);
3479}
3480
3481static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
3482 const Expr *E, const VarDecl *VD) {
3483 QualType T = E->getType();
3484
3485 // If it's thread_local, emit a call to its wrapper function instead.
3486 if (VD->getTLSKind() == VarDecl::TLS_Dynamic &&
3487 CGF.CGM.getCXXABI().usesThreadWrapperFunction(VD))
3488 return CGF.CGM.getCXXABI().EmitThreadLocalVarDeclLValue(CGF, VD, LValType: T);
3489 // Check if the variable is marked as declare target with link clause in
3490 // device codegen.
3491 if (CGF.getLangOpts().OpenMPIsTargetDevice) {
3492 Address Addr = emitDeclTargetVarDeclLValue(CGF, VD, T);
3493 if (Addr.isValid())
3494 return CGF.MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3495 }
3496
3497 // Global HLSL resource arrays initialized on access; create a temporary with
3498 // the initialized global resource array.
3499 if (CGF.getLangOpts().HLSL && VD->getType()->isHLSLResourceRecordArray()) {
3500 std::optional<LValue> LV =
3501 CGF.CGM.getHLSLRuntime().emitGlobalResourceArrayAsLValue(CGF, ArrayDecl: VD);
3502 if (LV.has_value())
3503 return LV.value();
3504 }
3505
3506 llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(D: VD);
3507
3508 if (VD->getTLSKind() != VarDecl::TLS_None)
3509 V = CGF.Builder.CreateThreadLocalAddress(Ptr: V);
3510
3511 llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(T: VD->getType());
3512 CharUnits Alignment = CGF.getContext().getDeclAlign(D: VD);
3513 Address Addr(V, RealVarTy, Alignment);
3514 // Emit reference to the private copy of the variable if it is an OpenMP
3515 // threadprivate variable.
3516 if (CGF.getLangOpts().OpenMP && !CGF.getLangOpts().OpenMPSimd &&
3517 VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
3518 return EmitThreadPrivateVarDeclLValue(CGF, VD, T, Addr, RealVarTy,
3519 Loc: E->getExprLoc());
3520 }
3521 LValue LV = VD->getType()->isReferenceType() ?
3522 CGF.EmitLoadOfReferenceLValue(RefAddr: Addr, RefTy: VD->getType(),
3523 Source: AlignmentSource::Decl) :
3524 CGF.MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3525 setObjCGCLValueClass(Ctx: CGF.getContext(), E, LV);
3526 return LV;
3527}
3528
3529llvm::Constant *CodeGenModule::getRawFunctionPointer(GlobalDecl GD,
3530 llvm::Type *Ty) {
3531 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
3532 if (FD->hasAttr<WeakRefAttr>()) {
3533 ConstantAddress aliasee = GetWeakRefReference(VD: FD);
3534 return aliasee.getPointer();
3535 }
3536
3537 llvm::Constant *V = GetAddrOfFunction(GD, Ty);
3538 return V;
3539}
3540
3541static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF, const Expr *E,
3542 GlobalDecl GD) {
3543 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
3544 llvm::Constant *V = CGF.CGM.getFunctionPointer(GD);
3545 QualType ETy = E->getType();
3546 if (ETy->isCFIUncheckedCalleeFunctionType()) {
3547 if (auto *GV = dyn_cast<llvm::GlobalValue>(Val: V))
3548 V = llvm::NoCFIValue::get(GV);
3549 }
3550 CharUnits Alignment = CGF.getContext().getDeclAlign(D: FD);
3551 return CGF.MakeAddrLValue(V, T: ETy, Alignment, Source: AlignmentSource::Decl);
3552}
3553
3554static LValue EmitCapturedFieldLValue(CodeGenFunction &CGF, const FieldDecl *FD,
3555 llvm::Value *ThisValue) {
3556
3557 return CGF.EmitLValueForLambdaField(Field: FD, ThisValue);
3558}
3559
3560/// Named Registers are named metadata pointing to the register name
3561/// which will be read from/written to as an argument to the intrinsic
3562/// @llvm.read/write_register.
3563/// So far, only the name is being passed down, but other options such as
3564/// register type, allocation type or even optimization options could be
3565/// passed down via the metadata node.
3566static LValue EmitGlobalNamedRegister(const VarDecl *VD, CodeGenModule &CGM) {
3567 SmallString<64> Name("llvm.named.register.");
3568 AsmLabelAttr *Asm = VD->getAttr<AsmLabelAttr>();
3569 assert(Asm->getLabel().size() < 64-Name.size() &&
3570 "Register name too big");
3571 Name.append(RHS: Asm->getLabel());
3572 llvm::NamedMDNode *M =
3573 CGM.getModule().getOrInsertNamedMetadata(Name);
3574 if (M->getNumOperands() == 0) {
3575 llvm::MDString *Str = llvm::MDString::get(Context&: CGM.getLLVMContext(),
3576 Str: Asm->getLabel());
3577 llvm::Metadata *Ops[] = {Str};
3578 M->addOperand(M: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Ops));
3579 }
3580
3581 CharUnits Alignment = CGM.getContext().getDeclAlign(D: VD);
3582
3583 llvm::Value *Ptr =
3584 llvm::MetadataAsValue::get(Context&: CGM.getLLVMContext(), MD: M->getOperand(i: 0));
3585 return LValue::MakeGlobalReg(V: Ptr, alignment: Alignment, type: VD->getType());
3586}
3587
3588/// Determine whether we can emit a reference to \p VD from the current
3589/// context, despite not necessarily having seen an odr-use of the variable in
3590/// this context.
3591static bool canEmitSpuriousReferenceToVariable(CodeGenFunction &CGF,
3592 const DeclRefExpr *E,
3593 const VarDecl *VD) {
3594 // For a variable declared in an enclosing scope, do not emit a spurious
3595 // reference even if we have a capture, as that will emit an unwarranted
3596 // reference to our capture state, and will likely generate worse code than
3597 // emitting a local copy.
3598 if (E->refersToEnclosingVariableOrCapture())
3599 return false;
3600
3601 // For a local declaration declared in this function, we can always reference
3602 // it even if we don't have an odr-use.
3603 if (VD->hasLocalStorage()) {
3604 return VD->getDeclContext() ==
3605 dyn_cast_or_null<DeclContext>(Val: CGF.CurCodeDecl);
3606 }
3607
3608 // For a global declaration, we can emit a reference to it if we know
3609 // for sure that we are able to emit a definition of it.
3610 VD = VD->getDefinition(C&: CGF.getContext());
3611 if (!VD)
3612 return false;
3613
3614 // Don't emit a spurious reference if it might be to a variable that only
3615 // exists on a different device / target.
3616 // FIXME: This is unnecessarily broad. Check whether this would actually be a
3617 // cross-target reference.
3618 if (CGF.getLangOpts().OpenMP || CGF.getLangOpts().CUDA ||
3619 CGF.getLangOpts().OpenCL) {
3620 return false;
3621 }
3622
3623 // We can emit a spurious reference only if the linkage implies that we'll
3624 // be emitting a non-interposable symbol that will be retained until link
3625 // time.
3626 switch (CGF.CGM.getLLVMLinkageVarDefinition(VD)) {
3627 case llvm::GlobalValue::ExternalLinkage:
3628 case llvm::GlobalValue::LinkOnceODRLinkage:
3629 case llvm::GlobalValue::WeakODRLinkage:
3630 case llvm::GlobalValue::InternalLinkage:
3631 case llvm::GlobalValue::PrivateLinkage:
3632 return true;
3633 default:
3634 return false;
3635 }
3636}
3637
3638/// Emit an LValue for a structured binding captured in an OpenMP region.
3639/// Handles extracting individual bindings from the captured decomposed
3640/// declaration (struct fields, array elements, etc.).
3641LValue CodeGenFunction::EmitOMPCapturedBindingLValue(const BindingDecl *BD) {
3642 assert(CapturedStmtInfo && "Expected to be inside a captured region");
3643 assert(CapturedStmtInfo->getKind() == CapturedRegionKind::CR_OpenMP &&
3644 "Expected OpenMP captured region");
3645 assert(CGM.getLangOpts().OpenMP && "Expected OpenMP to be enabled");
3646
3647 if (auto It = LocalDeclMap.find(Val: BD->getCanonicalDecl());
3648 It != LocalDeclMap.end())
3649 return MakeAddrLValue(Addr: It->second, T: BD->getType());
3650
3651 const auto *DD = cast<VarDecl>(Val: BD->getDecomposedDecl());
3652
3653 // Use getNonReferenceType() because we need the actual object type, not the
3654 // reference type. DeclRefExpr with VK_LValue requires a non-reference type
3655 // (AST invariant). EmitDeclRefLValue will load any reference for us.
3656 QualType DREType = DD->getType().getNonReferenceType();
3657 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(DD),
3658 /*RefersToEnclosingVariableOrCapture=*/true, DREType,
3659 VK_LValue, SourceLocation());
3660 LValue BaseLVal = EmitDeclRefLValue(E: &DRE);
3661
3662 // Ensure the Address has the correct element type for DD's type.
3663 // EmitDeclRefLValue might return an address with a different element type
3664 // if reference unwrapping occurred.
3665 Address BaseAddr = BaseLVal.getAddress();
3666 QualType DDType = DD->getType();
3667 llvm::Type *ExpectedTy = CGM.getTypes().ConvertTypeForMem(T: DDType);
3668 if (BaseAddr.getElementType() != ExpectedTy)
3669 BaseAddr = BaseAddr.withElementType(ElemTy: ExpectedTy);
3670
3671 // Now emit the binding expression (array subscript, member access, etc.)
3672 // by temporarily installing the decomposed storage address, then routing
3673 // through EmitLValue for the binding expression.
3674 Expr *BindingExpr = BD->getBinding();
3675 auto It = LocalDeclMap.find(Val: DD);
3676 bool WasMapped = It != LocalDeclMap.end();
3677 Address SavedAddr = WasMapped ? It->second : Address::invalid();
3678 Address MapAddr = BaseAddr;
3679 if (DD->getType()->isReferenceType()) {
3680 RawAddress RefSlot = CreateMemTemp(Ty: DD->getType(), Name: "omp.binding.ref");
3681 Builder.CreateStore(Val: BaseAddr.emitRawPointer(CGF&: *this), Addr: RefSlot);
3682 MapAddr = RefSlot;
3683 }
3684 if (WasMapped)
3685 It->second = MapAddr;
3686 else
3687 LocalDeclMap.insert(KV: {DD, MapAddr});
3688 llvm::scope_exit Guard([&] {
3689 if (WasMapped) {
3690 auto RestoreIt = LocalDeclMap.find(Val: DD);
3691 assert(RestoreIt != LocalDeclMap.end() && "DD should still be in map");
3692 RestoreIt->second = SavedAddr;
3693 } else {
3694 LocalDeclMap.erase(Val: DD);
3695 }
3696 });
3697
3698 return EmitLValue(E: BindingExpr);
3699}
3700
3701LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
3702 const NamedDecl *ND = E->getDecl();
3703 QualType T = E->getType();
3704
3705 assert(E->isNonOdrUse() != NOUR_Unevaluated &&
3706 "should not emit an unevaluated operand");
3707
3708 if (const auto *VD = dyn_cast<VarDecl>(Val: ND)) {
3709 // Global Named registers access via intrinsics only
3710 if (VD->getStorageClass() == SC_Register &&
3711 VD->hasAttr<AsmLabelAttr>() && !VD->isLocalVarDecl())
3712 return EmitGlobalNamedRegister(VD, CGM);
3713
3714 // If this DeclRefExpr does not constitute an odr-use of the variable,
3715 // we're not permitted to emit a reference to it in general, and it might
3716 // not be captured if capture would be necessary for a use. Emit the
3717 // constant value directly instead.
3718 if (E->isNonOdrUse() == NOUR_Constant &&
3719 (VD->getType()->isReferenceType() ||
3720 !canEmitSpuriousReferenceToVariable(CGF&: *this, E, VD))) {
3721 VD->getAnyInitializer(D&: VD);
3722 llvm::Constant *Val = ConstantEmitter(*this).emitAbstract(
3723 loc: E->getLocation(), value: *VD->evaluateValue(), T: VD->getType());
3724 assert(Val && "failed to emit constant expression");
3725
3726 Address Addr = Address::invalid();
3727 if (!VD->getType()->isReferenceType()) {
3728 // Spill the constant value to a global.
3729 Addr = CGM.createUnnamedGlobalFrom(D: *VD, Constant: Val,
3730 Align: getContext().getDeclAlign(D: VD));
3731 llvm::Type *VarTy = getTypes().ConvertTypeForMem(T: VD->getType());
3732 auto *PTy = llvm::PointerType::get(
3733 C&: getLLVMContext(), AddressSpace: getTypes().getTargetAddressSpace(T: VD->getType()));
3734 Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty: PTy, ElementTy: VarTy);
3735 } else {
3736 // Should we be using the alignment of the constant pointer we emitted?
3737 CharUnits Alignment =
3738 CGM.getNaturalTypeAlignment(T: E->getType(),
3739 /* BaseInfo= */ nullptr,
3740 /* TBAAInfo= */ nullptr,
3741 /* forPointeeType= */ true);
3742 Addr = makeNaturalAddressForPointer(Ptr: Val, T, Alignment);
3743 }
3744 return MakeAddrLValue(Addr, T, Source: AlignmentSource::Decl);
3745 }
3746
3747 // FIXME: Handle other kinds of non-odr-use DeclRefExprs.
3748
3749 // Check for captured variables.
3750 if (E->refersToEnclosingVariableOrCapture()) {
3751 VD = VD->getCanonicalDecl();
3752 if (auto *FD = LambdaCaptureFields.lookup(Val: VD))
3753 return EmitCapturedFieldLValue(CGF&: *this, FD, ThisValue: CXXABIThisValue);
3754 if (CapturedStmtInfo) {
3755 auto I = LocalDeclMap.find(Val: VD);
3756 if (I != LocalDeclMap.end()) {
3757 LValue CapLVal;
3758 if (VD->getType()->isReferenceType())
3759 CapLVal = EmitLoadOfReferenceLValue(RefAddr: I->second, RefTy: VD->getType(),
3760 Source: AlignmentSource::Decl);
3761 else
3762 CapLVal = MakeAddrLValue(Addr: I->second, T);
3763 // Mark lvalue as nontemporal if the variable is marked as nontemporal
3764 // in simd context.
3765 if (getLangOpts().OpenMP &&
3766 CGM.getOpenMPRuntime().isNontemporalDecl(VD))
3767 CapLVal.setNontemporal(/*Value=*/true);
3768 return CapLVal;
3769 }
3770 LValue CapLVal =
3771 EmitCapturedFieldLValue(CGF&: *this, FD: CapturedStmtInfo->lookup(VD),
3772 ThisValue: CapturedStmtInfo->getContextValue());
3773 Address LValueAddress = CapLVal.getAddress();
3774 CapLVal = MakeAddrLValue(Addr: Address(LValueAddress.emitRawPointer(CGF&: *this),
3775 LValueAddress.getElementType(),
3776 getContext().getDeclAlign(D: VD)),
3777 T: CapLVal.getType(),
3778 BaseInfo: LValueBaseInfo(AlignmentSource::Decl),
3779 TBAAInfo: CapLVal.getTBAAInfo());
3780 // Mark lvalue as nontemporal if the variable is marked as nontemporal
3781 // in simd context.
3782 if (getLangOpts().OpenMP &&
3783 CGM.getOpenMPRuntime().isNontemporalDecl(VD))
3784 CapLVal.setNontemporal(/*Value=*/true);
3785 return CapLVal;
3786 }
3787
3788 assert(isa<BlockDecl>(CurCodeDecl));
3789 Address addr = GetAddrOfBlockDecl(var: VD);
3790 return MakeAddrLValue(Addr: addr, T, Source: AlignmentSource::Decl);
3791 }
3792 }
3793
3794 // FIXME: We should be able to assert this for FunctionDecls as well!
3795 // FIXME: We should be able to assert this for all DeclRefExprs, not just
3796 // those with a valid source location.
3797 assert((ND->isUsed(false) || !isa<VarDecl>(ND) || E->isNonOdrUse() ||
3798 !E->getLocation().isValid()) &&
3799 "Should not use decl without marking it used!");
3800
3801 if (ND->hasAttr<WeakRefAttr>()) {
3802 const auto *VD = cast<ValueDecl>(Val: ND);
3803 ConstantAddress Aliasee = CGM.GetWeakRefReference(VD);
3804 return MakeAddrLValue(Addr: Aliasee, T, Source: AlignmentSource::Decl);
3805 }
3806
3807 if (const auto *VD = dyn_cast<VarDecl>(Val: ND)) {
3808 // Check if this is a global variable.
3809 if (VD->hasLinkage() || VD->isStaticDataMember())
3810 return EmitGlobalVarDeclLValue(CGF&: *this, E, VD);
3811
3812 Address addr = Address::invalid();
3813
3814 // The variable should generally be present in the local decl map.
3815 auto iter = LocalDeclMap.find(Val: VD);
3816 if (iter != LocalDeclMap.end()) {
3817 addr = iter->second;
3818
3819 // Otherwise, it might be static local we haven't emitted yet for
3820 // some reason; most likely, because it's in an outer function.
3821 } else if (VD->isStaticLocal()) {
3822 llvm::Constant *var = CGM.getOrCreateStaticVarDecl(
3823 D: *VD, Linkage: CGM.getLLVMLinkageVarDefinition(VD));
3824 addr = Address(
3825 var, ConvertTypeForMem(T: VD->getType()), getContext().getDeclAlign(D: VD));
3826
3827 // No other cases for now.
3828 } else {
3829 llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?");
3830 }
3831
3832 // Handle threadlocal function locals.
3833 if (VD->getTLSKind() != VarDecl::TLS_None)
3834 addr = addr.withPointer(
3835 NewPointer: Builder.CreateThreadLocalAddress(Ptr: addr.getBasePointer()),
3836 IsKnownNonNull: NotKnownNonNull);
3837
3838 // Check for OpenMP threadprivate variables.
3839 if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd &&
3840 VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
3841 return EmitThreadPrivateVarDeclLValue(
3842 CGF&: *this, VD, T, Addr: addr, RealVarTy: getTypes().ConvertTypeForMem(T: VD->getType()),
3843 Loc: E->getExprLoc());
3844 }
3845
3846 // Drill into block byref variables.
3847 bool isBlockByref = VD->isEscapingByref();
3848 if (isBlockByref) {
3849 addr = emitBlockByrefAddress(baseAddr: addr, V: VD);
3850 }
3851
3852 // Drill into reference types.
3853 LValue LV = VD->getType()->isReferenceType() ?
3854 EmitLoadOfReferenceLValue(RefAddr: addr, RefTy: VD->getType(), Source: AlignmentSource::Decl) :
3855 MakeAddrLValue(Addr: addr, T, Source: AlignmentSource::Decl);
3856
3857 bool isLocalStorage = VD->hasLocalStorage();
3858
3859 bool NonGCable = isLocalStorage &&
3860 !VD->getType()->isReferenceType() &&
3861 !isBlockByref;
3862 if (NonGCable) {
3863 LV.getQuals().removeObjCGCAttr();
3864 LV.setNonGC(true);
3865 }
3866
3867 bool isImpreciseLifetime =
3868 (isLocalStorage && !VD->hasAttr<ObjCPreciseLifetimeAttr>());
3869 if (isImpreciseLifetime)
3870 LV.setARCPreciseLifetime(ARCImpreciseLifetime);
3871 setObjCGCLValueClass(Ctx: getContext(), E, LV);
3872 return LV;
3873 }
3874
3875 if (const auto *FD = dyn_cast<FunctionDecl>(Val: ND))
3876 return EmitFunctionDeclLValue(CGF&: *this, E, GD: FD);
3877
3878 // FIXME: While we're emitting a binding from an enclosing scope, all other
3879 // DeclRefExprs we see should be implicitly treated as if they also refer to
3880 // an enclosing scope.
3881 if (const auto *BD = dyn_cast<BindingDecl>(Val: ND)) {
3882 if (E->refersToEnclosingVariableOrCapture()) {
3883 auto ApplyNontemporal = [&](LValue LV) {
3884 if (getLangOpts().OpenMP &&
3885 CGM.getOpenMPRuntime().isNontemporalDecl(VD: BD))
3886 LV.setNontemporal(/*Value=*/true);
3887 return LV;
3888 };
3889
3890 // Try direct lookup first.
3891 auto It = LocalDeclMap.find(Val: BD->getCanonicalDecl());
3892 if (It != LocalDeclMap.end()) {
3893 return ApplyNontemporal(
3894 MakeAddrLValue(Addr: It->second, T: E->getType(), Source: AlignmentSource::Decl));
3895 }
3896
3897 // OpenMP case: binding was captured via its decomposed decl.
3898 if (CapturedStmtInfo &&
3899 CapturedStmtInfo->getKind() == CapturedRegionKind::CR_OpenMP &&
3900 CGM.getLangOpts().OpenMP) {
3901 auto NameIt = OMPPrivatizedBindings.find(
3902 Val: cast<BindingDecl>(Val: BD->getCanonicalDecl()));
3903 if (NameIt != OMPPrivatizedBindings.end()) {
3904 return ApplyNontemporal(MakeAddrLValue(Addr: NameIt->second, T: E->getType(),
3905 Source: AlignmentSource::Decl));
3906 }
3907 return ApplyNontemporal(EmitOMPCapturedBindingLValue(BD));
3908 }
3909 // Non-OpenMP case: lambda capture.
3910 auto *FD = LambdaCaptureFields.lookup(Val: BD);
3911 return EmitCapturedFieldLValue(CGF&: *this, FD, ThisValue: CXXABIThisValue);
3912 }
3913 // Suppress debug location updates when visiting the binding, since the
3914 // binding may emit instructions that would otherwise be associated with the
3915 // binding itself, rather than the expression referencing the binding. (this
3916 // leads to jumpy debug stepping behavior where the location/debugger jump
3917 // back to the binding declaration, then back to the expression referencing
3918 // the binding)
3919 DisableDebugLocationUpdates D(*this);
3920 return EmitLValue(E: BD->getBinding(), IsKnownNonNull: NotKnownNonNull);
3921 }
3922
3923 // We can form DeclRefExprs naming GUID declarations when reconstituting
3924 // non-type template parameters into expressions.
3925 if (const auto *GD = dyn_cast<MSGuidDecl>(Val: ND))
3926 return MakeAddrLValue(Addr: CGM.GetAddrOfMSGuidDecl(GD), T,
3927 Source: AlignmentSource::Decl);
3928
3929 if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(Val: ND)) {
3930 ConstantAddress ATPO = CGM.GetAddrOfTemplateParamObject(TPO);
3931 auto AS = getLangASFromTargetAS(TargetAS: ATPO.getAddressSpace());
3932
3933 if (AS != T.getAddressSpace()) {
3934 auto TargetAS = getContext().getTargetAddressSpace(AS: T.getAddressSpace());
3935 llvm::Type *PtrTy =
3936 llvm::PointerType::get(C&: CGM.getLLVMContext(), AddressSpace: TargetAS);
3937 llvm::Constant *ASC = CGM.performAddrSpaceCast(Src: ATPO.getPointer(), DestTy: PtrTy);
3938 ATPO = ConstantAddress(ASC, ATPO.getElementType(), ATPO.getAlignment());
3939 }
3940
3941 return MakeAddrLValue(Addr: ATPO, T, Source: AlignmentSource::Decl);
3942 }
3943
3944 llvm_unreachable("Unhandled DeclRefExpr");
3945}
3946
3947LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
3948 // __extension__ doesn't affect lvalue-ness.
3949 if (E->getOpcode() == UO_Extension)
3950 return EmitLValue(E: E->getSubExpr());
3951
3952 QualType ExprTy = getContext().getCanonicalType(T: E->getSubExpr()->getType());
3953 switch (E->getOpcode()) {
3954 default: llvm_unreachable("Unknown unary operator lvalue!");
3955 case UO_Deref: {
3956 QualType T = E->getSubExpr()->getType()->getPointeeType();
3957 assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
3958
3959 LValueBaseInfo BaseInfo;
3960 TBAAAccessInfo TBAAInfo;
3961 Address Addr = EmitPointerWithAlignment(E: E->getSubExpr(), BaseInfo: &BaseInfo,
3962 TBAAInfo: &TBAAInfo);
3963 LValue LV = MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
3964 LV.getQuals().setAddressSpace(ExprTy.getAddressSpace());
3965
3966 // We should not generate __weak write barrier on indirect reference
3967 // of a pointer to object; as in void foo (__weak id *param); *param = 0;
3968 // But, we continue to generate __strong write barrier on indirect write
3969 // into a pointer to object.
3970 if (getLangOpts().ObjC &&
3971 getLangOpts().getGC() != LangOptions::NonGC &&
3972 LV.isObjCWeak())
3973 LV.setNonGC(!E->isOBJCGCCandidate(Ctx&: getContext()));
3974 return LV;
3975 }
3976 case UO_Real:
3977 case UO_Imag: {
3978 LValue LV = EmitLValue(E: E->getSubExpr());
3979 assert(LV.isSimple() && "real/imag on non-ordinary l-value");
3980
3981 // __real is valid on scalars. This is a faster way of testing that.
3982 // __imag can only produce an rvalue on scalars.
3983 if (E->getOpcode() == UO_Real &&
3984 !LV.getAddress().getElementType()->isStructTy()) {
3985 assert(E->getSubExpr()->getType()->isArithmeticType());
3986 return LV;
3987 }
3988
3989 QualType T = ExprTy->castAs<ComplexType>()->getElementType();
3990
3991 Address Component =
3992 (E->getOpcode() == UO_Real
3993 ? emitAddrOfRealComponent(complex: LV.getAddress(), complexType: LV.getType())
3994 : emitAddrOfImagComponent(complex: LV.getAddress(), complexType: LV.getType()));
3995 LValue ElemLV = MakeAddrLValue(Addr: Component, T, BaseInfo: LV.getBaseInfo(),
3996 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: T));
3997 ElemLV.getQuals().addQualifiers(Q: LV.getQuals());
3998 return ElemLV;
3999 }
4000 case UO_PreInc:
4001 case UO_PreDec: {
4002 LValue LV = EmitLValue(E: E->getSubExpr());
4003 bool isInc = E->getOpcode() == UO_PreInc;
4004
4005 if (E->getType()->isAnyComplexType())
4006 EmitComplexPrePostIncDec(E, LV, isInc, isPre: true/*isPre*/);
4007 else
4008 EmitScalarPrePostIncDec(E, LV, isInc, isPre: true/*isPre*/);
4009 return LV;
4010 }
4011 }
4012}
4013
4014LValue CodeGenFunction::EmitStringLiteralLValue(const StringLiteral *E) {
4015 return MakeAddrLValue(Addr: CGM.GetAddrOfConstantStringFromLiteral(S: E),
4016 T: E->getType(), Source: AlignmentSource::Decl);
4017}
4018
4019LValue CodeGenFunction::EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E) {
4020 return MakeAddrLValue(Addr: CGM.GetAddrOfConstantStringFromObjCEncode(E),
4021 T: E->getType(), Source: AlignmentSource::Decl);
4022}
4023
4024LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
4025 auto SL = E->getFunctionName();
4026 assert(SL != nullptr && "No StringLiteral name in PredefinedExpr");
4027 StringRef FnName = CurFn->getName();
4028 FnName.consume_front(Prefix: "\01");
4029 StringRef NameItems[] = {
4030 PredefinedExpr::getIdentKindName(IK: E->getIdentKind()), FnName};
4031 std::string GVName = llvm::join(Begin: NameItems, End: NameItems + 2, Separator: ".");
4032 if (auto *BD = dyn_cast_or_null<BlockDecl>(Val: CurCodeDecl)) {
4033 std::string Name = std::string(SL->getString());
4034 if (!Name.empty()) {
4035 unsigned Discriminator =
4036 CGM.getCXXABI().getMangleContext().getBlockId(BD, Local: true);
4037 if (Discriminator)
4038 Name += "_" + Twine(Discriminator + 1).str();
4039 auto C = CGM.GetAddrOfConstantCString(Str: Name, GlobalName: GVName);
4040 return MakeAddrLValue(Addr: C, T: E->getType(), Source: AlignmentSource::Decl);
4041 } else {
4042 auto C = CGM.GetAddrOfConstantCString(Str: std::string(FnName), GlobalName: GVName);
4043 return MakeAddrLValue(Addr: C, T: E->getType(), Source: AlignmentSource::Decl);
4044 }
4045 }
4046 auto C = CGM.GetAddrOfConstantStringFromLiteral(S: SL, Name: GVName);
4047 return MakeAddrLValue(Addr: C, T: E->getType(), Source: AlignmentSource::Decl);
4048}
4049
4050/// Emit a type description suitable for use by a runtime sanitizer library. The
4051/// format of a type descriptor is
4052///
4053/// \code
4054/// { i16 TypeKind, i16 TypeInfo }
4055/// \endcode
4056///
4057/// followed by an array of i8 containing the type name with extra information
4058/// for BitInt. TypeKind is TK_Integer(0) for an integer, TK_Float(1) for a
4059/// floating point value, TK_BitInt(2) for BitInt and TK_Unknown(0xFFFF) for
4060/// anything else.
4061llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) {
4062 // Only emit each type's descriptor once.
4063 if (llvm::Constant *C = CGM.getTypeDescriptorFromMap(Ty: T))
4064 return C;
4065
4066 uint16_t TypeKind = TK_Unknown;
4067 uint16_t TypeInfo = 0;
4068 bool IsBitInt = false;
4069
4070 if (T->isIntegerType()) {
4071 TypeKind = TK_Integer;
4072 TypeInfo = (llvm::Log2_32(Value: getContext().getTypeSize(T)) << 1) |
4073 (T->isSignedIntegerType() ? 1 : 0);
4074 // Follow suggestion from discussion of issue 64100.
4075 // So we can write the exact amount of bits in TypeName after '\0'
4076 // making it <diagnostic-like type name>.'\0'.<32-bit width>.
4077 if (T->isSignedIntegerType() && T->getAs<BitIntType>()) {
4078 // Do a sanity checks as we are using 32-bit type to store bit length.
4079 assert(getContext().getTypeSize(T) > 0 &&
4080 " non positive amount of bits in __BitInt type");
4081 assert(getContext().getTypeSize(T) <= 0xFFFFFFFF &&
4082 " too many bits in __BitInt type");
4083
4084 // Redefine TypeKind with the actual __BitInt type if we have signed
4085 // BitInt.
4086 TypeKind = TK_BitInt;
4087 IsBitInt = true;
4088 }
4089 } else if (T->isFloatingType()) {
4090 TypeKind = TK_Float;
4091 TypeInfo = getContext().getTypeSize(T);
4092 }
4093
4094 // Format the type name as if for a diagnostic, including quotes and
4095 // optionally an 'aka'.
4096 SmallString<32> Buffer;
4097 CGM.getDiags().ConvertArgToString(Kind: DiagnosticsEngine::ak_qualtype,
4098 Val: (intptr_t)T.getAsOpaquePtr(), Modifier: StringRef(),
4099 Argument: StringRef(), PrevArgs: {}, Output&: Buffer, QualTypeVals: {});
4100
4101 if (IsBitInt) {
4102 // The Structure is: 0 to end the string, 32 bit unsigned integer in target
4103 // endianness, zero.
4104 char S[6] = {'\0', '\0', '\0', '\0', '\0', '\0'};
4105 const auto *EIT = T->castAs<BitIntType>();
4106 uint32_t Bits = EIT->getNumBits();
4107 llvm::support::endian::write32(P: S + 1, V: Bits,
4108 E: getTarget().isBigEndian()
4109 ? llvm::endianness::big
4110 : llvm::endianness::little);
4111 StringRef Str = StringRef(S, sizeof(S) / sizeof(decltype(S[0])));
4112 Buffer.append(RHS: Str);
4113 }
4114
4115 llvm::Constant *Components[] = {
4116 Builder.getInt16(C: TypeKind), Builder.getInt16(C: TypeInfo),
4117 llvm::ConstantDataArray::getString(Context&: getLLVMContext(), Initializer: Buffer)
4118 };
4119 llvm::Constant *Descriptor = llvm::ConstantStruct::getAnon(V: Components);
4120
4121 auto *GV = new llvm::GlobalVariable(
4122 CGM.getModule(), Descriptor->getType(),
4123 /*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage, Descriptor);
4124 GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4125 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV);
4126
4127 // Remember the descriptor for this type.
4128 CGM.setTypeDescriptorInMap(Ty: T, C: GV);
4129
4130 return GV;
4131}
4132
4133llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) {
4134 llvm::Type *TargetTy = IntPtrTy;
4135
4136 if (V->getType() == TargetTy)
4137 return V;
4138
4139 // Floating-point types which fit into intptr_t are bitcast to integers
4140 // and then passed directly (after zero-extension, if necessary).
4141 if (V->getType()->isFloatingPointTy()) {
4142 unsigned Bits = V->getType()->getPrimitiveSizeInBits().getFixedValue();
4143 if (Bits <= TargetTy->getIntegerBitWidth())
4144 V = Builder.CreateBitCast(V, DestTy: llvm::Type::getIntNTy(C&: getLLVMContext(),
4145 N: Bits));
4146 }
4147
4148 // Integers which fit in intptr_t are zero-extended and passed directly.
4149 if (V->getType()->isIntegerTy() &&
4150 V->getType()->getIntegerBitWidth() <= TargetTy->getIntegerBitWidth())
4151 return Builder.CreateZExt(V, DestTy: TargetTy);
4152
4153 // Pointers are passed directly, everything else is passed by address.
4154 if (!V->getType()->isPointerTy()) {
4155 RawAddress Ptr = CreateDefaultAlignTempAlloca(Ty: V->getType());
4156 Builder.CreateStore(Val: V, Addr: Ptr);
4157 V = Ptr.getPointer();
4158 }
4159 return Builder.CreatePtrToInt(V, DestTy: TargetTy);
4160}
4161
4162/// Emit a representation of a SourceLocation for passing to a handler
4163/// in a sanitizer runtime library. The format for this data is:
4164/// \code
4165/// struct SourceLocation {
4166/// const char *Filename;
4167/// int32_t Line, Column;
4168/// };
4169/// \endcode
4170/// For an invalid SourceLocation, the Filename pointer is null.
4171llvm::Constant *CodeGenFunction::EmitCheckSourceLocation(SourceLocation Loc) {
4172 llvm::Constant *Filename;
4173 int Line, Column;
4174
4175 PresumedLoc PLoc = getContext().getSourceManager().getPresumedLoc(Loc);
4176 if (PLoc.isValid()) {
4177 StringRef FilenameString = PLoc.getFilename();
4178
4179 int PathComponentsToStrip =
4180 CGM.getCodeGenOpts().EmitCheckPathComponentsToStrip;
4181 if (PathComponentsToStrip < 0) {
4182 assert(PathComponentsToStrip != INT_MIN);
4183 int PathComponentsToKeep = -PathComponentsToStrip;
4184 auto I = llvm::sys::path::rbegin(path: FilenameString);
4185 auto E = llvm::sys::path::rend(path: FilenameString);
4186 while (I != E && --PathComponentsToKeep)
4187 ++I;
4188
4189 FilenameString = FilenameString.substr(Start: I - E);
4190 } else if (PathComponentsToStrip > 0) {
4191 auto I = llvm::sys::path::begin(path: FilenameString);
4192 auto E = llvm::sys::path::end(path: FilenameString);
4193 while (I != E && PathComponentsToStrip--)
4194 ++I;
4195
4196 if (I != E)
4197 FilenameString =
4198 FilenameString.substr(Start: I - llvm::sys::path::begin(path: FilenameString));
4199 else
4200 FilenameString = llvm::sys::path::filename(path: FilenameString);
4201 }
4202
4203 auto FilenameGV =
4204 CGM.GetAddrOfConstantCString(Str: std::string(FilenameString), GlobalName: ".src");
4205 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(
4206 GV: cast<llvm::GlobalVariable>(
4207 Val: FilenameGV.getPointer()->stripPointerCasts()));
4208 Filename = FilenameGV.getPointer();
4209 Line = PLoc.getLine();
4210 Column = PLoc.getColumn();
4211 } else {
4212 Filename = llvm::Constant::getNullValue(Ty: Int8PtrTy);
4213 Line = Column = 0;
4214 }
4215
4216 llvm::Constant *Data[] = {Filename, Builder.getInt32(C: Line),
4217 Builder.getInt32(C: Column)};
4218
4219 return llvm::ConstantStruct::getAnon(V: Data);
4220}
4221
4222namespace {
4223/// Specify under what conditions this check can be recovered
4224enum class CheckRecoverableKind {
4225 /// Always terminate program execution if this check fails.
4226 Unrecoverable,
4227 /// Check supports recovering, runtime has both fatal (noreturn) and
4228 /// non-fatal handlers for this check.
4229 Recoverable,
4230 /// Runtime conditionally aborts, always need to support recovery.
4231 AlwaysRecoverable
4232};
4233}
4234
4235static CheckRecoverableKind
4236getRecoverableKind(SanitizerKind::SanitizerOrdinal Ordinal) {
4237 if (Ordinal == SanitizerKind::SO_Vptr)
4238 return CheckRecoverableKind::AlwaysRecoverable;
4239 else if (Ordinal == SanitizerKind::SO_Return ||
4240 Ordinal == SanitizerKind::SO_Unreachable)
4241 return CheckRecoverableKind::Unrecoverable;
4242 else
4243 return CheckRecoverableKind::Recoverable;
4244}
4245
4246namespace {
4247struct SanitizerHandlerInfo {
4248 char const *const Name;
4249 unsigned Version;
4250};
4251}
4252
4253const SanitizerHandlerInfo SanitizerHandlers[] = {
4254#define SANITIZER_CHECK(Enum, Name, Version, Msg) {#Name, Version},
4255 LIST_SANITIZER_CHECKS
4256#undef SANITIZER_CHECK
4257};
4258
4259static void emitCheckHandlerCall(CodeGenFunction &CGF,
4260 llvm::FunctionType *FnType,
4261 ArrayRef<llvm::Value *> FnArgs,
4262 SanitizerHandler CheckHandler,
4263 CheckRecoverableKind RecoverKind, bool IsFatal,
4264 llvm::BasicBlock *ContBB, bool NoMerge) {
4265 assert(IsFatal || RecoverKind != CheckRecoverableKind::Unrecoverable);
4266 std::optional<ApplyDebugLocation> DL;
4267 if (!CGF.Builder.getCurrentDebugLocation()) {
4268 // Ensure that the call has at least an artificial debug location.
4269 DL.emplace(args&: CGF, args: SourceLocation());
4270 }
4271 bool NeedsAbortSuffix =
4272 IsFatal && RecoverKind != CheckRecoverableKind::Unrecoverable;
4273 bool MinimalRuntime = CGF.CGM.getCodeGenOpts().SanitizeMinimalRuntime;
4274 bool HandlerPreserveAllRegs =
4275 CGF.CGM.getCodeGenOpts().SanitizeHandlerPreserveAllRegs;
4276 const SanitizerHandlerInfo &CheckInfo = SanitizerHandlers[CheckHandler];
4277 const StringRef CheckName = CheckInfo.Name;
4278 std::string FnName = "__ubsan_handle_" + CheckName.str();
4279 if (CheckInfo.Version && !MinimalRuntime)
4280 FnName += "_v" + llvm::utostr(X: CheckInfo.Version);
4281 if (MinimalRuntime)
4282 FnName += "_minimal";
4283 if (NeedsAbortSuffix)
4284 FnName += "_abort";
4285 if (HandlerPreserveAllRegs && !NeedsAbortSuffix)
4286 FnName += "_preserve";
4287 bool MayReturn =
4288 !IsFatal || RecoverKind == CheckRecoverableKind::AlwaysRecoverable;
4289
4290 llvm::AttrBuilder B(CGF.getLLVMContext());
4291 if (!MayReturn) {
4292 B.addAttribute(Val: llvm::Attribute::NoReturn)
4293 .addAttribute(Val: llvm::Attribute::NoUnwind);
4294 }
4295 B.addUWTableAttr(Kind: llvm::UWTableKind::Default);
4296
4297 llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(
4298 Ty: FnType, Name: FnName,
4299 ExtraAttrs: llvm::AttributeList::get(C&: CGF.getLLVMContext(),
4300 Index: llvm::AttributeList::FunctionIndex, B),
4301 /*Local=*/true);
4302 llvm::CallInst *HandlerCall = CGF.EmitNounwindRuntimeCall(callee: Fn, args: FnArgs);
4303 NoMerge = NoMerge || !CGF.CGM.getCodeGenOpts().isOptimizedBuild() ||
4304 (CGF.CurCodeDecl && CGF.CurCodeDecl->hasAttr<OptimizeNoneAttr>());
4305 if (NoMerge)
4306 HandlerCall->addFnAttr(Kind: llvm::Attribute::NoMerge);
4307 if (HandlerPreserveAllRegs && !NeedsAbortSuffix) {
4308 // N.B. there is also a clang::CallingConv which is not what we want here.
4309 HandlerCall->setCallingConv(llvm::CallingConv::PreserveAll);
4310 }
4311 if (!MayReturn) {
4312 HandlerCall->setDoesNotReturn();
4313 CGF.Builder.CreateUnreachable();
4314 } else {
4315 CGF.Builder.CreateBr(Dest: ContBB);
4316 }
4317}
4318
4319void CodeGenFunction::EmitCheck(
4320 ArrayRef<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>> Checked,
4321 SanitizerHandler CheckHandler, ArrayRef<llvm::Constant *> StaticArgs,
4322 ArrayRef<llvm::Value *> DynamicArgs, const TrapReason *TR) {
4323 assert(IsSanitizerScope);
4324 assert(Checked.size() > 0);
4325 assert(CheckHandler >= 0 &&
4326 size_t(CheckHandler) < std::size(SanitizerHandlers));
4327 const StringRef CheckName = SanitizerHandlers[CheckHandler].Name;
4328
4329 llvm::Value *FatalCond = nullptr;
4330 llvm::Value *RecoverableCond = nullptr;
4331 llvm::Value *TrapCond = nullptr;
4332 bool NoMerge = false;
4333 // Expand checks into:
4334 // (Check1 || !allow_ubsan_check) && (Check2 || !allow_ubsan_check) ...
4335 // We need separate allow_ubsan_check intrinsics because they have separately
4336 // specified cutoffs.
4337 // This expression looks expensive but will be simplified after
4338 // LowerAllowCheckPass.
4339 for (auto &[Check, Ord] : Checked) {
4340 llvm::Value *GuardedCheck = Check;
4341 if (ClSanitizeGuardChecks ||
4342 (CGM.getCodeGenOpts().SanitizeSkipHotCutoffs[Ord] > 0)) {
4343 llvm::Value *Allow = Builder.CreateCall(
4344 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::allow_ubsan_check),
4345 Args: llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: Ord));
4346 GuardedCheck = Builder.CreateOr(LHS: Check, RHS: Builder.CreateNot(V: Allow));
4347 }
4348
4349 // -fsanitize-trap= overrides -fsanitize-recover=.
4350 llvm::Value *&Cond = CGM.getCodeGenOpts().SanitizeTrap.has(O: Ord) ? TrapCond
4351 : CGM.getCodeGenOpts().SanitizeRecover.has(O: Ord)
4352 ? RecoverableCond
4353 : FatalCond;
4354 Cond = Cond ? Builder.CreateAnd(LHS: Cond, RHS: GuardedCheck) : GuardedCheck;
4355
4356 if (!CGM.getCodeGenOpts().SanitizeMergeHandlers.has(O: Ord))
4357 NoMerge = true;
4358 }
4359
4360 if (TrapCond)
4361 EmitTrapCheck(Checked: TrapCond, CheckHandlerID: CheckHandler, NoMerge, TR);
4362 if (!FatalCond && !RecoverableCond)
4363 return;
4364
4365 llvm::Value *JointCond;
4366 if (FatalCond && RecoverableCond)
4367 JointCond = Builder.CreateAnd(LHS: FatalCond, RHS: RecoverableCond);
4368 else
4369 JointCond = FatalCond ? FatalCond : RecoverableCond;
4370 assert(JointCond);
4371
4372 CheckRecoverableKind RecoverKind = getRecoverableKind(Ordinal: Checked[0].second);
4373 assert(SanOpts.has(Checked[0].second));
4374#ifndef NDEBUG
4375 for (int i = 1, n = Checked.size(); i < n; ++i) {
4376 assert(RecoverKind == getRecoverableKind(Checked[i].second) &&
4377 "All recoverable kinds in a single check must be same!");
4378 assert(SanOpts.has(Checked[i].second));
4379 }
4380#endif
4381
4382 llvm::BasicBlock *Cont = createBasicBlock(name: "cont");
4383 llvm::BasicBlock *Handlers = createBasicBlock(name: "handler." + CheckName);
4384 llvm::Instruction *Branch = Builder.CreateCondBr(Cond: JointCond, True: Cont, False: Handlers);
4385 // Give hint that we very much don't expect to execute the handler
4386 llvm::MDBuilder MDHelper(getLLVMContext());
4387 llvm::MDNode *Node = MDHelper.createLikelyBranchWeights();
4388 Branch->setMetadata(KindID: llvm::LLVMContext::MD_prof, Node);
4389 EmitBlock(BB: Handlers);
4390
4391 // Clear arguments for the MinimalRuntime handler.
4392 if (CGM.getCodeGenOpts().SanitizeMinimalRuntime) {
4393 StaticArgs = {};
4394 DynamicArgs = {};
4395 }
4396
4397 // Handler functions take an i8* pointing to the (handler-specific) static
4398 // information block, followed by a sequence of intptr_t arguments
4399 // representing operand values.
4400 SmallVector<llvm::Value *, 4> Args;
4401 SmallVector<llvm::Type *, 4> ArgTypes;
4402
4403 Args.reserve(N: DynamicArgs.size() + 1);
4404 ArgTypes.reserve(N: DynamicArgs.size() + 1);
4405
4406 // Emit handler arguments and create handler function type.
4407 if (!StaticArgs.empty()) {
4408 llvm::Constant *Info = llvm::ConstantStruct::getAnon(V: StaticArgs);
4409 auto *InfoPtr = new llvm::GlobalVariable(
4410 CGM.getModule(), Info->getType(),
4411 // Non-constant global is used in a handler to deduplicate reports.
4412 // TODO: change deduplication logic and make it constant.
4413 /*isConstant=*/false, llvm::GlobalVariable::PrivateLinkage, Info, "",
4414 nullptr, llvm::GlobalVariable::NotThreadLocal,
4415 CGM.getDataLayout().getDefaultGlobalsAddressSpace());
4416 InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4417 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV: InfoPtr);
4418 Args.push_back(Elt: Builder.CreateAddrSpaceCast(V: InfoPtr, DestTy: CGM.VoidPtrTy));
4419 ArgTypes.push_back(Elt: CGM.VoidPtrTy);
4420 }
4421
4422 for (llvm::Value *DynamicArg : DynamicArgs) {
4423 Args.push_back(Elt: EmitCheckValue(V: DynamicArg));
4424 ArgTypes.push_back(Elt: IntPtrTy);
4425 }
4426
4427 llvm::FunctionType *FnType =
4428 llvm::FunctionType::get(Result: CGM.VoidTy, Params: ArgTypes, isVarArg: false);
4429
4430 if (!FatalCond || !RecoverableCond) {
4431 // Simple case: we need to generate a single handler call, either
4432 // fatal, or non-fatal.
4433 emitCheckHandlerCall(CGF&: *this, FnType, FnArgs: Args, CheckHandler, RecoverKind,
4434 IsFatal: (FatalCond != nullptr), ContBB: Cont, NoMerge);
4435 } else {
4436 // Emit two handler calls: first one for set of unrecoverable checks,
4437 // another one for recoverable.
4438 llvm::BasicBlock *NonFatalHandlerBB =
4439 createBasicBlock(name: "non_fatal." + CheckName);
4440 llvm::BasicBlock *FatalHandlerBB = createBasicBlock(name: "fatal." + CheckName);
4441 Builder.CreateCondBr(Cond: FatalCond, True: NonFatalHandlerBB, False: FatalHandlerBB);
4442 EmitBlock(BB: FatalHandlerBB);
4443 emitCheckHandlerCall(CGF&: *this, FnType, FnArgs: Args, CheckHandler, RecoverKind, IsFatal: true,
4444 ContBB: NonFatalHandlerBB, NoMerge);
4445 EmitBlock(BB: NonFatalHandlerBB);
4446 emitCheckHandlerCall(CGF&: *this, FnType, FnArgs: Args, CheckHandler, RecoverKind, IsFatal: false,
4447 ContBB: Cont, NoMerge);
4448 }
4449
4450 EmitBlock(BB: Cont);
4451}
4452
4453void CodeGenFunction::EmitCfiSlowPathCheck(
4454 SanitizerKind::SanitizerOrdinal Ordinal, llvm::Value *Cond,
4455 llvm::ConstantInt *TypeId, llvm::Value *Ptr,
4456 ArrayRef<llvm::Constant *> StaticArgs) {
4457 llvm::BasicBlock *Cont = createBasicBlock(name: "cfi.cont");
4458
4459 llvm::BasicBlock *CheckBB = createBasicBlock(name: "cfi.slowpath");
4460 llvm::CondBrInst *BI = Builder.CreateCondBr(Cond, True: Cont, False: CheckBB);
4461
4462 llvm::MDBuilder MDHelper(getLLVMContext());
4463 llvm::MDNode *Node = MDHelper.createLikelyBranchWeights();
4464 BI->setMetadata(KindID: llvm::LLVMContext::MD_prof, Node);
4465
4466 EmitBlock(BB: CheckBB);
4467
4468 bool WithDiag = !CGM.getCodeGenOpts().SanitizeTrap.has(O: Ordinal);
4469
4470 llvm::CallInst *CheckCall;
4471 llvm::FunctionCallee SlowPathFn;
4472 if (WithDiag) {
4473 llvm::Constant *Info = llvm::ConstantStruct::getAnon(V: StaticArgs);
4474 auto *InfoPtr =
4475 new llvm::GlobalVariable(CGM.getModule(), Info->getType(), false,
4476 llvm::GlobalVariable::PrivateLinkage, Info);
4477 InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4478 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV: InfoPtr);
4479
4480 SlowPathFn = CGM.getModule().getOrInsertFunction(
4481 Name: "__cfi_slowpath_diag",
4482 T: llvm::FunctionType::get(Result: VoidTy, Params: {Int64Ty, Int8PtrTy, Int8PtrTy},
4483 isVarArg: false));
4484 CheckCall = Builder.CreateCall(Callee: SlowPathFn, Args: {TypeId, Ptr, InfoPtr});
4485 } else {
4486 SlowPathFn = CGM.getModule().getOrInsertFunction(
4487 Name: "__cfi_slowpath",
4488 T: llvm::FunctionType::get(Result: VoidTy, Params: {Int64Ty, Int8PtrTy}, isVarArg: false));
4489 CheckCall = Builder.CreateCall(Callee: SlowPathFn, Args: {TypeId, Ptr});
4490 }
4491
4492 CGM.setDSOLocal(
4493 cast<llvm::GlobalValue>(Val: SlowPathFn.getCallee()->stripPointerCasts()));
4494 CheckCall->setDoesNotThrow();
4495
4496 EmitBlock(BB: Cont);
4497}
4498
4499// Emit a stub for __cfi_check function so that the linker knows about this
4500// symbol in LTO mode.
4501void CodeGenFunction::EmitCfiCheckStub() {
4502 llvm::Module *M = &CGM.getModule();
4503 ASTContext &C = getContext();
4504 QualType QInt64Ty = C.getIntTypeForBitwidth(DestWidth: 64, Signed: false);
4505
4506 auto *ArgCallsiteTypeId =
4507 ImplicitParamDecl::Create(C, T: QInt64Ty, ParamKind: ImplicitParamKind::Other);
4508 auto *ArgAddr =
4509 ImplicitParamDecl::Create(C, T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
4510 auto *ArgCFICheckFailData =
4511 ImplicitParamDecl::Create(C, T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
4512 FunctionArgList FnArgs{ArgCallsiteTypeId, ArgAddr, ArgCFICheckFailData};
4513 const CGFunctionInfo &FI =
4514 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: FnArgs);
4515
4516 llvm::Function *F = llvm::Function::Create(
4517 Ty: llvm::FunctionType::get(Result: VoidTy, Params: {Int64Ty, VoidPtrTy, VoidPtrTy}, isVarArg: false),
4518 Linkage: llvm::GlobalValue::WeakAnyLinkage, N: "__cfi_check", M);
4519 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F, /*IsThunk=*/false);
4520 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F);
4521 F->setAlignment(llvm::Align(4096));
4522 CGM.setDSOLocal(F);
4523
4524 llvm::LLVMContext &Ctx = M->getContext();
4525 llvm::BasicBlock *BB = llvm::BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: F);
4526 // CrossDSOCFI pass is not executed if there is no executable code.
4527 SmallVector<llvm::Value*> Args{F->getArg(i: 2), F->getArg(i: 1)};
4528 llvm::CallInst::Create(Func: M->getFunction(Name: "__cfi_check_fail"), Args, NameStr: "", InsertBefore: BB);
4529 llvm::ReturnInst::Create(C&: Ctx, retVal: nullptr, InsertBefore: BB);
4530}
4531
4532// This function is basically a switch over the CFI failure kind, which is
4533// extracted from CFICheckFailData (1st function argument). Each case is either
4534// llvm.trap or a call to one of the two runtime handlers, based on
4535// -fsanitize-trap and -fsanitize-recover settings. Default case (invalid
4536// failure kind) traps, but this should really never happen. CFICheckFailData
4537// can be nullptr if the calling module has -fsanitize-trap behavior for this
4538// check kind; in this case __cfi_check_fail traps as well.
4539void CodeGenFunction::EmitCfiCheckFail() {
4540 auto CheckHandler = SanitizerHandler::CFICheckFail;
4541 // TODO: the SanitizerKind is not yet determined for this check (and might
4542 // not even be available, if Data == nullptr). However, we still want to
4543 // annotate the instrumentation. We approximate this by using all the CFI
4544 // kinds.
4545 SanitizerDebugLocation SanScope(
4546 this,
4547 {SanitizerKind::SO_CFIVCall, SanitizerKind::SO_CFINVCall,
4548 SanitizerKind::SO_CFIDerivedCast, SanitizerKind::SO_CFIUnrelatedCast,
4549 SanitizerKind::SO_CFIICall},
4550 CheckHandler);
4551 auto *ArgData = ImplicitParamDecl::Create(
4552 C&: getContext(), T: getContext().VoidPtrTy, ParamKind: ImplicitParamKind::Other);
4553 auto *ArgAddr = ImplicitParamDecl::Create(
4554 C&: getContext(), T: getContext().VoidPtrTy, ParamKind: ImplicitParamKind::Other);
4555
4556 FunctionArgList Args{ArgData, ArgAddr};
4557 const CGFunctionInfo &FI =
4558 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: getContext().VoidTy, args: Args);
4559
4560 llvm::Function *F = llvm::Function::Create(
4561 Ty: llvm::FunctionType::get(Result: VoidTy, Params: {VoidPtrTy, VoidPtrTy}, isVarArg: false),
4562 Linkage: llvm::GlobalValue::WeakODRLinkage, N: "__cfi_check_fail", M: &CGM.getModule());
4563
4564 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F, /*IsThunk=*/false);
4565 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F);
4566 F->setVisibility(llvm::GlobalValue::HiddenVisibility);
4567
4568 StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidTy, Fn: F, FnInfo: FI, Args,
4569 Loc: SourceLocation());
4570
4571 ApplyDebugLocation ADL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
4572
4573 // This function is not affected by NoSanitizeList. This function does
4574 // not have a source location, but "src:*" would still apply. Revert any
4575 // changes to SanOpts made in StartFunction.
4576 SanOpts = CGM.getLangOpts().Sanitize;
4577
4578 llvm::Value *Data =
4579 EmitLoadOfScalar(Addr: GetAddrOfLocalVar(VD: ArgData), /*Volatile=*/false,
4580 Ty: CGM.getContext().VoidPtrTy, Loc: ArgData->getLocation());
4581 llvm::Value *Addr =
4582 EmitLoadOfScalar(Addr: GetAddrOfLocalVar(VD: ArgAddr), /*Volatile=*/false,
4583 Ty: CGM.getContext().VoidPtrTy, Loc: ArgAddr->getLocation());
4584
4585 // Data == nullptr means the calling module has trap behaviour for this check.
4586 llvm::Value *DataIsNotNullPtr =
4587 Builder.CreateICmpNE(LHS: Data, RHS: llvm::ConstantPointerNull::get(T: Int8PtrTy));
4588 // TODO: since there is no data, we don't know the CheckKind, and therefore
4589 // cannot inspect CGM.getCodeGenOpts().SanitizeMergeHandlers. We default to
4590 // NoMerge = false. Users can disable merging by disabling optimization.
4591 EmitTrapCheck(Checked: DataIsNotNullPtr, CheckHandlerID: SanitizerHandler::CFICheckFail,
4592 /*NoMerge=*/false);
4593
4594 llvm::StructType *SourceLocationTy =
4595 llvm::StructType::get(elt1: VoidPtrTy, elts: Int32Ty, elts: Int32Ty);
4596 llvm::StructType *CfiCheckFailDataTy =
4597 llvm::StructType::get(elt1: Int8Ty, elts: SourceLocationTy, elts: VoidPtrTy);
4598
4599 llvm::Value *V = Builder.CreateConstGEP2_32(
4600 Ty: CfiCheckFailDataTy, Ptr: Builder.CreatePointerCast(V: Data, DestTy: DefaultPtrTy), Idx0: 0, Idx1: 0);
4601
4602 Address CheckKindAddr(V, Int8Ty, getIntAlign());
4603 llvm::Value *CheckKind = Builder.CreateLoad(Addr: CheckKindAddr);
4604
4605 llvm::Value *AllVtables = llvm::MetadataAsValue::get(
4606 Context&: CGM.getLLVMContext(),
4607 MD: llvm::MDString::get(Context&: CGM.getLLVMContext(), Str: "all-vtables"));
4608 llvm::Value *ValidVtable = Builder.CreateZExt(
4609 V: Builder.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::type_test),
4610 Args: {Addr, AllVtables}),
4611 DestTy: IntPtrTy);
4612
4613 const std::pair<int, SanitizerKind::SanitizerOrdinal> CheckKinds[] = {
4614 {CFITCK_VCall, SanitizerKind::SO_CFIVCall},
4615 {CFITCK_NVCall, SanitizerKind::SO_CFINVCall},
4616 {CFITCK_DerivedCast, SanitizerKind::SO_CFIDerivedCast},
4617 {CFITCK_UnrelatedCast, SanitizerKind::SO_CFIUnrelatedCast},
4618 {CFITCK_ICall, SanitizerKind::SO_CFIICall}};
4619
4620 for (auto CheckKindOrdinalPair : CheckKinds) {
4621 int Kind = CheckKindOrdinalPair.first;
4622 SanitizerKind::SanitizerOrdinal Ordinal = CheckKindOrdinalPair.second;
4623
4624 // TODO: we could apply SanitizerAnnotateDebugInfo(Ordinal) instead of
4625 // relying on the SanitizerScope with all CFI ordinals
4626
4627 llvm::Value *Cond =
4628 Builder.CreateICmpNE(LHS: CheckKind, RHS: llvm::ConstantInt::get(Ty: Int8Ty, V: Kind));
4629 if (CGM.getLangOpts().Sanitize.has(O: Ordinal))
4630 EmitCheck(Checked: std::make_pair(x&: Cond, y&: Ordinal), CheckHandler: SanitizerHandler::CFICheckFail,
4631 StaticArgs: {}, DynamicArgs: {Data, Addr, ValidVtable});
4632 else
4633 // TODO: we can't rely on CGM.getCodeGenOpts().SanitizeMergeHandlers.
4634 // Although the compiler allows SanitizeMergeHandlers to be set
4635 // independently of CGM.getLangOpts().Sanitize, Driver/SanitizerArgs.cpp
4636 // requires that SanitizeMergeHandlers is a subset of Sanitize.
4637 EmitTrapCheck(Checked: Cond, CheckHandlerID: CheckHandler, /*NoMerge=*/false);
4638 }
4639
4640 FinishFunction();
4641 // The only reference to this function will be created during LTO link.
4642 // Make sure it survives until then.
4643 CGM.addUsedGlobal(GV: F);
4644}
4645
4646void CodeGenFunction::EmitUnreachable(SourceLocation Loc) {
4647 if (SanOpts.has(K: SanitizerKind::Unreachable)) {
4648 auto CheckOrdinal = SanitizerKind::SO_Unreachable;
4649 auto CheckHandler = SanitizerHandler::BuiltinUnreachable;
4650 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
4651 EmitCheck(Checked: std::make_pair(x: static_cast<llvm::Value *>(Builder.getFalse()),
4652 y&: CheckOrdinal),
4653 CheckHandler, StaticArgs: EmitCheckSourceLocation(Loc), DynamicArgs: {});
4654 }
4655 Builder.CreateUnreachable();
4656}
4657
4658void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked,
4659 SanitizerHandler CheckHandlerID,
4660 bool NoMerge, const TrapReason *TR) {
4661 llvm::BasicBlock *Cont = createBasicBlock(name: "cont");
4662
4663 // If we're optimizing, collapse all calls to trap down to just one per
4664 // check-type per function to save on code size.
4665 if ((int)TrapBBs.size() <= CheckHandlerID)
4666 TrapBBs.resize(N: CheckHandlerID + 1);
4667
4668 llvm::BasicBlock *&TrapBB = TrapBBs[CheckHandlerID];
4669
4670 llvm::DILocation *TrapLocation = Builder.getCurrentDebugLocation();
4671 llvm::StringRef TrapMessage;
4672 llvm::StringRef TrapCategory;
4673 auto DebugTrapReasonKind = CGM.getCodeGenOpts().getSanitizeDebugTrapReasons();
4674 if (TR && !TR->isEmpty() &&
4675 DebugTrapReasonKind ==
4676 CodeGenOptions::SanitizeDebugTrapReasonKind::Detailed) {
4677 TrapMessage = TR->getMessage();
4678 TrapCategory = TR->getCategory();
4679 } else {
4680 TrapMessage = GetUBSanTrapForHandler(ID: CheckHandlerID);
4681 TrapCategory = "Undefined Behavior Sanitizer";
4682 }
4683
4684 if (getDebugInfo() && !TrapMessage.empty() &&
4685 DebugTrapReasonKind !=
4686 CodeGenOptions::SanitizeDebugTrapReasonKind::None &&
4687 TrapLocation) {
4688 TrapLocation = getDebugInfo()->CreateTrapFailureMessageFor(
4689 TrapLocation, Category: TrapCategory, FailureMsg: TrapMessage);
4690 }
4691
4692 NoMerge = NoMerge || !CGM.getCodeGenOpts().isOptimizedBuild() ||
4693 (CurCodeDecl && CurCodeDecl->hasAttr<OptimizeNoneAttr>());
4694
4695 llvm::MDBuilder MDHelper(getLLVMContext());
4696 if (TrapBB && !NoMerge) {
4697 auto Call = TrapBB->begin();
4698 assert(isa<llvm::CallInst>(Call) && "Expected call in trap BB");
4699
4700 Call->applyMergedLocation(LocA: Call->getDebugLoc(), LocB: TrapLocation);
4701
4702 Builder.CreateCondBr(Cond: Checked, True: Cont, False: TrapBB,
4703 BranchWeights: MDHelper.createLikelyBranchWeights());
4704 } else {
4705 TrapBB = createBasicBlock(name: "trap");
4706 Builder.CreateCondBr(Cond: Checked, True: Cont, False: TrapBB,
4707 BranchWeights: MDHelper.createLikelyBranchWeights());
4708 EmitBlock(BB: TrapBB);
4709
4710 ApplyDebugLocation applyTrapDI(*this, TrapLocation);
4711
4712 llvm::CallInst *TrapCall;
4713 if (CGM.getCodeGenOpts().SanitizeTrapLoop)
4714 TrapCall =
4715 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::looptrap));
4716 else
4717 TrapCall = Builder.CreateCall(
4718 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::ubsantrap),
4719 Args: llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: CheckHandlerID));
4720
4721 if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
4722 auto A = llvm::Attribute::get(Context&: getLLVMContext(), Kind: "trap-func-name",
4723 Val: CGM.getCodeGenOpts().TrapFuncName);
4724 TrapCall->addFnAttr(Attr: A);
4725 }
4726 if (NoMerge)
4727 TrapCall->addFnAttr(Kind: llvm::Attribute::NoMerge);
4728 TrapCall->setDoesNotReturn();
4729 TrapCall->setDoesNotThrow();
4730 Builder.CreateUnreachable();
4731 }
4732
4733 EmitBlock(BB: Cont);
4734}
4735
4736llvm::CallInst *CodeGenFunction::EmitTrapCall(llvm::Intrinsic::ID IntrID,
4737 bool EnsureInsertPoint) {
4738 llvm::Function *TrapIntrinsic = CGM.getIntrinsic(IID: IntrID);
4739 llvm::CallInst *TrapCall = Builder.CreateCall(Callee: TrapIntrinsic);
4740
4741 if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
4742 auto A = llvm::Attribute::get(Context&: getLLVMContext(), Kind: "trap-func-name",
4743 Val: CGM.getCodeGenOpts().TrapFuncName);
4744 TrapCall->addFnAttr(Attr: A);
4745 }
4746
4747 if (InNoMergeAttributedStmt)
4748 TrapCall->addFnAttr(Kind: llvm::Attribute::NoMerge);
4749 if (TrapIntrinsic->doesNotThrow())
4750 TrapCall->setDoesNotThrow();
4751 if (TrapIntrinsic->doesNotReturn()) {
4752 TrapCall->setDoesNotReturn();
4753 Builder.CreateUnreachable();
4754 if (EnsureInsertPoint)
4755 EmitBlock(BB: createBasicBlock());
4756 else
4757 Builder.ClearInsertionPoint();
4758 }
4759 return TrapCall;
4760}
4761
4762void CodeGenFunction::EmitTrapCallAndMakeUnreachable() {
4763 llvm::CallInst *TrapCall =
4764 EmitTrapCall(IntrID: llvm::Intrinsic::trap, /*EnsureInsertPoint=*/false);
4765 TrapCall->setDoesNotReturn();
4766 TrapCall->setDoesNotThrow();
4767 if (HaveInsertPoint()) {
4768 Builder.CreateUnreachable();
4769 Builder.ClearInsertionPoint();
4770 }
4771}
4772
4773Address CodeGenFunction::EmitArrayToPointerDecay(const Expr *E,
4774 LValueBaseInfo *BaseInfo,
4775 TBAAAccessInfo *TBAAInfo) {
4776 assert(E->getType()->isArrayType() &&
4777 "Array to pointer decay must have array source type!");
4778
4779 // Expressions of array type can't be bitfields or vector elements.
4780 LValue LV = EmitLValue(E);
4781 Address Addr = LV.getAddress();
4782
4783 // If the array type was an incomplete type, we need to make sure
4784 // the decay ends up being the right type.
4785 llvm::Type *NewTy = ConvertType(T: E->getType());
4786 Addr = Addr.withElementType(ElemTy: NewTy);
4787
4788 // Note that VLA pointers are always decayed, so we don't need to do
4789 // anything here.
4790 if (!E->getType()->isVariableArrayType()) {
4791 assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
4792 "Expected pointer to array");
4793
4794 if (getLangOpts().EmitLogicalPointer) {
4795 // Array-to-pointer decay for an SGEP is a no-op as we don't do any
4796 // logical indexing. See #179951 for some additional context.
4797 auto *SGEP =
4798 Builder.CreateStructuredGEP(BaseType: NewTy, PtrBase: Addr.emitRawPointer(CGF&: *this), Indices: {});
4799 Addr = Address(SGEP, NewTy, Addr.getAlignment(), Addr.isKnownNonNull());
4800 } else {
4801 Addr = Builder.CreateConstArrayGEP(Addr, Index: 0, Name: "arraydecay");
4802 }
4803 }
4804
4805 // The result of this decay conversion points to an array element within the
4806 // base lvalue. However, since TBAA currently does not support representing
4807 // accesses to elements of member arrays, we conservatively represent accesses
4808 // to the pointee object as if it had no any base lvalue specified.
4809 // TODO: Support TBAA for member arrays.
4810 QualType EltType = E->getType()->castAsArrayTypeUnsafe()->getElementType();
4811 if (BaseInfo) *BaseInfo = LV.getBaseInfo();
4812 if (TBAAInfo) *TBAAInfo = CGM.getTBAAAccessInfo(AccessType: EltType);
4813
4814 return Addr.withElementType(ElemTy: ConvertTypeForMem(T: EltType));
4815}
4816
4817/// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an
4818/// array to pointer, return the array subexpression.
4819static const Expr *isSimpleArrayDecayOperand(const Expr *E) {
4820 // If this isn't just an array->pointer decay, bail out.
4821 const auto *CE = dyn_cast<CastExpr>(Val: E);
4822 if (!CE || CE->getCastKind() != CK_ArrayToPointerDecay)
4823 return nullptr;
4824
4825 // If this is a decay from variable width array, bail out.
4826 const Expr *SubExpr = CE->getSubExpr();
4827 if (SubExpr->getType()->isVariableArrayType())
4828 return nullptr;
4829
4830 return SubExpr;
4831}
4832
4833static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF,
4834 llvm::Type *elemType,
4835 llvm::Value *ptr,
4836 ArrayRef<llvm::Value*> indices,
4837 bool inbounds,
4838 bool signedIndices,
4839 SourceLocation loc,
4840 const llvm::Twine &name = "arrayidx") {
4841 if (inbounds && CGF.getLangOpts().EmitLogicalPointer)
4842 return CGF.Builder.CreateStructuredGEP(BaseType: elemType, PtrBase: ptr, Indices: indices);
4843
4844 if (inbounds) {
4845 return CGF.EmitCheckedInBoundsGEP(ElemTy: elemType, Ptr: ptr, IdxList: indices, SignedIndices: signedIndices,
4846 IsSubtraction: CodeGenFunction::NotSubtraction, Loc: loc,
4847 Name: name);
4848 } else {
4849 return CGF.Builder.CreateGEP(Ty: elemType, Ptr: ptr, IdxList: indices, Name: name);
4850 }
4851}
4852
4853static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
4854 ArrayRef<llvm::Value *> indices,
4855 llvm::Type *arrayType,
4856 llvm::Type *elementType, bool inbounds,
4857 bool signedIndices, SourceLocation loc,
4858 CharUnits align,
4859 const llvm::Twine &name = "arrayidx") {
4860 if (inbounds && CGF.getLangOpts().EmitLogicalPointer)
4861 return RawAddress(CGF.Builder.CreateStructuredGEP(BaseType: arrayType,
4862 PtrBase: addr.emitRawPointer(CGF),
4863 Indices: indices.drop_front()),
4864 elementType, align);
4865
4866 if (inbounds) {
4867 return CGF.EmitCheckedInBoundsGEP(Addr: addr, IdxList: indices, elementType, SignedIndices: signedIndices,
4868 IsSubtraction: CodeGenFunction::NotSubtraction, Loc: loc,
4869 Align: align, Name: name);
4870 } else {
4871 return CGF.Builder.CreateGEP(Addr: addr, IdxList: indices, ElementType: elementType, Align: align, Name: name);
4872 }
4873}
4874
4875static bool hasBPFPreserveStaticOffset(const RecordDecl *D) {
4876 return D && D->hasAttr<BPFPreserveStaticOffsetAttr>();
4877}
4878
4879static bool hasBPFPreserveStaticOffset(const Expr *E) {
4880 if (!E)
4881 return false;
4882 QualType PointeeType = E->getType()->getPointeeType();
4883 if (PointeeType.isNull())
4884 return false;
4885 if (const auto *BaseDecl = PointeeType->getAsRecordDecl())
4886 return hasBPFPreserveStaticOffset(D: BaseDecl);
4887 return false;
4888}
4889
4890// Wraps Addr with a call to llvm.preserve.static.offset intrinsic.
4891static Address wrapWithBPFPreserveStaticOffset(CodeGenFunction &CGF,
4892 Address &Addr) {
4893 if (!CGF.getTarget().getTriple().isBPF())
4894 return Addr;
4895
4896 llvm::Function *Fn =
4897 CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::preserve_static_offset);
4898 llvm::CallInst *Call = CGF.Builder.CreateCall(Callee: Fn, Args: {Addr.emitRawPointer(CGF)});
4899 return Address(Call, Addr.getElementType(), Addr.getAlignment());
4900}
4901
4902/// Given an array base, check whether its member access belongs to a record
4903/// with preserve_access_index attribute or not.
4904static bool IsPreserveAIArrayBase(CodeGenFunction &CGF, const Expr *ArrayBase) {
4905 if (!ArrayBase || !CGF.getDebugInfo())
4906 return false;
4907
4908 // Only support base as either a MemberExpr or DeclRefExpr.
4909 // DeclRefExpr to cover cases like:
4910 // struct s { int a; int b[10]; };
4911 // struct s *p;
4912 // p[1].a
4913 // p[1] will generate a DeclRefExpr and p[1].a is a MemberExpr.
4914 // p->b[5] is a MemberExpr example.
4915 const Expr *E = ArrayBase->IgnoreImpCasts();
4916 if (const auto *ME = dyn_cast<MemberExpr>(Val: E))
4917 return ME->getMemberDecl()->hasAttr<BPFPreserveAccessIndexAttr>();
4918
4919 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
4920 const auto *VarDef = dyn_cast<VarDecl>(Val: DRE->getDecl());
4921 if (!VarDef)
4922 return false;
4923
4924 const auto *PtrT = VarDef->getType()->getAs<PointerType>();
4925 if (!PtrT)
4926 return false;
4927
4928 const auto *PointeeT = PtrT->getPointeeType()
4929 ->getUnqualifiedDesugaredType();
4930 if (const auto *RecT = dyn_cast<RecordType>(Val: PointeeT))
4931 return RecT->getDecl()
4932 ->getMostRecentDecl()
4933 ->hasAttr<BPFPreserveAccessIndexAttr>();
4934 return false;
4935 }
4936
4937 return false;
4938}
4939
4940static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
4941 ArrayRef<llvm::Value *> indices,
4942 QualType eltType, bool inbounds,
4943 bool signedIndices, SourceLocation loc,
4944 QualType *arrayType = nullptr,
4945 const Expr *Base = nullptr,
4946 const llvm::Twine &name = "arrayidx") {
4947 // All the indices except that last must be zero.
4948#ifndef NDEBUG
4949 for (auto *idx : indices.drop_back())
4950 assert(isa<llvm::ConstantInt>(idx) &&
4951 cast<llvm::ConstantInt>(idx)->isZero());
4952#endif
4953
4954 // Determine the element size of the statically-sized base. This is
4955 // the thing that the indices are expressed in terms of.
4956 if (auto vla = CGF.getContext().getAsVariableArrayType(T: eltType)) {
4957 eltType = CodeGenUtils::getFixedSizeElementType(Ctx: CGF.getContext(), VLA: vla);
4958 }
4959
4960 // We can use that to compute the best alignment of the element.
4961 CharUnits eltSize = CGF.getContext().getTypeSizeInChars(T: eltType);
4962 CharUnits eltAlign =
4963 getArrayElementAlign(arrayAlign: addr.getAlignment(), idx: indices.back(), eltSize);
4964
4965 if (hasBPFPreserveStaticOffset(E: Base))
4966 addr = wrapWithBPFPreserveStaticOffset(CGF, Addr&: addr);
4967
4968 llvm::Value *eltPtr;
4969 auto LastIndex = dyn_cast<llvm::ConstantInt>(Val: indices.back());
4970 if (!LastIndex ||
4971 (!CGF.IsInPreservedAIRegion && !IsPreserveAIArrayBase(CGF, ArrayBase: Base))) {
4972 addr = emitArraySubscriptGEP(CGF, addr, indices,
4973 arrayType: arrayType ? CGF.ConvertTypeForMem(T: *arrayType)
4974 : nullptr,
4975 elementType: CGF.ConvertTypeForMem(T: eltType), inbounds,
4976 signedIndices, loc, align: eltAlign, name);
4977 return addr;
4978 } else {
4979 // Remember the original array subscript for bpf target
4980 unsigned idx = LastIndex->getZExtValue();
4981 llvm::DIType *DbgInfo = nullptr;
4982 if (arrayType)
4983 DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(Ty: *arrayType, Loc: loc);
4984 eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(
4985 ElTy: addr.getElementType(), Base: addr.emitRawPointer(CGF), Dimension: indices.size() - 1,
4986 LastIndex: idx, DbgInfo);
4987 }
4988
4989 return Address(eltPtr, CGF.ConvertTypeForMem(T: eltType), eltAlign);
4990}
4991
4992namespace {
4993
4994/// StructFieldAccess is a simple visitor class to grab the first l-value to
4995/// r-value cast Expr.
4996struct StructFieldAccess
4997 : public ConstStmtVisitor<StructFieldAccess, const Expr *> {
4998 const Expr *VisitCastExpr(const CastExpr *E) {
4999 if (E->getCastKind() == CK_LValueToRValue)
5000 return E;
5001 return Visit(S: E->getSubExpr());
5002 }
5003 const Expr *VisitParenExpr(const ParenExpr *E) {
5004 return Visit(S: E->getSubExpr());
5005 }
5006};
5007
5008} // end anonymous namespace
5009
5010/// The offset of a field from the beginning of the record.
5011static bool getFieldOffsetInBits(CodeGenFunction &CGF, const RecordDecl *RD,
5012 const FieldDecl *Field, int64_t &Offset) {
5013 ASTContext &Ctx = CGF.getContext();
5014 const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(D: RD);
5015 unsigned FieldNo = 0;
5016
5017 for (const FieldDecl *FD : RD->fields()) {
5018 if (FD == Field) {
5019 Offset += Layout.getFieldOffset(FieldNo);
5020 return true;
5021 }
5022
5023 QualType Ty = FD->getType();
5024 if (Ty->isRecordType())
5025 if (getFieldOffsetInBits(CGF, RD: Ty->getAsRecordDecl(), Field, Offset)) {
5026 Offset += Layout.getFieldOffset(FieldNo);
5027 return true;
5028 }
5029
5030 if (!RD->isUnion())
5031 ++FieldNo;
5032 }
5033
5034 return false;
5035}
5036
5037/// Returns the relative offset difference between \p FD1 and \p FD2.
5038/// \code
5039/// offsetof(struct foo, FD1) - offsetof(struct foo, FD2)
5040/// \endcode
5041/// Both fields must be within the same struct.
5042static std::optional<int64_t> getOffsetDifferenceInBits(CodeGenFunction &CGF,
5043 const FieldDecl *FD1,
5044 const FieldDecl *FD2) {
5045 const RecordDecl *FD1OuterRec =
5046 FD1->getParent()->getOuterLexicalRecordContext();
5047 const RecordDecl *FD2OuterRec =
5048 FD2->getParent()->getOuterLexicalRecordContext();
5049
5050 if (FD1OuterRec != FD2OuterRec)
5051 // Fields must be within the same RecordDecl.
5052 return std::optional<int64_t>();
5053
5054 int64_t FD1Offset = 0;
5055 if (!getFieldOffsetInBits(CGF, RD: FD1OuterRec, Field: FD1, Offset&: FD1Offset))
5056 return std::optional<int64_t>();
5057
5058 int64_t FD2Offset = 0;
5059 if (!getFieldOffsetInBits(CGF, RD: FD2OuterRec, Field: FD2, Offset&: FD2Offset))
5060 return std::optional<int64_t>();
5061
5062 return std::make_optional<int64_t>(t: FD1Offset - FD2Offset);
5063}
5064
5065/// Convert a '__sized_by' byte-count bound to an element-count bound so it can
5066/// be compared against an element index. \p BoundsVal is the loaded byte count
5067/// and \p PointeeTy is the pointer's pointee type. Returns \p BoundsVal
5068/// unchanged when no scaling is needed, otherwise returns a new `llvm::Value*`
5069/// that is element count.
5070static llvm::Value *convertSizedByBoundToElementCount(CodeGenFunction &CGF,
5071 llvm::Value *BoundsVal,
5072 QualType PointeeTy,
5073 bool CountSigned) {
5074 assert(BoundsVal->getType()->isIntegerTy());
5075 assert(!PointeeTy.isNull() && "pointee type is never null");
5076 assert(!PointeeTy->isFunctionType() &&
5077 "Sema guarantees a '__sized_by' pointee is a non-function type");
5078
5079 if (PointeeTy->isIncompleteType()) {
5080 // Sema enforces that only 'void' can be subscripted here (GNU extension,
5081 // 1-byte stride), so the byte count already equals the element count.
5082 assert(PointeeTy->isVoidType() && "expected a 'void' incomplete pointee");
5083 return BoundsVal;
5084 }
5085
5086 CharUnits ElemSize = CGF.getContext().getTypeSizeInChars(T: PointeeTy);
5087 if (ElemSize <= CharUnits::One())
5088 return BoundsVal;
5089
5090 int64_t ElemSizeQ = ElemSize.getQuantity();
5091 unsigned CountWidth = BoundsVal->getType()->getIntegerBitWidth();
5092
5093 // The divisor must be representable in the count field's type.
5094 bool ElemSizeFits =
5095 CountSigned ? llvm::isIntN(N: CountWidth, x: ElemSizeQ)
5096 : llvm::isUIntN(N: CountWidth, x: static_cast<uint64_t>(ElemSizeQ));
5097 if (!ElemSizeFits)
5098 // No whole element fits in any representable byte count. Use a bound of 0
5099 // to always trap.
5100 // FIXME: Sema should just reject this (#223525).
5101 return llvm::ConstantInt::get(Ty: BoundsVal->getType(), V: 0);
5102
5103 llvm::Value *ElemSizeV =
5104 llvm::ConstantInt::get(Ty: BoundsVal->getType(), V: ElemSizeQ);
5105 // Use signed division for a signed count field so a negative byte count stays
5106 // non-positive and is still rejected by the negative-bounds guard in
5107 // EmitBoundsCheckImpl (unsigned division would turn it into a large positive
5108 // count).
5109 return CountSigned ? CGF.Builder.CreateSDiv(LHS: BoundsVal, RHS: ElemSizeV)
5110 : CGF.Builder.CreateUDiv(LHS: BoundsVal, RHS: ElemSizeV);
5111}
5112
5113/// EmitCountedByBoundsChecking - If the array being accessed has a "counted_by"
5114/// attribute, generate bounds checking code. The "count" field is at the top
5115/// level of the struct or in an anonymous struct, that's also at the top level.
5116/// Future expansions may allow the "count" to reside at any place in the
5117/// struct, but the value of "counted_by" will be a "simple" path to the count,
5118/// i.e. "a.b.count", so we shouldn't need the full force of EmitLValue or
5119/// similar to emit the correct GEP.
5120void CodeGenFunction::EmitCountedByBoundsChecking(
5121 const Expr *ArrayExpr, QualType ArrayType, Address ArrayInst,
5122 QualType IndexType, llvm::Value *IndexVal, bool Accessed,
5123 bool FlexibleArray) {
5124 const auto *ME = dyn_cast<MemberExpr>(Val: ArrayExpr->IgnoreImpCasts());
5125 if (!ME || !ME->getMemberDecl()->getType()->isCountAttributedType())
5126 return;
5127
5128 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
5129 getLangOpts().getStrictFlexArraysLevel();
5130 if (FlexibleArray &&
5131 !ME->isFlexibleArrayMemberLike(Context: getContext(), StrictFlexArraysLevel))
5132 return;
5133
5134 const FieldDecl *FD = cast<FieldDecl>(Val: ME->getMemberDecl());
5135 const FieldDecl *CountFD = FD->findCountedByField();
5136 if (!CountFD)
5137 return;
5138
5139 if (std::optional<int64_t> Diff =
5140 getOffsetDifferenceInBits(CGF&: *this, FD1: CountFD, FD2: FD)) {
5141 if (!ArrayInst.isValid()) {
5142 // An invalid Address indicates we're checking a pointer array access.
5143 // Emit the checked L-Value here.
5144 LValue LV = EmitCheckedLValue(E: ArrayExpr, TCK: TCK_MemberAccess);
5145 ArrayInst = LV.getAddress();
5146 }
5147
5148 // FIXME: The 'static_cast' is necessary, otherwise the result turns into a
5149 // uint64_t, which messes things up if we have a negative offset difference.
5150 Diff = *Diff / static_cast<int64_t>(CGM.getContext().getCharWidth());
5151
5152 // Create a GEP with the byte offset between the counted object and the
5153 // count and use that to load the count value.
5154 Address CountAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
5155 Addr: ArrayInst, Ty: Int8PtrTy, ElementTy: Int8Ty);
5156
5157 llvm::Type *BoundsType = ConvertType(T: CountFD->getType());
5158 llvm::Value *BoundsVal =
5159 Builder.CreateInBoundsGEP(Ty: Int8Ty, Ptr: CountAddr.emitRawPointer(CGF&: *this),
5160 IdxList: Builder.getInt32(C: *Diff), Name: ".counted_by.gep");
5161 BoundsVal = Builder.CreateAlignedLoad(Ty: BoundsType, Addr: BoundsVal, Align: getIntAlign(),
5162 Name: ".counted_by.load");
5163
5164 const auto *CountAttributedTy = FD->getType()->getAs<CountAttributedType>();
5165 assert(CountAttributedTy && "expected FD to have a CountAttributedType");
5166
5167 // For the '_or_null' variants a null pointer describes no accessible
5168 // memory, so treat the bound as 0 when the pointer is null; any access then
5169 // traps.
5170 if (CountAttributedTy->isOrNull()) {
5171 // Load the pointer from its address rather than re-emitting the
5172 // member expression, which would re-evaluate a side-effecting base.
5173 llvm::Value *Ptr = Builder.CreateLoad(Addr: ArrayInst);
5174 llvm::Value *IsNull = Builder.CreateIsNull(Arg: Ptr);
5175 BoundsVal = Builder.CreateSelect(
5176 C: IsNull, True: llvm::ConstantInt::get(Ty: BoundsType, V: 0), False: BoundsVal);
5177 }
5178
5179 // For '__sized_by' the loaded bound is a byte count. Convert it to an
5180 // element count by dividing by the element size so the check can compare
5181 // the (element) index directly. '__counted_by' already counts elements so
5182 // needs no special handling.
5183 if (CountAttributedTy->isCountInBytes())
5184 BoundsVal = convertSizedByBoundToElementCount(
5185 CGF&: *this, BoundsVal, PointeeTy: ArrayType->getPointeeType(),
5186 CountSigned: CountFD->getType()->isSignedIntegerOrEnumerationType());
5187
5188 // Now emit the bounds checking.
5189 EmitBoundsCheckImpl(ArrayExpr, ArrayBaseType: ArrayType, IndexVal, IndexType, BoundsVal,
5190 BoundsType: CountFD->getType(), Accessed);
5191 }
5192}
5193
5194LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
5195 bool Accessed) {
5196 // The index must always be an integer, which is not an aggregate. Emit it
5197 // in lexical order (this complexity is, sadly, required by C++17).
5198 llvm::Value *IdxPre =
5199 (E->getLHS() == E->getIdx()) ? EmitScalarExpr(E: E->getIdx()) : nullptr;
5200 bool SignedIndices = false;
5201 auto EmitIdxAfterBase = [&, IdxPre](bool Promote) -> llvm::Value * {
5202 auto *Idx = IdxPre;
5203 if (E->getLHS() != E->getIdx()) {
5204 assert(E->getRHS() == E->getIdx() && "index was neither LHS nor RHS");
5205 Idx = EmitScalarExpr(E: E->getIdx());
5206 }
5207
5208 QualType IdxTy = E->getIdx()->getType();
5209 bool IdxSigned = IdxTy->isSignedIntegerOrEnumerationType();
5210 SignedIndices |= IdxSigned;
5211
5212 if (SanOpts.has(K: SanitizerKind::ArrayBounds))
5213 EmitBoundsCheck(ArrayExpr: E, ArrayExprBase: E->getBase(), IndexVal: Idx, IndexType: IdxTy, Accessed);
5214
5215 // Extend or truncate the index type to 32 or 64-bits.
5216 if (Promote && Idx->getType() != IntPtrTy)
5217 Idx = Builder.CreateIntCast(V: Idx, DestTy: IntPtrTy, isSigned: IdxSigned, Name: "idxprom");
5218
5219 return Idx;
5220 };
5221 IdxPre = nullptr;
5222
5223 // If the base is a vector type, then we are forming a vector element lvalue
5224 // with this subscript.
5225 if (E->getBase()->getType()->isSubscriptableVectorType() &&
5226 !isa<ExtVectorElementExpr>(Val: E->getBase())) {
5227 // Emit the vector as an lvalue to get its address.
5228 LValue LHS = EmitLValue(E: E->getBase());
5229 auto *Idx = EmitIdxAfterBase(/*Promote*/false);
5230 assert(LHS.isSimple() && "Can only subscript lvalue vectors here!");
5231 return LValue::MakeVectorElt(vecAddress: LHS.getAddress(), Idx, type: E->getBase()->getType(),
5232 BaseInfo: LHS.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5233 }
5234
5235 // The HLSL runtime handles subscript expressions on global resource arrays
5236 // and objects with HLSL buffer layouts.
5237 if (getLangOpts().HLSL) {
5238 std::optional<LValue> LV;
5239 if (E->getType()->isHLSLResourceRecord() ||
5240 E->getType()->isHLSLResourceRecordArray()) {
5241 LV = CGM.getHLSLRuntime().emitResourceArraySubscriptExpr(E, CGF&: *this);
5242 } else if (E->getType().getAddressSpace() == LangAS::hlsl_constant) {
5243 LV = CGM.getHLSLRuntime().emitBufferArraySubscriptExpr(E, CGF&: *this,
5244 EmitIdxAfterBase);
5245 }
5246 if (LV.has_value())
5247 return *LV;
5248 }
5249
5250 // All the other cases basically behave like simple offsetting.
5251
5252 // Handle the extvector case we ignored above.
5253 if (isa<ExtVectorElementExpr>(Val: E->getBase())) {
5254 LValue LV = EmitLValue(E: E->getBase());
5255 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5256 Address Addr = EmitExtVectorElementLValue(LV);
5257
5258 QualType EltType = LV.getType()->castAs<VectorType>()->getElementType();
5259 Addr = emitArraySubscriptGEP(CGF&: *this, addr: Addr, indices: Idx, eltType: EltType, /*inbounds*/ true,
5260 signedIndices: SignedIndices, loc: E->getExprLoc());
5261 return MakeAddrLValue(Addr, T: EltType, BaseInfo: LV.getBaseInfo(),
5262 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: EltType));
5263 }
5264
5265 LValueBaseInfo EltBaseInfo;
5266 TBAAAccessInfo EltTBAAInfo;
5267 Address Addr = Address::invalid();
5268 if (const VariableArrayType *vla =
5269 getContext().getAsVariableArrayType(T: E->getType())) {
5270 // The base must be a pointer, which is not an aggregate. Emit
5271 // it. It needs to be emitted first in case it's what captures
5272 // the VLA bounds.
5273 Addr = EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
5274 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5275
5276 // The element count here is the total number of non-VLA elements.
5277 llvm::Value *numElements = getVLASize(vla).NumElts;
5278
5279 // Effectively, the multiply by the VLA size is part of the GEP.
5280 // GEP indexes are signed, and scaling an index isn't permitted to
5281 // signed-overflow, so we use the same semantics for our explicit
5282 // multiply. We suppress this if overflow is not undefined behavior.
5283 if (getLangOpts().PointerOverflowDefined) {
5284 Idx = Builder.CreateMul(LHS: Idx, RHS: numElements);
5285 } else {
5286 Idx = Builder.CreateNSWMul(LHS: Idx, RHS: numElements);
5287 }
5288
5289 Addr = emitArraySubscriptGEP(CGF&: *this, addr: Addr, indices: Idx, eltType: vla->getElementType(),
5290 inbounds: !getLangOpts().PointerOverflowDefined,
5291 signedIndices: SignedIndices, loc: E->getExprLoc());
5292
5293 } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){
5294 // Indexing over an interface, as in "NSString *P; P[4];"
5295
5296 // Emit the base pointer.
5297 Addr = EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
5298 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5299
5300 CharUnits InterfaceSize = getContext().getTypeSizeInChars(T: OIT);
5301 llvm::Value *InterfaceSizeVal =
5302 llvm::ConstantInt::get(Ty: Idx->getType(), V: InterfaceSize.getQuantity());
5303
5304 llvm::Value *ScaledIdx = Builder.CreateMul(LHS: Idx, RHS: InterfaceSizeVal);
5305
5306 // We don't necessarily build correct LLVM struct types for ObjC
5307 // interfaces, so we can't rely on GEP to do this scaling
5308 // correctly, so we need to cast to i8*. FIXME: is this actually
5309 // true? A lot of other things in the fragile ABI would break...
5310 llvm::Type *OrigBaseElemTy = Addr.getElementType();
5311
5312 // Do the GEP.
5313 CharUnits EltAlign =
5314 getArrayElementAlign(arrayAlign: Addr.getAlignment(), idx: Idx, eltSize: InterfaceSize);
5315 llvm::Value *EltPtr =
5316 emitArraySubscriptGEP(CGF&: *this, elemType: Int8Ty, ptr: Addr.emitRawPointer(CGF&: *this),
5317 indices: ScaledIdx, inbounds: false, signedIndices: SignedIndices, loc: E->getExprLoc());
5318 Addr = Address(EltPtr, OrigBaseElemTy, EltAlign);
5319 } else if (const Expr *Array = isSimpleArrayDecayOperand(E: E->getBase())) {
5320 // If this is A[i] where A is an array, the frontend will have decayed the
5321 // base to be a ArrayToPointerDecay implicit cast. While correct, it is
5322 // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
5323 // "gep x, i" here. Emit one "gep A, 0, i".
5324 assert(Array->getType()->isArrayType() &&
5325 "Array to pointer decay must have array source type!");
5326 LValue ArrayLV;
5327 // For simple multidimensional array indexing, set the 'accessed' flag for
5328 // better bounds-checking of the base expression.
5329 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Array))
5330 ArrayLV = EmitArraySubscriptExpr(E: ASE, /*Accessed*/ true);
5331 else
5332 ArrayLV = EmitLValue(E: Array);
5333 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5334
5335 if (SanOpts.has(K: SanitizerKind::ArrayBounds))
5336 EmitCountedByBoundsChecking(ArrayExpr: Array, ArrayType: Array->getType(), ArrayInst: ArrayLV.getAddress(),
5337 IndexType: E->getIdx()->getType(), IndexVal: Idx, Accessed,
5338 /*FlexibleArray=*/true);
5339
5340 // Propagate the alignment from the array itself to the result.
5341 QualType arrayType = Array->getType();
5342 Addr = emitArraySubscriptGEP(
5343 CGF&: *this, addr: ArrayLV.getAddress(), indices: {CGM.getSize(numChars: CharUnits::Zero()), Idx},
5344 eltType: E->getType(), inbounds: !getLangOpts().PointerOverflowDefined, signedIndices: SignedIndices,
5345 loc: E->getExprLoc(), arrayType: &arrayType, Base: E->getBase());
5346 EltBaseInfo = ArrayLV.getBaseInfo();
5347 if (!CGM.getCodeGenOpts().NewStructPathTBAA) {
5348 // Since CodeGenTBAA::getTypeInfoHelper only handles array types for
5349 // new struct path TBAA, we must a use a plain access.
5350 EltTBAAInfo = CGM.getTBAAInfoForSubobject(Base: ArrayLV, AccessType: E->getType());
5351 } else if (ArrayLV.getTBAAInfo().isMayAlias()) {
5352 EltTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
5353 } else if (ArrayLV.getTBAAInfo().isIncomplete()) {
5354 // The array element is complete, even if the array is not.
5355 EltTBAAInfo = CGM.getTBAAAccessInfo(AccessType: E->getType());
5356 } else {
5357 // The TBAA access info from the array (base) lvalue is ordinary. We will
5358 // adapt it to create access info for the element.
5359 EltTBAAInfo = ArrayLV.getTBAAInfo();
5360
5361 // We retain the TBAA struct path (BaseType and Offset members) from the
5362 // array. In the TBAA representation, we map any array access to the
5363 // element at index 0, as the index is generally a runtime value. This
5364 // element has the same offset in the base type as the array itself.
5365 // If the array lvalue had no base type, there is no point trying to
5366 // generate one, since an array itself is not a valid base type.
5367
5368 // We also retain the access type from the base lvalue, but the access
5369 // size must be updated to the size of an individual element.
5370 EltTBAAInfo.Size =
5371 getContext().getTypeSizeInChars(T: E->getType()).getQuantity();
5372 }
5373 } else {
5374 // The base must be a pointer; emit it with an estimate of its alignment.
5375 Address BaseAddr =
5376 EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &EltBaseInfo, TBAAInfo: &EltTBAAInfo);
5377 auto *Idx = EmitIdxAfterBase(/*Promote*/true);
5378 QualType ptrType = E->getBase()->getType();
5379 Addr = emitArraySubscriptGEP(CGF&: *this, addr: BaseAddr, indices: Idx, eltType: E->getType(),
5380 inbounds: !getLangOpts().PointerOverflowDefined,
5381 signedIndices: SignedIndices, loc: E->getExprLoc(), arrayType: &ptrType,
5382 Base: E->getBase());
5383
5384 if (SanOpts.has(K: SanitizerKind::ArrayBounds)) {
5385 StructFieldAccess Visitor;
5386 const Expr *Base = Visitor.Visit(S: E->getBase());
5387
5388 if (const auto *CE = dyn_cast_if_present<CastExpr>(Val: Base);
5389 CE && CE->getCastKind() == CK_LValueToRValue)
5390 EmitCountedByBoundsChecking(ArrayExpr: CE, ArrayType: ptrType, ArrayInst: Address::invalid(),
5391 IndexType: E->getIdx()->getType(), IndexVal: Idx, Accessed,
5392 /*FlexibleArray=*/false);
5393 }
5394 }
5395
5396 LValue LV = MakeAddrLValue(Addr, T: E->getType(), BaseInfo: EltBaseInfo, TBAAInfo: EltTBAAInfo);
5397
5398 if (getLangOpts().ObjC &&
5399 getLangOpts().getGC() != LangOptions::NonGC) {
5400 LV.setNonGC(!E->isOBJCGCCandidate(Ctx&: getContext()));
5401 setObjCGCLValueClass(Ctx: getContext(), E, LV);
5402 }
5403 return LV;
5404}
5405
5406llvm::Value *CodeGenFunction::EmitMatrixIndexExpr(const Expr *E) {
5407 llvm::Value *Idx = EmitScalarExpr(E);
5408 if (Idx->getType() == IntPtrTy)
5409 return Idx;
5410 bool IsSigned = E->getType()->isSignedIntegerOrEnumerationType();
5411 return Builder.CreateIntCast(V: Idx, DestTy: IntPtrTy, isSigned: IsSigned);
5412}
5413
5414LValue CodeGenFunction::EmitMatrixSingleSubscriptExpr(
5415 const MatrixSingleSubscriptExpr *E) {
5416 LValue Base = EmitLValue(E: E->getBase());
5417 llvm::Value *RowIdx = EmitMatrixIndexExpr(E: E->getRowIdx());
5418
5419 RawAddress MatAddr = Base.getAddress();
5420 if (getLangOpts().HLSL &&
5421 E->getBase()->getType().getAddressSpace() == LangAS::hlsl_constant)
5422 MatAddr = CGM.getHLSLRuntime().createBufferMatrixTempAddress(LV: Base, CGF&: *this);
5423
5424 return LValue::MakeMatrixRow(Addr: MaybeConvertMatrixAddress(Addr: MatAddr, CGF&: *this),
5425 RowIdx, MatrixTy: E->getBase()->getType(),
5426 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5427}
5428
5429LValue CodeGenFunction::EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E) {
5430 assert(
5431 !E->isIncomplete() &&
5432 "incomplete matrix subscript expressions should be rejected during Sema");
5433 LValue Base = EmitLValue(E: E->getBase());
5434
5435 // Extend or truncate the index type to 32 or 64-bits if needed.
5436 llvm::Value *RowIdx = EmitMatrixIndexExpr(E: E->getRowIdx());
5437 llvm::Value *ColIdx = EmitMatrixIndexExpr(E: E->getColumnIdx());
5438 llvm::MatrixBuilder MB(Builder);
5439 const auto *MatrixTy = E->getBase()->getType()->castAs<ConstantMatrixType>();
5440 unsigned NumCols = MatrixTy->getNumColumns();
5441 unsigned NumRows = MatrixTy->getNumRows();
5442 bool IsMatrixRowMajor =
5443 isMatrixRowMajor(LangOpts: getLangOpts(), T: E->getBase()->getType());
5444 llvm::Value *FinalIdx =
5445 MB.CreateIndex(RowIdx, ColumnIdx: ColIdx, NumRows, NumCols, IsMatrixRowMajor);
5446
5447 return LValue::MakeMatrixElt(
5448 matAddress: MaybeConvertMatrixAddress(Addr: Base.getAddress(), CGF&: *this), Idx: FinalIdx,
5449 type: E->getBase()->getType(), BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5450}
5451
5452static Address emitOMPArraySectionBase(CodeGenFunction &CGF, const Expr *Base,
5453 LValueBaseInfo &BaseInfo,
5454 TBAAAccessInfo &TBAAInfo,
5455 QualType BaseTy, QualType ElTy,
5456 bool IsLowerBound) {
5457 LValue BaseLVal;
5458 if (auto *ASE = dyn_cast<ArraySectionExpr>(Val: Base->IgnoreParenImpCasts())) {
5459 BaseLVal = CGF.EmitArraySectionExpr(E: ASE, IsLowerBound);
5460 if (BaseTy->isArrayType()) {
5461 Address Addr = BaseLVal.getAddress();
5462 BaseInfo = BaseLVal.getBaseInfo();
5463
5464 // If the array type was an incomplete type, we need to make sure
5465 // the decay ends up being the right type.
5466 llvm::Type *NewTy = CGF.ConvertType(T: BaseTy);
5467 Addr = Addr.withElementType(ElemTy: NewTy);
5468
5469 // Note that VLA pointers are always decayed, so we don't need to do
5470 // anything here.
5471 if (!BaseTy->isVariableArrayType()) {
5472 assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
5473 "Expected pointer to array");
5474 Addr = CGF.Builder.CreateConstArrayGEP(Addr, Index: 0, Name: "arraydecay");
5475 }
5476
5477 return Addr.withElementType(ElemTy: CGF.ConvertTypeForMem(T: ElTy));
5478 }
5479 LValueBaseInfo TypeBaseInfo;
5480 TBAAAccessInfo TypeTBAAInfo;
5481 CharUnits Align =
5482 CGF.CGM.getNaturalTypeAlignment(T: ElTy, BaseInfo: &TypeBaseInfo, TBAAInfo: &TypeTBAAInfo);
5483 BaseInfo.mergeForCast(Info: TypeBaseInfo);
5484 TBAAInfo = CGF.CGM.mergeTBAAInfoForCast(SourceInfo: TBAAInfo, TargetInfo: TypeTBAAInfo);
5485 return Address(CGF.Builder.CreateLoad(Addr: BaseLVal.getAddress()),
5486 CGF.ConvertTypeForMem(T: ElTy), Align);
5487 }
5488 return CGF.EmitPointerWithAlignment(E: Base, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
5489}
5490
5491LValue CodeGenFunction::EmitArraySectionExpr(const ArraySectionExpr *E,
5492 bool IsLowerBound) {
5493
5494 assert(!E->isOpenACCArraySection() &&
5495 "OpenACC Array section codegen not implemented");
5496
5497 QualType BaseTy = ArraySectionExpr::getBaseOriginalType(Base: E->getBase());
5498 QualType ResultExprTy;
5499 if (auto *AT = getContext().getAsArrayType(T: BaseTy))
5500 ResultExprTy = AT->getElementType();
5501 else
5502 ResultExprTy = BaseTy->getPointeeType();
5503 llvm::Value *Idx = nullptr;
5504 if (IsLowerBound || E->getColonLocFirst().isInvalid()) {
5505 // Requesting lower bound or upper bound, but without provided length and
5506 // without ':' symbol for the default length -> length = 1.
5507 // Idx = LowerBound ?: 0;
5508 if (auto *LowerBound = E->getLowerBound()) {
5509 Idx = Builder.CreateIntCast(
5510 V: EmitScalarExpr(E: LowerBound), DestTy: IntPtrTy,
5511 isSigned: LowerBound->getType()->hasSignedIntegerRepresentation());
5512 } else
5513 Idx = llvm::ConstantInt::getNullValue(Ty: IntPtrTy);
5514 } else {
5515 // Try to emit length or lower bound as constant. If this is possible, 1
5516 // is subtracted from constant length or lower bound. Otherwise, emit LLVM
5517 // IR (LB + Len) - 1.
5518 auto &C = CGM.getContext();
5519 auto *Length = E->getLength();
5520 llvm::APSInt ConstLength;
5521 if (Length) {
5522 // Idx = LowerBound + Length - 1;
5523 if (std::optional<llvm::APSInt> CL = Length->getIntegerConstantExpr(Ctx: C)) {
5524 ConstLength = CL->zextOrTrunc(width: PointerWidthInBits);
5525 Length = nullptr;
5526 }
5527 auto *LowerBound = E->getLowerBound();
5528 llvm::APSInt ConstLowerBound(PointerWidthInBits, /*isUnsigned=*/false);
5529 if (LowerBound) {
5530 if (std::optional<llvm::APSInt> LB =
5531 LowerBound->getIntegerConstantExpr(Ctx: C)) {
5532 ConstLowerBound = LB->zextOrTrunc(width: PointerWidthInBits);
5533 LowerBound = nullptr;
5534 }
5535 }
5536 if (!Length)
5537 --ConstLength;
5538 else if (!LowerBound)
5539 --ConstLowerBound;
5540
5541 if (Length || LowerBound) {
5542 auto *LowerBoundVal =
5543 LowerBound
5544 ? Builder.CreateIntCast(
5545 V: EmitScalarExpr(E: LowerBound), DestTy: IntPtrTy,
5546 isSigned: LowerBound->getType()->hasSignedIntegerRepresentation())
5547 : llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLowerBound);
5548 auto *LengthVal =
5549 Length
5550 ? Builder.CreateIntCast(
5551 V: EmitScalarExpr(E: Length), DestTy: IntPtrTy,
5552 isSigned: Length->getType()->hasSignedIntegerRepresentation())
5553 : llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLength);
5554 Idx = Builder.CreateAdd(LHS: LowerBoundVal, RHS: LengthVal, Name: "lb_add_len",
5555 /*HasNUW=*/false,
5556 HasNSW: !getLangOpts().PointerOverflowDefined);
5557 if (Length && LowerBound) {
5558 Idx = Builder.CreateSub(
5559 LHS: Idx, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, /*V=*/1), Name: "idx_sub_1",
5560 /*HasNUW=*/false, HasNSW: !getLangOpts().PointerOverflowDefined);
5561 }
5562 } else
5563 Idx = llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLength + ConstLowerBound);
5564 } else {
5565 // Idx = ArraySize - 1;
5566 QualType ArrayTy = BaseTy->isPointerType()
5567 ? E->getBase()->IgnoreParenImpCasts()->getType()
5568 : BaseTy;
5569 if (auto *VAT = C.getAsVariableArrayType(T: ArrayTy)) {
5570 Length = VAT->getSizeExpr();
5571 if (std::optional<llvm::APSInt> L = Length->getIntegerConstantExpr(Ctx: C)) {
5572 ConstLength = *L;
5573 Length = nullptr;
5574 }
5575 } else {
5576 auto *CAT = C.getAsConstantArrayType(T: ArrayTy);
5577 assert(CAT && "unexpected type for array initializer");
5578 ConstLength = CAT->getSize();
5579 }
5580 if (Length) {
5581 auto *LengthVal = Builder.CreateIntCast(
5582 V: EmitScalarExpr(E: Length), DestTy: IntPtrTy,
5583 isSigned: Length->getType()->hasSignedIntegerRepresentation());
5584 Idx = Builder.CreateSub(
5585 LHS: LengthVal, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, /*V=*/1), Name: "len_sub_1",
5586 /*HasNUW=*/false, HasNSW: !getLangOpts().PointerOverflowDefined);
5587 } else {
5588 ConstLength = ConstLength.zextOrTrunc(width: PointerWidthInBits);
5589 --ConstLength;
5590 Idx = llvm::ConstantInt::get(Ty: IntPtrTy, V: ConstLength);
5591 }
5592 }
5593 }
5594 assert(Idx);
5595
5596 Address EltPtr = Address::invalid();
5597 LValueBaseInfo BaseInfo;
5598 TBAAAccessInfo TBAAInfo;
5599 if (auto *VLA = getContext().getAsVariableArrayType(T: ResultExprTy)) {
5600 // The base must be a pointer, which is not an aggregate. Emit
5601 // it. It needs to be emitted first in case it's what captures
5602 // the VLA bounds.
5603 Address Base =
5604 emitOMPArraySectionBase(CGF&: *this, Base: E->getBase(), BaseInfo, TBAAInfo,
5605 BaseTy, ElTy: VLA->getElementType(), IsLowerBound);
5606 // The element count here is the total number of non-VLA elements.
5607 llvm::Value *NumElements = getVLASize(vla: VLA).NumElts;
5608
5609 // Effectively, the multiply by the VLA size is part of the GEP.
5610 // GEP indexes are signed, and scaling an index isn't permitted to
5611 // signed-overflow, so we use the same semantics for our explicit
5612 // multiply. We suppress this if overflow is not undefined behavior.
5613 if (getLangOpts().PointerOverflowDefined)
5614 Idx = Builder.CreateMul(LHS: Idx, RHS: NumElements);
5615 else
5616 Idx = Builder.CreateNSWMul(LHS: Idx, RHS: NumElements);
5617 EltPtr = emitArraySubscriptGEP(CGF&: *this, addr: Base, indices: Idx, eltType: VLA->getElementType(),
5618 inbounds: !getLangOpts().PointerOverflowDefined,
5619 /*signedIndices=*/false, loc: E->getExprLoc());
5620 } else if (const Expr *Array = isSimpleArrayDecayOperand(E: E->getBase())) {
5621 // If this is A[i] where A is an array, the frontend will have decayed the
5622 // base to be a ArrayToPointerDecay implicit cast. While correct, it is
5623 // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
5624 // "gep x, i" here. Emit one "gep A, 0, i".
5625 assert(Array->getType()->isArrayType() &&
5626 "Array to pointer decay must have array source type!");
5627 LValue ArrayLV;
5628 // For simple multidimensional array indexing, set the 'accessed' flag for
5629 // better bounds-checking of the base expression.
5630 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Array))
5631 ArrayLV = EmitArraySubscriptExpr(E: ASE, /*Accessed*/ true);
5632 else
5633 ArrayLV = EmitLValue(E: Array);
5634
5635 // Propagate the alignment from the array itself to the result.
5636 EltPtr = emitArraySubscriptGEP(
5637 CGF&: *this, addr: ArrayLV.getAddress(), indices: {CGM.getSize(numChars: CharUnits::Zero()), Idx},
5638 eltType: ResultExprTy, inbounds: !getLangOpts().PointerOverflowDefined,
5639 /*signedIndices=*/false, loc: E->getExprLoc());
5640 BaseInfo = ArrayLV.getBaseInfo();
5641 TBAAInfo = CGM.getTBAAInfoForSubobject(Base: ArrayLV, AccessType: ResultExprTy);
5642 } else {
5643 Address Base =
5644 emitOMPArraySectionBase(CGF&: *this, Base: E->getBase(), BaseInfo, TBAAInfo, BaseTy,
5645 ElTy: ResultExprTy, IsLowerBound);
5646 EltPtr = emitArraySubscriptGEP(CGF&: *this, addr: Base, indices: Idx, eltType: ResultExprTy,
5647 inbounds: !getLangOpts().PointerOverflowDefined,
5648 /*signedIndices=*/false, loc: E->getExprLoc());
5649 }
5650
5651 return MakeAddrLValue(Addr: EltPtr, T: ResultExprTy, BaseInfo, TBAAInfo);
5652}
5653
5654LValue CodeGenFunction::
5655EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
5656 // Emit the base vector as an l-value.
5657 LValue Base;
5658
5659 // ExtVectorElementExpr's base can either be a vector or pointer to vector.
5660 if (E->isArrow()) {
5661 // If it is a pointer to a vector, emit the address and form an lvalue with
5662 // it.
5663 LValueBaseInfo BaseInfo;
5664 TBAAAccessInfo TBAAInfo;
5665 Address Ptr = EmitPointerWithAlignment(E: E->getBase(), BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
5666 const auto *PT = E->getBase()->getType()->castAs<PointerType>();
5667 Base = MakeAddrLValue(Addr: Ptr, T: PT->getPointeeType(), BaseInfo, TBAAInfo);
5668 Base.getQuals().removeObjCGCAttr();
5669 } else if (E->getBase()->isGLValue()) {
5670 // Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
5671 // emit the base as an lvalue.
5672 assert(E->getBase()->getType()->isVectorType());
5673 Base = EmitLValue(E: E->getBase());
5674 } else {
5675 // Otherwise, the base is a normal rvalue (as in (V+V).x), emit it as such.
5676 assert(E->getBase()->getType()->isVectorType() &&
5677 "Result must be a vector");
5678 llvm::Value *Vec = EmitScalarExpr(E: E->getBase());
5679
5680 // Store the vector to memory (because LValue wants an address).
5681 Address VecMem = CreateMemTemp(Ty: E->getBase()->getType());
5682 // need to zero extend an hlsl boolean vector to store it back to memory
5683 QualType Ty = E->getBase()->getType();
5684 llvm::Type *LTy = convertTypeForLoadStore(ASTTy: Ty, LLVMTy: Vec->getType());
5685 if (LTy->getScalarSizeInBits() > Vec->getType()->getScalarSizeInBits())
5686 Vec = Builder.CreateZExt(V: Vec, DestTy: LTy);
5687 Builder.CreateStore(Val: Vec, Addr: VecMem);
5688 Base = MakeAddrLValue(Addr: VecMem, T: Ty, Source: AlignmentSource::Decl);
5689 }
5690
5691 QualType type =
5692 E->getType().withCVRQualifiers(CVR: Base.getQuals().getCVRQualifiers());
5693
5694 // Encode the element access list into a vector of unsigned indices.
5695 SmallVector<uint32_t, 4> Indices;
5696 E->getEncodedElementAccess(Elts&: Indices);
5697
5698 if (Base.isSimple()) {
5699 llvm::Constant *CV =
5700 llvm::ConstantDataVector::get(Context&: getLLVMContext(), Elts: Indices);
5701 return LValue::MakeExtVectorElt(Addr: Base.getAddress(), Elts: CV, type,
5702 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5703 }
5704
5705 if (Base.isMatrixRow()) {
5706 if (auto *RowIdx =
5707 llvm::dyn_cast<llvm::ConstantInt>(Val: Base.getMatrixRowIdx())) {
5708 llvm::SmallVector<llvm::Constant *> MatIndices;
5709 QualType MatTy = Base.getType();
5710 const ConstantMatrixType *MT = MatTy->castAs<ConstantMatrixType>();
5711 unsigned NumCols = Indices.size();
5712 unsigned NumRows = MT->getNumRows();
5713 unsigned Row = RowIdx->getZExtValue();
5714 QualType VecQT = E->getBase()->getType();
5715 if (NumCols != MT->getNumColumns()) {
5716 const auto *EVT = VecQT->getAs<ExtVectorType>();
5717 QualType ElemQT = EVT->getElementType();
5718 VecQT = getContext().getExtVectorType(VectorType: ElemQT, NumElts: NumCols);
5719 }
5720 for (unsigned C = 0; C < NumCols; ++C) {
5721 unsigned Col = Indices[C];
5722 unsigned Linear = Col * NumRows + Row;
5723 MatIndices.push_back(Elt: llvm::ConstantInt::get(Ty: Int32Ty, V: Linear));
5724 }
5725
5726 llvm::Constant *ConstIdxs = llvm::ConstantVector::get(V: MatIndices);
5727 return LValue::MakeExtVectorElt(Addr: Base.getMatrixAddress(), Elts: ConstIdxs, type: VecQT,
5728 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5729 }
5730 llvm::Constant *Cols =
5731 llvm::ConstantDataVector::get(Context&: getLLVMContext(), Elts: Indices);
5732 // Note: intentionally not using E.getType() so we can reuse isMatrixRow()
5733 // implementations in EmitLoadOfLValue & EmitStoreThroughLValue and don't
5734 // need the LValue to have its own number of rows and columns when the
5735 // type is a vector.
5736 return LValue::MakeMatrixRowSwizzle(
5737 MatAddr: Base.getMatrixAddress(), RowIdx: Base.getMatrixRowIdx(), Cols, MatrixTy: Base.getType(),
5738 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5739 }
5740
5741 assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
5742
5743 llvm::Constant *BaseElts = Base.getExtVectorElts();
5744 SmallVector<llvm::Constant *, 4> CElts;
5745
5746 for (unsigned Index : Indices)
5747 CElts.push_back(Elt: BaseElts->getAggregateElement(Elt: Index));
5748 llvm::Constant *CV = llvm::ConstantVector::get(V: CElts);
5749 return LValue::MakeExtVectorElt(Addr: Base.getExtVectorAddress(), Elts: CV, type,
5750 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
5751}
5752
5753bool CodeGenFunction::isUnderlyingBasePointerConstantNull(const Expr *E) {
5754 const Expr *UnderlyingBaseExpr = E->IgnoreParens();
5755 while (auto *BaseMemberExpr = dyn_cast<MemberExpr>(Val: UnderlyingBaseExpr))
5756 UnderlyingBaseExpr = BaseMemberExpr->getBase()->IgnoreParens();
5757 return getContext().isSentinelNullExpr(E: UnderlyingBaseExpr);
5758}
5759
5760LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
5761 if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(CGF&: *this, ME: E)) {
5762 EmitIgnoredExpr(E: E->getBase());
5763 return EmitDeclRefLValue(E: DRE);
5764 }
5765
5766 if (getLangOpts().HLSL) {
5767 QualType QT = E->getType();
5768 if (QT.getAddressSpace() == LangAS::hlsl_constant)
5769 return CGM.getHLSLRuntime().emitBufferMemberExpr(CGF&: *this, E);
5770
5771 if (QT->isHLSLResourceRecord() || QT->isHLSLResourceRecordArray()) {
5772 std::optional<LValue> LV;
5773 LV = CGM.getHLSLRuntime().emitResourceMemberExpr(CGF&: *this, E);
5774 if (LV.has_value())
5775 return *LV;
5776 }
5777 }
5778
5779 Expr *BaseExpr = E->getBase();
5780 // Check whether the underlying base pointer is a constant null.
5781 // If so, we do not set inbounds flag for GEP to avoid breaking some
5782 // old-style offsetof idioms.
5783 bool IsInBounds = !getLangOpts().PointerOverflowDefined &&
5784 !isUnderlyingBasePointerConstantNull(E: BaseExpr);
5785 // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar.
5786 LValue BaseLV;
5787 if (E->isArrow()) {
5788 LValueBaseInfo BaseInfo;
5789 TBAAAccessInfo TBAAInfo;
5790 Address Addr = EmitPointerWithAlignment(E: BaseExpr, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
5791 QualType PtrTy = BaseExpr->getType()->getPointeeType();
5792 SanitizerSet SkippedChecks;
5793 bool IsBaseCXXThis = IsWrappedCXXThis(Obj: BaseExpr);
5794 if (IsBaseCXXThis)
5795 SkippedChecks.set(K: SanitizerKind::Alignment, Value: true);
5796 if (IsBaseCXXThis || isa<DeclRefExpr>(Val: BaseExpr))
5797 SkippedChecks.set(K: SanitizerKind::Null, Value: true);
5798 EmitTypeCheck(TCK: TCK_MemberAccess, Loc: E->getExprLoc(), Addr, Type: PtrTy,
5799 /*Alignment=*/CharUnits::Zero(), SkippedChecks);
5800 BaseLV = MakeAddrLValue(Addr, T: PtrTy, BaseInfo, TBAAInfo);
5801 } else
5802 BaseLV = EmitCheckedLValue(E: BaseExpr, TCK: TCK_MemberAccess);
5803
5804 NamedDecl *ND = E->getMemberDecl();
5805 if (auto *Field = dyn_cast<FieldDecl>(Val: ND)) {
5806 LValue LV = EmitLValueForField(Base: BaseLV, Field, IsInBounds);
5807 setObjCGCLValueClass(Ctx: getContext(), E, LV);
5808 if (getLangOpts().OpenMP) {
5809 // If the member was explicitly marked as nontemporal, mark it as
5810 // nontemporal. If the base lvalue is marked as nontemporal, mark access
5811 // to children as nontemporal too.
5812 if ((IsWrappedCXXThis(Obj: BaseExpr) &&
5813 CGM.getOpenMPRuntime().isNontemporalDecl(VD: Field)) ||
5814 BaseLV.isNontemporal())
5815 LV.setNontemporal(/*Value=*/true);
5816 }
5817 return LV;
5818 }
5819
5820 if (const auto *FD = dyn_cast<FunctionDecl>(Val: ND))
5821 return EmitFunctionDeclLValue(CGF&: *this, E, GD: FD);
5822
5823 llvm_unreachable("Unhandled member declaration!");
5824}
5825
5826/// Given that we are currently emitting a lambda, emit an l-value for
5827/// one of its members.
5828///
5829LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field,
5830 llvm::Value *ThisValue) {
5831 bool HasExplicitObjectParameter = false;
5832 const auto *MD = dyn_cast_if_present<CXXMethodDecl>(Val: CurCodeDecl);
5833 if (MD) {
5834 HasExplicitObjectParameter = MD->isExplicitObjectMemberFunction();
5835 assert(MD->getParent()->isLambda());
5836 assert(MD->getParent() == Field->getParent());
5837 }
5838 LValue LambdaLV;
5839 if (HasExplicitObjectParameter) {
5840 const VarDecl *D = cast<CXXMethodDecl>(Val: CurCodeDecl)->getParamDecl(i: 0);
5841 auto It = LocalDeclMap.find(Val: D);
5842 assert(It != LocalDeclMap.end() && "explicit parameter not loaded?");
5843 Address AddrOfExplicitObject = It->getSecond();
5844 if (D->getType()->isReferenceType())
5845 LambdaLV = EmitLoadOfReferenceLValue(RefAddr: AddrOfExplicitObject, RefTy: D->getType(),
5846 Source: AlignmentSource::Decl);
5847 else
5848 LambdaLV = MakeAddrLValue(Addr: AddrOfExplicitObject,
5849 T: D->getType().getNonReferenceType());
5850
5851 // Make sure we have an lvalue to the lambda itself and not a derived class.
5852 auto *ThisTy = D->getType().getNonReferenceType()->getAsCXXRecordDecl();
5853 auto *LambdaTy = cast<CXXRecordDecl>(Val: Field->getParent());
5854 if (ThisTy != LambdaTy) {
5855 const CXXCastPath &BasePathArray = getContext().LambdaCastPaths.at(Val: MD);
5856 Address Base = GetAddressOfBaseClass(
5857 Value: LambdaLV.getAddress(), Derived: ThisTy, PathBegin: BasePathArray.begin(),
5858 PathEnd: BasePathArray.end(), /*NullCheckValue=*/false, Loc: SourceLocation());
5859 CanQualType T = getContext().getCanonicalTagType(TD: LambdaTy);
5860 LambdaLV = MakeAddrLValue(Addr: Base, T);
5861 }
5862 } else {
5863 CanQualType LambdaTagType =
5864 getContext().getCanonicalTagType(TD: Field->getParent());
5865 LambdaLV = MakeNaturalAlignAddrLValue(V: ThisValue, T: LambdaTagType);
5866 }
5867 return EmitLValueForField(Base: LambdaLV, Field);
5868}
5869
5870LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field) {
5871 return EmitLValueForLambdaField(Field, ThisValue: CXXABIThisValue);
5872}
5873
5874/// Get the field index in the debug info. The debug info structure/union
5875/// will ignore the unnamed bitfields.
5876unsigned CodeGenFunction::getDebugInfoFIndex(const RecordDecl *Rec,
5877 unsigned FieldIndex) {
5878 unsigned I = 0, Skipped = 0;
5879
5880 for (auto *F : Rec->getDefinition()->fields()) {
5881 if (I == FieldIndex)
5882 break;
5883 if (F->isUnnamedBitField())
5884 Skipped++;
5885 I++;
5886 }
5887
5888 return FieldIndex - Skipped;
5889}
5890
5891/// Get the address of a zero-sized field within a record. The resulting
5892/// address doesn't necessarily have the right type.
5893static Address emitAddrOfZeroSizeField(CodeGenFunction &CGF, Address Base,
5894 const FieldDecl *Field,
5895 bool IsInBounds) {
5896 CharUnits Offset = CGF.getContext().toCharUnitsFromBits(
5897 BitSize: CGF.getContext().getFieldOffset(FD: Field));
5898 if (Offset.isZero())
5899 return Base;
5900 Base = Base.withElementType(ElemTy: CGF.Int8Ty);
5901 if (!IsInBounds)
5902 return CGF.Builder.CreateConstByteGEP(Addr: Base, Offset);
5903 return CGF.Builder.CreateConstInBoundsByteGEP(Addr: Base, Offset);
5904}
5905
5906/// Drill down to the storage of a field without walking into reference types,
5907/// and without respect for pointer field protection.
5908///
5909/// The resulting address doesn't necessarily have the right type.
5910static Address emitRawAddrOfFieldStorage(CodeGenFunction &CGF, Address base,
5911 const FieldDecl *field,
5912 bool IsInBounds) {
5913 if (CodeGenUtils::isEmptyFieldForLayout(Ctx: CGF.getContext(), FD: field))
5914 return emitAddrOfZeroSizeField(CGF, Base: base, Field: field, IsInBounds);
5915
5916 const RecordDecl *rec = field->getParent();
5917
5918 unsigned idx =
5919 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(FD: field);
5920 llvm::Type *StructType =
5921 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMType();
5922
5923 if (CGF.getLangOpts().EmitLogicalPointer)
5924 return RawAddress(
5925 CGF.Builder.CreateStructuredGEP(BaseType: StructType, PtrBase: base.emitRawPointer(CGF),
5926 Indices: {CGF.Builder.getSize(N: idx)}),
5927 base.getElementType(), base.getAlignment());
5928
5929 if (!IsInBounds)
5930 return CGF.Builder.CreateConstGEP2_32(Addr: base, Idx0: 0, Idx1: idx, Name: field->getName());
5931
5932 return CGF.Builder.CreateStructGEP(Addr: base, Index: idx, Name: field->getName());
5933}
5934
5935/// Drill down to the storage of a field without walking into reference types,
5936/// wrapping the address in an llvm.protected.field.ptr intrinsic for the
5937/// pointer field protection feature if necessary.
5938///
5939/// The resulting address doesn't necessarily have the right type.
5940static Address emitAddrOfFieldStorage(CodeGenFunction &CGF, Address base,
5941 const FieldDecl *field, bool IsInBounds) {
5942 Address Addr = emitRawAddrOfFieldStorage(CGF, base, field, IsInBounds);
5943
5944 if (!CGF.getContext().isPFPField(Field: field))
5945 return Addr;
5946
5947 return CGF.EmitAddressOfPFPField(RecordPtr: base, FieldPtr: Addr, Field: field);
5948}
5949
5950static Address emitPreserveStructAccess(CodeGenFunction &CGF, LValue base,
5951 Address addr, const FieldDecl *field) {
5952 const RecordDecl *rec = field->getParent();
5953 llvm::DIType *DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(
5954 Ty: base.getType(), Loc: rec->getLocation());
5955
5956 unsigned idx =
5957 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(FD: field);
5958
5959 return CGF.Builder.CreatePreserveStructAccessIndex(
5960 Addr: addr, Index: idx, FieldIndex: CGF.getDebugInfoFIndex(Rec: rec, FieldIndex: field->getFieldIndex()), DbgInfo);
5961}
5962
5963static bool hasAnyVptr(const QualType Type, const ASTContext &Context) {
5964 const auto *RD = Type.getTypePtr()->getAsCXXRecordDecl();
5965 if (!RD)
5966 return false;
5967
5968 if (RD->isDynamicClass())
5969 return true;
5970
5971 for (const auto &Base : RD->bases())
5972 if (hasAnyVptr(Type: Base.getType(), Context))
5973 return true;
5974
5975 for (const FieldDecl *Field : RD->fields())
5976 if (hasAnyVptr(Type: Field->getType(), Context))
5977 return true;
5978
5979 return false;
5980}
5981
5982LValue CodeGenFunction::EmitLValueForField(LValue base, const FieldDecl *field,
5983 bool IsInBounds) {
5984 LValueBaseInfo BaseInfo = base.getBaseInfo();
5985
5986 if (field->isBitField()) {
5987 const CGRecordLayout &RL =
5988 CGM.getTypes().getCGRecordLayout(field->getParent());
5989 const CGBitFieldInfo &Info = RL.getBitFieldInfo(FD: field);
5990 const bool UseVolatile = CodeGenUtils::isAAPCS(TargetInfo: CGM.getTarget()) &&
5991 CGM.getCodeGenOpts().AAPCSBitfieldWidth &&
5992 Info.VolatileStorageSize != 0 &&
5993 field->getType()
5994 .withCVRQualifiers(CVR: base.getVRQualifiers())
5995 .isVolatileQualified();
5996 Address Addr = base.getAddress();
5997 unsigned Idx = RL.getLLVMFieldNo(FD: field);
5998 const RecordDecl *rec = field->getParent();
5999 if (hasBPFPreserveStaticOffset(D: rec))
6000 Addr = wrapWithBPFPreserveStaticOffset(CGF&: *this, Addr);
6001 if (!UseVolatile) {
6002 if (!IsInPreservedAIRegion &&
6003 (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
6004 if (Idx != 0) {
6005 // For structs, we GEP to the field that the record layout suggests.
6006 if (!IsInBounds)
6007 Addr = Builder.CreateConstGEP2_32(Addr, Idx0: 0, Idx1: Idx, Name: field->getName());
6008 else
6009 Addr = Builder.CreateStructGEP(Addr, Index: Idx, Name: field->getName());
6010 }
6011 } else {
6012 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType(
6013 Ty: getContext().getCanonicalTagType(TD: rec), L: rec->getLocation());
6014 Addr = Builder.CreatePreserveStructAccessIndex(
6015 Addr, Index: Idx, FieldIndex: getDebugInfoFIndex(Rec: rec, FieldIndex: field->getFieldIndex()),
6016 DbgInfo);
6017 }
6018 }
6019 const unsigned SS =
6020 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
6021 // Get the access type.
6022 llvm::Type *FieldIntTy = llvm::Type::getIntNTy(C&: getLLVMContext(), N: SS);
6023 Addr = Addr.withElementType(ElemTy: FieldIntTy);
6024 if (UseVolatile) {
6025 const unsigned VolatileOffset = Info.VolatileStorageOffset.getQuantity();
6026 if (VolatileOffset)
6027 Addr = Builder.CreateConstInBoundsGEP(Addr, Index: VolatileOffset);
6028 }
6029
6030 QualType fieldType =
6031 field->getType().withCVRQualifiers(CVR: base.getVRQualifiers());
6032 // TODO: Support TBAA for bit fields.
6033 LValueBaseInfo FieldBaseInfo(BaseInfo.getAlignmentSource());
6034 return LValue::MakeBitfield(Addr, Info, type: fieldType, BaseInfo: FieldBaseInfo,
6035 TBAAInfo: TBAAAccessInfo());
6036 }
6037
6038 // Fields of may-alias structures are may-alias themselves.
6039 // FIXME: this should get propagated down through anonymous structs
6040 // and unions.
6041 QualType FieldType = field->getType();
6042 const RecordDecl *rec = field->getParent();
6043 AlignmentSource BaseAlignSource = BaseInfo.getAlignmentSource();
6044 LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(Source: BaseAlignSource));
6045 TBAAAccessInfo FieldTBAAInfo;
6046 if (base.getTBAAInfo().isMayAlias() ||
6047 rec->hasAttr<MayAliasAttr>() || FieldType->isVectorType()) {
6048 FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
6049 } else if (rec->isUnion()) {
6050 // TODO: Support TBAA for unions.
6051 FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
6052 } else {
6053 // If no base type been assigned for the base access, then try to generate
6054 // one for this base lvalue.
6055 FieldTBAAInfo = base.getTBAAInfo();
6056 if (!FieldTBAAInfo.BaseType) {
6057 FieldTBAAInfo.BaseType = CGM.getTBAABaseTypeInfo(QTy: base.getType());
6058 assert(!FieldTBAAInfo.Offset &&
6059 "Nonzero offset for an access with no base type!");
6060 }
6061
6062 // Adjust offset to be relative to the base type.
6063 const ASTRecordLayout &Layout =
6064 getContext().getASTRecordLayout(D: field->getParent());
6065 unsigned CharWidth = getContext().getCharWidth();
6066 if (FieldTBAAInfo.BaseType)
6067 FieldTBAAInfo.Offset +=
6068 Layout.getFieldOffset(FieldNo: field->getFieldIndex()) / CharWidth;
6069
6070 // Update the final access type and size.
6071 FieldTBAAInfo.AccessType = CGM.getTBAATypeInfo(QTy: FieldType);
6072 FieldTBAAInfo.Size =
6073 getContext().getTypeSizeInChars(T: FieldType).getQuantity();
6074 }
6075
6076 Address addr = base.getAddress();
6077 if (hasBPFPreserveStaticOffset(D: rec))
6078 addr = wrapWithBPFPreserveStaticOffset(CGF&: *this, Addr&: addr);
6079
6080 unsigned RecordCVR = base.getVRQualifiers();
6081 if (rec->isUnion()) {
6082 // For unions, there is no pointer adjustment.
6083 if (CGM.getCodeGenOpts().StrictVTablePointers &&
6084 hasAnyVptr(Type: FieldType, Context: getContext()))
6085 // Because unions can easily skip invariant.barriers, we need to add
6086 // a barrier every time CXXRecord field with vptr is referenced.
6087 addr = Builder.CreateLaunderInvariantGroup(Addr: addr);
6088
6089 if (IsInPreservedAIRegion ||
6090 (getDebugInfo() && rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
6091 // Remember the original union field index
6092 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(Ty: base.getType(),
6093 Loc: rec->getLocation());
6094 addr =
6095 Address(Builder.CreatePreserveUnionAccessIndex(
6096 Base: addr.emitRawPointer(CGF&: *this),
6097 FieldIndex: getDebugInfoFIndex(Rec: rec, FieldIndex: field->getFieldIndex()), DbgInfo),
6098 addr.getElementType(), addr.getAlignment());
6099 }
6100
6101 if (FieldType->isReferenceType())
6102 addr = addr.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: FieldType));
6103 } else {
6104 if (!IsInPreservedAIRegion &&
6105 (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>()))
6106 // For structs, we GEP to the field that the record layout suggests.
6107 addr = emitAddrOfFieldStorage(CGF&: *this, base: addr, field, IsInBounds);
6108 else
6109 // Remember the original struct field index
6110 addr = emitPreserveStructAccess(CGF&: *this, base, addr, field);
6111 }
6112
6113 // If this is a reference field, load the reference right now.
6114 if (FieldType->isReferenceType()) {
6115 LValue RefLVal =
6116 MakeAddrLValue(Addr: addr, T: FieldType, BaseInfo: FieldBaseInfo, TBAAInfo: FieldTBAAInfo);
6117 if (RecordCVR & Qualifiers::Volatile)
6118 RefLVal.getQuals().addVolatile();
6119 addr = EmitLoadOfReference(RefLVal, PointeeBaseInfo: &FieldBaseInfo, PointeeTBAAInfo: &FieldTBAAInfo);
6120
6121 // Qualifiers on the struct don't apply to the referencee.
6122 RecordCVR = 0;
6123 FieldType = FieldType->getPointeeType();
6124 }
6125
6126 // Make sure that the address is pointing to the right type. This is critical
6127 // for both unions and structs.
6128 addr = addr.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T: FieldType));
6129
6130 if (field->hasAttr<AnnotateAttr>())
6131 addr = EmitFieldAnnotations(D: field, V: addr);
6132
6133 LValue LV = MakeAddrLValue(Addr: addr, T: FieldType, BaseInfo: FieldBaseInfo, TBAAInfo: FieldTBAAInfo);
6134 LV.getQuals().addCVRQualifiers(mask: RecordCVR);
6135
6136 // __weak attribute on a field is ignored.
6137 if (LV.getQuals().getObjCGCAttr() == Qualifiers::Weak)
6138 LV.getQuals().removeObjCGCAttr();
6139
6140 return LV;
6141}
6142
6143LValue
6144CodeGenFunction::EmitLValueForFieldInitialization(LValue Base,
6145 const FieldDecl *Field) {
6146 QualType FieldType = Field->getType();
6147
6148 if (!FieldType->isReferenceType())
6149 return EmitLValueForField(base: Base, field: Field);
6150
6151 Address V = emitAddrOfFieldStorage(
6152 CGF&: *this, base: Base.getAddress(), field: Field,
6153 /*IsInBounds=*/!getLangOpts().PointerOverflowDefined);
6154
6155 // Make sure that the address is pointing to the right type.
6156 llvm::Type *llvmType = ConvertTypeForMem(T: FieldType);
6157 V = V.withElementType(ElemTy: llvmType);
6158
6159 // TODO: Generate TBAA information that describes this access as a structure
6160 // member access and not just an access to an object of the field's type. This
6161 // should be similar to what we do in EmitLValueForField().
6162 LValueBaseInfo BaseInfo = Base.getBaseInfo();
6163 AlignmentSource FieldAlignSource = BaseInfo.getAlignmentSource();
6164 LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(Source: FieldAlignSource));
6165 return MakeAddrLValue(Addr: V, T: FieldType, BaseInfo: FieldBaseInfo,
6166 TBAAInfo: CGM.getTBAAInfoForSubobject(Base, AccessType: FieldType));
6167}
6168
6169LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr *E){
6170 if (E->isFileScope()) {
6171 ConstantAddress GlobalPtr = CGM.GetAddrOfConstantCompoundLiteral(E);
6172 return MakeAddrLValue(Addr: GlobalPtr, T: E->getType(), Source: AlignmentSource::Decl);
6173 }
6174 if (E->getType()->isVariablyModifiedType())
6175 // make sure to emit the VLA size.
6176 EmitVariablyModifiedType(Ty: E->getType());
6177
6178 Address DeclPtr = CreateMemTempWithoutCast(Ty: E->getType(), Name: ".compoundliteral");
6179 const Expr *InitExpr = E->getInitializer();
6180 LValue Result = MakeAddrLValue(Addr: DeclPtr, T: E->getType(), Source: AlignmentSource::Decl);
6181
6182 if (!getLangOpts().CPlusPlus) {
6183 if (HaveInsertPoint() && !hasLabelBeenSeenInCurrentScope() &&
6184 EmitLifetimeStart(Addr: DeclPtr.getBasePointer()))
6185 pushCleanupAfterFullExpr<CallLifetimeEnd>(Kind: NormalEHLifetimeMarker,
6186 A: DeclPtr);
6187 }
6188
6189 EmitAnyExprToMem(E: InitExpr, Location: DeclPtr, Quals: E->getType().getQualifiers(),
6190 /*Init*/ IsInit: true);
6191
6192 // Block-scope compound literals are destroyed at the end of the enclosing
6193 // scope in C.
6194 if (!getLangOpts().CPlusPlus)
6195 if (QualType::DestructionKind DtorKind = E->getType().isDestructedType())
6196 pushLifetimeExtendedDestroy(dtorKind: DtorKind, addr: DeclPtr, type: E->getType());
6197
6198 return Result;
6199}
6200
6201LValue CodeGenFunction::EmitInitListLValue(const InitListExpr *E) {
6202 if (!E->isGLValue())
6203 // Initializing an aggregate temporary in C++11: T{...}.
6204 return EmitAggExprToLValue(E);
6205
6206 // An lvalue initializer list must be initializing a reference.
6207 assert(E->isTransparent() && "non-transparent glvalue init list");
6208 return EmitLValue(E: E->getInit(Init: 0));
6209}
6210
6211/// Emit the operand of a glvalue conditional operator. This is either a glvalue
6212/// or a (possibly-parenthesized) throw-expression. If this is a throw, no
6213/// LValue is returned and the current block has been terminated.
6214static std::optional<LValue> EmitLValueOrThrowExpression(CodeGenFunction &CGF,
6215 const Expr *Operand) {
6216 if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Val: Operand->IgnoreParens())) {
6217 CGF.EmitCXXThrowExpr(E: ThrowExpr, /*KeepInsertionPoint*/false);
6218 return std::nullopt;
6219 }
6220
6221 return CGF.EmitLValue(E: Operand);
6222}
6223
6224namespace {
6225// Handle the case where the condition is a constant evaluatable simple integer,
6226// which means we don't have to separately handle the true/false blocks.
6227std::optional<LValue> HandleConditionalOperatorLValueSimpleCase(
6228 CodeGenFunction &CGF, const AbstractConditionalOperator *E) {
6229 const Expr *condExpr = E->getCond();
6230 bool CondExprBool;
6231 if (CGF.ConstantFoldsToSimpleInteger(Cond: condExpr, Result&: CondExprBool)) {
6232 const Expr *Live = E->getTrueExpr(), *Dead = E->getFalseExpr();
6233 if (!CondExprBool)
6234 std::swap(a&: Live, b&: Dead);
6235
6236 if (!CGF.ContainsLabel(S: Dead)) {
6237 // If the true case is live, we need to track its region.
6238 CGF.incrementProfileCounter(ExecSkip: CondExprBool ? CGF.UseExecPath
6239 : CGF.UseSkipPath,
6240 S: E, /*UseBoth=*/true);
6241 CGF.markStmtMaybeUsed(S: Dead);
6242 // If a throw expression we emit it and return an undefined lvalue
6243 // because it can't be used.
6244 if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Val: Live->IgnoreParens())) {
6245 CGF.EmitCXXThrowExpr(E: ThrowExpr);
6246 llvm::Type *ElemTy = CGF.ConvertType(T: Dead->getType());
6247 llvm::Type *Ty = CGF.DefaultPtrTy;
6248 return CGF.MakeAddrLValue(
6249 Addr: Address(llvm::UndefValue::get(T: Ty), ElemTy, CharUnits::One()),
6250 T: Dead->getType());
6251 }
6252 return CGF.EmitLValue(E: Live);
6253 }
6254 }
6255 return std::nullopt;
6256}
6257struct ConditionalInfo {
6258 llvm::BasicBlock *lhsBlock, *rhsBlock;
6259 std::optional<LValue> LHS, RHS;
6260};
6261
6262// Create and generate the 3 blocks for a conditional operator.
6263// Leaves the 'current block' in the continuation basic block.
6264template<typename FuncTy>
6265ConditionalInfo EmitConditionalBlocks(CodeGenFunction &CGF,
6266 const AbstractConditionalOperator *E,
6267 const FuncTy &BranchGenFunc) {
6268 ConditionalInfo Info{.lhsBlock: CGF.createBasicBlock(name: "cond.true"),
6269 .rhsBlock: CGF.createBasicBlock(name: "cond.false"), .LHS: std::nullopt,
6270 .RHS: std::nullopt};
6271 llvm::BasicBlock *endBlock = CGF.createBasicBlock(name: "cond.end");
6272
6273 CodeGenFunction::ConditionalEvaluation eval(CGF);
6274 CGF.EmitBranchOnBoolExpr(Cond: E->getCond(), TrueBlock: Info.lhsBlock, FalseBlock: Info.rhsBlock,
6275 TrueCount: CGF.getProfileCount(S: E));
6276
6277 // Any temporaries created here are conditional.
6278 CGF.EmitBlock(BB: Info.lhsBlock);
6279 CGF.incrementProfileCounter(ExecSkip: CGF.UseExecPath, S: E);
6280 eval.begin(CGF);
6281 Info.LHS = BranchGenFunc(CGF, E->getTrueExpr());
6282 eval.end(CGF);
6283 Info.lhsBlock = CGF.Builder.GetInsertBlock();
6284
6285 if (Info.LHS)
6286 CGF.Builder.CreateBr(Dest: endBlock);
6287
6288 // Any temporaries created here are conditional.
6289 CGF.EmitBlock(BB: Info.rhsBlock);
6290 CGF.incrementProfileCounter(ExecSkip: CGF.UseSkipPath, S: E);
6291 eval.begin(CGF);
6292 Info.RHS = BranchGenFunc(CGF, E->getFalseExpr());
6293 eval.end(CGF);
6294 Info.rhsBlock = CGF.Builder.GetInsertBlock();
6295 CGF.EmitBlock(BB: endBlock);
6296
6297 return Info;
6298}
6299} // namespace
6300
6301void CodeGenFunction::EmitIgnoredConditionalOperator(
6302 const AbstractConditionalOperator *E) {
6303 if (!E->isGLValue()) {
6304 // ?: here should be an aggregate.
6305 assert(hasAggregateEvaluationKind(E->getType()) &&
6306 "Unexpected conditional operator!");
6307 return (void)EmitAggExprToLValue(E);
6308 }
6309
6310 OpaqueValueMapping binding(*this, E);
6311 if (HandleConditionalOperatorLValueSimpleCase(CGF&: *this, E))
6312 return;
6313
6314 EmitConditionalBlocks(CGF&: *this, E, BranchGenFunc: [](CodeGenFunction &CGF, const Expr *E) {
6315 CGF.EmitIgnoredExpr(E);
6316 return LValue{};
6317 });
6318}
6319LValue CodeGenFunction::EmitConditionalOperatorLValue(
6320 const AbstractConditionalOperator *expr) {
6321 if (!expr->isGLValue()) {
6322 // ?: here should be an aggregate.
6323 assert(hasAggregateEvaluationKind(expr->getType()) &&
6324 "Unexpected conditional operator!");
6325 return EmitAggExprToLValue(E: expr);
6326 }
6327
6328 OpaqueValueMapping binding(*this, expr);
6329 if (std::optional<LValue> Res =
6330 HandleConditionalOperatorLValueSimpleCase(CGF&: *this, E: expr))
6331 return *Res;
6332
6333 ConditionalInfo Info = EmitConditionalBlocks(
6334 CGF&: *this, E: expr, BranchGenFunc: [](CodeGenFunction &CGF, const Expr *E) {
6335 return EmitLValueOrThrowExpression(CGF, Operand: E);
6336 });
6337
6338 if ((Info.LHS && !Info.LHS->isSimple()) ||
6339 (Info.RHS && !Info.RHS->isSimple()))
6340 return EmitUnsupportedLValue(E: expr, Name: "conditional operator");
6341
6342 if (Info.LHS && Info.RHS) {
6343 Address lhsAddr = Info.LHS->getAddress();
6344 Address rhsAddr = Info.RHS->getAddress();
6345 Address result = mergeAddressesInConditionalExpr(
6346 LHS: lhsAddr, RHS: rhsAddr, LHSBlock: Info.lhsBlock, RHSBlock: Info.rhsBlock,
6347 MergeBlock: Builder.GetInsertBlock(), MergedType: expr->getType());
6348 AlignmentSource alignSource =
6349 std::max(a: Info.LHS->getBaseInfo().getAlignmentSource(),
6350 b: Info.RHS->getBaseInfo().getAlignmentSource());
6351 TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForConditionalOperator(
6352 InfoA: Info.LHS->getTBAAInfo(), InfoB: Info.RHS->getTBAAInfo());
6353 return MakeAddrLValue(Addr: result, T: expr->getType(), BaseInfo: LValueBaseInfo(alignSource),
6354 TBAAInfo);
6355 } else {
6356 assert((Info.LHS || Info.RHS) &&
6357 "both operands of glvalue conditional are throw-expressions?");
6358 return Info.LHS ? *Info.LHS : *Info.RHS;
6359 }
6360}
6361
6362/// EmitCastLValue - Casts are never lvalues unless that cast is to a reference
6363/// type. If the cast is to a reference, we can have the usual lvalue result,
6364/// otherwise if a cast is needed by the code generator in an lvalue context,
6365/// then it must mean that we need the address of an aggregate in order to
6366/// access one of its members. This can happen for all the reasons that casts
6367/// are permitted with aggregate result, including noop aggregate casts, and
6368/// cast from scalar to union.
6369LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
6370 llvm::scope_exit RestoreCurCast([this, Prev = CurCast] { CurCast = Prev; });
6371 CurCast = E;
6372 switch (E->getCastKind()) {
6373 case CK_ToVoid:
6374 case CK_BitCast:
6375 case CK_LValueToRValueBitCast:
6376 case CK_ArrayToPointerDecay:
6377 case CK_FunctionToPointerDecay:
6378 case CK_NullToMemberPointer:
6379 case CK_NullToPointer:
6380 case CK_IntegralToPointer:
6381 case CK_PointerToIntegral:
6382 case CK_PointerToBoolean:
6383 case CK_IntegralCast:
6384 case CK_BooleanToSignedIntegral:
6385 case CK_IntegralToBoolean:
6386 case CK_IntegralToFloating:
6387 case CK_FloatingToIntegral:
6388 case CK_FloatingToBoolean:
6389 case CK_FloatingCast:
6390 case CK_FloatingRealToComplex:
6391 case CK_FloatingComplexToReal:
6392 case CK_FloatingComplexToBoolean:
6393 case CK_FloatingComplexCast:
6394 case CK_FloatingComplexToIntegralComplex:
6395 case CK_IntegralRealToComplex:
6396 case CK_IntegralComplexToReal:
6397 case CK_IntegralComplexToBoolean:
6398 case CK_IntegralComplexCast:
6399 case CK_IntegralComplexToFloatingComplex:
6400 case CK_DerivedToBaseMemberPointer:
6401 case CK_BaseToDerivedMemberPointer:
6402 case CK_MemberPointerToBoolean:
6403 case CK_ReinterpretMemberPointer:
6404 case CK_AnyPointerToBlockPointerCast:
6405 case CK_ARCProduceObject:
6406 case CK_ARCConsumeObject:
6407 case CK_ARCReclaimReturnedObject:
6408 case CK_ARCExtendBlockObject:
6409 case CK_CopyAndAutoreleaseBlockObject:
6410 case CK_IntToOCLSampler:
6411 case CK_FloatingToFixedPoint:
6412 case CK_FixedPointToFloating:
6413 case CK_FixedPointCast:
6414 case CK_FixedPointToBoolean:
6415 case CK_FixedPointToIntegral:
6416 case CK_IntegralToFixedPoint:
6417 case CK_MatrixCast:
6418 case CK_HLSLVectorTruncation:
6419 case CK_HLSLMatrixTruncation:
6420 case CK_HLSLArrayRValue:
6421 case CK_HLSLElementwiseCast:
6422 case CK_HLSLAggregateSplatCast:
6423 return EmitUnsupportedLValue(E, Name: "unexpected cast lvalue");
6424
6425 case CK_Dependent:
6426 llvm_unreachable("dependent cast kind in IR gen!");
6427
6428 case CK_BuiltinFnToFnPtr:
6429 llvm_unreachable("builtin functions are handled elsewhere");
6430
6431 // These are never l-values; just use the aggregate emission code.
6432 case CK_NonAtomicToAtomic:
6433 case CK_AtomicToNonAtomic:
6434 return EmitAggExprToLValue(E);
6435
6436 case CK_Dynamic: {
6437 LValue LV = EmitLValue(E: E->getSubExpr());
6438 Address V = LV.getAddress();
6439 const auto *DCE = cast<CXXDynamicCastExpr>(Val: E);
6440 return MakeNaturalAlignRawAddrLValue(V: EmitDynamicCast(V, DCE), T: E->getType());
6441 }
6442
6443 case CK_ConstructorConversion:
6444 case CK_UserDefinedConversion:
6445 case CK_CPointerToObjCPointerCast:
6446 case CK_BlockPointerToObjCPointerCast:
6447 case CK_LValueToRValue:
6448 return EmitLValue(E: E->getSubExpr());
6449
6450 case CK_NoOp: {
6451 // CK_NoOp can model a qualification conversion, which can remove an array
6452 // bound and change the IR type.
6453 // FIXME: Once pointee types are removed from IR, remove this.
6454 LValue LV = EmitLValue(E: E->getSubExpr());
6455 // Propagate the volatile qualifer to LValue, if exist in E.
6456 if (E->changesVolatileQualification())
6457 LV.getQuals() = E->getType().getQualifiers();
6458 if (LV.isSimple()) {
6459 Address V = LV.getAddress();
6460 if (V.isValid()) {
6461 llvm::Type *T = ConvertTypeForMem(T: E->getType());
6462 if (V.getElementType() != T)
6463 LV.setAddress(V.withElementType(ElemTy: T));
6464 }
6465 }
6466 return LV;
6467 }
6468
6469 case CK_UncheckedDerivedToBase:
6470 case CK_DerivedToBase: {
6471 auto *DerivedClassDecl = E->getSubExpr()->getType()->castAsCXXRecordDecl();
6472 LValue LV = EmitLValue(E: E->getSubExpr());
6473 Address This = LV.getAddress();
6474
6475 // Perform the derived-to-base conversion
6476 Address Base = GetAddressOfBaseClass(
6477 Value: This, Derived: DerivedClassDecl, PathBegin: E->path_begin(), PathEnd: E->path_end(),
6478 /*NullCheckValue=*/false, Loc: E->getExprLoc());
6479
6480 // TODO: Support accesses to members of base classes in TBAA. For now, we
6481 // conservatively pretend that the complete object is of the base class
6482 // type.
6483 return MakeAddrLValue(Addr: Base, T: E->getType(), BaseInfo: LV.getBaseInfo(),
6484 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
6485 }
6486 case CK_ToUnion:
6487 return EmitAggExprToLValue(E);
6488 case CK_BaseToDerived: {
6489 auto *DerivedClassDecl = E->getType()->castAsCXXRecordDecl();
6490 LValue LV = EmitLValue(E: E->getSubExpr());
6491
6492 // Perform the base-to-derived conversion
6493 Address Derived = GetAddressOfDerivedClass(
6494 Value: LV.getAddress(), Derived: DerivedClassDecl, PathBegin: E->path_begin(), PathEnd: E->path_end(),
6495 /*NullCheckValue=*/false);
6496
6497 // C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is
6498 // performed and the object is not of the derived type.
6499 if (sanitizePerformTypeCheck())
6500 EmitTypeCheck(TCK: TCK_DowncastReference, Loc: E->getExprLoc(), Addr: Derived,
6501 Type: E->getType());
6502
6503 if (SanOpts.has(K: SanitizerKind::CFIDerivedCast))
6504 EmitVTablePtrCheckForCast(T: E->getType(), Derived,
6505 /*MayBeNull=*/false, TCK: CFITCK_DerivedCast,
6506 Loc: E->getBeginLoc());
6507
6508 return MakeAddrLValue(Addr: Derived, T: E->getType(), BaseInfo: LV.getBaseInfo(),
6509 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
6510 }
6511 case CK_LValueBitCast: {
6512 // This must be a reinterpret_cast (or c-style equivalent).
6513 const auto *CE = cast<ExplicitCastExpr>(Val: E);
6514
6515 CGM.EmitExplicitCastExprType(E: CE, CGF: this);
6516 LValue LV = EmitLValue(E: E->getSubExpr());
6517 Address V = LV.getAddress().withElementType(
6518 ElemTy: ConvertTypeForMem(T: CE->getTypeAsWritten()->getPointeeType()));
6519
6520 if (SanOpts.has(K: SanitizerKind::CFIUnrelatedCast))
6521 EmitVTablePtrCheckForCast(T: E->getType(), Derived: V,
6522 /*MayBeNull=*/false, TCK: CFITCK_UnrelatedCast,
6523 Loc: E->getBeginLoc());
6524
6525 return MakeAddrLValue(Addr: V, T: E->getType(), BaseInfo: LV.getBaseInfo(),
6526 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
6527 }
6528 case CK_AddressSpaceConversion: {
6529 LValue LV = EmitLValue(E: E->getSubExpr());
6530 QualType DestTy = getContext().getPointerType(T: E->getType());
6531 llvm::Value *V =
6532 performAddrSpaceCast(Src: LV.getPointer(CGF&: *this), DestTy: ConvertType(T: DestTy));
6533 return MakeAddrLValue(Addr: Address(V, ConvertTypeForMem(T: E->getType()),
6534 LV.getAddress().getAlignment()),
6535 T: E->getType(), BaseInfo: LV.getBaseInfo(), TBAAInfo: LV.getTBAAInfo());
6536 }
6537 case CK_ObjCObjectLValueCast: {
6538 LValue LV = EmitLValue(E: E->getSubExpr());
6539 Address V = LV.getAddress().withElementType(ElemTy: ConvertType(T: E->getType()));
6540 return MakeAddrLValue(Addr: V, T: E->getType(), BaseInfo: LV.getBaseInfo(),
6541 TBAAInfo: CGM.getTBAAInfoForSubobject(Base: LV, AccessType: E->getType()));
6542 }
6543 case CK_ZeroToOCLOpaqueType:
6544 llvm_unreachable("NULL to OpenCL opaque type lvalue cast is not valid");
6545
6546 case CK_VectorSplat: {
6547 // LValue results of vector splats are only supported in HLSL.
6548 if (!getLangOpts().HLSL)
6549 return EmitUnsupportedLValue(E, Name: "unexpected cast lvalue");
6550 return EmitLValue(E: E->getSubExpr());
6551 }
6552 }
6553
6554 llvm_unreachable("Unhandled lvalue cast kind?");
6555}
6556
6557LValue CodeGenFunction::EmitOpaqueValueLValue(const OpaqueValueExpr *e) {
6558 assert(OpaqueValueMappingData::shouldBindAsLValue(e));
6559 return getOrCreateOpaqueLValueMapping(e);
6560}
6561
6562std::pair<LValue, LValue>
6563CodeGenFunction::EmitHLSLOutArgLValues(const HLSLOutArgExpr *E, QualType Ty) {
6564 // Emitting the casted temporary through an opaque value.
6565 LValue BaseLV = EmitLValue(E: E->getArgLValue());
6566 OpaqueValueMappingData::bind(CGF&: *this, ov: E->getOpaqueArgLValue(), lv: BaseLV);
6567
6568 QualType ExprTy = E->getType();
6569 Address OutTemp = CreateIRTempWithoutCast(Ty: ExprTy);
6570 LValue TempLV = MakeAddrLValue(Addr: OutTemp, T: ExprTy);
6571
6572 // Start the lifetime before the copy-in so that the temporary is live when
6573 // the initial value is written. This ensures the store is within the
6574 // lifetime and is not killed by a store undef inserted at lifetime.start.
6575 EmitLifetimeStart(Addr: OutTemp.getBasePointer());
6576
6577 if (E->isInOut())
6578 EmitInitializationToLValue(E: E->getCastedTemporary()->getSourceExpr(),
6579 LV: TempLV);
6580
6581 OpaqueValueMappingData::bind(CGF&: *this, ov: E->getCastedTemporary(), lv: TempLV);
6582 return std::make_pair(x&: BaseLV, y&: TempLV);
6583}
6584
6585LValue CodeGenFunction::EmitHLSLOutArgExpr(const HLSLOutArgExpr *E,
6586 CallArgList &Args, QualType Ty) {
6587
6588 auto [BaseLV, TempLV] = EmitHLSLOutArgLValues(E, Ty);
6589
6590 llvm::Value *Addr = TempLV.getAddress().getBasePointer();
6591 llvm::Type *ElTy = ConvertTypeForMem(T: TempLV.getType());
6592
6593 Address TmpAddr(Addr, ElTy, TempLV.getAlignment());
6594 Args.addWriteback(srcLV: BaseLV, temporary: TmpAddr, toUse: nullptr, writebackExpr: E->getWritebackCast());
6595 Args.add(rvalue: RValue::get(Addr: TmpAddr, CGF&: *this), type: Ty);
6596 return TempLV;
6597}
6598
6599LValue
6600CodeGenFunction::getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e) {
6601 assert(OpaqueValueMapping::shouldBindAsLValue(e));
6602
6603 llvm::DenseMap<const OpaqueValueExpr*,LValue>::iterator
6604 it = OpaqueLValues.find(Val: e);
6605
6606 if (it != OpaqueLValues.end())
6607 return it->second;
6608
6609 assert(e->isUnique() && "LValue for a nonunique OVE hasn't been emitted");
6610 return EmitLValue(E: e->getSourceExpr());
6611}
6612
6613RValue
6614CodeGenFunction::getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e) {
6615 assert(!OpaqueValueMapping::shouldBindAsLValue(e));
6616
6617 llvm::DenseMap<const OpaqueValueExpr*,RValue>::iterator
6618 it = OpaqueRValues.find(Val: e);
6619
6620 if (it != OpaqueRValues.end())
6621 return it->second;
6622
6623 assert(e->isUnique() && "RValue for a nonunique OVE hasn't been emitted");
6624 return EmitAnyExpr(E: e->getSourceExpr());
6625}
6626
6627bool CodeGenFunction::isOpaqueValueEmitted(const OpaqueValueExpr *E) {
6628 if (OpaqueValueMapping::shouldBindAsLValue(expr: E))
6629 return OpaqueLValues.contains(Val: E);
6630 return OpaqueRValues.contains(Val: E);
6631}
6632
6633RValue CodeGenFunction::EmitRValueForField(LValue LV,
6634 const FieldDecl *FD,
6635 SourceLocation Loc) {
6636 QualType FT = FD->getType();
6637 LValue FieldLV = EmitLValueForField(base: LV, field: FD);
6638 switch (getEvaluationKind(T: FT)) {
6639 case TEK_Complex:
6640 return RValue::getComplex(C: EmitLoadOfComplex(src: FieldLV, loc: Loc));
6641 case TEK_Aggregate:
6642 return FieldLV.asAggregateRValue();
6643 case TEK_Scalar:
6644 // This routine is used to load fields one-by-one to perform a copy, so
6645 // don't load reference fields.
6646 if (FD->getType()->isReferenceType())
6647 return RValue::get(V: FieldLV.getPointer(CGF&: *this));
6648 // Call EmitLoadOfScalar except when the lvalue is a bitfield to emit a
6649 // primitive load.
6650 if (FieldLV.isBitField())
6651 return EmitLoadOfLValue(LV: FieldLV, Loc);
6652 return RValue::get(V: EmitLoadOfScalar(lvalue: FieldLV, Loc));
6653 }
6654 llvm_unreachable("bad evaluation kind");
6655}
6656
6657//===--------------------------------------------------------------------===//
6658// Expression Emission
6659//===--------------------------------------------------------------------===//
6660
6661RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
6662 ReturnValueSlot ReturnValue,
6663 llvm::CallBase **CallOrInvoke) {
6664 llvm::CallBase *CallOrInvokeStorage;
6665 if (!CallOrInvoke) {
6666 CallOrInvoke = &CallOrInvokeStorage;
6667 }
6668
6669 llvm::scope_exit AddCoroElideSafeOnExit([&] {
6670 if (E->isCoroElideSafe()) {
6671 auto *I = *CallOrInvoke;
6672 if (I)
6673 I->addFnAttr(Kind: llvm::Attribute::CoroElideSafe);
6674 }
6675 });
6676
6677 // Builtins never have block type.
6678 if (E->getCallee()->getType()->isBlockPointerType())
6679 return EmitBlockCallExpr(E, ReturnValue, CallOrInvoke);
6680
6681 if (const auto *CE = dyn_cast<CXXMemberCallExpr>(Val: E))
6682 return EmitCXXMemberCallExpr(E: CE, ReturnValue, CallOrInvoke);
6683
6684 if (const auto *CE = dyn_cast<CUDAKernelCallExpr>(Val: E))
6685 return EmitCUDAKernelCallExpr(E: CE, ReturnValue, CallOrInvoke);
6686
6687 // A CXXOperatorCallExpr is created even for explicit object methods, but
6688 // these should be treated like static function call.
6689 if (const auto *CE = dyn_cast<CXXOperatorCallExpr>(Val: E))
6690 if (const auto *MD =
6691 dyn_cast_if_present<CXXMethodDecl>(Val: CE->getCalleeDecl());
6692 MD && MD->isImplicitObjectMemberFunction())
6693 return EmitCXXOperatorMemberCallExpr(E: CE, MD, ReturnValue, CallOrInvoke);
6694
6695 CGCallee callee = EmitCallee(E: E->getCallee());
6696
6697 if (callee.isBuiltin()) {
6698 return EmitBuiltinExpr(GD: callee.getBuiltinDecl(), BuiltinID: callee.getBuiltinID(),
6699 E, ReturnValue);
6700 }
6701
6702 if (callee.isPseudoDestructor()) {
6703 return EmitCXXPseudoDestructorExpr(E: callee.getPseudoDestructorExpr());
6704 }
6705
6706 return EmitCall(FnType: E->getCallee()->getType(), Callee: callee, E, ReturnValue,
6707 /*Chain=*/nullptr, CallOrInvoke);
6708}
6709
6710/// Emit a CallExpr without considering whether it might be a subclass.
6711RValue CodeGenFunction::EmitSimpleCallExpr(const CallExpr *E,
6712 ReturnValueSlot ReturnValue,
6713 llvm::CallBase **CallOrInvoke) {
6714 CGCallee Callee = EmitCallee(E: E->getCallee());
6715 return EmitCall(FnType: E->getCallee()->getType(), Callee, E, ReturnValue,
6716 /*Chain=*/nullptr, CallOrInvoke);
6717}
6718
6719static CGCallee EmitDirectCallee(CodeGenFunction &CGF, GlobalDecl GD) {
6720 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
6721
6722 if (auto builtinID = FD->getBuiltinID()) {
6723 std::string NoBuiltinFD = ("no-builtin-" + FD->getName()).str();
6724 std::string NoBuiltins = "no-builtins";
6725
6726 StringRef Ident = CGF.CGM.getMangledName(GD);
6727 std::string FDInlineName = (Ident + ".inline").str();
6728
6729 bool IsPredefinedLibFunction =
6730 CGF.getContext().BuiltinInfo.isPredefinedLibFunction(ID: builtinID);
6731 bool HasAttributeNoBuiltin =
6732 CGF.CurFn->getAttributes().hasFnAttr(Kind: NoBuiltinFD) ||
6733 CGF.CurFn->getAttributes().hasFnAttr(Kind: NoBuiltins);
6734
6735 // When directing calling an inline builtin, call it through it's mangled
6736 // name to make it clear it's not the actual builtin.
6737 if (CGF.CurFn->getName() != FDInlineName &&
6738 CodeGenUtils::onlyHasInlineBuiltinDeclaration(FD)) {
6739 llvm::Constant *CalleePtr = CGF.CGM.getRawFunctionPointer(GD);
6740 llvm::Function *Fn = llvm::cast<llvm::Function>(Val: CalleePtr);
6741 llvm::Module *M = Fn->getParent();
6742 llvm::Function *Clone = M->getFunction(Name: FDInlineName);
6743 if (!Clone) {
6744 Clone = llvm::Function::Create(Ty: Fn->getFunctionType(),
6745 Linkage: llvm::GlobalValue::InternalLinkage,
6746 AddrSpace: Fn->getAddressSpace(), N: FDInlineName, M);
6747 Clone->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
6748 }
6749 return CGCallee::forDirect(functionPtr: Clone, abstractInfo: GD);
6750 }
6751
6752 // Replaceable builtins provide their own implementation of a builtin. If we
6753 // are in an inline builtin implementation, avoid trivial infinite
6754 // recursion. Honor __attribute__((no_builtin("foo"))) or
6755 // __attribute__((no_builtin)) on the current function unless foo is
6756 // not a predefined library function which means we must generate the
6757 // builtin no matter what.
6758 else if (!IsPredefinedLibFunction || !HasAttributeNoBuiltin)
6759 return CGCallee::forBuiltin(builtinID, builtinDecl: FD);
6760 }
6761
6762 llvm::Constant *CalleePtr = CGF.CGM.getRawFunctionPointer(GD);
6763 if (CGF.CGM.getLangOpts().CUDA && !CGF.CGM.getLangOpts().CUDAIsDevice &&
6764 FD->hasAttr<CUDAGlobalAttr>())
6765 CalleePtr = CGF.CGM.getCUDARuntime().getKernelStub(
6766 Handle: cast<llvm::GlobalValue>(Val: CalleePtr->stripPointerCasts()));
6767
6768 return CGCallee::forDirect(functionPtr: CalleePtr, abstractInfo: GD);
6769}
6770
6771static GlobalDecl getGlobalDeclForDirectCall(const FunctionDecl *FD) {
6772 if (DeviceKernelAttr::isOpenCLSpelling(A: FD->getAttr<DeviceKernelAttr>()))
6773 return GlobalDecl(FD, KernelReferenceKind::Stub);
6774 return GlobalDecl(FD);
6775}
6776
6777CGCallee CodeGenFunction::EmitCallee(const Expr *E) {
6778 E = E->IgnoreParens();
6779
6780 // A WebAssembly funcref is an opaque reference type and llvm only accepts
6781 // function pointers as the call target. To make an indirect call through a
6782 // reference type, first use the llvm.wasm.funcref.to_ptr intrinsic to make a
6783 // fake function pointer to it. The backend lowers the resulting indirect call
6784 // to a table.set into a single element dummy table + call_indirect 0.
6785 auto ConvertFuncrefToPtr = [&](llvm::Value *CalleePtr) -> llvm::Value * {
6786 if (auto *TET = dyn_cast<llvm::TargetExtType>(Val: CalleePtr->getType());
6787 TET && TET->getName() == "wasm.funcref") {
6788 llvm::Function *ToPtr =
6789 CGM.getIntrinsic(IID: llvm::Intrinsic::wasm_funcref_to_ptr);
6790 return Builder.CreateCall(Callee: ToPtr, Args: {CalleePtr});
6791 }
6792 return CalleePtr;
6793 };
6794
6795 // Look through function-to-pointer decay.
6796 if (auto ICE = dyn_cast<ImplicitCastExpr>(Val: E)) {
6797 if (ICE->getCastKind() == CK_FunctionToPointerDecay ||
6798 ICE->getCastKind() == CK_BuiltinFnToFnPtr) {
6799 return EmitCallee(E: ICE->getSubExpr());
6800 }
6801
6802 // Try to remember the original __ptrauth qualifier for loads of
6803 // function pointers.
6804 if (ICE->getCastKind() == CK_LValueToRValue) {
6805 const Expr *SubExpr = ICE->getSubExpr();
6806 if (const auto *PtrType = SubExpr->getType()->getAs<PointerType>()) {
6807 std::pair<llvm::Value *, CGPointerAuthInfo> Result =
6808 EmitOrigPointerRValue(E);
6809
6810 QualType FunctionType = PtrType->getPointeeType();
6811 assert(FunctionType->isFunctionType());
6812
6813 GlobalDecl GD;
6814 if (const auto *VD =
6815 dyn_cast_or_null<VarDecl>(Val: E->getReferencedDeclOfCallee())) {
6816 GD = GlobalDecl(VD);
6817 }
6818 CGCalleeInfo CalleeInfo(FunctionType->castAs<clang::FunctionType>(),
6819 GD);
6820 CGCallee Callee(CalleeInfo, ConvertFuncrefToPtr(Result.first),
6821 Result.second);
6822 return Callee;
6823 }
6824 }
6825
6826 // Resolve direct calls.
6827 } else if (auto DRE = dyn_cast<DeclRefExpr>(Val: E)) {
6828 if (auto FD = dyn_cast<FunctionDecl>(Val: DRE->getDecl())) {
6829 return EmitDirectCallee(CGF&: *this, GD: getGlobalDeclForDirectCall(FD));
6830 }
6831 } else if (auto ME = dyn_cast<MemberExpr>(Val: E)) {
6832 if (auto FD = dyn_cast<FunctionDecl>(Val: ME->getMemberDecl())) {
6833 EmitIgnoredExpr(E: ME->getBase());
6834 return EmitDirectCallee(CGF&: *this, GD: FD);
6835 }
6836
6837 // Look through template substitutions.
6838 } else if (auto NTTP = dyn_cast<SubstNonTypeTemplateParmExpr>(Val: E)) {
6839 return EmitCallee(E: NTTP->getReplacement());
6840
6841 // Treat pseudo-destructor calls differently.
6842 } else if (auto PDE = dyn_cast<CXXPseudoDestructorExpr>(Val: E)) {
6843 return CGCallee::forPseudoDestructor(E: PDE);
6844 }
6845
6846 // Otherwise, we have an indirect reference.
6847 llvm::Value *calleePtr;
6848 QualType functionType;
6849 if (auto ptrType = E->getType()->getAs<PointerType>()) {
6850 calleePtr = EmitScalarExpr(E);
6851 functionType = ptrType->getPointeeType();
6852 } else {
6853 functionType = E->getType();
6854 calleePtr = EmitLValue(E, IsKnownNonNull: KnownNonNull).getPointer(CGF&: *this);
6855 }
6856 assert(functionType->isFunctionType());
6857
6858 GlobalDecl GD;
6859 if (const auto *VD =
6860 dyn_cast_or_null<VarDecl>(Val: E->getReferencedDeclOfCallee()))
6861 GD = GlobalDecl(VD);
6862
6863 CGCalleeInfo calleeInfo(functionType->castAs<clang::FunctionType>(), GD);
6864 CGPointerAuthInfo pointerAuth = CGM.getFunctionPointerAuthInfo(T: functionType);
6865 CGCallee callee(calleeInfo, ConvertFuncrefToPtr(calleePtr), pointerAuth);
6866 return callee;
6867}
6868
6869LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
6870 // Comma expressions just emit their LHS then their RHS as an l-value.
6871 if (E->getOpcode() == BO_Comma) {
6872 EmitIgnoredExpr(E: E->getLHS());
6873 EnsureInsertPoint();
6874 return EmitLValue(E: E->getRHS());
6875 }
6876
6877 if (E->getOpcode() == BO_PtrMemD ||
6878 E->getOpcode() == BO_PtrMemI)
6879 return EmitPointerToDataMemberBinaryExpr(E);
6880
6881 assert(E->getOpcode() == BO_Assign && "unexpected binary l-value");
6882
6883 // Create a Key Instructions source location atom group that covers both
6884 // LHS and RHS expressions. Nested RHS expressions may get subsequently
6885 // separately grouped (1 below):
6886 //
6887 // 1. `a = b = c` -> Two atoms.
6888 // 2. `x = new(1)` -> One atom (for both addr store and value store).
6889 // 3. Complex and agg assignment -> One atom.
6890 ApplyAtomGroup Grp(getDebugInfo());
6891
6892 // Note that in all of these cases, __block variables need the RHS
6893 // evaluated first just in case the variable gets moved by the RHS.
6894
6895 switch (getEvaluationKind(T: E->getType())) {
6896 case TEK_Scalar: {
6897 if (PointerAuthQualifier PtrAuth =
6898 E->getLHS()->getType().getPointerAuth()) {
6899 LValue LV = EmitCheckedLValue(E: E->getLHS(), TCK: TCK_Store);
6900 LValue CopiedLV = LV;
6901 CopiedLV.getQuals().removePointerAuth();
6902 llvm::Value *RV =
6903 EmitPointerAuthQualify(Qualifier: PtrAuth, PointerExpr: E->getRHS(), StorageAddress: CopiedLV.getAddress());
6904 EmitNullabilityCheck(LHS: CopiedLV, RHS: RV, Loc: E->getExprLoc());
6905 EmitStoreThroughLValue(Src: RValue::get(V: RV), Dst: CopiedLV);
6906 return LV;
6907 }
6908
6909 switch (E->getLHS()->getType().getObjCLifetime()) {
6910 case Qualifiers::OCL_Strong:
6911 return EmitARCStoreStrong(e: E, /*ignored*/ false).first;
6912
6913 case Qualifiers::OCL_Autoreleasing:
6914 return EmitARCStoreAutoreleasing(e: E).first;
6915
6916 // No reason to do any of these differently.
6917 case Qualifiers::OCL_None:
6918 case Qualifiers::OCL_ExplicitNone:
6919 case Qualifiers::OCL_Weak:
6920 break;
6921 }
6922
6923 // TODO: Can we de-duplicate this code with the corresponding code in
6924 // CGExprScalar, similar to the way EmitCompoundAssignmentLValue works?
6925 RValue RV;
6926 llvm::Value *Previous = nullptr;
6927 QualType SrcType = E->getRHS()->getType();
6928 // Check if LHS is a bitfield, if RHS contains an implicit cast expression
6929 // we want to extract that value and potentially (if the bitfield sanitizer
6930 // is enabled) use it to check for an implicit conversion.
6931 if (E->getLHS()->refersToBitField()) {
6932 llvm::Value *RHS =
6933 EmitWithOriginalRHSBitfieldAssignment(E, Previous: &Previous, SrcType: &SrcType);
6934 RV = RValue::get(V: RHS);
6935 } else
6936 RV = EmitAnyExpr(E: E->getRHS());
6937
6938 LValue LV = EmitCheckedLValue(E: E->getLHS(), TCK: TCK_Store);
6939
6940 if (RV.isScalar())
6941 EmitNullabilityCheck(LHS: LV, RHS: RV.getScalarVal(), Loc: E->getExprLoc());
6942
6943 if (LV.isBitField()) {
6944 llvm::Value *Result = nullptr;
6945 // If bitfield sanitizers are enabled we want to use the result
6946 // to check whether a truncation or sign change has occurred.
6947 if (SanOpts.has(K: SanitizerKind::ImplicitBitfieldConversion))
6948 EmitStoreThroughBitfieldLValue(Src: RV, Dst: LV, Result: &Result);
6949 else
6950 EmitStoreThroughBitfieldLValue(Src: RV, Dst: LV);
6951
6952 // If the expression contained an implicit conversion, make sure
6953 // to use the value before the scalar conversion.
6954 llvm::Value *Src = Previous ? Previous : RV.getScalarVal();
6955 QualType DstType = E->getLHS()->getType();
6956 EmitBitfieldConversionCheck(Src, SrcType, Dst: Result, DstType,
6957 Info: LV.getBitFieldInfo(), Loc: E->getExprLoc());
6958 } else
6959 EmitStoreThroughLValue(Src: RV, Dst: LV);
6960
6961 if (getLangOpts().OpenMP)
6962 CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF&: *this,
6963 LHS: E->getLHS());
6964 return LV;
6965 }
6966
6967 case TEK_Complex:
6968 return EmitComplexAssignmentLValue(E);
6969
6970 case TEK_Aggregate:
6971 // If the lang opt is HLSL and the LHS is a constant array
6972 // then we are performing a copy assignment and call a special
6973 // function because EmitAggExprToLValue emits to a temporary LValue
6974 if (getLangOpts().HLSL && E->getLHS()->getType()->isConstantArrayType())
6975 return EmitHLSLArrayAssignLValue(E);
6976
6977 return EmitAggExprToLValue(E);
6978 }
6979 llvm_unreachable("bad evaluation kind");
6980}
6981
6982// This function implements trivial copy assignment for HLSL's
6983// assignable constant arrays.
6984LValue CodeGenFunction::EmitHLSLArrayAssignLValue(const BinaryOperator *E) {
6985 // Don't emit an LValue for the RHS because it might not be an LValue
6986 LValue LHS = EmitLValue(E: E->getLHS());
6987
6988 // If the RHS is a global resource array, copy all individual resources
6989 // into LHS.
6990 if (E->getRHS()->getType()->isHLSLResourceRecordArray()) {
6991 AggValueSlot Slot = AggValueSlot::forAddr(
6992 addr: LHS.getAddress(), quals: Qualifiers(), isDestructed: AggValueSlot::IsDestructed_t(true),
6993 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsAliased_t(false),
6994 mayOverlap: AggValueSlot::DoesNotOverlap);
6995 if (CGM.getHLSLRuntime().emitGlobalResourceArray(CGF&: *this, E: E->getRHS(), DestSlot&: Slot))
6996 return LHS;
6997 }
6998
6999 // In C the RHS of an assignment operator is an RValue.
7000 // EmitAggregateAssign takes an LValue for the RHS. Instead we can call
7001 // EmitInitializationToLValue to emit an RValue into an LValue.
7002 EmitInitializationToLValue(E: E->getRHS(), LV: LHS);
7003 return LHS;
7004}
7005
7006LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E,
7007 llvm::CallBase **CallOrInvoke) {
7008 RValue RV = EmitCallExpr(E, ReturnValue: ReturnValueSlot(), CallOrInvoke);
7009
7010 if (!RV.isScalar())
7011 return MakeAddrLValue(Addr: RV.getAggregateAddress(), T: E->getType(),
7012 Source: AlignmentSource::Decl);
7013
7014 assert(E->getCallReturnType(getContext())->isReferenceType() &&
7015 "Can't have a scalar return unless the return type is a "
7016 "reference type!");
7017
7018 return MakeNaturalAlignPointeeAddrLValue(V: RV.getScalarVal(), T: E->getType());
7019}
7020
7021LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {
7022 // FIXME: This shouldn't require another copy.
7023 return EmitAggExprToLValue(E);
7024}
7025
7026LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
7027 assert(E->getType()->getAsCXXRecordDecl()->hasTrivialDestructor()
7028 && "binding l-value to type which needs a temporary");
7029 AggValueSlot Slot = CreateAggTemp(T: E->getType());
7030 EmitCXXConstructExpr(E, Dest: Slot);
7031 return MakeAddrLValue(Addr: Slot.getAddress(), T: E->getType(), Source: AlignmentSource::Decl);
7032}
7033
7034LValue
7035CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
7036 return MakeNaturalAlignRawAddrLValue(V: EmitCXXTypeidExpr(E), T: E->getType());
7037}
7038
7039Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) {
7040 return CGM.GetAddrOfMSGuidDecl(GD: E->getGuidDecl())
7041 .withElementType(ElemTy: ConvertType(T: E->getType()));
7042}
7043
7044LValue CodeGenFunction::EmitCXXUuidofLValue(const CXXUuidofExpr *E) {
7045 return MakeAddrLValue(Addr: EmitCXXUuidofExpr(E), T: E->getType(),
7046 Source: AlignmentSource::Decl);
7047}
7048
7049LValue
7050CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) {
7051 AggValueSlot Slot = CreateAggTemp(T: E->getType(), Name: "temp.lvalue");
7052 Slot.setExternallyDestructed();
7053 EmitAggExpr(E: E->getSubExpr(), AS: Slot);
7054 EmitCXXTemporary(Temporary: E->getTemporary(), TempType: E->getType(), Ptr: Slot.getAddress());
7055 return MakeAddrLValue(Addr: Slot.getAddress(), T: E->getType(), Source: AlignmentSource::Decl);
7056}
7057
7058LValue CodeGenFunction::EmitObjCMessageExprLValue(const ObjCMessageExpr *E) {
7059 RValue RV = EmitObjCMessageExpr(E);
7060
7061 if (!RV.isScalar())
7062 return MakeAddrLValue(Addr: RV.getAggregateAddress(), T: E->getType(),
7063 Source: AlignmentSource::Decl);
7064
7065 assert(E->getMethodDecl()->getReturnType()->isReferenceType() &&
7066 "Can't have a scalar return unless the return type is a "
7067 "reference type!");
7068
7069 return MakeNaturalAlignPointeeAddrLValue(V: RV.getScalarVal(), T: E->getType());
7070}
7071
7072LValue CodeGenFunction::EmitObjCSelectorLValue(const ObjCSelectorExpr *E) {
7073 Address V =
7074 CGM.getObjCRuntime().GetAddrOfSelector(CGF&: *this, Sel: E->getSelector());
7075 return MakeAddrLValue(Addr: V, T: E->getType(), Source: AlignmentSource::Decl);
7076}
7077
7078llvm::Value *CodeGenFunction::EmitIvarOffset(const ObjCInterfaceDecl *Interface,
7079 const ObjCIvarDecl *Ivar) {
7080 return CGM.getObjCRuntime().EmitIvarOffset(CGF&: *this, Interface, Ivar);
7081}
7082
7083llvm::Value *
7084CodeGenFunction::EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,
7085 const ObjCIvarDecl *Ivar) {
7086 llvm::Value *OffsetValue = EmitIvarOffset(Interface, Ivar);
7087 QualType PointerDiffType = getContext().getPointerDiffType();
7088 return Builder.CreateZExtOrTrunc(V: OffsetValue,
7089 DestTy: getTypes().ConvertType(T: PointerDiffType));
7090}
7091
7092LValue CodeGenFunction::EmitLValueForIvar(QualType ObjectTy,
7093 llvm::Value *BaseValue,
7094 const ObjCIvarDecl *Ivar,
7095 unsigned CVRQualifiers) {
7096 return CGM.getObjCRuntime().EmitObjCValueForIvar(CGF&: *this, ObjectTy, BaseValue,
7097 Ivar, CVRQualifiers);
7098}
7099
7100LValue CodeGenFunction::EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E) {
7101 // FIXME: A lot of the code below could be shared with EmitMemberExpr.
7102 llvm::Value *BaseValue = nullptr;
7103 const Expr *BaseExpr = E->getBase();
7104 Qualifiers BaseQuals;
7105 QualType ObjectTy;
7106 if (E->isArrow()) {
7107 BaseValue = EmitScalarExpr(E: BaseExpr);
7108 ObjectTy = BaseExpr->getType()->getPointeeType();
7109 BaseQuals = ObjectTy.getQualifiers();
7110 } else {
7111 LValue BaseLV = EmitLValue(E: BaseExpr);
7112 BaseValue = BaseLV.getPointer(CGF&: *this);
7113 ObjectTy = BaseExpr->getType();
7114 BaseQuals = ObjectTy.getQualifiers();
7115 }
7116
7117 LValue LV =
7118 EmitLValueForIvar(ObjectTy, BaseValue, Ivar: E->getDecl(),
7119 CVRQualifiers: BaseQuals.getCVRQualifiers());
7120 setObjCGCLValueClass(Ctx: getContext(), E, LV);
7121 return LV;
7122}
7123
7124LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {
7125 // Can only get l-value for message expression returning aggregate type
7126 RValue RV = EmitAnyExprToTemp(E);
7127 return MakeAddrLValue(Addr: RV.getAggregateAddress(), T: E->getType(),
7128 Source: AlignmentSource::Decl);
7129}
7130
7131RValue CodeGenFunction::EmitCall(QualType CalleeType,
7132 const CGCallee &OrigCallee, const CallExpr *E,
7133 ReturnValueSlot ReturnValue,
7134 llvm::Value *Chain,
7135 llvm::CallBase **CallOrInvoke,
7136 CGFunctionInfo const **ResolvedFnInfo) {
7137 // Get the actual function type. The callee type will always be a pointer to
7138 // function type or a block pointer type.
7139 assert(CalleeType->isFunctionPointerType() &&
7140 "Call must have function pointer type!");
7141
7142 const Decl *TargetDecl =
7143 OrigCallee.getAbstractInfo().getCalleeDecl().getDecl();
7144
7145 assert((!isa_and_present<FunctionDecl>(TargetDecl) ||
7146 !cast<FunctionDecl>(TargetDecl)->isImmediateFunction()) &&
7147 "trying to emit a call to an immediate function");
7148
7149 CalleeType = getContext().getCanonicalType(T: CalleeType);
7150
7151 auto PointeeType = cast<PointerType>(Val&: CalleeType)->getPointeeType();
7152
7153 CGCallee Callee = OrigCallee;
7154
7155 bool CFIUnchecked = CalleeType->hasPointeeToCFIUncheckedCalleeFunctionType();
7156
7157 if (SanOpts.has(K: SanitizerKind::Function) &&
7158 (!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl)) &&
7159 !isa<FunctionNoProtoType>(Val: PointeeType) && !CFIUnchecked) {
7160 if (llvm::Constant *PrefixSig =
7161 CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM)) {
7162 auto CheckOrdinal = SanitizerKind::SO_Function;
7163 auto CheckHandler = SanitizerHandler::FunctionTypeMismatch;
7164 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
7165 auto *TypeHash = getUBSanFunctionTypeHash(T: PointeeType);
7166
7167 llvm::Type *PrefixSigType = PrefixSig->getType();
7168 llvm::StructType *PrefixStructTy = llvm::StructType::get(
7169 Context&: CGM.getLLVMContext(), Elements: {PrefixSigType, Int32Ty}, /*isPacked=*/true);
7170
7171 llvm::Value *CalleePtr = Callee.getFunctionPointer();
7172 if (CGM.getCodeGenOpts().PointerAuth.FunctionPointers) {
7173 // Use raw pointer since we are using the callee pointer as data here.
7174 Address Addr =
7175 Address(CalleePtr, CalleePtr->getType(),
7176 CharUnits::fromQuantity(
7177 Quantity: CalleePtr->getPointerAlignment(DL: CGM.getDataLayout())),
7178 Callee.getPointerAuthInfo(), nullptr);
7179 CalleePtr = Addr.emitRawPointer(CGF&: *this);
7180 }
7181
7182 // On 32-bit Arm, the low bit of a function pointer indicates whether
7183 // it's using the Arm or Thumb instruction set. The actual first
7184 // instruction lives at the same address either way, so we must clear
7185 // that low bit before using the function address to find the prefix
7186 // structure.
7187 //
7188 // This applies to both Arm and Thumb target triples, because
7189 // either one could be used in an interworking context where it
7190 // might be passed function pointers of both types.
7191 llvm::Value *AlignedCalleePtr;
7192 if (CGM.getTriple().isARM() || CGM.getTriple().isThumb()) {
7193 AlignedCalleePtr = Builder.CreateIntrinsic(
7194 RetTy: CalleePtr->getType(), ID: llvm::Intrinsic::ptrmask,
7195 Args: {CalleePtr, llvm::ConstantInt::getSigned(Ty: IntPtrTy, V: ~1)});
7196 } else {
7197 AlignedCalleePtr = CalleePtr;
7198 }
7199
7200 llvm::Value *CalleePrefixStruct = AlignedCalleePtr;
7201 llvm::Value *CalleeSigPtr =
7202 Builder.CreateConstGEP2_32(Ty: PrefixStructTy, Ptr: CalleePrefixStruct, Idx0: -1, Idx1: 0);
7203 llvm::Value *CalleeSig =
7204 Builder.CreateAlignedLoad(Ty: PrefixSigType, Addr: CalleeSigPtr, Align: getIntAlign());
7205 llvm::Value *CalleeSigMatch = Builder.CreateICmpEQ(LHS: CalleeSig, RHS: PrefixSig);
7206
7207 llvm::BasicBlock *Cont = createBasicBlock(name: "cont");
7208 llvm::BasicBlock *TypeCheck = createBasicBlock(name: "typecheck");
7209 Builder.CreateCondBr(Cond: CalleeSigMatch, True: TypeCheck, False: Cont);
7210
7211 EmitBlock(BB: TypeCheck);
7212 llvm::Value *CalleeTypeHash = Builder.CreateAlignedLoad(
7213 Ty: Int32Ty,
7214 Addr: Builder.CreateConstGEP2_32(Ty: PrefixStructTy, Ptr: CalleePrefixStruct, Idx0: -1, Idx1: 1),
7215 Align: getPointerAlign());
7216 llvm::Value *CalleeTypeHashMatch =
7217 Builder.CreateICmpEQ(LHS: CalleeTypeHash, RHS: TypeHash);
7218 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc: E->getBeginLoc()),
7219 EmitCheckTypeDescriptor(T: CalleeType)};
7220 EmitCheck(Checked: std::make_pair(x&: CalleeTypeHashMatch, y&: CheckOrdinal), CheckHandler,
7221 StaticArgs: StaticData, DynamicArgs: {CalleePtr});
7222
7223 Builder.CreateBr(Dest: Cont);
7224 EmitBlock(BB: Cont);
7225 }
7226 }
7227
7228 const auto *FnType = cast<FunctionType>(Val&: PointeeType);
7229
7230 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
7231 FD && DeviceKernelAttr::isOpenCLSpelling(A: FD->getAttr<DeviceKernelAttr>()))
7232 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FnType);
7233
7234 // If we are checking indirect calls and this call is indirect, check that the
7235 // function pointer is a member of the bit set for the function type.
7236 if (SanOpts.has(K: SanitizerKind::CFIICall) &&
7237 (!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl)) && !CFIUnchecked) {
7238 auto CheckOrdinal = SanitizerKind::SO_CFIICall;
7239 auto CheckHandler = SanitizerHandler::CFICheckFail;
7240 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
7241 EmitSanitizerStatReport(SSK: llvm::SanStat_CFI_ICall);
7242
7243 llvm::Metadata *MD =
7244 CGM.CreateMetadataIdentifierForFnType(T: QualType(FnType, 0));
7245
7246 llvm::Value *TypeId = llvm::MetadataAsValue::get(Context&: getLLVMContext(), MD);
7247
7248 llvm::Value *CalleePtr = Callee.getFunctionPointer();
7249 llvm::Value *TypeTest = Builder.CreateCall(
7250 Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::type_test), Args: {CalleePtr, TypeId});
7251
7252 auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
7253 llvm::Constant *StaticData[] = {
7254 llvm::ConstantInt::get(Ty: Int8Ty, V: CFITCK_ICall),
7255 EmitCheckSourceLocation(Loc: E->getBeginLoc()),
7256 EmitCheckTypeDescriptor(T: QualType(FnType, 0)),
7257 };
7258 if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) {
7259 EmitCfiSlowPathCheck(Ordinal: CheckOrdinal, Cond: TypeTest, TypeId: CrossDsoTypeId, Ptr: CalleePtr,
7260 StaticArgs: StaticData);
7261 } else {
7262 EmitCheck(Checked: std::make_pair(x&: TypeTest, y&: CheckOrdinal), CheckHandler,
7263 StaticArgs: StaticData, DynamicArgs: {CalleePtr, llvm::UndefValue::get(T: IntPtrTy)});
7264 }
7265 }
7266
7267 CallArgList Args;
7268 if (Chain)
7269 Args.add(rvalue: RValue::get(V: Chain), type: CGM.getContext().VoidPtrTy);
7270
7271 // C++17 requires that we evaluate arguments to a call using assignment syntax
7272 // right-to-left, and that we evaluate arguments to certain other operators
7273 // left-to-right. Note that we allow this to override the order dictated by
7274 // the calling convention on the MS ABI, which means that parameter
7275 // destruction order is not necessarily reverse construction order.
7276 // FIXME: Revisit this based on C++ committee response to unimplementability.
7277 EvaluationOrder Order = EvaluationOrder::Default;
7278 bool StaticOperator = false;
7279 if (auto *OCE = dyn_cast<CXXOperatorCallExpr>(Val: E)) {
7280 if (OCE->isAssignmentOp())
7281 Order = EvaluationOrder::ForceRightToLeft;
7282 else {
7283 switch (OCE->getOperator()) {
7284 case OO_LessLess:
7285 case OO_GreaterGreater:
7286 case OO_AmpAmp:
7287 case OO_PipePipe:
7288 case OO_Comma:
7289 case OO_ArrowStar:
7290 Order = EvaluationOrder::ForceLeftToRight;
7291 break;
7292 default:
7293 break;
7294 }
7295 }
7296
7297 if (const auto *MD =
7298 dyn_cast_if_present<CXXMethodDecl>(Val: OCE->getCalleeDecl());
7299 MD && MD->isStatic())
7300 StaticOperator = true;
7301 }
7302
7303 auto Arguments = E->arguments();
7304 if (StaticOperator) {
7305 // If we're calling a static operator, we need to emit the object argument
7306 // and ignore it.
7307 EmitIgnoredExpr(E: E->getArg(Arg: 0));
7308 Arguments = drop_begin(RangeOrContainer&: Arguments, N: 1);
7309 }
7310 EmitCallArgs(Args, Prototype: dyn_cast<FunctionProtoType>(Val: FnType), ArgRange: Arguments,
7311 AC: E->getDirectCallee(), /*ParamsToSkip=*/0, Order);
7312
7313 const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(
7314 Args, Ty: FnType, /*ChainCall=*/Chain, ABIInfoFD: getCurrentFunctionDecl());
7315
7316 if (ResolvedFnInfo)
7317 *ResolvedFnInfo = &FnInfo;
7318
7319 // HIP function pointer contains kernel handle when it is used in triple
7320 // chevron. The kernel stub needs to be loaded from kernel handle and used
7321 // as callee.
7322 if (CGM.getLangOpts().HIP && !CGM.getLangOpts().CUDAIsDevice &&
7323 isa<CUDAKernelCallExpr>(Val: E) &&
7324 (!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl))) {
7325 llvm::Value *Handle = Callee.getFunctionPointer();
7326 auto *Stub = Builder.CreateLoad(
7327 Addr: Address(Handle, Handle->getType(), CGM.getPointerAlign()));
7328 Callee.setFunctionPointer(Stub);
7329 }
7330
7331 // Insert function pointer lookup if this is a target call
7332 //
7333 // This is used for the indirect function case, virtual function case is
7334 // handled in ItaniumCXXABI.cpp
7335 if (getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU() &&
7336 (!TargetDecl || !isa<FunctionDecl>(Val: TargetDecl))) {
7337 const Expr *CalleeExpr = E->getCallee()->IgnoreParenImpCasts();
7338 const DeclRefExpr *DRE = nullptr;
7339 while (CalleeExpr) {
7340 if ((DRE = dyn_cast<DeclRefExpr>(Val: CalleeExpr)))
7341 break;
7342 if (const auto *ME = dyn_cast<MemberExpr>(Val: CalleeExpr))
7343 CalleeExpr = ME->getBase()->IgnoreParenImpCasts();
7344 else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: CalleeExpr))
7345 CalleeExpr = ASE->getBase()->IgnoreParenImpCasts();
7346 else
7347 break;
7348 }
7349
7350 const auto *VD = DRE ? dyn_cast<VarDecl>(Val: DRE->getDecl()) : nullptr;
7351 if (VD && VD->hasAttr<OMPTargetIndirectCallAttr>()) {
7352 auto *FuncPtrTy = llvm::PointerType::get(
7353 C&: CGM.getLLVMContext(), AddressSpace: CGM.getDataLayout().getProgramAddressSpace());
7354 llvm::Type *RtlFnArgs[] = {FuncPtrTy};
7355 llvm::FunctionCallee DeviceRtlFn = CGM.CreateRuntimeFunction(
7356 Ty: llvm::FunctionType::get(Result: FuncPtrTy, Params: RtlFnArgs, isVarArg: false),
7357 Name: "__llvm_omp_indirect_call_lookup");
7358 llvm::Value *Func = Callee.getFunctionPointer();
7359 llvm::Type *BackupTy = Func->getType();
7360 Func = Builder.CreatePointerBitCastOrAddrSpaceCast(V: Func, DestTy: FuncPtrTy);
7361 Func = EmitRuntimeCall(callee: DeviceRtlFn, args: {Func});
7362 Func = Builder.CreatePointerBitCastOrAddrSpaceCast(V: Func, DestTy: BackupTy);
7363 Callee.setFunctionPointer(Func);
7364 }
7365 }
7366
7367 llvm::CallBase *LocalCallOrInvoke = nullptr;
7368 RValue Call = EmitCall(CallInfo: FnInfo, Callee, ReturnValue, Args, CallOrInvoke: &LocalCallOrInvoke,
7369 IsMustTail: E == MustTailCall, Loc: E->getExprLoc());
7370
7371 if (auto *CalleeDecl = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
7372 if (CalleeDecl->hasAttr<RestrictAttr>() ||
7373 CalleeDecl->hasAttr<MallocSpanAttr>() ||
7374 CalleeDecl->hasAttr<AllocSizeAttr>()) {
7375 // Function has 'malloc' (aka. 'restrict') or 'alloc_size' attribute.
7376 if (SanOpts.has(K: SanitizerKind::AllocToken)) {
7377 // Set !alloc_token metadata.
7378 EmitAllocToken(CB: LocalCallOrInvoke, E);
7379 }
7380 }
7381 }
7382 if (CallOrInvoke)
7383 *CallOrInvoke = LocalCallOrInvoke;
7384
7385 return Call;
7386}
7387
7388LValue CodeGenFunction::
7389EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) {
7390 Address BaseAddr = Address::invalid();
7391 if (E->getOpcode() == BO_PtrMemI) {
7392 BaseAddr = EmitPointerWithAlignment(E: E->getLHS());
7393 } else {
7394 BaseAddr = EmitLValue(E: E->getLHS()).getAddress();
7395 }
7396
7397 llvm::Value *OffsetV = EmitScalarExpr(E: E->getRHS());
7398 const auto *MPT = E->getRHS()->getType()->castAs<MemberPointerType>();
7399
7400 LValueBaseInfo BaseInfo;
7401 TBAAAccessInfo TBAAInfo;
7402 bool IsInBounds = !getLangOpts().PointerOverflowDefined &&
7403 !isUnderlyingBasePointerConstantNull(E: E->getLHS());
7404 Address MemberAddr = EmitCXXMemberDataPointerAddress(
7405 E, base: BaseAddr, memberPtr: OffsetV, memberPtrType: MPT, IsInBounds, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
7406
7407 return MakeAddrLValue(Addr: MemberAddr, T: MPT->getPointeeType(), BaseInfo, TBAAInfo);
7408}
7409
7410/// Given the address of a temporary variable, produce an r-value of
7411/// its type.
7412RValue CodeGenFunction::convertTempToRValue(Address addr,
7413 QualType type,
7414 SourceLocation loc) {
7415 LValue lvalue = MakeAddrLValue(Addr: addr, T: type, Source: AlignmentSource::Decl);
7416 switch (getEvaluationKind(T: type)) {
7417 case TEK_Complex:
7418 return RValue::getComplex(C: EmitLoadOfComplex(src: lvalue, loc));
7419 case TEK_Aggregate:
7420 return lvalue.asAggregateRValue();
7421 case TEK_Scalar:
7422 return RValue::get(V: EmitLoadOfScalar(lvalue, Loc: loc));
7423 }
7424 llvm_unreachable("bad evaluation kind");
7425}
7426
7427void CodeGenFunction::SetFPAccuracy(llvm::Value *Val, float Accuracy) {
7428 assert(Val->getType()->isFPOrFPVectorTy());
7429 if (Accuracy == 0.0 || !isa<llvm::Instruction>(Val))
7430 return;
7431
7432 llvm::MDBuilder MDHelper(getLLVMContext());
7433 llvm::MDNode *Node = MDHelper.createFPMath(Accuracy);
7434
7435 cast<llvm::Instruction>(Val)->setMetadata(KindID: llvm::LLVMContext::MD_fpmath, Node);
7436}
7437
7438void CodeGenFunction::SetSqrtFPAccuracy(llvm::Value *Val) {
7439 llvm::Type *EltTy = Val->getType()->getScalarType();
7440 if (!EltTy->isFloatTy() && !EltTy->isHalfTy())
7441 return;
7442
7443 if ((getLangOpts().OpenCL &&
7444 !CGM.getCodeGenOpts().OpenCLCorrectlyRoundedDivSqrt) ||
7445 (getLangOpts().HIP && getLangOpts().CUDAIsDevice &&
7446 !CGM.getCodeGenOpts().HIPCorrectlyRoundedDivSqrt)) {
7447 // OpenCL v1.1 s7.4: minimum accuracy of single precision sqrt is 3 ulp.
7448 // OpenCL v3.0 s7.4: minimum accuracy of half precision sqrt is 1.5 ulp.
7449 //
7450 // OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt
7451 // build option allows an application to specify that single precision
7452 // floating-point divide (x/y and 1/x) and sqrt used in the program
7453 // source are correctly rounded.
7454 //
7455 // TODO: CUDA has a prec-sqrt flag
7456 SetFPAccuracy(Val, Accuracy: EltTy->isFloatTy() ? 3.0f : 1.5f);
7457 }
7458}
7459
7460void CodeGenFunction::SetDivFPAccuracy(llvm::Value *Val) {
7461 llvm::Type *EltTy = Val->getType()->getScalarType();
7462 if (!EltTy->isFloatTy() && !EltTy->isHalfTy())
7463 return;
7464
7465 if ((getLangOpts().OpenCL &&
7466 !CGM.getCodeGenOpts().OpenCLCorrectlyRoundedDivSqrt) ||
7467 (getLangOpts().HIP && getLangOpts().CUDAIsDevice &&
7468 !CGM.getCodeGenOpts().HIPCorrectlyRoundedDivSqrt)) {
7469 // OpenCL v1.1 s7.4: minimum accuracy of single precision / is 2.5 ulp.
7470 // OpenCL v3.0 s7.4: minimum accuracy of half precision / is 1 ulp.
7471 //
7472 // OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt
7473 // build option allows an application to specify that single precision
7474 // floating-point divide (x/y and 1/x) and sqrt used in the program
7475 // source are correctly rounded.
7476 //
7477 // TODO: CUDA has a prec-div flag
7478 SetFPAccuracy(Val, Accuracy: EltTy->isFloatTy() ? 2.5f : 1.f);
7479 }
7480}
7481
7482namespace {
7483 struct LValueOrRValue {
7484 LValue LV;
7485 RValue RV;
7486 };
7487}
7488
7489static LValueOrRValue emitPseudoObjectExpr(CodeGenFunction &CGF,
7490 const PseudoObjectExpr *E,
7491 bool forLValue,
7492 AggValueSlot slot) {
7493 SmallVector<CodeGenFunction::OpaqueValueMappingData, 4> opaques;
7494
7495 // Find the result expression, if any.
7496 const Expr *resultExpr = E->getResultExpr();
7497 LValueOrRValue result;
7498
7499 for (PseudoObjectExpr::const_semantics_iterator
7500 i = E->semantics_begin(), e = E->semantics_end(); i != e; ++i) {
7501 const Expr *semantic = *i;
7502
7503 // If this semantic expression is an opaque value, bind it
7504 // to the result of its source expression.
7505 if (const auto *ov = dyn_cast<OpaqueValueExpr>(Val: semantic)) {
7506 // Skip unique OVEs.
7507 if (ov->isUnique()) {
7508 assert(ov != resultExpr &&
7509 "A unique OVE cannot be used as the result expression");
7510 continue;
7511 }
7512
7513 // If this is the result expression, we may need to evaluate
7514 // directly into the slot.
7515 typedef CodeGenFunction::OpaqueValueMappingData OVMA;
7516 OVMA opaqueData;
7517 if (ov == resultExpr && ov->isPRValue() && !forLValue &&
7518 CodeGenFunction::hasAggregateEvaluationKind(T: ov->getType())) {
7519 CGF.EmitAggExpr(E: ov->getSourceExpr(), AS: slot);
7520 LValue LV = CGF.MakeAddrLValue(Addr: slot.getAddress(), T: ov->getType(),
7521 Source: AlignmentSource::Decl);
7522 opaqueData = OVMA::bind(CGF, ov, lv: LV);
7523 result.RV = slot.asRValue();
7524
7525 // Otherwise, emit as normal.
7526 } else {
7527 opaqueData = OVMA::bind(CGF, ov, e: ov->getSourceExpr());
7528
7529 // If this is the result, also evaluate the result now.
7530 if (ov == resultExpr) {
7531 if (forLValue)
7532 result.LV = CGF.EmitLValue(E: ov);
7533 else
7534 result.RV = CGF.EmitAnyExpr(E: ov, aggSlot: slot);
7535 }
7536 }
7537
7538 opaques.push_back(Elt: opaqueData);
7539
7540 // Otherwise, if the expression is the result, evaluate it
7541 // and remember the result.
7542 } else if (semantic == resultExpr) {
7543 if (forLValue)
7544 result.LV = CGF.EmitLValue(E: semantic);
7545 else
7546 result.RV = CGF.EmitAnyExpr(E: semantic, aggSlot: slot);
7547
7548 // Otherwise, evaluate the expression in an ignored context.
7549 } else {
7550 CGF.EmitIgnoredExpr(E: semantic);
7551 }
7552 }
7553
7554 // Unbind all the opaques now.
7555 for (CodeGenFunction::OpaqueValueMappingData &opaque : opaques)
7556 opaque.unbind(CGF);
7557
7558 return result;
7559}
7560
7561RValue CodeGenFunction::EmitPseudoObjectRValue(const PseudoObjectExpr *E,
7562 AggValueSlot slot) {
7563 return emitPseudoObjectExpr(CGF&: *this, E, forLValue: false, slot).RV;
7564}
7565
7566LValue CodeGenFunction::EmitPseudoObjectLValue(const PseudoObjectExpr *E) {
7567 return emitPseudoObjectExpr(CGF&: *this, E, forLValue: true, slot: AggValueSlot::ignored()).LV;
7568}
7569
7570void CodeGenFunction::FlattenAccessAndTypeLValue(
7571 LValue Val, SmallVectorImpl<LValue> &AccessList) {
7572
7573 llvm::SmallVector<
7574 std::tuple<LValue, QualType, llvm::SmallVector<llvm::Value *, 4>>, 16>
7575 WorkList;
7576 llvm::IntegerType *IdxTy = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: 32);
7577 WorkList.push_back(Elt: {Val, Val.getType(), {llvm::ConstantInt::get(Ty: IdxTy, V: 0)}});
7578
7579 while (!WorkList.empty()) {
7580 auto [LVal, T, IdxList] = WorkList.pop_back_val();
7581 T = T.getCanonicalType().getUnqualifiedType();
7582 if (const auto *CAT = dyn_cast<ConstantArrayType>(Val&: T)) {
7583 uint64_t Size = CAT->getZExtSize();
7584 for (int64_t I = Size - 1; I > -1; I--) {
7585 llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList;
7586 IdxListCopy.push_back(Elt: llvm::ConstantInt::get(Ty: IdxTy, V: I));
7587 WorkList.emplace_back(Args&: LVal, Args: CAT->getElementType(), Args&: IdxListCopy);
7588 }
7589 } else if (const auto *RT = dyn_cast<RecordType>(Val&: T)) {
7590 const RecordDecl *Record = RT->getDecl()->getDefinitionOrSelf();
7591 assert(!Record->isUnion() && "Union types not supported in flat cast.");
7592
7593 const CXXRecordDecl *CXXD = dyn_cast<CXXRecordDecl>(Val: Record);
7594
7595 llvm::SmallVector<
7596 std::tuple<LValue, QualType, llvm::SmallVector<llvm::Value *, 4>>, 16>
7597 ReverseList;
7598 if (CXXD && CXXD->isStandardLayout())
7599 Record = CXXD->getStandardLayoutBaseWithFields();
7600
7601 // deal with potential base classes
7602 if (CXXD && !CXXD->isStandardLayout()) {
7603 if (CXXD->getNumBases() > 0) {
7604 assert(CXXD->getNumBases() == 1 &&
7605 "HLSL doesn't support multiple inheritance.");
7606 auto Base = CXXD->bases_begin();
7607 llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList;
7608 IdxListCopy.push_back(Elt: llvm::ConstantInt::get(
7609 Ty: IdxTy, V: 0)); // base struct should be at index zero
7610 ReverseList.emplace_back(Args&: LVal, Args: Base->getType(), Args&: IdxListCopy);
7611 }
7612 }
7613
7614 const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(Record);
7615
7616 llvm::Type *LLVMT = ConvertTypeForMem(T);
7617 CharUnits Align = getContext().getTypeAlignInChars(T);
7618 LValue RLValue;
7619 bool createdGEP = false;
7620 for (auto *FD : Record->fields()) {
7621 if (FD->isBitField()) {
7622 if (FD->isUnnamedBitField())
7623 continue;
7624 if (!createdGEP) {
7625 createdGEP = true;
7626 Address GEP = Builder.CreateInBoundsGEP(Addr: LVal.getAddress(), IdxList,
7627 ElementType: LLVMT, Align, Name: "gep");
7628 RLValue = MakeAddrLValue(Addr: GEP, T);
7629 }
7630 LValue FieldLVal = EmitLValueForField(base: RLValue, field: FD, IsInBounds: true);
7631 ReverseList.push_back(Elt: {FieldLVal, FD->getType(), {}});
7632 } else {
7633 llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList;
7634 IdxListCopy.push_back(
7635 Elt: llvm::ConstantInt::get(Ty: IdxTy, V: Layout.getLLVMFieldNo(FD)));
7636 ReverseList.emplace_back(Args&: LVal, Args: FD->getType(), Args&: IdxListCopy);
7637 }
7638 }
7639
7640 std::reverse(first: ReverseList.begin(), last: ReverseList.end());
7641 llvm::append_range(C&: WorkList, R&: ReverseList);
7642 } else if (const auto *VT = dyn_cast<VectorType>(Val&: T)) {
7643 llvm::Type *LLVMT = ConvertTypeForMem(T);
7644 CharUnits Align = getContext().getTypeAlignInChars(T);
7645 Address GEP = Builder.CreateInBoundsGEP(Addr: LVal.getAddress(), IdxList, ElementType: LLVMT,
7646 Align, Name: "vector.gep");
7647 LValue Base = MakeAddrLValue(Addr: GEP, T);
7648 for (unsigned I = 0, E = VT->getNumElements(); I < E; I++) {
7649 llvm::Constant *Idx = llvm::ConstantInt::get(Ty: IdxTy, V: I);
7650 LValue LV =
7651 LValue::MakeVectorElt(vecAddress: Base.getAddress(), Idx, type: VT->getElementType(),
7652 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
7653 AccessList.emplace_back(Args&: LV);
7654 }
7655 } else if (const auto *MT = dyn_cast<ConstantMatrixType>(Val&: T)) {
7656 // Matrices are represented as flat arrays in memory, but has a vector
7657 // value type. So we use ConvertMatrixAddress to convert the address from
7658 // array to vector, and extract elements similar to the vector case above.
7659 // The matrix elements are iterated over in row-major order regardless of
7660 // the memory layout of the matrix.
7661 llvm::Type *LLVMT = ConvertTypeForMem(T);
7662 CharUnits Align = getContext().getTypeAlignInChars(T);
7663 Address GEP = Builder.CreateInBoundsGEP(Addr: LVal.getAddress(), IdxList, ElementType: LLVMT,
7664 Align, Name: "matrix.gep");
7665 LValue Base = MakeAddrLValue(Addr: GEP, T);
7666 Address MatAddr = MaybeConvertMatrixAddress(Addr: Base.getAddress(), CGF&: *this);
7667 unsigned NumRows = MT->getNumRows();
7668 unsigned NumCols = MT->getNumColumns();
7669 bool IsMatrixRowMajor = isMatrixRowMajor(LangOpts: getLangOpts(), T);
7670 llvm::MatrixBuilder MB(Builder);
7671 for (unsigned Row = 0; Row < MT->getNumRows(); Row++) {
7672 for (unsigned Col = 0; Col < MT->getNumColumns(); Col++) {
7673 llvm::Value *RowIdx = llvm::ConstantInt::get(Ty: IdxTy, V: Row);
7674 llvm::Value *ColIdx = llvm::ConstantInt::get(Ty: IdxTy, V: Col);
7675 llvm::Value *Idx = MB.CreateIndex(RowIdx, ColumnIdx: ColIdx, NumRows, NumCols,
7676 IsMatrixRowMajor);
7677 LValue LV =
7678 LValue::MakeMatrixElt(matAddress: MatAddr, Idx, type: MT->getElementType(),
7679 BaseInfo: Base.getBaseInfo(), TBAAInfo: TBAAAccessInfo());
7680 AccessList.emplace_back(Args&: LV);
7681 }
7682 }
7683 } else { // a scalar/builtin type
7684 if (!IdxList.empty()) {
7685 llvm::Type *LLVMT = ConvertTypeForMem(T);
7686 CharUnits Align = getContext().getTypeAlignInChars(T);
7687 Address GEP = Builder.CreateInBoundsGEP(Addr: LVal.getAddress(), IdxList,
7688 ElementType: LLVMT, Align, Name: "gep");
7689 AccessList.emplace_back(Args: MakeAddrLValue(Addr: GEP, T));
7690 } else // must be a bitfield we already created an lvalue for
7691 AccessList.emplace_back(Args&: LVal);
7692 }
7693 }
7694}
7695